Python 88.3%
TypeScript 7.6%
Shell 4.1%
1#!/usr/bin/env python32"""Re-run the atlas pipeline and report what changed.34 python3 scripts/update_atlas.py # full run (network: indexes, pages, /v1/models)5 python3 scripts/update_atlas.py --no-crawl # skip docs crawl (offline rebuild + diff)6 python3 scripts/update_atlas.py --no-live # skip /v1/models discovery78Steps9 1. Snapshot current generated/*.json into reports/snapshots/<UTC timestamp>/ (baseline for diff).10 2. Refresh llms.txt indexes + crawl all doc pages (scripts/crawl_docs.py --refresh-index --follow) for both providers11 → sources/<provider>/pages-changes.json lists new/changed/removed pages.12 3. Live model discovery (scripts/discover_openai_models.py, scripts/discover_anthropic_models.py) → models-diff.json.13 4. Rebuild generated/ from fragments (scripts/build_generated.py).14 5. Diff baseline vs new generated files → reports/changes.md (+ reports/changes.json):15 NEW/REMOVED MODELS, MODEL ID CHANGES (alias/snapshot), NEW/REMOVED ENDPOINTS, NEW TOOLS, PRICING CHANGES,16 CONTEXT WINDOW CHANGES, NEW BETA FEATURES, DEPRECATIONS, CHANGED DOC PAGES.17 6. Link check (offline mode) → reports/link-check.json.1819NOTE: fragments (generated/fragments/**) are curated by agents/humans from the sources. A re-run detects drift20(live model list vs catalogue, changed doc pages) and flags it in reports/changes.md; it does not rewrite the21curated fragments automatically. The section "PAGES TO RE-REVIEW" tells a maintainer (or an agent) exactly which22source pages changed so the corresponding fragments can be refreshed.23"""24from __future__ import annotations2526import argparse27import json28import shutil29import subprocess30import sys31from datetime import datetime, timezone32from pathlib import Path3334ROOT = Path(__file__).resolve().parent.parent35GEN = ROOT / "generated"36REPORTS = ROOT / "reports"37PY = sys.executable383940def now() -> str:41 return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")424344def run(cmd: list[str], check: bool = False) -> int:45 print("$", " ".join(cmd), flush=True)46 r = subprocess.run(cmd, cwd=ROOT)47 if check and r.returncode:48 sys.exit(r.returncode)49 return r.returncode505152def load(p: Path) -> list[dict]:53 if not p.exists():54 return []55 try:56 d = json.loads(p.read_text())57 return d if isinstance(d, list) else []58 except Exception: # noqa: BLE00159 return []606162def index_by(recs: list[dict], *fields: str) -> dict[tuple, dict]:63 return {tuple(str(r.get(f)) for f in fields): r for r in recs}646566def diff_sets(old: dict, new: dict) -> tuple[list, list]:67 return sorted(k for k in new if k not in old), sorted(k for k in old if k not in new)686970def main() -> int:71 ap = argparse.ArgumentParser()72 ap.add_argument("--no-crawl", action="store_true")73 ap.add_argument("--no-live", action="store_true")74 ap.add_argument("--no-links", action="store_true")75 args = ap.parse_args()7677 stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")78 snap = REPORTS / "snapshots" / stamp79 snap.mkdir(parents=True, exist_ok=True)80 for f in GEN.glob("*.json"):81 shutil.copy2(f, snap / f.name)82 print(f"baseline snapshot: {snap.relative_to(ROOT)}")8384 if not args.no_crawl:85 for prov in ("openai", "anthropic", "gemini"):86 run([PY, "scripts/crawl_docs.py", prov, "--refresh-index", "--follow"])87 # docs.x.ai refuses direct .md fetches from non-browser clients → rebuild pages from the official llms-full.txt88 run([PY, "scripts/split_xai_llms_full.py"])89 if not args.no_live:90 for s in ("scripts/discover_openai_models.py", "scripts/discover_anthropic_models.py",91 "scripts/discover_xai_models.py", "scripts/discover_gemini_models.py"):92 if (ROOT / s).exists():93 run([PY, s])94 run([PY, "scripts/build_generated.py"], check=True)9596 old_models = index_by(load(snap / "models.json"), "provider", "id")97 new_models = index_by(load(GEN / "models.json"), "provider", "id")98 new_m, rem_m = diff_sets(old_models, new_models)99 ctx_changes, price_changes, status_changes = [], [], []100 for k, m in new_models.items():101 o = old_models.get(k)102 if not o:103 continue104 if o.get("context_window") != m.get("context_window") or o.get("max_output") != m.get("max_output"):105 ctx_changes.append((k, o.get("context_window"), m.get("context_window"), o.get("max_output"), m.get("max_output")))106 if json.dumps(o.get("pricing"), sort_keys=True, default=str) != json.dumps(m.get("pricing"), sort_keys=True, default=str):107 price_changes.append((k, o.get("pricing"), m.get("pricing")))108 if sorted(o.get("status") or []) != sorted(m.get("status") or []):109 status_changes.append((k, o.get("status"), m.get("status")))110111 old_ep = index_by(load(snap / "endpoints.json"), "provider", "method", "path")112 new_ep = index_by(load(GEN / "endpoints.json"), "provider", "method", "path")113 new_e, rem_e = diff_sets(old_ep, new_ep)114115 old_t = index_by(load(snap / "tools.json"), "provider", "type")116 new_t = index_by(load(GEN / "tools.json"), "provider", "type")117 new_tools, rem_tools = diff_sets(old_t, new_t)118119 old_p = index_by(load(snap / "pricing.json"), "provider", "model_or_service", "dimension", "tier", "unit")120 new_p = index_by(load(GEN / "pricing.json"), "provider", "model_or_service", "dimension", "tier", "unit")121 new_prices, rem_prices = diff_sets(old_p, new_p)122 changed_prices = [(k, old_p[k].get("price"), new_p[k].get("price")) for k in new_p if k in old_p and old_p[k].get("price") != new_p[k].get("price")]123124 old_params = index_by(load(snap / "parameters.json"), "provider", "endpoint", "parameter")125 new_params = index_by(load(GEN / "parameters.json"), "provider", "endpoint", "parameter")126 new_par, rem_par = diff_sets(old_params, new_params)127128 old_beta = index_by(load(snap / "beta-headers.json"), "provider", "value")129 new_beta = index_by(load(GEN / "beta-headers.json"), "provider", "value")130 new_b, rem_b = diff_sets(old_beta, new_beta)131132 old_dep = index_by(load(snap / "deprecations.json"), "provider", "subject", "shutdown_date")133 new_dep = index_by(load(GEN / "deprecations.json"), "provider", "subject", "shutdown_date")134 new_d, _ = diff_sets(old_dep, new_dep)135136 # live discovery drift: ids in the live list but absent from the catalogue and vice-versa137 drift = {}138 for prov in ("openai", "anthropic", "xai", "gemini"):139 raw = ROOT / f"sources/{prov}/models-api-raw.json"140 try:141 j = json.loads(raw.read_text())142 live_ids = {d.get("id") or d.get("name", "").replace("models/", "") for d in (j.get("data") or j.get("models") or [])}143 except Exception: # noqa: BLE001144 live_ids = set()145 cat_ids = {k[1] for k in new_models if k[0] == prov}146 drift[prov] = {"live_not_in_catalogue": sorted(live_ids - cat_ids),147 # only generation-capable, non-alias, non-retired records are expected in the live list148 "catalogue_live_verified_but_gone": sorted(149 k[1] for k, m in new_models.items() if k[0] == prov and "LIVE_VERIFIED" in (m.get("status") or [])150 and live_ids and k[1] not in live_ids151 and not ({"RETIRED", "DEPRECATED"} & set(m.get("status") or []))152 and m.get("record_kind", m.get("kind", "model")) not in ("alias", "snapshot", "id_only")153 and not m.get("alias_of") and not m.get("resolves_to")154 and (m.get("canonical_model") or k[1]) == k[1])}155 md = ROOT / "sources" / prov / "models-diff.json"156 if md.exists():157 try:158 drift[prov]["models_api_diff"] = json.loads(md.read_text())159 except Exception: # noqa: BLE001160 pass161162 pages = {}163 for prov in ("openai", "anthropic", "xai", "gemini"):164 pc = ROOT / "sources" / prov / "pages-changes.json"165 if pc.exists():166 pages[prov] = json.loads(pc.read_text())167168 changes = {"run_at": now(), "baseline_snapshot": str(snap.relative_to(ROOT)),169 "new_models": new_m, "removed_models": rem_m, "model_status_changes": status_changes,170 "context_window_changes": ctx_changes, "model_pricing_changes": price_changes,171 "new_endpoints": new_e, "removed_endpoints": rem_e, "new_tools": new_tools, "removed_tools": rem_tools,172 "new_prices": new_prices, "removed_prices": rem_prices, "changed_prices": changed_prices,173 "new_parameters": new_par, "removed_parameters": rem_par, "new_beta_headers": new_b,174 "removed_beta_headers": rem_b, "new_deprecations": new_d, "live_drift": drift, "doc_pages": pages}175 REPORTS.mkdir(exist_ok=True)176 (REPORTS / "changes.json").write_text(json.dumps(changes, indent=1, default=str))177178 def sec(title: str, items) -> str:179 items = list(items)180 body = "\n".join(f"- {i}" for i in items) if items else "- (none)"181 return f"## {title} ({len(items)})\n\n{body}\n\n"182183 def fmt(keys):184 return [" / ".join(k) for k in keys]185186 md = [f"# Atlas change report\n\nRun: {changes['run_at']} \nBaseline: `{changes['baseline_snapshot']}`\n\n"]187 md.append(sec("NEW MODELS", fmt(new_m)))188 md.append(sec("REMOVED MODELS", fmt(rem_m)))189 md.append(sec("MODEL ID CHANGES / STATUS CHANGES", [f"{' / '.join(k)}: {o} → {n}" for k, o, n in status_changes]))190 md.append(sec("NEW ENDPOINTS", fmt(new_e)))191 md.append(sec("REMOVED ENDPOINTS", fmt(rem_e)))192 md.append(sec("NEW TOOLS", fmt(new_tools)))193 md.append(sec("REMOVED TOOLS", fmt(rem_tools)))194 md.append(sec("PRICING CHANGES", [f"{' / '.join(k)}: {o} → {n}" for k, o, n in changed_prices]195 + [f"NEW {' / '.join(k)}" for k in new_prices] + [f"REMOVED {' / '.join(k)}" for k in rem_prices]))196 md.append(sec("CONTEXT WINDOW CHANGES", [f"{' / '.join(k)}: ctx {a}→{b}, max_out {c}→{d}" for k, a, b, c, d in ctx_changes]))197 md.append(sec("NEW PARAMETERS", fmt(new_par)))198 md.append(sec("REMOVED PARAMETERS", fmt(rem_par)))199 md.append(sec("NEW BETA FEATURES", fmt(new_b)))200 md.append(sec("DEPRECATIONS (new entries)", fmt(new_d)))201 for prov, d in drift.items():202 md.append(sec(f"LIVE DRIFT — {prov}: live ids missing from catalogue", d["live_not_in_catalogue"]))203 md.append(sec(f"LIVE DRIFT — {prov}: catalogue LIVE_VERIFIED ids no longer listed", d["catalogue_live_verified_but_gone"]))204 if "models_api_diff" in d:205 mad = d["models_api_diff"]206 md.append(sec(f"MODELS API DIFF — {prov}: new", mad.get("new", mad.get("added", []))))207 md.append(sec(f"MODELS API DIFF — {prov}: removed", mad.get("removed", [])))208 for prov, p in pages.items():209 md.append(sec(f"PAGES TO RE-REVIEW — {prov}: changed", p.get("changed", [])))210 md.append(sec(f"PAGES TO RE-REVIEW — {prov}: new", p.get("new", [])))211 md.append(sec(f"PAGES TO RE-REVIEW — {prov}: removed", p.get("removed", [])))212 (REPORTS / "changes.md").write_text("".join(md))213 print(f"wrote {REPORTS / 'changes.md'}")214215 if not args.no_links:216 run([PY, "scripts/verify_links.py", "--offline"])217 return 0218219220if __name__ == "__main__":221 sys.exit(main())222