#!/usr/bin/env python3
"""Re-run the atlas pipeline and report what changed.

    python3 scripts/update_atlas.py                 # full run (network: indexes, pages, /v1/models)
    python3 scripts/update_atlas.py --no-crawl      # skip docs crawl (offline rebuild + diff)
    python3 scripts/update_atlas.py --no-live       # skip /v1/models discovery

Steps
 1. Snapshot current generated/*.json into reports/snapshots/<UTC timestamp>/ (baseline for diff).
 2. Refresh llms.txt indexes + crawl all doc pages (scripts/crawl_docs.py --refresh-index --follow) for both providers
    → sources/<provider>/pages-changes.json lists new/changed/removed pages.
 3. Live model discovery (scripts/discover_openai_models.py, scripts/discover_anthropic_models.py) → models-diff.json.
 4. Rebuild generated/ from fragments (scripts/build_generated.py).
 5. Diff baseline vs new generated files → reports/changes.md (+ reports/changes.json):
    NEW/REMOVED MODELS, MODEL ID CHANGES (alias/snapshot), NEW/REMOVED ENDPOINTS, NEW TOOLS, PRICING CHANGES,
    CONTEXT WINDOW CHANGES, NEW BETA FEATURES, DEPRECATIONS, CHANGED DOC PAGES.
 6. Link check (offline mode) → reports/link-check.json.

NOTE: fragments (generated/fragments/**) are curated by agents/humans from the sources. A re-run detects drift
(live model list vs catalogue, changed doc pages) and flags it in reports/changes.md; it does not rewrite the
curated fragments automatically. The section "PAGES TO RE-REVIEW" tells a maintainer (or an agent) exactly which
source pages changed so the corresponding fragments can be refreshed.
"""
from __future__ import annotations

import argparse
import json
import shutil
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
GEN = ROOT / "generated"
REPORTS = ROOT / "reports"
PY = sys.executable


def now() -> str:
    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")


def run(cmd: list[str], check: bool = False) -> int:
    print("$", " ".join(cmd), flush=True)
    r = subprocess.run(cmd, cwd=ROOT)
    if check and r.returncode:
        sys.exit(r.returncode)
    return r.returncode


def load(p: Path) -> list[dict]:
    if not p.exists():
        return []
    try:
        d = json.loads(p.read_text())
        return d if isinstance(d, list) else []
    except Exception:  # noqa: BLE001
        return []


def index_by(recs: list[dict], *fields: str) -> dict[tuple, dict]:
    return {tuple(str(r.get(f)) for f in fields): r for r in recs}


def diff_sets(old: dict, new: dict) -> tuple[list, list]:
    return sorted(k for k in new if k not in old), sorted(k for k in old if k not in new)


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--no-crawl", action="store_true")
    ap.add_argument("--no-live", action="store_true")
    ap.add_argument("--no-links", action="store_true")
    args = ap.parse_args()

    stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
    snap = REPORTS / "snapshots" / stamp
    snap.mkdir(parents=True, exist_ok=True)
    for f in GEN.glob("*.json"):
        shutil.copy2(f, snap / f.name)
    print(f"baseline snapshot: {snap.relative_to(ROOT)}")

    if not args.no_crawl:
        for prov in ("openai", "anthropic", "gemini"):
            run([PY, "scripts/crawl_docs.py", prov, "--refresh-index", "--follow"])
        # docs.x.ai refuses direct .md fetches from non-browser clients → rebuild pages from the official llms-full.txt
        run([PY, "scripts/split_xai_llms_full.py"])
    if not args.no_live:
        for s in ("scripts/discover_openai_models.py", "scripts/discover_anthropic_models.py",
                  "scripts/discover_xai_models.py", "scripts/discover_gemini_models.py"):
            if (ROOT / s).exists():
                run([PY, s])
    run([PY, "scripts/build_generated.py"], check=True)

    old_models = index_by(load(snap / "models.json"), "provider", "id")
    new_models = index_by(load(GEN / "models.json"), "provider", "id")
    new_m, rem_m = diff_sets(old_models, new_models)
    ctx_changes, price_changes, status_changes = [], [], []
    for k, m in new_models.items():
        o = old_models.get(k)
        if not o:
            continue
        if o.get("context_window") != m.get("context_window") or o.get("max_output") != m.get("max_output"):
            ctx_changes.append((k, o.get("context_window"), m.get("context_window"), o.get("max_output"), m.get("max_output")))
        if json.dumps(o.get("pricing"), sort_keys=True, default=str) != json.dumps(m.get("pricing"), sort_keys=True, default=str):
            price_changes.append((k, o.get("pricing"), m.get("pricing")))
        if sorted(o.get("status") or []) != sorted(m.get("status") or []):
            status_changes.append((k, o.get("status"), m.get("status")))

    old_ep = index_by(load(snap / "endpoints.json"), "provider", "method", "path")
    new_ep = index_by(load(GEN / "endpoints.json"), "provider", "method", "path")
    new_e, rem_e = diff_sets(old_ep, new_ep)

    old_t = index_by(load(snap / "tools.json"), "provider", "type")
    new_t = index_by(load(GEN / "tools.json"), "provider", "type")
    new_tools, rem_tools = diff_sets(old_t, new_t)

    old_p = index_by(load(snap / "pricing.json"), "provider", "model_or_service", "dimension", "tier", "unit")
    new_p = index_by(load(GEN / "pricing.json"), "provider", "model_or_service", "dimension", "tier", "unit")
    new_prices, rem_prices = diff_sets(old_p, new_p)
    changed_prices = [(k, old_p[k].get("price"), new_p[k].get("price")) for k in new_p if k in old_p and old_p[k].get("price") != new_p[k].get("price")]

    old_params = index_by(load(snap / "parameters.json"), "provider", "endpoint", "parameter")
    new_params = index_by(load(GEN / "parameters.json"), "provider", "endpoint", "parameter")
    new_par, rem_par = diff_sets(old_params, new_params)

    old_beta = index_by(load(snap / "beta-headers.json"), "provider", "value")
    new_beta = index_by(load(GEN / "beta-headers.json"), "provider", "value")
    new_b, rem_b = diff_sets(old_beta, new_beta)

    old_dep = index_by(load(snap / "deprecations.json"), "provider", "subject", "shutdown_date")
    new_dep = index_by(load(GEN / "deprecations.json"), "provider", "subject", "shutdown_date")
    new_d, _ = diff_sets(old_dep, new_dep)

    # live discovery drift: ids in the live list but absent from the catalogue and vice-versa
    drift = {}
    for prov in ("openai", "anthropic", "xai", "gemini"):
        raw = ROOT / f"sources/{prov}/models-api-raw.json"
        try:
            j = json.loads(raw.read_text())
            live_ids = {d.get("id") or d.get("name", "").replace("models/", "") for d in (j.get("data") or j.get("models") or [])}
        except Exception:  # noqa: BLE001
            live_ids = set()
        cat_ids = {k[1] for k in new_models if k[0] == prov}
        drift[prov] = {"live_not_in_catalogue": sorted(live_ids - cat_ids),
                       # only generation-capable, non-alias, non-retired records are expected in the live list
                       "catalogue_live_verified_but_gone": sorted(
                           k[1] for k, m in new_models.items() if k[0] == prov and "LIVE_VERIFIED" in (m.get("status") or [])
                           and live_ids and k[1] not in live_ids
                           and not ({"RETIRED", "DEPRECATED"} & set(m.get("status") or []))
                           and m.get("record_kind", m.get("kind", "model")) not in ("alias", "snapshot", "id_only")
                           and not m.get("alias_of") and not m.get("resolves_to")
                           and (m.get("canonical_model") or k[1]) == k[1])}
        md = ROOT / "sources" / prov / "models-diff.json"
        if md.exists():
            try:
                drift[prov]["models_api_diff"] = json.loads(md.read_text())
            except Exception:  # noqa: BLE001
                pass

    pages = {}
    for prov in ("openai", "anthropic", "xai", "gemini"):
        pc = ROOT / "sources" / prov / "pages-changes.json"
        if pc.exists():
            pages[prov] = json.loads(pc.read_text())

    changes = {"run_at": now(), "baseline_snapshot": str(snap.relative_to(ROOT)),
               "new_models": new_m, "removed_models": rem_m, "model_status_changes": status_changes,
               "context_window_changes": ctx_changes, "model_pricing_changes": price_changes,
               "new_endpoints": new_e, "removed_endpoints": rem_e, "new_tools": new_tools, "removed_tools": rem_tools,
               "new_prices": new_prices, "removed_prices": rem_prices, "changed_prices": changed_prices,
               "new_parameters": new_par, "removed_parameters": rem_par, "new_beta_headers": new_b,
               "removed_beta_headers": rem_b, "new_deprecations": new_d, "live_drift": drift, "doc_pages": pages}
    REPORTS.mkdir(exist_ok=True)
    (REPORTS / "changes.json").write_text(json.dumps(changes, indent=1, default=str))

    def sec(title: str, items) -> str:
        items = list(items)
        body = "\n".join(f"- {i}" for i in items) if items else "- (none)"
        return f"## {title} ({len(items)})\n\n{body}\n\n"

    def fmt(keys):
        return [" / ".join(k) for k in keys]

    md = [f"# Atlas change report\n\nRun: {changes['run_at']}  \nBaseline: `{changes['baseline_snapshot']}`\n\n"]
    md.append(sec("NEW MODELS", fmt(new_m)))
    md.append(sec("REMOVED MODELS", fmt(rem_m)))
    md.append(sec("MODEL ID CHANGES / STATUS CHANGES", [f"{' / '.join(k)}: {o} → {n}" for k, o, n in status_changes]))
    md.append(sec("NEW ENDPOINTS", fmt(new_e)))
    md.append(sec("REMOVED ENDPOINTS", fmt(rem_e)))
    md.append(sec("NEW TOOLS", fmt(new_tools)))
    md.append(sec("REMOVED TOOLS", fmt(rem_tools)))
    md.append(sec("PRICING CHANGES", [f"{' / '.join(k)}: {o} → {n}" for k, o, n in changed_prices]
               + [f"NEW {' / '.join(k)}" for k in new_prices] + [f"REMOVED {' / '.join(k)}" for k in rem_prices]))
    md.append(sec("CONTEXT WINDOW CHANGES", [f"{' / '.join(k)}: ctx {a}→{b}, max_out {c}→{d}" for k, a, b, c, d in ctx_changes]))
    md.append(sec("NEW PARAMETERS", fmt(new_par)))
    md.append(sec("REMOVED PARAMETERS", fmt(rem_par)))
    md.append(sec("NEW BETA FEATURES", fmt(new_b)))
    md.append(sec("DEPRECATIONS (new entries)", fmt(new_d)))
    for prov, d in drift.items():
        md.append(sec(f"LIVE DRIFT — {prov}: live ids missing from catalogue", d["live_not_in_catalogue"]))
        md.append(sec(f"LIVE DRIFT — {prov}: catalogue LIVE_VERIFIED ids no longer listed", d["catalogue_live_verified_but_gone"]))
        if "models_api_diff" in d:
            mad = d["models_api_diff"]
            md.append(sec(f"MODELS API DIFF — {prov}: new", mad.get("new", mad.get("added", []))))
            md.append(sec(f"MODELS API DIFF — {prov}: removed", mad.get("removed", [])))
    for prov, p in pages.items():
        md.append(sec(f"PAGES TO RE-REVIEW — {prov}: changed", p.get("changed", [])))
        md.append(sec(f"PAGES TO RE-REVIEW — {prov}: new", p.get("new", [])))
        md.append(sec(f"PAGES TO RE-REVIEW — {prov}: removed", p.get("removed", [])))
    (REPORTS / "changes.md").write_text("".join(md))
    print(f"wrote {REPORTS / 'changes.md'}")

    if not args.no_links:
        run([PY, "scripts/verify_links.py", "--offline"])
    return 0


if __name__ == "__main__":
    sys.exit(main())