"""API 1.1 contract tests against the live local database (docs/API.md). Read-only. Counts are never asserted as fixed numbers.""" from __future__ import annotations from collections.abc import AsyncIterator import pytest from httpx import ASGITransport, AsyncClient from aiatlas import db from aiatlas.api.main import app from aiatlas.config import settings from aiatlas.services import cache ADMIN = {"x-aia-admin-token": settings.admin_token or "dev-admin-token"} MODEL, MODEL_B, BENCH = "claude-opus-5", "claude-sonnet-5", "gpqa" @pytest.fixture async def client() -> AsyncIterator[AsyncClient]: await cache.cache_invalidate() async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test", timeout=120) as c: yield c await cache.close() await db.dispose() async def test_models_canonical_universe_and_artifacts(client: AsyncClient) -> None: r = await client.get("/api/v1/models", params={"limit": 200}) assert r.status_code == 200 body = r.json() assert body["universe"] == "canonical models" and body["items"] assert all(it["entity_type"] == "model" for it in body["items"]) assert all("identity_confidence" in it for it in body["items"]) r2 = await client.get("/api/v1/models", params={"limit": 200, "include": "artifacts"}) assert r2.status_code == 200 and r2.json()["universe"] == "models+artifacts" and r2.json()["total"] >= body["total"] for it in r2.json()["items"]: if it["entity_type"] == "artifact": assert "canonical" in it and "artifact_kind" in it facets = (await client.get("/api/v1/models", params={"facets": 1, "limit": 1})).json()["facets"] assert {"families", "licenses", "trust", "definitions"} <= set(facets) assert all({"value", "label", "category", "count"} <= set(x) for x in facets["licenses"]) lic = await client.get("/api/v1/models", params={"license": "Apache 2.0", "limit": 5}) # raw label → canonical key assert lic.status_code == 200 assert (await client.get("/api/v1/models", params={"family": "Claude", "limit": 2})).status_code == 200 async def test_model_detail_v11_blocks(client: AsyncClient) -> None: d = (await client.get(f"/api/v1/models/{MODEL}")).json() for block in ("family", "artifacts", "deployments", "identity", "openness", "version_history", "benchmarks"): assert block in d, block assert {"items", "total"} <= set(d["artifacts"]) and {"canonical_model", "official_checkpoints", "third_party_artifacts", "provider_deployments", "api_aliases"} <= set(d["identity"]) assert d["openness"]["category"] in ("open-source", "open-weights", "restricted-weights", "proprietary", "unknown") and "dimensions" in d["openness"] assert all({"property", "transitions"} <= set(v) for v in d["version_history"]) if d["deployments"]: dep = d["deployments"][0] assert {"model", "provider", "prices", "features", "status", "valid_from"} <= set(dep) and {"input", "output", "native_units"} <= set(dep["prices"]) assert {"items", "total_rows", "note"} <= set(d["benchmarks"]) for b in d["benchmarks"]["items"]: for m in b["metrics"]: for g in m["groups"]: assert {"config_key", "comparability_group", "n_rows", "best"} <= set(g) and "trust_level" in g["best"] async def test_leaderboard_one_row_per_model(client: AsyncClient) -> None: r = await client.get(f"/api/v1/benchmarks/{BENCH}/leaderboard", params={"limit": 1000}) assert r.status_code == 200 body = r.json() assert body["group"] and {"metric", "config_key", "label", "n", "model_count"} <= set(body["group"]) and body["groups"] ids = [it["model"]["id"] for it in body["items"]] assert ids and len(ids) == len(set(ids)), "one row per model" ranks = [it["rank"] for it in body["items"]] assert ranks[0] == 1 and ranks == sorted(ranks) first = body["items"][0] assert {"rank", "model", "score", "trust_level", "config", "config_key", "comparability", "comparability_reasons", "delta_rank", "n_rows"} <= set(first) assert first["comparability"] in ("comparable", "partially-comparable", "not-comparable") only = (await client.get(f"/api/v1/benchmarks/{BENCH}/leaderboard", params={"comparable_only": 1, "limit": 50})).json() assert all(it["comparability"] == "comparable" for it in only["items"]) alias = await client.get("/api/v1/benchmarks/GPQA Diamond/leaderboard", params={"limit": 3}) # alias resolution assert alias.status_code == 200 and alias.json()["benchmark"]["slug"] == BENCH listing = (await client.get("/api/v1/benchmarks")).json() b = next(x for x in listing["items"] if x["slug"] == BENCH) assert {"family", "variant", "metric", "direction", "result_count", "model_count", "leader", "groups", "trust_mix", "category"} <= set(b) res = (await client.get(f"/api/v1/benchmarks/{BENCH}/results", params={"limit": 3, "metric": "accuracy"})).json() assert res["items"] and all(it["metric"].lower() == "accuracy" for it in res["items"]) fr = (await client.get(f"/api/v1/benchmarks/{BENCH}/frontier")).json() assert "series" in fr and all({"group", "points", "current_leader"} <= set(s) for s in fr["series"]) async def test_matrix_shape(client: AsyncClient) -> None: body = (await client.get("/api/v1/benchmarks/matrix", params={"limit": 5})).json() assert {"columns", "rows", "total_rows", "methodology", "min_cells"} <= set(body) assert 1 <= len(body["columns"]) <= 12 and all({"id", "slug", "metric", "config_key", "group_label"} <= set(c) for c in body["columns"]) col_ids = [c["id"] for c in body["columns"]] for row in body["rows"]: assert set(row["cells"]) == set(col_ids) and row["n_cells"] >= 3 and {"model", "mean_rank"} <= set(row) for cell in row["cells"].values(): if cell: assert {"score", "rank", "trust_level", "config_key", "comparability"} <= set(cell) sub = (await client.get("/api/v1/benchmarks/matrix", params={"benchmarks": f"{BENCH},swe-bench-verified", "org": "anthropic"})).json() assert [c["slug"] for c in sub["columns"]] == [BENCH, "swe-bench-verified"] async def test_changes_default_excludes_backfill(client: AsyncClient) -> None: body = (await client.get("/api/v1/changes", params={"limit": 20})).json() assert body["date_field"] == "occurred" and body["include_backfill"] is False assert all(e["is_backfill"] is False and "occurred_at" in e for e in body["items"]) if body["items"] and body["next_before"]: assert body["next_before"] == body["items"][-1]["occurred_at"] obs = (await client.get("/api/v1/changes", params={"limit": 3, "include_backfill": 1, "date_field": "observed"})).json() assert obs["date_field"] == "observed" and obs["total"] >= body["total"] or obs["total"] == 10000 daily = (await client.get("/api/v1/changes/daily")).json() assert {"date", "counts", "sections", "new_models", "today", "backfill_excluded"} <= set(daily) for s in daily["today"]: assert {"key", "label", "items", "total"} <= set(s) and all({"sources", "documents", "grouped_events"} <= set(i) for i in s["items"]) tl = (await client.get(f"/api/v1/entities/{MODEL}/timeline", params={"limit": 3})).json() assert tl["date_field"] == "occurred" and all(e["is_backfill"] is False for e in tl["items"]) gt = (await client.get("/api/v1/timeline", params={"limit": 5})).json() assert gt["include_backfill"] is False async def test_stats_definitions(client: AsyncClient) -> None: s = (await client.get("/api/v1/stats")).json() assert {"organizations_total", "artifacts", "model_families", "definitions", "change_events_live_24h", "change_events_24h"} <= set(s) assert s["organizations_total"] == sum(s["entities"].get(t, 0) for t in ("company", "organization", "lab", "university")) companies = (await client.get("/api/v1/companies", params={"limit": 1})).json() assert companies["total"] == s["organizations_total"] assert "models" in s["definitions"] and "artifact" in s["entities"] and "model_family" in s["entities"] async def test_frontier_shape(client: AsyncClient) -> None: body = (await client.get("/api/v1/frontier")).json() for key in ("latest_major_models", "benchmark_frontier", "price_frontier", "context_frontier", "open_weight_frontier", "efficiency_frontier", "agentic_frontier", "multimodal_frontier", "recent_frontier_movements", "methodology"): assert key in body, key for b in body["benchmark_frontier"]: assert {"benchmark", "group", "leader", "second", "gap"} <= set(b) and b["group"]["n"] >= 20 assert {"cheapest_output", "cheapest_output_1m_context", "frontier_models", "composition"} <= set(body["price_frontier"]) assert {"points", "frontier"} <= set(body["efficiency_frontier"]) assert "dimensions" in body["open_weight_frontier"] and "score" not in str(body["open_weight_frontier"]["dimensions"]) async def test_pareto_frontier_ids_are_efficient(client: AsyncClient) -> None: body = (await client.get("/api/v1/pareto", params={"benchmark": BENCH})).json() assert {"group", "points", "frontier", "methodology", "x", "y"} <= set(body) pts = {p["id"]: p for p in body["points"]} front = set(body["frontier"]) assert front <= set(pts) hib = body["group"]["higher_is_better"] for fid in front: # no other point is at least as good on both axes and strictly better on one p = pts[fid] for o in pts.values(): if o["id"] == fid: continue better_y = o["y"] > p["y"] if hib else o["y"] < p["y"] ge_y = o["y"] >= p["y"] if hib else o["y"] <= p["y"] assert not (o["x"] <= p["x"] and ge_y and (o["x"] < p["x"] or better_y)), (fid, o["id"]) assert all(p["pareto"] == (p["id"] in front) for p in body["points"]) assert (await client.get("/api/v1/pareto", params={"benchmark": BENCH, "x": "latency"})).status_code == 400 async def test_cost_routes(client: AsyncClient) -> None: body = (await client.get("/api/v1/cost", params={"model": MODEL, "input_tokens": 1000, "output_tokens": 500, "requests_per_day": 100, "cached_share": 0.5})).json() assert {"items", "methodology", "inputs"} <= set(body) for it in body["items"]: c, p = it["cost"], it["deployment"]["prices"] if c["per_request"] is not None and p["input"] is not None and p["output"] is not None: assert c["daily"] == pytest.approx(c["per_request"] * 100) and c["monthly"] == pytest.approx(c["daily"] * 30) ctx = (await client.get("/api/v1/cost/context", params={"tokens": 100000, "limit": 5})).json() assert ctx["items"] and all(it["context_length"] >= 100000 and it["cost_usd"] == pytest.approx(it["deployment"]["prices"]["input"] * 0.1) for it in ctx["items"]) deps = (await client.get("/api/v1/deployments", params={"model": MODEL})).json() assert deps["items"] and all(d["status"] == "active" for d in deps["items"]) async def test_find_a_model_returns_why(client: AsyncClient) -> None: body = (await client.get("/api/v1/find-a-model", params={"use_case": "coding", "limit": 5})).json() assert {"matches", "filters_applied", "note", "rules"} <= set(body) and body["matches"] for m in body["matches"]: assert m["why"] and {"model", "observed"} <= set(m) and "best_rank" in m["observed"] assert "score" not in m # no composite winner score local = (await client.get("/api/v1/find-a-model", params={"use_case": "local", "memory_gb": 64, "limit": 3})).json() assert all("estimated_fit" in m and m["estimated_fit"]["estimated"] is True for m in local["matches"]) assert (await client.get("/api/v1/find-a-model", params={"use_case": "nope"})).status_code == 400 async def test_open_run_locally_families_graph(client: AsyncClient) -> None: op = (await client.get("/api/v1/open", params={"limit": 3})).json() assert {"items", "summary", "note"} <= set(op) and all({"licence", "dimensions", "best_results", "hardware_fit", "providers"} <= set(i) for i in op["items"]) rl = (await client.get("/api/v1/run-locally", params={"memory_gb": 64, "limit": 3})).json() assert rl["estimated"] is True and all(i["fit"]["fits"] and "breakdown" in i["fit"] for i in rl["items"]) fams = (await client.get("/api/v1/families", params={"limit": 3})).json() assert fams["items"] and all({"name", "model_count", "canonical", "licenses", "benchmark_best"} <= set(f) for f in fams["items"]) fd = await client.get(f"/api/v1/families/{fams['items'][0]['slug']}") assert fd.status_code == 200 and {"members", "timeline", "lineage", "artifacts_count"} <= set(fd.json()) g = (await client.get("/api/v1/graph/explore", params={"node": MODEL, "mode": "company", "depth": 2, "limit": 20})).json() assert {"nodes", "edges", "truncated", "counts"} <= set(g) and len(g["nodes"]) <= 20 assert (await client.get("/api/v1/graph/explore", params={"node": MODEL, "mode": "nope"})).status_code == 400 async def test_search_compiler_v2_response(client: AsyncClient) -> None: body = (await client.get("/api/v1/search", params={"q": "Anthropic models released since 2025"})).json() q = body["query"] assert q["version"] == 2 and {"compiled", "residual", "unrecognised"} <= set(q) assert q["organization"] == "Anthropic" and q["year_from"] == 2025 assert any(c["filter"] == "organization" and isinstance(c["value"], dict) for c in q["compiled"]) assert body["items"] and all(it["organization"]["slug"] == "anthropic" for it in body["items"]) unknown = (await client.get("/api/v1/search", params={"q": "Foobarbaz models released since 2025"})).json()["query"] assert "organization" not in unknown and "Foobarbaz" in unknown["unrecognised"] async def test_frontier_price_join_and_cache_namespace(client: AsyncClient) -> None: """Regression: frontier ids and prices come from the SAME database — cheapest_frontier must exist whenever frontier offers exist.""" from aiatlas.services import cache assert cache.api_key("x").startswith(f"aia:api:{cache.namespace()}:") and len(cache.namespace()) == 8 idx = (await client.get("/api/v1/prices/index", params={"days": 7})).json() offers_today = idx["series"][-1]["sample"]["frontier_offers"] if offers_today > 0: assert idx["cheapest_frontier"] is not None and idx["cheapest_frontier"]["output"] > 0 cheapest = (await client.get("/api/v1/prices", params={"sort": "cheapest_frontier", "limit": 5})).json() assert cheapest["items"] and cheapest["items"][0]["output_per_mtok"] > 0 fr = (await client.get("/api/v1/frontier")).json() assert fr["price_frontier"]["cheapest_output"] is not None and fr["price_frontier"]["frontier_models"] > 0 for key in ("new_listings_30d", "delistings_30d", "price_changes_30d"): assert {"count", "items", "definition"} <= set(idx[key]) and len(idx[key]["items"]) <= 50 async def test_pulse_items_and_price_delta(client: AsyncClient) -> None: from aiatlas.api.routers.intelligence import price_delta d = price_delta({"input_per_mtok": 1.0, "output_per_mtok": 4.0}, {"input_per_mtok": 0.5, "output_per_mtok": 5.0}) assert d["output_percent"] == 25.0 and d["input_percent"] == -50.0 and d["percent"] == 25.0 assert price_delta(None, {"output_per_mtok": 1}) is None and price_delta(2.0, 3.0)["percent"] == 50.0 body = (await client.get("/api/v1/pulse", params={"days": 90})).json() pc = body["counters"]["price_changes"] assert "items" in pc and "median_percent" in pc and isinstance(body["counters"]["new_models_1m_context"]["items"], list) if pc["value"]: assert pc["items"] and {"summary", "percent_change", "provider", "occurred_at"} <= set(pc["items"][0]) async def test_provider_detail_deployments_and_listing_status(client: AsyncClient) -> None: providers = (await client.get("/api/v1/providers")).json()["items"] slug = providers[0]["slug"] d = (await client.get(f"/api/v1/providers/{slug}")).json() assert "deployments" in d and "removed" in d outs = [x["prices"]["output"] for x in d["deployments"] if x["prices"]["output"] is not None] assert outs == sorted(outs) and all(x["status"] == "active" for x in d["deployments"]) and all(x["status"] == "delisted" for x in d["removed"]) gone = (await client.get("/api/v1/deployments", params={"current": 0, "status": "delisted", "limit": 5})).json() assert gone["status"] == "delisted" and all(x["status"] == "delisted" for x in gone["items"]) both = (await client.get("/api/v1/deployments", params={"current": 0, "limit": 200})).json() assert both["status"] == "all" and both["total"] >= gone["total"] async def test_models_best_price_and_family_scores_and_matrix_cells(client: AsyncClient) -> None: rows = (await client.get("/api/v1/models", params={"limit": 50, "sort": "cheapest"})).json()["items"] priced = [r for r in rows if r.get("best_price")] assert priced and {"input_per_mtok", "output_per_mtok", "provider", "providers"} <= set(priced[0]["best_price"]) and priced[0]["best_price"]["provider"]["slug"] fams = (await client.get("/api/v1/families", params={"limit": 3})).json()["items"] for b in fams[0]["benchmark_best"].values(): assert {"rank", "score", "metric", "model"} <= set(b) fd = (await client.get(f"/api/v1/families/{fams[0]['slug']}")).json() assert all("benchmark_best" in m and all({"benchmark", "rank", "score"} <= set(x) for x in m["benchmark_best"]) for m in fd["members"]) mx = (await client.get("/api/v1/benchmarks/matrix", params={"limit": 3, "since": "2024-01-01"})).json() assert mx["filters"]["since"] == "2024-01-01" for row in mx["rows"]: assert row["model"]["release_date"] and row["model"]["release_date"][:10] >= "2024-01-01" assert all("observed_at" in c for c in row["cells"].values() if c) pts = (await client.get("/api/v1/pareto", params={"benchmark": BENCH})).json()["points"] assert pts and {"context_length", "parameter_count"} <= set(pts[0]) fw = (await client.get("/api/v1/explore/framework", params={"limit": 20})).json() assert all("kind" in it for it in fw["items"]) op = (await client.get("/api/v1/open", params={"limit": 1})).json() assert op["summary"]["new_30d"] <= op["total"] and "new_30d_definition" in op["summary"] async def test_etag_304(client: AsyncClient) -> None: r1 = await client.get("/api/v1/stats") etag = r1.headers.get("etag") assert etag and etag.startswith('W/"') and "max-age" in r1.headers.get("cache-control", "") r2 = await client.get("/api/v1/stats", headers={"if-none-match": etag}) assert r2.status_code == 304 and r2.content == b"" and r2.headers.get("etag") == etag admin = await client.get("/api/v1/admin/overview", headers=ADMIN) assert "etag" not in admin.headers # admin responses are never cacheable async def test_admin_rate_limit_before_auth(client: AsyncClient) -> None: from aiatlas.api.common import _hits _hits.clear() codes = [(await client.get("/api/v1/admin/overview", headers={"x-aia-admin-token": "wrong"})).status_code for _ in range(11)] assert codes[:10] == [401] * 10 and codes[10] == 429, codes # 10 failed auths per minute per IP, then 429 before any token check _hits.clear() assert (await client.get("/api/v1/admin/overview", headers=ADMIN)).status_code == 200 async def test_admin_workbenches_and_audit(client: AsyncClient) -> None: from aiatlas.api.common import _hits _hits.clear() q = (await client.get("/api/v1/admin/quality", headers=ADMIN)).json() for key in ("duplicate_candidates", "taxonomy_violations", "impossible_values", "conflicting_t1_claims", "models_without_organization", "orphan_benchmark_results", "unresolved_provider_deployments", "quantisations_typed_as_models", "stale_sources", "empty_public_categories", "quarantined_runs_pending", "review_queue_priority"): assert key in q, key assert {"count", "sample"} <= set(q["models_without_organization"]) er = (await client.get("/api/v1/admin/entity-resolution", params={"limit": 3}, headers=ADMIN)).json() assert {"items", "decisions"} <= set(er) and all({"a", "b", "signals", "hint"} <= set(i) for i in er["items"]) for path in ("/api/v1/admin/anomalies", "/api/v1/admin/quarantine", "/api/v1/admin/audit?limit=5"): assert (await client.get(path, headers=ADMIN)).status_code == 200, path audit = (await client.get("/api/v1/admin/audit?limit=5", headers=ADMIN)).json() assert audit["items"] and audit["items"][0]["actor"] == "admin" and audit["items"][0]["action"].startswith("GET /api/v1/admin/") assert (await client.post("/api/v1/admin/quarantine/nope", headers=ADMIN, json={"action": "release"})).status_code in (404, 501) assert (await client.post("/api/v1/admin/anomalies/nope", headers=ADMIN, json={"status": "resolved"})).status_code == 404 async def test_claims_provenance_licenses_misc(client: AsyncClient) -> None: cl = (await client.get(f"/api/v1/entities/{MODEL}/claims", params={"limit": 2})).json() assert cl["items"] and cl["status"] == "current" detail = (await client.get(f"/api/v1/claims/{cl['items'][0]['id']}")).json() assert {"claim", "entity", "chain", "source", "extractor", "evidence"} <= set(detail) and {"snapshot_id", "document_url", "archived"} <= set(detail["evidence"]) prov = (await client.get(f"/api/v1/entities/{MODEL}/provenance/context_length")).json() assert {"value", "source", "tier", "observed_at", "claim_id", "conflicts", "history_count", "snapshot_id"} <= set(prov) assert (await client.get(f"/api/v1/entities/{MODEL}/provenance/not_a_property")).status_code == 404 lic = (await client.get("/api/v1/licenses")).json() assert lic["items"] and all({"key", "commercial_use", "models"} <= set(i) for i in lic["items"]) assert (await client.get("/api/v1/licenses/apache-2.0")).json()["key"] == "Apache-2.0" meth = (await client.get("/api/v1/methodology")).json() assert {"openness", "trust_levels", "comparability", "counters", "anomaly_checks", "event_semantics"} <= set(meth) tr = (await client.get("/api/v1/trending", params={"kind": "most_changed"})).json() assert tr["kind"] == "most_changed" and "definition" in tr cmp_ = (await client.get("/api/v1/compare", params={"ids": f"{MODEL},{MODEL_B}", "diff_only": 1, "mode": "models"})).json() assert cmp_["diff_only"] is True and "comparability" in cmp_ for d in cmp_["dimensions"]: vals = [repr(sorted(map(str, it["values"][d["key"]])) if isinstance(it["values"][d["key"]], list) else it["values"][d["key"]]) for it in cmp_["items"]] assert len(set(vals)) > 1, d["key"] diff = (await client.get(f"/api/v1/models/{MODEL}/diff/{MODEL_B}")).json() assert all("delta" in d for d in diff["dimensions"]) assert (await client.get("/api/v1/compare", params={"ids": f"{MODEL},{MODEL_B}", "mode": "providers"})).status_code == 400 df = (await client.get("/api/v1/diff", params={"a": "2026-01-01", "b": "2026-12-31", "limit": 5})).json() assert {"new_benchmark_leaders", "provider_changes", "hardware_changes", "context_changes", "retired_models"} <= set(df) and df["include_artifacts"] is False tm = (await client.get("/api/v1/time-machine", params={"date": "2025-06-01", "scope": "models", "limit": 3})).json() assert {"reconstructed", "first_entity_at", "note", "models"} <= set(tm) and all("attributes_as_of" in m for m in tm["models"]["items"]) pulse = (await client.get("/api/v1/pulse", params={"days": 7})).json() assert all({"value", "definition"} <= set(v) for v in pulse["counters"].values()) idx = (await client.get("/api/v1/prices/index", params={"days": 14})).json() assert {"series", "cheapest_frontier", "distribution", "new_listings_30d", "delistings_30d", "methodology"} <= set(idx) assert all({"median_frontier_output", "median_open_output", "median_embedding_input", "sample"} <= set(s) for s in idx["series"]) prov_list = (await client.get("/api/v1/providers")).json()["items"] assert all({"input_price_distribution", "output_price_distribution", "models_added_30d", "models_removed_30d", "price_changes_30d", "organizations_covered", "features_supported"} <= set(p) for p in prov_list) hw = (await client.get("/api/v1/hardware/apple-m3-ultra/fit", params={"limit": 3})).json() assert hw["estimated"] is True and hw["memory_options_gb"]