SPB Git forge
spboucher.ai ↗

spb/ai-atlas

Public
41commits 1branches 0releases
4.6 MBsize
maindefault branch
12 days agolast push
HTML 77.2% TypeScript 10.5% Python 9.6% JavaScript 2.5%

API polish: provenance source names, methodology labels, status vocabulary, zero-filled stats

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 13 days ago (Sep 11, 2026) parent 513f07e

7 changed files +106 −7

modified src/aiatlas/api/common.py +67 −2
@@ -65,7 +65,6 @@ class AtlasJSONResponse(JSONResponse):
65 65 # ------------------------------------------------------------------------------------------------------------------ pagination & parsing
66 66
67 67
68 −
69 68 class Pagination:
70 69 def __init__(self, limit: int = Query(50, ge=1, le=200), offset: int = Query(0, ge=0)):
71 70 self.limit = limit
@@ -266,6 +265,62 @@ def summary_attributes(entity_type: str, attrs: dict[str, Any] | None) -> dict[s
266 265 return out
267 266
268 267
268 +STATUS_VOCAB = ("active", "preview", "deprecated", "retired", "announced", "limited-availability", "unknown")
269 +STATUS_MAP = {"available": "active", "ga": "active", "general-availability": "active", "released": "active", "live": "active", "beta": "preview",
270 + "alpha": "preview", "experimental": "preview", "coming-soon": "announced", "upcoming": "announced", "sunset": "retired", "discontinued": "retired",
271 + "archived": "retired", "legacy": "deprecated", "limited": "limited-availability", "merged": "retired"}
272 +
273 +
274 +def normalize_status(value: Any) -> str:
275 + s = str(value or "").strip().lower().replace("_", "-").replace(" ", "-")
276 + s = STATUS_MAP.get(s, s)
277 + return s if s in STATUS_VOCAB else "unknown"
278 +
279 +
280 +MAIN_ENTITY_TYPES = ("model", "company", "paper", "provider", "benchmark", "hardware", "framework", "dataset", "tool", "repository")
281 +
282 +EVENT_TYPE_LABELS: dict[str, tuple[str, int]] = { # event_type -> (human label, default importance 0–3)
283 + "NEW_MODEL": ("New model", 3), "MODEL_UPDATED": ("Model updated", 1), "PRICE_CHANGED": ("Price change", 2), "CONTEXT_CHANGED": ("Context window change", 2),
284 + "MAX_OUTPUT_CHANGED": ("Max output change", 1), "PARAMETERS_CHANGED": ("Parameter count change", 2), "KNOWLEDGE_CUTOFF_CHANGED": ("Knowledge cutoff change", 1),
285 + "RELEASE_DATE_CHANGED": ("Release date change", 1), "STATUS_CHANGED": ("Status change", 2), "LICENSE_CHANGED": ("License change", 2),
286 + "OPENNESS_CHANGED": ("Openness change", 2), "CAPABILITIES_CHANGED": ("Capabilities change", 1), "PROPERTY_CHANGED": ("Property change", 1),
287 + "DEPRECATION_ANNOUNCED": ("Deprecation announced", 2), "RETIREMENT_ANNOUNCED": ("Retirement announced", 2), "BENCHMARK_RESULT": ("Benchmark result", 1),
288 + "BENCHMARK_UPDATED": ("Benchmark updated", 1), "NEW_PAPER": ("New paper", 2), "NEW_COMPANY": ("New company", 2), "NEW_ORGANIZATION": ("New organization", 2),
289 + "NEW_LAB": ("New lab", 2), "NEW_UNIVERSITY": ("New university", 1), "NEW_PROVIDER": ("New provider", 2), "NEW_HARDWARE": ("New hardware", 2),
290 + "NEW_FRAMEWORK": ("New framework", 2), "NEW_LIBRARY": ("New library", 1), "NEW_BENCHMARK": ("New benchmark", 2), "NEW_DATASET": ("New dataset", 2),
291 + "NEW_REPOSITORY": ("New repository", 1), "NEW_TOOL": ("New tool", 1), "PROVIDER_LISTED": ("Listed by provider", 2), "PROVIDER_DELISTED": ("Delisted by provider", 2),
292 + "ANNOUNCEMENT": ("Announcement", 2), "RELEASE": ("Release", 2), "VERSION_RELEASED": ("Version released", 1), "DOCUMENT_CHANGED": ("Source document changed", 0),
293 + "ENTITY_MERGED": ("Duplicate merged", 1), "CLAIM_RETRACTED": ("Claim retracted", 1),
294 +}
295 +
296 +
297 +def event_type_label(event_type: str) -> str:
298 + known = EVENT_TYPE_LABELS.get(event_type)
299 + return known[0] if known else event_type.replace("_", " ").capitalize()
300 +
301 +
302 +def event_type_importance(event_type: str) -> int:
303 + known = EVENT_TYPE_LABELS.get(event_type)
304 + return known[1] if known else 1
305 +
306 +
307 +async def enrich_provenance(conn: AsyncConnection, *provenances: dict[str, Any] | None) -> None:
308 + """Add `source_name` (from `sources.name`) to every provenance entry that carries a `source_id` — in place."""
309 + ids = {v.get("source_id") for p in provenances if p for v in p.values() if isinstance(v, dict) and v.get("source_id")}
310 + if not ids:
311 + return
312 + from aiatlas.db import fetch_all
313 +
314 + rows = await fetch_all(conn, "select id, name, domain, tier from sources where id = any(cast(:ids as text[]))", ids=sorted(ids))
315 + names = {r["id"]: r for r in rows}
316 + for p in provenances:
317 + for v in (p or {}).values():
318 + if isinstance(v, dict) and v.get("source_id") in names:
319 + src = names[v["source_id"]]
320 + v.setdefault("source_name", src["name"])
321 + v.setdefault("source_domain", src["domain"])
322 +
323 +
269 324 def org_of(row: dict[str, Any], prefix: str = "") -> dict[str, Any] | None:
270 325 oid = row.get(f"{prefix}organization_id")
271 326 if not oid:
@@ -279,8 +334,11 @@ def entity_summary(row: dict[str, Any], prefix: str = "") -> dict[str, Any] | No
279 334 return None
280 335 etype = g(f"{prefix}entity_type") or ""
281 336 desc = g(f"{prefix}description")
337 + raw_status = g(f"{prefix}status")
338 + status = normalize_status(raw_status)
282 339 return {"id": g(f"{prefix}id"), "entity_type": etype, "slug": g(f"{prefix}slug"), "name": g(f"{prefix}canonical_name"),
283 − "description": (desc[:280] if isinstance(desc, str) and len(desc) > 280 else desc), "status": g(f"{prefix}status"),
340 + "description": (desc[:280] if isinstance(desc, str) and len(desc) > 280 else desc), "status": status,
341 + **({"status_raw": raw_status} if raw_status and raw_status != status else {}),
284 342 "organization": org_of(row, prefix), "attributes": summary_attributes(etype, g(f"{prefix}attributes")),
285 343 "quality": g(f"{prefix}quality") or {}, "counts": g(f"{prefix}counts") or {},
286 344 "first_seen_at": g(f"{prefix}first_seen_at"), "last_seen_at": g(f"{prefix}last_seen_at"), "updated_at": g(f"{prefix}updated_at")}
@@ -374,12 +432,15 @@ __all__ = [
374 432 "ENTITY_FROM",
375 433 "EVENT_COLS",
376 434 "EVENT_FROM",
435 + "EVENT_TYPE_LABELS",
436 + "MAIN_ENTITY_TYPES",
377 437 "PAGINATION",
378 438 "PRICE_COLS",
379 439 "PRICE_FROM",
380 440 "RESULT_COLS",
381 441 "RESULT_FROM",
382 442 "RESULT_ORDER",
443 + "STATUS_VOCAB",
383 444 "TYPE_LABELS",
384 445 "ApiError",
385 446 "AtlasJSONResponse",
@@ -393,11 +454,15 @@ __all__ = [
393 454 "csv",
394 455 "day_bounds",
395 456 "dumps",
457 + "enrich_provenance",
396 458 "entity_cols",
397 459 "entity_join",
398 460 "entity_summary",
461 + "event_type_importance",
462 + "event_type_label",
399 463 "flip_order",
400 464 "normalize",
465 + "normalize_status",
401 466 "num_expr",
402 467 "org_of",
403 468 "page",
modified src/aiatlas/api/detail.py +5 −1
@@ -18,6 +18,7 @@ from aiatlas.api.common import (
18 18 RESULT_FROM,
19 19 RESULT_ORDER,
20 20 change_event,
21 + enrich_provenance,
21 22 entity_summary,
22 23 price_row,
23 24 result_row,
@@ -180,9 +181,12 @@ async def entity_detail(row: dict[str, Any]) -> dict[str, Any]:
180 181 async with connection() as conn:
181 182 return await fn(conn, *args, **kw)
182 183
184 + provenance = dict(row.get("provenance") or {})
185 +
183 186 async def base(conn: AsyncConnection) -> dict[str, Any]:
184 187 aliases = await fetch_all(conn, "select alias from entity_aliases where entity_id = :id order by kind, alias limit 200", id=eid)
185 188 idents = await fetch_all(conn, "select scheme, value from entity_identifiers where entity_id = :id order by scheme, value limit 200", id=eid)
189 + await enrich_provenance(conn, provenance)
186 190 return {"aliases": [a["alias"] for a in aliases], "identifiers": [{"scheme": i["scheme"], "value": i["value"]} for i in idents]}
187 191
188 192 tasks: dict[str, Any] = {"base": one(base), "relations": one(relations_grouped, eid), "sources": one(sources_of, eid), "timeline": one(timeline_of, eid, etype)}
@@ -215,7 +219,7 @@ async def entity_detail(row: dict[str, Any]) -> dict[str, Any]:
215 219
216 220 detail = entity_summary(row) or {}
217 221 detail["attributes"] = row.get("attributes") or {}
218 − detail["provenance"] = row.get("provenance") or {}
222 + detail["provenance"] = provenance
219 223 detail.update(blocks.pop("base"))
220 224 detail["relations"] = blocks.pop("relations")
221 225 detail["sources"] = blocks.pop("sources")
modified src/aiatlas/api/routers/changes.py +3 −1
@@ -17,6 +17,7 @@ from aiatlas.api.common import (
17 17 csv,
18 18 day_bounds,
19 19 entity_summary,
20 + event_type_label,
20 21 parse_date,
21 22 parse_ts,
22 23 resolve_id,
@@ -119,7 +120,8 @@ async def changes_categories(request: Request, days: int = Query(7, ge=1, le=365
119 120 async with connection() as conn:
120 121 rows = await fetch_all(conn, """select category, event_type, count(*) as count from change_events where observed_at > now() - make_interval(days => :d)
121 122 and event_type <> 'DOCUMENT_CHANGED' group by 1, 2 order by 3 desc, 1, 2""", d=days)
122 − return {"days": days, "items": [{"category": r["category"], "label": CATEGORY_LABELS.get(r["category"], r["category"].title()), "event_type": r["event_type"], "count": int(r["count"])} for r in rows]}
123 + return {"days": days, "items": [{"category": r["category"], "label": CATEGORY_LABELS.get(r["category"], r["category"].title()), "event_type": r["event_type"],
124 + "event_label": event_type_label(r["event_type"]), "count": int(r["count"])} for r in rows]}
123 125
124 126
125 127 __all__ = ["CATEGORY_LABELS", "ApiError", "router"]
modified src/aiatlas/api/routers/compare.py +2 −0
@@ -14,6 +14,7 @@ from aiatlas.api.common import (
14 14 ApiError,
15 15 cached,
16 16 csv,
17 + enrich_provenance,
17 18 entity_summary,
18 19 price_row,
19 20 resolve_entity,
@@ -104,6 +105,7 @@ async def compare(request: Request, ids: str = Query(..., description="2–6 slu
104 105 agg = await fetch_all(conn, "select benchmark_id, count(*) as n from benchmark_results where benchmark_id = any(cast(:ids as text[])) and valid_to is null group by 1", ids=eids)
105 106 for a in agg:
106 107 extra[a["benchmark_id"]] = {"result_count": int(a["n"])}
108 + await enrich_provenance(conn, *(r.get("provenance") for r in rows))
107 109 for r in rows:
108 110 attrs = r.get("attributes") or {}
109 111 prov = r.get("provenance") or {}
modified src/aiatlas/api/routers/misc.py +16 −2
@@ -8,7 +8,17 @@ from typing import Any
8 8 from fastapi import APIRouter, Depends, Query, Request
9 9 from pydantic import BaseModel, Field
10 10
11 −from aiatlas.api.common import ENTITY_COLS, ApiError, cached, entity_summary, rate_limit
11 +from aiatlas.api.common import (
12 + ENTITY_COLS,
13 + EVENT_TYPE_LABELS,
14 + STATUS_VOCAB,
15 + ApiError,
16 + cached,
17 + entity_summary,
18 + event_type_importance,
19 + event_type_label,
20 + rate_limit,
21 +)
12 22 from aiatlas.db import connection, execute, fetch_all, fetch_one, transaction
13 23 from aiatlas.ids import ENTITY_TYPES
14 24 from aiatlas.services.quality import EXPECTED_FIELDS, QUALITY_VERSION
@@ -41,8 +51,12 @@ async def methodology(request: Request) -> dict[str, Any]:
41 51 async with connection() as conn:
42 52 metrics = await fetch_all(conn, "select key, label, version, description, formula from metric_definitions order by key")
43 53 event_types = await fetch_all(conn, "select event_type, category, count(*) as count, max(observed_at) as last_seen_at from change_events group by 1, 2 order by 3 desc")
54 + seen = {r["event_type"] for r in event_types}
55 + types = [{**r, "count": int(r["count"]), "label": event_type_label(r["event_type"]), "importance": event_type_importance(r["event_type"])} for r in event_types]
56 + types += [{"event_type": t, "category": None, "count": 0, "last_seen_at": None, "label": lbl, "importance": imp}
57 + for t, (lbl, imp) in EVENT_TYPE_LABELS.items() if t not in seen]
44 58 return {"metrics": metrics, "quality_version": QUALITY_VERSION, "expected_fields": EXPECTED_FIELDS, "confidence_levels": CONFIDENCE_LEVELS, "tiers": TIERS,
45 − "event_types": [{**r, "count": int(r["count"])} for r in event_types], "extractors": EXTRACTORS,
59 + "event_types": types, "status_vocabulary": list(STATUS_VOCAB), "extractors": EXTRACTORS,
46 60 "principles": ["Never fabricate: missing data is reported as unavailable.", "Every fact carries provenance (source, snapshot, URL, tier, confidence, extractor).",
47 61 "History is append-only: claims, prices and benchmark results are never overwritten.",
48 62 "Conflicts between sources are stored side by side and flagged for review.", "Live counters and feeds are computed from the database."]}
modified src/aiatlas/api/routers/stats.py +2 −1
@@ -7,7 +7,7 @@ from typing import Any
7 7
8 8 from fastapi import APIRouter, Query, Request
9 9
10 −from aiatlas.api.common import cached
10 +from aiatlas.api.common import MAIN_ENTITY_TYPES, cached
11 11 from aiatlas.db import connection
12 12 from aiatlas.sdk.archive import archive_size
13 13 from aiatlas.services.stats import history, live_counts
@@ -20,6 +20,7 @@ router = APIRouter(prefix="/api/v1/stats", tags=["stats"])
20 20 async def stats(request: Request) -> dict[str, Any]:
21 21 async with connection() as conn:
22 22 counts = await live_counts(conn)
23 + counts["entities"] = {**{t: 0 for t in MAIN_ENTITY_TYPES}, **counts["entities"]} # zero-filled: the homepage never renders a missing key
23 24 counts["archive"] = await asyncio.to_thread(archive_size)
24 25 counts["computed_at"] = datetime.now(UTC)
25 26 return counts
modified tests/test_api.py +11 −0
@@ -45,6 +45,8 @@ async def test_stats_keys_and_live(client: AsyncClient) -> None:
45 45 assert key in s, key
46 46 assert s["entities_total"] == sum(s["entities"].values()) > 0
47 47 assert s["entities"].get("model", 0) > 0
48 + for t in ("model", "company", "paper", "provider", "benchmark", "hardware", "framework", "dataset", "tool", "repository"):
49 + assert t in s["entities"], t # zero-filled for the homepage
48 50 assert {"raw_bytes", "raw_files", "text_bytes", "text_files"} <= set(s["archive"])
49 51 hist = await client.get("/api/v1/stats/history?days=30")
50 52 assert hist.status_code == 200 and "items" in hist.json()
@@ -92,6 +94,8 @@ async def test_entity_detail_blocks(client: AsyncClient) -> None:
92 94 "providers", "quality", "counts"):
93 95 assert block in d, block
94 96 assert d["provenance"] and all({"tier", "confidence", "extractor", "observed_at"} <= set(v) for v in d["provenance"].values())
97 + assert any(v.get("source_name") for v in d["provenance"].values() if v.get("source_id"))
98 + assert d["status"] in ("active", "preview", "deprecated", "retired", "announced", "limited-availability", "unknown")
95 99 assert d["prices"] and d["prices"][0]["provider"]["slug"] and d["prices"][0]["input_per_mtok"] is not None
96 100 assert d["timeline"] and any(e["event_type"] == "NEW_MODEL" for e in d["timeline"])
97 101 assert d["sources"] and {"url", "tier", "doc_type", "snapshots"} <= set(d["sources"][0])
@@ -182,6 +186,13 @@ async def test_listings_and_misc(client: AsyncClient) -> None:
182 186 assert r.status_code == 200, (path, r.text[:200])
183 187 prices = (await client.get("/api/v1/prices", params={"model": MODEL})).json()
184 188 assert prices["items"] and prices["items"][0]["model"]["slug"] == MODEL
189 + meth = (await client.get("/api/v1/methodology")).json()
190 + assert meth["metrics"] and {"key", "label", "version", "description"} <= set(meth["metrics"][0])
191 + assert all({"event_type", "label", "importance", "count"} <= set(t) for t in meth["event_types"])
192 + assert next(t for t in meth["event_types"] if t["event_type"] == "NEW_MODEL")["label"] == "New model"
193 + srcs = (await client.get("/api/v1/sources")).json()["items"]
194 + linked = [c for s in srcs for c in s["connectors"]]
195 + assert linked and {"name", "label", "health", "last_success_at", "interval_seconds"} <= set(linked[0])
185 196 fit = (await client.get("/api/v1/hardware/fit", params={"memory_gb": 24})).json()
186 197 assert fit["estimated"] is True and fit["assumptions"] and "items" in fit
187 198 assert (await client.get("/api/v1/hardware/fit", params={"memory_gb": 24, "quant": "2bit"})).status_code == 400
188 199