HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""/methodology · /trending · POST /views · /sitemap · /api-keys/me · /licenses."""2from __future__ import annotations34import hashlib5from datetime import UTC, datetime6from typing import Any78from fastapi import APIRouter, Depends, Query, Request9from pydantic import BaseModel, Field1011from aiatlas.api.common import (12 ENTITY_COLS,13 ENTITY_FROM,14 EVENT_TYPE_LABELS,15 STATUS_VOCAB,16 ApiError,17 cached,18 entity_summary,19 event_type_importance,20 event_type_label,21 rate_limit,22)23from aiatlas.db import connection, execute, fetch_all, fetch_one, transaction24from aiatlas.ids import ENTITY_TYPES25from aiatlas.ontology import anomalies as anomaly_checks26from aiatlas.ontology.benchmarks import CONDITION_KEYS, IGNORED_KEYS, TASK_KEYS, TRUST_LABELS, TRUST_LEVELS27from aiatlas.ontology.licenses import CATEGORIES, LICENSES, normalize_license28from aiatlas.ontology.openness import OPENNESS_CATEGORIES, OPENNESS_DEFINITIONS, OPENNESS_DIMENSIONS, OPENNESS_LABELS29from aiatlas.services import hardware_fit as hf30from aiatlas.services.finder import RULES as FINDER_RULES31from aiatlas.services.frontier import FRONTIER_METHODOLOGY32from aiatlas.services.quality import EXPECTED_FIELDS, QUALITY_VERSION33from aiatlas.services.stats import DEFINITIONS as COUNTER_DEFINITIONS3435router = APIRouter(prefix="/api/v1", tags=["misc"])3637CONFIDENCE_LEVELS = [38 {"key": "verified", "label": "Verified", "description": "Confirmed by at least two independent sources, one of them tier 1."},39 {"key": "high", "label": "High", "description": "Stated explicitly by an official (tier 1) source and extracted deterministically."},40 {"key": "medium", "label": "Medium", "description": "Quality secondary source, curated registry, or LLM extraction from an official document."},41 {"key": "low", "label": "Low", "description": "Community or unverified source, or LLM extraction from a secondary document."},42 {"key": "conflicted", "label": "Conflicted", "description": "Another source states a different value; both claims are kept and flagged, never averaged."},43]44TIERS = [45 {"tier": 1, "label": "Official / primary", "description": "The organization's own site, docs, pricing pages, model cards, filings."},46 {"tier": 2, "label": "Quality secondary", "description": "Peer-reviewed venues, arXiv, curated registries, major leaderboards."},47 {"tier": 3, "label": "Community", "description": "Community-maintained hubs, forums, wikis."},48 {"tier": 4, "label": "Unverified", "description": "Anything else; never overrides a better tier."},49]50EXTRACTORS = [51 {"key": "deterministic", "description": "Rule-based parsers (tables, JSON-LD, meta tags, embedded JSON, regex) — always runs first."},52 {"key": "curated", "description": "Hand-maintained registries shipped with the code (organizations, providers, benchmarks, hardware)."},53 {"key": "llm", "description": "Local LLM extraction validated against a strict JSON schema; medium/low confidence, never overrides deterministic tier-1 claims."},54]55COMPARABILITY_RULES = {56 "comparable": "Same benchmark variant, same canonical metric, same task-defining configuration (variant, evaluator/harness, shots, pass regime, scaffold…) and same conditions.",57 "partially-comparable": "Same task, but conditions differ (reasoning effort, thinking budget, temperature, judge, tool use, max tokens…).",58 "not-comparable": "Different benchmark variant, different metric, or a task-defining configuration key differs.",59 "task_keys": list(TASK_KEYS), "condition_keys": list(CONDITION_KEYS), "ignored_keys": sorted(IGNORED_KEYS),60 "group": "A comparability group is (benchmark, canonical metric, config_key) where config_key = sha1 of the task keys + metric (12 hex chars).",61 "leaderboard": "One row per canonical model: its best current row inside the chosen group; effort variants folded into a model share its rows.",62}63ANOMALY_CHECKS = [64 {"check": "params_too_large", "severity": "critical", "description": f"parameter_count above {anomaly_checks.MAX_PARAMS:.0e}"},65 {"check": "params_too_small", "severity": "warning", "description": "parameter_count below 100K"},66 {"check": "active_gt_total", "severity": "critical", "description": "active parameters exceed total parameters"},67 {"check": "context_too_large", "severity": "critical", "description": "context_length above 100M tokens"},68 {"check": "context_too_small", "severity": "warning", "description": "context_length below 256 tokens"},69 {"check": "max_output_gt_context", "severity": "warning", "description": "max_output_tokens exceeds context_length"},70 {"check": "release_in_future", "severity": "critical", "description": "release_date after today"},71 {"check": "release_too_old", "severity": "warning", "description": "release_date before 2010"},72 {"check": "deprecated_before_release", "severity": "critical", "description": "deprecation_date before release_date"},73 {"check": "retired_before_release", "severity": "critical", "description": "retirement_date before release_date"},74 {"check": "retired_before_deprecated", "severity": "warning", "description": "retirement_date before deprecation_date"},75 {"check": "cutoff_after_release", "severity": "warning", "description": "knowledge_cutoff after release_date"},76 {"check": "deprecated_future_release", "severity": "critical", "description": "deprecated/retired status with a future release date"},77 {"check": "open_without_weights_url", "severity": "info", "description": "labelled open but no weights location recorded"},78 {"check": "negative_price", "severity": "critical", "description": "a price field is negative"},79 {"check": "price_too_high", "severity": "warning", "description": f"a price exceeds ${anomaly_checks.MAX_PRICE_PER_MTOK:g} per 1M tokens"},80 {"check": "zero_output_price", "severity": "warning", "description": "output price 0 while input is positive and the offer is not free"},81 {"check": "input_gt_output_price", "severity": "info", "description": "input price more than 4× the output price"},82 {"check": "cached_gt_input_price", "severity": "warning", "description": "cached input price above the input price"},83 {"check": "price_jump_100x", "severity": "critical", "description": "a price moved by more than 100× between two observations"},84 {"check": "score_above_max", "severity": "critical", "description": "benchmark score above the metric maximum"},85 {"check": "score_below_min", "severity": "critical", "description": "benchmark score below the metric minimum"},86 {"check": "evaluated_before_release", "severity": "warning", "description": "result evaluated more than 45 days before the model's release"},87 {"check": "memory_implausible", "severity": "critical", "description": "hardware memory_gb ≤ 0 or above 100 000"},88 {"check": "bandwidth_implausible", "severity": "warning", "description": "memory bandwidth ≤ 0 or above 100 000 GB/s"},89 {"check": "tdp_implausible", "severity": "warning", "description": "TDP ≤ 0 or above 200 000 W"},90]919293@router.get("/methodology")94@cached(600)95async def methodology(request: Request) -> dict[str, Any]:96 async with connection() as conn:97 metrics = await fetch_all(conn, "select key, label, version, description, formula from metric_definitions order by key")98 event_types = await fetch_all(conn, "select event_type, category, count(*) as count, max(observed_at) as last_seen_at from change_events group by 1, 2 order by 3 desc")99 seen = {r["event_type"] for r in event_types}100 types = [{**r, "count": int(r["count"]), "label": event_type_label(r["event_type"]), "importance": event_type_importance(r["event_type"])} for r in event_types]101 types += [{"event_type": t, "category": None, "count": 0, "last_seen_at": None, "label": lbl, "importance": imp}102 for t, (lbl, imp) in EVENT_TYPE_LABELS.items() if t not in seen]103 return {"metrics": metrics, "quality_version": QUALITY_VERSION, "expected_fields": EXPECTED_FIELDS, "confidence_levels": CONFIDENCE_LEVELS, "tiers": TIERS,104 "event_types": types, "status_vocabulary": list(STATUS_VOCAB), "extractors": EXTRACTORS,105 "openness": {"categories": list(OPENNESS_CATEGORIES), "labels": OPENNESS_LABELS, "definitions": OPENNESS_DEFINITIONS, "dimensions": list(OPENNESS_DIMENSIONS),106 "note": "Categories are derived from measurable dimensions and the licence ontology; a custom community licence is never 'open-source'."},107 "trust_levels": [{"key": k, "label": TRUST_LABELS[k]} for k in TRUST_LEVELS],108 "comparability": COMPARABILITY_RULES,109 "counters": COUNTER_DEFINITIONS,110 "anomaly_checks": ANOMALY_CHECKS,111 "event_semantics": {"occurred_at": "coalesce(effective_at, observed_at) — when the change happened (effective date when a source states it)",112 "observed_at": "when AI Atlas first saw the change", "recorded_at": "when the row was written",113 "is_backfill": "true for history imported when a source is first crawled (never shown as 'today' in feeds)",114 "group_key": "one release / announcement seen through several documents shares a group_key"},115 "hardware_fit": {"assumptions": hf.ASSUMPTIONS, "bytes_per_param": hf.BYTES_PER_PARAM, "reserved_gb": hf.RESERVED_GB},116 "frontier": FRONTIER_METHODOLOGY,117 "find_a_model": FINDER_RULES,118 "licence_categories": list(CATEGORIES),119 "principles": ["Never fabricate: missing data is reported as unavailable.", "Every fact carries provenance (source, snapshot, URL, tier, confidence, extractor).",120 "History is append-only: claims, prices and benchmark results are never overwritten.",121 "Conflicts between sources are stored side by side and flagged for review.", "Live counters and feeds are computed from the database.",122 "No composite 'best model' score: rankings are per benchmark comparability group; finders return the observed dimensions."]}123124125@router.get("/trending")126@cached(300)127async def trending(request: Request, days: int = Query(7, ge=1, le=90), limit: int = Query(12, ge=1, le=60), type: str | None = Query(None, alias="type"),128 kind: str = Query("views", pattern="^(views|most_changed|new_listings|new_results)$")) -> dict[str, Any]:129 """`kind=views` (page views, v1) · `most_changed` (events per entity) · `new_listings` (PROVIDER_LISTED) · `new_results` (benchmark result events) — separate lists, never merged."""130 async with connection() as conn:131 if kind == "views":132 rows = await fetch_all(conn, f"""133 with v as (select split_part(regexp_replace(path, '[?#].*$', ''), '/', 3) as slug, sum(views) as views from page_views134 where day >= ((now() at time zone 'UTC') - make_interval(days => :d))::date and path ~ '^/[a-z-]+/[^/?#]+' group by 1)135 select v.views as n, {ENTITY_COLS} from v join entities e on e.slug = v.slug left join entities eo on eo.id = e.organization_id136 where e.merged_into is null {"and e.entity_type = :t" if type else ""} order by v.views desc, e.updated_at desc limit :lim""", d=days, lim=limit, t=type)137 items = [{**(entity_summary(r) or {}), "views": int(r["n"] or 0)} for r in rows]138 definition = "Page views recorded by the site beacon in the window."139 else:140 cond = {"most_changed": "ev.event_type <> 'DOCUMENT_CHANGED'", "new_listings": "ev.event_type in ('PROVIDER_LISTED','PROVIDER_DELISTED')",141 "new_results": "ev.event_type in ('BENCHMARK_RESULT','BENCHMARK_UPDATED')"}[kind]142 rows = await fetch_all(conn, f"""143 with v as (select ev.entity_id, count(*) as n, max(ev.occurred_at) as last_at from change_events ev144 where ev.is_backfill = false and ev.occurred_at > now() - make_interval(days => :d) and {cond} group by 1)145 select v.n, v.last_at, {ENTITY_COLS} from v join entities e on e.id = v.entity_id left join entities eo on eo.id = e.organization_id146 where e.merged_into is null {"and e.entity_type = :t" if type else ""} order by v.n desc, v.last_at desc limit :lim""", d=days, lim=limit, t=type)147 items = [{**(entity_summary(r) or {}), "events": int(r["n"] or 0), "last_event_at": r["last_at"]} for r in rows]148 definition = {"most_changed": "Entities with the most non-backfill events (any type except source-document changes) that occurred in the window.",149 "new_listings": "Entities with the most provider listing / delisting events in the window.",150 "new_results": "Entities with the most new or updated benchmark result events in the window."}[kind]151 return {"days": days, "kind": kind, "items": items, "definition": definition}152153154@router.get("/licenses")155@cached(600)156async def licenses(request: Request) -> dict[str, Any]:157 """Licence ontology + how many canonical models use each key (canonical `license_key` or a raw label the ontology maps to it)."""158 async with connection() as conn:159 rows = await fetch_all(conn, """select coalesce(e.attributes->>'license_key', e.attributes->>'license') as raw, count(*) as n from entities e160 where e.entity_type = 'model' and e.merged_into is null and (e.attributes ? 'license' or e.attributes ? 'license_key') group by 1""")161 counts: dict[str, int] = {}162 unclassified: dict[str, int] = {}163 for r in rows:164 key = r["raw"] if r["raw"] in LICENSES else normalize_license(r["raw"])165 if key:166 counts[key] = counts.get(key, 0) + int(r["n"])167 elif r["raw"]:168 unclassified[r["raw"]] = unclassified.get(r["raw"], 0) + int(r["n"])169 items = [{**info.as_dict(), "aliases": list(info.aliases), "models": counts.get(key, 0)} for key, info in LICENSES.items()]170 items.sort(key=lambda x: (-x["models"], x["label"]))171 return {"items": items, "total": len(items), "categories": list(CATEGORIES), "unclassified": [{"raw": k, "models": v} for k, v in sorted(unclassified.items(), key=lambda kv: -kv[1])],172 "note": "Permissions are read from the licence text (null = the text is ambiguous). Counts cover canonical models only."}173174175@router.get("/licenses/{key}")176@cached(600)177async def license_detail(request: Request, key: str, limit: int = Query(50, ge=1, le=200)) -> dict[str, Any]:178 canon = key if key in LICENSES else normalize_license(key)179 info = LICENSES.get(canon) if canon else None180 if not info:181 raise ApiError(404, f"unknown licence {key!r}")182 raw = sorted({info.key.lower(), *(a.lower() for a in info.aliases), *([info.spdx.lower()] if info.spdx else [])})183 async with connection() as conn:184 rows = await fetch_all(conn, f"""select {ENTITY_COLS} from {ENTITY_FROM} where e.entity_type = 'model' and e.merged_into is null185 and (e.attributes->>'license_key' = :k or lower(e.attributes->>'license') = any(cast(:raw as text[])))186 order by e.attributes->>'release_date' desc nulls last, e.canonical_name limit :lim""", k=info.key, raw=raw, lim=limit)187 total = await fetch_one(conn, """select count(*) as n from entities e where e.entity_type = 'model' and e.merged_into is null188 and (e.attributes->>'license_key' = :k or lower(e.attributes->>'license') = any(cast(:raw as text[])))""", k=info.key, raw=raw)189 return {**info.as_dict(), "aliases": list(info.aliases), "models": {"items": [entity_summary(r) for r in rows], "total": int((total or {}).get("n") or 0), "limit": limit, "offset": 0}}190191192class ViewBeacon(BaseModel):193 path: str = Field(..., min_length=1, max_length=300)194195196@router.post("/views", dependencies=[Depends(rate_limit("views"))])197async def record_view(body: ViewBeacon) -> dict[str, Any]:198 path = body.path.strip()199 if not path.startswith("/") or "\n" in path or "//" in path:200 raise ApiError(400, "path must be a site-relative path")201 path = path.split("?", 1)[0].split("#", 1)[0][:300]202 async with transaction() as conn:203 await execute(conn, "insert into page_views (path, day, views) values (:p, :d, 1) on conflict (path, day) do update set views = page_views.views + 1",204 p=path, d=datetime.now(UTC).date())205 return {"ok": True}206207208@router.get("/sitemap")209@cached(600)210async def sitemap(request: Request, type: str | None = Query(None, alias="type"), limit: int = Query(5000, ge=1, le=5000), offset: int = Query(0, ge=0)) -> dict[str, Any]:211 if type and type not in ENTITY_TYPES:212 raise ApiError(400, f"unknown entity type {type!r}")213 where = "e.merged_into is null" + (" and e.entity_type = :t" if type else "")214 async with connection() as conn:215 rows = await fetch_all(conn, f"select e.slug, e.entity_type, e.updated_at from entities e where {where} order by e.updated_at desc, e.id limit :lim offset :off", t=type, lim=limit, off=offset)216 total = await fetch_one(conn, f"select count(*) as n from entities e where {where}", t=type)217 return {"items": rows, "total": int(total["n"]) if total else 0, "limit": limit, "offset": offset}218219220@router.get("/api-keys/me")221async def api_key_me(request: Request) -> dict[str, Any]:222 key = request.headers.get("x-api-key") or ""223 if not key:224 raise ApiError(401, "x-api-key header required")225 digest = hashlib.sha256(key.encode()).hexdigest()226 async with transaction() as conn:227 row = await fetch_one(conn, "update api_keys set last_used_at = now(), usage_count = usage_count + 1 where key_hash = :h and enabled returning label, plan, rate_per_min, usage_count, created_at, last_used_at", h=digest)228 if not row:229 raise ApiError(401, "unknown or disabled API key")230 return row231232233__all__ = ["ANOMALY_CHECKS", "COMPARABILITY_RULES", "router"]234