HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""/benchmarks · /benchmarks/{slug} · /benchmarks/{slug}/results · /benchmarks/{slug}/leaderboard · /benchmarks/{slug}/history ·2/benchmarks/{slug}/frontier · /benchmarks/matrix.34API 1.1: results are organised in comparability GROUPS (canonical metric × config_key); leaderboards are ONE row per canonical model (best5current row inside the chosen group); benchmarks resolve by slug, alias or id; every benchmark exposes `family`, `variant`, `metric`,6`direction`, `groups`, `trust_mix`."""7from __future__ import annotations89from collections import defaultdict10from datetime import UTC, datetime11from typing import Any1213from fastapi import APIRouter, Query, Request1415from aiatlas.api.common import (16 PAGINATION,17 RESULT_COLS,18 RESULT_FROM,19 ApiError,20 Pagination,21 cached,22 csv,23 entity_summary,24 page,25 parse_ts,26 resolve_entity,27 resolve_id,28 result_row,29)30from aiatlas.api.detail import leaderboard as legacy_leaderboard31from aiatlas.api.routers.entities import detail_for_type32from aiatlas.db import connection, fetch_all, fetch_val33from aiatlas.ontology.benchmarks import TRUST_LABELS, comparability34from aiatlas.services.frontier import (35 all_primary_groups,36 benchmark_meta,37 frontier_series,38 group_rows,39 group_summary,40 leaderboard_rows,41 load_results,42 primary_group,43 rank_rows,44)4546router = APIRouter(prefix="/api/v1/benchmarks", tags=["benchmarks"])47GROUPING_NOTE = ("Rows are grouped by comparability group = canonical metric × config_key (hash of the task-defining configuration keys: variant, "48 "evaluator, harness, shots, pass regime…). A leaderboard shows one row per canonical model: its best current row inside the group. "49 "Reasoning effort, temperature or judge differences keep rows in the same group but mark them partially comparable.")505152async def _resolve_benchmark(conn: Any, slug: str) -> dict[str, Any]:53 return await resolve_entity(conn, slug, ("benchmark",), aliases=True)545556def _pick_group(groups: list[dict[str, Any]], attrs: dict[str, Any] | None, *, metric: str | None, config_key: str | None) -> dict[str, Any] | None:57 cands = groups58 if metric:59 m = metric.strip().lower()60 cands = [g for g in cands if g["metric"] == m or (g["metric"] or "").lower() == m]61 if config_key:62 cands = [g for g in cands if g["config_key"] == config_key]63 if metric or config_key:64 return max(cands, key=lambda g: (g["model_count"], g["n"])) if cands else None65 return primary_group(attrs, groups)666768# ------------------------------------------------------------------------------------------------------------------ listing697071@router.get("")72@cached(300)73async def list_benchmarks(request: Request, category: str | None = None) -> dict[str, Any]:74 async with connection() as conn:75 meta = await benchmark_meta(conn)76 rows = await load_results(conn)77 by_bench: dict[str, list[dict[str, Any]]] = defaultdict(list)78 for r in rows:79 by_bench[r["benchmark_id"]].append(r)80 items = []81 for bid, m in meta.items():82 if category and (m.get("category") or "").lower() != category.lower():83 continue84 brows = by_bench.get(bid, [])85 groups = list(group_rows(brows).values())86 pg = primary_group(m["attributes"], groups)87 leader = None88 second = None89 if pg:90 ranked = rank_rows(pg["rows"], pg["higher_is_better"])91 if ranked:92 lb = leaderboard_rows(pg)93 leader = lb[0] if lb else None94 second = lb[1] if len(lb) > 1 else None95 trust_mix: dict[str, int] = defaultdict(int)96 for r in brows:97 trust_mix[r["trust_level"]] += 198 items.append({99 "id": bid, "entity_type": "benchmark", "slug": m["slug"], "name": m["name"], "category": m.get("category"), "family": m.get("family"), "variant": m.get("variant"),100 "metric": m.get("metric"), "unit": m.get("unit"), "direction": m.get("direction"), "attributes": m["attributes"],101 "result_count": len(brows), "model_count": len({r["model_id"] for r in brows}),102 "leader": leader, "second": second, "top": {"model": leader["model"], "score": leader["score"]} if leader else None,103 "primary_group": group_summary(pg) if pg else None, "groups": sorted((group_summary(g) for g in groups), key=lambda g: (-g["model_count"], g["label"])),104 "trust_mix": dict(trust_mix), "trust_labels": {k: TRUST_LABELS.get(k, k) for k in trust_mix},105 })106 items.sort(key=lambda x: (-x["result_count"], x["name"]))107 return {"items": items, "total": len(items), "note": GROUPING_NOTE}108109110@router.get("/matrix")111@cached(300)112async def matrix(request: Request, benchmarks: str | None = None, models: str | None = None, org: str | None = None, family: str | None = None,113 limit: int = Query(60, ge=1, le=300), comparable_only: int = Query(0, ge=0, le=1), min_cells: int = Query(3, ge=1, le=30),114 since: str | None = Query(None, description="model release_date ≥ YYYY-MM-DD"), until: str | None = Query(None, description="model release_date ≤ YYYY-MM-DD")) -> dict[str, Any]:115 """Rows = canonical models, columns = benchmarks, cell = best current score in the primary comparability group (+ trust, config_key, observed_at)."""116 from aiatlas.api.common import parse_date117118 since_d, until_d = parse_date(since, "since"), parse_date(until, "until")119 async with connection() as conn:120 groups = await all_primary_groups(conn)121 meta = await benchmark_meta(conn)122 wanted_b: list[str] | None = None123 if benchmarks:124 wanted_b = []125 for key in csv(benchmarks):126 b = await _resolve_benchmark(conn, key)127 wanted_b.append(b["id"])128 wanted_m: set[str] | None = None129 if models:130 wanted_m = {await resolve_id(conn, k, ("model",)) or "" for k in csv(models)}131 if org:132 rows = await fetch_all(conn, "select e.id from entities e join entities o on o.id = e.organization_id where e.entity_type = 'model' and e.merged_into is null and (o.slug = :o or o.id = :o or o.canonical_name ilike :o)", o=org)133 wanted_m = (wanted_m or set()) | {r["id"] for r in rows}134 if family:135 rows = await fetch_all(conn, "select e.id from entities e left join entities f on f.id = e.family_id where e.entity_type = 'model' and e.merged_into is null and (f.slug = :f or f.id = :f or e.attributes->>'family' ilike :f)", f=family)136 wanted_m = (wanted_m or set()) | {r["id"] for r in rows}137 if wanted_b is None:138 wanted_b = [bid for bid, _ in sorted(groups.items(), key=lambda kv: -kv[1]["n"])[:12]]139 columns = []140 cells: dict[str, dict[str, dict[str, Any]]] = defaultdict(dict)141 model_ref: dict[str, dict[str, Any]] = {}142 for bid in wanted_b:143 g = groups.get(bid)144 m = meta.get(bid) or {"id": bid, "slug": None, "name": None}145 columns.append({"id": bid, "slug": m.get("slug"), "name": m.get("name"), "category": m.get("category"), "metric": g["metric"] if g else m.get("metric"),146 "config_key": g["config_key"] if g else None, "group_label": g["label"] if g else None, "higher_is_better": g["higher_is_better"] if g else None,147 "n_models": g["model_count"] if g else 0})148 if not g:149 continue150 leader_cfg = None151 ranked = rank_rows(g["rows"], g["higher_is_better"])152 if ranked:153 leader_cfg = ranked[0]154 for r in ranked:155 if wanted_m is not None and r["model_id"] not in wanted_m:156 continue157 rel = str((r.get("model_attrs") or {}).get("release_date") or "")[:10]158 if (since_d or until_d) and (not rel or (since_d and rel < since_d.isoformat()) or (until_d and rel > until_d.isoformat())):159 continue160 level, _ = comparability(leader_cfg["config"] if leader_cfg else None, r.get("config"), leader_cfg["metric"] if leader_cfg else None, r.get("metric")) if leader_cfg else ("comparable", [])161 if comparable_only and level != "comparable":162 continue163 cells[r["model_id"]][bid] = {"score": r["score"], "rank": r["rank"], "trust_level": r["trust_level"], "config_key": r["config_key"], "comparability": level,164 "result_id": r["id"], "observed_at": r["observed_at"], "evaluated_at": r.get("evaluated_at")}165 model_ref.setdefault(r["model_id"], {"id": r["model_id"], "slug": r["model_slug"], "name": r["model_name"], "organization": r.get("org_name"), "organization_slug": r.get("org_slug"),166 "openness": (r.get("model_attrs") or {}).get("openness"), "release_date": (r.get("model_attrs") or {}).get("release_date")})167 rows_out = []168 for mid, c in cells.items():169 if wanted_m is None and len(c) < min_cells:170 continue171 rows_out.append({"model": model_ref[mid], "cells": {bid: c.get(bid) for bid in wanted_b}, "n_cells": len(c),172 "mean_rank": round(sum(x["rank"] for x in c.values()) / len(c), 2)})173 rows_out.sort(key=lambda x: (-x["n_cells"], x["mean_rank"], x["model"]["name"] or ""))174 return {"columns": columns, "rows": rows_out[:limit], "total_rows": len(rows_out), "comparable_only": bool(comparable_only), "min_cells": min_cells if wanted_m is None else None,175 "filters": {k: v for k, v in {"since": since, "until": until, "org": org, "family": family, "models": models}.items() if v},176 "methodology": GROUPING_NOTE + " Each cell is the model's best current row in the benchmark's primary group; `mean_rank` is only a sort key, not a composite score."}177178179# ------------------------------------------------------------------------------------------------------------------ one benchmark180181182@router.get("/{slug}")183@cached(300)184async def get_benchmark(request: Request, slug: str) -> dict[str, Any]:185 async with connection() as conn:186 bench = await _resolve_benchmark(conn, slug)187 rows = await load_results(conn, benchmark_ids=[bench["id"]])188 meta = (await benchmark_meta(conn, [bench["id"]])).get(bench["id"], {})189 detail = await detail_for_type(bench["slug"], ("benchmark",))190 groups = list(group_rows(rows).values())191 pg = primary_group(bench.get("attributes"), groups)192 detail.update({"family": meta.get("family"), "variant": meta.get("variant"), "metric": meta.get("metric"), "direction": meta.get("direction"), "category": meta.get("category"),193 "groups": sorted((group_summary(g) for g in groups), key=lambda g: (-g["model_count"], g["label"])), "primary_group": group_summary(pg) if pg else None,194 "result_count": len(rows), "model_count": len({r["model_id"] for r in rows}), "leaderboard": leaderboard_rows(pg)[:25] if pg else [],195 "trust_mix": dict(defaultdict(int, {k: sum(1 for r in rows if r["trust_level"] == k) for k in {r["trust_level"] for r in rows}}))})196 return detail197198199@router.get("/{slug}/results")200@cached(300)201async def benchmark_results(request: Request, slug: str, p: Pagination = PAGINATION, config: str | None = Query(None, max_length=200), history: int = Query(0, ge=0, le=1),202 metric: str | None = Query(None, max_length=80), config_key: str | None = Query(None, max_length=40)) -> dict[str, Any]:203 """v1 behaviour (one row per result, sorted by score) + `metric=` and `config_key=` filters."""204 async with connection() as conn:205 bench = await _resolve_benchmark(conn, slug)206 items = await legacy_leaderboard(conn, bench["id"], limit=p.limit, offset=p.offset, config=config, history=bool(history), metric=metric, config_key=config_key)207 where = "r.benchmark_id = :id" + ("" if history else " and r.valid_to is null") + (" and r.config::text ilike :cfg" if config else "") \208 + (" and lower(r.metric) = lower(:metric)" if metric else "") + (" and r.config_key = :ck" if config_key else "")209 total = await fetch_val(conn, f"select count(*) from benchmark_results r where {where}", id=bench["id"], cfg=f"%{config}%" if config else None, metric=metric, ck=config_key)210 out = page(items, int(total or 0), p)211 out["benchmark"] = entity_summary(bench)212 return out213214215@router.get("/{slug}/leaderboard")216@cached(300)217async def benchmark_leaderboard(request: Request, slug: str, metric: str | None = None, config_key: str | None = None, trust: str | None = None, org: str | None = None,218 since: str | None = None, until: str | None = None, comparable_only: int = Query(0, ge=0, le=1), limit: int = Query(100, ge=1, le=1000),219 offset: int = Query(0, ge=0)) -> dict[str, Any]:220 """ONE row per canonical model — best current row inside the chosen comparability group (default: primary group)."""221 since_ts, until_ts = parse_ts(since, "since"), parse_ts(until, "until")222 async with connection() as conn:223 bench = await _resolve_benchmark(conn, slug)224 rows = await load_results(conn, benchmark_ids=[bench["id"]])225 closed = await load_results(conn, benchmark_ids=[bench["id"]], current_only=False)226 org_id = await resolve_id(conn, org) if org else None227 groups = list(group_rows(rows).values())228 g = _pick_group(groups, bench.get("attributes"), metric=metric, config_key=config_key)229 if not g:230 return {"benchmark": entity_summary(bench), "group": None, "groups": [group_summary(x) for x in groups], "items": [], "total": 0,231 "note": "no current results in the requested group" if (metric or config_key) else "no current results for this benchmark"}232 trust_set = set(csv(trust)) if trust else None233 sel = [r for r in g["rows"] if (trust_set is None or r["trust_level"] in trust_set) and (org_id is None or r.get("organization_id") == org_id)234 and (since_ts is None or (r.get("evaluated_at") or r["observed_at"]) >= since_ts) and (until_ts is None or (r.get("evaluated_at") or r["observed_at"]) <= until_ts)]235 history_rows = [r for r in closed if r.get("valid_to") is not None and r["metric_canonical"] == g["metric"] and r["config_key"] == g["config_key"]]236 sub = {**g, "rows": sel}237 items = leaderboard_rows(sub, history_rows=history_rows or None, comparable_only=bool(comparable_only))238 return {"benchmark": entity_summary(bench), "group": group_summary(g), "groups": sorted((group_summary(x) for x in groups), key=lambda x: (-x["model_count"], x["label"])),239 "items": items[offset:offset + limit], "total": len(items), "limit": limit, "offset": offset, "comparable_only": bool(comparable_only),240 "filters": {k: v for k, v in {"metric": metric, "config_key": config_key, "trust": trust, "org": org, "since": since, "until": until}.items() if v},241 "history_available": bool(history_rows), "methodology": GROUPING_NOTE + " delta_rank compares with the ranking built from the closed (previous) rows of the same group."}242243244@router.get("/{slug}/frontier")245@cached(300)246async def benchmark_frontier(request: Request, slug: str, metric: str | None = None, config_key: str | None = None) -> dict[str, Any]:247 """History of the leader per comparability group: a point each time a new best score appears (ordered by coalesce(evaluated_at, observed_at))."""248 async with connection() as conn:249 bench = await _resolve_benchmark(conn, slug)250 rows = await load_results(conn, benchmark_ids=[bench["id"]], current_only=False)251 groups = list(group_rows(rows).values())252 if metric or config_key:253 groups = [g for g in groups if (not metric or g["metric"] == metric.strip().lower()) and (not config_key or g["config_key"] == config_key)]254 pg = primary_group(bench.get("attributes"), groups)255 series = []256 for g in sorted(groups, key=lambda x: (-x["model_count"], x["label"])):257 pts = frontier_series(g["rows"], g["higher_is_better"])258 series.append({"group": group_summary(g), "primary": pg is not None and g["config_key"] == pg["config_key"] and g["metric"] == pg["metric"], "points": pts,259 "current_leader": pts[-1] if pts else None})260 return {"benchmark": entity_summary(bench), "series": series, "generated_at": datetime.now(UTC),261 "methodology": "Includes closed rows (history). A point is emitted whenever a result beats every earlier result of the same group, "262 "ordered by evaluated_at when the source publishes it, else observed_at."}263264265@router.get("/{slug}/history")266@cached(300)267async def benchmark_history(request: Request, slug: str, model: str | None = None, limit: int = Query(1000, ge=1, le=5000)) -> dict[str, Any]:268 async with connection() as conn:269 bench = await _resolve_benchmark(conn, slug)270 where = ["r.benchmark_id = :id"]271 params: dict[str, Any] = {"id": bench["id"], "lim": limit}272 if model:273 where.append("r.model_id = :model")274 params["model"] = await resolve_id(conn, model)275 rows = await fetch_all(conn, f"select {RESULT_COLS}, r.config_key, r.trust_level from {RESULT_FROM} where {' and '.join(where)} order by r.observed_at asc, r.id limit :lim", **params)276 items = []277 for r in rows:278 it = result_row(r)279 it["config_key"], it["trust_level"] = r.get("config_key"), r.get("trust_level")280 items.append(it)281 return {"benchmark": entity_summary(bench), "items": items}282283284__all__ = ["ApiError", "router"]285