HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""Benchmark comparability groups, per-model leaderboards and the "frontier" model set (API 1.1). Deterministic, no LLM.23A *group* is (benchmark, canonical metric, config_key): the comparability-relevant part of the result configuration hashed by4`ontology.benchmarks.config_key` (task keys + metric). Rows written before migration 0003 carry a NULL `config_key` / `trust_level`;5both are recomputed here from `config` and the source key so the API behaves the same before and after canonicalisation.67Leaderboards are ONE row per canonical model: the best current row of that model inside the chosen group. Effort variants folded into8a model land in the same group (reasoning effort is a *condition*, not a task key) and are summarised in `config`."""9from __future__ import annotations1011from collections import Counter, defaultdict12from datetime import UTC, datetime, timedelta13from typing import Any1415from sqlalchemy.ext.asyncio import AsyncConnection1617from aiatlas.db import fetch_all, fetch_val18from aiatlas.ontology.benchmarks import (19 CONDITION_KEYS,20 METRICS,21 TASK_KEYS,22 TRUST_LABELS,23 comparability,24 config_key,25 family_of,26 normalize_metric,27 trust_level,28)29from aiatlas.services import cache3031RESULT_SELECT = """32 select r.id, r.model_id, r.benchmark_id, r.score, r.metric, r.unit, r.higher_is_better, r.config, r.evaluated_at, r.observed_at, r.source_url, r.tier,33 r.confidence, r.valid_to, r.config_key, r.trust_level, r.variant, r.run_group, r.is_current, r.extractor, s.key as source_key,34 m.slug as model_slug, m.canonical_name as model_name, m.entity_type as model_type, m.merged_into as model_merged_into, m.organization_id,35 m.attributes as model_attrs, m.family_id, mo.slug as org_slug, mo.canonical_name as org_name,36 b.slug as benchmark_slug, b.canonical_name as benchmark_name, b.attributes as benchmark_attrs37 from benchmark_results r38 join entities m on m.id = r.model_id left join entities mo on mo.id = m.organization_id39 join entities b on b.id = r.benchmark_id40 left join sources s on s.id = r.source_id41"""42CURRENT = "r.valid_to is null and r.is_current"43CANONICAL_MODEL = "m.entity_type = 'model' and m.merged_into is null"44FRONTIER_CACHE_KEY = "frontier:model-ids:v1"45FRONTIER_TTL_S = 60046FRONTIER_METHODOLOGY = ("Frontier models = canonical models released in the last 12 months by organizations with at least 3 canonical models, "47 "OR holding a top-10 rank in the primary comparability group of at least one benchmark. Artifacts and folded variants are excluded. "48 "No composite score is used to pick them.")495051# ------------------------------------------------------------------------------------------------------------------ row enrichment525354def enrich(row: dict[str, Any]) -> dict[str, Any]:55 """Fill `config_key` / `trust_level` / `metric_canonical` when the writer has not (pre-0003 rows)."""56 cfg = row.get("config") or {}57 metric = normalize_metric(row.get("metric")) or (row.get("metric") or "").strip().lower() or "score"58 row["metric_canonical"] = metric59 if not row.get("config_key"):60 row["config_key"] = config_key(cfg, row.get("metric"))61 if not row.get("trust_level"):62 row["trust_level"] = trust_level(row.get("source_key"), cfg, extractor=row.get("extractor") or "deterministic")63 return row646566def group_label(metric: str, cfg: dict[str, Any] | None) -> str:67 parts = [f"{k}={cfg[k]}" for k in TASK_KEYS if cfg and cfg.get(k) not in (None, "", [], {})]68 return metric + (" · " + " · ".join(str(p) for p in parts) if parts else "")697071def config_summary(cfg: dict[str, Any] | None) -> dict[str, Any]:72 """Task + condition keys only (bookkeeping keys such as aa_slug are dropped)."""73 return {k: cfg[k] for k in (*TASK_KEYS, *CONDITION_KEYS) if cfg and cfg.get(k) not in (None, "", [], {})}747576def direction_of(rows: list[dict[str, Any]], metric: str) -> bool:77 votes = Counter(bool(r.get("higher_is_better")) for r in rows if r.get("higher_is_better") is not None)78 if votes:79 return votes.most_common(1)[0][0]80 spec = METRICS.get(metric)81 return bool(spec["higher_is_better"]) if spec else True828384# ------------------------------------------------------------------------------------------------------------------ loading858687async def load_results(conn: AsyncConnection, *, benchmark_ids: list[str] | None = None, model_ids: list[str] | None = None, current_only: bool = True,88 canonical_models_only: bool = True, limit: int = 50_000) -> list[dict[str, Any]]:89 where = []90 params: dict[str, Any] = {"lim": limit}91 if benchmark_ids is not None:92 where.append("r.benchmark_id = any(cast(:bids as text[]))")93 params["bids"] = benchmark_ids94 if model_ids is not None:95 where.append("r.model_id = any(cast(:mids as text[]))")96 params["mids"] = model_ids97 if current_only:98 where.append(CURRENT)99 if canonical_models_only:100 where.append(CANONICAL_MODEL)101 sql = RESULT_SELECT + " where " + (" and ".join(where) or "true") + " order by r.benchmark_id, r.observed_at desc limit :lim"102 return [enrich(r) for r in await fetch_all(conn, sql, **params)]103104105# ------------------------------------------------------------------------------------------------------------------ grouping106107108def group_rows(rows: list[dict[str, Any]]) -> dict[tuple[str, str, str], dict[str, Any]]:109 """{(benchmark_id, metric, config_key): {rows, metric, config_key, label, n, models, higher_is_better, representative config}}."""110 groups: dict[tuple[str, str, str], dict[str, Any]] = {}111 for r in rows:112 k = (r["benchmark_id"], r["metric_canonical"], r["config_key"])113 g = groups.get(k)114 if g is None:115 g = groups[k] = {"benchmark_id": r["benchmark_id"], "metric": r["metric_canonical"], "config_key": r["config_key"], "rows": [], "models": set(),116 "config": {k2: r["config"][k2] for k2 in TASK_KEYS if (r.get("config") or {}).get(k2) not in (None, "", [], {})}}117 g["rows"].append(r)118 g["models"].add(r["model_id"])119 for g in groups.values():120 g["n"] = len(g["rows"])121 g["model_count"] = len(g["models"])122 g["label"] = group_label(g["metric"], g["config"])123 g["higher_is_better"] = direction_of(g["rows"], g["metric"])124 g["trust_mix"] = dict(Counter(r["trust_level"] for r in g["rows"]))125 return groups126127128def primary_metric(benchmark_attrs: dict[str, Any] | None, groups: list[dict[str, Any]]) -> str | None:129 """Registry metric when it has current rows, else the most populated metric that is not a LiveBench per-category average."""130 declared = normalize_metric((benchmark_attrs or {}).get("metric"))131 present = Counter()132 for g in groups:133 present[g["metric"]] += g["n"]134 if declared and present.get(declared):135 return declared136 ranked = [m for m, _ in present.most_common() if not m.startswith("category:")] or [m for m, _ in present.most_common()]137 return ranked[0] if ranked else declared138139140def primary_group(benchmark_attrs: dict[str, Any] | None, groups: list[dict[str, Any]]) -> dict[str, Any] | None:141 metric = primary_metric(benchmark_attrs, groups)142 cands = [g for g in groups if g["metric"] == metric]143 if not cands:144 return None145 return max(cands, key=lambda g: (g["model_count"], g["n"], g["config_key"]))146147148def best_per_model(rows: list[dict[str, Any]], higher_is_better: bool) -> list[dict[str, Any]]:149 best: dict[str, dict[str, Any]] = {}150 for r in rows:151 cur = best.get(r["model_id"])152 if cur is None or (r["score"] > cur["score"] if higher_is_better else r["score"] < cur["score"]) or \153 (r["score"] == cur["score"] and (r.get("evaluated_at") or r["observed_at"]) > (cur.get("evaluated_at") or cur["observed_at"])):154 best[r["model_id"]] = r155 return sorted(best.values(), key=lambda r: (-r["score"] if higher_is_better else r["score"], r["model_name"] or ""))156157158def rank_rows(rows: list[dict[str, Any]], higher_is_better: bool) -> list[dict[str, Any]]:159 """Competition ranking (1, 2, 2, 4) over best-per-model rows."""160 ranked = best_per_model(rows, higher_is_better)161 out: list[dict[str, Any]] = []162 prev_score, prev_rank = None, 0163 for i, r in enumerate(ranked, start=1):164 rank = prev_rank if prev_score is not None and r["score"] == prev_score else i165 prev_score, prev_rank = r["score"], rank166 out.append({**r, "rank": rank})167 return out168169170def leaderboard_rows(group: dict[str, Any], *, history_rows: list[dict[str, Any]] | None = None, comparable_only: bool = False) -> list[dict[str, Any]]:171 """Public leaderboard rows for one group. `history_rows` (closed rows of the same group) give the previous rank per model."""172 hib = group["higher_is_better"]173 ranked = rank_rows(group["rows"], hib)174 leader = ranked[0] if ranked else None175 prev_rank: dict[str, int] = {}176 if history_rows:177 prev_rank = {r["model_id"]: r["rank"] for r in rank_rows(history_rows, hib)}178 out: list[dict[str, Any]] = []179 for r in ranked:180 level, reasons = comparability(leader["config"] if leader else None, r.get("config"), leader["metric"] if leader else None, r.get("metric")) if leader else ("comparable", [])181 if comparable_only and level != "comparable":182 continue183 prev = prev_rank.get(r["model_id"])184 out.append({185 "rank": r["rank"], "model": _model_ref(r), "score": r["score"], "metric": r["metric_canonical"], "unit": r.get("unit"), "higher_is_better": hib,186 "delta_rank": (prev - r["rank"]) if prev is not None else None, "previous_rank": prev,187 "trust_level": r["trust_level"], "trust_label": TRUST_LABELS.get(r["trust_level"], r["trust_level"]), "config": config_summary(r.get("config")),188 "config_key": r["config_key"], "comparability": level, "comparability_reasons": reasons, "evaluated_at": r.get("evaluated_at"), "observed_at": r["observed_at"],189 "source_url": r.get("source_url"), "tier": r.get("tier"), "result_id": r["id"], "n_rows": sum(1 for x in group["rows"] if x["model_id"] == r["model_id"]),190 })191 return out192193194def _model_ref(r: dict[str, Any]) -> dict[str, Any]:195 attrs = r.get("model_attrs") or {}196 return {"id": r["model_id"], "slug": r["model_slug"], "name": r["model_name"], "entity_type": r.get("model_type", "model"),197 "organization": {"id": r.get("organization_id"), "slug": r.get("org_slug"), "name": r.get("org_name")} if r.get("organization_id") else None,198 "attributes": {k: attrs[k] for k in ("openness", "parameter_count", "context_length", "release_date", "modalities", "license", "family") if attrs.get(k) not in (None, "", [])}}199200201def group_summary(g: dict[str, Any]) -> dict[str, Any]:202 return {"metric": g["metric"], "config_key": g["config_key"], "label": g["label"], "n": g["n"], "model_count": g["model_count"], "config": g["config"],203 "higher_is_better": g["higher_is_better"], "trust_mix": g["trust_mix"]}204205206def frontier_series(rows: list[dict[str, Any]], higher_is_better: bool) -> list[dict[str, Any]]:207 """History of the leader: each time a new best score appears (ordered by coalesce(evaluated_at, observed_at))."""208 ordered = sorted(rows, key=lambda r: ((r.get("evaluated_at") or r["observed_at"]), r["observed_at"], r["id"]))209 best: float | None = None210 out: list[dict[str, Any]] = []211 for r in ordered:212 s = float(r["score"])213 better = best is None or (s > best if higher_is_better else s < best)214 if better:215 best = s216 out.append({"date": (r.get("evaluated_at") or r["observed_at"]), "model": _model_ref(r), "score": s, "trust_level": r["trust_level"],217 "config": config_summary(r.get("config")), "result_id": r["id"]})218 return out219220221def leader_at(rows: list[dict[str, Any]], at: datetime, benchmark_attrs: dict[str, Any] | None) -> dict[str, Any] | None:222 """Leader of the primary group as it was known at `at`: rows observed at or before `at` and not closed before `at`."""223 visible = [r for r in rows if r["observed_at"] <= at and (r.get("valid_to") is None or r["valid_to"] > at)]224 if not visible:225 return None226 groups = list(group_rows(visible).values())227 pg = primary_group(benchmark_attrs, groups)228 if not pg:229 return None230 ranked = rank_rows(pg["rows"], pg["higher_is_better"])231 if not ranked:232 return None233 top = ranked[0]234 return {"model": _model_ref(top), "score": top["score"], "metric": pg["metric"], "config_key": pg["config_key"], "group_label": pg["label"],235 "trust_level": top["trust_level"], "n_models": pg["model_count"], "as_of": at}236237238# ------------------------------------------------------------------------------------------------------------------ benchmark catalogue239240241async def benchmark_meta(conn: AsyncConnection, ids: list[str] | None = None) -> dict[str, dict[str, Any]]:242 where = "e.entity_type = 'benchmark' and e.merged_into is null" + (" and e.id = any(cast(:ids as text[]))" if ids is not None else "")243 rows = await fetch_all(conn, f"select e.id, e.slug, e.canonical_name, e.attributes from entities e where {where}", ids=ids)244 out: dict[str, dict[str, Any]] = {}245 for r in rows:246 attrs = r["attributes"] or {}247 fam, variant = family_of(r["slug"])248 out[r["id"]] = {"id": r["id"], "slug": r["slug"], "name": r["canonical_name"], "attributes": attrs, "category": attrs.get("category"),249 "family": attrs.get("family") or fam, "variant": attrs.get("variant") or variant,250 "metric": normalize_metric(attrs.get("metric")) or attrs.get("metric"), "unit": attrs.get("unit"),251 "direction": attrs.get("direction") or ("higher" if METRICS.get(normalize_metric(attrs.get("metric")) or "", {}).get("higher_is_better", True) else "lower")}252 return out253254255async def all_primary_groups(conn: AsyncConnection, *, min_results: int = 1) -> dict[str, dict[str, Any]]:256 """{benchmark_id: primary group (with rows)} across every benchmark, from current rows of canonical models."""257 meta = await benchmark_meta(conn)258 rows = await load_results(conn)259 by_bench: dict[str, list[dict[str, Any]]] = defaultdict(list)260 for r in rows:261 by_bench[r["benchmark_id"]].append(r)262 out: dict[str, dict[str, Any]] = {}263 for bid, brows in by_bench.items():264 groups = list(group_rows(brows).values())265 pg = primary_group(meta.get(bid, {}).get("attributes"), groups)266 if pg and pg["n"] >= min_results:267 pg = dict(pg)268 pg["benchmark"] = meta.get(bid) or {"id": bid, "slug": brows[0]["benchmark_slug"], "name": brows[0]["benchmark_name"]}269 pg["all_groups"] = groups270 out[bid] = pg271 return out272273274# ------------------------------------------------------------------------------------------------------------------ frontier model set275276277async def frontier_model_ids(conn: AsyncConnection, *, use_cache: bool = True) -> tuple[set[str], dict[str, Any]]:278 """Frontier composition (see FRONTIER_METHODOLOGY). Returns (ids, sample counts)."""279 if use_cache:280 hit = await cache.cache_get(FRONTIER_CACHE_KEY)281 if hit and hit.get("ids"):282 # guard: the cached set must still be live canonical models of THIS database (cache keys are namespaced per DB, but a283 # canonicalisation run can merge ids away between refreshes) — otherwise recompute284 live = await fetch_val(conn, "select count(*) from entities where id = any(cast(:ids as text[])) and entity_type = 'model' and merged_into is null", ids=hit["ids"])285 if int(live or 0) == len(hit["ids"]):286 return set(hit["ids"]), hit["composition"]287 since = (datetime.now(UTC) - timedelta(days=365)).date().isoformat()288 recent = await fetch_all(conn, """289 select e.id from entities e where e.entity_type = 'model' and e.merged_into is null and e.attributes->>'release_date' >= :since290 and e.organization_id in (select organization_id from entities where entity_type = 'model' and merged_into is null and organization_id is not null291 group by organization_id having count(*) >= 3)""", since=since)292 recent_ids = {r["id"] for r in recent}293 top10: set[str] = set()294 groups = await all_primary_groups(conn)295 for g in groups.values():296 for r in rank_rows(g["rows"], g["higher_is_better"]):297 if r["rank"] <= 10:298 top10.add(r["model_id"])299 ids = recent_ids | top10300 composition = {"recent_by_active_orgs": len(recent_ids), "top10_on_a_benchmark": len(top10), "total": len(ids), "since": since}301 await cache.cache_set(FRONTIER_CACHE_KEY, {"ids": sorted(ids), "composition": composition}, FRONTIER_TTL_S)302 return ids, composition303304305__all__ = ["CANONICAL_MODEL", "CURRENT", "FRONTIER_METHODOLOGY", "RESULT_SELECT", "all_primary_groups", "benchmark_meta", "best_per_model", "config_summary",306 "direction_of", "enrich", "frontier_model_ids", "frontier_series", "group_label", "group_rows", "group_summary", "leader_at", "leaderboard_rows",307 "load_results", "primary_group", "primary_metric", "rank_rows"]308