HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""Benchmark ontology — families, variants, metric bounds, trust levels and the comparability of two result configurations.23Two scores are only *directly comparable* when they measure the same benchmark variant with the same metric under configurations that4do not change the task: same dataset revision/variant, same evaluator or harness class, same scaffold when the benchmark is agentic,5same shot/pass regime. Reasoning effort, sampling temperature and judge differences make them *partially comparable*. Different6variants (SWE-bench Verified vs Lite), different metrics, or different pass regimes are *not directly comparable*.7"""8from __future__ import annotations910import hashlib11import json12import re13from typing import Any1415COMPARABLE, PARTIAL, NOT_COMPARABLE = "comparable", "partially-comparable", "not-comparable"1617# Benchmark family → known variants (registry keys). Used to build `variant_of` relations and the family attribute; the registry YAML18# carries the authoritative per-entry `family`/`variant` fields, this table is the fallback for benchmarks created from other sources.19FAMILIES: dict[str, dict[str, Any]] = {20 "swe-bench": {"label": "SWE-bench", "variants": {"swe-bench-full": "full", "swe-bench-verified": "Verified", "swe-bench-lite": "Lite",21 "swe-bench-multimodal": "Multimodal", "swe-bench-multilingual": "Multilingual", "swe-bench-pro": "Pro"}},22 "aime": {"label": "AIME", "variants": {"aime-2024": "2024", "aime-2025": "2025", "aime-2026": "2026"}},23 "mmlu": {"label": "MMLU", "variants": {"mmlu": "original", "mmlu-pro": "Pro", "mmlu-redux": "Redux", "mmmlu": "multilingual"}},24 "mmmu": {"label": "MMMU", "variants": {"mmmu": "original", "mmmu-pro": "Pro"}},25 "gpqa": {"label": "GPQA", "variants": {"gpqa": "main", "gpqa-diamond": "Diamond"}},26 "tau-bench": {"label": "τ-bench", "variants": {"tau-bench": "v1", "tau2-bench": "τ²"}},27 "livebench": {"label": "LiveBench", "variants": {"livebench": "global", "livebench-2": "global"}},28 "arc-agi": {"label": "ARC-AGI", "variants": {"arc-agi": "1", "arc-agi-2": "2", "arc-agi-3": "3"}},29 "humaneval": {"label": "HumanEval", "variants": {"humaneval": "original", "humaneval-plus": "Plus", "mbpp": None}},30 "livecodebench": {"label": "LiveCodeBench", "variants": {"livecodebench": "rolling"}},31 "math": {"label": "MATH", "variants": {"math": "full", "math-500": "500"}},32 "terminal-bench": {"label": "Terminal-Bench", "variants": {"terminal-bench": "1.0", "terminal-bench-2": "2.0"}},33 "ifeval": {"label": "IFEval", "variants": {"ifeval": "original", "ifbench": "IFBench"}},34 "artificial-analysis-intelligence-index": {"label": "Artificial Analysis Intelligence Index", "variants": {"artificial-analysis-intelligence-index": "index"}},35 "humanitys-last-exam": {"label": "Humanity's Last Exam", "variants": {"humanitys-last-exam": "full"}},36 "scicode": {"label": "SciCode", "variants": {"scicode": "main"}},37 "aider-polyglot": {"label": "Aider polyglot", "variants": {"aider-polyglot": "polyglot"}},38 "lmarena": {"label": "LMArena", "variants": {"lmarena-text": "text", "lmarena-vision": "vision", "lmarena-webdev": "webdev"}},39 "mteb": {"label": "MTEB", "variants": {"mteb": "v1", "mteb-v2": "v2", "mmteb": "MMTEB"}},40}41_KEY_TO_FAMILY: dict[str, tuple[str, str | None]] = {k: (fam, v) for fam, spec in FAMILIES.items() for k, v in spec["variants"].items()}424344def family_of(benchmark_key: str) -> tuple[str | None, str | None]:45 """(family key, variant label) for a registry key / slug."""46 if benchmark_key in _KEY_TO_FAMILY:47 return _KEY_TO_FAMILY[benchmark_key]48 for fam in FAMILIES:49 if benchmark_key.startswith(fam):50 return fam, benchmark_key[len(fam):].strip("-") or None51 return None, None525354# ---------------------------------------------------------------------------------------------- metrics55METRICS: dict[str, dict[str, Any]] = {56 # canonical metric → bounds and direction. `None` bound = unbounded.57 "accuracy": {"min": 0, "max": 100, "higher_is_better": True, "unit": "%"},58 "pass@1": {"min": 0, "max": 100, "higher_is_better": True, "unit": "%"},59 "pass^1": {"min": 0, "max": 100, "higher_is_better": True, "unit": "%"},60 "pass^k": {"min": 0, "max": 100, "higher_is_better": True, "unit": "%"},61 "resolved": {"min": 0, "max": 100, "higher_is_better": True, "unit": "%"},62 "pass_rate_2": {"min": 0, "max": 100, "higher_is_better": True, "unit": "%"},63 "percent_cases_well_formed": {"min": 0, "max": 100, "higher_is_better": True, "unit": "%"},64 "global_average": {"min": 0, "max": 100, "higher_is_better": True, "unit": "%"},65 "average score": {"min": 0, "max": 100, "higher_is_better": True, "unit": "%"},66 "mean score": {"min": 0, "max": 100, "higher_is_better": True, "unit": "%"},67 "index": {"min": 0, "max": 100, "higher_is_better": True, "unit": ""},68 "elo": {"min": 0, "max": None, "higher_is_better": True, "unit": ""},69 "score": {"min": None, "max": None, "higher_is_better": True, "unit": ""},70 "win_rate": {"min": 0, "max": 100, "higher_is_better": True, "unit": "%"},71 "f1": {"min": 0, "max": 100, "higher_is_better": True, "unit": "%"},72 "ndcg": {"min": 0, "max": 100, "higher_is_better": True, "unit": "%"},73 "perplexity": {"min": 0, "max": None, "higher_is_better": False, "unit": ""},74 "latency_ms": {"min": 0, "max": None, "higher_is_better": False, "unit": "ms"},75 "cost_usd": {"min": 0, "max": None, "higher_is_better": False, "unit": "USD"},76}77_METRIC_ALIASES = {78 "acc": "accuracy", "exact match": "accuracy", "em": "accuracy", "pass@1": "pass@1", "pass at 1": "pass@1", "pass^1": "pass^1", "pass1": "pass^1",79 "pass rate (2 attempts)": "pass_rate_2", "pass rate": "pass_rate_2", "percent resolved": "resolved", "% resolved": "resolved", "resolved rate": "resolved",80 "elo / bradley–terry score": "elo", "elo / bradley-terry score": "elo", "arena score": "elo", "bradley-terry": "elo", "mean score": "mean score",81 "average score": "average score", "global average": "global_average", "prompt-level strict accuracy": "accuracy", "strict accuracy": "accuracy",82 "intelligence index": "index", "composite": "index", "ppl": "perplexity",83}848586def normalize_metric(raw: str | None) -> str | None:87 if not raw:88 return None89 s = raw.strip().lower()90 if s.startswith("category:"):91 return raw.strip() # LiveBench per-category averages keep their label; they are a separate metric each92 if s in METRICS:93 return s94 return _METRIC_ALIASES.get(s) or (s if re.fullmatch(r"[a-z0-9_@^%.\- ]+", s) else None)959697def metric_bounds(metric: str | None, unit: str | None = None) -> tuple[float | None, float | None]:98 m = normalize_metric(metric)99 if m and m.startswith("category:"):100 return 0, 100101 spec = METRICS.get(m or "")102 if spec:103 return spec["min"], spec["max"]104 if unit == "%":105 return 0, 100106 return None, None107108109# ---------------------------------------------------------------------------------------------- trust levels110TRUST_LEVELS = ("official-model-card", "official-benchmark", "peer-reviewed", "independent-evaluator", "community", "unverified")111TRUST_LABELS = {112 "official-model-card": "Official model card / technical report (self-reported)",113 "official-benchmark": "Official benchmark leaderboard (submissions checked by the benchmark owner)",114 "peer-reviewed": "Peer-reviewed paper",115 "independent-evaluator": "Independent third-party evaluator",116 "community": "Community-run leaderboard or submission",117 "unverified": "Unverified / unknown provenance",118}119# source key (domain) → trust level of results it publishes120SOURCE_TRUST: dict[str, str] = {121 "swebench.com": "official-benchmark", "aider.chat": "official-benchmark", "livebench.ai": "official-benchmark", "artificialanalysis.ai": "independent-evaluator",122 "lmarena.ai": "independent-evaluator", "scale.com": "independent-evaluator", "epoch.ai": "independent-evaluator", "vals.ai": "independent-evaluator",123 "huggingface.co": "community", "github.com": "community", "arxiv.org": "peer-reviewed", "openreview.net": "peer-reviewed",124}125OFFICIAL_LAB_SOURCES = {"docs.claude.com", "platform.openai.com", "ai.google.dev", "docs.mistral.ai", "api-docs.deepseek.com", "docs.cohere.com",126 "docs.x.ai", "ai.meta.com", "llama.com", "qwenlm.github.io", "developer.nvidia.com", "machinelearning.apple.com"}127128129def trust_level(source_key: str | None, config: dict[str, Any] | None = None, *, extractor: str = "deterministic") -> str:130 cfg = config or {}131 if source_key in SOURCE_TRUST:132 level = SOURCE_TRUST[source_key]133 if level == "official-benchmark" and cfg.get("checked_by_swebench") is False:134 return "community"135 if level == "official-benchmark" and str(cfg.get("submission", "")).lower() in ("self-reported", "self reported", "unverified"):136 return "community"137 return level138 if source_key in OFFICIAL_LAB_SOURCES or cfg.get("self_reported") or cfg.get("source_kind") == "model_card":139 return "official-model-card"140 if extractor == "llm":141 return "unverified"142 return "unverified"143144145# ---------------------------------------------------------------------------------------------- comparability146# Config keys that change the *task* (must match for full comparability).147TASK_KEYS = ("variant", "board", "harness", "evaluator", "subset", "split", "shots", "pass_count", "attempts", "language", "scaffold", "agent", "system")148# Config keys that identify the *run* (a LiveBench release, an AA index version, a dataset revision): a newer run supersedes the older one149# for the same task (`run_group`, one current row per task), and two runs of the same task are only partially comparable.150RUN_KEYS = ("release", "index_version", "version", "dataset_revision")151# Config keys that change the *conditions* (mismatch → partially comparable).152CONDITION_KEYS = ("reasoning_effort", "reasoning", "thinking_budget", "temperature", "judge", "tools", "tool_use", "max_tokens", "context_length",153 "sampling", "aggregation", "edit_format", "model_tag", *RUN_KEYS)154# Keys that are pure bookkeeping (never affect comparability).155IGNORED_KEYS = {"aa_slug", "livebench_model_id", "api_model_id", "date", "submission", "checked_by_swebench", "open_source_system", "system_org",156 "total_cost_usd", "cost_per_instance_usd", "dirname", "command", "versions", "test_cases", "seconds_per_case", "estimated",157 "livebench_hf_link", "source_kind", "self_reported", "subtasks", "notes", "url"}158159160def _clean(v: Any) -> Any:161 if isinstance(v, str):162 return v.strip().lower()163 return v164165166def config_key(config: dict[str, Any] | None, metric: str | None = None) -> str:167 """Stable hash of the comparability-relevant part of a result configuration (task keys + metric). Run keys (release, index168 version) are deliberately excluded: they define `run_group`, so the newest run of a task replaces the older ones on leaderboards."""169 cfg = config or {}170 core = {k: _clean(cfg[k]) for k in TASK_KEYS if k in cfg and cfg[k] not in (None, "", [], {})}171 core["metric"] = normalize_metric(metric) or ""172 return hashlib.sha1(json.dumps(core, sort_keys=True, default=str).encode()).hexdigest()[:12]173174175def comparability(a_cfg: dict[str, Any] | None, b_cfg: dict[str, Any] | None, a_metric: str | None = None, b_metric: str | None = None,176 *, same_benchmark: bool = True) -> tuple[str, list[str]]:177 """Return (level, reasons). `same_benchmark` False → not comparable outright."""178 reasons: list[str] = []179 if not same_benchmark:180 return NOT_COMPARABLE, ["different benchmark variants"]181 ma, mb = normalize_metric(a_metric), normalize_metric(b_metric)182 if ma != mb and (ma or mb):183 return NOT_COMPARABLE, [f"different metrics ({a_metric} vs {b_metric})"]184 a, b = a_cfg or {}, b_cfg or {}185 for k in TASK_KEYS:186 if k in a or k in b:187 va, vb = _clean(a.get(k)), _clean(b.get(k))188 if va != vb and va not in (None, "") and vb not in (None, ""):189 reasons.append(f"{k}: {a.get(k)} vs {b.get(k)}")190 if reasons:191 return NOT_COMPARABLE, reasons192 for k in CONDITION_KEYS:193 if k in a or k in b:194 va, vb = _clean(a.get(k)), _clean(b.get(k))195 if va != vb:196 reasons.append(f"{k}: {a.get(k) if k in a else 'unspecified'} vs {b.get(k) if k in b else 'unspecified'}")197 if reasons:198 return PARTIAL, reasons199 return COMPARABLE, ["same variant, metric and evaluation conditions"]200201202def variant_from_config(config: dict[str, Any] | None) -> str | None:203 cfg = config or {}204 for k in ("variant", "board", "subset", "split"):205 v = cfg.get(k)206 if isinstance(v, str) and v.strip():207 return v.strip()208 return None209210211def run_group_from_config(config: dict[str, Any] | None) -> str | None:212 """The 'run' a result belongs to (a LiveBench release, an aider run date, an AA index version…). One current row per run group."""213 cfg = config or {}214 for k in ("release", "index_version", "version", "date", "dataset_revision"):215 v = cfg.get(k)216 if v not in (None, ""):217 return str(v)218 return None219220221__all__ = ["COMPARABLE", "CONDITION_KEYS", "FAMILIES", "IGNORED_KEYS", "METRICS", "NOT_COMPARABLE", "PARTIAL", "RUN_KEYS", "TASK_KEYS", "TRUST_LABELS",222 "TRUST_LEVELS", "comparability", "config_key", "family_of", "metric_bounds", "normalize_metric", "run_group_from_config", "trust_level",223 "variant_from_config"]224