# Benchmark definitions (entities of type `benchmark`). Results come from connectors; definitions are curated with sources.
benchmarks:
  - key: mmlu
    name: MMLU
    aliases: [Massive Multitask Language Understanding]
    category: knowledge
    task: 57-subject multiple-choice questions
    metric: accuracy
    unit: "%"
    creator: Hendrycks et al.
    website: https://github.com/hendrycks/test
    paper: https://arxiv.org/abs/2009.03300
    known_limitations: Widespread training-data contamination; saturated at the frontier.
    source_url: https://arxiv.org/abs/2009.03300
  - key: mmlu-pro
    name: MMLU-Pro
    category: knowledge
    task: harder, 10-option MMLU variant
    metric: accuracy
    unit: "%"
    website: https://github.com/TIGER-AI-Lab/MMLU-Pro
    paper: https://arxiv.org/abs/2406.01574
    source_url: https://arxiv.org/abs/2406.01574
  - key: gpqa
    name: GPQA
    aliases: [GPQA Diamond]
    category: reasoning
    task: graduate-level science questions
    metric: accuracy
    unit: "%"
    website: https://github.com/idavidrein/gpqa
    paper: https://arxiv.org/abs/2311.12022
    source_url: https://arxiv.org/abs/2311.12022
  - key: humaneval
    name: HumanEval
    category: coding
    task: Python function synthesis from docstrings
    metric: pass@1
    unit: "%"
    website: https://github.com/openai/human-eval
    paper: https://arxiv.org/abs/2107.03374
    known_limitations: Saturated; small (164 problems).
    source_url: https://arxiv.org/abs/2107.03374
  - key: swe-bench-verified
    name: SWE-bench Verified
    aliases: [SWE-bench]
    category: coding
    task: resolve real GitHub issues (500 human-validated instances)
    metric: resolved
    unit: "%"
    website: https://www.swebench.com
    paper: https://arxiv.org/abs/2310.06770
    known_limitations: Scaffold/agent dependent; results are not comparable across harnesses.
    source_url: https://www.swebench.com/
  - key: aider-polyglot
    name: Aider polyglot
    aliases: [Aider polyglot coding leaderboard]
    category: coding
    task: 225 Exercism exercises in 6 languages, edit-format aware
    metric: pass rate (2 attempts)
    unit: "%"
    website: https://aider.chat/docs/leaderboards/
    known_limitations: Depends on aider's edit format and prompting; cost column depends on provider pricing.
    source_url: https://aider.chat/docs/leaderboards/
  - key: livebench
    name: LiveBench
    category: general
    task: contamination-limited, monthly refreshed questions across 6 categories
    metric: average score
    unit: "%"
    website: https://livebench.ai
    paper: https://arxiv.org/abs/2406.19314
    source_url: https://livebench.ai/
  - key: math-500
    name: MATH-500
    aliases: [MATH]
    category: math
    task: competition mathematics
    metric: accuracy
    unit: "%"
    paper: https://arxiv.org/abs/2103.03874
    source_url: https://arxiv.org/abs/2103.03874
  - key: aime-2025
    name: AIME 2025
    aliases: [AIME]
    category: math
    task: American Invitational Mathematics Examination problems
    metric: accuracy
    unit: "%"
    known_limitations: 30 problems per year; high variance; often reported with majority voting.
    source_url: https://maa.org/maa-invitational-competitions/
  - key: arc-agi
    name: ARC-AGI
    aliases: [ARC-AGI-1, ARC-AGI-2, Abstraction and Reasoning Corpus]
    category: reasoning
    task: novel visual abstraction puzzles
    metric: accuracy
    unit: "%"
    website: https://arcprize.org
    source_url: https://arcprize.org/
  - key: humanitys-last-exam
    name: Humanity's Last Exam
    aliases: [HLE]
    category: knowledge
    task: expert-written frontier questions
    metric: accuracy
    unit: "%"
    website: https://lastexam.ai
    paper: https://arxiv.org/abs/2501.14249
    source_url: https://lastexam.ai/
  - key: lmarena-text
    name: LMArena text leaderboard
    aliases: [Chatbot Arena, LMSYS Chatbot Arena, Arena Elo]
    category: preference
    task: crowdsourced pairwise human preference
    metric: Elo / Bradley–Terry score
    unit: ""
    website: https://lmarena.ai
    known_limitations: Style bias; sampling of prompts by users.
    source_url: https://lmarena.ai/
  - key: mmmu
    name: MMMU
    category: multimodal
    task: college-level multimodal understanding
    metric: accuracy
    unit: "%"
    website: https://mmmu-benchmark.github.io
    paper: https://arxiv.org/abs/2311.16502
    source_url: https://arxiv.org/abs/2311.16502
  - key: tau-bench
    name: τ-bench
    aliases: [tau-bench, TAU-bench]
    category: agentic
    task: tool-agent-user interaction in retail/airline domains
    metric: pass^1
    unit: "%"
    paper: https://arxiv.org/abs/2406.12045
    source_url: https://arxiv.org/abs/2406.12045
  - key: terminal-bench
    name: Terminal-Bench
    category: agentic
    task: terminal tasks solved by agents
    metric: accuracy
    unit: "%"
    website: https://www.tbench.ai
    source_url: https://www.tbench.ai/
  - key: ifeval
    name: IFEval
    category: instruction-following
    task: verifiable instruction following
    metric: prompt-level strict accuracy
    unit: "%"
    paper: https://arxiv.org/abs/2311.07911
    source_url: https://arxiv.org/abs/2311.07911
  - key: mteb
    name: MTEB
    aliases: [Massive Text Embedding Benchmark]
    category: embeddings
    task: embedding tasks across retrieval, classification, clustering…
    metric: mean score
    unit: ""
    website: https://huggingface.co/spaces/mteb/leaderboard
    paper: https://arxiv.org/abs/2210.07316
    source_url: https://arxiv.org/abs/2210.07316
  - key: artificial-analysis-intelligence-index
    name: Artificial Analysis Intelligence Index
    category: composite
    task: composite of several evaluations run by Artificial Analysis
    metric: index
    unit: ""
    website: https://artificialanalysis.ai
    known_limitations: Proprietary composite; methodology versions change.
    source_url: https://artificialanalysis.ai/methodology