# Benchmarks reported by the leaderboard connectors (SWE-bench boards, Artificial Analysis component evaluations). Fragment of registry/benchmarks.yaml.
benchmarks:
  - key: swe-bench-lite
    name: SWE-bench Lite
    category: coding
    task: resolve real GitHub issues (300-instance subset)
    metric: resolved
    unit: "%"
    website: https://www.swebench.com
    paper: https://arxiv.org/abs/2310.06770
    known_limitations: Scaffold/agent dependent; results are not comparable across harnesses.
    source_url: https://www.swebench.com/
  - key: swe-bench-full
    name: SWE-bench (full test split)
    aliases: [SWE-bench Test, SWE-bench Full]
    category: coding
    task: resolve real GitHub issues (2,294 instances)
    metric: resolved
    unit: "%"
    website: https://www.swebench.com
    paper: https://arxiv.org/abs/2310.06770
    source_url: https://www.swebench.com/
  - key: swe-bench-multimodal
    name: SWE-bench Multimodal
    category: coding
    task: resolve visual JavaScript issues (screenshots + code)
    metric: resolved
    unit: "%"
    website: https://www.swebench.com/multimodal
    source_url: https://www.swebench.com/
  - key: swe-bench-multilingual
    name: SWE-bench Multilingual
    category: coding
    task: resolve GitHub issues across 9 programming languages
    metric: resolved
    unit: "%"
    website: https://www.swebench.com/multilingual
    source_url: https://www.swebench.com/
  - key: mmmu-pro
    name: MMMU-Pro
    category: multimodal
    task: robust multimodal understanding (10-option, vision-only variants)
    metric: accuracy
    unit: "%"
    paper: https://arxiv.org/abs/2409.02813
    source_url: https://arxiv.org/abs/2409.02813
  - key: livecodebench
    name: LiveCodeBench
    category: coding
    task: contamination-free competitive programming problems
    metric: pass@1
    unit: "%"
    website: https://livecodebench.github.io
    paper: https://arxiv.org/abs/2403.07974
    source_url: https://arxiv.org/abs/2403.07974
  - key: scicode
    name: SciCode
    category: coding
    task: research-level scientific coding problems
    metric: accuracy
    unit: "%"
    website: https://scicode-bench.github.io
    paper: https://arxiv.org/abs/2407.13168
    source_url: https://arxiv.org/abs/2407.13168
  - key: ifbench
    name: IFBench
    category: instruction-following
    task: precise instruction following with novel constraints
    metric: accuracy
    unit: "%"
    paper: https://arxiv.org/abs/2507.02833
    source_url: https://arxiv.org/abs/2507.02833
  - key: tau2-bench
    name: τ²-bench
    aliases: [tau2-bench, TAU2-bench, tau-squared bench]
    category: agentic
    task: dual-control tool-agent-user interaction (telecom, retail, airline)
    metric: pass^1
    unit: "%"
    paper: https://arxiv.org/abs/2506.07982
    source_url: https://arxiv.org/abs/2506.07982