HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1<!DOCTYPE html>2<html lang="en">3<head>4 <meta charset="UTF-8">5 <meta name="viewport" content="width=device-width, initial-scale=1.0">6 <title>SWE-bench Leaderboards</title>7 <link rel="stylesheet" href="css/main.css">8 <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.2.0/css/all.min.css">9 <script10 async11 src="https://www.googletagmanager.com/gtag/js?id=G-H9XFCMDPNS"12 ></script>13 <script>14 window.dataLayer = window.dataLayer || [];15 function gtag() {16 dataLayer.push(arguments);17 }18 gtag("js", new Date());1920 gtag("config", "G-H9XFCMDPNS");21 </script>22 <script>23 // Initialize theme immediately to prevent flash24 (function() {25 const storedTheme = localStorage.getItem('theme');26 const prefersDark = window.matchMedia('(prefers-color-scheme: dark)').matches;27 if (storedTheme === 'dark' || (!storedTheme && prefersDark)) {28 document.documentElement.classList.add('dark-mode');29 }30 })();31 </script>32 33 <link rel="icon" href="favicon.ico" type="image/x-icon">3435</head>36<body class="page-leaderboard">37 <div class="sidebar-overlay"></div>38<header class="mobile-header">39 <a href="/" class="mobile-logo"><img src="img/swe-llama.svg" alt="SWE-bench logo" class="sidebar-logo-icon">SWE-bench</a>40 <button class="sidebar-opener" aria-label="Open menu">41 <i class="fas fa-bars"></i>42 </button>43</header>4445<aside class="sidebar" aria-label="Sidebar">46 <div class="sidebar-header">47 <a href="/" class="sidebar-logo"><img src="img/swe-llama.svg" alt="SWE-bench logo" class="sidebar-logo-icon">SWE-bench</a>48 <button class="mobile-nav-toggle" aria-label="Close menu">49 <i class="fas fa-times"></i>50 </button>51 </div>52 <nav class="sidebar-nav">53 <ul>54 <li><a href="index.html" class="nav-link" data-page="index">Leaderboards</a></li>55 <li class="nav-section-title">Benchmarks</li>56 <li><a href="original.html" class="nav-link" data-page="original">SWE-bench</a></li>57 <li><a href="verified.html" class="nav-link" data-page="verified">SWE-bench Verified</a></li>58 <li><a href="multilingual.html" class="nav-link" data-page="multilingual">SWE-bench Multilingual</a></li>59 <li><a href="multimodal.html" class="nav-link" data-page="multimodal">SWE-bench Multimodal</a></li>60 <li><a href="lite.html" class="nav-link" data-page="lite">SWE-bench Lite</a></li>61 <li class="nav-section-title">About</li>62 <li><a href="https://openreview.net/pdf?id=VTF8yNQM66" class="nav-link" data-page="paper">Paper <i class="fas fa-external-link-alt"></i></a></li>63 <li><a href="https://swebench.com/SWE-bench/" class="nav-link" data-page="docs">Docs <i class="fas fa-external-link-alt"></i></a></li>64 <li><a href="blog.html" class="nav-link" data-page="blog">Blog</a></li>65 <li><a href="contact.html" class="nav-link" data-page="contact">Contact</a></li>66 <li><a href="citations.html" class="nav-link" data-page="citations">Citations</a></li>67 <li><a href="press.html" class="nav-link" data-page="press">Press</a></li>68 <li><a href="submit.html" class="nav-link" data-page="submit">Submit</a></li> 69 <li class="nav-section-title">SWE-bench Family</li>70 <li><a href="https://codeclash.ai/" class="nav-link" data-page="codeclash"><img src="img/codeclash.svg" alt="CodeClash icon" class="sidebar-family-icon">CodeClash <i class="fas fa-external-link-alt"></i></a></li>71 <li><a href="https://mini-swe-agent.com/" class="nav-link" data-page="mini-swe-agent"><img src="img/mini-icon.svg" alt="mini-SWE-agent icon" class="sidebar-family-icon">mini-SWE-agent <i class="fas fa-external-link-alt"></i></a></li>72 <li><a href="https://programbench.com/" class="nav-link" data-page="programbench"><img src="img/programbench-icon.png" alt="ProgramBench icon" class="sidebar-family-icon">ProgramBench <i class="fas fa-external-link-alt"></i></a></li>73 <li><a href="https://swe-agent.com/latest/" class="nav-link" data-page="swe-agent"><img src="img/swe-agent-icon.svg" alt="SWE-agent icon" class="sidebar-family-icon">SWE-agent <span style="font-size: 0.8em;">(legacy)</span> <i class="fas fa-external-link-alt"></i></a></li>74 <li><a href="https://swebench.com/sb-cli/" class="nav-link" data-page="sb-cli"><img src="img/sb-cli-icon.svg" alt="SWE-bench CLI icon" class="sidebar-family-icon">SWE-bench CLI <i class="fas fa-external-link-alt"></i></a></li>75 <li><a href="https://swe-rex.com/latest/" class="nav-link" data-page="swe-rex"><img src="img/swe-rex-icon.svg" alt="SWE-ReX icon" class="sidebar-family-icon">SWE-ReX <i class="fas fa-external-link-alt"></i></a></li>76 <li><a href="https://swesmith.com/" class="nav-link" data-page="swe-smith"><img src="img/swe-smith-icon.svg" alt="SWE-smith icon" class="sidebar-family-icon">SWE-smith <i class="fas fa-external-link-alt"></i></a></li>77 </ul>78 </nav>79 <div class="sidebar-footer">80 <a href="https://github.com/swe-bench/SWE-bench" target="_blank" rel="noopener noreferrer" aria-label="SWE-bench on GitHub">81 <i class="fab fa-github"></i>82 </a>83 84 <a href="https://www.youtube.com/@SWE-bench" target="_blank" rel="noopener noreferrer" aria-label="SWE-bench on YouTube">85 <i class="fab fa-youtube"></i>86 </a>87 88 <a href="https://twitter.com/SWEbench" target="_blank" rel="noopener noreferrer" aria-label="SWE-bench on Twitter">89 <i class="fab fa-twitter"></i>90 </a>91 92 <a href="https://join.slack.com/t/swe-bench/shared_invite/zt-3rljl10bn-WED_gNRuFQu6YcILJMgeSw" target="_blank" rel="noopener noreferrer" aria-label="Join the SWE-bench Slack">93 <i class="fab fa-slack"></i>94 </a>95 <button id="theme-toggle" class="theme-toggle" aria-label="Toggle dark mode">96 <i class="fas fa-moon"></i>97 </button>98 </div>99</aside>100 101 <div class="main-content">102 103 <header class="page-header">104 <div class="container">105 <div class="header-logo-container">106 <h1>107 <img src="img/swe-llama.svg" alt="SWE-bench Logo" class="header-logo-small">Official Leaderboards108 </h1>109 </div>110 </div>111 </header>112113 <section class="container">114115 <div class="content-section" id="leaderboard">116 <div class="leaderboard-tabs">117 <div class="leaderboard-tab-list">118 <button class="tablinks" data-leaderboard="Verified">Verified</button>119 <button class="tablinks" data-leaderboard="Multimodal">Multimodal</button>120 <button class="tablinks" data-leaderboard="Multilingual">Multilingual</button>121 <button class="tablinks" data-leaderboard="Lite">Lite</button>122 <button class="tablinks" data-leaderboard="Test">Full</button>123 </div>124 </div>125126 127 <script src="js/leaderboardFilters.js"></script>128129130<div class="mb-2 leaderboard-header-container">131 <div id="leaderboard-description-text">132 <!-- Dynamic text will be inserted here -->133 </div>134 <div class="filter-controls-group border-top pt-2" id="filter-bar">135 <button id="compare-btn" class="button button-primary button-icon" type="button" title="Compare selected results" style="display: none;">136 <i class="fa-solid fa-chart-column"></i>137 <span>Compare results</span>138 </button>139140 <div class="filter-menu-container">141 <button id="filter-toggle-btn" class="button button-secondary button-icon filter-toggle-icon" type="button"142 aria-haspopup="true" aria-expanded="false" aria-label="Filter results" title="Filter results">143 <i class="fa-solid fa-filter"></i>144 </button>145 <div id="filter-menu" class="filter-menu" style="display: none;"></div>146 </div>147148 <div id="bash-only-toggle-wrapper" class="preset-toggle" style="display: none;">149 <label title="Show only runs in the mini-SWE-agent environment, so scores compare models rather than harnesses">150 <input type="checkbox" id="bash-only-toggle">151 <span>Bash Only</span>152 </label>153 </div>154155 <div id="filter-chips" class="filter-chips" style="display: none;"></div>156 </div>157</div>158159<!-- Container for dynamically loaded table content -->160<div id="leaderboard-container">161 <!-- Initial table will be loaded here -->162</div>163164<script type="application/json" id="leaderboard-data">[{"name":"Multilingual","results":[{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":true,"cost":105.55067604999999,"date":"2026-02-13","folder":"20260213_mini-v2.0.0a0_gemini-3-flash","instance_calls":52.45333333333333,"instance_cost":0.3518355868333333,"logo":["img/logos/20260213_mini-v2.0.0a0_gemini-3-flash.svg","img/mini-icon.svg"],"logs":"s3://swe-bench-submissions/multilingual/20260213_mini-v2.0.0a0_gemini-3-flash/logs","mini-swe-agent_version":"2.0.0a0","model_display":"Gemini 3 Flash","model_org":"Google DeepMind","model_release_date":20251217,"name":"Gemini 3 Flash","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":72.7,"site":"https://ai.google.dev/gemini-api/docs/models","tags":["Model: gemini-3-flash","Org: Google DeepMind","System: Attempts - 1","Mini: 2.0.0a0"],"trajs":"s3://swe-bench-submissions/multilingual/20260213_mini-v2.0.0a0_gemini-3-flash/trajs","trajs_docent":"https://docent.transluce.org/dashboard/babc7da2-92e1-494d-8c53-b45e7ec30eeb","warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":true,"cost":198.892073,"date":"2026-02-13","folder":"20260213_mini-v2.0.0a0_claude-4-6-opus","instance_calls":28.85333333333333,"instance_cost":0.6629735766666667,"logo":["img/logos/20260213_mini-v2.0.0a0_claude-4-6-opus.jpg","img/mini-icon.svg"],"logs":"s3://swe-bench-submissions/multilingual/20260213_mini-v2.0.0a0_claude-4-6-opus/logs","mini-swe-agent_version":"2.0.0a0","model_display":"Claude 4.6 Opus","model_org":"Anthropic","model_release_date":20260205,"name":"Claude 4.6 Opus","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":72.0,"site":"https://www.anthropic.com/news/claude-opus-4-5","tags":["Model: claude-opus-4-6","Org: Anthropic","System: Attempts - 1","Mini: 2.0.0a0"],"trajs":"s3://swe-bench-submissions/multilingual/20260213_mini-v2.0.0a0_claude-4-6-opus/trajs","trajs_docent":"https://docent.transluce.org/dashboard/8dc7c4e5-2423-470c-88d8-12dafb1870ec","warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":true,"cost":250.06589275,"date":"2026-02-13","folder":"20260213_mini-v2.0.0a0_claude-4-5-opus","instance_calls":30.796666666666667,"instance_cost":0.8335529758333333,"logo":["img/logos/20260213_mini-v2.0.0a0_claude-4-5-opus.jpg","img/mini-icon.svg"],"logs":"s3://swe-bench-submissions/multilingual/20260213_mini-v2.0.0a0_claude-4-5-opus/logs","mini-swe-agent_version":"2.0.0a0","model_display":"Claude 4.5 Opus","model_org":"Anthropic","model_release_date":20251124,"name":"Claude 4.5 Opus","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":70.7,"site":"https://www.anthropic.com/news/claude-opus-4-5","tags":["Model: claude-opus-4-5-20251101","Org: Anthropic","System: Attempts - 1","Mini: 2.0.0a0"],"trajs":"s3://swe-bench-submissions/multilingual/20260213_mini-v2.0.0a0_claude-4-5-opus/trajs","trajs_docent":"https://docent.transluce.org/dashboard/bd05bc79-ff04-41a8-8c6c-8a14866c3206","warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":true,"cost":192.74231006,"date":"2026-02-13","folder":"20260213_mini-v2.0.0a0_glm-5","instance_calls":67.56333333333333,"instance_cost":0.6424743668666667,"logo":["img/logos/20260213_mini-v2.0.0a0_glm-5.svg","img/mini-icon.svg"],"logs":"s3://swe-bench-submissions/multilingual/20260213_mini-v2.0.0a0_glm-5/logs","mini-swe-agent_version":"2.0.0a0","model_display":"GLM 5","model_org":"Z.ai","model_release_date":20260211,"name":"GLM 5","os_model":true,"os_system":true,"reasoning_effort":null,"resolved":69.7,"site":"https://z.ai","tags":["Model: glm-5","Org: Z.ai","System: Attempts - 1","Mini: 2.0.0a0"],"trajs":"s3://swe-bench-submissions/multilingual/20260213_mini-v2.0.0a0_glm-5/trajs","trajs_docent":"https://docent.transluce.org/dashboard/7d28aece-48d0-4e07-bb03-936d982614bf","warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":true,"cost":306.3256754,"date":"2026-02-13","folder":"20260213_mini-v2.0.0a0_gemini-3-pro","instance_calls":48.85,"instance_cost":1.0210855846666667,"logo":["img/logos/20260213_mini-v2.0.0a0_gemini-3-pro.svg","img/mini-icon.svg"],"logs":"s3://swe-bench-submissions/multilingual/20260213_mini-v2.0.0a0_gemini-3-pro/logs","mini-swe-agent_version":"2.0.0a0","model_display":"Gemini 3 Pro","model_org":"Google DeepMind","model_release_date":20251118,"name":"Gemini 3 Pro","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":68.7,"site":"https://ai.google.dev/gemini-api/docs/models","tags":["Model: gemini-3-pro","Org: Google DeepMind","System: Attempts - 1","Mini: 2.0.0a0"],"trajs":"s3://swe-bench-submissions/multilingual/20260213_mini-v2.0.0a0_gemini-3-pro/trajs","trajs_docent":"https://docent.transluce.org/dashboard/ff4871a2-1c92-4a8d-8275-b3aefeb0e6a5","warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":true,"cost":30.05951837,"date":"2026-02-16","folder":"20260216_mini-v2.0.0a0_minimax-2-5","instance_calls":74.16498316498317,"instance_cost":0.10121049956228956,"logo":["img/logos/20260216_mini-v2.0.0a0_minimax-2-5.png","img/mini-icon.svg"],"logs":"s3://swe-bench-submissions/multilingual/20260216_mini-v2.0.0a0_minimax-2-5/logs","mini-swe-agent_version":"2.0.0a0","model_display":"MiniMax M2.5","model_org":"Minimax","model_release_date":20260212,"name":"MiniMax M2.5","os_model":true,"os_system":true,"reasoning_effort":null,"resolved":68.3,"site":"https://www.minimax.io","tags":["Model: minimax-2.5","Org: Minimax","System: Attempts - 1","Mini: 2.0.0a0"],"trajs":"s3://swe-bench-submissions/multilingual/20260216_mini-v2.0.0a0_minimax-2-5/trajs","trajs_docent":"https://docent.transluce.org/dashboard/718e213c-83f3-4265-bc7b-e71243b5f1f9","warning":null}]},{"name":"Test","results":[{"agent":"Sonar Foundation Agent","agent_org":"Sonar","checked":false,"cost":null,"date":"2025-12-19","folder":"20251219_sonar-foundation-agent_claude-opus-4-5","instance_calls":null,"instance_cost":null,"logo":["img/logos/20251219_sonar-foundation-agent_claude-opus-4-5.png"],"logs":"s3://swe-bench-submissions/test/20251219_sonar-foundation-agent_claude-opus-4-5/logs","model_display":"Claude 4.5 Opus","model_org":"Anthropic","model_release_date":null,"name":"Sonar Foundation Agent + Claude 4.5 Opus","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":52.62,"site":"https://www.sonarsource.com","tags":["Model: claude-opus-4-5","Org: Sonar","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/test/20251219_sonar-foundation-agent_claude-opus-4-5/trajs","trajs_docent":false,"warning":null},{"agent":"Salesforce AI Research SAGE","agent_org":"Salesforce AI Research","checked":false,"cost":null,"date":"2025-10-27","folder":"20251027_salesforce_SAGE","instance_calls":null,"instance_cost":null,"logo":["img/logos/20251027_salesforce_SAGE.png"],"logs":"s3://swe-bench-submissions/test/20251027_salesforce_SAGE/logs","model_display":"Multiple","model_org":"Anthropic","model_release_date":null,"name":"Salesforce AI Research SAGE (bash-only)","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":44.25,"site":"https://www.salesforce.com/blog/sage-swe/","tags":["Model: claude-sonnet-4.5","Model: gpt-5","Org: Salesforce AI Research","System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/test/20251027_salesforce_SAGE/trajs","trajs_docent":false,"warning":null},{"agent":"Atlassian Rovo Dev","agent_org":"Atlassian","checked":false,"cost":null,"date":"2025-06-05","folder":"20250605_atlassian-rovo-dev","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250605_atlassian-rovo-dev.png"],"logs":"s3://swe-bench-submissions/test/20250605_atlassian-rovo-dev/logs","model_display":"Undisclosed","model_org":null,"model_release_date":null,"name":"Atlassian Rovo Dev (2025-06-05)","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":41.98,"site":"https://www.atlassian.com/rovo-dev","tags":["Org: Atlassian","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/test/20250605_atlassian-rovo-dev/trajs","trajs_docent":false,"warning":null},{"agent":"Amazon Q Developer Agent","agent_org":"AWS","checked":false,"cost":null,"date":"2025-05-22","folder":"20250522_amazon-q-developer-agent-20250405-dev","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250522_amazon-q-developer-agent-20250405-dev.jpg"],"logs":"s3://swe-bench-submissions/test/20250522_amazon-q-developer-agent-20250405-dev/logs","model_display":"Undisclosed","model_org":null,"model_release_date":null,"name":"Amazon Q Developer Agent (v20250405-dev)","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":37.1,"site":"https://aws.amazon.com/q/developer/","tags":["Org: AWS","System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/test/20250522_amazon-q-developer-agent-20250405-dev/trajs","trajs_docent":false,"warning":null},{"agent":"SWE-agent 1.0","agent_org":"SWE-agent","checked":true,"cost":null,"date":"2025-02-27","folder":"20250227_sweagent-claude-3-7-20250219","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250227_sweagent-claude-3-7-20250219.png"],"logs":"s3://swe-bench-submissions/test/20250227_sweagent-claude-3-7-20250219/logs/","model_display":"Claude 3.7 Sonnet","model_org":"Anthropic","model_release_date":null,"name":"SWE-agent 1.0 (Claude 3.7 Sonnet)","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":33.83,"site":"https://github.com/swe-agent/swe-agent","tags":["Model: claude-3-7-sonnet-20250219","Org: SWE-agent","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/test/20250227_sweagent-claude-3-7-20250219/trajs/","trajs_docent":false,"warning":null},{"agent":"Amazon Q Developer Agent","agent_org":"AWS","checked":false,"cost":null,"date":"2025-01-31","folder":"20250131_amazon-q-developer-agent-20241202-dev","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250131_amazon-q-developer-agent-20241202-dev.jpg"],"logs":"s3://swe-bench-submissions/test/20250131_amazon-q-developer-agent-20241202-dev/logs/","model_display":"Undisclosed","model_org":null,"model_release_date":null,"name":"Amazon Q Developer Agent (v20241202-dev)","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":29.99,"site":"https://aws.amazon.com/q/developer/","tags":["Org: AWS","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/test/20250131_amazon-q-developer-agent-20241202-dev/trajs/","trajs_docent":false,"warning":null}]},{"name":"Verified","results":[{"agent":"Sonar Foundation Agent","agent_org":"Sonar","checked":false,"cost":null,"date":"2025-12-05","folder":"20251205_sonar-foundation-agent_claude-opus-4-5","instance_calls":null,"instance_cost":null,"logo":["img/logos/20251205_sonar-foundation-agent_claude-opus-4-5.png"],"logs":"s3://swe-bench-submissions/verified/20251205_sonar-foundation-agent_claude-opus-4-5/logs","model_display":"Claude 4.5 Opus","model_org":"Anthropic","model_release_date":null,"name":"Sonar Foundation Agent + Claude 4.5 Opus","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":79.2,"site":"https://www.sonarsource.com","tags":["Model: claude-opus-4-5","Org: Sonar","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20251205_sonar-foundation-agent_claude-opus-4-5/trajs","trajs_docent":false,"warning":null},{"agent":"live-SWE-agent","agent_org":"UIUC","checked":false,"cost":null,"date":"2025-12-15","folder":"20251215_livesweagent_claude-opus-4-5","instance_calls":null,"instance_cost":null,"logo":["img/logos/20251215_livesweagent_claude-opus-4-5.png"],"logs":"s3://swe-bench-submissions/verified/20251215_livesweagent_claude-opus-4-5/logs","model_display":"Claude 4.5 Opus","model_org":"Anthropic","model_release_date":null,"name":"live-SWE-agent + Claude 4.5 Opus medium (20251101)","os_model":false,"os_system":true,"reasoning_effort":"medium","resolved":79.2,"site":"https://github.com/OpenAutoCoder/live-swe-agent","tags":["Model: claude-opus-4-5-20251101","Org: UIUC","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20251215_livesweagent_claude-opus-4-5/trajs","trajs_docent":false,"warning":null},{"agent":"TRAE","agent_org":"ByteDance","checked":false,"cost":null,"date":"2025-09-28","folder":"20250928_trae_doubao_seed_code","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250928_trae_doubao_seed_code.jpg"],"logs":"s3://swe-bench-submissions/verified/20250928_trae_doubao_seed_code/logs","model_display":"Doubao-Seed-Code","model_org":"ByteDance","model_release_date":null,"name":"TRAE + Doubao-Seed-Code","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":78.8,"site":["https://github.com/bytedance/trae-agent","https://seed.bytedance.com/"],"tags":["Model: Doubao-Seed-Code","Model: Doubao-Seed-1.6","Org: ByteDance","System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/verified/20250928_trae_doubao_seed_code/trajs","trajs_docent":false,"warning":null},{"agent":"live-SWE-agent","agent_org":"UIUC","checked":"false (See README.md for info on how to get your results verified)","cost":null,"date":"2025-11-20","folder":"20251120_livesweagent_gemini-3-pro-preview","instance_calls":null,"instance_cost":null,"logo":["img/logos/20251120_livesweagent_gemini-3-pro-preview.png"],"logs":"s3://swe-bench-submissions/verified/20251120_livesweagent_gemini-3-pro-preview/logs","model_display":"Gemini 3 Pro Preview","model_org":"Google DeepMind","model_release_date":null,"name":"live-SWE-agent + Gemini 3 Pro Preview (2025-11-18)","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":77.4,"site":"https://github.com/OpenAutoCoder/live-swe-agent","tags":["Model: gemini-3-pro-preview","Org: UIUC","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20251120_livesweagent_gemini-3-pro-preview/trajs","trajs_docent":false,"warning":null},{"agent":"EPAM AI/Run Developer Agent","agent_org":"EPAM Systems, Inc.","checked":false,"cost":null,"date":"2025-08-04","folder":"20250804_epam-ai-run-claude-4-sonnet","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250804_epam-ai-run-claude-4-sonnet.jpg"],"logs":"s3://swe-bench-submissions/verified/20250804_epam-ai-run-claude-4-sonnet/logs","model_display":"Claude 4 Sonnet","model_org":"Anthropic","model_release_date":null,"name":"EPAM AI/Run Developer Agent v20250719 + Claude 4 Sonnet","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":76.8,"site":"https://www.epam.com/services/artificial-intelligence","tags":["Model: claude-sonnet-4-20250514","Org: EPAM Systems, Inc.","System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/verified/20250804_epam-ai-run-claude-4-sonnet/trajs","trajs_docent":false,"warning":null},{"agent":"Atlassian Rovo Dev","agent_org":"Atlassian","checked":false,"cost":null,"date":"2025-09-02","folder":"20250902_atlassian-rovo-dev","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250902_atlassian-rovo-dev.png"],"logs":"s3://swe-bench-submissions/verified/20250902_atlassian-rovo-dev/logs","model_display":"Multiple","model_org":"Anthropic","model_release_date":null,"name":"Atlassian Rovo Dev (2025-09-02)","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":76.8,"site":"https://www.atlassian.com/rovo-dev","tags":["Model: claude-sonnet-4-20250514","Model: gpt-5","Org: Atlassian","System: Attempts - 2"],"trajs":"s3://swe-bench-submissions/verified/20250902_atlassian-rovo-dev/trajs","trajs_docent":false,"warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":null,"cost":376.9539985,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_claude-4-5-opus-high","instance_calls":32.896,"instance_cost":0.753907997,"logo":["img/logos/20260217_mini-v2.0.0_claude-4-5-opus-high.jpg","img/mini-icon.svg"],"logs":null,"mini-swe-agent_version":"2.0.0","model_display":"Claude 4.5 Opus","model_org":"Anthropic","model_release_date":20251124,"name":"Claude 4.5 Opus (high)","os_model":false,"os_system":true,"reasoning_effort":"high","resolved":76.8,"site":"https://www.anthropic.com/claude","tags":["Model: claude-4-5-opus","Org: Anthropic","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_claude-4-5-opus-high/trajs","trajs_docent":"https://docent.transluce.org/dashboard/b038912e-0133-4594-b093-92806f8ffb17","warning":null},{"agent":"ACoder","agent_org":"ACoder","checked":false,"cost":null,"date":"2025-08-19","folder":"20250819_ACoder","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250819_ACoder.png"],"logs":"s3://swe-bench-submissions/verified/20250819_ACoder/logs","model_display":"Multiple","model_org":"Anthropic","model_release_date":null,"name":"ACoder","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":76.4,"site":"https://github.com/ACoder-AI/ACoder","tags":["Model: claude-4-sonnet","Model: claude-4.1-opus","Model: gpt-5-0807-global","Model: gemini-2.5-pro-06-17","Org: ACoder","System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/verified/20250819_ACoder/trajs","trajs_docent":false,"warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":null,"cost":177.98206539999998,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_gemini-3-flash-high","instance_calls":56.126,"instance_cost":0.35596413079999994,"logo":["img/logos/20260217_mini-v2.0.0_gemini-3-flash-high.svg","img/mini-icon.svg"],"logs":null,"mini-swe-agent_version":"2.0.0","model_display":"Gemini 3 Flash","model_org":"Google DeepMind","model_release_date":20251217,"name":"Gemini 3 Flash (high)","os_model":false,"os_system":true,"reasoning_effort":"high","resolved":75.8,"site":"https://ai.google.dev/gemini-api/docs/models","tags":["Model: gemini-3-flash-preview","Org: Google DeepMind","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_gemini-3-flash-high/trajs","trajs_docent":"https://docent.transluce.org/dashboard/1ebbdd7a-55b3-4015-9b83-5978cc7fb618","warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":null,"cost":36.644993922,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_minimax-2-5-high","instance_calls":60.45,"instance_cost":0.073289987844,"logo":["img/logos/20260217_mini-v2.0.0_minimax-2-5-high.png","img/mini-icon.svg"],"logs":null,"mini-swe-agent_version":"2.0.0","model_display":"MiniMax M2.5","model_org":"Minimax","model_release_date":20260212,"name":"MiniMax M2.5 (high)","os_model":true,"os_system":true,"reasoning_effort":"high","resolved":75.8,"site":"https://www.minimax.io","tags":["Model: minimax-m2.5","Org: Minimax","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_minimax-2-5-high/trajs","trajs_docent":"https://docent.transluce.org/dashboard/5b77e003-7328-4003-879e-9b55dd3a0b6f","warning":null},{"agent":"Warp","agent_org":"Warp","checked":false,"cost":null,"date":"2025-09-01","folder":"20250901_warp","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250901_warp.png"],"logs":"s3://swe-bench-submissions/verified/20250901_warp/logs","model_display":"Multiple","model_org":"OpenAI","model_release_date":null,"name":"Warp","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":75.6,"site":"https://www.warp.dev","tags":["Model: gpt-5","Model: claude-sonnet-4-20250514","Org: Warp","System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/verified/20250901_warp/trajs","trajs_docent":false,"warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":null,"cost":275.76132225,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_claude-4-6-opus","instance_calls":28.932,"instance_cost":0.5515226445,"logo":["img/logos/20260217_mini-v2.0.0_claude-4-6-opus.jpg","img/mini-icon.svg"],"logs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_claude-4-6-opus/logs","mini-swe-agent_version":"2.0.0","model_display":"Claude 4.6 Opus","model_org":"Anthropic","model_release_date":20260205,"name":"Claude 4.6 Opus","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":75.6,"site":"https://www.anthropic.com/claude","tags":["Model: claude-opus-4-6","Org: Anthropic","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_claude-4-6-opus/trajs","trajs_docent":"https://docent.transluce.org/dashboard/9243cc78-d399-402f-be97-e366ff63282c","warning":null},{"agent":"TRAE","agent_org":"TRAE","checked":false,"cost":null,"date":"2025-06-12","folder":"20250612_trae","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250612_trae.png"],"logs":"s3://swe-bench-submissions/verified/20250612_trae/logs","model_display":"Multiple","model_org":"Anthropic","model_release_date":null,"name":"TRAE + Claude Sonnet 4 + Opus 4 + Sonnet 3.7 + Gemini 2.5 Pro","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":75.2,"site":"https://github.com/bytedance/trae-agent","tags":["Model: claude-4-sonnet-20250522","Model: claude-4-opus-20250522","Model: claude-3-7-sonnet-20250219","Model: gemini-2.5-pro-preview-06-05","Org: TRAE","System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/verified/20250612_trae/trajs","trajs_docent":false,"warning":null},{"agent":"Harness AI","agent_org":"Harness","checked":false,"cost":null,"date":"2025-07-31","folder":"20250731_harness_ai","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250731_harness_ai.svg"],"logs":"s3://swe-bench-submissions/verified/20250731_harness_ai/logs","model_display":"Claude Sonnet 4","model_org":"Anthropic","model_release_date":null,"name":"Harness AI","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":74.8,"site":"https://www.harness.io/","tags":["Model: claude-sonnet-4-20250514","Org: Harness","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20250731_harness_ai/trajs","trajs_docent":false,"warning":null},{"agent":"Sonar Foundation Agent","agent_org":"Sonar","checked":false,"cost":null,"date":"2025-11-03","folder":"20251103_sonar-foundation-agent_claude-sonnet-4-5","instance_calls":null,"instance_cost":null,"logo":["img/logos/20251103_sonar-foundation-agent_claude-sonnet-4-5.png"],"logs":"s3://swe-bench-submissions/verified/20251103_sonar-foundation-agent_claude-sonnet-4-5/logs","model_display":"Claude 4.5 Sonnet","model_org":"Anthropic","model_release_date":null,"name":"Sonar Foundation Agent + Claude 4.5 Sonnet","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":74.8,"site":"https://www.sonarsource.com","tags":["Model: claude-sonnet-4-5","Org: Sonar","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20251103_sonar-foundation-agent_claude-sonnet-4-5/trajs","trajs_docent":false,"warning":null},{"agent":"Lingxi-v1.5","agent_org":"Lingxi","checked":false,"cost":null,"date":"2025-07-20","folder":"20250720_Lingxi-v1.5_claude-4-sonnet-20250514","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250720_Lingxi-v1.5_claude-4-sonnet-20250514.png"],"logs":"s3://swe-bench-submissions/verified/20250720_Lingxi-v1.5_claude-4-sonnet-20250514/logs","model_display":"Claude 4 Sonnet","model_org":"Anthropic","model_release_date":null,"name":"Lingxi-v1.5_claude-4-sonnet-20250514","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":74.6,"site":"https://github.com/nimasteryang/Lingxi","tags":["Model: claude-4-sonnet-20250514","Org: Lingxi","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20250720_Lingxi-v1.5_claude-4-sonnet-20250514/trajs","trajs_docent":false,"warning":null},{"agent":"JoyCode","agent_org":"JoyCode","checked":false,"cost":null,"date":"2025-09-15","folder":"20250915_JoyCode","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250915_JoyCode.png"],"logs":"s3://swe-bench-submissions/verified/20250915_JoyCode/logs","model_display":"Multiple","model_org":"Anthropic","model_release_date":null,"name":"JoyCode + Claude 4 Sonnet + GPT-4.1","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":74.6,"site":"https://github.com/jd-opensource/joycode-agent","tags":["Model: claude-4-sonnet","Model: gpt-4.1","Org: JoyCode","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20250915_JoyCode/trajs","trajs_docent":false,"warning":null},{"agent":"Refact.ai Agent","agent_org":"Refact.ai","checked":false,"cost":null,"date":"2025-06-03","folder":"20250603_Refact_Agent_claude-4-sonnet","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250603_Refact_Agent_claude-4-sonnet.png"],"logs":"s3://swe-bench-submissions/verified/20250603_Refact_Agent_claude-4-sonnet/logs","model_display":"Multiple","model_org":"Anthropic","model_release_date":null,"name":"Refact.ai Agent + Claude 4 Sonnet + o4-mini","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":74.4,"site":"https://refact.ai","tags":["Model: claude-4-sonnet","Model: o4-mini","Org: Refact.ai","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20250603_Refact_Agent_claude-4-sonnet/trajs","trajs_docent":false,"warning":null},{"agent":"Prometheus-v1.2.1","agent_org":"EuniAI","checked":false,"cost":null,"date":"2025-10-15","folder":"20251015_Prometheus_v1.2.1_gpt5","instance_calls":null,"instance_cost":null,"logo":["img/logos/20251015_Prometheus_v1.2.1_gpt5.jpg"],"logs":"s3://swe-bench-submissions/verified/20251015_Prometheus_v1.2.1_gpt5/logs","model_display":"GPT-5","model_org":"OpenAI","model_release_date":null,"name":"Prometheus-v1.2.1 + GPT-5","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":74.4,"site":"https://euni.ai/","tags":["Model: gpt-5-2025-08-07","Org: EuniAI","Org: Delysium","System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/verified/20251015_Prometheus_v1.2.1_gpt5/trajs","trajs_docent":false,"warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":true,"cost":360.6224985,"date":"2025-11-24","folder":"20251124_mini-v1.16.0_claude-opus-4-5-20251101","instance_calls":38.042,"instance_cost":0.721244997,"logo":["img/logos/20251124_mini-v1.16.0_claude-opus-4-5-20251101.jpg","img/mini-icon.svg"],"logs":"s3://swe-bench-submissions/bash-only/20251124_mini-v1.16.0_claude-opus-4-5-20251101/logs","mini-swe-agent_version":"1.16.0","model_display":"Claude 4.5 Opus","model_org":"Anthropic","model_release_date":20251124,"name":"Claude 4.5 Opus (20251101) (medium)","os_model":false,"os_system":true,"reasoning_effort":"medium","resolved":74.4,"site":"https://www.anthropic.com/news/claude-opus-4-5","tags":["Model: claude-opus-4-5-20251101","Org: Anthropic","System: Attempts - 1","Mini: 1.16.0"],"trajs":"s3://swe-bench-submissions/bash-only/20251124_mini-v1.16.0_claude-opus-4-5-20251101/trajs","trajs_docent":"https://docent.transluce.org/dashboard/6f6e2390-00c7-479f-9a2a-87b53b7477cb","warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":true,"cost":229.978759475,"date":"2025-11-18","folder":"20251118_mini-v1.15.0_gemini-3-pro-preview-20251118","instance_calls":40.326,"instance_cost":0.45995751895000003,"logo":["img/logos/20251118_mini-v1.15.0_gemini-3-pro-preview-20251118.svg","img/mini-icon.svg"],"logs":"s3://swe-bench-submissions/bash-only/20251118_mini-v1.15.0_gemini-3-pro-preview-20251118/logs","mini-swe-agent_version":"1.15.0","model_display":"Gemini 3 Pro Preview","model_org":"Google DeepMind","model_release_date":20251118,"name":"Gemini 3 Pro Preview (2025-11-18)","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":74.2,"site":"https://ai.google.dev/gemini-api/docs/models","tags":["Model: gemini-3-pro-preview","Org: Google DeepMind","System: Attempts - 1","Mini: 1.15.0"],"trajs":"s3://swe-bench-submissions/bash-only/20251118_mini-v1.15.0_gemini-3-pro-preview-20251118/trajs","trajs_docent":"https://docent.transluce.org/dashboard/3641b17f-034e-4b36-aa66-471dfed837d6","warning":null},{"agent":"Salesforce AI Research SAGE","agent_org":"Salesforce AI Research","checked":false,"cost":null,"date":"2025-11-03","folder":"20251103_SalesforceAIResearch_SAGE_OpenHands","instance_calls":null,"instance_cost":null,"logo":["img/logos/20251103_SalesforceAIResearch_SAGE_OpenHands.png"],"logs":"s3://swe-bench-submissions/verified/20251103_SalesforceAIResearch_SAGE_OpenHands/logs","model_display":"Multiple","model_org":"Anthropic","model_release_date":null,"name":"Salesforce AI Research SAGE (OpenHands)","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":73.8,"site":"https://www.salesforce.com/blog/sage-swe/","tags":["Model: claude-sonnet-4.5","Model: gpt-5","Org: Salesforce AI Research","System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/verified/20251103_SalesforceAIResearch_SAGE_OpenHands/trajs","trajs_docent":false,"warning":null},{"agent":"Tools","agent_org":null,"checked":false,"cost":null,"date":"2025-05-22","folder":"20250522_tools_claude-4-opus","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250522_tools_claude-4-opus.jpg"],"logs":"s3://swe-bench-submissions/verified/20250522_tools_claude-4-opus/logs","model_display":"Claude 4 Opus","model_org":"Anthropic","model_release_date":null,"name":"Tools + Claude 4 Opus (2025-05-22)","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":73.2,"site":"https://www.anthropic.com/","tags":["Model: claude-4-opus-20250514","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20250522_tools_claude-4-opus/trajs","trajs_docent":false,"warning":null},{"agent":"Salesforce AI Research SAGE","agent_org":"Salesforce AI Research","checked":false,"cost":null,"date":"2025-10-21","folder":"20251021_SalesforceAIResearch_SAGE_bash_only","instance_calls":null,"instance_cost":null,"logo":["img/logos/20251021_SalesforceAIResearch_SAGE_bash_only.png"],"logs":"s3://swe-bench-submissions/verified/20251021_SalesforceAIResearch_SAGE_bash_only/logs","model_display":"Multiple","model_org":"Anthropic","model_release_date":null,"name":"Salesforce AI Research SAGE (bash-only)","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":73.0,"site":"https://www.salesforce.com/blog/sage-swe/","tags":["Model: claude-sonnet-4.5","Model: gpt-5","Org: Salesforce AI Research","System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/verified/20251021_SalesforceAIResearch_SAGE_bash_only/trajs","trajs_docent":false,"warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":null,"cost":267.19456811,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_glm-5-high","instance_calls":76.176,"instance_cost":0.53438913622,"logo":["img/logos/20260217_mini-v2.0.0_glm-5-high.svg","img/mini-icon.svg"],"logs":null,"mini-swe-agent_version":"2.0.0","model_display":"GLM 5","model_org":"Z-AI","model_release_date":20260211,"name":"GLM 5 (high)","os_model":true,"os_system":true,"reasoning_effort":"high","resolved":72.8,"site":"https://z.ai","tags":["Model: glm-5","Org: Z-AI","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_glm-5-high/trajs","trajs_docent":"https://docent.transluce.org/dashboard/5fc22eb4-1f44-405e-9dff-40a61f9f6ef6","warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":null,"cost":236.78350885,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_gpt-5-2-high","instance_calls":35.046,"instance_cost":0.4735670177,"logo":["img/logos/20260217_mini-v2.0.0_gpt-5-2-high.svg","img/mini-icon.svg"],"logs":null,"mini-swe-agent_version":"2.0.0","model_display":"GPT 5.2","model_org":"OpenAI","model_release_date":20251211,"name":"GPT 5.2 (high)","os_model":false,"os_system":true,"reasoning_effort":"high","resolved":72.8,"site":"https://platform.openai.com","tags":["Model: gpt-5-2","Org: OpenAI","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_gpt-5-2-high/trajs","trajs_docent":"https://docent.transluce.org/dashboard/c2c7f5be-2c1a-472f-a34a-92e9e1df4f01","warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":null,"cost":224.71194935,"date":"2026-02-19","folder":"20260219_mini-v2.0.0_gpt-5-2-codex","instance_calls":28.072,"instance_cost":0.4494238987,"logo":["img/logos/20260219_mini-v2.0.0_gpt-5-2-codex.svg","img/mini-icon.svg"],"logs":"s3://swe-bench-submissions/bash-only/20260219_mini-v2.0.0_gpt-5-2-codex/logs","mini-swe-agent_version":"2.0.0","model_display":"GPT 5.2 Codex","model_org":"OpenAI","model_release_date":20251211,"name":"GPT 5.2 Codex","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":72.8,"site":"https://platform.openai.com/docs/models/gpt-5","tags":["Model: gpt-5-2-codex","Org: OpenAI","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260219_mini-v2.0.0_gpt-5-2-codex/trajs","trajs_docent":"https://docent.transluce.org/dashboard/fb22a2e4-0a41-4d41-8e1e-388d4cb50d80","warning":null},{"agent":"Tools","agent_org":null,"checked":false,"cost":null,"date":"2025-05-22","folder":"20250522_tools_claude-4-sonnet","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250522_tools_claude-4-sonnet.jpg"],"logs":"s3://swe-bench-submissions/verified/20250522_tools_claude-4-sonnet/logs","model_display":"Claude 4 Sonnet","model_org":"Anthropic","model_release_date":null,"name":"Tools + Claude 4 Sonnet (2025-05-22)","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":72.4,"site":"https://www.anthropic.com/","tags":["Model: claude-4-sonnet-20250514","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20250522_tools_claude-4-sonnet/trajs","trajs_docent":false,"warning":null},{"agent":"OpenHands","agent_org":"OpenHands","checked":true,"cost":null,"date":"2025-08-07","folder":"20250807_openhands_gpt5","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250807_openhands_gpt5.png"],"logs":"s3://swe-bench-submissions/verified/20250807_openhands_gpt5/logs","model_display":"GPT-5","model_org":"OpenAI","model_release_date":null,"name":"OpenHands + GPT-5","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":71.8,"site":"https://all-hands.dev/","tags":["Model: openai/gpt-5-2025-08-07","Org: OpenHands","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20250807_openhands_gpt5/trajs","trajs_docent":false,"warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":null,"cost":260.1295018,"date":"2025-12-11","folder":"20251211_mini-v1.17.2_gpt-5.2-2025-12-11-high","instance_calls":19.762,"instance_cost":0.5202590036,"logo":["img/logos/20251211_mini-v1.17.2_gpt-5.2-2025-12-11-high.svg","img/mini-icon.svg"],"logs":"s3://swe-bench-submissions/bash-only/20251211_mini-v1.17.2_gpt-5.2-2025-12-11-high/logs","mini-swe-agent_version":"1.17.2","model_display":"GPT 5.2","model_org":"OpenAI","model_release_date":20251211,"name":"GPT 5.2 (2025-12-11) (high)","os_model":false,"os_system":true,"reasoning_effort":"high","resolved":71.8,"site":"https://platform.openai.com/docs/models/gpt-5","tags":["Model: gpt-5.2-2025-12-11","Org: OpenAI","System: Attempts - 1","Mini: 1.17.2"],"trajs":"s3://swe-bench-submissions/bash-only/20251211_mini-v1.17.2_gpt-5.2-2025-12-11-high/trajs","trajs_docent":"https://docent.transluce.org/dashboard/93ef4179-a475-4f0b-88d1-46b5aa75e19b","warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":null,"cost":328.9482492,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_claude-4-5-sonnet-high","instance_calls":48.3,"instance_cost":0.6578964984000001,"logo":["img/logos/20260217_mini-v2.0.0_claude-4-5-sonnet-high.jpg","img/mini-icon.svg"],"logs":null,"mini-swe-agent_version":"2.0.0","model_display":"Claude 4.5 Sonnet","model_org":"Anthropic","model_release_date":20250929,"name":"Claude 4.5 Sonnet (high)","os_model":false,"os_system":true,"reasoning_effort":"high","resolved":71.4,"site":"https://www.anthropic.com/claude","tags":["Model: claude-sonnet-4-5-20250929","Org: Anthropic","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_claude-4-5-sonnet-high/trajs","trajs_docent":"https://docent.transluce.org/dashboard/737e5dd2-8555-435a-9fbd-1c6907c972f1","warning":null},{"agent":"Bloop","agent_org":"Bloop AI","checked":false,"cost":null,"date":"2025-07-10","folder":"20250710_bloop","instance_calls":null,"instance_cost":null,"logo":null,"logs":"s3://swe-bench-submissions/verified/20250710_bloop/logs","model_display":"Claude 4 Sonnet","model_org":"Anthropic","model_release_date":null,"name":"Bloop","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":71.2,"site":"https://bloop.ai/","tags":["Model: claude-4-sonnet-20250514","Org: Bloop AI","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20250710_bloop/trajs","trajs_docent":false,"warning":null},{"agent":"Qodo Command","agent_org":"Qodo","checked":false,"cost":null,"date":"2025-07-15","folder":"20250715_qodo_command","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250715_qodo_command.png"],"logs":"s3://swe-bench-submissions/verified/20250715_qodo_command/logs","model_display":"Multiple","model_org":"Anthropic","model_release_date":null,"name":"Qodo Command","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":71.2,"site":"https://www.qodo.ai/","tags":["Model: claude-sonnet-4-20250514","Model: gpt-4o","Org: Qodo","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20250715_qodo_command/trajs","trajs_docent":false,"warning":null},{"agent":"Prometheus-v1.2","agent_org":"EuniAI","checked":false,"cost":null,"date":"2025-09-29","folder":"20250929_Prometheus_v1.2_gpt5","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250929_Prometheus_v1.2_gpt5.jpg"],"logs":"s3://swe-bench-submissions/verified/20250929_Prometheus_v1.2_gpt5/logs","model_display":"GPT-5","model_org":"OpenAI","model_release_date":null,"name":"Prometheus-v1.2 + GPT-5","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":71.2,"site":"https://euni.ai/","tags":["Model: gpt-5-2025-08-07","Org: EuniAI","Org: Delysium","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20250929_Prometheus_v1.2_gpt5/trajs","trajs_docent":false,"warning":null},{"agent":"Lingxi v1.5","agent_org":"Huawei","checked":false,"cost":null,"date":"2025-10-14","folder":"20251014_Lingxi_kimi_k2","instance_calls":null,"instance_cost":null,"logo":["img/logos/20251014_Lingxi_kimi_k2.png"],"logs":"s3://swe-bench-submissions/verified/20251014_Lingxi_kimi_k2/logs","model_display":"Kimi K2","model_org":"Moonshot AI","model_release_date":null,"name":"Lingxi v1.5 x Kimi K2","os_model":true,"os_system":true,"reasoning_effort":null,"resolved":71.2,"site":"https://github.com/lingxi-agent/Lingxi/tree/master","tags":["Model: kimi-k2-0905-preview","Org: Huawei","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20251014_Lingxi_kimi_k2/trajs","trajs_docent":false,"warning":null},{"agent":"Warp","agent_org":"Warp","checked":false,"cost":null,"date":"2025-06-23","folder":"20250623_warp","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250623_warp.png"],"logs":"s3://swe-bench-submissions/verified/20250623_warp/logs","model_display":"Multiple","model_org":"Anthropic","model_release_date":null,"name":"Warp","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":71.0,"site":"https://www.warp.dev","tags":["Model: claude-sonnet-4-20250514","Model: claude-3-7-sonnet-20250219","Model: gemini-2.5-pro","Model: gpt-4.1-2025-04-14","Org: Warp","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20250623_warp/trajs","trajs_docent":false,"warning":null},{"agent":"Moatless Tools","agent_org":"Moatless AI","checked":true,"cost":null,"date":"2025-06-11","folder":"20250611_moatless_claude-4-sonnet-20250514","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250611_moatless_claude-4-sonnet-20250514.png"],"logs":"s3://swe-bench-submissions/verified/20250611_moatless_claude-4-sonnet-20250514/logs","model_display":"Claude 4 Sonnet","model_org":"Anthropic","model_release_date":null,"name":"Moatless Tools + Claude 4 Sonnet","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":70.8,"site":"https://github.com/aorwall/moatless-tools","tags":["Model: claude-4-sonnet-20250514","Org: Moatless AI","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20250611_moatless_claude-4-sonnet-20250514/trajs","trajs_docent":false,"warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":null,"cost":73.276870372,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_kimi-k2-5-high","instance_calls":51.178,"instance_cost":0.14655374074400002,"logo":["img/logos/20260217_mini-v2.0.0_kimi-k2-5-high.jpg","img/mini-icon.svg"],"logs":null,"mini-swe-agent_version":"2.0.0","model_display":"Kimi K2.5","model_org":"Moonshot AI","model_release_date":20260127,"name":"Kimi K2.5 (high)","os_model":true,"os_system":true,"reasoning_effort":"high","resolved":70.8,"site":"https://moonshotai.github.io/Kimi-K2/","tags":["Model: kimi-k2.5","Org: Moonshot AI","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_kimi-k2-5-high/trajs","trajs_docent":"https://docent.transluce.org/dashboard/c4f6e550-6e49-436e-96fa-3aafd9a0c936","warning":null},{"agent":"TRAE","agent_org":null,"checked":false,"cost":null,"date":"2025-05-19","folder":"20250519_trae","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250519_trae.png"],"logs":"s3://swe-bench-submissions/verified/20250519_trae/logs","model_display":"Undisclosed","model_org":null,"model_release_date":null,"name":"TRAE","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":70.6,"site":"https://www.trae.ai/","tags":["System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/verified/20250519_trae/trajs","trajs_docent":false,"warning":null},{"agent":"mini-SWE-agent","agent_org":"SWE-agent","checked":true,"cost":279.16737045,"date":"2025-09-29","folder":"20250929_mini-v1.13.3_sonnet-4-5-20250929","instance_calls":50.988,"instance_cost":0.5583347409,"logo":["img/logos/20250929_mini-v1.13.3_sonnet-4-5-20250929.jpg","img/mini-icon.svg"],"logs":"s3://swe-bench-submissions/bash-only/20250929_mini-v1.13.3_sonnet-4-5-20250929/logs","mini-swe-agent_version":"1.13.3","model_display":"Claude 4.5 Sonnet","model_org":"Anthropic","model_release_date":20250929,"name":"Claude 4.5 Sonnet (20250929)","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":70.6,"site":"https://www.anthropic.com/news/claude-sonnet-4-5","tags":["Model: claude-sonnet-4-5-20250929","Org: Anthropic","System: Attempts - 1","Mini: 1.13.3"],"trajs":"s3://swe-bench-submissions/bash-only/20250929_mini-v1.13.3_sonnet-4-5-20250929/trajs","trajs_docent":"https://docent.transluce.org/dashboard/a4844da1-fbb9-4d61-b82c-f46e471f748a","warning":null}]},{"name":"Lite","results":[{"agent":"ExpeRepair-v1.0","agent_org":null,"checked":false,"cost":null,"date":"2025-06-25","folder":"20250625_ExpeRepair-v1_claude-4-sonnet-20250514","instance_calls":null,"instance_cost":null,"logo":null,"logs":"s3://swe-bench-submissions/lite/20250625_ExpeRepair-v1_claude-4-sonnet-20250514/logs","model_display":"Claude 4 Sonnet","model_org":"Anthropic","model_release_date":null,"name":"ExpeRepair-v1.0 + Claude 4 Sonnet","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":60.33,"site":"https://github.com/ExpeRepair/ExpeRepair","tags":["Model: claude-4-sonnet-20250514","Model: o3-mini","Model: o4-mini","System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/lite/20250625_ExpeRepair-v1_claude-4-sonnet-20250514/trajs","trajs_docent":false,"warning":null},{"agent":"Refact.ai Agent","agent_org":"Refact.ai","checked":false,"cost":null,"date":"2025-04-25","folder":"20250425_Refact_Agent","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250425_Refact_Agent.png"],"logs":"s3://swe-bench-submissions/lite/20250425_Refact_Agent/logs","model_display":"Multiple","model_org":"Anthropic","model_release_date":null,"name":"Refact.ai Agent","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":60.0,"site":"https://refact.ai","tags":["Model: claude-3-7-sonnet-20250219","Model: o4-mini","Org: Refact.ai","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/lite/20250425_Refact_Agent/trajs","trajs_docent":false,"warning":null},{"agent":"KGCompass","agent_org":"GLEAM-Lab","checked":false,"cost":null,"date":"2025-09-06","folder":"20250906_KGCompass_claude-4-sonnet-20250514","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250906_KGCompass_claude-4-sonnet-20250514.png"],"logs":"s3://swe-bench-submissions/lite/20250906_KGCompass_claude-4-sonnet-20250514/logs","model_display":"Claude 4 Sonnet","model_org":"Anthropic","model_release_date":null,"name":"KGCompass + Claude 4 Sonnet (20250514)","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":58.33,"site":"https://github.com/GLEAM-Lab/KGCompass","tags":["Model: claude-4-sonnet-20250514","Org: GLEAM-Lab","System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/lite/20250906_KGCompass_claude-4-sonnet-20250514/trajs","trajs_docent":false,"warning":null},{"agent":"SWE-agent","agent_org":"SWE-agent","checked":true,"cost":null,"date":"2025-05-26","folder":"20250526_sweagent_claude-4-sonnet-20250514","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250526_sweagent_claude-4-sonnet-20250514.png"],"logs":"s3://swe-bench-submissions/lite/20250522_sweagent_claude-4-sonnet-20250514/trajs","model_display":"Claude 4 Sonnet","model_org":"Anthropic","model_release_date":null,"name":"SWE-agent + Claude 4 Sonnet","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":56.67,"site":"https://github.com/SWE-agent/SWE-agent","tags":["Model: claude-4-sonnet-20250514","Org: SWE-agent","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/lite/20250522_sweagent_claude-4-sonnet-20250514/logs","trajs_docent":false,"warning":null},{"agent":"Isoform","agent_org":null,"checked":false,"cost":null,"date":"2025-01-14","folder":"20250114_Isoform","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250114_Isoform.png"],"logs":"s3://swe-bench-submissions/lite/20250114_Isoform/logs","model_display":"Undisclosed","model_org":null,"model_release_date":null,"name":"Isoform","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":55.0,"site":"https://www.isoform.ai","tags":["System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/lite/20250114_Isoform/trajs","trajs_docent":false,"warning":null},{"agent":"SemAgent_Multi-v1.0","agent_org":"Columbia","checked":false,"cost":null,"date":"2025-06-25","folder":"20250625_SemAgent_Multi-v1_Claude3.7Sonnet_Gemini2.5Pro","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250625_SemAgent_Multi-v1_Claude3.7Sonnet_Gemini2.5Pro.jpg"],"logs":"s3://swe-bench-submissions/lite/20250625_SemAgent_Multi-v1_Claude3.7Sonnet_Gemini2.5Pro/logs","model_display":"Multiple","model_org":"Anthropic","model_release_date":null,"name":"SemAgent_Multi-v1.0","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":51.67,"site":"https://arxiv.org/abs/2506.16650","tags":["Model: claude-3-7-sonnet-20250219","Model: gemini-2.5-pro-preview-05-06","Org: Columbia","Org: SemAgent","System: Attempts - 2+"],"trajs":"https://github.com/AnvithPabba/SemAgent_Multi/tree/main/trajs/SWEBench_Lite/20250625_SemAgent_Multi-v1.0_Claude3.7Sonnet_Gemini2.5Pro","trajs_docent":false,"warning":null}]},{"name":"Multimodal","results":[{"agent":"GUIRepair","agent_org":"GUIRepair","checked":true,"cost":null,"date":"2025-07-01","folder":"20250701_GUIRepair_o3","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250701_GUIRepair_o3.png"],"logs":null,"model_display":"o3","model_org":"OpenAI","model_release_date":null,"name":"GUIRepair + o3 (2025-04-16)","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":35.98,"site":"https://sites.google.com/view/guirepair","tags":["Model: o3-2025-04-16","Org: GUIRepair","System: Attempts - 1"],"trajs":null,"trajs_docent":false,"warning":null},{"agent":"Codefuse_Pycfuse_SVR","agent_org":"Codefuse","checked":true,"cost":null,"date":"2025-11-17","folder":"20251117_codefuse_pycfuse_svr","instance_calls":null,"instance_cost":null,"logo":["img/logos/20251117_codefuse_pycfuse_svr.png"],"logs":null,"model_display":"o3","model_org":"OpenAI","model_release_date":null,"name":"Codefuse_Pycfuse_SVR","os_model":false,"os_system":false,"reasoning_effort":null,"resolved":35.98,"site":"https://github.com/codefuse-ai/codefuse-svr?tab=readme-ov-file","tags":["Model: o3-2025-04-16","Org: Codefuse","System: Attempts - 1"],"trajs":null,"trajs_docent":false,"warning":null},{"agent":"Refact.ai Agent","agent_org":"Refact.ai","checked":false,"cost":null,"date":"2025-06-11","folder":"20250611_Refact_Agent_claude-4-sonnet","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250611_Refact_Agent_claude-4-sonnet.png"],"logs":false,"model_display":"Claude 4 Sonnet","model_org":"Anthropic","model_release_date":null,"name":"Refact.ai Agent","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":35.59,"site":"https://refact.ai","tags":["Model: claude-4-sonnet","Org: Refact.ai","System: Attempts - 1"],"trajs":false,"trajs_docent":false,"warning":null},{"agent":"OpenHands-Versa","agent_org":"OpenHands","checked":true,"cost":null,"date":"2025-05-28","folder":"20250528_OpenHands-Versa-claude4","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250528_OpenHands-Versa-claude4.png"],"logs":false,"model_display":"Claude Sonnet 4","model_org":"Anthropic","model_release_date":null,"name":"OpenHands-Versa (Claude-Sonnet 4)","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":34.43,"site":"https://github.com/adityasoni9998/OpenHands-Versa","tags":["Model: claude-4-sonnet-20250514","Org: OpenHands","System: Attempts - 1"],"trajs":false,"trajs_docent":false,"warning":null},{"agent":"GUIRepair","agent_org":"GUIRepair","checked":true,"cost":null,"date":"2025-05-31","folder":"20250531_GUIRepair_o4mini","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250531_GUIRepair_o4mini.png"],"logs":null,"model_display":"o4-mini","model_org":"OpenAI","model_release_date":null,"name":"GUIRepair + o4-mini (2025-04-16)","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":33.85,"site":"https://sites.google.com/view/guirepair","tags":["Model: o4-mini-2025-04-16","Org: GUIRepair","System: Attempts - 1"],"trajs":null,"trajs_docent":false,"warning":null},{"agent":"OpenHands-Versa","agent_org":"OpenHands","checked":true,"cost":null,"date":"2025-05-09","folder":"20250509_OpenHands-Versa_claude3.7","instance_calls":null,"instance_cost":null,"logo":["img/logos/20250509_OpenHands-Versa_claude3.7.png"],"logs":false,"model_display":"Claude 3.7 Sonnet","model_org":"Anthropic","model_release_date":null,"name":"OpenHands-Versa (Claude-3.7 Sonnet)","os_model":false,"os_system":true,"reasoning_effort":null,"resolved":31.33,"site":"https://github.com/adityasoni9998/OpenHands-Versa","tags":["Model: claude-3-7-sonnet-20250219","Org: OpenHands","System: Attempts - 1"],"trajs":false,"trajs_docent":false,"warning":null}]}]</script>165166<!-- Chart.js and analysis for comparison -->167<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>168<script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns"></script>169<script src="js/modelReleaseDates.js"></script>170<script src="js/multilingualLanguageMap.js"></script>171<script src="js/charts/barChart.js"></script>172<script src="js/charts/scatterChart.js"></script>173<script src="js/charts/cumulativeCostChart.js"></script>174<script src="js/charts/groupedBarChart.js"></script>175<script src="js/charts/groupedBarChartByLanguage.js"></script>176<script src="js/charts/resolvedVsLimitChart.js"></script>177<script src="js/charts/resolvedVsAvgCostChart.js"></script>178<script src="js/charts/resolvedInstancesMatrix.js"></script>179<script src="js/charts/resolvedVsReleaseDateChart.js"></script>180<script src="js/analysis.js"></script>181182<!-- Compare Results Modal -->183<div id="compare-modal" class="modal" aria-hidden="true">184 <div class="modal-backdrop" data-close="true"></div>185 <div class="modal-dialog modal-dialog-large" role="dialog" aria-modal="true" aria-labelledby="compare-modal-title">186 <div class="modal-header">187 <h3 id="compare-modal-title" class="m-0">Compare results</h3>188 <button class="modal-close" aria-label="Close" data-close="true"><i class="fa fa-times"></i></button>189 </div>190 <div class="modal-body">191 <div class="d-flex align-items-center gap-2 mb-2 flex-wrap">192 <select id="compare-chart-type" class="form-control form-select" style="flex: 1 1 200px;">193 <option value="bar" selected>Resolved (bar chart)</option>194 <option value="grouped-bar">Resolved by repository</option>195 <option value="grouped-bar-language">Resolved by language</option>196 <option value="resolved-instances-matrix">Resolved instances matrix</option>197 <option value="scatter">Resolved vs cost (scatter plot)</option>198 <option value="resolved-vs-release-date">Resolved vs model release date</option>199 <option value="resolved-vs-avg-cost">Resolved vs average cost</option>200 <option value="resolved-vs-cost-limit">Resolved vs cost limit</option>201 <option value="resolved-vs-step-limit">Resolved vs step limit</option>202 <option value="cumulative-cost">Cumulative cost distribution</option>203 <option value="cumulative-cost-resolved">Cumulative cost distribution (resolved only)</option>204 <option value="cumulative-steps">Cumulative step distribution</option>205 <option value="cumulative-steps-resolved">Cumulative step distribution (resolved only)</option>206 </select>207 <button id="chart-theme-toggle" class="btn btn-sm btn-outline-secondary" type="button" title="Switch theme">208 <i class="fa fa-sun"></i> Light209 </button>210 <button id="quickselect-btn" class="btn btn-sm btn-outline-secondary" type="button" title="Quick select models">211 <i class="fa fa-check-square"></i> Selection212 </button>213 <button id="download-json-btn" class="btn btn-sm btn-outline-secondary" type="button" title="Download data as JSON">214 <i class="fa fa-download"></i> JSON215 </button>216 <button id="download-png-btn" class="btn btn-sm btn-outline-secondary" type="button" title="Download chart as PNG">217 <i class="fa fa-download"></i> PNG218 </button>219 <button id="copy-link-btn" class="btn btn-sm btn-outline-secondary" type="button" title="Copy shareable link">220 <i class="fa fa-link"></i> Copy Link221 </button>222 </div>223 <div class="d-flex align-items-center gap-2 mb-2 flex-wrap">224 <select id="matrix-chunk-selector" class="form-control form-select" style="flex: 1 1 200px; display: none;">225 <option value="0">Instances 1-100</option>226 <option value="100">Instances 101-200</option>227 <option value="200">Instances 201-300</option>228 <option value="300">Instances 301-400</option>229 <option value="400">Instances 401-500</option>230 </select>231 </div>232 <div class="mt-2 chart-container">233 <canvas id="compare-chart"></canvas>234 <div id="compare-empty" class="text-muted mt-2">Select models via the checkboxes, then click <em>Compare results</em>.</div>235 </div>236 </div>237 </div>238 239 240</div>241242<!-- No Selection Modal -->243<div id="no-selection-modal" class="modal" aria-hidden="true">244 <div class="modal-backdrop" data-close="true"></div>245 <div class="modal-dialog modal-dialog-small" role="dialog" aria-modal="true" aria-labelledby="no-selection-modal-title">246 <div class="modal-header">247 <h3 id="no-selection-modal-title" class="m-0">No models selected</h3>248 <button class="modal-close" aria-label="Close" data-close="true"><i class="fa fa-times"></i></button>249 </div>250 <div class="modal-body">251 <p class="mb-3">Select at least one model using the checkboxes in the first columns, or click one of the following buttons for a pre-defined selection.</p>252 <p class="mb-2 fw-medium">Quick select:</p>253 <div class="d-flex gap-2 flex-wrap">254 <button id="select-top-10" class="button button-primary" type="button">Select top 10</button>255 <button id="select-top-20" class="button button-secondary" type="button">Select top 20</button>256 <button id="select-all" class="button button-secondary" type="button">Select all</button>257 <button id="select-all-ow" class="button button-secondary" type="button">Select all (open weights)</button>258 </div>259 </div>260 </div>261</div>262263<div class="leaderboard-notes">264 <p class="benchmark-cards-note">Each entry reports <b>% Resolved</b>, the percentage of task instances solved.</p>265 <ul class="leaderboard-legend">266 <li>267 <span class="model-tag model-tag-open"><i class="fa-solid fa-lock-open"></i></span>268 Open-weights model269 </li>270 <li>271 <span class="model-tag model-tag-verified"><i class="fa-solid fa-circle-check"></i></span>272 Run performed or directly checked by the SWE-bench team273 </li>274 </ul>275 <div class="benchmark-cards">276 <div class="benchmark-card">277 <div class="benchmark-card-header">278 <h4>SWE-bench</h4>279 <span class="benchmark-card-count">2294 instances</span>280 </div>281 <p>The original benchmark: real GitHub issues from 12 Python repositories.</p>282 <a class="benchmark-card-link" href="original.html">Read the post <i class="fa-solid fa-arrow-right"></i></a>283 </div>284 <div class="benchmark-card">285 <div class="benchmark-card-header">286 <h4>Bash Only</h4>287 <span class="benchmark-card-count">500 instances</span>288 </div>289 <p>The default Verified view: every model in the same <a href="https://github.com/SWE-agent/mini-swe-agent">mini-SWE-agent</a> environment.</p>290 <a class="benchmark-card-link" href="verified.html">Read the post <i class="fa-solid fa-arrow-right"></i></a>291 </div>292 <div class="benchmark-card">293 <div class="benchmark-card-header">294 <h4>Lite</h4>295 <span class="benchmark-card-count">300 instances</span>296 </div>297 <p>A subset curated for less costly evaluation.</p>298 <a class="benchmark-card-link" href="lite.html">Read the post <i class="fa-solid fa-arrow-right"></i></a>299 </div>300 <div class="benchmark-card">301 <div class="benchmark-card-header">302 <h4>Verified</h4>303 <span class="benchmark-card-count">500 instances</span>304 </div>305 <p>A human-filtered subset of SWE-bench.</p>306 <a class="benchmark-card-link" href="https://openai.com/index/introducing-swe-bench-verified/">Read the post <i class="fa-solid fa-arrow-right"></i></a>307 </div>308 <div class="benchmark-card">309 <div class="benchmark-card-header">310 <h4>Multilingual</h4>311 <span class="benchmark-card-count">300 instances</span>312 </div>313 <p>Tasks from 42 repositories across 9 programming languages.</p>314 <a class="benchmark-card-link" href="multilingual.html">Read the post <i class="fa-solid fa-arrow-right"></i></a>315 </div>316 <div class="benchmark-card">317 <div class="benchmark-card-header">318 <h4>Multimodal</h4>319 <span class="benchmark-card-count">480 instances</span>320 </div>321 <p>Issues described with visual elements.</p>322 <a class="benchmark-card-link" href="multimodal.html">Read the post <i class="fa-solid fa-arrow-right"></i></a>323 </div>324 </div>325</div>326327 <a href="viewer.html" class="view-details-button">Analyze results in detail</a>328 </div>329 </section>330331 <section class="container">332 <div class="content-section" id="news">333 <h2 class="content-section-title">News</h2>334 <ol class="news-timeline">335 <li class="news-item">336 <span class="news-marker"><img src="img/programbench-icon.png" alt="ProgramBench icon"></span>337 <span class="news-date">May 2026</span>338 <span class="news-text">We released <b>ProgramBench</b> to benchmark whether models can code meaningful software artifacts from scratch.339 <a class="news-link" href="https://programbench.com" target="_blank" rel="noopener noreferrer">Link <i class="fa-solid fa-arrow-right"></i></a>340 </span>341 </li>342 <li class="news-item">343 <span class="news-marker"><img src="img/codeclash.svg" alt="CodeClash icon"></span>344 <span class="news-date">Nov 2025</span>345 <span class="news-text">Introducing <b>CodeClash</b>, our new eval of LMs as goal (not task) oriented developers.346 <a class="news-link" href="https://codeclash.ai" target="_blank" rel="noopener noreferrer">Link <i class="fa-solid fa-arrow-right"></i></a>347 </span>348 </li>349 <li class="news-item">350 <span class="news-marker"><img src="img/mini-icon.svg" alt="mini-SWE-agent icon"></span>351 <span class="news-date">Jul 2025</span>352 <span class="news-text"><b>mini-SWE-agent</b> scores 65% on SWE-bench Verified in 100 lines of Python.353 <a class="news-link" href="https://github.com/SWE-agent/mini-swe-agent" target="_blank" rel="noopener noreferrer">Link <i class="fa-solid fa-arrow-right"></i></a>354 </span>355 </li>356 <li class="news-item">357 <span class="news-marker"><img src="img/swe-smith-icon.svg" alt="SWE-smith icon"></span>358 <span class="news-date">May 2025</span>359 <span class="news-text"><b>SWE-smith</b> is out! Train your own models for software engineering agents.360 <a class="news-link" href="https://swesmith.com" target="_blank" rel="noopener noreferrer">Link <i class="fa-solid fa-arrow-right"></i></a>361 </span>362 </li>363 <li class="news-item">364 <span class="news-marker"><img src="img/swe-agent-icon.svg" alt="SWE-agent icon"></span>365 <span class="news-date">Mar 2025</span>366 <span class="news-text"><b>SWE-agent 1.0</b> is the open source SOTA on SWE-bench Lite.367 <a class="news-link" href="https://github.com/SWE-agent/SWE-agent" target="_blank" rel="noopener noreferrer">Link <i class="fa-solid fa-arrow-right"></i></a>368 </span>369 </li>370 <li class="news-item">371 <span class="news-marker"><img src="img/swe-llama-mm.svg" alt="SWE-bench Multimodal icon"></span>372 <span class="news-date">Oct 2024</span>373 <span class="news-text">Introducing <b>SWE-bench Multimodal</b>.374 <a class="news-link" href="multimodal.html">Link <i class="fa-solid fa-arrow-right"></i></a>375 </span>376 </li>377 <li class="news-item">378 <span class="news-marker"><img src="img/openai-icon.svg" alt="OpenAI icon"></span>379 <span class="news-date">Aug 2024</span>380 <span class="news-text">SWE-bench x OpenAI = <b>SWE-bench Verified</b>.381 <a class="news-link" href="https://openai.com/index/introducing-swe-bench-verified/" target="_blank" rel="noopener noreferrer">Report <i class="fa-solid fa-arrow-right"></i></a>382 </span>383 </li>384 <li class="news-item">385 <span class="news-marker"><img src="img/swe-llama.svg" alt="SWE-bench icon"></span>386 <span class="news-date">Jun 2024</span>387 <span class="news-text"><b>Docker</b>-ized SWE-bench for easier evaluation.388 <a class="news-link" href="https://www.swebench.com/SWE-bench/guides/docker_setup/" target="_blank" rel="noopener noreferrer">Report <i class="fa-solid fa-arrow-right"></i></a>389 </span>390 </li>391 <li class="news-item">392 <span class="news-marker"><img src="img/swe-agent-icon.svg" alt="SWE-agent icon"></span>393 <span class="news-date">Mar 2024</span>394 <span class="news-text">Check out <b>SWE-agent</b> (12.47% on SWE-bench).395 <a class="news-link" href="https://github.com/SWE-agent/SWE-agent" target="_blank" rel="noopener noreferrer">Link <i class="fa-solid fa-arrow-right"></i></a>396 </span>397 </li>398 <li class="news-item">399 <span class="news-marker"><img src="img/swe-llama.svg" alt="SWE-bench icon"></span>400 <span class="news-date">Mar 2024</span>401 <span class="news-text">Released <b>SWE-bench Lite</b>.402 <a class="news-link" href="lite.html">Report <i class="fa-solid fa-arrow-right"></i></a>403 </span>404 </li>405 </ol>406 </div>407 </section>408409 <section class="container">410 <div class="content-section" id="about">411 <h2 class="content-section-title">Acknowledgements</h2>412 <p>413 We thank the following institutions for their generous support:414 Open Philanthropy, AWS, Modal, Andreessen Horowitz, OpenAI, and Anthropic.415 </p>416 </div>417 </section>418419 420 <footer class="footer">421 <div class="container">422 <div class="footer-content">423 <div class="footer-copyright">424 © 2026 SWE-bench Team. All rights reserved.425 </div>426 <div class="footer-links">427 <a href="https://github.com/swe-bench/SWE-bench" target="_blank">GitHub</a>428 <a href="https://huggingface.co/collections/SWE-bench/benchmarks-68113bc99eb3a64a91ea33c9" target="_blank">HuggingFace</a>429 <a href="https://arxiv.org/abs/2310.06770" target="_blank">Paper</a>430 </div>431 </div>432 </div>433 </footer>434 </div>435436 <!-- Common JS files -->437 <script src="js/mainResults.js"></script>438 <script src="js/citation.js"></script>439 <script src="js/citationFormat.js"></script>440 441 <!-- Page-specific JS files will be injected here -->442 443444 <script>445 function initTheme() {446 const themeToggle = document.getElementById('theme-toggle');447 const prefersDarkScheme = window.matchMedia('(prefers-color-scheme: dark)');448 449 // Move dark-mode class from documentElement to body if present450 if (document.documentElement.classList.contains('dark-mode')) {451 document.body.classList.add('dark-mode');452 document.documentElement.classList.remove('dark-mode');453 }454 455 if (themeToggle) {456 themeToggle.addEventListener('click', () => {457 document.body.classList.toggle('dark-mode');458 459 const isDarkMode = document.body.classList.contains('dark-mode');460 localStorage.setItem('theme', isDarkMode ? 'dark' : 'light');461 });462 }463 464 prefersDarkScheme.addEventListener('change', (e) => {465 if (!localStorage.getItem('theme')) {466 if (e.matches) {467 document.body.classList.add('dark-mode');468 } else {469 document.body.classList.remove('dark-mode');470 }471 }472 });473 }474 475 document.addEventListener('DOMContentLoaded', function() {476 initTheme();477 478 const currentPath = window.location.pathname;479 const currentPage = currentPath.split('/').pop().split('.')[0] || 'index';480 481 const navLinks = document.querySelectorAll('.nav-link');482 navLinks.forEach(link => {483 const linkPage = link.getAttribute('data-page');484 if (linkPage === currentPage) {485 link.classList.add('active');486 }487 488 if (currentPage === 'index' && window.location.hash) {489 const currentHash = window.location.hash.substring(1);490 if (linkPage === currentHash) {491 link.classList.add('active');492 }493 }494 });495 496 const sidebarOpener = document.querySelector('.sidebar-opener');497 const sidebarCloser = document.querySelector('.sidebar .mobile-nav-toggle');498 const sidebar = document.querySelector('.sidebar');499 const overlay = document.querySelector('.sidebar-overlay');500 501 if ((sidebarOpener || sidebarCloser) && sidebar && overlay) {502 function toggleMenu() {503 sidebar.classList.toggle('open');504 overlay.classList.toggle('active');505 }506 507 function openMenu() {508 sidebar.classList.add('open');509 overlay.classList.add('active');510 }511 512 function closeMenu() {513 sidebar.classList.remove('open');514 overlay.classList.remove('active');515 }516 517 if (sidebarOpener) {518 sidebarOpener.addEventListener('click', function(e) {519 e.preventDefault();520 openMenu();521 });522 }523 524 if (sidebarCloser) {525 sidebarCloser.addEventListener('click', function(e) {526 e.preventDefault();527 closeMenu();528 });529 }530 531 overlay.addEventListener('click', function(e) {532 e.preventDefault();533 closeMenu();534 });535 536 navLinks.forEach(link => {537 link.addEventListener('click', () => {538 if (sidebar.classList.contains('open')) {539 closeMenu();540 }541 });542 });543 544 document.addEventListener('keydown', function(e) {545 if (e.key === 'Escape' && sidebar.classList.contains('open')) {546 closeMenu();547 }548 });549 }550 });551 </script>552553 554555</body>556</html>