HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1- dirname: 2025-02-25-20-23-07--gemini-pro2 test_cases: 2253 model: Gemini 2.0 Pro exp-02-054 edit_format: whole5 commit_hash: 2fccd476 pass_rate_1: 20.47 pass_rate_2: 35.68 pass_num_1: 469 pass_num_2: 8010 percent_cases_well_formed: 100.011 error_outputs: 43012 num_malformed_responses: 013 num_with_malformed_responses: 014 user_asks: 1315 lazy_comments: 016 syntax_errors: 017 indentation_errors: 018 exhausted_context_windows: 019 test_timeouts: 520 total_tests: 22521 command: aider --model gemini/gemini-2.0-pro-exp-02-0522 date: 2025-02-2523 versions: 0.75.2.dev24 seconds_per_case: 34.825 total_cost: 0.00002627- dirname: 2024-12-21-18-41-18--polyglot-gpt-4o-mini28 test_cases: 22529 model: gpt-4o-mini-2024-07-1830 edit_format: whole31 commit_hash: a755079-dirty32 pass_rate_1: 0.933 pass_rate_2: 3.634 pass_num_1: 235 pass_num_2: 836 percent_cases_well_formed: 100.037 error_outputs: 038 num_malformed_responses: 039 num_with_malformed_responses: 040 user_asks: 3641 lazy_comments: 042 syntax_errors: 043 indentation_errors: 044 exhausted_context_windows: 045 test_timeouts: 346 total_tests: 22547 command: aider --model gpt-4o-mini-2024-07-1848 date: 2024-12-2149 versions: 0.69.2.dev50 seconds_per_case: 17.351 total_cost: 0.32365253- dirname: 2025-01-17-19-44-33--sonnet-baseline-jan-1754 test_cases: 22555 model: claude-3-5-sonnet-2024102256 edit_format: diff57 commit_hash: 6451d5958 pass_rate_1: 22.259 pass_rate_2: 51.660 pass_num_1: 5061 pass_num_2: 11662 percent_cases_well_formed: 99.663 error_outputs: 264 num_malformed_responses: 165 num_with_malformed_responses: 166 user_asks: 1167 lazy_comments: 068 syntax_errors: 069 indentation_errors: 070 exhausted_context_windows: 171 test_timeouts: 872 total_tests: 22573 command: aider --model claude-3-5-sonnet-2024102274 date: 2025-01-1775 versions: 0.71.2.dev76 seconds_per_case: 21.477 total_cost: 14.406378 79- dirname: 2024-12-30-20-57-12--gpt-4o-2024-11-20-ex-as-sys80 test_cases: 22581 model: gpt-4o-2024-11-2082 edit_format: diff83 commit_hash: 09ee197-dirty84 pass_rate_1: 4.985 pass_rate_2: 18.286 pass_num_1: 1187 pass_num_2: 4188 percent_cases_well_formed: 95.189 error_outputs: 1290 num_malformed_responses: 1291 num_with_malformed_responses: 1192 user_asks: 5393 lazy_comments: 094 syntax_errors: 095 indentation_errors: 096 exhausted_context_windows: 097 test_timeouts: 1298 total_tests: 22599 command: aider --model gpt-4o-2024-11-20100 date: 2024-12-30101 versions: 0.70.1.dev102 seconds_per_case: 12.1103 total_cost: 6.7351104 105- dirname: 2024-12-30-20-44-54--gpt4o-ex-as-sys-clean-prompt106 test_cases: 225107 model: gpt-4o-2024-08-06108 edit_format: diff109 commit_hash: 09ee197-dirty110 pass_rate_1: 4.9111 pass_rate_2: 23.1112 pass_num_1: 11113 pass_num_2: 52114 percent_cases_well_formed: 94.2115 error_outputs: 21116 num_malformed_responses: 21117 num_with_malformed_responses: 13118 user_asks: 65119 lazy_comments: 0120 syntax_errors: 0121 indentation_errors: 0122 exhausted_context_windows: 0123 test_timeouts: 3124 total_tests: 225125 command: aider --model gpt-4o-2024-08-06126 date: 2024-12-30127 versions: 0.70.1.dev128 seconds_per_case: 16.0129 total_cost: 7.0286130 131- dirname: 2024-12-21-19-23-03--polyglot-o1-hard-diff132 test_cases: 224133 model: o1-2024-12-17 (high)134 edit_format: diff135 commit_hash: a755079-dirty136 pass_rate_1: 23.7137 pass_rate_2: 61.7138 pass_num_1: 53139 pass_num_2: 139140 percent_cases_well_formed: 91.5141 error_outputs: 25142 num_malformed_responses: 24143 num_with_malformed_responses: 19144 user_asks: 16145 lazy_comments: 0146 syntax_errors: 0147 indentation_errors: 0148 exhausted_context_windows: 0149 test_timeouts: 2150 total_tests: 225151 command: aider --model openrouter/openai/o1152 date: 2024-12-21153 versions: 0.69.2.dev154 seconds_per_case: 133.2155 total_cost: 186.4958156157- dirname: 2024-12-21-20-56-21--polyglot-deepseek-diff158 test_cases: 225159 model: DeepSeek Chat V2.5160 edit_format: diff161 commit_hash: a755079-dirty162 pass_rate_1: 5.3163 pass_rate_2: 17.8164 pass_num_1: 12165 pass_num_2: 40166 percent_cases_well_formed: 92.9167 error_outputs: 42168 num_malformed_responses: 37169 num_with_malformed_responses: 16170 user_asks: 23171 lazy_comments: 0172 syntax_errors: 0173 indentation_errors: 0174 exhausted_context_windows: 5175 test_timeouts: 5176 total_tests: 225177 command: aider --model deepseek/deepseek-chat178 date: 2024-12-21179 versions: 0.69.2.dev180 seconds_per_case: 184.0181 total_cost: 0.5101182183- dirname: 2024-12-21-21-46-27--polyglot-haiku-diff184 test_cases: 225185 model: claude-3-5-haiku-20241022186 edit_format: diff187 commit_hash: a755079-dirty188 pass_rate_1: 7.1189 pass_rate_2: 28.0190 pass_num_1: 16191 pass_num_2: 63192 percent_cases_well_formed: 91.1193 error_outputs: 31194 num_malformed_responses: 30195 num_with_malformed_responses: 20196 user_asks: 13197 lazy_comments: 0198 syntax_errors: 0199 indentation_errors: 0200 exhausted_context_windows: 1201 test_timeouts: 9202 total_tests: 225203 command: aider --model claude-3-5-haiku-20241022204 date: 2024-12-21205 versions: 0.69.2.dev206 seconds_per_case: 31.8207 total_cost: 6.0583208209- dirname: 2024-12-22-13-22-32--polyglot-qwen-diff210 test_cases: 225211 model: Qwen2.5-Coder-32B-Instruct212 edit_format: diff213 commit_hash: 6d7e8be-dirty214 pass_rate_1: 4.4215 pass_rate_2: 8.0216 pass_num_1: 10217 pass_num_2: 18218 percent_cases_well_formed: 71.6219 error_outputs: 158220 num_malformed_responses: 148221 num_with_malformed_responses: 64222 user_asks: 132223 lazy_comments: 0224 syntax_errors: 0225 indentation_errors: 0226 exhausted_context_windows: 1227 test_timeouts: 2228 total_tests: 225229 command: "aider --model openai/Qwen/Qwen2.5-Coder-32B-Instruct # via hyperbolic"230 date: 2024-12-22231 versions: 0.69.2.dev232 seconds_per_case: 84.4233 total_cost: 0.0000234235- dirname: 2024-12-22-21-26-35--polyglot-o1mini-whole236 test_cases: 225237 model: o1-mini-2024-09-12238 edit_format: whole239 commit_hash: 37df899240 pass_rate_1: 5.8241 pass_rate_2: 32.9242 pass_num_1: 13243 pass_num_2: 74244 percent_cases_well_formed: 96.9245 error_outputs: 8246 num_malformed_responses: 8247 num_with_malformed_responses: 7248 user_asks: 27249 lazy_comments: 0250 syntax_errors: 0251 indentation_errors: 0252 exhausted_context_windows: 0253 test_timeouts: 3254 total_tests: 225255 command: aider --model o1-mini256 date: 2024-12-22257 versions: 0.69.2.dev258 seconds_per_case: 34.7259 total_cost: 18.5770260261- dirname: 2024-12-22-18-43-25--gemini-exp-1206-polyglot-whole-2262 test_cases: 225263 model: gemini-exp-1206264 edit_format: whole265 commit_hash: b1bc2f8266 pass_rate_1: 19.6267 pass_rate_2: 38.2268 pass_num_1: 44269 pass_num_2: 86270 percent_cases_well_formed: 98.2271 error_outputs: 8272 num_malformed_responses: 8273 num_with_malformed_responses: 4274 user_asks: 32275 lazy_comments: 0276 syntax_errors: 0277 indentation_errors: 0278 exhausted_context_windows: 0279 test_timeouts: 9280 total_tests: 225281 command: aider --model gemini/gemini-exp-1206282 date: 2024-12-22283 versions: 0.69.2.dev284 seconds_per_case: 45.5285 total_cost: 0.0000286 287- dirname: 2024-12-22-20-08-13--gemini-2.0-flash-exp-polyglot-whole288 test_cases: 225289 model: gemini-2.0-flash-exp290 edit_format: whole291 commit_hash: b1bc2f8292 pass_rate_1: 11.6293 pass_rate_2: 22.2294 pass_num_1: 26295 pass_num_2: 50296 percent_cases_well_formed: 100.0297 error_outputs: 1298 num_malformed_responses: 0299 num_with_malformed_responses: 0300 user_asks: 9301 lazy_comments: 0302 syntax_errors: 0303 indentation_errors: 0304 exhausted_context_windows: 1305 test_timeouts: 8306 total_tests: 225307 command: aider --model gemini/gemini-2.0-flash-exp308 date: 2024-12-22309 versions: 0.69.2.dev310 seconds_per_case: 12.2311 total_cost: 0.0000312 313- dirname: 2024-12-23-01-11-56--yi-test314 test_cases: 225315 model: yi-lightning316 edit_format: whole317 commit_hash: 2b1625e318 pass_rate_1: 5.8319 pass_rate_2: 12.9320 pass_num_1: 13321 pass_num_2: 29322 percent_cases_well_formed: 92.9323 error_outputs: 87324 num_malformed_responses: 72325 num_with_malformed_responses: 16326 user_asks: 107327 lazy_comments: 0328 syntax_errors: 0329 indentation_errors: 0330 exhausted_context_windows: 1331 test_timeouts: 6332 total_tests: 225333 command: aider --model openai/yi-lightning334 date: 2024-12-23335 versions: 0.69.2.dev336 seconds_per_case: 146.7337 total_cost: 0.0000338339- dirname: 2024-12-25-13-31-51--deepseekv3preview-diff2340 test_cases: 225341 model: DeepSeek Chat V3 (prev)342 edit_format: diff343 commit_hash: 0a23c4a-dirty344 pass_rate_1: 22.7345 pass_rate_2: 48.4346 pass_num_1: 51347 pass_num_2: 109348 percent_cases_well_formed: 98.7349 error_outputs: 7350 num_malformed_responses: 7351 num_with_malformed_responses: 3352 user_asks: 19353 lazy_comments: 0354 syntax_errors: 0355 indentation_errors: 0356 exhausted_context_windows: 0357 test_timeouts: 8358 total_tests: 225359 command: aider --model deepseek/deepseek-chat360 date: 2024-12-25361 versions: 0.69.2.dev362 seconds_per_case: 34.8363 total_cost: 0.3369364365- dirname: 2024-12-26-00-55-20--Qwen2.5-Coder-32B-Instruct366 test_cases: 225367 model: Qwen2.5-Coder-32B-Instruct368 edit_format: whole369 commit_hash: b51768b0370 pass_rate_1: 4.9371 pass_rate_2: 16.4372 pass_num_1: 11373 pass_num_2: 37374 percent_cases_well_formed: 99.6375 error_outputs: 1376 num_malformed_responses: 1377 num_with_malformed_responses: 1378 user_asks: 33379 lazy_comments: 0380 syntax_errors: 0381 indentation_errors: 0382 exhausted_context_windows: 0383 test_timeouts: 6384 total_tests: 225385 command: aider --model openai/Qwen2.5-Coder-32B-Instruct386 date: 2024-12-26387 versions: 0.69.2.dev388 seconds_per_case: 42.0389 total_cost: 0.0000390391- dirname: 2025-01-13-18-17-25--codestral-whole2392 test_cases: 225393 model: Codestral 25.01394 edit_format: whole395 commit_hash: 0cba898-dirty396 pass_rate_1: 4.0397 pass_rate_2: 11.1398 pass_num_1: 9399 pass_num_2: 25400 percent_cases_well_formed: 100.0401 error_outputs: 0402 num_malformed_responses: 0403 num_with_malformed_responses: 0404 user_asks: 47405 lazy_comments: 0406 syntax_errors: 0407 indentation_errors: 0408 exhausted_context_windows: 0409 test_timeouts: 4410 total_tests: 225411 command: aider --model mistral/codestral-latest412 date: 2025-01-13413 versions: 0.71.2.dev414 seconds_per_case: 9.3415 total_cost: 1.9834416417- dirname: 2025-01-20-19-11-38--ds-turns-upd-cur-msgs-fix-with-summarizer418 test_cases: 225419 model: DeepSeek R1420 edit_format: diff421 commit_hash: 5650697-dirty422 pass_rate_1: 26.7423 pass_rate_2: 56.9424 pass_num_1: 60425 pass_num_2: 128426 percent_cases_well_formed: 96.9427 error_outputs: 8428 num_malformed_responses: 7429 num_with_malformed_responses: 7430 user_asks: 15431 lazy_comments: 0432 syntax_errors: 0433 indentation_errors: 0434 exhausted_context_windows: 1435 test_timeouts: 5436 total_tests: 225437 command: aider --model deepseek/deepseek-reasoner438 date: 2025-01-20439 versions: 0.71.2.dev440 seconds_per_case: 113.7441 total_cost: 5.4193442443- dirname: 2025-01-23-19-14-48--r1-architect-sonnet444 test_cases: 225445 model: DeepSeek R1 + claude-3-5-sonnet-20241022446 edit_format: architect447 commit_hash: 05a77c7448 editor_model: claude-3-5-sonnet-20241022449 editor_edit_format: editor-diff450 pass_rate_1: 27.1451 pass_rate_2: 64.0452 pass_num_1: 61453 pass_num_2: 144454 percent_cases_well_formed: 100.0455 error_outputs: 2456 num_malformed_responses: 0457 num_with_malformed_responses: 0458 user_asks: 392459 lazy_comments: 6460 syntax_errors: 0461 indentation_errors: 0462 exhausted_context_windows: 0463 test_timeouts: 5464 total_tests: 225465 command: aider --architect --model r1 --editor-model sonnet466 date: 2025-01-23467 versions: 0.72.3.dev468 seconds_per_case: 251.6469 total_cost: 13.2933470471- dirname: 2025-01-28-16-00-03--qwen-max-2025-01-25-polyglot-diff472 test_cases: 225473 model: qwen-max-2025-01-25474 edit_format: diff475 commit_hash: ae7d459476 pass_rate_1: 9.3477 pass_rate_2: 21.8478 pass_num_1: 21479 pass_num_2: 49480 percent_cases_well_formed: 90.2481 error_outputs: 46482 num_malformed_responses: 44483 num_with_malformed_responses: 22484 user_asks: 23485 lazy_comments: 0486 syntax_errors: 0487 indentation_errors: 0488 exhausted_context_windows: 0489 test_timeouts: 9490 total_tests: 225491 command: OPENAI_API_BASE=https://dashscope-intl.aliyuncs.com/compatible-mode/v1 aider --model openai/qwen-max-2025-01-25492 date: 2025-01-28493 versions: 0.72.4.dev494 seconds_per_case: 39.5495496- dirname: 2025-01-31-20-27-46--o3-mini-diff2497 test_cases: 225498 model: o3-mini (medium)499 edit_format: diff500 commit_hash: 2fb517b-dirty501 pass_rate_1: 19.1502 pass_rate_2: 53.8503 pass_num_1: 43504 pass_num_2: 121505 percent_cases_well_formed: 95.1506 error_outputs: 28507 num_malformed_responses: 28508 num_with_malformed_responses: 11509 user_asks: 17510 lazy_comments: 0511 syntax_errors: 0512 indentation_errors: 0513 exhausted_context_windows: 0514 test_timeouts: 2515 total_tests: 225516 command: aider --model o3-mini517 date: 2025-01-31518 versions: 0.72.4.dev519 seconds_per_case: 47.2520 total_cost: 8.8599521522- dirname: 2025-01-31-20-42-47--o3-mini-diff-high523 test_cases: 224524 model: o3-mini (high)525 edit_format: diff526 commit_hash: b0d58d1-dirty527 pass_rate_1: 21.0528 pass_rate_2: 60.4529 pass_num_1: 47530 pass_num_2: 136531 percent_cases_well_formed: 93.3532 error_outputs: 26533 num_malformed_responses: 24534 num_with_malformed_responses: 15535 user_asks: 19536 lazy_comments: 0537 syntax_errors: 0538 indentation_errors: 0539 exhausted_context_windows: 1540 test_timeouts: 7541 total_tests: 225542 command: aider --model o3-mini --reasoning-effort high543 date: 2025-01-31544 versions: 0.72.4.dev545 seconds_per_case: 124.6546 total_cost: 18.1584547548- dirname: 2025-01-21-22-51-49--gemini-2.0-flash-thinking-exp-01-21-polyglot-diff549 test_cases: 225550 model: gemini-2.0-flash-thinking-exp-01-21551 edit_format: diff552 commit_hash: 843720a553 pass_rate_1: 5.8554 pass_rate_2: 18.2555 pass_num_1: 13556 pass_num_2: 41557 percent_cases_well_formed: 77.8558 error_outputs: 182559 num_malformed_responses: 180560 num_with_malformed_responses: 50561 user_asks: 26562 lazy_comments: 0563 syntax_errors: 0564 indentation_errors: 0565 exhausted_context_windows: 2566 test_timeouts: 7567 total_tests: 225568 command: aider --model gemini/gemini-2.0-flash-thinking-exp-01-21569 date: 2025-01-21570 versions: 0.72.2.dev571 seconds_per_case: 24.2572 total_cost: 0.0000573574- dirname: 2025-02-15-19-51-22--chatgpt4o-feb15-diff575 test_cases: 223576 model: chatgpt-4o-latest (2025-02-15)577 edit_format: diff578 commit_hash: 108ce18-dirty579 pass_rate_1: 9.0580 pass_rate_2: 27.1581 pass_num_1: 20582 pass_num_2: 61583 percent_cases_well_formed: 93.3584 error_outputs: 66585 num_malformed_responses: 21586 num_with_malformed_responses: 15587 user_asks: 57588 lazy_comments: 0589 syntax_errors: 0590 indentation_errors: 0591 exhausted_context_windows: 0592 test_timeouts: 2593 total_tests: 225594 command: aider --model chatgpt-4o-latest595 date: 2025-02-15596 versions: 0.74.3.dev597 seconds_per_case: 12.4598 total_cost: 14.3703599600- dirname: 2025-02-24-19-54-07--sonnet37-diff601 test_cases: 225602 model: claude-3-7-sonnet-20250219 (no thinking)603 edit_format: diff604 commit_hash: 75e9ee6605 pass_rate_1: 24.4606 pass_rate_2: 60.4607 pass_num_1: 55608 pass_num_2: 136609 percent_cases_well_formed: 93.3610 error_outputs: 16611 num_malformed_responses: 16612 num_with_malformed_responses: 15613 user_asks: 12614 lazy_comments: 0615 syntax_errors: 0616 indentation_errors: 0617 exhausted_context_windows: 0618 test_timeouts: 0619 total_tests: 225620 command: aider --model sonnet621 date: 2025-02-24622 versions: 0.74.4.dev623 seconds_per_case: 28.3624 total_cost: 17.7191625626- dirname: 2025-02-24-21-47-23--sonnet37-diff-think-32k-64k627 test_cases: 225628 model: claude-3-7-sonnet-20250219 (32k thinking tokens)629 edit_format: diff630 commit_hash: 60d11a6, 93edbda631 pass_rate_1: 29.3632 pass_rate_2: 64.9633 pass_num_1: 66634 pass_num_2: 146635 percent_cases_well_formed: 97.8636 error_outputs: 66637 num_malformed_responses: 5638 num_with_malformed_responses: 5639 user_asks: 5640 lazy_comments: 0641 syntax_errors: 0642 indentation_errors: 0643 exhausted_context_windows: 0644 test_timeouts: 1645 total_tests: 225646 command: "aider --model anthropic/claude-3-7-sonnet-20250219 --thinking-tokens 32k"647 date: 2025-02-24648 versions: 0.75.1.dev649 seconds_per_case: 105.2650 total_cost: 36.8343651652- dirname: 2025-02-27-20-26-15--gpt45-diff3653 test_cases: 224654 model: gpt-4.5-preview655 edit_format: diff656 commit_hash: b462e55-dirty657 pass_rate_1: 22.3658 pass_rate_2: 44.9659 pass_num_1: 50660 pass_num_2: 101661 percent_cases_well_formed: 97.3662 error_outputs: 10663 num_malformed_responses: 8664 num_with_malformed_responses: 6665 user_asks: 15666 lazy_comments: 0667 syntax_errors: 0668 indentation_errors: 0669 exhausted_context_windows: 1670 test_timeouts: 2671 total_tests: 225672 command: aider --model openai/gpt-4.5-preview673 date: 2025-02-27674 versions: 0.75.2.dev675 seconds_per_case: 113.5676 total_cost: 183.1802677678- dirname: 2025-03-06-17-40-24--qwq32b-diff-temp-topp-ex-sys-remind-user-for-real679 test_cases: 225680 model: QwQ-32B681 edit_format: diff682 commit_hash: 51d118f-dirty683 pass_rate_1: 8.0684 pass_rate_2: 20.9685 pass_num_1: 18686 pass_num_2: 47687 percent_cases_well_formed: 67.6688 error_outputs: 145689 num_malformed_responses: 143690 num_with_malformed_responses: 73691 user_asks: 17692 lazy_comments: 0693 syntax_errors: 0694 indentation_errors: 0695 exhausted_context_windows: 1696 test_timeouts: 4697 total_tests: 225698 command: aider --model fireworks_ai/accounts/fireworks/models/qwq-32b699 date: 2025-03-06700 versions: 0.75.3.dev701 seconds_per_case: 228.6702 total_cost: 0.0000703704- dirname: 2025-03-07-15-11-27--qwq32b-arch-temp-topp-again705 test_cases: 225706 model: QwQ-32B + Qwen 2.5 Coder Instruct707 edit_format: architect708 commit_hash: 52162a5709 editor_model: fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct710 editor_edit_format: editor-diff711 pass_rate_1: 9.8712 pass_rate_2: 26.2713 pass_num_1: 22714 pass_num_2: 59715 percent_cases_well_formed: 100.0716 error_outputs: 122717 num_malformed_responses: 0718 num_with_malformed_responses: 0719 user_asks: 489720 lazy_comments: 8721 syntax_errors: 0722 indentation_errors: 0723 exhausted_context_windows: 1724 test_timeouts: 2725 total_tests: 225726 command: aider --model fireworks_ai/accounts/fireworks/models/qwq-32b --architect727 date: 2025-03-07728 versions: 0.75.3.dev729 seconds_per_case: 137.4730 total_cost: 0731732- dirname: 2025-03-14-23-40-00--cmda-quality-whole2733 test_cases: 225734 model: command-a-03-2025-quality735 edit_format: whole736 commit_hash: a1aa63f737 pass_rate_1: 2.2738 pass_rate_2: 12.0739 pass_num_1: 5740 pass_num_2: 27741 percent_cases_well_formed: 99.6742 error_outputs: 2743 num_malformed_responses: 1744 num_with_malformed_responses: 1745 user_asks: 215746 lazy_comments: 0747 syntax_errors: 0748 indentation_errors: 0749 exhausted_context_windows: 1750 test_timeouts: 4751 total_tests: 225752 command: OPENAI_API_BASE=https://api.cohere.ai/compatibility/v1 aider --model openai/command-a-03-2025-quality753 date: 2025-03-14754 versions: 0.77.1.dev755 seconds_per_case: 85.1756 total_cost: 0.0000757758- dirname: 2025-03-15-01-21-24--gemma3-27b-or759 test_cases: 225760 model: gemma-3-27b-it761 edit_format: whole762 commit_hash: fd21f51-dirty763 pass_rate_1: 1.8764 pass_rate_2: 4.9765 pass_num_1: 4766 pass_num_2: 11767 percent_cases_well_formed: 100.0768 error_outputs: 3769 num_malformed_responses: 0770 num_with_malformed_responses: 0771 user_asks: 181772 lazy_comments: 0773 syntax_errors: 0774 indentation_errors: 0775 exhausted_context_windows: 1776 test_timeouts: 3777 total_tests: 225778 command: aider --model openrouter/google/gemma-3-27b-it779 date: 2025-03-15780 versions: 0.77.1.dev781 seconds_per_case: 79.7782 total_cost: 0.0000783784- dirname: 2025-03-24-15-41-33--deepseek-v3-0324-polyglot-diff785 test_cases: 225786 model: DeepSeek V3 (0324)787 edit_format: diff788 commit_hash: 502b863789 pass_rate_1: 28.0790 pass_rate_2: 55.1791 pass_num_1: 63792 pass_num_2: 124793 percent_cases_well_formed: 99.6794 error_outputs: 32795 num_malformed_responses: 1796 num_with_malformed_responses: 1797 user_asks: 96798 lazy_comments: 0799 syntax_errors: 0800 indentation_errors: 0801 exhausted_context_windows: 2802 test_timeouts: 4803 total_tests: 225804 command: aider --model deepseek/deepseek-chat805 date: 2025-03-24806 versions: 0.78.1.dev807 seconds_per_case: 290.0808 total_cost: 1.1164809810- dirname: 2025-04-12-04-55-50--gemini-25-pro-diff-fenced811 test_cases: 225812 model: Gemini 2.5 Pro Preview 03-25813 edit_format: diff-fenced814 commit_hash: 0282574815 pass_rate_1: 40.9816 pass_rate_2: 72.9817 pass_num_1: 92818 pass_num_2: 164819 percent_cases_well_formed: 92.4820 error_outputs: 21821 num_malformed_responses: 21822 num_with_malformed_responses: 17823 user_asks: 69824 lazy_comments: 0825 syntax_errors: 0826 indentation_errors: 0827 exhausted_context_windows: 0828 test_timeouts: 2829 total_tests: 225830 command: aider --model gemini/gemini-2.5-pro-preview-03-25831 date: 2025-04-12832 versions: 0.81.3.dev833 seconds_per_case: 45.3834 total_cost: 0 # incorrect: 6.3174835836- dirname: 2025-03-29-05-24-55--chatgpt4o-mar28-diff837 test_cases: 225838 model: chatgpt-4o-latest (2025-03-29)839 edit_format: diff840 commit_hash: 0decbad841 pass_rate_1: 16.4842 pass_rate_2: 45.3843 pass_num_1: 37844 pass_num_2: 102845 percent_cases_well_formed: 64.4846 error_outputs: 85847 num_malformed_responses: 85848 num_with_malformed_responses: 80849 user_asks: 174850 lazy_comments: 0851 syntax_errors: 0852 indentation_errors: 0853 exhausted_context_windows: 0854 test_timeouts: 4855 total_tests: 225856 command: aider --model chatgpt-4o-latest857 date: 2025-03-29858 versions: 0.79.3.dev859 seconds_per_case: 10.3860 total_cost: 19.7416861862- dirname: 2025-04-04-02-57-25--qalpha-diff-exsys863 test_cases: 225864 model: Quasar Alpha865 edit_format: diff866 commit_hash: 8a34a6c-dirty867 pass_rate_1: 21.8868 pass_rate_2: 54.7869 pass_num_1: 49870 pass_num_2: 123871 percent_cases_well_formed: 98.2872 error_outputs: 4873 num_malformed_responses: 4874 num_with_malformed_responses: 4875 user_asks: 187876 lazy_comments: 0877 syntax_errors: 0878 indentation_errors: 0879 exhausted_context_windows: 0880 test_timeouts: 4881 total_tests: 225882 command: aider --model openrouter/openrouter/quasar-alpha883 date: 2025-04-04884 versions: 0.80.5.dev885 seconds_per_case: 14.8886 total_cost: 0.0000887888- dirname: 2025-04-06-08-39-52--llama-4-maverick-17b-128e-instruct-polyglot-whole889 test_cases: 225890 model: Llama 4 Maverick891 edit_format: whole892 commit_hash: 9445a31893 pass_rate_1: 4.4894 pass_rate_2: 15.6895 pass_num_1: 10896 pass_num_2: 35897 percent_cases_well_formed: 99.1898 error_outputs: 12899 num_malformed_responses: 2900 num_with_malformed_responses: 2901 user_asks: 248902 lazy_comments: 0903 syntax_errors: 0904 indentation_errors: 0905 exhausted_context_windows: 0906 test_timeouts: 4907 total_tests: 225908 command: aider --model nvidia_nim/meta/llama-4-maverick-17b-128e-instruct909 date: 2025-04-06910 versions: 0.81.2.dev911 seconds_per_case: 20.5912 total_cost: 0.0000913914- dirname: 2025-04-10-04-21-31--grok3-diff-exuser915 test_cases: 225916 model: Grok 3 Beta917 edit_format: diff918 commit_hash: 2dd40fc-dirty919 pass_rate_1: 22.2920 pass_rate_2: 53.3921 pass_num_1: 50922 pass_num_2: 120923 percent_cases_well_formed: 99.6924 error_outputs: 1925 num_malformed_responses: 1926 num_with_malformed_responses: 1927 user_asks: 68928 lazy_comments: 0929 syntax_errors: 0930 indentation_errors: 0931 exhausted_context_windows: 0932 test_timeouts: 2933 total_tests: 225934 command: aider --model openrouter/x-ai/grok-3-beta935 date: 2025-04-10936 versions: 0.81.2.dev937 seconds_per_case: 15.3938 total_cost: 11.0338939940- dirname: 2025-04-10-18-47-24--grok3-mini-whole-exuser941 test_cases: 225942 model: Grok 3 Mini Beta (low)943 edit_format: whole944 commit_hash: 14ffe77-dirty945 pass_rate_1: 11.1946 pass_rate_2: 34.7947 pass_num_1: 25948 pass_num_2: 78949 percent_cases_well_formed: 100.0950 error_outputs: 3951 num_malformed_responses: 0952 num_with_malformed_responses: 0953 user_asks: 73954 lazy_comments: 0955 syntax_errors: 0956 indentation_errors: 0957 exhausted_context_windows: 0958 test_timeouts: 5959 total_tests: 225960 command: aider --model openrouter/x-ai/grok-3-mini-beta961 date: 2025-04-10962 versions: 0.81.2.dev963 seconds_per_case: 35.1964 total_cost: 0.7856965966- dirname: 2025-04-10-23-59-02--xai-grok3-mini-whole-high967 test_cases: 225968 model: Grok 3 Mini Beta (high)969 edit_format: whole970 commit_hash: 8ee33da-dirty971 pass_rate_1: 17.3972 pass_rate_2: 49.3973 pass_num_1: 39974 pass_num_2: 111975 percent_cases_well_formed: 99.6976 error_outputs: 1977 num_malformed_responses: 1978 num_with_malformed_responses: 1979 user_asks: 64980 lazy_comments: 0981 syntax_errors: 0982 indentation_errors: 0983 exhausted_context_windows: 0984 test_timeouts: 0985 total_tests: 225986 command: aider --model xai/grok-3-mini-beta --reasoning-effort high987 date: 2025-04-10988 versions: 0.81.3.dev989 seconds_per_case: 79.1990 total_cost: 0.7346991 992- dirname: 2025-04-10-19-02-44--oalpha-diff-exsys993 test_cases: 225994 model: Optimus Alpha995 edit_format: diff996 commit_hash: 532bc45-dirty997 pass_rate_1: 21.3998 pass_rate_2: 52.9999 pass_num_1: 481000 pass_num_2: 1191001 percent_cases_well_formed: 97.31002 error_outputs: 71003 num_malformed_responses: 61004 num_with_malformed_responses: 61005 user_asks: 1821006 lazy_comments: 01007 syntax_errors: 01008 indentation_errors: 01009 exhausted_context_windows: 01010 test_timeouts: 31011 total_tests: 2251012 command: aider --model openrouter/openrouter/optimus-alpha1013 date: 2025-04-101014 versions: 0.81.2.dev1015 seconds_per_case: 18.41016 total_cost: 0.000010171018- dirname: 2025-04-14-21-05-54--gpt41-diff-exuser1019 test_cases: 2251020 model: gpt-4.11021 edit_format: diff1022 commit_hash: 7a87db5-dirty1023 pass_rate_1: 20.01024 pass_rate_2: 52.41025 pass_num_1: 451026 pass_num_2: 1181027 percent_cases_well_formed: 98.21028 error_outputs: 61029 num_malformed_responses: 51030 num_with_malformed_responses: 41031 user_asks: 1711032 lazy_comments: 01033 syntax_errors: 01034 indentation_errors: 01035 exhausted_context_windows: 11036 test_timeouts: 51037 total_tests: 2251038 command: aider --model gpt-4.11039 date: 2025-04-141040 versions: 0.81.4.dev1041 seconds_per_case: 20.51042 total_cost: 9.855610431044- dirname: 2025-04-14-21-27-53--gpt41mini-diff1045 test_cases: 2251046 model: gpt-4.1-mini1047 edit_format: diff1048 commit_hash: ffb743e-dirty1049 pass_rate_1: 11.11050 pass_rate_2: 32.41051 pass_num_1: 251052 pass_num_2: 731053 percent_cases_well_formed: 92.41054 error_outputs: 641055 num_malformed_responses: 621056 num_with_malformed_responses: 171057 user_asks: 1591058 lazy_comments: 01059 syntax_errors: 01060 indentation_errors: 01061 exhausted_context_windows: 21062 test_timeouts: 21063 total_tests: 2251064 command: aider --model gpt-4.1-mini1065 date: 2025-04-141066 versions: 0.81.4.dev1067 seconds_per_case: 19.51068 total_cost: 1.991810691070- dirname: 2025-04-14-22-46-01--gpt41nano-diff1071 test_cases: 2251072 model: gpt-4.1-nano1073 edit_format: whole1074 commit_hash: 71d1591-dirty1075 pass_rate_1: 3.11076 pass_rate_2: 8.91077 pass_num_1: 71078 pass_num_2: 201079 percent_cases_well_formed: 94.21080 error_outputs: 201081 num_malformed_responses: 201082 num_with_malformed_responses: 131083 user_asks: 3161084 lazy_comments: 01085 syntax_errors: 01086 indentation_errors: 01087 exhausted_context_windows: 01088 test_timeouts: 81089 total_tests: 2251090 command: aider --model gpt-4.1-nano1091 date: 2025-04-141092 versions: 0.81.4.dev1093 seconds_per_case: 12.01094 total_cost: 0.428110951096- dirname: 2025-04-16-22-01-58--o4-mini-high-diff-exsys1097 test_cases: 2251098 model: o4-mini (high)1099 edit_format: diff1100 commit_hash: b66901f-dirty1101 pass_rate_1: 19.61102 pass_rate_2: 72.01103 pass_num_1: 441104 pass_num_2: 1621105 percent_cases_well_formed: 90.71106 error_outputs: 261107 num_malformed_responses: 241108 num_with_malformed_responses: 211109 user_asks: 661110 lazy_comments: 01111 syntax_errors: 01112 indentation_errors: 01113 exhausted_context_windows: 11114 test_timeouts: 21115 total_tests: 2251116 command: aider --model o4-mini1117 date: 2025-04-161118 versions: 0.82.1.dev1119 seconds_per_case: 176.51120 total_cost: 19.639911211122- dirname: 2025-04-19-14-43-04--o4-mini-patch1123 test_cases: 2251124 model: openhands-lm-32b-v0.11125 edit_format: whole1126 commit_hash: c08336f1127 pass_rate_1: 4.01128 pass_rate_2: 10.21129 pass_num_1: 91130 pass_num_2: 231131 percent_cases_well_formed: 95.11132 error_outputs: 551133 num_malformed_responses: 411134 num_with_malformed_responses: 111135 user_asks: 1661136 lazy_comments: 01137 syntax_errors: 01138 indentation_errors: 01139 exhausted_context_windows: 01140 test_timeouts: 111141 total_tests: 2251142 command: aider --model openrouter/all-hands/openhands-lm-32b-v0.11143 date: 2025-04-191144 versions: 0.82.2.dev1145 seconds_per_case: 195.61146 total_cost: 0.000011471148- dirname: 2025-04-20-19-54-31--flash25-diff-no-think1149 test_cases: 2251150 model: gemini-2.5-flash-preview-04-17 (default)1151 edit_format: diff1152 commit_hash: 7fcce5d-dirty1153 pass_rate_1: 21.81154 pass_rate_2: 47.11155 pass_num_1: 491156 pass_num_2: 1061157 percent_cases_well_formed: 85.31158 error_outputs: 601159 num_malformed_responses: 551160 num_with_malformed_responses: 331161 user_asks: 821162 lazy_comments: 11163 syntax_errors: 01164 indentation_errors: 01165 exhausted_context_windows: 51166 test_timeouts: 41167 total_tests: 2251168 command: aider --model gemini/gemini-2.5-flash-preview-04-171169 date: 2025-04-201170 versions: 0.82.3.dev1171 seconds_per_case: 50.11172 total_cost: 1.845111731174- dirname: 2025-05-07-19-32-40--gemini0506-diff-fenced-completion_cost1175 test_cases: 2251176 model: Gemini 2.5 Pro Preview 05-061177 edit_format: diff-fenced1178 commit_hash: 3b08327-dirty1179 pass_rate_1: 36.41180 pass_rate_2: 76.91181 pass_num_1: 821182 pass_num_2: 1731183 percent_cases_well_formed: 97.31184 error_outputs: 151185 num_malformed_responses: 71186 num_with_malformed_responses: 61187 user_asks: 1051188 lazy_comments: 01189 syntax_errors: 01190 indentation_errors: 01191 exhausted_context_windows: 01192 test_timeouts: 21193 total_tests: 2251194 command: aider --model gemini/gemini-2.5-pro-preview-05-061195 date: 2025-05-071196 versions: 0.82.4.dev1197 seconds_per_case: 165.31198 total_cost: 37.410411991200- dirname: 2025-05-08-03-20-24--qwen3-32b-default1201 test_cases: 2251202 model: Qwen3 32B1203 edit_format: diff1204 commit_hash: aaacee5-dirty, aeaf2591205 pass_rate_1: 14.21206 pass_rate_2: 40.01207 pass_num_1: 321208 pass_num_2: 901209 percent_cases_well_formed: 83.61210 error_outputs: 1191211 num_malformed_responses: 501212 num_with_malformed_responses: 371213 user_asks: 971214 lazy_comments: 01215 syntax_errors: 01216 indentation_errors: 01217 exhausted_context_windows: 121218 prompt_tokens: 3175911219 completion_tokens: 1204181220 test_timeouts: 51221 total_tests: 2251222 command: aider --model openrouter/qwen/qwen3-32b1223 date: 2025-05-081224 versions: 0.82.4.dev1225 seconds_per_case: 372.21226 total_cost: 0.760312271228- dirname: 2025-05-09-17-02-02--qwen3-235b-a22b.unthink_16k_diff1229 test_cases: 2251230 model: Qwen3 235B A22B diff, no think, Alibaba API1231 edit_format: diff1232 commit_hash: 91d7fbd-dirty1233 pass_rate_1: 28.91234 pass_rate_2: 59.61235 pass_num_1: 651236 pass_num_2: 1341237 percent_cases_well_formed: 92.91238 error_outputs: 221239 num_malformed_responses: 221240 num_with_malformed_responses: 161241 user_asks: 1111242 lazy_comments: 01243 syntax_errors: 01244 indentation_errors: 01245 exhausted_context_windows: 01246 prompt_tokens: 28161921247 completion_tokens: 3420621248 test_timeouts: 11249 total_tests: 2251250 command: aider --model openai/qwen3-235b-a22b1251 date: 2025-05-091252 versions: 0.82.4.dev1253 seconds_per_case: 45.41254 total_cost: 0.000012551256- dirname: 2025-05-24-21-17-54--sonnet4-diff-exuser1257 test_cases: 2251258 model: claude-sonnet-4-20250514 (no thinking)1259 edit_format: diff1260 commit_hash: ef3f8bb-dirty1261 pass_rate_1: 20.41262 pass_rate_2: 56.41263 pass_num_1: 461264 pass_num_2: 1271265 percent_cases_well_formed: 98.21266 error_outputs: 61267 num_malformed_responses: 41268 num_with_malformed_responses: 41269 user_asks: 1291270 lazy_comments: 01271 syntax_errors: 01272 indentation_errors: 01273 exhausted_context_windows: 11274 prompt_tokens: 34606631275 completion_tokens: 4333731276 test_timeouts: 71277 total_tests: 2251278 command: aider --model claude-sonnet-4-202505141279 date: 2025-05-241280 versions: 0.83.3.dev1281 seconds_per_case: 29.81282 total_cost: 15.81551283 1284- dirname: 2025-05-24-22-10-36--sonnet4-diff-exuser-think32k1285 test_cases: 2251286 model: claude-sonnet-4-20250514 (32k thinking)1287 edit_format: diff1288 commit_hash: e3cb9071289 thinking_tokens: 320001290 pass_rate_1: 25.81291 pass_rate_2: 61.31292 pass_num_1: 581293 pass_num_2: 1381294 percent_cases_well_formed: 97.31295 error_outputs: 101296 num_malformed_responses: 101297 num_with_malformed_responses: 61298 user_asks: 1111299 lazy_comments: 01300 syntax_errors: 01301 indentation_errors: 01302 exhausted_context_windows: 01303 prompt_tokens: 28630681304 completion_tokens: 12710741305 test_timeouts: 61306 total_tests: 2251307 command: aider --model claude-sonnet-4-202505141308 date: 2025-05-241309 versions: 0.83.3.dev1310 seconds_per_case: 79.91311 total_cost: 26.575513121313- dirname: 2025-05-25-19-57-20--opus4-diff-exuser1314 test_cases: 2251315 model: claude-opus-4-20250514 (no think)1316 edit_format: diff1317 commit_hash: 9ef32111318 pass_rate_1: 32.91319 pass_rate_2: 70.71320 pass_num_1: 741321 pass_num_2: 1591322 percent_cases_well_formed: 98.71323 error_outputs: 31324 num_malformed_responses: 31325 num_with_malformed_responses: 31326 user_asks: 1051327 lazy_comments: 01328 syntax_errors: 01329 indentation_errors: 01330 exhausted_context_windows: 01331 prompt_tokens: 26714371332 completion_tokens: 3807171333 test_timeouts: 31334 total_tests: 2251335 command: aider --model claude-opus-4-202505141336 date: 2025-05-251337 versions: 0.83.3.dev1338 seconds_per_case: 42.51339 total_cost: 68.625313401341- dirname: 2025-05-25-20-40-51--opus4-diff-exuser1342 test_cases: 2251343 model: claude-opus-4-20250514 (32k thinking)1344 edit_format: diff1345 commit_hash: 9ef32111346 thinking_tokens: 320001347 pass_rate_1: 37.31348 pass_rate_2: 72.01349 pass_num_1: 841350 pass_num_2: 1621351 percent_cases_well_formed: 97.31352 error_outputs: 101353 num_malformed_responses: 61354 num_with_malformed_responses: 61355 user_asks: 971356 lazy_comments: 01357 syntax_errors: 01358 indentation_errors: 01359 exhausted_context_windows: 01360 prompt_tokens: 25675141361 completion_tokens: 3631421362 test_timeouts: 41363 total_tests: 2251364 command: aider --model claude-opus-4-202505141365 date: 2025-05-251366 versions: 0.83.3.dev1367 seconds_per_case: 44.11368 total_cost: 65.748413691370- dirname: 2025-05-26-15-56-31--flash25-05-20-24k-think # dirname is misleading1371 test_cases: 2251372 model: gemini-2.5-flash-preview-05-20 (no think)1373 edit_format: diff1374 commit_hash: 214b811-dirty1375 thinking_tokens: 0 # <-- no thinking1376 pass_rate_1: 20.91377 pass_rate_2: 44.01378 pass_num_1: 471379 pass_num_2: 991380 percent_cases_well_formed: 93.81381 error_outputs: 161382 num_malformed_responses: 161383 num_with_malformed_responses: 141384 user_asks: 791385 lazy_comments: 01386 syntax_errors: 01387 indentation_errors: 01388 exhausted_context_windows: 01389 prompt_tokens: 55124581390 completion_tokens: 5141451391 test_timeouts: 41392 total_tests: 2251393 command: aider --model gemini/gemini-2.5-flash-preview-05-201394 date: 2025-05-261395 versions: 0.83.3.dev1396 seconds_per_case: 12.21397 total_cost: 1.13541398 1399- dirname: 2025-05-25-22-58-44--flash25-05-20-24k-think1400 test_cases: 2251401 model: gemini-2.5-flash-preview-05-20 (24k think)1402 edit_format: diff1403 commit_hash: a8568c3-dirty1404 thinking_tokens: 245761405 pass_rate_1: 26.21406 pass_rate_2: 55.11407 pass_num_1: 591408 pass_num_2: 1241409 percent_cases_well_formed: 95.61410 error_outputs: 151411 num_malformed_responses: 151412 num_with_malformed_responses: 101413 user_asks: 1011414 lazy_comments: 01415 syntax_errors: 01416 indentation_errors: 01417 exhausted_context_windows: 01418 prompt_tokens: 36667921419 completion_tokens: 27031621420 test_timeouts: 41421 total_tests: 2251422 command: aider --model gemini/gemini-2.5-flash-preview-05-201423 date: 2025-05-251424 versions: 0.83.3.dev1425 seconds_per_case: 53.91426 total_cost: 8.562514271428- dirname: 2025-06-06-18-38-56--gemini0605-diff-fenced1429 test_cases: 2251430 model: gemini-2.5-pro-preview-06-05 (default think)1431 edit_format: diff-fenced1432 commit_hash: 4c161f9-dirty1433 pass_rate_1: 44.91434 pass_rate_2: 79.11435 pass_num_1: 1011436 pass_num_2: 1781437 percent_cases_well_formed: 100.01438 error_outputs: 41439 num_malformed_responses: 01440 num_with_malformed_responses: 01441 user_asks: 1051442 lazy_comments: 01443 syntax_errors: 01444 indentation_errors: 01445 exhausted_context_windows: 41446 prompt_tokens: 27512961447 completion_tokens: 41421971448 test_timeouts: 11449 total_tests: 2251450 command: aider --model gemini/gemini-2.5-pro-preview-06-051451 date: 2025-06-061452 versions: 0.84.1.dev1453 seconds_per_case: 175.21454 total_cost: 45.596114551456- dirname: 2025-06-06-16-36-21--gemini0605-32k-think-diff-fenced1457 test_cases: 2251458 model: gemini-2.5-pro-preview-06-05 (32k think)1459 edit_format: diff-fenced1460 commit_hash: f827f221461 thinking_tokens: 327681462 pass_rate_1: 46.21463 pass_rate_2: 83.11464 pass_num_1: 1041465 pass_num_2: 1871466 percent_cases_well_formed: 99.61467 error_outputs: 11468 num_malformed_responses: 11469 num_with_malformed_responses: 11470 user_asks: 1121471 lazy_comments: 01472 syntax_errors: 01473 indentation_errors: 01474 exhausted_context_windows: 01475 prompt_tokens: 27199611476 completion_tokens: 46482271477 test_timeouts: 01478 total_tests: 2251479 command: aider --model gemini/gemini-2.5-pro-preview-06-05 --thinking-tokens 32k1480 date: 2025-06-061481 versions: 0.84.1.dev1482 seconds_per_case: 200.31483 total_cost: 49.882214841485- dirname: 2025-06-06-16-47-07--r1-diff1486 test_cases: 2241487 model: DeepSeek R1 (0528)1488 edit_format: diff1489 commit_hash: 4c161f9-dirty1490 pass_rate_1: 34.41491 pass_rate_2: 71.41492 pass_num_1: 771493 pass_num_2: 1601494 percent_cases_well_formed: 94.61495 error_outputs: 281496 num_malformed_responses: 151497 num_with_malformed_responses: 121498 user_asks: 1051499 lazy_comments: 01500 syntax_errors: 01501 indentation_errors: 01502 exhausted_context_windows: 01503 prompt_tokens: 26441691504 completion_tokens: 18421681505 test_timeouts: 21506 total_tests: 2251507 command: aider --model deepseek/deepseek-reasoner1508 date: 2025-06-061509 versions: 0.84.1.dev1510 seconds_per_case: 716.61511 total_cost: 4.801615121513- dirname: 2025-06-25-21-04-24--o3-price-reduction-high1514 test_cases: 2251515 model: o3 (high)1516 edit_format: diff1517 commit_hash: c48fea61518 reasoning_effort: high1519 pass_rate_1: 40.01520 pass_rate_2: 81.31521 pass_num_1: 901522 pass_num_2: 1831523 percent_cases_well_formed: 94.71524 error_outputs: 251525 num_malformed_responses: 231526 num_with_malformed_responses: 121527 user_asks: 1161528 lazy_comments: 01529 syntax_errors: 01530 indentation_errors: 01531 exhausted_context_windows: 11532 prompt_tokens: 31489321533 completion_tokens: 20476151534 test_timeouts: 21535 total_tests: 2251536 command: aider --model o3 --reasoning-effort high1537 date: 2025-06-251538 versions: 0.84.1.dev1539 seconds_per_case: 197.31540 total_cost: 21.225915411542- dirname: 2025-06-25-20-30-16--o3-price-reduction1543 test_cases: 2251544 model: o31545 edit_format: diff1546 commit_hash: c48fea61547 pass_rate_1: 40.91548 pass_rate_2: 76.91549 pass_num_1: 921550 pass_num_2: 1731551 percent_cases_well_formed: 93.81552 error_outputs: 221553 num_malformed_responses: 221554 num_with_malformed_responses: 141555 user_asks: 1081556 lazy_comments: 21557 syntax_errors: 01558 indentation_errors: 01559 exhausted_context_windows: 01560 prompt_tokens: 28931891561 completion_tokens: 11547671562 test_timeouts: 11563 total_tests: 2251564 command: aider --model o31565 date: 2025-06-251566 versions: 0.84.1.dev1567 seconds_per_case: 101.71568 total_cost: 13.751715691570- dirname: 2025-06-27-23-53-57--o3-mini-high-diff-arch1571 test_cases: 2241572 model: o3 (high) + gpt-4.11573 edit_format: architect1574 commit_hash: 4f4f00f-dirty1575 editor_model: gpt-4.11576 editor_edit_format: editor-diff1577 reasoning_effort: high1578 pass_rate_1: 34.81579 pass_rate_2: 78.21580 pass_num_1: 781581 pass_num_2: 1761582 percent_cases_well_formed: 100.01583 error_outputs: 181584 num_malformed_responses: 01585 num_with_malformed_responses: 01586 user_asks: 1721587 lazy_comments: 01588 syntax_errors: 01589 indentation_errors: 01590 exhausted_context_windows: 11591 prompt_tokens: 13068771592 completion_tokens: 13271541593 test_timeouts: 11594 total_tests: 2251595 command: aider --model o31596 date: 2025-06-271597 versions: 0.85.1.dev1598 seconds_per_case: 121.81599 total_cost: 17.551816001601- dirname: 2025-06-28-00-38-18--o3-pro-high1602 test_cases: 2251603 model: o3-pro (high)1604 edit_format: diff1605 commit_hash: 53183801606 reasoning_effort: high1607 pass_rate_1: 43.61608 pass_rate_2: 84.91609 pass_num_1: 981610 pass_num_2: 1911611 percent_cases_well_formed: 97.81612 error_outputs: 201613 num_malformed_responses: 81614 num_with_malformed_responses: 51615 user_asks: 1001616 lazy_comments: 01617 syntax_errors: 01618 indentation_errors: 01619 exhausted_context_windows: 01620 prompt_tokens: 23726361621 completion_tokens: 12359021622 test_timeouts: 11623 total_tests: 2251624 command: aider --model o3-pro1625 date: 2025-06-281626 versions: 0.85.1.dev1627 seconds_per_case: 449.01628 total_cost: 146.324916291630- dirname: 2025-07-11-19-37-40--xai-or-grok4-high1631 test_cases: 2251632 model: grok-4 (high)1633 edit_format: diff1634 commit_hash: f7870b6-dirty1635 reasoning_effort: high1636 pass_rate_1: 40.91637 pass_rate_2: 79.61638 pass_num_1: 921639 pass_num_2: 1791640 percent_cases_well_formed: 97.31641 error_outputs: 111642 num_malformed_responses: 81643 num_with_malformed_responses: 61644 user_asks: 1331645 lazy_comments: 01646 syntax_errors: 01647 indentation_errors: 01648 exhausted_context_windows: 01649 prompt_tokens: 28153471650 completion_tokens: 34114801651 test_timeouts: 01652 total_tests: 2251653 command: aider --model openrouter/x-ai/grok-41654 date: 2025-07-111655 versions: 0.85.2.dev1656 seconds_per_case: 403.21657 total_cost: 59.618216581659- dirname: 2025-07-17-17-41-54--kimi-k2-diff-or-pricing1660 test_cases: 2251661 model: Kimi K21662 edit_format: diff1663 commit_hash: 915ebff-dirty1664 pass_rate_1: 20.41665 pass_rate_2: 59.11666 pass_num_1: 461667 pass_num_2: 1331668 percent_cases_well_formed: 92.91669 error_outputs: 191670 num_malformed_responses: 191671 num_with_malformed_responses: 161672 user_asks: 611673 lazy_comments: 01674 syntax_errors: 01675 indentation_errors: 01676 exhausted_context_windows: 01677 prompt_tokens: 23551411678 completion_tokens: 3638461679 test_timeouts: 41680 total_tests: 2251681 command: aider --model openrouter/moonshotai/kimi-k21682 date: 2025-07-171683 versions: 0.85.3.dev1684 seconds_per_case: 67.61685 total_cost: 1.235716861687- dirname: 2025-08-06-04-54-48--gpt-oss-120b-high-polyglot1688 test_cases: 2251689 model: gpt-oss-120b (high)1690 edit_format: diff1691 commit_hash: 1af0e591692 pass_rate_1: 13.81693 pass_rate_2: 41.81694 pass_num_1: 311695 pass_num_2: 941696 percent_cases_well_formed: 79.11697 error_outputs: 951698 num_malformed_responses: 771699 num_with_malformed_responses: 471700 user_asks: 1421701 lazy_comments: 01702 syntax_errors: 01703 indentation_errors: 01704 exhausted_context_windows: 01705 prompt_tokens: 31237681706 completion_tokens: 8564951707 test_timeouts: 41708 total_tests: 2251709 command: aider --model openrouter/openai/gpt-oss-120b --reasoning-effort high1710 date: 2025-08-061711 versions: 0.85.3.dev1712 seconds_per_case: 35.51713 total_cost: 0.740617141715- dirname: 2025-08-23-15-47-21--gpt-5-high1716 test_cases: 2251717 model: gpt-5 (high)1718 edit_format: diff1719 commit_hash: 32faf821720 reasoning_effort: high1721 pass_rate_1: 52.01722 pass_rate_2: 88.01723 pass_num_1: 1171724 pass_num_2: 1981725 percent_cases_well_formed: 91.61726 error_outputs: 231727 num_malformed_responses: 221728 num_with_malformed_responses: 191729 user_asks: 961730 lazy_comments: 31731 syntax_errors: 01732 indentation_errors: 01733 exhausted_context_windows: 01734 prompt_tokens: 26755611735 completion_tokens: 26234291736 test_timeouts: 31737 total_tests: 2251738 command: aider --model openai/gpt-51739 date: 2025-08-231740 versions: 0.86.2.dev1741 seconds_per_case: 194.01742 total_cost: 29.082917431744- dirname: 2025-08-25-13-23-27--gpt-5-medium1745 test_cases: 2251746 model: gpt-5 (medium)1747 edit_format: diff1748 commit_hash: 32faf821749 reasoning_effort: medium1750 pass_rate_1: 49.81751 pass_rate_2: 86.71752 pass_num_1: 1121753 pass_num_2: 1951754 percent_cases_well_formed: 88.41755 error_outputs: 401756 num_malformed_responses: 401757 num_with_malformed_responses: 261758 user_asks: 1021759 lazy_comments: 01760 syntax_errors: 01761 indentation_errors: 01762 exhausted_context_windows: 01763 prompt_tokens: 28272611764 completion_tokens: 14687991765 test_timeouts: 01766 total_tests: 2251767 command: aider --model openai/gpt-51768 date: 2025-08-251769 versions: 0.86.2.dev1770 seconds_per_case: 118.71771 total_cost: 17.693017721773- dirname: 2025-08-25-14-16-37--gpt-5-low1774 test_cases: 2251775 model: gpt-5 (low)1776 edit_format: diff1777 commit_hash: 32faf821778 reasoning_effort: low1779 pass_rate_1: 43.11780 pass_rate_2: 81.31781 pass_num_1: 971782 pass_num_2: 1831783 percent_cases_well_formed: 86.71784 error_outputs: 461785 num_malformed_responses: 461786 num_with_malformed_responses: 301787 user_asks: 1131788 lazy_comments: 11789 syntax_errors: 01790 indentation_errors: 01791 exhausted_context_windows: 01792 prompt_tokens: 25340591793 completion_tokens: 7795681794 test_timeouts: 11795 total_tests: 2251796 command: aider --model openai/gpt-51797 date: 2025-08-251798 versions: 0.86.2.dev1799 seconds_per_case: 62.41800 total_cost: 10.371318011802- dirname: 2025-10-03-09-45-34--deepseek-v3.2-reasoner1803 test_cases: 2251804 model: DeepSeek-V3.2-Exp (Reasoner)1805 edit_format: diff1806 commit_hash: cbb53761807 pass_rate_1: 39.61808 pass_rate_2: 74.21809 pass_num_1: 891810 pass_num_2: 1671811 percent_cases_well_formed: 97.31812 error_outputs: 81813 num_malformed_responses: 61814 num_with_malformed_responses: 61815 user_asks: 671816 lazy_comments: 01817 syntax_errors: 01818 indentation_errors: 01819 exhausted_context_windows: 11820 prompt_tokens: 21914461821 completion_tokens: 16451291822 test_timeouts: 11823 total_tests: 2251824 command: aider --model deepseek/deepseek-reasoner1825 date: 2025-10-031826 versions: 0.86.2.dev1827 seconds_per_case: 291.21828 total_cost: 1.304518291830- dirname: 2025-10-03-09-21-36--deepseek-v3.2-chat1831 test_cases: 2251832 model: DeepSeek-V3.2-Exp (Chat)1833 edit_format: diff1834 commit_hash: cbb53761835 pass_rate_1: 38.71836 pass_rate_2: 70.21837 pass_num_1: 871838 pass_num_2: 1581839 percent_cases_well_formed: 98.21840 error_outputs: 61841 num_malformed_responses: 41842 num_with_malformed_responses: 41843 user_asks: 601844 lazy_comments: 01845 syntax_errors: 01846 indentation_errors: 01847 exhausted_context_windows: 11848 prompt_tokens: 22668681849 completion_tokens: 5734771850 test_timeouts: 41851 total_tests: 2251852 command: aider --model deepseek/deepseek-chat1853 date: 2025-10-031854 versions: 0.86.2.dev1855 seconds_per_case: 104.01856 total_cost: 0.87561857