- dirname: 2025-02-25-20-23-07--gemini-pro
  test_cases: 225
  model: Gemini 2.0 Pro exp-02-05
  edit_format: whole
  commit_hash: 2fccd47
  pass_rate_1: 20.4
  pass_rate_2: 35.6
  pass_num_1: 46
  pass_num_2: 80
  percent_cases_well_formed: 100.0
  error_outputs: 430
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 13
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 5
  total_tests: 225
  command: aider --model gemini/gemini-2.0-pro-exp-02-05
  date: 2025-02-25
  versions: 0.75.2.dev
  seconds_per_case: 34.8
  total_cost: 0.0000

- dirname: 2024-12-21-18-41-18--polyglot-gpt-4o-mini
  test_cases: 225
  model: gpt-4o-mini-2024-07-18
  edit_format: whole
  commit_hash: a755079-dirty
  pass_rate_1: 0.9
  pass_rate_2: 3.6
  pass_num_1: 2
  pass_num_2: 8
  percent_cases_well_formed: 100.0
  error_outputs: 0
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 36
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 3
  total_tests: 225
  command: aider --model gpt-4o-mini-2024-07-18
  date: 2024-12-21
  versions: 0.69.2.dev
  seconds_per_case: 17.3
  total_cost: 0.3236

- dirname: 2025-01-17-19-44-33--sonnet-baseline-jan-17
  test_cases: 225
  model: claude-3-5-sonnet-20241022
  edit_format: diff
  commit_hash: 6451d59
  pass_rate_1: 22.2
  pass_rate_2: 51.6
  pass_num_1: 50
  pass_num_2: 116
  percent_cases_well_formed: 99.6
  error_outputs: 2
  num_malformed_responses: 1
  num_with_malformed_responses: 1
  user_asks: 11
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 8
  total_tests: 225
  command: aider --model claude-3-5-sonnet-20241022
  date: 2025-01-17
  versions: 0.71.2.dev
  seconds_per_case: 21.4
  total_cost: 14.4063
  
- dirname: 2024-12-30-20-57-12--gpt-4o-2024-11-20-ex-as-sys
  test_cases: 225
  model: gpt-4o-2024-11-20
  edit_format: diff
  commit_hash: 09ee197-dirty
  pass_rate_1: 4.9
  pass_rate_2: 18.2
  pass_num_1: 11
  pass_num_2: 41
  percent_cases_well_formed: 95.1
  error_outputs: 12
  num_malformed_responses: 12
  num_with_malformed_responses: 11
  user_asks: 53
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 12
  total_tests: 225
  command: aider --model gpt-4o-2024-11-20
  date: 2024-12-30
  versions: 0.70.1.dev
  seconds_per_case: 12.1
  total_cost: 6.7351
  
- dirname: 2024-12-30-20-44-54--gpt4o-ex-as-sys-clean-prompt
  test_cases: 225
  model: gpt-4o-2024-08-06
  edit_format: diff
  commit_hash: 09ee197-dirty
  pass_rate_1: 4.9
  pass_rate_2: 23.1
  pass_num_1: 11
  pass_num_2: 52
  percent_cases_well_formed: 94.2
  error_outputs: 21
  num_malformed_responses: 21
  num_with_malformed_responses: 13
  user_asks: 65
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 3
  total_tests: 225
  command: aider --model gpt-4o-2024-08-06
  date: 2024-12-30
  versions: 0.70.1.dev
  seconds_per_case: 16.0
  total_cost: 7.0286
  
- dirname: 2024-12-21-19-23-03--polyglot-o1-hard-diff
  test_cases: 224
  model: o1-2024-12-17 (high)
  edit_format: diff
  commit_hash: a755079-dirty
  pass_rate_1: 23.7
  pass_rate_2: 61.7
  pass_num_1: 53
  pass_num_2: 139
  percent_cases_well_formed: 91.5
  error_outputs: 25
  num_malformed_responses: 24
  num_with_malformed_responses: 19
  user_asks: 16
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 2
  total_tests: 225
  command: aider --model openrouter/openai/o1
  date: 2024-12-21
  versions: 0.69.2.dev
  seconds_per_case: 133.2
  total_cost: 186.4958

- dirname: 2024-12-21-20-56-21--polyglot-deepseek-diff
  test_cases: 225
  model: DeepSeek Chat V2.5
  edit_format: diff
  commit_hash: a755079-dirty
  pass_rate_1: 5.3
  pass_rate_2: 17.8
  pass_num_1: 12
  pass_num_2: 40
  percent_cases_well_formed: 92.9
  error_outputs: 42
  num_malformed_responses: 37
  num_with_malformed_responses: 16
  user_asks: 23
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 5
  test_timeouts: 5
  total_tests: 225
  command: aider --model deepseek/deepseek-chat
  date: 2024-12-21
  versions: 0.69.2.dev
  seconds_per_case: 184.0
  total_cost: 0.5101

- dirname: 2024-12-21-21-46-27--polyglot-haiku-diff
  test_cases: 225
  model: claude-3-5-haiku-20241022
  edit_format: diff
  commit_hash: a755079-dirty
  pass_rate_1: 7.1
  pass_rate_2: 28.0
  pass_num_1: 16
  pass_num_2: 63
  percent_cases_well_formed: 91.1
  error_outputs: 31
  num_malformed_responses: 30
  num_with_malformed_responses: 20
  user_asks: 13
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 9
  total_tests: 225
  command: aider --model claude-3-5-haiku-20241022
  date: 2024-12-21
  versions: 0.69.2.dev
  seconds_per_case: 31.8
  total_cost: 6.0583

- dirname: 2024-12-22-13-22-32--polyglot-qwen-diff
  test_cases: 225
  model: Qwen2.5-Coder-32B-Instruct
  edit_format: diff
  commit_hash: 6d7e8be-dirty
  pass_rate_1: 4.4
  pass_rate_2: 8.0
  pass_num_1: 10
  pass_num_2: 18
  percent_cases_well_formed: 71.6
  error_outputs: 158
  num_malformed_responses: 148
  num_with_malformed_responses: 64
  user_asks: 132
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 2
  total_tests: 225
  command: "aider --model openai/Qwen/Qwen2.5-Coder-32B-Instruct # via hyperbolic"
  date: 2024-12-22
  versions: 0.69.2.dev
  seconds_per_case: 84.4
  total_cost: 0.0000

- dirname: 2024-12-22-21-26-35--polyglot-o1mini-whole
  test_cases: 225
  model: o1-mini-2024-09-12
  edit_format: whole
  commit_hash: 37df899
  pass_rate_1: 5.8
  pass_rate_2: 32.9
  pass_num_1: 13
  pass_num_2: 74
  percent_cases_well_formed: 96.9
  error_outputs: 8
  num_malformed_responses: 8
  num_with_malformed_responses: 7
  user_asks: 27
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 3
  total_tests: 225
  command: aider --model o1-mini
  date: 2024-12-22
  versions: 0.69.2.dev
  seconds_per_case: 34.7
  total_cost: 18.5770

- dirname: 2024-12-22-18-43-25--gemini-exp-1206-polyglot-whole-2
  test_cases: 225
  model: gemini-exp-1206
  edit_format: whole
  commit_hash: b1bc2f8
  pass_rate_1: 19.6
  pass_rate_2: 38.2
  pass_num_1: 44
  pass_num_2: 86
  percent_cases_well_formed: 98.2
  error_outputs: 8
  num_malformed_responses: 8
  num_with_malformed_responses: 4
  user_asks: 32
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 9
  total_tests: 225
  command: aider --model gemini/gemini-exp-1206
  date: 2024-12-22
  versions: 0.69.2.dev
  seconds_per_case: 45.5
  total_cost: 0.0000
  
- dirname: 2024-12-22-20-08-13--gemini-2.0-flash-exp-polyglot-whole
  test_cases: 225
  model: gemini-2.0-flash-exp
  edit_format: whole
  commit_hash: b1bc2f8
  pass_rate_1: 11.6
  pass_rate_2: 22.2
  pass_num_1: 26
  pass_num_2: 50
  percent_cases_well_formed: 100.0
  error_outputs: 1
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 9
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 8
  total_tests: 225
  command: aider --model gemini/gemini-2.0-flash-exp
  date: 2024-12-22
  versions: 0.69.2.dev
  seconds_per_case: 12.2
  total_cost: 0.0000
  
- dirname: 2024-12-23-01-11-56--yi-test
  test_cases: 225
  model: yi-lightning
  edit_format: whole
  commit_hash: 2b1625e
  pass_rate_1: 5.8
  pass_rate_2: 12.9
  pass_num_1: 13
  pass_num_2: 29
  percent_cases_well_formed: 92.9
  error_outputs: 87
  num_malformed_responses: 72
  num_with_malformed_responses: 16
  user_asks: 107
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 6
  total_tests: 225
  command: aider --model openai/yi-lightning
  date: 2024-12-23
  versions: 0.69.2.dev
  seconds_per_case: 146.7
  total_cost: 0.0000

- dirname: 2024-12-25-13-31-51--deepseekv3preview-diff2
  test_cases: 225
  model: DeepSeek Chat V3 (prev)
  edit_format: diff
  commit_hash: 0a23c4a-dirty
  pass_rate_1: 22.7
  pass_rate_2: 48.4
  pass_num_1: 51
  pass_num_2: 109
  percent_cases_well_formed: 98.7
  error_outputs: 7
  num_malformed_responses: 7
  num_with_malformed_responses: 3
  user_asks: 19
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 8
  total_tests: 225
  command: aider --model deepseek/deepseek-chat
  date: 2024-12-25
  versions: 0.69.2.dev
  seconds_per_case: 34.8
  total_cost: 0.3369

- dirname: 2024-12-26-00-55-20--Qwen2.5-Coder-32B-Instruct
  test_cases: 225
  model: Qwen2.5-Coder-32B-Instruct
  edit_format: whole
  commit_hash: b51768b0
  pass_rate_1: 4.9
  pass_rate_2: 16.4
  pass_num_1: 11
  pass_num_2: 37
  percent_cases_well_formed: 99.6
  error_outputs: 1
  num_malformed_responses: 1
  num_with_malformed_responses: 1
  user_asks: 33
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 6
  total_tests: 225
  command: aider --model openai/Qwen2.5-Coder-32B-Instruct
  date: 2024-12-26
  versions: 0.69.2.dev
  seconds_per_case: 42.0
  total_cost: 0.0000

- dirname: 2025-01-13-18-17-25--codestral-whole2
  test_cases: 225
  model: Codestral 25.01
  edit_format: whole
  commit_hash: 0cba898-dirty
  pass_rate_1: 4.0
  pass_rate_2: 11.1
  pass_num_1: 9
  pass_num_2: 25
  percent_cases_well_formed: 100.0
  error_outputs: 0
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 47
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 4
  total_tests: 225
  command: aider --model mistral/codestral-latest
  date: 2025-01-13
  versions: 0.71.2.dev
  seconds_per_case: 9.3
  total_cost: 1.9834

- dirname: 2025-01-20-19-11-38--ds-turns-upd-cur-msgs-fix-with-summarizer
  test_cases: 225
  model: DeepSeek R1
  edit_format: diff
  commit_hash: 5650697-dirty
  pass_rate_1: 26.7
  pass_rate_2: 56.9
  pass_num_1: 60
  pass_num_2: 128
  percent_cases_well_formed: 96.9
  error_outputs: 8
  num_malformed_responses: 7
  num_with_malformed_responses: 7
  user_asks: 15
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 5
  total_tests: 225
  command: aider --model deepseek/deepseek-reasoner
  date: 2025-01-20
  versions: 0.71.2.dev
  seconds_per_case: 113.7
  total_cost: 5.4193

- dirname: 2025-01-23-19-14-48--r1-architect-sonnet
  test_cases: 225
  model: DeepSeek R1 + claude-3-5-sonnet-20241022
  edit_format: architect
  commit_hash: 05a77c7
  editor_model: claude-3-5-sonnet-20241022
  editor_edit_format: editor-diff
  pass_rate_1: 27.1
  pass_rate_2: 64.0
  pass_num_1: 61
  pass_num_2: 144
  percent_cases_well_formed: 100.0
  error_outputs: 2
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 392
  lazy_comments: 6
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 5
  total_tests: 225
  command: aider --architect --model r1 --editor-model sonnet
  date: 2025-01-23
  versions: 0.72.3.dev
  seconds_per_case: 251.6
  total_cost: 13.2933

- dirname: 2025-01-28-16-00-03--qwen-max-2025-01-25-polyglot-diff
  test_cases: 225
  model: qwen-max-2025-01-25
  edit_format: diff
  commit_hash: ae7d459
  pass_rate_1: 9.3
  pass_rate_2: 21.8
  pass_num_1: 21
  pass_num_2: 49
  percent_cases_well_formed: 90.2
  error_outputs: 46
  num_malformed_responses: 44
  num_with_malformed_responses: 22
  user_asks: 23
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 9
  total_tests: 225
  command: OPENAI_API_BASE=https://dashscope-intl.aliyuncs.com/compatible-mode/v1 aider --model openai/qwen-max-2025-01-25
  date: 2025-01-28
  versions: 0.72.4.dev
  seconds_per_case: 39.5

- dirname: 2025-01-31-20-27-46--o3-mini-diff2
  test_cases: 225
  model: o3-mini (medium)
  edit_format: diff
  commit_hash: 2fb517b-dirty
  pass_rate_1: 19.1
  pass_rate_2: 53.8
  pass_num_1: 43
  pass_num_2: 121
  percent_cases_well_formed: 95.1
  error_outputs: 28
  num_malformed_responses: 28
  num_with_malformed_responses: 11
  user_asks: 17
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 2
  total_tests: 225
  command: aider --model o3-mini
  date: 2025-01-31
  versions: 0.72.4.dev
  seconds_per_case: 47.2
  total_cost: 8.8599

- dirname: 2025-01-31-20-42-47--o3-mini-diff-high
  test_cases: 224
  model: o3-mini (high)
  edit_format: diff
  commit_hash: b0d58d1-dirty
  pass_rate_1: 21.0
  pass_rate_2: 60.4
  pass_num_1: 47
  pass_num_2: 136
  percent_cases_well_formed: 93.3
  error_outputs: 26
  num_malformed_responses: 24
  num_with_malformed_responses: 15
  user_asks: 19
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 7
  total_tests: 225
  command: aider --model o3-mini --reasoning-effort high
  date: 2025-01-31
  versions: 0.72.4.dev
  seconds_per_case: 124.6
  total_cost: 18.1584

- dirname: 2025-01-21-22-51-49--gemini-2.0-flash-thinking-exp-01-21-polyglot-diff
  test_cases: 225
  model: gemini-2.0-flash-thinking-exp-01-21
  edit_format: diff
  commit_hash: 843720a
  pass_rate_1: 5.8
  pass_rate_2: 18.2
  pass_num_1: 13
  pass_num_2: 41
  percent_cases_well_formed: 77.8
  error_outputs: 182
  num_malformed_responses: 180
  num_with_malformed_responses: 50
  user_asks: 26
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 2
  test_timeouts: 7
  total_tests: 225
  command: aider --model gemini/gemini-2.0-flash-thinking-exp-01-21
  date: 2025-01-21
  versions: 0.72.2.dev
  seconds_per_case: 24.2
  total_cost: 0.0000

- dirname: 2025-02-15-19-51-22--chatgpt4o-feb15-diff
  test_cases: 223
  model: chatgpt-4o-latest (2025-02-15)
  edit_format: diff
  commit_hash: 108ce18-dirty
  pass_rate_1: 9.0
  pass_rate_2: 27.1
  pass_num_1: 20
  pass_num_2: 61
  percent_cases_well_formed: 93.3
  error_outputs: 66
  num_malformed_responses: 21
  num_with_malformed_responses: 15
  user_asks: 57
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 2
  total_tests: 225
  command: aider --model chatgpt-4o-latest
  date: 2025-02-15
  versions: 0.74.3.dev
  seconds_per_case: 12.4
  total_cost: 14.3703

- dirname: 2025-02-24-19-54-07--sonnet37-diff
  test_cases: 225
  model: claude-3-7-sonnet-20250219 (no thinking)
  edit_format: diff
  commit_hash: 75e9ee6
  pass_rate_1: 24.4
  pass_rate_2: 60.4
  pass_num_1: 55
  pass_num_2: 136
  percent_cases_well_formed: 93.3
  error_outputs: 16
  num_malformed_responses: 16
  num_with_malformed_responses: 15
  user_asks: 12
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 0
  total_tests: 225
  command: aider --model sonnet
  date: 2025-02-24
  versions: 0.74.4.dev
  seconds_per_case: 28.3
  total_cost: 17.7191

- dirname: 2025-02-24-21-47-23--sonnet37-diff-think-32k-64k
  test_cases: 225
  model: claude-3-7-sonnet-20250219 (32k thinking tokens)
  edit_format: diff
  commit_hash: 60d11a6, 93edbda
  pass_rate_1: 29.3
  pass_rate_2: 64.9
  pass_num_1: 66
  pass_num_2: 146
  percent_cases_well_formed: 97.8
  error_outputs: 66
  num_malformed_responses: 5
  num_with_malformed_responses: 5
  user_asks: 5
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 1
  total_tests: 225
  command: "aider --model anthropic/claude-3-7-sonnet-20250219 --thinking-tokens 32k"
  date: 2025-02-24
  versions: 0.75.1.dev
  seconds_per_case: 105.2
  total_cost: 36.8343

- dirname: 2025-02-27-20-26-15--gpt45-diff3
  test_cases: 224
  model: gpt-4.5-preview
  edit_format: diff
  commit_hash: b462e55-dirty
  pass_rate_1: 22.3
  pass_rate_2: 44.9
  pass_num_1: 50
  pass_num_2: 101
  percent_cases_well_formed: 97.3
  error_outputs: 10
  num_malformed_responses: 8
  num_with_malformed_responses: 6
  user_asks: 15
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 2
  total_tests: 225
  command: aider --model openai/gpt-4.5-preview
  date: 2025-02-27
  versions: 0.75.2.dev
  seconds_per_case: 113.5
  total_cost: 183.1802

- dirname: 2025-03-06-17-40-24--qwq32b-diff-temp-topp-ex-sys-remind-user-for-real
  test_cases: 225
  model: QwQ-32B
  edit_format: diff
  commit_hash: 51d118f-dirty
  pass_rate_1: 8.0
  pass_rate_2: 20.9
  pass_num_1: 18
  pass_num_2: 47
  percent_cases_well_formed: 67.6
  error_outputs: 145
  num_malformed_responses: 143
  num_with_malformed_responses: 73
  user_asks: 17
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 4
  total_tests: 225
  command: aider --model fireworks_ai/accounts/fireworks/models/qwq-32b
  date: 2025-03-06
  versions: 0.75.3.dev
  seconds_per_case: 228.6
  total_cost: 0.0000

- dirname: 2025-03-07-15-11-27--qwq32b-arch-temp-topp-again
  test_cases: 225
  model: QwQ-32B + Qwen 2.5 Coder Instruct
  edit_format: architect
  commit_hash: 52162a5
  editor_model: fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct
  editor_edit_format: editor-diff
  pass_rate_1: 9.8
  pass_rate_2: 26.2
  pass_num_1: 22
  pass_num_2: 59
  percent_cases_well_formed: 100.0
  error_outputs: 122
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 489
  lazy_comments: 8
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 2
  total_tests: 225
  command: aider --model fireworks_ai/accounts/fireworks/models/qwq-32b --architect
  date: 2025-03-07
  versions: 0.75.3.dev
  seconds_per_case: 137.4
  total_cost: 0

- dirname: 2025-03-14-23-40-00--cmda-quality-whole2
  test_cases: 225
  model: command-a-03-2025-quality
  edit_format: whole
  commit_hash: a1aa63f
  pass_rate_1: 2.2
  pass_rate_2: 12.0
  pass_num_1: 5
  pass_num_2: 27
  percent_cases_well_formed: 99.6
  error_outputs: 2
  num_malformed_responses: 1
  num_with_malformed_responses: 1
  user_asks: 215
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 4
  total_tests: 225
  command: OPENAI_API_BASE=https://api.cohere.ai/compatibility/v1 aider --model openai/command-a-03-2025-quality
  date: 2025-03-14
  versions: 0.77.1.dev
  seconds_per_case: 85.1
  total_cost: 0.0000

- dirname: 2025-03-15-01-21-24--gemma3-27b-or
  test_cases: 225
  model: gemma-3-27b-it
  edit_format: whole
  commit_hash: fd21f51-dirty
  pass_rate_1: 1.8
  pass_rate_2: 4.9
  pass_num_1: 4
  pass_num_2: 11
  percent_cases_well_formed: 100.0
  error_outputs: 3
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 181
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 3
  total_tests: 225
  command: aider --model openrouter/google/gemma-3-27b-it
  date: 2025-03-15
  versions: 0.77.1.dev
  seconds_per_case: 79.7
  total_cost: 0.0000

- dirname: 2025-03-24-15-41-33--deepseek-v3-0324-polyglot-diff
  test_cases: 225
  model: DeepSeek V3 (0324)
  edit_format: diff
  commit_hash: 502b863
  pass_rate_1: 28.0
  pass_rate_2: 55.1
  pass_num_1: 63
  pass_num_2: 124
  percent_cases_well_formed: 99.6
  error_outputs: 32
  num_malformed_responses: 1
  num_with_malformed_responses: 1
  user_asks: 96
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 2
  test_timeouts: 4
  total_tests: 225
  command: aider --model deepseek/deepseek-chat
  date: 2025-03-24
  versions: 0.78.1.dev
  seconds_per_case: 290.0
  total_cost: 1.1164

- dirname: 2025-04-12-04-55-50--gemini-25-pro-diff-fenced
  test_cases: 225
  model: Gemini 2.5 Pro Preview 03-25
  edit_format: diff-fenced
  commit_hash: 0282574
  pass_rate_1: 40.9
  pass_rate_2: 72.9
  pass_num_1: 92
  pass_num_2: 164
  percent_cases_well_formed: 92.4
  error_outputs: 21
  num_malformed_responses: 21
  num_with_malformed_responses: 17
  user_asks: 69
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 2
  total_tests: 225
  command: aider --model gemini/gemini-2.5-pro-preview-03-25
  date: 2025-04-12
  versions: 0.81.3.dev
  seconds_per_case: 45.3
  total_cost: 0 # incorrect: 6.3174

- dirname: 2025-03-29-05-24-55--chatgpt4o-mar28-diff
  test_cases: 225
  model: chatgpt-4o-latest (2025-03-29)
  edit_format: diff
  commit_hash: 0decbad
  pass_rate_1: 16.4
  pass_rate_2: 45.3
  pass_num_1: 37
  pass_num_2: 102
  percent_cases_well_formed: 64.4
  error_outputs: 85
  num_malformed_responses: 85
  num_with_malformed_responses: 80
  user_asks: 174
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 4
  total_tests: 225
  command: aider --model chatgpt-4o-latest
  date: 2025-03-29
  versions: 0.79.3.dev
  seconds_per_case: 10.3
  total_cost: 19.7416

- dirname: 2025-04-04-02-57-25--qalpha-diff-exsys
  test_cases: 225
  model: Quasar Alpha
  edit_format: diff
  commit_hash: 8a34a6c-dirty
  pass_rate_1: 21.8
  pass_rate_2: 54.7
  pass_num_1: 49
  pass_num_2: 123
  percent_cases_well_formed: 98.2
  error_outputs: 4
  num_malformed_responses: 4
  num_with_malformed_responses: 4
  user_asks: 187
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 4
  total_tests: 225
  command: aider --model openrouter/openrouter/quasar-alpha
  date: 2025-04-04
  versions: 0.80.5.dev
  seconds_per_case: 14.8
  total_cost: 0.0000

- dirname: 2025-04-06-08-39-52--llama-4-maverick-17b-128e-instruct-polyglot-whole
  test_cases: 225
  model: Llama 4 Maverick
  edit_format: whole
  commit_hash: 9445a31
  pass_rate_1: 4.4
  pass_rate_2: 15.6
  pass_num_1: 10
  pass_num_2: 35
  percent_cases_well_formed: 99.1
  error_outputs: 12
  num_malformed_responses: 2
  num_with_malformed_responses: 2
  user_asks: 248
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 4
  total_tests: 225
  command: aider --model nvidia_nim/meta/llama-4-maverick-17b-128e-instruct
  date: 2025-04-06
  versions: 0.81.2.dev
  seconds_per_case: 20.5
  total_cost: 0.0000

- dirname: 2025-04-10-04-21-31--grok3-diff-exuser
  test_cases: 225
  model: Grok 3 Beta
  edit_format: diff
  commit_hash: 2dd40fc-dirty
  pass_rate_1: 22.2
  pass_rate_2: 53.3
  pass_num_1: 50
  pass_num_2: 120
  percent_cases_well_formed: 99.6
  error_outputs: 1
  num_malformed_responses: 1
  num_with_malformed_responses: 1
  user_asks: 68
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 2
  total_tests: 225
  command: aider --model openrouter/x-ai/grok-3-beta
  date: 2025-04-10
  versions: 0.81.2.dev
  seconds_per_case: 15.3
  total_cost: 11.0338

- dirname: 2025-04-10-18-47-24--grok3-mini-whole-exuser
  test_cases: 225
  model: Grok 3 Mini Beta (low)
  edit_format: whole
  commit_hash: 14ffe77-dirty
  pass_rate_1: 11.1
  pass_rate_2: 34.7
  pass_num_1: 25
  pass_num_2: 78
  percent_cases_well_formed: 100.0
  error_outputs: 3
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 73
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 5
  total_tests: 225
  command: aider --model openrouter/x-ai/grok-3-mini-beta
  date: 2025-04-10
  versions: 0.81.2.dev
  seconds_per_case: 35.1
  total_cost: 0.7856

- dirname: 2025-04-10-23-59-02--xai-grok3-mini-whole-high
  test_cases: 225
  model: Grok 3 Mini Beta (high)
  edit_format: whole
  commit_hash: 8ee33da-dirty
  pass_rate_1: 17.3
  pass_rate_2: 49.3
  pass_num_1: 39
  pass_num_2: 111
  percent_cases_well_formed: 99.6
  error_outputs: 1
  num_malformed_responses: 1
  num_with_malformed_responses: 1
  user_asks: 64
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 0
  total_tests: 225
  command: aider --model xai/grok-3-mini-beta --reasoning-effort high
  date: 2025-04-10
  versions: 0.81.3.dev
  seconds_per_case: 79.1
  total_cost: 0.7346
  
- dirname: 2025-04-10-19-02-44--oalpha-diff-exsys
  test_cases: 225
  model: Optimus Alpha
  edit_format: diff
  commit_hash: 532bc45-dirty
  pass_rate_1: 21.3
  pass_rate_2: 52.9
  pass_num_1: 48
  pass_num_2: 119
  percent_cases_well_formed: 97.3
  error_outputs: 7
  num_malformed_responses: 6
  num_with_malformed_responses: 6
  user_asks: 182
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 3
  total_tests: 225
  command: aider --model openrouter/openrouter/optimus-alpha
  date: 2025-04-10
  versions: 0.81.2.dev
  seconds_per_case: 18.4
  total_cost: 0.0000

- dirname: 2025-04-14-21-05-54--gpt41-diff-exuser
  test_cases: 225
  model: gpt-4.1
  edit_format: diff
  commit_hash: 7a87db5-dirty
  pass_rate_1: 20.0
  pass_rate_2: 52.4
  pass_num_1: 45
  pass_num_2: 118
  percent_cases_well_formed: 98.2
  error_outputs: 6
  num_malformed_responses: 5
  num_with_malformed_responses: 4
  user_asks: 171
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 5
  total_tests: 225
  command: aider --model gpt-4.1
  date: 2025-04-14
  versions: 0.81.4.dev
  seconds_per_case: 20.5
  total_cost: 9.8556

- dirname: 2025-04-14-21-27-53--gpt41mini-diff
  test_cases: 225
  model: gpt-4.1-mini
  edit_format: diff
  commit_hash: ffb743e-dirty
  pass_rate_1: 11.1
  pass_rate_2: 32.4
  pass_num_1: 25
  pass_num_2: 73
  percent_cases_well_formed: 92.4
  error_outputs: 64
  num_malformed_responses: 62
  num_with_malformed_responses: 17
  user_asks: 159
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 2
  test_timeouts: 2
  total_tests: 225
  command: aider --model gpt-4.1-mini
  date: 2025-04-14
  versions: 0.81.4.dev
  seconds_per_case: 19.5
  total_cost: 1.9918

- dirname: 2025-04-14-22-46-01--gpt41nano-diff
  test_cases: 225
  model: gpt-4.1-nano
  edit_format: whole
  commit_hash: 71d1591-dirty
  pass_rate_1: 3.1
  pass_rate_2: 8.9
  pass_num_1: 7
  pass_num_2: 20
  percent_cases_well_formed: 94.2
  error_outputs: 20
  num_malformed_responses: 20
  num_with_malformed_responses: 13
  user_asks: 316
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 8
  total_tests: 225
  command: aider --model gpt-4.1-nano
  date: 2025-04-14
  versions: 0.81.4.dev
  seconds_per_case: 12.0
  total_cost: 0.4281

- dirname: 2025-04-16-22-01-58--o4-mini-high-diff-exsys
  test_cases: 225
  model: o4-mini (high)
  edit_format: diff
  commit_hash: b66901f-dirty
  pass_rate_1: 19.6
  pass_rate_2: 72.0
  pass_num_1: 44
  pass_num_2: 162
  percent_cases_well_formed: 90.7
  error_outputs: 26
  num_malformed_responses: 24
  num_with_malformed_responses: 21
  user_asks: 66
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 2
  total_tests: 225
  command: aider --model o4-mini
  date: 2025-04-16
  versions: 0.82.1.dev
  seconds_per_case: 176.5
  total_cost: 19.6399

- dirname: 2025-04-19-14-43-04--o4-mini-patch
  test_cases: 225
  model: openhands-lm-32b-v0.1
  edit_format: whole
  commit_hash: c08336f
  pass_rate_1: 4.0
  pass_rate_2: 10.2
  pass_num_1: 9
  pass_num_2: 23
  percent_cases_well_formed: 95.1
  error_outputs: 55
  num_malformed_responses: 41
  num_with_malformed_responses: 11
  user_asks: 166
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 11
  total_tests: 225
  command: aider --model openrouter/all-hands/openhands-lm-32b-v0.1
  date: 2025-04-19
  versions: 0.82.2.dev
  seconds_per_case: 195.6
  total_cost: 0.0000

- dirname: 2025-04-20-19-54-31--flash25-diff-no-think
  test_cases: 225
  model: gemini-2.5-flash-preview-04-17 (default)
  edit_format: diff
  commit_hash: 7fcce5d-dirty
  pass_rate_1: 21.8
  pass_rate_2: 47.1
  pass_num_1: 49
  pass_num_2: 106
  percent_cases_well_formed: 85.3
  error_outputs: 60
  num_malformed_responses: 55
  num_with_malformed_responses: 33
  user_asks: 82
  lazy_comments: 1
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 5
  test_timeouts: 4
  total_tests: 225
  command: aider --model gemini/gemini-2.5-flash-preview-04-17
  date: 2025-04-20
  versions: 0.82.3.dev
  seconds_per_case: 50.1
  total_cost: 1.8451

- dirname: 2025-05-07-19-32-40--gemini0506-diff-fenced-completion_cost
  test_cases: 225
  model: Gemini 2.5 Pro Preview 05-06
  edit_format: diff-fenced
  commit_hash: 3b08327-dirty
  pass_rate_1: 36.4
  pass_rate_2: 76.9
  pass_num_1: 82
  pass_num_2: 173
  percent_cases_well_formed: 97.3
  error_outputs: 15
  num_malformed_responses: 7
  num_with_malformed_responses: 6
  user_asks: 105
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 2
  total_tests: 225
  command: aider --model gemini/gemini-2.5-pro-preview-05-06
  date: 2025-05-07
  versions: 0.82.4.dev
  seconds_per_case: 165.3
  total_cost: 37.4104

- dirname: 2025-05-08-03-20-24--qwen3-32b-default
  test_cases: 225
  model: Qwen3 32B
  edit_format: diff
  commit_hash: aaacee5-dirty, aeaf259
  pass_rate_1: 14.2
  pass_rate_2: 40.0
  pass_num_1: 32
  pass_num_2: 90
  percent_cases_well_formed: 83.6
  error_outputs: 119
  num_malformed_responses: 50
  num_with_malformed_responses: 37
  user_asks: 97
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 12
  prompt_tokens: 317591
  completion_tokens: 120418
  test_timeouts: 5
  total_tests: 225
  command: aider --model openrouter/qwen/qwen3-32b
  date: 2025-05-08
  versions: 0.82.4.dev
  seconds_per_case: 372.2
  total_cost: 0.7603

- dirname: 2025-05-09-17-02-02--qwen3-235b-a22b.unthink_16k_diff
  test_cases: 225
  model: Qwen3 235B A22B diff, no think, Alibaba API
  edit_format: diff
  commit_hash: 91d7fbd-dirty
  pass_rate_1: 28.9
  pass_rate_2: 59.6
  pass_num_1: 65
  pass_num_2: 134
  percent_cases_well_formed: 92.9
  error_outputs: 22
  num_malformed_responses: 22
  num_with_malformed_responses: 16
  user_asks: 111
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 2816192
  completion_tokens: 342062
  test_timeouts: 1
  total_tests: 225
  command: aider --model openai/qwen3-235b-a22b
  date: 2025-05-09
  versions: 0.82.4.dev
  seconds_per_case: 45.4
  total_cost: 0.0000

- dirname: 2025-05-24-21-17-54--sonnet4-diff-exuser
  test_cases: 225
  model: claude-sonnet-4-20250514 (no thinking)
  edit_format: diff
  commit_hash: ef3f8bb-dirty
  pass_rate_1: 20.4
  pass_rate_2: 56.4
  pass_num_1: 46
  pass_num_2: 127
  percent_cases_well_formed: 98.2
  error_outputs: 6
  num_malformed_responses: 4
  num_with_malformed_responses: 4
  user_asks: 129
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  prompt_tokens: 3460663
  completion_tokens: 433373
  test_timeouts: 7
  total_tests: 225
  command: aider --model claude-sonnet-4-20250514
  date: 2025-05-24
  versions: 0.83.3.dev
  seconds_per_case: 29.8
  total_cost: 15.8155
  
- dirname: 2025-05-24-22-10-36--sonnet4-diff-exuser-think32k
  test_cases: 225
  model: claude-sonnet-4-20250514 (32k thinking)
  edit_format: diff
  commit_hash: e3cb907
  thinking_tokens: 32000
  pass_rate_1: 25.8
  pass_rate_2: 61.3
  pass_num_1: 58
  pass_num_2: 138
  percent_cases_well_formed: 97.3
  error_outputs: 10
  num_malformed_responses: 10
  num_with_malformed_responses: 6
  user_asks: 111
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 2863068
  completion_tokens: 1271074
  test_timeouts: 6
  total_tests: 225
  command: aider --model claude-sonnet-4-20250514
  date: 2025-05-24
  versions: 0.83.3.dev
  seconds_per_case: 79.9
  total_cost: 26.5755

- dirname: 2025-05-25-19-57-20--opus4-diff-exuser
  test_cases: 225
  model: claude-opus-4-20250514 (no think)
  edit_format: diff
  commit_hash: 9ef3211
  pass_rate_1: 32.9
  pass_rate_2: 70.7
  pass_num_1: 74
  pass_num_2: 159
  percent_cases_well_formed: 98.7
  error_outputs: 3
  num_malformed_responses: 3
  num_with_malformed_responses: 3
  user_asks: 105
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 2671437
  completion_tokens: 380717
  test_timeouts: 3
  total_tests: 225
  command: aider --model claude-opus-4-20250514
  date: 2025-05-25
  versions: 0.83.3.dev
  seconds_per_case: 42.5
  total_cost: 68.6253

- dirname: 2025-05-25-20-40-51--opus4-diff-exuser
  test_cases: 225
  model: claude-opus-4-20250514 (32k thinking)
  edit_format: diff
  commit_hash: 9ef3211
  thinking_tokens: 32000
  pass_rate_1: 37.3
  pass_rate_2: 72.0
  pass_num_1: 84
  pass_num_2: 162
  percent_cases_well_formed: 97.3
  error_outputs: 10
  num_malformed_responses: 6
  num_with_malformed_responses: 6
  user_asks: 97
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 2567514
  completion_tokens: 363142
  test_timeouts: 4
  total_tests: 225
  command: aider --model claude-opus-4-20250514
  date: 2025-05-25
  versions: 0.83.3.dev
  seconds_per_case: 44.1
  total_cost: 65.7484

- dirname: 2025-05-26-15-56-31--flash25-05-20-24k-think # dirname is misleading
  test_cases: 225
  model: gemini-2.5-flash-preview-05-20 (no think)
  edit_format: diff
  commit_hash: 214b811-dirty
  thinking_tokens: 0 # <-- no thinking
  pass_rate_1: 20.9
  pass_rate_2: 44.0
  pass_num_1: 47
  pass_num_2: 99
  percent_cases_well_formed: 93.8
  error_outputs: 16
  num_malformed_responses: 16
  num_with_malformed_responses: 14
  user_asks: 79
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 5512458
  completion_tokens: 514145
  test_timeouts: 4
  total_tests: 225
  command: aider --model gemini/gemini-2.5-flash-preview-05-20
  date: 2025-05-26
  versions: 0.83.3.dev
  seconds_per_case: 12.2
  total_cost: 1.1354
  
- dirname: 2025-05-25-22-58-44--flash25-05-20-24k-think
  test_cases: 225
  model: gemini-2.5-flash-preview-05-20 (24k think)
  edit_format: diff
  commit_hash: a8568c3-dirty
  thinking_tokens: 24576
  pass_rate_1: 26.2
  pass_rate_2: 55.1
  pass_num_1: 59
  pass_num_2: 124
  percent_cases_well_formed: 95.6
  error_outputs: 15
  num_malformed_responses: 15
  num_with_malformed_responses: 10
  user_asks: 101
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 3666792
  completion_tokens: 2703162
  test_timeouts: 4
  total_tests: 225
  command: aider --model gemini/gemini-2.5-flash-preview-05-20
  date: 2025-05-25
  versions: 0.83.3.dev
  seconds_per_case: 53.9
  total_cost: 8.5625

- dirname: 2025-06-06-18-38-56--gemini0605-diff-fenced
  test_cases: 225
  model: gemini-2.5-pro-preview-06-05 (default think)
  edit_format: diff-fenced
  commit_hash: 4c161f9-dirty
  pass_rate_1: 44.9
  pass_rate_2: 79.1
  pass_num_1: 101
  pass_num_2: 178
  percent_cases_well_formed: 100.0
  error_outputs: 4
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 105
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 4
  prompt_tokens: 2751296
  completion_tokens: 4142197
  test_timeouts: 1
  total_tests: 225
  command: aider --model gemini/gemini-2.5-pro-preview-06-05
  date: 2025-06-06
  versions: 0.84.1.dev
  seconds_per_case: 175.2
  total_cost: 45.5961

- dirname: 2025-06-06-16-36-21--gemini0605-32k-think-diff-fenced
  test_cases: 225
  model: gemini-2.5-pro-preview-06-05 (32k think)
  edit_format: diff-fenced
  commit_hash: f827f22
  thinking_tokens: 32768
  pass_rate_1: 46.2
  pass_rate_2: 83.1
  pass_num_1: 104
  pass_num_2: 187
  percent_cases_well_formed: 99.6
  error_outputs: 1
  num_malformed_responses: 1
  num_with_malformed_responses: 1
  user_asks: 112
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 2719961
  completion_tokens: 4648227
  test_timeouts: 0
  total_tests: 225
  command: aider --model gemini/gemini-2.5-pro-preview-06-05 --thinking-tokens 32k
  date: 2025-06-06
  versions: 0.84.1.dev
  seconds_per_case: 200.3
  total_cost: 49.8822

- dirname: 2025-06-06-16-47-07--r1-diff
  test_cases: 224
  model: DeepSeek R1 (0528)
  edit_format: diff
  commit_hash: 4c161f9-dirty
  pass_rate_1: 34.4
  pass_rate_2: 71.4
  pass_num_1: 77
  pass_num_2: 160
  percent_cases_well_formed: 94.6
  error_outputs: 28
  num_malformed_responses: 15
  num_with_malformed_responses: 12
  user_asks: 105
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 2644169
  completion_tokens: 1842168
  test_timeouts: 2
  total_tests: 225
  command: aider --model deepseek/deepseek-reasoner
  date: 2025-06-06
  versions: 0.84.1.dev
  seconds_per_case: 716.6
  total_cost: 4.8016

- dirname: 2025-06-25-21-04-24--o3-price-reduction-high
  test_cases: 225
  model: o3 (high)
  edit_format: diff
  commit_hash: c48fea6
  reasoning_effort: high
  pass_rate_1: 40.0
  pass_rate_2: 81.3
  pass_num_1: 90
  pass_num_2: 183
  percent_cases_well_formed: 94.7
  error_outputs: 25
  num_malformed_responses: 23
  num_with_malformed_responses: 12
  user_asks: 116
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  prompt_tokens: 3148932
  completion_tokens: 2047615
  test_timeouts: 2
  total_tests: 225
  command: aider --model o3 --reasoning-effort high
  date: 2025-06-25
  versions: 0.84.1.dev
  seconds_per_case: 197.3
  total_cost: 21.2259

- dirname: 2025-06-25-20-30-16--o3-price-reduction
  test_cases: 225
  model: o3
  edit_format: diff
  commit_hash: c48fea6
  pass_rate_1: 40.9
  pass_rate_2: 76.9
  pass_num_1: 92
  pass_num_2: 173
  percent_cases_well_formed: 93.8
  error_outputs: 22
  num_malformed_responses: 22
  num_with_malformed_responses: 14
  user_asks: 108
  lazy_comments: 2
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 2893189
  completion_tokens: 1154767
  test_timeouts: 1
  total_tests: 225
  command: aider --model o3
  date: 2025-06-25
  versions: 0.84.1.dev
  seconds_per_case: 101.7
  total_cost: 13.7517

- dirname: 2025-06-27-23-53-57--o3-mini-high-diff-arch
  test_cases: 224
  model: o3 (high) + gpt-4.1
  edit_format: architect
  commit_hash: 4f4f00f-dirty
  editor_model: gpt-4.1
  editor_edit_format: editor-diff
  reasoning_effort: high
  pass_rate_1: 34.8
  pass_rate_2: 78.2
  pass_num_1: 78
  pass_num_2: 176
  percent_cases_well_formed: 100.0
  error_outputs: 18
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 172
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  prompt_tokens: 1306877
  completion_tokens: 1327154
  test_timeouts: 1
  total_tests: 225
  command: aider --model o3
  date: 2025-06-27
  versions: 0.85.1.dev
  seconds_per_case: 121.8
  total_cost: 17.5518

- dirname: 2025-06-28-00-38-18--o3-pro-high
  test_cases: 225
  model: o3-pro (high)
  edit_format: diff
  commit_hash: 5318380
  reasoning_effort: high
  pass_rate_1: 43.6
  pass_rate_2: 84.9
  pass_num_1: 98
  pass_num_2: 191
  percent_cases_well_formed: 97.8
  error_outputs: 20
  num_malformed_responses: 8
  num_with_malformed_responses: 5
  user_asks: 100
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 2372636
  completion_tokens: 1235902
  test_timeouts: 1
  total_tests: 225
  command: aider --model o3-pro
  date: 2025-06-28
  versions: 0.85.1.dev
  seconds_per_case: 449.0
  total_cost: 146.3249

- dirname: 2025-07-11-19-37-40--xai-or-grok4-high
  test_cases: 225
  model: grok-4 (high)
  edit_format: diff
  commit_hash: f7870b6-dirty
  reasoning_effort: high
  pass_rate_1: 40.9
  pass_rate_2: 79.6
  pass_num_1: 92
  pass_num_2: 179
  percent_cases_well_formed: 97.3
  error_outputs: 11
  num_malformed_responses: 8
  num_with_malformed_responses: 6
  user_asks: 133
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 2815347
  completion_tokens: 3411480
  test_timeouts: 0
  total_tests: 225
  command: aider --model openrouter/x-ai/grok-4
  date: 2025-07-11
  versions: 0.85.2.dev
  seconds_per_case: 403.2
  total_cost: 59.6182

- dirname: 2025-07-17-17-41-54--kimi-k2-diff-or-pricing
  test_cases: 225
  model: Kimi K2
  edit_format: diff
  commit_hash: 915ebff-dirty
  pass_rate_1: 20.4
  pass_rate_2: 59.1
  pass_num_1: 46
  pass_num_2: 133
  percent_cases_well_formed: 92.9
  error_outputs: 19
  num_malformed_responses: 19
  num_with_malformed_responses: 16
  user_asks: 61
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 2355141
  completion_tokens: 363846
  test_timeouts: 4
  total_tests: 225
  command: aider --model openrouter/moonshotai/kimi-k2
  date: 2025-07-17
  versions: 0.85.3.dev
  seconds_per_case: 67.6
  total_cost: 1.2357

- dirname: 2025-08-06-04-54-48--gpt-oss-120b-high-polyglot
  test_cases: 225
  model: gpt-oss-120b (high)
  edit_format: diff
  commit_hash: 1af0e59
  pass_rate_1: 13.8
  pass_rate_2: 41.8
  pass_num_1: 31
  pass_num_2: 94
  percent_cases_well_formed: 79.1
  error_outputs: 95
  num_malformed_responses: 77
  num_with_malformed_responses: 47
  user_asks: 142
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 3123768
  completion_tokens: 856495
  test_timeouts: 4
  total_tests: 225
  command: aider --model openrouter/openai/gpt-oss-120b --reasoning-effort high
  date: 2025-08-06
  versions: 0.85.3.dev
  seconds_per_case: 35.5
  total_cost: 0.7406

- dirname: 2025-08-23-15-47-21--gpt-5-high
  test_cases: 225
  model: gpt-5 (high)
  edit_format: diff
  commit_hash: 32faf82
  reasoning_effort: high
  pass_rate_1: 52.0
  pass_rate_2: 88.0
  pass_num_1: 117
  pass_num_2: 198
  percent_cases_well_formed: 91.6
  error_outputs: 23
  num_malformed_responses: 22
  num_with_malformed_responses: 19
  user_asks: 96
  lazy_comments: 3
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 2675561
  completion_tokens: 2623429
  test_timeouts: 3
  total_tests: 225
  command: aider --model openai/gpt-5
  date: 2025-08-23
  versions: 0.86.2.dev
  seconds_per_case: 194.0
  total_cost: 29.0829

- dirname: 2025-08-25-13-23-27--gpt-5-medium
  test_cases: 225
  model: gpt-5 (medium)
  edit_format: diff
  commit_hash: 32faf82
  reasoning_effort: medium
  pass_rate_1: 49.8
  pass_rate_2: 86.7
  pass_num_1: 112
  pass_num_2: 195
  percent_cases_well_formed: 88.4
  error_outputs: 40
  num_malformed_responses: 40
  num_with_malformed_responses: 26
  user_asks: 102
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 2827261
  completion_tokens: 1468799
  test_timeouts: 0
  total_tests: 225
  command: aider --model openai/gpt-5
  date: 2025-08-25
  versions: 0.86.2.dev
  seconds_per_case: 118.7
  total_cost: 17.6930

- dirname: 2025-08-25-14-16-37--gpt-5-low
  test_cases: 225
  model: gpt-5 (low)
  edit_format: diff
  commit_hash: 32faf82
  reasoning_effort: low
  pass_rate_1: 43.1
  pass_rate_2: 81.3
  pass_num_1: 97
  pass_num_2: 183
  percent_cases_well_formed: 86.7
  error_outputs: 46
  num_malformed_responses: 46
  num_with_malformed_responses: 30
  user_asks: 113
  lazy_comments: 1
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  prompt_tokens: 2534059
  completion_tokens: 779568
  test_timeouts: 1
  total_tests: 225
  command: aider --model openai/gpt-5
  date: 2025-08-25
  versions: 0.86.2.dev
  seconds_per_case: 62.4
  total_cost: 10.3713

- dirname: 2025-10-03-09-45-34--deepseek-v3.2-reasoner
  test_cases: 225
  model: DeepSeek-V3.2-Exp (Reasoner)
  edit_format: diff
  commit_hash: cbb5376
  pass_rate_1: 39.6
  pass_rate_2: 74.2
  pass_num_1: 89
  pass_num_2: 167
  percent_cases_well_formed: 97.3
  error_outputs: 8
  num_malformed_responses: 6
  num_with_malformed_responses: 6
  user_asks: 67
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  prompt_tokens: 2191446
  completion_tokens: 1645129
  test_timeouts: 1
  total_tests: 225
  command: aider --model deepseek/deepseek-reasoner
  date: 2025-10-03
  versions: 0.86.2.dev
  seconds_per_case: 291.2
  total_cost: 1.3045

- dirname: 2025-10-03-09-21-36--deepseek-v3.2-chat
  test_cases: 225
  model: DeepSeek-V3.2-Exp (Chat)
  edit_format: diff
  commit_hash: cbb5376
  pass_rate_1: 38.7
  pass_rate_2: 70.2
  pass_num_1: 87
  pass_num_2: 158
  percent_cases_well_formed: 98.2
  error_outputs: 6
  num_malformed_responses: 4
  num_with_malformed_responses: 4
  user_asks: 60
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  prompt_tokens: 2266868
  completion_tokens: 573477
  test_timeouts: 4
  total_tests: 225
  command: aider --model deepseek/deepseek-chat
  date: 2025-10-03
  versions: 0.86.2.dev
  seconds_per_case: 104.0
  total_cost: 0.8756