spb/note-spboucher-ai
Public
1{2 "generated": "2026-11-01T07:18:30.317Z",3 "runs": 836,4 "trials_per_cell": 2,5 "total_estimated_cost_usd": 17.243599999999997,6 "categories": [7 {8 "id": "document_analysis",9 "label": "Document analysis"10 },11 {12 "id": "json_generation",13 "label": "JSON generation"14 },15 {16 "id": "complex_reasoning",17 "label": "Complex reasoning"18 },19 {20 "id": "long_context",21 "label": "Long-context retrieval"22 },23 {24 "id": "english_writing",25 "label": "English writing"26 },27 {28 "id": "french_writing",29 "label": "French writing"30 },31 {32 "id": "programming",33 "label": "Programming"34 },35 {36 "id": "simple_reasoning",37 "label": "Simple reasoning"38 },39 {40 "id": "knowledge_synthesis",41 "label": "Knowledge synthesis"42 },43 {44 "id": "tool_calling",45 "label": "Tool calling"46 },47 {48 "id": "mathematics",49 "label": "Mathematics"50 }51 ],52 "pricing_sources": {53 "openai": {54 "url": "https://developers.openai.com/api/docs/pricing",55 "fetched": "2026-11-01",56 "note": "Standard tier, short context (≤272K input)"57 },58 "anthropic": {59 "url": "https://platform.claude.com/docs/en/about-claude/pricing",60 "fetched": "2026-11-01",61 "note": "First-party API rates"62 }63 },64 "judges": [65 "claude-sonnet-5-5",66 "gpt-5.4"67 ],68 "models": [69 {70 "model": "chat-latest",71 "provider": "openai",72 "runs": 22,73 "errors": 0,74 "pass_rate": 1,75 "mean_score": 1,76 "judged_mean": 5,77 "ttft_median_ms": 740.5,78 "total_median_ms": 1392.5,79 "tps_median": 157.05,80 "cost_total": 0.4256,81 "cost_mean": 0.019345454545454544,82 "in_tokens": 113620,83 "out_tokens": 3506,84 "reasoning_tokens": 128,85 "by_category": {86 "document_analysis": {87 "runs": 2,88 "errors": 0,89 "pass_rate": 1,90 "score": 1,91 "judged": null,92 "ttft_ms": 824,93 "total_ms": 1258,94 "tps": 195.95,95 "cost_mean": 0.00505,96 "out_tokens": 8597 },98 "json_generation": {99 "runs": 2,100 "errors": 0,101 "pass_rate": 1,102 "score": 1,103 "judged": null,104 "ttft_ms": 706.5,105 "total_ms": 1091,106 "tps": 174.65,107 "cost_mean": 0.00277,108 "out_tokens": 67109 },110 "complex_reasoning": {111 "runs": 2,112 "errors": 0,113 "pass_rate": 1,114 "score": 1,115 "judged": null,116 "ttft_ms": 1582,117 "total_ms": 2846.5,118 "tps": 269.25,119 "cost_mean": 0.010794999999999999,120 "out_tokens": 336.5121 },122 "long_context": {123 "runs": 2,124 "errors": 0,125 "pass_rate": 1,126 "score": 1,127 "judged": null,128 "ttft_ms": 1056.5,129 "total_ms": 1250.5,130 "tps": 58.1,131 "cost_mean": 0.15248499999999998,132 "out_tokens": 11133 },134 "english_writing": {135 "runs": 2,136 "errors": 0,137 "pass_rate": 1,138 "score": 1,139 "judged": 5,140 "ttft_ms": 766,141 "total_ms": 2856.5,142 "tps": 115.5,143 "cost_mean": 0.007549999999999999,144 "out_tokens": 241145 },146 "french_writing": {147 "runs": 2,148 "errors": 0,149 "pass_rate": 1,150 "score": 1,151 "judged": 5,152 "ttft_ms": 735.5,153 "total_ms": 2906,154 "tps": 138.4,155 "cost_mean": 0.009475,156 "out_tokens": 300.5157 },158 "programming": {159 "runs": 2,160 "errors": 0,161 "pass_rate": 1,162 "score": 1,163 "judged": null,164 "ttft_ms": 786,165 "total_ms": 1392,166 "tps": 203.9,167 "cost_mean": 0.00408,168 "out_tokens": 123.5169 },170 "simple_reasoning": {171 "runs": 2,172 "errors": 0,173 "pass_rate": 1,174 "score": 1,175 "judged": null,176 "ttft_ms": 692.5,177 "total_ms": 900.5,178 "tps": 92.4,179 "cost_mean": 0.001195,180 "out_tokens": 19181 },182 "knowledge_synthesis": {183 "runs": 2,184 "errors": 0,185 "pass_rate": 1,186 "score": 1,187 "judged": 5,188 "ttft_ms": 687.5,189 "total_ms": 2518.5,190 "tps": 157.05,191 "cost_mean": 0.00978,192 "out_tokens": 287.5193 },194 "tool_calling": {195 "runs": 2,196 "errors": 0,197 "pass_rate": 1,198 "score": 1,199 "judged": null,200 "ttft_ms": 622,201 "total_ms": 899.5,202 "tps": 147.7,203 "cost_mean": 0.001985,204 "out_tokens": 41205 },206 "mathematics": {207 "runs": 2,208 "errors": 0,209 "pass_rate": 1,210 "score": 1,211 "judged": null,212 "ttft_ms": 726,213 "total_ms": 1969.5,214 "tps": 193.7,215 "cost_mean": 0.007634999999999999,216 "out_tokens": 241217 }218 }219 },220 {221 "model": "gpt-5.3-codex",222 "provider": "openai",223 "runs": 22,224 "errors": 0,225 "pass_rate": 1,226 "mean_score": 1,227 "judged_mean": 4.666666666666667,228 "ttft_median_ms": 601.5,229 "total_median_ms": 1417.5,230 "tps_median": 93.95,231 "cost_total": 0.253197,232 "cost_mean": 0.011508954545454546,233 "in_tokens": 113620,234 "out_tokens": 3883,235 "reasoning_tokens": 0,236 "by_category": {237 "document_analysis": {238 "runs": 2,239 "errors": 0,240 "pass_rate": 1,241 "score": 1,242 "judged": null,243 "ttft_ms": 602,244 "total_ms": 1296.5,245 "tps": 122.44999999999999,246 "cost_mean": 0.002065,247 "out_tokens": 85248 },249 "json_generation": {250 "runs": 2,251 "errors": 0,252 "pass_rate": 1,253 "score": 1,254 "judged": null,255 "ttft_ms": 702,256 "total_ms": 1284,257 "tps": 115.25,258 "cost_mean": 0.001204,259 "out_tokens": 67260 },261 "complex_reasoning": {262 "runs": 2,263 "errors": 0,264 "pass_rate": 1,265 "score": 1,266 "judged": null,267 "ttft_ms": 700.5,268 "total_ms": 4894.5,269 "tps": 111.75,270 "cost_mean": 0.006810999999999999,271 "out_tokens": 469272 },273 "long_context": {274 "runs": 2,275 "errors": 0,276 "pass_rate": 1,277 "score": 1,278 "judged": null,279 "ttft_ms": 706.5,280 "total_ms": 974.5,281 "tps": 42.150000000000006,282 "cost_mean": 0.09675225,283 "out_tokens": 11284 },285 "english_writing": {286 "runs": 2,287 "errors": 0,288 "pass_rate": 1,289 "score": 1,290 "judged": 5,291 "ttft_ms": 526,292 "total_ms": 3880.5,293 "tps": 67.9,294 "cost_mean": 0.00329,295 "out_tokens": 227296 },297 "french_writing": {298 "runs": 2,299 "errors": 0,300 "pass_rate": 1,301 "score": 1,302 "judged": 5,303 "ttft_ms": 728,304 "total_ms": 4085,305 "tps": 80.9,306 "cost_mean": 0.003962,307 "out_tokens": 271.5308 },309 "programming": {310 "runs": 2,311 "errors": 0,312 "pass_rate": 1,313 "score": 1,314 "judged": null,315 "ttft_ms": 548,316 "total_ms": 1417.5,317 "tps": 140.45,318 "cost_mean": 0.0018392500000000002,319 "out_tokens": 122320 },321 "simple_reasoning": {322 "runs": 2,323 "errors": 0,324 "pass_rate": 1,325 "score": 1,326 "judged": null,327 "ttft_ms": 542,328 "total_ms": 831,329 "tps": 65.9,330 "cost_mean": 0.00048475,331 "out_tokens": 19332 },333 "knowledge_synthesis": {334 "runs": 2,335 "errors": 0,336 "pass_rate": 1,337 "score": 1,338 "judged": 4,339 "ttft_ms": 609,340 "total_ms": 3382.5,341 "tps": 93.55000000000001,342 "cost_mean": 0.00402325,343 "out_tokens": 258.5344 },345 "tool_calling": {346 "runs": 2,347 "errors": 0,348 "pass_rate": 1,349 "score": 1,350 "judged": null,351 "ttft_ms": 623,352 "total_ms": 1083,353 "tps": 89.05000000000001,354 "cost_mean": 0.00083825,355 "out_tokens": 41356 },357 "mathematics": {358 "runs": 2,359 "errors": 0,360 "pass_rate": 1,361 "score": 1,362 "judged": null,363 "ttft_ms": 538.5,364 "total_ms": 3267.5,365 "tps": 136,366 "cost_mean": 0.00532875,367 "out_tokens": 370.5368 }369 }370 },371 {372 "model": "gpt-4.1",373 "provider": "openai",374 "runs": 22,375 "errors": 0,376 "pass_rate": 1,377 "mean_score": 1,378 "judged_mean": 4.166666666666667,379 "ttft_median_ms": 449.5,380 "total_median_ms": 1722,381 "tps_median": 162.95,382 "cost_total": 0.2591960000000001,383 "cost_mean": 0.011781636363636367,384 "in_tokens": 113634,385 "out_tokens": 5479,386 "reasoning_tokens": 0,387 "by_category": {388 "document_analysis": {389 "runs": 2,390 "errors": 0,391 "pass_rate": 1,392 "score": 1,393 "judged": null,394 "ttft_ms": 477.5,395 "total_ms": 1064.5,396 "tps": 174,397 "cost_mean": 0.001812,398 "out_tokens": 101399 },400 "json_generation": {401 "runs": 2,402 "errors": 0,403 "pass_rate": 1,404 "score": 1,405 "judged": null,406 "ttft_ms": 522.5,407 "total_ms": 890.5,408 "tps": 160.7,409 "cost_mean": 0.00078,410 "out_tokens": 59411 },412 "complex_reasoning": {413 "runs": 2,414 "errors": 0,415 "pass_rate": 1,416 "score": 1,417 "judged": null,418 "ttft_ms": 421.5,419 "total_ms": 4315,420 "tps": 253.25,421 "cost_mean": 0.00817,422 "out_tokens": 986423 },424 "long_context": {425 "runs": 2,426 "errors": 0,427 "pass_rate": 1,428 "score": 1,429 "judged": null,430 "ttft_ms": 2847.5,431 "total_ms": 3115,432 "tps": 33.2,433 "cost_mean": 0.104512,434 "out_tokens": 8435 },436 "english_writing": {437 "runs": 2,438 "errors": 0,439 "pass_rate": 1,440 "score": 1,441 "judged": 5,442 "ttft_ms": 420.5,443 "total_ms": 1776,444 "tps": 161.7,445 "cost_mean": 0.001882,446 "out_tokens": 219447 },448 "french_writing": {449 "runs": 2,450 "errors": 0,451 "pass_rate": 1,452 "score": 1,453 "judged": 5,454 "ttft_ms": 345.5,455 "total_ms": 1943,456 "tps": 179.25,457 "cost_mean": 0.00245,458 "out_tokens": 283459 },460 "programming": {461 "runs": 2,462 "errors": 0,463 "pass_rate": 1,464 "score": 1,465 "judged": null,466 "ttft_ms": 371,467 "total_ms": 1115,468 "tps": 184.5,469 "cost_mean": 0.001192,470 "out_tokens": 130471 },472 "simple_reasoning": {473 "runs": 2,474 "errors": 0,475 "pass_rate": 1,476 "score": 1,477 "judged": null,478 "ttft_ms": 392.5,479 "total_ms": 631,480 "tps": 67.05000000000001,481 "cost_mean": 0.00038,482 "out_tokens": 16483 },484 "knowledge_synthesis": {485 "runs": 2,486 "errors": 0,487 "pass_rate": 1,488 "score": 1,489 "judged": 2.5,490 "ttft_ms": 545.5,491 "total_ms": 1812.5,492 "tps": 216.05,493 "cost_mean": 0.002652,494 "out_tokens": 273.5495 },496 "tool_calling": {497 "runs": 2,498 "errors": 0,499 "pass_rate": 1,500 "score": 1,501 "judged": null,502 "ttft_ms": 501.5,503 "total_ms": 789,504 "tps": 132.2,505 "cost_mean": 0.000596,506 "out_tokens": 38507 },508 "mathematics": {509 "runs": 2,510 "errors": 0,511 "pass_rate": 1,512 "score": 1,513 "judged": null,514 "ttft_ms": 637,515 "total_ms": 2967.5,516 "tps": 266.9,517 "cost_mean": 0.0051719999999999995,518 "out_tokens": 626519 }520 }521 },522 {523 "model": "gpt-6-sol",524 "provider": "openai",525 "runs": 22,526 "errors": 0,527 "pass_rate": 1,528 "mean_score": 1,529 "judged_mean": 5,530 "ttft_median_ms": 1106,531 "total_median_ms": 1732,532 "tps_median": 149.39999999999998,533 "cost_total": 0.16421719999999998,534 "cost_mean": 0.007464418181818181,535 "in_tokens": 113620,536 "out_tokens": 3633,537 "reasoning_tokens": 1358,538 "by_category": {539 "document_analysis": {540 "runs": 2,541 "errors": 0,542 "pass_rate": 1,543 "score": 1,544 "judged": null,545 "ttft_ms": 955,546 "total_ms": 1538,547 "tps": 185.8,548 "cost_mean": 0.0020800000000000003,549 "out_tokens": 108550 },551 "json_generation": {552 "runs": 2,553 "errors": 0,554 "pass_rate": 1,555 "score": 1,556 "judged": null,557 "ttft_ms": 1200.5,558 "total_ms": 1732,559 "tps": 127.64999999999999,560 "cost_mean": 0.000974,561 "out_tokens": 67562 },563 "complex_reasoning": {564 "runs": 2,565 "errors": 0,566 "pass_rate": 1,567 "score": 1,568 "judged": null,569 "ttft_ms": 1983,570 "total_ms": 2787,571 "tps": 254.15,572 "cost_mean": 0.002325,573 "out_tokens": 204.5574 },575 "long_context": {576 "runs": 2,577 "errors": 0,578 "pass_rate": 1,579 "score": 1,580 "judged": null,581 "ttft_ms": 1154.5,582 "total_ms": 1452.5,583 "tps": 38.35,584 "cost_mean": 0.0608316,585 "out_tokens": 11586 },587 "english_writing": {588 "runs": 2,589 "errors": 0,590 "pass_rate": 1,591 "score": 1,592 "judged": 5,593 "ttft_ms": 2360,594 "total_ms": 4195,595 "tps": 241.05,596 "cost_mean": 0.003848,597 "out_tokens": 372598 },599 "french_writing": {600 "runs": 2,601 "errors": 0,602 "pass_rate": 1,603 "score": 1,604 "judged": 5,605 "ttft_ms": 2382,606 "total_ms": 4593.5,607 "tps": 213.2,608 "cost_mean": 0.004373999999999999,609 "out_tokens": 419610 },611 "programming": {612 "runs": 2,613 "errors": 0,614 "pass_rate": 1,615 "score": 1,616 "judged": null,617 "ttft_ms": 1837,618 "total_ms": 2589.5,619 "tps": 311.70000000000005,620 "cost_mean": 0.002495,621 "out_tokens": 234.5622 },623 "simple_reasoning": {624 "runs": 2,625 "errors": 0,626 "pass_rate": 1,627 "score": 1,628 "judged": null,629 "ttft_ms": 849,630 "total_ms": 1187,631 "tps": 104.94999999999999,632 "cost_mean": 0.000595,633 "out_tokens": 34.5634 },635 "knowledge_synthesis": {636 "runs": 2,637 "errors": 0,638 "pass_rate": 1,639 "score": 1,640 "judged": 5,641 "ttft_ms": 933.5,642 "total_ms": 2514.5,643 "tps": 164.5,644 "cost_mean": 0.003062,645 "out_tokens": 260646 },647 "tool_calling": {648 "runs": 2,649 "errors": 0,650 "pass_rate": 1,651 "score": 1,652 "judged": null,653 "ttft_ms": 828.5,654 "total_ms": 1251,655 "tps": 97.25,656 "cost_mean": 0.000712,657 "out_tokens": 41658 },659 "mathematics": {660 "runs": 2,661 "errors": 0,662 "pass_rate": 1,663 "score": 1,664 "judged": null,665 "ttft_ms": 940,666 "total_ms": 1281,667 "tps": 81.6,668 "cost_mean": 0.000812,669 "out_tokens": 65670 }671 }672 },673 {674 "model": "gpt-5.4",675 "provider": "openai",676 "runs": 22,677 "errors": 0,678 "pass_rate": 1,679 "mean_score": 1,680 "judged_mean": 5,681 "ttft_median_ms": 715.5,682 "total_median_ms": 1817.5,683 "tps_median": 85.69999999999999,684 "cost_total": 0.3470499999999999,685 "cost_mean": 0.015774999999999997,686 "in_tokens": 113620,687 "out_tokens": 4200,688 "reasoning_tokens": 0,689 "by_category": {690 "document_analysis": {691 "runs": 2,692 "errors": 0,693 "pass_rate": 1,694 "score": 1,695 "judged": null,696 "ttft_ms": 878,697 "total_ms": 1630.5,698 "tps": 114.30000000000001,699 "cost_mean": 0.002525,700 "out_tokens": 85701 },702 "json_generation": {703 "runs": 2,704 "errors": 0,705 "pass_rate": 1,706 "score": 1,707 "judged": null,708 "ttft_ms": 787,709 "total_ms": 1453.5,710 "tps": 100.65,711 "cost_mean": 0.001385,712 "out_tokens": 67713 },714 "complex_reasoning": {715 "runs": 2,716 "errors": 0,717 "pass_rate": 1,718 "score": 1,719 "judged": null,720 "ttft_ms": 740,721 "total_ms": 8292,722 "tps": 85.69999999999999,723 "cost_mean": 0.010055000000000001,724 "out_tokens": 647725 },726 "long_context": {727 "runs": 2,728 "errors": 0,729 "pass_rate": 1,730 "score": 1,731 "judged": null,732 "ttft_ms": 1132,733 "total_ms": 1526.5,734 "tps": 32.7,735 "cost_mean": 0.1381625,736 "out_tokens": 11737 },738 "english_writing": {739 "runs": 2,740 "errors": 0,741 "pass_rate": 1,742 "score": 1,743 "judged": 5,744 "ttft_ms": 589,745 "total_ms": 4781.5,746 "tps": 54,747 "cost_mean": 0.0035575,748 "out_tokens": 226.5749 },750 "french_writing": {751 "runs": 2,752 "errors": 0,753 "pass_rate": 1,754 "score": 1,755 "judged": 5,756 "ttft_ms": 783,757 "total_ms": 7653.5,758 "tps": 40.75,759 "cost_mean": 0.0044225,760 "out_tokens": 279.5761 },762 "programming": {763 "runs": 2,764 "errors": 0,765 "pass_rate": 1,766 "score": 1,767 "judged": null,768 "ttft_ms": 796.5,769 "total_ms": 1764.5,770 "tps": 112.25,771 "cost_mean": 0.001815,772 "out_tokens": 108.5773 },774 "simple_reasoning": {775 "runs": 2,776 "errors": 0,777 "pass_rate": 1,778 "score": 1,779 "judged": null,780 "ttft_ms": 628,781 "total_ms": 982.5,782 "tps": 53.95,783 "cost_mean": 0.0005975,784 "out_tokens": 19785 },786 "knowledge_synthesis": {787 "runs": 2,788 "errors": 0,789 "pass_rate": 1,790 "score": 1,791 "judged": 5,792 "ttft_ms": 734,793 "total_ms": 4693.5,794 "tps": 72.8,795 "cost_mean": 0.0048975,796 "out_tokens": 288797 },798 "tool_calling": {799 "runs": 2,800 "errors": 0,801 "pass_rate": 1,802 "score": 1,803 "judged": null,804 "ttft_ms": 652.5,805 "total_ms": 1011.5,806 "tps": 114.1,807 "cost_mean": 0.0009925,808 "out_tokens": 41809 },810 "mathematics": {811 "runs": 2,812 "errors": 0,813 "pass_rate": 1,814 "score": 1,815 "judged": null,816 "ttft_ms": 645,817 "total_ms": 3608,818 "tps": 110.5,819 "cost_mean": 0.005115,820 "out_tokens": 327.5821 }822 }823 },824 {825 "model": "claude-fable-5-1",826 "provider": "anthropic",827 "runs": 22,828 "errors": 0,829 "pass_rate": 1,830 "mean_score": 1,831 "judged_mean": 5,832 "ttft_median_ms": 389,833 "total_median_ms": 2098.5,834 "tps_median": 118.3,835 "cost_total": 1.9785900000000003,836 "cost_mean": 0.08993590909090911,837 "in_tokens": 168774,838 "out_tokens": 5817,839 "reasoning_tokens": 709,840 "by_category": {841 "document_analysis": {842 "runs": 2,843 "errors": 0,844 "pass_rate": 1,845 "score": 1,846 "judged": null,847 "ttft_ms": 727.5,848 "total_ms": 1647,849 "tps": 128.3,850 "cost_mean": 0.0161,851 "out_tokens": 118852 },853 "json_generation": {854 "runs": 2,855 "errors": 0,856 "pass_rate": 1,857 "score": 1,858 "judged": null,859 "ttft_ms": 737,860 "total_ms": 1587.5,861 "tps": 123.4,862 "cost_mean": 0.01021,863 "out_tokens": 105864 },865 "complex_reasoning": {866 "runs": 2,867 "errors": 0,868 "pass_rate": 1,869 "score": 1,870 "judged": null,871 "ttft_ms": 1766,872 "total_ms": 5206,873 "tps": 140.2,874 "cost_mean": 0.026165,875 "out_tokens": 476.5876 },877 "long_context": {878 "runs": 2,879 "errors": 0,880 "pass_rate": 1,881 "score": 1,882 "judged": null,883 "ttft_ms": 1388,884 "total_ms": 1444,885 "tps": 220.55,886 "cost_mean": 0.80968,887 "out_tokens": 11888 },889 "english_writing": {890 "runs": 2,891 "errors": 0,892 "pass_rate": 1,893 "score": 1,894 "judged": 5,895 "ttft_ms": 340.5,896 "total_ms": 7003,897 "tps": 64.19999999999999,898 "cost_mean": 0.022289999999999997,899 "out_tokens": 425900 },901 "french_writing": {902 "runs": 2,903 "errors": 0,904 "pass_rate": 1,905 "score": 1,906 "judged": 5,907 "ttft_ms": 342,908 "total_ms": 7573,909 "tps": 74.80000000000001,910 "cost_mean": 0.02844,911 "out_tokens": 536912 },913 "programming": {914 "runs": 2,915 "errors": 0,916 "pass_rate": 1,917 "score": 1,918 "judged": null,919 "ttft_ms": 360.5,920 "total_ms": 2098.5,921 "tps": 118.3,922 "cost_mean": 0.011495,923 "out_tokens": 205.5924 },925 "simple_reasoning": {926 "runs": 2,927 "errors": 0,928 "pass_rate": 1,929 "score": 1,930 "judged": null,931 "ttft_ms": 331.5,932 "total_ms": 740.5,933 "tps": 46.4,934 "cost_mean": 0.00276,935 "out_tokens": 19936 },937 "knowledge_synthesis": {938 "runs": 2,939 "errors": 0,940 "pass_rate": 1,941 "score": 1,942 "judged": 5,943 "ttft_ms": 2914,944 "total_ms": 8164.5,945 "tps": 125.94999999999999,946 "cost_mean": 0.036559999999999995,947 "out_tokens": 661948 },949 "tool_calling": {950 "runs": 2,951 "errors": 0,952 "pass_rate": 1,953 "score": 1,954 "judged": null,955 "ttft_ms": 422.5,956 "total_ms": 1559,957 "tps": 121.1,958 "cost_mean": 0.013645000000000001,959 "out_tokens": 135.5960 },961 "mathematics": {962 "runs": 2,963 "errors": 0,964 "pass_rate": 1,965 "score": 1,966 "judged": null,967 "ttft_ms": 350.5,968 "total_ms": 2466,969 "tps": 102.2,970 "cost_mean": 0.01195,971 "out_tokens": 216972 }973 }974 },975 {976 "model": "gpt-5.2",977 "provider": "openai",978 "runs": 22,979 "errors": 0,980 "pass_rate": 1,981 "mean_score": 1,982 "judged_mean": 4.333333333333333,983 "ttft_median_ms": 675.5,984 "total_median_ms": 2144.5,985 "tps_median": 85.35,986 "cost_total": 0.16192260000000006,987 "cost_mean": 0.007360118181818184,988 "in_tokens": 113620,989 "out_tokens": 3541,990 "reasoning_tokens": 0,991 "by_category": {992 "document_analysis": {993 "runs": 2,994 "errors": 0,995 "pass_rate": 1,996 "score": 1,997 "judged": null,998 "ttft_ms": 792,999 "total_ms": 1564.5,1000 "tps": 110,1001 "cost_mean": 0.002065,1002 "out_tokens": 851003 },1004 "json_generation": {1005 "runs": 2,1006 "errors": 0,1007 "pass_rate": 1,1008 "score": 1,1009 "judged": null,1010 "ttft_ms": 704.5,1011 "total_ms": 1407.5,1012 "tps": 97.6,1013 "cost_mean": 0.001204,1014 "out_tokens": 671015 },1016 "complex_reasoning": {1017 "runs": 2,1018 "errors": 0,1019 "pass_rate": 1,1020 "score": 1,1021 "judged": null,1022 "ttft_ms": 571.5,1023 "total_ms": 4959,1024 "tps": 87.9,1025 "cost_mean": 0.005656,1026 "out_tokens": 386.51027 },1028 "long_context": {1029 "runs": 2,1030 "errors": 0,1031 "pass_rate": 1,1032 "score": 1,1033 "judged": null,1034 "ttft_ms": 779.5,1035 "total_ms": 1019.5,1036 "tps": 46,1037 "cost_mean": 0.053509049999999996,1038 "out_tokens": 111039 },1040 "english_writing": {1041 "runs": 2,1042 "errors": 0,1043 "pass_rate": 1,1044 "score": 1,1045 "judged": 5,1046 "ttft_ms": 570.5,1047 "total_ms": 4333.5,1048 "tps": 57.55,1049 "cost_mean": 0.003143,1050 "out_tokens": 216.51051 },1052 "french_writing": {1053 "runs": 2,1054 "errors": 0,1055 "pass_rate": 1,1056 "score": 1,1057 "judged": 5,1058 "ttft_ms": 717.5,1059 "total_ms": 3830.5,1060 "tps": 81.25,1061 "cost_mean": 0.003689,1062 "out_tokens": 2521063 },1064 "programming": {1065 "runs": 2,1066 "errors": 0,1067 "pass_rate": 1,1068 "score": 1,1069 "judged": null,1070 "ttft_ms": 801.5,1071 "total_ms": 2144.5,1072 "tps": 102.80000000000001,1073 "cost_mean": 0.00206325,1074 "out_tokens": 1381075 },1076 "simple_reasoning": {1077 "runs": 2,1078 "errors": 0,1079 "pass_rate": 1,1080 "score": 1,1081 "judged": null,1082 "ttft_ms": 629,1083 "total_ms": 999.5,1084 "tps": 52.55,1085 "cost_mean": 0.00048475,1086 "out_tokens": 191087 },1088 "knowledge_synthesis": {1089 "runs": 2,1090 "errors": 0,1091 "pass_rate": 1,1092 "score": 1,1093 "judged": 3,1094 "ttft_ms": 584.5,1095 "total_ms": 3930,1096 "tps": 72.7,1097 "cost_mean": 0.0037992499999999997,1098 "out_tokens": 242.51099 },1100 "tool_calling": {1101 "runs": 2,1102 "errors": 0,1103 "pass_rate": 1,1104 "score": 1,1105 "judged": null,1106 "ttft_ms": 710.5,1107 "total_ms": 1148,1108 "tps": 93.7,1109 "cost_mean": 0.00083825,1110 "out_tokens": 411111 },1112 "mathematics": {1113 "runs": 2,1114 "errors": 0,1115 "pass_rate": 1,1116 "score": 1,1117 "judged": null,1118 "ttft_ms": 708.5,1119 "total_ms": 4149.5,1120 "tps": 89.94999999999999,1121 "cost_mean": 0.00450975,1122 "out_tokens": 3121123 }1124 }1125 },1126 {1127 "model": "claude-sonnet-5",1128 "provider": "anthropic",1129 "runs": 22,1130 "errors": 0,1131 "pass_rate": 1,1132 "mean_score": 1,1133 "judged_mean": 4.333333333333333,1134 "ttft_median_ms": 949.5,1135 "total_median_ms": 2168.5,1136 "tps_median": 145.05,1137 "cost_total": 0.415752,1138 "cost_mean": 0.018897818181818182,1139 "in_tokens": 168866,1140 "out_tokens": 7802,1141 "reasoning_tokens": 2116,1142 "by_category": {1143 "document_analysis": {1144 "runs": 2,1145 "errors": 0,1146 "pass_rate": 1,1147 "score": 1,1148 "judged": null,1149 "ttft_ms": 971,1150 "total_ms": 1678.5,1151 "tps": 196.15,1152 "cost_mean": 0.0034260000000000002,1153 "out_tokens": 1391154 },1155 "json_generation": {1156 "runs": 2,1157 "errors": 0,1158 "pass_rate": 1,1159 "score": 1,1160 "judged": null,1161 "ttft_ms": 1258.5,1162 "total_ms": 1904.5,1163 "tps": 162.6,1164 "cost_mean": 0.002038,1165 "out_tokens": 1051166 },1167 "complex_reasoning": {1168 "runs": 2,1169 "errors": 0,1170 "pass_rate": 1,1171 "score": 1,1172 "judged": null,1173 "ttft_ms": 6056.5,1174 "total_ms": 11139,1175 "tps": 252.3,1176 "cost_mean": 0.012919,1177 "out_tokens": 1245.51178 },1179 "long_context": {1180 "runs": 2,1181 "errors": 0,1182 "pass_rate": 1,1183 "score": 1,1184 "judged": null,1185 "ttft_ms": 1202,1186 "total_ms": 1569.5,1187 "tps": 30.200000000000003,1188 "cost_mean": 0.161932,1189 "out_tokens": 111190 },1191 "english_writing": {1192 "runs": 2,1193 "errors": 0,1194 "pass_rate": 1,1195 "score": 1,1196 "judged": 5,1197 "ttft_ms": 949.5,1198 "total_ms": 6332.5,1199 "tps": 88.05000000000001,1200 "cost_mean": 0.0049440000000000005,1201 "out_tokens": 4741202 },1203 "french_writing": {1204 "runs": 2,1205 "errors": 0,1206 "pass_rate": 1,1207 "score": 1,1208 "judged": 5,1209 "ttft_ms": 1487.5,1210 "total_ms": 7222,1211 "tps": 110.45,1212 "cost_mean": 0.006659,1213 "out_tokens": 633.51214 },1215 "programming": {1216 "runs": 2,1217 "errors": 0,1218 "pass_rate": 1,1219 "score": 1,1220 "judged": null,1221 "ttft_ms": 494,1222 "total_ms": 1814,1223 "tps": 145.05,1224 "cost_mean": 0.002155,1225 "out_tokens": 191.51226 },1227 "simple_reasoning": {1228 "runs": 2,1229 "errors": 0,1230 "pass_rate": 1,1231 "score": 1,1232 "judged": null,1233 "ttft_ms": 926,1234 "total_ms": 1158.5,1235 "tps": 343.1,1236 "cost_mean": 0.001118,1237 "out_tokens": 761238 },1239 "knowledge_synthesis": {1240 "runs": 2,1241 "errors": 0,1242 "pass_rate": 1,1243 "score": 1,1244 "judged": 3,1245 "ttft_ms": 479.5,1246 "total_ms": 4911,1247 "tps": 101.65,1248 "cost_mean": 0.005202999999999999,1249 "out_tokens": 450.51250 },1251 "tool_calling": {1252 "runs": 2,1253 "errors": 0,1254 "pass_rate": 1,1255 "score": 1,1256 "judged": null,1257 "ttft_ms": 751.5,1258 "total_ms": 1455,1259 "tps": 200.45,1260 "cost_mean": 0.002896,1261 "out_tokens": 1391262 },1263 "mathematics": {1264 "runs": 2,1265 "errors": 0,1266 "pass_rate": 1,1267 "score": 1,1268 "judged": null,1269 "ttft_ms": 2783.5,1270 "total_ms": 4106,1271 "tps": 117.85,1272 "cost_mean": 0.004586,1273 "out_tokens": 4361274 }1275 }1276 },1277 {1278 "model": "gpt-5.1",1279 "provider": "openai",1280 "runs": 22,1281 "errors": 0,1282 "pass_rate": 1,1283 "mean_score": 1,1284 "judged_mean": 4.5,1285 "ttft_median_ms": 649.5,1286 "total_median_ms": 2181,1287 "tps_median": 138.85000000000002,1288 "cost_total": 0.12493499999999999,1289 "cost_mean": 0.0056788636363636355,1290 "in_tokens": 113620,1291 "out_tokens": 4483,1292 "reasoning_tokens": 0,1293 "by_category": {1294 "document_analysis": {1295 "runs": 2,1296 "errors": 0,1297 "pass_rate": 1,1298 "score": 1,1299 "judged": null,1300 "ttft_ms": 826.5,1301 "total_ms": 1363,1302 "tps": 171.60000000000002,1303 "cost_mean": 0.001535,1304 "out_tokens": 911305 },1306 "json_generation": {1307 "runs": 2,1308 "errors": 0,1309 "pass_rate": 1,1310 "score": 1,1311 "judged": null,1312 "ttft_ms": 653.5,1313 "total_ms": 1112.5,1314 "tps": 159.45,1315 "cost_mean": 0.00092,1316 "out_tokens": 731317 },1318 "complex_reasoning": {1319 "runs": 2,1320 "errors": 0,1321 "pass_rate": 1,1322 "score": 1,1323 "judged": null,1324 "ttft_ms": 614,1325 "total_ms": 4113,1326 "tps": 189.2,1327 "cost_mean": 0.00679,1328 "out_tokens": 661.51329 },1330 "long_context": {1331 "runs": 2,1332 "errors": 0,1333 "pass_rate": 1,1334 "score": 1,1335 "judged": null,1336 "ttft_ms": 1506.5,1337 "total_ms": 1775,1338 "tps": 64.05,1339 "cost_mean": 0.03820875,1340 "out_tokens": 171341 },1342 "english_writing": {1343 "runs": 2,1344 "errors": 0,1345 "pass_rate": 1,1346 "score": 1,1347 "judged": 5,1348 "ttft_ms": 559.5,1349 "total_ms": 4746.5,1350 "tps": 52.8,1351 "cost_mean": 0.0022849999999999997,1352 "out_tokens": 220.51353 },1354 "french_writing": {1355 "runs": 2,1356 "errors": 0,1357 "pass_rate": 1,1358 "score": 1,1359 "judged": 5,1360 "ttft_ms": 987,1361 "total_ms": 3910,1362 "tps": 101.05,1363 "cost_mean": 0.00307,1364 "out_tokens": 295.51365 },1366 "programming": {1367 "runs": 2,1368 "errors": 0,1369 "pass_rate": 1,1370 "score": 1,1371 "judged": null,1372 "ttft_ms": 808.5,1373 "total_ms": 1711.5,1374 "tps": 161.65,1375 "cost_mean": 0.00155375,1376 "out_tokens": 1461377 },1378 "simple_reasoning": {1379 "runs": 2,1380 "errors": 0,1381 "pass_rate": 1,1382 "score": 1,1383 "judged": null,1384 "ttft_ms": 584.5,1385 "total_ms": 833,1386 "tps": 100.69999999999999,1387 "cost_mean": 0.00040625,1388 "out_tokens": 251389 },1390 "knowledge_synthesis": {1391 "runs": 2,1392 "errors": 0,1393 "pass_rate": 1,1394 "score": 1,1395 "judged": 3.5,1396 "ttft_ms": 594.5,1397 "total_ms": 2606,1398 "tps": 132.95,1399 "cost_mean": 0.0029537499999999998,1400 "out_tokens": 266.51401 },1402 "tool_calling": {1403 "runs": 2,1404 "errors": 0,1405 "pass_rate": 1,1406 "score": 1,1407 "judged": null,1408 "ttft_ms": 672.5,1409 "total_ms": 1023.5,1410 "tps": 134.10000000000002,1411 "cost_mean": 0.00065875,1412 "out_tokens": 471413 },1414 "mathematics": {1415 "runs": 2,1416 "errors": 0,1417 "pass_rate": 1,1418 "score": 1,1419 "judged": null,1420 "ttft_ms": 680,1421 "total_ms": 2868.5,1422 "tps": 183,1423 "cost_mean": 0.00408625,1424 "out_tokens": 398.51425 }1426 }1427 },1428 {1429 "model": "gpt-6.1-sol",1430 "provider": "openai",1431 "runs": 22,1432 "errors": 0,1433 "pass_rate": 1,1434 "mean_score": 1,1435 "judged_mean": 4.666666666666667,1436 "ttft_median_ms": 1444,1437 "total_median_ms": 2183.5,1438 "tps_median": 88.75,1439 "cost_total": 0.15463759999999996,1440 "cost_mean": 0.007028981818181817,1441 "in_tokens": 113620,1442 "out_tokens": 3227,1443 "reasoning_tokens": 619,1444 "by_category": {1445 "document_analysis": {1446 "runs": 2,1447 "errors": 0,1448 "pass_rate": 1,1449 "score": 1,1450 "judged": null,1451 "ttft_ms": 1136.5,1452 "total_ms": 1875.5,1453 "tps": 151.64999999999998,1454 "cost_mean": 0.00212,1455 "out_tokens": 1121456 },1457 "json_generation": {1458 "runs": 2,1459 "errors": 0,1460 "pass_rate": 1,1461 "score": 1,1462 "judged": null,1463 "ttft_ms": 1059.5,1464 "total_ms": 1674,1465 "tps": 115.75,1466 "cost_mean": 0.000974,1467 "out_tokens": 671468 },1469 "complex_reasoning": {1470 "runs": 2,1471 "errors": 0,1472 "pass_rate": 1,1473 "score": 1,1474 "judged": null,1475 "ttft_ms": 2701,1476 "total_ms": 4874,1477 "tps": 126.4,1478 "cost_mean": 0.00298,1479 "out_tokens": 2701480 },1481 "long_context": {1482 "runs": 2,1483 "errors": 0,1484 "pass_rate": 1,1485 "score": 1,1486 "judged": null,1487 "ttft_ms": 1422,1488 "total_ms": 1687.5,1489 "tps": 42.2,1490 "cost_mean": 0.0580718,1491 "out_tokens": 111492 },1493 "english_writing": {1494 "runs": 2,1495 "errors": 0,1496 "pass_rate": 1,1497 "score": 1,1498 "judged": 5,1499 "ttft_ms": 1979,1500 "total_ms": 7634.5,1501 "tps": 44,1502 "cost_mean": 0.002618,1503 "out_tokens": 2491504 },1505 "french_writing": {1506 "runs": 2,1507 "errors": 0,1508 "pass_rate": 1,1509 "score": 1,1510 "judged": 5,1511 "ttft_ms": 2749.5,1512 "total_ms": 7974,1513 "tps": 60.2,1514 "cost_mean": 0.0033290000000000004,1515 "out_tokens": 314.51516 },1517 "programming": {1518 "runs": 2,1519 "errors": 0,1520 "pass_rate": 1,1521 "score": 1,1522 "judged": null,1523 "ttft_ms": 3534,1524 "total_ms": 4413,1525 "tps": 247.70000000000002,1526 "cost_mean": 0.0023150000000000002,1527 "out_tokens": 216.51528 },1529 "simple_reasoning": {1530 "runs": 2,1531 "errors": 0,1532 "pass_rate": 1,1533 "score": 1,1534 "judged": null,1535 "ttft_ms": 1110.5,1536 "total_ms": 1417.5,1537 "tps": 93.05000000000001,1538 "cost_mean": 0.00052,1539 "out_tokens": 271540 },1541 "knowledge_synthesis": {1542 "runs": 2,1543 "errors": 0,1544 "pass_rate": 1,1545 "score": 1,1546 "judged": 4,1547 "ttft_ms": 846,1548 "total_ms": 4112,1549 "tps": 72.75,1550 "cost_mean": 0.002832,1551 "out_tokens": 2371552 },1553 "tool_calling": {1554 "runs": 2,1555 "errors": 0,1556 "pass_rate": 1,1557 "score": 1,1558 "judged": null,1559 "ttft_ms": 794.5,1560 "total_ms": 1173.5,1561 "tps": 108.69999999999999,1562 "cost_mean": 0.000712,1563 "out_tokens": 411564 },1565 "mathematics": {1566 "runs": 2,1567 "errors": 0,1568 "pass_rate": 1,1569 "score": 1,1570 "judged": null,1571 "ttft_ms": 1805,1572 "total_ms": 2183.5,1573 "tps": 74.44999999999999,1574 "cost_mean": 0.000847,1575 "out_tokens": 68.51576 }1577 }1578 },1579 {1580 "model": "claude-sonnet-4-5-20250929",1581 "provider": "anthropic",1582 "runs": 22,1583 "errors": 0,1584 "pass_rate": 1,1585 "mean_score": 1,1586 "judged_mean": 4.333333333333333,1587 "ttft_median_ms": 774.5,1588 "total_median_ms": 2337,1589 "tps_median": 89.85,1590 "cost_total": 0.474456,1591 "cost_mean": 0.021566181818181818,1592 "in_tokens": 129532,1593 "out_tokens": 5724,1594 "reasoning_tokens": 0,1595 "by_category": {1596 "document_analysis": {1597 "runs": 2,1598 "errors": 0,1599 "pass_rate": 1,1600 "score": 1,1601 "judged": null,1602 "ttft_ms": 1042,1603 "total_ms": 1914.5,1604 "tps": 115.80000000000001,1605 "cost_mean": 0.004035,1606 "out_tokens": 1011607 },1608 "json_generation": {1609 "runs": 2,1610 "errors": 0,1611 "pass_rate": 1,1612 "score": 1,1613 "judged": null,1614 "ttft_ms": 1074,1615 "total_ms": 1832,1616 "tps": 105.55000000000001,1617 "cost_mean": 0.002394,1618 "out_tokens": 801619 },1620 "complex_reasoning": {1621 "runs": 2,1622 "errors": 0,1623 "pass_rate": 1,1624 "score": 1,1625 "judged": null,1626 "ttft_ms": 591,1627 "total_ms": 10630.5,1628 "tps": 83.80000000000001,1629 "cost_mean": 0.01311,1630 "out_tokens": 8421631 },1632 "long_context": {1633 "runs": 2,1634 "errors": 0,1635 "pass_rate": 1,1636 "score": 1,1637 "judged": null,1638 "ttft_ms": 1624,1639 "total_ms": 2080,1640 "tps": 28.8,1641 "cost_mean": 0.18549,1642 "out_tokens": 131643 },1644 "english_writing": {1645 "runs": 2,1646 "errors": 0,1647 "pass_rate": 1,1648 "score": 1,1649 "judged": 5,1650 "ttft_ms": 725.5,1651 "total_ms": 6388,1652 "tps": 43.150000000000006,1653 "cost_mean": 0.003882,1654 "out_tokens": 2441655 },1656 "french_writing": {1657 "runs": 2,1658 "errors": 0,1659 "pass_rate": 1,1660 "score": 1,1661 "judged": 5,1662 "ttft_ms": 561.5,1663 "total_ms": 8078.5,1664 "tps": 49.5,1665 "cost_mean": 0.005955,1666 "out_tokens": 3721667 },1668 "programming": {1669 "runs": 2,1670 "errors": 0,1671 "pass_rate": 1,1672 "score": 1,1673 "judged": null,1674 "ttft_ms": 589.5,1675 "total_ms": 2337,1676 "tps": 99.6,1677 "cost_mean": 0.002853,1678 "out_tokens": 1741679 },1680 "simple_reasoning": {1681 "runs": 2,1682 "errors": 0,1683 "pass_rate": 1,1684 "score": 1,1685 "judged": null,1686 "ttft_ms": 985.5,1687 "total_ms": 1149.5,1688 "tps": 122.4,1689 "cost_mean": 0.000741,1690 "out_tokens": 201691 },1692 "knowledge_synthesis": {1693 "runs": 2,1694 "errors": 0,1695 "pass_rate": 1,1696 "score": 1,1697 "judged": 3,1698 "ttft_ms": 630,1699 "total_ms": 7236,1700 "tps": 41.55,1701 "cost_mean": 0.004907999999999999,1702 "out_tokens": 2741703 },1704 "tool_calling": {1705 "runs": 2,1706 "errors": 0,1707 "pass_rate": 1,1708 "score": 1,1709 "judged": null,1710 "ttft_ms": 1197.5,1711 "total_ms": 1718.5,1712 "tps": 230.05,1713 "cost_mean": 0.004242,1714 "out_tokens": 1201715 },1716 "mathematics": {1717 "runs": 2,1718 "errors": 0,1719 "pass_rate": 1,1720 "score": 1,1721 "judged": null,1722 "ttft_ms": 774.5,1723 "total_ms": 7696,1724 "tps": 89.85,1725 "cost_mean": 0.009618,1726 "out_tokens": 6221727 }1728 }1729 },1730 {1731 "model": "claude-haiku-5-5",1732 "provider": "anthropic",1733 "runs": 22,1734 "errors": 0,1735 "pass_rate": 1,1736 "mean_score": 1,1737 "judged_mean": 4.833333333333333,1738 "ttft_median_ms": 1301,1739 "total_median_ms": 2404,1740 "tps_median": 486.8,1741 "cost_total": 0.023120900000000003,1742 "cost_mean": 0.0010509500000000001,1743 "in_tokens": 168774,1744 "out_tokens": 12487,1745 "reasoning_tokens": 6979,1746 "by_category": {1747 "document_analysis": {1748 "runs": 2,1749 "errors": 0,1750 "pass_rate": 1,1751 "score": 1,1752 "judged": null,1753 "ttft_ms": 2934,1754 "total_ms": 3251,1755 "tps": 636.95,1756 "cost_mean": 0.00020275000000000002,1757 "out_tokens": 201.51758 },1759 "json_generation": {1760 "runs": 2,1761 "errors": 0,1762 "pass_rate": 1,1763 "score": 1,1764 "judged": null,1765 "ttft_ms": 1049,1766 "total_ms": 1319,1767 "tps": 389.35,1768 "cost_mean": 0.00010209999999999999,1769 "out_tokens": 1051770 },1771 "complex_reasoning": {1772 "runs": 2,1773 "errors": 0,1774 "pass_rate": 1,1775 "score": 1,1776 "judged": null,1777 "ttft_ms": 2643,1778 "total_ms": 4890,1779 "tps": 530.05,1780 "cost_mean": 0.00061815,1781 "out_tokens": 1189.51782 },1783 "long_context": {1784 "runs": 2,1785 "errors": 0,1786 "pass_rate": 1,1787 "score": 1,1788 "judged": null,1789 "ttft_ms": 2369.5,1790 "total_ms": 2404,1791 "tps": 4265.85,1792 "cost_mean": 0.008162550000000001,1793 "out_tokens": 142.51794 },1795 "english_writing": {1796 "runs": 2,1797 "errors": 0,1798 "pass_rate": 1,1799 "score": 1,1800 "judged": 5,1801 "ttft_ms": 442.5,1802 "total_ms": 2749,1803 "tps": 180.55,1804 "cost_mean": 0.00021715,1805 "out_tokens": 413.51806 },1807 "french_writing": {1808 "runs": 2,1809 "errors": 0,1810 "pass_rate": 1,1811 "score": 1,1812 "judged": 4.5,1813 "ttft_ms": 6442.5,1814 "total_ms": 7764,1815 "tps": 1654.15,1816 "cost_mean": 0.0011124,1817 "out_tokens": 21921818 },1819 "programming": {1820 "runs": 2,1821 "errors": 0,1822 "pass_rate": 1,1823 "score": 1,1824 "judged": null,1825 "ttft_ms": 345.5,1826 "total_ms": 888.5,1827 "tps": 351.25,1828 "cost_mean": 0.0001067,1829 "out_tokens": 1891830 },1831 "simple_reasoning": {1832 "runs": 2,1833 "errors": 0,1834 "pass_rate": 1,1835 "score": 1,1836 "judged": null,1837 "ttft_ms": 603,1838 "total_ms": 728,1839 "tps": 710.9,1840 "cost_mean": 0.0000566,1841 "out_tokens": 771842 },1843 "knowledge_synthesis": {1844 "runs": 2,1845 "errors": 0,1846 "pass_rate": 1,1847 "score": 1,1848 "judged": 5,1849 "ttft_ms": 4008,1850 "total_ms": 5274,1851 "tps": 890.55,1852 "cost_mean": 0.0005986,1853 "out_tokens": 11271854 },1855 "tool_calling": {1856 "runs": 2,1857 "errors": 0,1858 "pass_rate": 1,1859 "score": 1,1860 "judged": null,1861 "ttft_ms": 532.5,1862 "total_ms": 868,1863 "tps": 387.3,1864 "cost_mean": 0.0001337,1865 "out_tokens": 1301866 },1867 "mathematics": {1868 "runs": 2,1869 "errors": 0,1870 "pass_rate": 1,1871 "score": 1,1872 "judged": null,1873 "ttft_ms": 1145.5,1874 "total_ms": 2074.5,1875 "tps": 297.8,1876 "cost_mean": 0.00024974999999999997,1877 "out_tokens": 476.51878 }1879 }1880 },1881 {1882 "model": "gpt-5.6-sol",1883 "provider": "openai",1884 "runs": 22,1885 "errors": 0,1886 "pass_rate": 1,1887 "mean_score": 1,1888 "judged_mean": 5,1889 "ttft_median_ms": 1107.5,1890 "total_median_ms": 2466,1891 "tps_median": 132.5,1892 "cost_total": 0.3350544,1893 "cost_mean": 0.015229745454545453,1894 "in_tokens": 113620,1895 "out_tokens": 3964,1896 "reasoning_tokens": 1123,1897 "by_category": {1898 "document_analysis": {1899 "runs": 2,1900 "errors": 0,1901 "pass_rate": 1,1902 "score": 1,1903 "judged": null,1904 "ttft_ms": 774.5,1905 "total_ms": 1327.5,1906 "tps": 153.85,1907 "cost_mean": 0.0037,1908 "out_tokens": 851909 },1910 "json_generation": {1911 "runs": 2,1912 "errors": 0,1913 "pass_rate": 1,1914 "score": 1,1915 "judged": null,1916 "ttft_ms": 743.5,1917 "total_ms": 1333,1918 "tps": 113.75,1919 "cost_mean": 0.001948,1920 "out_tokens": 671921 },1922 "complex_reasoning": {1923 "runs": 2,1924 "errors": 0,1925 "pass_rate": 1,1926 "score": 1,1927 "judged": null,1928 "ttft_ms": 3244,1929 "total_ms": 4249.5,1930 "tps": 329.20000000000005,1931 "cost_mean": 0.00718,1932 "out_tokens": 3311933 },1934 "long_context": {1935 "runs": 2,1936 "errors": 0,1937 "pass_rate": 1,1938 "score": 1,1939 "judged": null,1940 "ttft_ms": 1641,1941 "total_ms": 1933,1942 "tps": 37.849999999999994,1943 "cost_mean": 0.1216632,1944 "out_tokens": 111945 },1946 "english_writing": {1947 "runs": 2,1948 "errors": 0,1949 "pass_rate": 1,1950 "score": 1,1951 "judged": 5,1952 "ttft_ms": 1089.5,1953 "total_ms": 3794,1954 "tps": 108.85,1955 "cost_mean": 0.006136,1956 "out_tokens": 2941957 },1958 "french_writing": {1959 "runs": 2,1960 "errors": 0,1961 "pass_rate": 1,1962 "score": 1,1963 "judged": 5,1964 "ttft_ms": 1252,1965 "total_ms": 4038,1966 "tps": 116.95,1967 "cost_mean": 0.006888,1968 "out_tokens": 3261969 },1970 "programming": {1971 "runs": 2,1972 "errors": 0,1973 "pass_rate": 1,1974 "score": 1,1975 "judged": null,1976 "ttft_ms": 1839.5,1977 "total_ms": 2626,1978 "tps": 271.20000000000005,1979 "cost_mean": 0.0044399999999999995,1980 "out_tokens": 2071981 },1982 "simple_reasoning": {1983 "runs": 2,1984 "errors": 0,1985 "pass_rate": 1,1986 "score": 1,1987 "judged": null,1988 "ttft_ms": 787,1989 "total_ms": 1087,1990 "tps": 63.3,1991 "cost_mean": 0.00088,1992 "out_tokens": 191993 },1994 "knowledge_synthesis": {1995 "runs": 2,1996 "errors": 0,1997 "pass_rate": 1,1998 "score": 1,1999 "judged": 5,2000 "ttft_ms": 1906,2001 "total_ms": 4141.5,2002 "tps": 152.5,2003 "cost_mean": 0.007743999999999999,2004 "out_tokens": 3412005 },2006 "tool_calling": {2007 "runs": 2,2008 "errors": 0,2009 "pass_rate": 1,2010 "score": 1,2011 "judged": null,2012 "ttft_ms": 860.5,2013 "total_ms": 1244,2014 "tps": 144.85000000000002,2015 "cost_mean": 0.001714,2016 "out_tokens": 55.52017 },2018 "mathematics": {2019 "runs": 2,2020 "errors": 0,2021 "pass_rate": 1,2022 "score": 1,2023 "judged": null,2024 "ttft_ms": 1170,2025 "total_ms": 2600,2026 "tps": 132.5,2027 "cost_mean": 0.005234,2028 "out_tokens": 245.52029 }2030 }2031 },2032 {2033 "model": "gpt-5.5",2034 "provider": "openai",2035 "runs": 22,2036 "errors": 0,2037 "pass_rate": 1,2038 "mean_score": 1,2039 "judged_mean": 4.833333333333333,2040 "ttft_median_ms": 1755,2041 "total_median_ms": 2472,2042 "tps_median": 146.55,2043 "cost_total": 0.4671800000000001,2044 "cost_mean": 0.02123545454545455,2045 "in_tokens": 113620,2046 "out_tokens": 4892,2047 "reasoning_tokens": 2460,2048 "by_category": {2049 "document_analysis": {2050 "runs": 2,2051 "errors": 0,2052 "pass_rate": 1,2053 "score": 1,2054 "judged": null,2055 "ttft_ms": 890,2056 "total_ms": 1613,2057 "tps": 118.1,2058 "cost_mean": 0.00505,2059 "out_tokens": 852060 },2061 "json_generation": {2062 "runs": 2,2063 "errors": 0,2064 "pass_rate": 1,2065 "score": 1,2066 "judged": null,2067 "ttft_ms": 1008.5,2068 "total_ms": 1639,2069 "tps": 139.89999999999998,2070 "cost_mean": 0.003385,2071 "out_tokens": 87.52072 },2073 "complex_reasoning": {2074 "runs": 2,2075 "errors": 0,2076 "pass_rate": 1,2077 "score": 1,2078 "judged": null,2079 "ttft_ms": 4300.5,2080 "total_ms": 5810.5,2081 "tps": 255.85000000000002,2082 "cost_mean": 0.01225,2083 "out_tokens": 3852084 },2085 "long_context": {2086 "runs": 2,2087 "errors": 0,2088 "pass_rate": 1,2089 "score": 1,2090 "judged": null,2091 "ttft_ms": 2015.5,2092 "total_ms": 2255,2093 "tps": 403,2094 "cost_mean": 0.15496000000000001,2095 "out_tokens": 93.52096 },2097 "english_writing": {2098 "runs": 2,2099 "errors": 0,2100 "pass_rate": 1,2101 "score": 1,2102 "judged": 5,2103 "ttft_ms": 1137,2104 "total_ms": 4611,2105 "tps": 71.25,2106 "cost_mean": 0.007730000000000001,2107 "out_tokens": 2472108 },2109 "french_writing": {2110 "runs": 2,2111 "errors": 0,2112 "pass_rate": 1,2113 "score": 1,2114 "judged": 5,2115 "ttft_ms": 9677,2116 "total_ms": 13390.5,2117 "tps": 213.05,2118 "cost_mean": 0.02245,2119 "out_tokens": 7332120 },2121 "programming": {2122 "runs": 2,2123 "errors": 0,2124 "pass_rate": 1,2125 "score": 1,2126 "judged": null,2127 "ttft_ms": 2119,2128 "total_ms": 3176,2129 "tps": 199.85000000000002,2130 "cost_mean": 0.00669,2131 "out_tokens": 210.52132 },2133 "simple_reasoning": {2134 "runs": 2,2135 "errors": 0,2136 "pass_rate": 1,2137 "score": 1,2138 "judged": null,2139 "ttft_ms": 1398,2140 "total_ms": 1776.5,2141 "tps": 191.60000000000002,2142 "cost_mean": 0.002785,2143 "out_tokens": 722144 },2145 "knowledge_synthesis": {2146 "runs": 2,2147 "errors": 0,2148 "pass_rate": 1,2149 "score": 1,2150 "judged": 4.5,2151 "ttft_ms": 2466.5,2152 "total_ms": 4838.5,2153 "tps": 160.9,2154 "cost_mean": 0.012209999999999999,2155 "out_tokens": 368.52156 },2157 "tool_calling": {2158 "runs": 2,2159 "errors": 0,2160 "pass_rate": 1,2161 "score": 1,2162 "judged": null,2163 "ttft_ms": 894.5,2164 "total_ms": 1262,2165 "tps": 111.7,2166 "cost_mean": 0.001985,2167 "out_tokens": 412168 },2169 "mathematics": {2170 "runs": 2,2171 "errors": 0,2172 "pass_rate": 1,2173 "score": 1,2174 "judged": null,2175 "ttft_ms": 1952.5,2176 "total_ms": 2295.5,2177 "tps": 79.65,2178 "cost_mean": 0.004095,2179 "out_tokens": 1232180 }2181 }2182 },2183 {2184 "model": "claude-opus-4-5-20251101",2185 "provider": "anthropic",2186 "runs": 22,2187 "errors": 0,2188 "pass_rate": 1,2189 "mean_score": 1,2190 "judged_mean": 4.666666666666667,2191 "ttft_median_ms": 861.5,2192 "total_median_ms": 2475.5,2193 "tps_median": 97.8,2194 "cost_total": 0.78326,2195 "cost_mean": 0.03560272727272727,2196 "in_tokens": 129532,2197 "out_tokens": 5424,2198 "reasoning_tokens": 0,2199 "by_category": {2200 "document_analysis": {2201 "runs": 2,2202 "errors": 0,2203 "pass_rate": 1,2204 "score": 1,2205 "judged": null,2206 "ttft_ms": 861.5,2207 "total_ms": 1650.5,2208 "tps": 128,2209 "cost_mean": 0.006725,2210 "out_tokens": 1012211 },2212 "json_generation": {2213 "runs": 2,2214 "errors": 0,2215 "pass_rate": 1,2216 "score": 1,2217 "judged": null,2218 "ttft_ms": 864.5,2219 "total_ms": 1384.5,2220 "tps": 132.7,2221 "cost_mean": 0.003715,2222 "out_tokens": 692223 },2224 "complex_reasoning": {2225 "runs": 2,2226 "errors": 0,2227 "pass_rate": 1,2228 "score": 1,2229 "judged": null,2230 "ttft_ms": 816,2231 "total_ms": 9210.5,2232 "tps": 83.25,2233 "cost_mean": 0.01825,2234 "out_tokens": 6982235 },2236 "long_context": {2237 "runs": 2,2238 "errors": 0,2239 "pass_rate": 1,2240 "score": 1,2241 "judged": null,2242 "ttft_ms": 1994.5,2243 "total_ms": 2146,2244 "tps": 85.9,2245 "cost_mean": 0.30915,2246 "out_tokens": 132247 },2248 "english_writing": {2249 "runs": 2,2250 "errors": 0,2251 "pass_rate": 1,2252 "score": 1,2253 "judged": 5,2254 "ttft_ms": 646.5,2255 "total_ms": 6864,2256 "tps": 42.2,2257 "cost_mean": 0.006932499999999999,2258 "out_tokens": 262.52259 },2260 "french_writing": {2261 "runs": 2,2262 "errors": 0,2263 "pass_rate": 1,2264 "score": 1,2265 "judged": 5,2266 "ttft_ms": 656.5,2267 "total_ms": 7292.5,2268 "tps": 57.65,2269 "cost_mean": 0.010187499999999999,2270 "out_tokens": 382.52271 },2272 "programming": {2273 "runs": 2,2274 "errors": 0,2275 "pass_rate": 1,2276 "score": 1,2277 "judged": null,2278 "ttft_ms": 820,2279 "total_ms": 2475.5,2280 "tps": 104.55000000000001,2281 "cost_mean": 0.00473,2282 "out_tokens": 1732283 },2284 "simple_reasoning": {2285 "runs": 2,2286 "errors": 0,2287 "pass_rate": 1,2288 "score": 1,2289 "judged": null,2290 "ttft_ms": 1088,2291 "total_ms": 1182.5,2292 "tps": 210.9,2293 "cost_mean": 0.001235,2294 "out_tokens": 202295 },2296 "knowledge_synthesis": {2297 "runs": 2,2298 "errors": 0,2299 "pass_rate": 1,2300 "score": 1,2301 "judged": 4,2302 "ttft_ms": 990.5,2303 "total_ms": 7481.5,2304 "tps": 48.150000000000006,2305 "cost_mean": 0.009142500000000001,2306 "out_tokens": 312.52307 },2308 "tool_calling": {2309 "runs": 2,2310 "errors": 0,2311 "pass_rate": 1,2312 "score": 1,2313 "judged": null,2314 "ttft_ms": 1352.5,2315 "total_ms": 1748.5,2316 "tps": 303,2317 "cost_mean": 0.00707,2318 "out_tokens": 1202319 },2320 "mathematics": {2321 "runs": 2,2322 "errors": 0,2323 "pass_rate": 1,2324 "score": 1,2325 "judged": null,2326 "ttft_ms": 571.5,2327 "total_ms": 6305.5,2328 "tps": 97.8,2329 "cost_mean": 0.014492499999999998,2330 "out_tokens": 560.52331 }2332 }2333 },2334 {2335 "model": "claude-opus-5-5",2336 "provider": "anthropic",2337 "runs": 22,2338 "errors": 0,2339 "pass_rate": 1,2340 "mean_score": 1,2341 "judged_mean": 4.833333333333333,2342 "ttft_median_ms": 1703,2343 "total_median_ms": 2782.5,2344 "tps_median": 160.6,2345 "cost_total": 0.9427159999999999,2346 "cost_mean": 0.04285072727272727,2347 "in_tokens": 168774,2348 "out_tokens": 13381,2349 "reasoning_tokens": 7584,2350 "by_category": {2351 "document_analysis": {2352 "runs": 2,2353 "errors": 0,2354 "pass_rate": 1,2355 "score": 1,2356 "judged": null,2357 "ttft_ms": 1019.5,2358 "total_ms": 1820.5,2359 "tps": 147.95,2360 "cost_mean": 0.00644,2361 "out_tokens": 1182362 },2363 "json_generation": {2364 "runs": 2,2365 "errors": 0,2366 "pass_rate": 1,2367 "score": 1,2368 "judged": null,2369 "ttft_ms": 1703,2370 "total_ms": 2361.5,2371 "tps": 159.35,2372 "cost_mean": 0.004084,2373 "out_tokens": 1052374 },2375 "complex_reasoning": {2376 "runs": 2,2377 "errors": 0,2378 "pass_rate": 1,2379 "score": 1,2380 "judged": null,2381 "ttft_ms": 2424,2382 "total_ms": 6117,2383 "tps": 184.45,2384 "cost_mean": 0.014536,2385 "out_tokens": 6802386 },2387 "long_context": {2388 "runs": 2,2389 "errors": 0,2390 "pass_rate": 1,2391 "score": 1,2392 "judged": null,2393 "ttft_ms": 2306.5,2394 "total_ms": 2336,2395 "tps": 375.15,2396 "cost_mean": 0.323872,2397 "out_tokens": 112398 },2399 "english_writing": {2400 "runs": 2,2401 "errors": 0,2402 "pass_rate": 1,2403 "score": 1,2404 "judged": 5,2405 "ttft_ms": 12624.5,2406 "total_ms": 15484,2407 "tps": 667.1500000000001,2408 "cost_mean": 0.038606,2409 "out_tokens": 1909.52410 },2411 "french_writing": {2412 "runs": 2,2413 "errors": 0,2414 "pass_rate": 1,2415 "score": 1,2416 "judged": 5,2417 "ttft_ms": 14963,2418 "total_ms": 18435.5,2419 "tps": 698.8,2420 "cost_mean": 0.049145999999999995,2421 "out_tokens": 2424.52422 },2423 "programming": {2424 "runs": 2,2425 "errors": 0,2426 "pass_rate": 1,2427 "score": 1,2428 "judged": null,2429 "ttft_ms": 1170,2430 "total_ms": 2647.5,2431 "tps": 185.3,2432 "cost_mean": 0.005938000000000001,2433 "out_tokens": 272.52434 },2435 "simple_reasoning": {2436 "runs": 2,2437 "errors": 0,2438 "pass_rate": 1,2439 "score": 1,2440 "judged": null,2441 "ttft_ms": 835.5,2442 "total_ms": 1282.5,2443 "tps": 132.1,2444 "cost_mean": 0.001904,2445 "out_tokens": 592446 },2447 "knowledge_synthesis": {2448 "runs": 2,2449 "errors": 0,2450 "pass_rate": 1,2451 "score": 1,2452 "judged": 4.5,2453 "ttft_ms": 2205,2454 "total_ms": 5931.5,2455 "tps": 145.45,2456 "cost_mean": 0.012244,2457 "out_tokens": 5422458 },2459 "tool_calling": {2460 "runs": 2,2461 "errors": 0,2462 "pass_rate": 1,2463 "score": 1,2464 "judged": null,2465 "ttft_ms": 946,2466 "total_ms": 1873,2467 "tps": 140.14999999999998,2468 "cost_mean": 0.005348,2469 "out_tokens": 1302470 },2471 "mathematics": {2472 "runs": 2,2473 "errors": 0,2474 "pass_rate": 1,2475 "score": 1,2476 "judged": null,2477 "ttft_ms": 901.5,2478 "total_ms": 3753,2479 "tps": 134.05,2480 "cost_mean": 0.00924,2481 "out_tokens": 4392482 }2483 }2484 },2485 {2486 "model": "claude-sonnet-5-5",2487 "provider": "anthropic",2488 "runs": 22,2489 "errors": 0,2490 "pass_rate": 1,2491 "mean_score": 1,2492 "judged_mean": 4.666666666666667,2493 "ttft_median_ms": 1019.5,2494 "total_median_ms": 2853,2495 "tps_median": 201.95,2496 "cost_total": 0.432488,2497 "cost_mean": 0.019658545454545453,2498 "in_tokens": 168774,2499 "out_tokens": 9494,2500 "reasoning_tokens": 2881,2501 "by_category": {2502 "document_analysis": {2503 "runs": 2,2504 "errors": 0,2505 "pass_rate": 1,2506 "score": 1,2507 "judged": null,2508 "ttft_ms": 1609.5,2509 "total_ms": 2040.5,2510 "tps": 275.05,2511 "cost_mean": 0.00322,2512 "out_tokens": 1182513 },2514 "json_generation": {2515 "runs": 2,2516 "errors": 0,2517 "pass_rate": 1,2518 "score": 1,2519 "judged": null,2520 "ttft_ms": 1719,2521 "total_ms": 2151.5,2522 "tps": 242.89999999999998,2523 "cost_mean": 0.002042,2524 "out_tokens": 1052525 },2526 "complex_reasoning": {2527 "runs": 2,2528 "errors": 0,2529 "pass_rate": 1,2530 "score": 1,2531 "judged": null,2532 "ttft_ms": 2570.5,2533 "total_ms": 5096,2534 "tps": 297.4,2535 "cost_mean": 0.007977999999999999,2536 "out_tokens": 7512537 },2538 "long_context": {2539 "runs": 2,2540 "errors": 0,2541 "pass_rate": 1,2542 "score": 1,2543 "judged": null,2544 "ttft_ms": 2243,2545 "total_ms": 2270,2546 "tps": 415,2547 "cost_mean": 0.161936,2548 "out_tokens": 112549 },2550 "english_writing": {2551 "runs": 2,2552 "errors": 0,2553 "pass_rate": 1,2554 "score": 1,2555 "judged": 5,2556 "ttft_ms": 463,2557 "total_ms": 4608.5,2558 "tps": 115.1,2559 "cost_mean": 0.004968,2560 "out_tokens": 4762561 },2562 "french_writing": {2563 "runs": 2,2564 "errors": 0,2565 "pass_rate": 1,2566 "score": 1,2567 "judged": 5,2568 "ttft_ms": 7827.5,2569 "total_ms": 10002,2570 "tps": 768.95,2571 "cost_mean": 0.016973000000000002,2572 "out_tokens": 1664.52573 },2574 "programming": {2575 "runs": 2,2576 "errors": 0,2577 "pass_rate": 1,2578 "score": 1,2579 "judged": null,2580 "ttft_ms": 798,2581 "total_ms": 4436.5,2582 "tps": 201.95,2583 "cost_mean": 0.007589,2584 "out_tokens": 734.52585 },2586 "simple_reasoning": {2587 "runs": 2,2588 "errors": 0,2589 "pass_rate": 1,2590 "score": 1,2591 "judged": null,2592 "ttft_ms": 688.5,2593 "total_ms": 1222,2594 "tps": 35.599999999999994,2595 "cost_mean": 0.000552,2596 "out_tokens": 192597 },2598 "knowledge_synthesis": {2599 "runs": 2,2600 "errors": 0,2601 "pass_rate": 1,2602 "score": 1,2603 "judged": 4,2604 "ttft_ms": 853,2605 "total_ms": 3735.5,2606 "tps": 146.15,2607 "cost_mean": 0.004912,2608 "out_tokens": 4212609 },2610 "tool_calling": {2611 "runs": 2,2612 "errors": 0,2613 "pass_rate": 1,2614 "score": 1,2615 "judged": null,2616 "ttft_ms": 766.5,2617 "total_ms": 1619,2618 "tps": 152.5,2619 "cost_mean": 0.002674,2620 "out_tokens": 1302621 },2622 "mathematics": {2623 "runs": 2,2624 "errors": 0,2625 "pass_rate": 1,2626 "score": 1,2627 "judged": null,2628 "ttft_ms": 808.5,2629 "total_ms": 2853,2630 "tps": 155.05,2631 "cost_mean": 0.0034000000000000002,2632 "out_tokens": 3172633 }2634 }2635 },2636 {2637 "model": "claude-opus-4-8",2638 "provider": "anthropic",2639 "runs": 22,2640 "errors": 0,2641 "pass_rate": 1,2642 "mean_score": 1,2643 "judged_mean": 4.666666666666667,2644 "ttft_median_ms": 487.5,2645 "total_median_ms": 2865,2646 "tps_median": 103.8,2647 "cost_total": 0.9819399999999999,2648 "cost_mean": 0.04463363636363636,2649 "in_tokens": 168738,2650 "out_tokens": 5530,2651 "reasoning_tokens": 0,2652 "by_category": {2653 "document_analysis": {2654 "runs": 2,2655 "errors": 0,2656 "pass_rate": 1,2657 "score": 1,2658 "judged": null,2659 "ttft_ms": 2111,2660 "total_ms": 3173.5,2661 "tps": 110.15,2662 "cost_mean": 0.008015,2663 "out_tokens": 1172664 },2665 "json_generation": {2666 "runs": 2,2667 "errors": 0,2668 "pass_rate": 1,2669 "score": 1,2670 "judged": null,2671 "ttft_ms": 1108.5,2672 "total_ms": 2021,2673 "tps": 114,2674 "cost_mean": 0.00507,2675 "out_tokens": 1042676 },2677 "complex_reasoning": {2678 "runs": 2,2679 "errors": 0,2680 "pass_rate": 1,2681 "score": 1,2682 "judged": null,2683 "ttft_ms": 455,2684 "total_ms": 5179,2685 "tps": 97.75,2686 "cost_mean": 0.0126975,2687 "out_tokens": 461.52688 },2689 "long_context": {2690 "runs": 2,2691 "errors": 0,2692 "pass_rate": 1,2693 "score": 1,2694 "judged": null,2695 "ttft_ms": 1920,2696 "total_ms": 2182,2697 "tps": 41.95,2698 "cost_mean": 0.40483,2699 "out_tokens": 112700 },2701 "english_writing": {2702 "runs": 2,2703 "errors": 0,2704 "pass_rate": 1,2705 "score": 1,2706 "judged": 5,2707 "ttft_ms": 454.5,2708 "total_ms": 7459,2709 "tps": 64.65,2710 "cost_mean": 0.0117725,2711 "out_tokens": 450.52712 },2713 "french_writing": {2714 "runs": 2,2715 "errors": 0,2716 "pass_rate": 1,2717 "score": 1,2718 "judged": 5,2719 "ttft_ms": 447.5,2720 "total_ms": 9605,2721 "tps": 59.7,2722 "cost_mean": 0.014447499999999999,2723 "out_tokens": 545.52724 },2725 "programming": {2726 "runs": 2,2727 "errors": 0,2728 "pass_rate": 1,2729 "score": 1,2730 "judged": null,2731 "ttft_ms": 553.5,2732 "total_ms": 1918.5,2733 "tps": 125.75,2734 "cost_mean": 0.0048874999999999995,2735 "out_tokens": 171.52736 },2737 "simple_reasoning": {2738 "runs": 2,2739 "errors": 0,2740 "pass_rate": 1,2741 "score": 1,2742 "judged": null,2743 "ttft_ms": 486,2744 "total_ms": 662,2745 "tps": 110.19999999999999,2746 "cost_mean": 0.00137,2747 "out_tokens": 192748 },2749 "knowledge_synthesis": {2750 "runs": 2,2751 "errors": 0,2752 "pass_rate": 1,2753 "score": 1,2754 "judged": 4,2755 "ttft_ms": 495.5,2756 "total_ms": 5562.5,2757 "tps": 87.4,2758 "cost_mean": 0.012782499999999999,2759 "out_tokens": 441.52760 },2761 "tool_calling": {2762 "runs": 2,2763 "errors": 0,2764 "pass_rate": 1,2765 "score": 1,2766 "judged": null,2767 "ttft_ms": 508.5,2768 "total_ms": 1669,2769 "tps": 124.19999999999999,2770 "cost_mean": 0.007045,2771 "out_tokens": 1442772 },2773 "mathematics": {2774 "runs": 2,2775 "errors": 0,2776 "pass_rate": 1,2777 "score": 1,2778 "judged": null,2779 "ttft_ms": 435.5,2780 "total_ms": 3139.5,2781 "tps": 110.95,2782 "cost_mean": 0.0080525,2783 "out_tokens": 299.52784 }2785 }2786 },2787 {2788 "model": "o3",2789 "provider": "openai",2790 "runs": 22,2791 "errors": 0,2792 "pass_rate": 1,2793 "mean_score": 1,2794 "judged_mean": 4.333333333333333,2795 "ttft_median_ms": 1762.5,2796 "total_median_ms": 2943.5,2797 "tps_median": 533,2798 "cost_total": 0.32102,2799 "cost_mean": 0.014591818181818183,2800 "in_tokens": 113614,2801 "out_tokens": 11724,2802 "reasoning_tokens": 8512,2803 "by_category": {2804 "document_analysis": {2805 "runs": 2,2806 "errors": 0,2807 "pass_rate": 1,2808 "score": 1,2809 "judged": null,2810 "ttft_ms": 1638,2811 "total_ms": 2220.5,2812 "tps": 592.3499999999999,2813 "cost_mean": 0.003548,2814 "out_tokens": 318.52815 },2816 "json_generation": {2817 "runs": 2,2818 "errors": 0,2819 "pass_rate": 1,2820 "score": 1,2821 "judged": null,2822 "ttft_ms": 1224.5,2823 "total_ms": 1664.5,2824 "tps": 454.9,2825 "cost_mean": 0.00186,2826 "out_tokens": 194.52827 },2828 "complex_reasoning": {2829 "runs": 2,2830 "errors": 0,2831 "pass_rate": 1,2832 "score": 1,2833 "judged": null,2834 "ttft_ms": 5316,2835 "total_ms": 5720,2836 "tps": 2221.35,2837 "cost_mean": 0.007516,2838 "out_tokens": 904.52839 },2840 "long_context": {2841 "runs": 2,2842 "errors": 0,2843 "pass_rate": 1,2844 "score": 1,2845 "judged": null,2846 "ttft_ms": 3176,2847 "total_ms": 3456.5,2848 "tps": 423.2,2849 "cost_mean": 0.111322,2850 "out_tokens": 115.52851 },2852 "english_writing": {2853 "runs": 2,2854 "errors": 0,2855 "pass_rate": 1,2856 "score": 1,2857 "judged": 5,2858 "ttft_ms": 2516,2859 "total_ms": 4191.5,2860 "tps": 468.55,2861 "cost_mean": 0.005572,2862 "out_tokens": 680.52863 },2864 "french_writing": {2865 "runs": 2,2866 "errors": 0,2867 "pass_rate": 1,2868 "score": 1,2869 "judged": 4.5,2870 "ttft_ms": 6182.5,2871 "total_ms": 7472,2872 "tps": 1105.05,2873 "cost_mean": 0.011652,2874 "out_tokens": 1433.52875 },2876 "programming": {2877 "runs": 2,2878 "errors": 0,2879 "pass_rate": 1,2880 "score": 1,2881 "judged": null,2882 "ttft_ms": 1925.5,2883 "total_ms": 2572,2884 "tps": 701.0999999999999,2885 "cost_mean": 0.003778,2886 "out_tokens": 453.52887 },2888 "simple_reasoning": {2889 "runs": 2,2890 "errors": 0,2891 "pass_rate": 1,2892 "score": 1,2893 "judged": null,2894 "ttft_ms": 1455.5,2895 "total_ms": 1748.5,2896 "tps": 1004.5,2897 "cost_mean": 0.002498,2898 "out_tokens": 2812899 },2900 "knowledge_synthesis": {2901 "runs": 2,2902 "errors": 0,2903 "pass_rate": 1,2904 "score": 1,2905 "judged": 3.5,2906 "ttft_ms": 1336,2907 "total_ms": 3022,2908 "tps": 261.75,2909 "cost_mean": 0.00395,2910 "out_tokens": 4362911 },2912 "tool_calling": {2913 "runs": 2,2914 "errors": 0,2915 "pass_rate": 1,2916 "score": 1,2917 "judged": null,2918 "ttft_ms": 742,2919 "total_ms": 1044,2920 "tps": 317.95000000000005,2921 "cost_mean": 0.00106,2922 "out_tokens": 95.52923 },2924 "mathematics": {2925 "runs": 2,2926 "errors": 0,2927 "pass_rate": 1,2928 "score": 1,2929 "judged": null,2930 "ttft_ms": 3940.5,2931 "total_ms": 4663.5,2932 "tps": 231.4,2933 "cost_mean": 0.0077540000000000005,2934 "out_tokens": 9492935 }2936 }2937 },2938 {2939 "model": "gpt-6-astra",2940 "provider": "openai",2941 "runs": 22,2942 "errors": 0,2943 "pass_rate": 1,2944 "mean_score": 1,2945 "judged_mean": 5,2946 "ttft_median_ms": 2376,2947 "total_median_ms": 3056.5,2948 "tps_median": 77.85,2949 "cost_total": 0.809736,2950 "cost_mean": 0.03680618181818182,2951 "in_tokens": 113620,2952 "out_tokens": 3406,2953 "reasoning_tokens": 776,2954 "by_category": {2955 "document_analysis": {2956 "runs": 2,2957 "errors": 0,2958 "pass_rate": 1,2959 "score": 1,2960 "judged": null,2961 "ttft_ms": 1307.5,2962 "total_ms": 2154.5,2963 "tps": 126.35000000000001,2964 "cost_mean": 0.01035,2965 "out_tokens": 1072966 },2967 "json_generation": {2968 "runs": 2,2969 "errors": 0,2970 "pass_rate": 1,2971 "score": 1,2972 "judged": null,2973 "ttft_ms": 1995,2974 "total_ms": 2804.5,2975 "tps": 93.5,2976 "cost_mean": 0.00487,2977 "out_tokens": 672978 },2979 "complex_reasoning": {2980 "runs": 2,2981 "errors": 0,2982 "pass_rate": 1,2983 "score": 1,2984 "judged": null,2985 "ttft_ms": 3496,2986 "total_ms": 6345,2987 "tps": 96.95,2988 "cost_mean": 0.015224999999999999,2989 "out_tokens": 276.52990 },2991 "long_context": {2992 "runs": 2,2993 "errors": 0,2994 "pass_rate": 1,2995 "score": 1,2996 "judged": null,2997 "ttft_ms": 2465.5,2998 "total_ms": 2827.5,2999 "tps": 30.4,3000 "cost_mean": 0.30415800000000004,3001 "out_tokens": 113002 },3003 "english_writing": {3004 "runs": 2,3005 "errors": 0,3006 "pass_rate": 1,3007 "score": 1,3008 "judged": 5,3009 "ttft_ms": 2419.5,3010 "total_ms": 8367.5,3011 "tps": 43.45,3012 "cost_mean": 0.013565,3013 "out_tokens": 258.53014 },3015 "french_writing": {3016 "runs": 2,3017 "errors": 0,3018 "pass_rate": 1,3019 "score": 1,3020 "judged": 5,3021 "ttft_ms": 2570,3022 "total_ms": 9359,3023 "tps": 45.150000000000006,3024 "cost_mean": 0.016245,3025 "out_tokens": 306.53026 },3027 "programming": {3028 "runs": 2,3029 "errors": 0,3030 "pass_rate": 1,3031 "score": 1,3032 "judged": null,3033 "ttft_ms": 3521,3034 "total_ms": 4526,3035 "tps": 208.55,3036 "cost_mean": 0.011300000000000001,3037 "out_tokens": 2113038 },3039 "simple_reasoning": {3040 "runs": 2,3041 "errors": 0,3042 "pass_rate": 1,3043 "score": 1,3044 "judged": null,3045 "ttft_ms": 1421.5,3046 "total_ms": 1873.5,3047 "tps": 93.1,3048 "cost_mean": 0.00305,3049 "out_tokens": 363050 },3051 "knowledge_synthesis": {3052 "runs": 2,3053 "errors": 0,3054 "pass_rate": 1,3055 "score": 1,3056 "judged": 5,3057 "ttft_ms": 2987.5,3058 "total_ms": 7010.5,3059 "tps": 76,3060 "cost_mean": 0.01761,3061 "out_tokens": 3063062 },3063 "tool_calling": {3064 "runs": 2,3065 "errors": 0,3066 "pass_rate": 1,3067 "score": 1,3068 "judged": null,3069 "ttft_ms": 1530.5,3070 "total_ms": 1973,3071 "tps": 93.94999999999999,3072 "cost_mean": 0.00356,3073 "out_tokens": 413074 },3075 "mathematics": {3076 "runs": 2,3077 "errors": 0,3078 "pass_rate": 1,3079 "score": 1,3080 "judged": null,3081 "ttft_ms": 2493,3082 "total_ms": 2858.5,3083 "tps": 75.15,3084 "cost_mean": 0.004935,3085 "out_tokens": 82.53086 }3087 }3088 },3089 {3090 "model": "gpt-6.1-terra",3091 "provider": "openai",3092 "runs": 22,3093 "errors": 0,3094 "pass_rate": 1,3095 "mean_score": 1,3096 "judged_mean": 4.666666666666667,3097 "ttft_median_ms": 1514.5,3098 "total_median_ms": 3151,3099 "tps_median": 92.6,3100 "cost_total": 0.0809488,3101 "cost_mean": 0.003679490909090909,3102 "in_tokens": 113620,3103 "out_tokens": 3953,3104 "reasoning_tokens": 1303,3105 "by_category": {3106 "document_analysis": {3107 "runs": 2,3108 "errors": 0,3109 "pass_rate": 1,3110 "score": 1,3111 "judged": null,3112 "ttft_ms": 888.5,3113 "total_ms": 1951,3114 "tps": 80,3115 "cost_mean": 0.000925,3116 "out_tokens": 853117 },3118 "json_generation": {3119 "runs": 2,3120 "errors": 0,3121 "pass_rate": 1,3122 "score": 1,3123 "judged": null,3124 "ttft_ms": 1085.5,3125 "total_ms": 2026.5,3126 "tps": 84.05000000000001,3127 "cost_mean": 0.0005395,3128 "out_tokens": 77.53129 },3130 "complex_reasoning": {3131 "runs": 2,3132 "errors": 0,3133 "pass_rate": 1,3134 "score": 1,3135 "judged": null,3136 "ttft_ms": 2656.5,3137 "total_ms": 4411,3138 "tps": 174.5,3139 "cost_mean": 0.0016675000000000001,3140 "out_tokens": 305.53141 },3142 "long_context": {3143 "runs": 2,3144 "errors": 0,3145 "pass_rate": 1,3146 "score": 1,3147 "judged": null,3148 "ttft_ms": 921.5,3149 "total_ms": 1217,3150 "tps": 37.45,3151 "cost_mean": 0.0290359,3152 "out_tokens": 113153 },3154 "english_writing": {3155 "runs": 2,3156 "errors": 0,3157 "pass_rate": 1,3158 "score": 1,3159 "judged": 5,3160 "ttft_ms": 2098.5,3161 "total_ms": 5373.5,3162 "tps": 94.35,3163 "cost_mean": 0.0015965,3164 "out_tokens": 306.53165 },3166 "french_writing": {3167 "runs": 2,3168 "errors": 0,3169 "pass_rate": 1,3170 "score": 1,3171 "judged": 5,3172 "ttft_ms": 1514.5,3173 "total_ms": 4905,3174 "tps": 95.7,3175 "cost_mean": 0.0017144999999999999,3176 "out_tokens": 324.53177 },3178 "programming": {3179 "runs": 2,3180 "errors": 0,3181 "pass_rate": 1,3182 "score": 1,3183 "judged": null,3184 "ttft_ms": 1833.5,3185 "total_ms": 3197,3186 "tps": 142.4,3187 "cost_mean": 0.0010400000000000001,3188 "out_tokens": 1933189 },3190 "simple_reasoning": {3191 "runs": 2,3192 "errors": 0,3193 "pass_rate": 1,3194 "score": 1,3195 "judged": null,3196 "ttft_ms": 1024.5,3197 "total_ms": 1447,3198 "tps": 104.19999999999999,3199 "cost_mean": 0.00034250000000000003,3200 "out_tokens": 43.53201 },3202 "knowledge_synthesis": {3203 "runs": 2,3204 "errors": 0,3205 "pass_rate": 1,3206 "score": 1,3207 "judged": 4,3208 "ttft_ms": 2715.5,3209 "total_ms": 5120.5,3210 "tps": 157.15,3211 "cost_mean": 0.0021260000000000003,3212 "out_tokens": 3793213 },3214 "tool_calling": {3215 "runs": 2,3216 "errors": 0,3217 "pass_rate": 1,3218 "score": 1,3219 "judged": null,3220 "ttft_ms": 821,3221 "total_ms": 1412,3222 "tps": 69.4,3223 "cost_mean": 0.000356,3224 "out_tokens": 413225 },3226 "mathematics": {3227 "runs": 2,3228 "errors": 0,3229 "pass_rate": 1,3230 "score": 1,3231 "judged": null,3232 "ttft_ms": 1954,3233 "total_ms": 3430,3234 "tps": 80.6,3235 "cost_mean": 0.001131,3236 "out_tokens": 2103237 }3238 }3239 },3240 {3241 "model": "claude-opus-4-7",3242 "provider": "anthropic",3243 "runs": 22,3244 "errors": 0,3245 "pass_rate": 1,3246 "mean_score": 1,3247 "judged_mean": 4.333333333333333,3248 "ttft_median_ms": 686.5,3249 "total_median_ms": 3320.5,3250 "tps_median": 91.35,3251 "cost_total": 0.9817649999999998,3252 "cost_mean": 0.04462568181818181,3253 "in_tokens": 169618,3254 "out_tokens": 5347,3255 "reasoning_tokens": 0,3256 "by_category": {3257 "document_analysis": {3258 "runs": 2,3259 "errors": 0,3260 "pass_rate": 1,3261 "score": 1,3262 "judged": null,3263 "ttft_ms": 2202,3264 "total_ms": 3266.5,3265 "tps": 112.80000000000001,3266 "cost_mean": 0.008115,3267 "out_tokens": 1203268 },3269 "json_generation": {3270 "runs": 2,3271 "errors": 0,3272 "pass_rate": 1,3273 "score": 1,3274 "judged": null,3275 "ttft_ms": 1585,3276 "total_ms": 2583.5,3277 "tps": 107.1,3278 "cost_mean": 0.00517,3279 "out_tokens": 1073280 },3281 "complex_reasoning": {3282 "runs": 2,3283 "errors": 0,3284 "pass_rate": 1,3285 "score": 1,3286 "judged": null,3287 "ttft_ms": 613,3288 "total_ms": 4529.5,3289 "tps": 91.35,3290 "cost_mean": 0.010110000000000001,3291 "out_tokens": 3573292 },3293 "long_context": {3294 "runs": 2,3295 "errors": 0,3296 "pass_rate": 1,3297 "score": 1,3298 "judged": null,3299 "ttft_ms": 2775,3300 "total_ms": 2984.5,3301 "tps": 67.3,3302 "cost_mean": 0.40493,3303 "out_tokens": 143304 },3305 "english_writing": {3306 "runs": 2,3307 "errors": 0,3308 "pass_rate": 1,3309 "score": 1,3310 "judged": 5,3311 "ttft_ms": 531,3312 "total_ms": 8492.5,3313 "tps": 59.25,3314 "cost_mean": 0.0123225,3315 "out_tokens": 471.53316 },3317 "french_writing": {3318 "runs": 2,3319 "errors": 0,3320 "pass_rate": 1,3321 "score": 1,3322 "judged": 5,3323 "ttft_ms": 1160,3324 "total_ms": 11929.5,3325 "tps": 52.95,3326 "cost_mean": 0.015085000000000001,3327 "out_tokens": 5703328 },3329 "programming": {3330 "runs": 2,3331 "errors": 0,3332 "pass_rate": 1,3333 "score": 1,3334 "judged": null,3335 "ttft_ms": 479.5,3336 "total_ms": 1789.5,3337 "tps": 125.19999999999999,3338 "cost_mean": 0.004725,3339 "out_tokens": 1643340 },3341 "simple_reasoning": {3342 "runs": 2,3343 "errors": 0,3344 "pass_rate": 1,3345 "score": 1,3346 "judged": null,3347 "ttft_ms": 734.5,3348 "total_ms": 894.5,3349 "tps": 159.8,3350 "cost_mean": 0.00147,3351 "out_tokens": 223352 },3353 "knowledge_synthesis": {3354 "runs": 2,3355 "errors": 0,3356 "pass_rate": 1,3357 "score": 1,3358 "judged": 3,3359 "ttft_ms": 574.5,3360 "total_ms": 6505,3361 "tps": 71.55000000000001,3362 "cost_mean": 0.012344999999999998,3363 "out_tokens": 4233364 },3365 "tool_calling": {3366 "runs": 2,3367 "errors": 0,3368 "pass_rate": 1,3369 "score": 1,3370 "judged": null,3371 "ttft_ms": 992,3372 "total_ms": 1948.5,3373 "tps": 152.64999999999998,3374 "cost_mean": 0.009045,3375 "out_tokens": 1463376 },3377 "mathematics": {3378 "runs": 2,3379 "errors": 0,3380 "pass_rate": 1,3381 "score": 1,3382 "judged": null,3383 "ttft_ms": 458.5,3384 "total_ms": 4039,3385 "tps": 76.3,3386 "cost_mean": 0.007565,3387 "out_tokens": 2793388 }3389 }3390 },3391 {3392 "model": "claude-opus-5",3393 "provider": "anthropic",3394 "runs": 22,3395 "errors": 0,3396 "pass_rate": 1,3397 "mean_score": 1,3398 "judged_mean": 4.666666666666667,3399 "ttft_median_ms": 2530,3400 "total_median_ms": 4043,3401 "tps_median": 160.89999999999998,3402 "cost_total": 1.1158750000000002,3403 "cost_mean": 0.05072159090909092,3404 "in_tokens": 168730,3405 "out_tokens": 10889,3406 "reasoning_tokens": 5458,3407 "by_category": {3408 "document_analysis": {3409 "runs": 2,3410 "errors": 0,3411 "pass_rate": 1,3412 "score": 1,3413 "judged": null,3414 "ttft_ms": 2203,3415 "total_ms": 3265.5,3416 "tps": 111.1,3417 "cost_mean": 0.00804,3418 "out_tokens": 1183419 },3420 "json_generation": {3421 "runs": 2,3422 "errors": 0,3423 "pass_rate": 1,3424 "score": 1,3425 "judged": null,3426 "ttft_ms": 1354,3427 "total_ms": 2252.5,3428 "tps": 116.85,3429 "cost_mean": 0.005095,3430 "out_tokens": 1053431 },3432 "complex_reasoning": {3433 "runs": 2,3434 "errors": 0,3435 "pass_rate": 1,3436 "score": 1,3437 "judged": null,3438 "ttft_ms": 4756.5,3439 "total_ms": 8775.5,3440 "tps": 168.89999999999998,3441 "cost_mean": 0.0179975,3442 "out_tokens": 673.53443 },3444 "long_context": {3445 "runs": 2,3446 "errors": 0,3447 "pass_rate": 1,3448 "score": 1,3449 "judged": null,3450 "ttft_ms": 2696,3451 "total_ms": 2739,3452 "tps": 263.54999999999995,3453 "cost_mean": 0.40483,3454 "out_tokens": 113455 },3456 "english_writing": {3457 "runs": 2,3458 "errors": 0,3459 "pass_rate": 1,3460 "score": 1,3461 "judged": 5,3462 "ttft_ms": 9286.5,3463 "total_ms": 14506,3464 "tps": 245.64999999999998,3465 "cost_mean": 0.0260725,3466 "out_tokens": 1022.53467 },3468 "french_writing": {3469 "runs": 2,3470 "errors": 0,3471 "pass_rate": 1,3472 "score": 1,3473 "judged": 5,3474 "ttft_ms": 17383.5,3475 "total_ms": 22242,3476 "tps": 390.75,3477 "cost_mean": 0.047385,3478 "out_tokens": 18633479 },3480 "programming": {3481 "runs": 2,3482 "errors": 0,3483 "pass_rate": 1,3484 "score": 1,3485 "judged": null,3486 "ttft_ms": 4119.5,3487 "total_ms": 7068.5,3488 "tps": 156.60000000000002,3489 "cost_mean": 0.011574999999999998,3490 "out_tokens": 4393491 },3492 "simple_reasoning": {3493 "runs": 2,3494 "errors": 0,3495 "pass_rate": 1,3496 "score": 1,3497 "judged": null,3498 "ttft_ms": 2304.5,3499 "total_ms": 2482.5,3500 "tps": 444.34999999999997,3501 "cost_mean": 0.00272,3502 "out_tokens": 733503 },3504 "knowledge_synthesis": {3505 "runs": 2,3506 "errors": 0,3507 "pass_rate": 1,3508 "score": 1,3509 "judged": 4,3510 "ttft_ms": 4503,3511 "total_ms": 9943,3512 "tps": 121.35,3513 "cost_mean": 0.01812,3514 "out_tokens": 6553515 },3516 "tool_calling": {3517 "runs": 2,3518 "errors": 0,3519 "pass_rate": 1,3520 "score": 1,3521 "judged": null,3522 "ttft_ms": 1556,3523 "total_ms": 2426,3524 "tps": 182.3,3525 "cost_mean": 0.007375,3526 "out_tokens": 1583527 },3528 "mathematics": {3529 "runs": 2,3530 "errors": 0,3531 "pass_rate": 1,3532 "score": 1,3533 "judged": null,3534 "ttft_ms": 1415.5,3535 "total_ms": 4043,3536 "tps": 99.9,3537 "cost_mean": 0.0087275,3538 "out_tokens": 326.53539 }3540 }3541 },3542 {3543 "model": "claude-opus-4-6",3544 "provider": "anthropic",3545 "runs": 22,3546 "errors": 0,3547 "pass_rate": 1,3548 "mean_score": 1,3549 "judged_mean": 4.833333333333333,3550 "ttft_median_ms": 1621.5,3551 "total_median_ms": 4197.5,3552 "tps_median": 74.5,3553 "cost_total": 0.77559,3554 "cost_mean": 0.03525409090909091,3555 "in_tokens": 129538,3556 "out_tokens": 5116,3557 "reasoning_tokens": 0,3558 "by_category": {3559 "document_analysis": {3560 "runs": 2,3561 "errors": 0,3562 "pass_rate": 1,3563 "score": 1,3564 "judged": null,3565 "ttft_ms": 2807,3566 "total_ms": 3685,3567 "tps": 115,3568 "cost_mean": 0.00673,3569 "out_tokens": 1013570 },3571 "json_generation": {3572 "runs": 2,3573 "errors": 0,3574 "pass_rate": 1,3575 "score": 1,3576 "judged": null,3577 "ttft_ms": 3382.5,3578 "total_ms": 3993,3579 "tps": 113.19999999999999,3580 "cost_mean": 0.00372,3581 "out_tokens": 693582 },3583 "complex_reasoning": {3584 "runs": 2,3585 "errors": 0,3586 "pass_rate": 1,3587 "score": 1,3588 "judged": null,3589 "ttft_ms": 1414,3590 "total_ms": 12800,3591 "tps": 61.4,3592 "cost_mean": 0.018250000000000002,3593 "out_tokens": 6983594 },3595 "long_context": {3596 "runs": 2,3597 "errors": 0,3598 "pass_rate": 1,3599 "score": 1,3600 "judged": null,3601 "ttft_ms": 2326,3602 "total_ms": 2823,3603 "tps": 26.4,3604 "cost_mean": 0.30915,3605 "out_tokens": 133606 },3607 "english_writing": {3608 "runs": 2,3609 "errors": 0,3610 "pass_rate": 1,3611 "score": 1,3612 "judged": 5,3613 "ttft_ms": 1475.5,3614 "total_ms": 8633,3615 "tps": 36.7,3616 "cost_mean": 0.0069325,3617 "out_tokens": 262.53618 },3619 "french_writing": {3620 "runs": 2,3621 "errors": 0,3622 "pass_rate": 1,3623 "score": 1,3624 "judged": 5,3625 "ttft_ms": 1509.5,3626 "total_ms": 9183.5,3627 "tps": 50.55,3628 "cost_mean": 0.0103,3629 "out_tokens": 3873630 },3631 "programming": {3632 "runs": 2,3633 "errors": 0,3634 "pass_rate": 1,3635 "score": 1,3636 "judged": null,3637 "ttft_ms": 2020.5,3638 "total_ms": 3737.5,3639 "tps": 96.19999999999999,3640 "cost_mean": 0.00453,3641 "out_tokens": 1653642 },3643 "simple_reasoning": {3644 "runs": 2,3645 "errors": 0,3646 "pass_rate": 1,3647 "score": 1,3648 "judged": null,3649 "ttft_ms": 1441,3650 "total_ms": 1718,3651 "tps": 76.05,3652 "cost_mean": 0.001235,3653 "out_tokens": 203654 },3655 "knowledge_synthesis": {3656 "runs": 2,3657 "errors": 0,3658 "pass_rate": 1,3659 "score": 1,3660 "judged": 4.5,3661 "ttft_ms": 1718.5,3662 "total_ms": 8823,3663 "tps": 42.2,3664 "cost_mean": 0.008817499999999999,3665 "out_tokens": 299.53666 },3667 "tool_calling": {3668 "runs": 2,3669 "errors": 0,3670 "pass_rate": 1,3671 "score": 1,3672 "judged": null,3673 "ttft_ms": 1791,3674 "total_ms": 2906.5,3675 "tps": 118.5,3676 "cost_mean": 0.007375,3677 "out_tokens": 1323678 },3679 "mathematics": {3680 "runs": 2,3681 "errors": 0,3682 "pass_rate": 1,3683 "score": 1,3684 "judged": null,3685 "ttft_ms": 1582.5,3686 "total_ms": 6329.5,3687 "tps": 86.55,3688 "cost_mean": 0.010755,3689 "out_tokens": 4113690 }3691 }3692 },3693 {3694 "model": "claude-sonnet-4-6",3695 "provider": "anthropic",3696 "runs": 22,3697 "errors": 0,3698 "pass_rate": 1,3699 "mean_score": 1,3700 "judged_mean": 5,3701 "ttft_median_ms": 735,3702 "total_median_ms": 4707.5,3703 "tps_median": 89.6,3704 "cost_total": 0.4661640000000001,3705 "cost_mean": 0.02118927272727273,3706 "in_tokens": 129538,3707 "out_tokens": 5170,3708 "reasoning_tokens": 0,3709 "by_category": {3710 "document_analysis": {3711 "runs": 2,3712 "errors": 0,3713 "pass_rate": 1,3714 "score": 1,3715 "judged": null,3716 "ttft_ms": 3623,3717 "total_ms": 4525.5,3718 "tps": 111.94999999999999,3719 "cost_mean": 0.004038,3720 "out_tokens": 1013721 },3722 "json_generation": {3723 "runs": 2,3724 "errors": 0,3725 "pass_rate": 1,3726 "score": 1,3727 "judged": null,3728 "ttft_ms": 2725.5,3729 "total_ms": 3381,3730 "tps": 105.4,3731 "cost_mean": 0.002232,3732 "out_tokens": 693733 },3734 "complex_reasoning": {3735 "runs": 2,3736 "errors": 0,3737 "pass_rate": 1,3738 "score": 1,3739 "judged": null,3740 "ttft_ms": 659.5,3741 "total_ms": 10339,3742 "tps": 69.85,3743 "cost_mean": 0.01062,3744 "out_tokens": 6763745 },3746 "long_context": {3747 "runs": 2,3748 "errors": 0,3749 "pass_rate": 1,3750 "score": 1,3751 "judged": null,3752 "ttft_ms": 2302.5,3753 "total_ms": 2669,3754 "tps": 35.5,3755 "cost_mean": 0.18549,3756 "out_tokens": 133757 },3758 "english_writing": {3759 "runs": 2,3760 "errors": 0,3761 "pass_rate": 1,3762 "score": 1,3763 "judged": 5,3764 "ttft_ms": 646,3765 "total_ms": 7520.5,3766 "tps": 40.05,3767 "cost_mean": 0.0043395,3768 "out_tokens": 274.53769 },3770 "french_writing": {3771 "runs": 2,3772 "errors": 0,3773 "pass_rate": 1,3774 "score": 1,3775 "judged": 5,3776 "ttft_ms": 714,3777 "total_ms": 8037.5,3778 "tps": 50.3,3779 "cost_mean": 0.005895,3780 "out_tokens": 3683781 },3782 "programming": {3783 "runs": 2,3784 "errors": 0,3785 "pass_rate": 1,3786 "score": 1,3787 "judged": null,3788 "ttft_ms": 673.5,3789 "total_ms": 2598.5,3790 "tps": 91.5,3791 "cost_mean": 0.002883,3792 "out_tokens": 1763793 },3794 "simple_reasoning": {3795 "runs": 2,3796 "errors": 0,3797 "pass_rate": 1,3798 "score": 1,3799 "judged": null,3800 "ttft_ms": 1070.5,3801 "total_ms": 1231,3802 "tps": 124.9,3803 "cost_mean": 0.000741,3804 "out_tokens": 203805 },3806 "knowledge_synthesis": {3807 "runs": 2,3808 "errors": 0,3809 "pass_rate": 1,3810 "score": 1,3811 "judged": 5,3812 "ttft_ms": 872.5,3813 "total_ms": 7648,3814 "tps": 45.75,3815 "cost_mean": 0.005448,3816 "out_tokens": 3103817 },3818 "tool_calling": {3819 "runs": 2,3820 "errors": 0,3821 "pass_rate": 1,3822 "score": 1,3823 "judged": null,3824 "ttft_ms": 681,3825 "total_ms": 1792.5,3826 "tps": 117.9,3827 "cost_mean": 0.00441,3828 "out_tokens": 1313829 },3830 "mathematics": {3831 "runs": 2,3832 "errors": 0,3833 "pass_rate": 1,3834 "score": 1,3835 "judged": null,3836 "ttft_ms": 611,3837 "total_ms": 5533.5,3838 "tps": 90.69999999999999,3839 "cost_mean": 0.0069855,3840 "out_tokens": 446.53841 }3842 }3843 },3844 {3845 "model": "gpt-5",3846 "provider": "openai",3847 "runs": 22,3848 "errors": 0,3849 "pass_rate": 1,3850 "mean_score": 1,3851 "judged_mean": 4.333333333333333,3852 "ttft_median_ms": 4121,3853 "total_median_ms": 4969.5,3854 "tps_median": 962.45,3855 "cost_total": 0.26522899999999994,3856 "cost_mean": 0.012055863636363634,3857 "in_tokens": 113620,3858 "out_tokens": 17346,3859 "reasoning_tokens": 13696,3860 "by_category": {3861 "document_analysis": {3862 "runs": 2,3863 "errors": 0,3864 "pass_rate": 1,3865 "score": 1,3866 "judged": null,3867 "ttft_ms": 5090.5,3868 "total_ms": 5656,3869 "tps": 1595.9499999999998,3870 "cost_mean": 0.0093,3871 "out_tokens": 867.53872 },3873 "json_generation": {3874 "runs": 2,3875 "errors": 0,3876 "pass_rate": 1,3877 "score": 1,3878 "judged": null,3879 "ttft_ms": 2910,3880 "total_ms": 3430,3881 "tps": 1050.35,3882 "cost_mean": 0.00564,3883 "out_tokens": 5453884 },3885 "complex_reasoning": {3886 "runs": 2,3887 "errors": 0,3888 "pass_rate": 1,3889 "score": 1,3890 "judged": null,3891 "ttft_ms": 4953,3892 "total_ms": 5845,3893 "tps": 1215.55,3894 "cost_mean": 0.010825000000000001,3895 "out_tokens": 10653896 },3897 "long_context": {3898 "runs": 2,3899 "errors": 0,3900 "pass_rate": 1,3901 "score": 1,3902 "judged": null,3903 "ttft_ms": 3088.5,3904 "total_ms": 3382,3905 "tps": 796.45,3906 "cost_mean": 0.04594575,3907 "out_tokens": 207.53908 },3909 "english_writing": {3910 "runs": 2,3911 "errors": 0,3912 "pass_rate": 1,3913 "score": 1,3914 "judged": 5,3915 "ttft_ms": 9221.5,3916 "total_ms": 10391.5,3917 "tps": 1243.15,3918 "cost_mean": 0.0146,3919 "out_tokens": 14523920 },3921 "french_writing": {3922 "runs": 2,3923 "errors": 0,3924 "pass_rate": 1,3925 "score": 1,3926 "judged": 5,3927 "ttft_ms": 7258,3928 "total_ms": 9252.5,3929 "tps": 763.4,3930 "cost_mean": 0.01303,3931 "out_tokens": 1291.53932 },3933 "programming": {3934 "runs": 2,3935 "errors": 0,3936 "pass_rate": 1,3937 "score": 1,3938 "judged": null,3939 "ttft_ms": 3734,3940 "total_ms": 4640.5,3941 "tps": 960.75,3942 "cost_mean": 0.00878375,3943 "out_tokens": 8693944 },3945 "simple_reasoning": {3946 "runs": 2,3947 "errors": 0,3948 "pass_rate": 1,3949 "score": 1,3950 "judged": null,3951 "ttft_ms": 1297,3952 "total_ms": 1580.5,3953 "tps": 830.9,3954 "cost_mean": 0.0024462499999999996,3955 "out_tokens": 2293956 },3957 "knowledge_synthesis": {3958 "runs": 2,3959 "errors": 0,3960 "pass_rate": 1,3961 "score": 1,3962 "judged": 3,3963 "ttft_ms": 5380.5,3964 "total_ms": 7232,3965 "tps": 566.55,3966 "cost_mean": 0.01069875,3967 "out_tokens": 10413968 },3969 "tool_calling": {3970 "runs": 2,3971 "errors": 0,3972 "pass_rate": 1,3973 "score": 1,3974 "judged": null,3975 "ttft_ms": 1965,3976 "total_ms": 2430,3977 "tps": 574.1,3978 "cost_mean": 0.00260875,3979 "out_tokens": 2423980 },3981 "mathematics": {3982 "runs": 2,3983 "errors": 0,3984 "pass_rate": 1,3985 "score": 1,3986 "judged": null,3987 "ttft_ms": 7273.5,3988 "total_ms": 7600,3989 "tps": 197,3990 "cost_mean": 0.008736250000000001,3991 "out_tokens": 863.53992 }3993 }3994 },3995 {3996 "model": "gpt-5-mini",3997 "provider": "openai",3998 "runs": 22,3999 "errors": 0,4000 "pass_rate": 1,4001 "mean_score": 1,4002 "judged_mean": 4.833333333333333,4003 "ttft_median_ms": 4395,4004 "total_median_ms": 5673,4005 "tps_median": 374.55,4006 "cost_total": 0.056407,4007 "cost_mean": 0.0025639545454545455,4008 "in_tokens": 113620,4009 "out_tokens": 14001,4010 "reasoning_tokens": 10176,4011 "by_category": {4012 "document_analysis": {4013 "runs": 2,4014 "errors": 0,4015 "pass_rate": 1,4016 "score": 1,4017 "judged": null,4018 "ttft_ms": 4189.5,4019 "total_ms": 5282.5,4020 "tps": 378.8,4021 "cost_mean": 0.000953,4022 "out_tokens": 4144023 },4024 "json_generation": {4025 "runs": 2,4026 "errors": 0,4027 "pass_rate": 1,4028 "score": 1,4029 "judged": null,4030 "ttft_ms": 2894,4031 "total_ms": 3835.5,4032 "tps": 241.25,4033 "cost_mean": 0.000492,4034 "out_tokens": 2274035 },4036 "complex_reasoning": {4037 "runs": 2,4038 "errors": 0,4039 "pass_rate": 1,4040 "score": 1,4041 "judged": null,4042 "ttft_ms": 9508.5,4043 "total_ms": 12444,4044 "tps": 349.75,4045 "cost_mean": 0.0020740000000000003,4046 "out_tokens": 1019.54047 },4048 "long_context": {4049 "runs": 2,4050 "errors": 0,4051 "pass_rate": 1,4052 "score": 1,4053 "judged": null,4054 "ttft_ms": 2807.5,4055 "total_ms": 3102.5,4056 "tps": 471.75,4057 "cost_mean": 0.01406575,4058 "out_tokens": 1334059 },4060 "english_writing": {4061 "runs": 2,4062 "errors": 0,4063 "pass_rate": 1,4064 "score": 1,4065 "judged": 5,4066 "ttft_ms": 8008,4067 "total_ms": 10396.5,4068 "tps": 364.35,4069 "cost_mean": 0.001692,4070 "out_tokens": 8384071 },4072 "french_writing": {4073 "runs": 2,4074 "errors": 0,4075 "pass_rate": 1,4076 "score": 1,4077 "judged": 5,4078 "ttft_ms": 13945.5,4079 "total_ms": 16488.5,4080 "tps": 558.05,4081 "cost_mean": 0.00286,4082 "out_tokens": 1418.54083 },4084 "programming": {4085 "runs": 2,4086 "errors": 0,4087 "pass_rate": 1,4088 "score": 1,4089 "judged": null,4090 "ttft_ms": 4148.5,4091 "total_ms": 5509,4092 "tps": 401.75,4093 "cost_mean": 0.00111375,4094 "out_tokens": 547.54095 },4096 "simple_reasoning": {4097 "runs": 2,4098 "errors": 0,4099 "pass_rate": 1,4100 "score": 1,4101 "judged": null,4102 "ttft_ms": 2540.5,4103 "total_ms": 2858,4104 "tps": 829.3,4105 "cost_mean": 0.00055525,4106 "out_tokens": 2624107 },4108 "knowledge_synthesis": {4109 "runs": 2,4110 "errors": 0,4111 "pass_rate": 1,4112 "score": 1,4113 "judged": 4.5,4114 "ttft_ms": 6918,4115 "total_ms": 9179.5,4116 "tps": 393.45000000000005,4117 "cost_mean": 0.00183175,4118 "out_tokens": 8874119 },4120 "tool_calling": {4121 "runs": 2,4122 "errors": 0,4123 "pass_rate": 1,4124 "score": 1,4125 "judged": null,4126 "ttft_ms": 1149,4127 "total_ms": 1626.5,4128 "tps": 205.14999999999998,4129 "cost_mean": 0.00023075,4130 "out_tokens": 96.54131 },4132 "mathematics": {4133 "runs": 2,4134 "errors": 0,4135 "pass_rate": 1,4136 "score": 1,4137 "judged": null,4138 "ttft_ms": 6625,4139 "total_ms": 8280.5,4140 "tps": 156.05,4141 "cost_mean": 0.00233525,4142 "out_tokens": 1157.54143 }4144 }4145 },4146 {4147 "model": "gpt-4o",4148 "provider": "openai",4149 "runs": 22,4150 "errors": 0,4151 "pass_rate": 0.9545454545454546,4152 "mean_score": 0.9962272727272729,4153 "judged_mean": 3.8333333333333335,4154 "ttft_median_ms": 656,4155 "total_median_ms": 2401.5,4156 "tps_median": 106.25,4157 "cost_total": 0.3303150000000001,4158 "cost_mean": 0.015014318181818186,4159 "in_tokens": 113634,4160 "out_tokens": 4623,4161 "reasoning_tokens": 0,4162 "by_category": {4163 "document_analysis": {4164 "runs": 2,4165 "errors": 0,4166 "pass_rate": 1,4167 "score": 1,4168 "judged": null,4169 "ttft_ms": 709.5,4170 "total_ms": 1207,4171 "tps": 160.05,4172 "cost_mean": 0.002045,4173 "out_tokens": 794174 },4175 "json_generation": {4176 "runs": 2,4177 "errors": 0,4178 "pass_rate": 1,4179 "score": 1,4180 "judged": null,4181 "ttft_ms": 507,4182 "total_ms": 977,4183 "tps": 134.05,4184 "cost_mean": 0.000985,4185 "out_tokens": 604186 },4187 "complex_reasoning": {4188 "runs": 2,4189 "errors": 0,4190 "pass_rate": 1,4191 "score": 1,4192 "judged": null,4193 "ttft_ms": 670.5,4194 "total_ms": 3947,4195 "tps": 131.45,4196 "cost_mean": 0.0046675,4197 "out_tokens": 431.54198 },4199 "long_context": {4200 "runs": 2,4201 "errors": 0,4202 "pass_rate": 1,4203 "score": 1,4204 "judged": null,4205 "ttft_ms": 3213.5,4206 "total_ms": 3409,4207 "tps": 42.45,4208 "cost_mean": 0.13808,4209 "out_tokens": 84210 },4211 "english_writing": {4212 "runs": 2,4213 "errors": 0,4214 "pass_rate": 1,4215 "score": 1,4216 "judged": 5,4217 "ttft_ms": 767.5,4218 "total_ms": 2344.5,4219 "tps": 117.9,4220 "cost_mean": 0.0020175,4221 "out_tokens": 185.54222 },4223 "french_writing": {4224 "runs": 2,4225 "errors": 0,4226 "pass_rate": 0.5,4227 "score": 0.9585,4228 "judged": 4,4229 "ttft_ms": 702.5,4230 "total_ms": 3305,4231 "tps": 106.25,4232 "cost_mean": 0.0029975,4233 "out_tokens": 276.54234 },4235 "programming": {4236 "runs": 2,4237 "errors": 0,4238 "pass_rate": 1,4239 "score": 1,4240 "judged": null,4241 "ttft_ms": 1073,4242 "total_ms": 2237,4243 "tps": 116.85,4244 "cost_mean": 0.001415,4245 "out_tokens": 122.54246 },4247 "simple_reasoning": {4248 "runs": 2,4249 "errors": 0,4250 "pass_rate": 1,4251 "score": 1,4252 "judged": null,4253 "ttft_ms": 622,4254 "total_ms": 832,4255 "tps": 79.25,4256 "cost_mean": 0.000475,4257 "out_tokens": 164258 },4259 "knowledge_synthesis": {4260 "runs": 2,4261 "errors": 0,4262 "pass_rate": 1,4263 "score": 1,4264 "judged": 2.5,4265 "ttft_ms": 692,4266 "total_ms": 2541.5,4267 "tps": 108.8,4268 "cost_mean": 0.002585,4269 "out_tokens": 200.54270 },4271 "tool_calling": {4272 "runs": 2,4273 "errors": 0,4274 "pass_rate": 1,4275 "score": 1,4276 "judged": null,4277 "ttft_ms": 610.5,4278 "total_ms": 993,4279 "tps": 99.5,4280 "cost_mean": 0.000745,4281 "out_tokens": 384282 },4283 "mathematics": {4284 "runs": 2,4285 "errors": 0,4286 "pass_rate": 1,4287 "score": 1,4288 "judged": null,4289 "ttft_ms": 656,4290 "total_ms": 8235.5,4291 "tps": 123.5,4292 "cost_mean": 0.009145,4293 "out_tokens": 8944294 }4295 }4296 },4297 {4298 "model": "gpt-5.6-luna",4299 "provider": "openai",4300 "runs": 22,4301 "errors": 0,4302 "pass_rate": 0.9545454545454546,4303 "mean_score": 0.9772727272727273,4304 "judged_mean": 4.666666666666667,4305 "ttft_median_ms": 1398,4306 "total_median_ms": 3014,4307 "tps_median": 136.45,4308 "cost_total": 0.018583520000000006,4309 "cost_mean": 0.0008447054545454548,4310 "in_tokens": 113620,4311 "out_tokens": 4829,4312 "reasoning_tokens": 1858,4313 "by_category": {4314 "document_analysis": {4315 "runs": 2,4316 "errors": 0,4317 "pass_rate": 1,4318 "score": 1,4319 "judged": null,4320 "ttft_ms": 911.5,4321 "total_ms": 1586.5,4322 "tps": 184.9,4323 "cost_mean": 0.0002494,4324 "out_tokens": 124.54325 },4326 "json_generation": {4327 "runs": 2,4328 "errors": 0,4329 "pass_rate": 1,4330 "score": 1,4331 "judged": null,4332 "ttft_ms": 689.5,4333 "total_ms": 1329,4334 "tps": 105.4,4335 "cost_mean": 0.0001108,4336 "out_tokens": 674337 },4338 "complex_reasoning": {4339 "runs": 2,4340 "errors": 0,4341 "pass_rate": 1,4342 "score": 1,4343 "judged": null,4344 "ttft_ms": 2832.5,4345 "total_ms": 4203.5,4346 "tps": 326.4,4347 "cost_mean": 0.0005625999999999999,4348 "out_tokens": 445.54349 },4350 "long_context": {4351 "runs": 2,4352 "errors": 0,4353 "pass_rate": 1,4354 "score": 1,4355 "judged": null,4356 "ttft_ms": 1057.5,4357 "total_ms": 1326.5,4358 "tps": 41.900000000000006,4359 "cost_mean": 0.0060853600000000015,4360 "out_tokens": 114361 },4362 "english_writing": {4363 "runs": 2,4364 "errors": 0,4365 "pass_rate": 1,4366 "score": 1,4367 "judged": 5,4368 "ttft_ms": 1268,4369 "total_ms": 4736,4370 "tps": 95.25,4371 "cost_mean": 0.00040340000000000005,4372 "out_tokens": 325.54373 },4374 "french_writing": {4375 "runs": 2,4376 "errors": 0,4377 "pass_rate": 1,4378 "score": 1,4379 "judged": 5,4380 "ttft_ms": 1906,4381 "total_ms": 4849,4382 "tps": 145.6,4383 "cost_mean": 0.0005325999999999999,4384 "out_tokens": 428.54385 },4386 "programming": {4387 "runs": 2,4388 "errors": 0,4389 "pass_rate": 1,4390 "score": 1,4391 "judged": null,4392 "ttft_ms": 1963.5,4393 "total_ms": 3067,4394 "tps": 238.75,4395 "cost_mean": 0.00033120000000000003,4396 "out_tokens": 263.54397 },4398 "simple_reasoning": {4399 "runs": 2,4400 "errors": 0,4401 "pass_rate": 1,4402 "score": 1,4403 "judged": null,4404 "ttft_ms": 1299,4405 "total_ms": 1657,4406 "tps": 162.8,4407 "cost_mean": 0.0000946,4408 "out_tokens": 584409 },4410 "knowledge_synthesis": {4411 "runs": 2,4412 "errors": 0,4413 "pass_rate": 1,4414 "score": 1,4415 "judged": 4,4416 "ttft_ms": 1662,4417 "total_ms": 4621,4418 "tps": 112.8,4419 "cost_mean": 0.0004464,4420 "out_tokens": 333.54421 },4422 "tool_calling": {4423 "runs": 2,4424 "errors": 0,4425 "pass_rate": 1,4426 "score": 1,4427 "judged": null,4428 "ttft_ms": 994.5,4429 "total_ms": 1377,4430 "tps": 138.95,4431 "cost_mean": 0.0000938,4432 "out_tokens": 534433 },4434 "mathematics": {4435 "runs": 2,4436 "errors": 0,4437 "pass_rate": 0.5,4438 "score": 0.75,4439 "judged": null,4440 "ttft_ms": 2430.5,4441 "total_ms": 3678.5,4442 "tps": 105.05,4443 "cost_mean": 0.00038159999999999995,4444 "out_tokens": 304.54445 }4446 }4447 },4448 {4449 "model": "gpt-4.1-mini",4450 "provider": "openai",4451 "runs": 22,4452 "errors": 0,4453 "pass_rate": 0.9090909090909091,4454 "mean_score": 0.9735,4455 "judged_mean": 3.8333333333333335,4456 "ttft_median_ms": 449,4457 "total_median_ms": 1683.5,4458 "tps_median": 153.3,4459 "cost_total": 0.053280799999999996,4460 "cost_mean": 0.0024218545454545454,4461 "in_tokens": 113634,4462 "out_tokens": 5228,4463 "reasoning_tokens": 0,4464 "by_category": {4465 "document_analysis": {4466 "runs": 2,4467 "errors": 0,4468 "pass_rate": 1,4469 "score": 1,4470 "judged": null,4471 "ttft_ms": 391.5,4472 "total_ms": 940.5,4473 "tps": 145,4474 "cost_mean": 0.00032720000000000004,4475 "out_tokens": 794476 },4477 "json_generation": {4478 "runs": 2,4479 "errors": 0,4480 "pass_rate": 1,4481 "score": 1,4482 "judged": null,4483 "ttft_ms": 421.5,4484 "total_ms": 839.5,4485 "tps": 141.2,4486 "cost_mean": 0.000156,4487 "out_tokens": 594488 },4489 "complex_reasoning": {4490 "runs": 2,4491 "errors": 0,4492 "pass_rate": 1,4493 "score": 1,4494 "judged": null,4495 "ttft_ms": 365,4496 "total_ms": 5553.5,4497 "tps": 189.45,4498 "cost_mean": 0.0016308000000000002,4499 "out_tokens": 9844500 },4501 "long_context": {4502 "runs": 2,4503 "errors": 0,4504 "pass_rate": 1,4505 "score": 1,4506 "judged": null,4507 "ttft_ms": 1922.5,4508 "total_ms": 2153.5,4509 "tps": 35.4,4510 "cost_mean": 0.021824,4511 "out_tokens": 84512 },4513 "english_writing": {4514 "runs": 2,4515 "errors": 0,4516 "pass_rate": 1,4517 "score": 1,4518 "judged": 5,4519 "ttft_ms": 416,4520 "total_ms": 1670,4521 "tps": 174.6,4522 "cost_mean": 0.00037640000000000004,4523 "out_tokens": 2194524 },4525 "french_writing": {4526 "runs": 2,4527 "errors": 0,4528 "pass_rate": 0.5,4529 "score": 0.9585,4530 "judged": 4,4531 "ttft_ms": 501,4532 "total_ms": 2025.5,4533 "tps": 176.8,4534 "cost_mean": 0.0004676,4535 "out_tokens": 2694536 },4537 "programming": {4538 "runs": 2,4539 "errors": 0,4540 "pass_rate": 1,4541 "score": 1,4542 "judged": null,4543 "ttft_ms": 444,4544 "total_ms": 1209,4545 "tps": 156.25,4546 "cost_mean": 0.00022160000000000002,4547 "out_tokens": 119.54548 },4549 "simple_reasoning": {4550 "runs": 2,4551 "errors": 0,4552 "pass_rate": 1,4553 "score": 1,4554 "judged": null,4555 "ttft_ms": 983,4556 "total_ms": 1233.5,4557 "tps": 64.15,4558 "cost_mean": 0.000076,4559 "out_tokens": 164560 },4561 "knowledge_synthesis": {4562 "runs": 2,4563 "errors": 0,4564 "pass_rate": 1,4565 "score": 1,4566 "judged": 2.5,4567 "ttft_ms": 445,4568 "total_ms": 2108.5,4569 "tps": 149.55,4570 "cost_mean": 0.0004904,4571 "out_tokens": 248.54572 },4573 "tool_calling": {4574 "runs": 2,4575 "errors": 0,4576 "pass_rate": 1,4577 "score": 1,4578 "judged": null,4579 "ttft_ms": 548.5,4580 "total_ms": 832.5,4581 "tps": 135.85,4582 "cost_mean": 0.00011920000000000001,4583 "out_tokens": 384584 },4585 "mathematics": {4586 "runs": 2,4587 "errors": 0,4588 "pass_rate": 0.5,4589 "score": 0.75,4590 "judged": null,4591 "ttft_ms": 708.5,4592 "total_ms": 4104,4593 "tps": 173.75,4594 "cost_mean": 0.0009512000000000001,4595 "out_tokens": 5744596 }4597 }4598 },4599 {4600 "model": "claude-haiku-4-5-20251001",4601 "provider": "anthropic",4602 "runs": 22,4603 "errors": 0,4604 "pass_rate": 0.9090909090909091,4605 "mean_score": 0.9697272727272727,4606 "judged_mean": 3.3333333333333335,4607 "ttft_median_ms": 502,4608 "total_median_ms": 1966.5,4609 "tps_median": 134.55,4610 "cost_total": 0.15852199999999997,4611 "cost_mean": 0.007205545454545453,4612 "in_tokens": 129532,4613 "out_tokens": 5798,4614 "reasoning_tokens": 0,4615 "by_category": {4616 "document_analysis": {4617 "runs": 2,4618 "errors": 0,4619 "pass_rate": 1,4620 "score": 1,4621 "judged": null,4622 "ttft_ms": 1132.5,4623 "total_ms": 1880.5,4624 "tps": 152.39999999999998,4625 "cost_mean": 0.00141,4626 "out_tokens": 1144627 },4628 "json_generation": {4629 "runs": 2,4630 "errors": 0,4631 "pass_rate": 1,4632 "score": 1,4633 "judged": null,4634 "ttft_ms": 1039.5,4635 "total_ms": 1524.5,4636 "tps": 142.55,4637 "cost_mean": 0.000743,4638 "out_tokens": 694639 },4640 "complex_reasoning": {4641 "runs": 2,4642 "errors": 0,4643 "pass_rate": 1,4644 "score": 1,4645 "judged": null,4646 "ttft_ms": 404.5,4647 "total_ms": 7524,4648 "tps": 131.8,4649 "cost_mean": 0.0048425,4650 "out_tokens": 936.54651 },4652 "long_context": {4653 "runs": 2,4654 "errors": 0,4655 "pass_rate": 1,4656 "score": 1,4657 "judged": null,4658 "ttft_ms": 865.5,4659 "total_ms": 973,4660 "tps": 121.5,4661 "cost_mean": 0.06183,4662 "out_tokens": 134663 },4664 "english_writing": {4665 "runs": 2,4666 "errors": 0,4667 "pass_rate": 1,4668 "score": 1,4669 "judged": 4,4670 "ttft_ms": 373,4671 "total_ms": 3227,4672 "tps": 89.9,4673 "cost_mean": 0.0013565,4674 "out_tokens": 256.54675 },4676 "french_writing": {4677 "runs": 2,4678 "errors": 0,4679 "pass_rate": 1,4680 "score": 1,4681 "judged": 3.5,4682 "ttft_ms": 384.5,4683 "total_ms": 3746,4684 "tps": 102.35,4685 "cost_mean": 0.0018449999999999999,4686 "out_tokens": 3444687 },4688 "programming": {4689 "runs": 2,4690 "errors": 0,4691 "pass_rate": 1,4692 "score": 1,4693 "judged": null,4694 "ttft_ms": 553,4695 "total_ms": 1930.5,4696 "tps": 151.3,4697 "cost_mean": 0.0011235,4698 "out_tokens": 208.54699 },4700 "simple_reasoning": {4701 "runs": 2,4702 "errors": 0,4703 "pass_rate": 0,4704 "score": 0.667,4705 "judged": null,4706 "ttft_ms": 508,4707 "total_ms": 598,4708 "tps": 221.55,4709 "cost_mean": 0.000247,4710 "out_tokens": 204711 },4712 "knowledge_synthesis": {4713 "runs": 2,4714 "errors": 0,4715 "pass_rate": 1,4716 "score": 1,4717 "judged": 2.5,4718 "ttft_ms": 368,4719 "total_ms": 3060.5,4720 "tps": 94.6,4721 "cost_mean": 0.0015184999999999999,4722 "out_tokens": 250.54723 },4724 "tool_calling": {4725 "runs": 2,4726 "errors": 0,4727 "pass_rate": 1,4728 "score": 1,4729 "judged": null,4730 "ttft_ms": 536.5,4731 "total_ms": 1165,4732 "tps": 192.55,4733 "cost_mean": 0.001419,4734 "out_tokens": 1214735 },4736 "mathematics": {4737 "runs": 2,4738 "errors": 0,4739 "pass_rate": 1,4740 "score": 1,4741 "judged": null,4742 "ttft_ms": 372.5,4743 "total_ms": 4615,4744 "tps": 133.35000000000002,4745 "cost_mean": 0.0029259999999999998,4746 "out_tokens": 5664747 }4748 }4749 },4750 {4751 "model": "gpt-5-nano",4752 "provider": "openai",4753 "runs": 22,4754 "errors": 0,4755 "pass_rate": 0.9545454545454546,4756 "mean_score": 0.9659090909090909,4757 "judged_mean": 3.6666666666666665,4758 "ttft_median_ms": 6133,4759 "total_median_ms": 6849.5,4760 "tps_median": 1362.1,4761 "cost_total": 0.0140714,4762 "cost_mean": 0.0006396090909090909,4763 "in_tokens": 113620,4764 "out_tokens": 27168,4765 "reasoning_tokens": 24384,4766 "by_category": {4767 "document_analysis": {4768 "runs": 2,4769 "errors": 0,4770 "pass_rate": 1,4771 "score": 1,4772 "judged": null,4773 "ttft_ms": 9956,4774 "total_ms": 10721,4775 "tps": 1359.9,4776 "cost_mean": 0.00044120000000000004,4777 "out_tokens": 1040.54778 },4779 "json_generation": {4780 "runs": 2,4781 "errors": 0,4782 "pass_rate": 1,4783 "score": 1,4784 "judged": null,4785 "ttft_ms": 2280.5,4786 "total_ms": 2955,4787 "tps": 952,4788 "cost_mean": 0.0002278,4789 "out_tokens": 550.54790 },4791 "complex_reasoning": {4792 "runs": 2,4793 "errors": 0,4794 "pass_rate": 1,4795 "score": 1,4796 "judged": null,4797 "ttft_ms": 5995,4798 "total_ms": 6223,4799 "tps": 5903.200000000001,4800 "cost_mean": 0.000546,4801 "out_tokens": 1347.54802 },4803 "long_context": {4804 "runs": 2,4805 "errors": 0,4806 "pass_rate": 1,4807 "score": 1,4808 "judged": null,4809 "ttft_ms": 1907.5,4810 "total_ms": 2222,4811 "tps": 705.3,4812 "cost_mean": 0.0015985500000000002,4813 "out_tokens": 192.54814 },4815 "english_writing": {4816 "runs": 2,4817 "errors": 0,4818 "pass_rate": 1,4819 "score": 1,4820 "judged": 5,4821 "ttft_ms": 11278.5,4822 "total_ms": 12370,4823 "tps": 2324.3,4824 "cost_mean": 0.0009774,4825 "out_tokens": 2435.54826 },4827 "french_writing": {4828 "runs": 2,4829 "errors": 0,4830 "pass_rate": 0.5,4831 "score": 0.625,4832 "judged": 3,4833 "ttft_ms": 9189,4834 "total_ms": 11226.5,4835 "tps": 2000.2,4836 "cost_mean": 0.0010896,4837 "out_tokens": 2712.54838 },4839 "programming": {4840 "runs": 2,4841 "errors": 0,4842 "pass_rate": 1,4843 "score": 1,4844 "judged": null,4845 "ttft_ms": 6243,4846 "total_ms": 7041,4847 "tps": 2127.05,4848 "cost_mean": 0.00068495,4849 "out_tokens": 17034850 },4851 "simple_reasoning": {4852 "runs": 2,4853 "errors": 0,4854 "pass_rate": 1,4855 "score": 1,4856 "judged": null,4857 "ttft_ms": 1874,4858 "total_ms": 2087.5,4859 "tps": 2235.9,4860 "cost_mean": 0.00019645000000000003,4861 "out_tokens": 475.54862 },4863 "knowledge_synthesis": {4864 "runs": 2,4865 "errors": 0,4866 "pass_rate": 1,4867 "score": 1,4868 "judged": 3,4869 "ttft_ms": 6880,4870 "total_ms": 8291.5,4871 "tps": 1267.7,4872 "cost_mean": 0.00072255,4873 "out_tokens": 1777.54874 },4875 "tool_calling": {4876 "runs": 2,4877 "errors": 0,4878 "pass_rate": 1,4879 "score": 1,4880 "judged": null,4881 "ttft_ms": 1292.5,4882 "total_ms": 1613.5,4883 "tps": 741.5999999999999,4884 "cost_mean": 0.00010255,4885 "out_tokens": 237.54886 },4887 "mathematics": {4888 "runs": 2,4889 "errors": 0,4890 "pass_rate": 1,4891 "score": 1,4892 "judged": null,4893 "ttft_ms": 7906,4894 "total_ms": 8184.5,4895 "tps": 322.9,4896 "cost_mean": 0.00044865000000000004,4897 "out_tokens": 1111.54898 }4899 }4900 },4901 {4902 "model": "gpt-5.4-mini",4903 "provider": "openai",4904 "runs": 22,4905 "errors": 0,4906 "pass_rate": 0.9545454545454546,4907 "mean_score": 0.9545454545454546,4908 "judged_mean": 4.666666666666667,4909 "ttft_median_ms": 623.5,4910 "total_median_ms": 1202.5,4911 "tps_median": 166.65,4912 "cost_total": 0.06353849999999998,4913 "cost_mean": 0.0028881136363636357,4914 "in_tokens": 113620,4915 "out_tokens": 3439,4916 "reasoning_tokens": 0,4917 "by_category": {4918 "document_analysis": {4919 "runs": 2,4920 "errors": 0,4921 "pass_rate": 1,4922 "score": 1,4923 "judged": null,4924 "ttft_ms": 733.5,4925 "total_ms": 1127,4926 "tps": 216,4927 "cost_mean": 0.0007575,4928 "out_tokens": 854929 },4930 "json_generation": {4931 "runs": 2,4932 "errors": 0,4933 "pass_rate": 1,4934 "score": 1,4935 "judged": null,4936 "ttft_ms": 705.5,4937 "total_ms": 1126,4938 "tps": 159.4,4939 "cost_mean": 0.0004155,4940 "out_tokens": 674941 },4942 "complex_reasoning": {4943 "runs": 2,4944 "errors": 0,4945 "pass_rate": 0.5,4946 "score": 0.5,4947 "judged": null,4948 "ttft_ms": 485.5,4949 "total_ms": 1996,4950 "tps": 206.85,4951 "cost_mean": 0.00151125,4952 "out_tokens": 312.54953 },4954 "long_context": {4955 "runs": 2,4956 "errors": 0,4957 "pass_rate": 1,4958 "score": 1,4959 "judged": null,4960 "ttft_ms": 869,4961 "total_ms": 1068,4962 "tps": 56.1,4963 "cost_mean": 0.02287275,4964 "out_tokens": 114965 },4966 "english_writing": {4967 "runs": 2,4968 "errors": 0,4969 "pass_rate": 1,4970 "score": 1,4971 "judged": 5,4972 "ttft_ms": 501,4973 "total_ms": 1988.5,4974 "tps": 152.2,4975 "cost_mean": 0.00106725,4976 "out_tokens": 226.54977 },4978 "french_writing": {4979 "runs": 2,4980 "errors": 0,4981 "pass_rate": 1,4982 "score": 1,4983 "judged": 5,4984 "ttft_ms": 605,4985 "total_ms": 2124,4986 "tps": 169.85,4987 "cost_mean": 0.00123,4988 "out_tokens": 2584989 },4990 "programming": {4991 "runs": 2,4992 "errors": 0,4993 "pass_rate": 1,4994 "score": 1,4995 "judged": null,4996 "ttft_ms": 555,4997 "total_ms": 1096,4998 "tps": 246.75,4999 "cost_mean": 0.000657,5000 "out_tokens": 133.55001 },5002 "simple_reasoning": {5003 "runs": 2,5004 "errors": 0,5005 "pass_rate": 1,5006 "score": 1,5007 "judged": null,5008 "ttft_ms": 722.5,5009 "total_ms": 1016.5,5010 "tps": 64.7,5011 "cost_mean": 0.00017925,5012 "out_tokens": 195013 },5014 "knowledge_synthesis": {5015 "runs": 2,5016 "errors": 0,5017 "pass_rate": 1,5018 "score": 1,5019 "judged": 4,5020 "ttft_ms": 623,5021 "total_ms": 1963,5022 "tps": 180.35000000000002,5023 "cost_mean": 0.0012487499999999999,5024 "out_tokens": 2395025 },5026 "tool_calling": {5027 "runs": 2,5028 "errors": 0,5029 "pass_rate": 1,5030 "score": 1,5031 "judged": null,5032 "ttft_ms": 600.5,5033 "total_ms": 935,5034 "tps": 123.3,5035 "cost_mean": 0.00029775,5036 "out_tokens": 415037 },5038 "mathematics": {5039 "runs": 2,5040 "errors": 0,5041 "pass_rate": 1,5042 "score": 1,5043 "judged": null,5044 "ttft_ms": 604.5,5045 "total_ms": 1753,5046 "tps": 284.5,5047 "cost_mean": 0.0015322500000000002,5048 "out_tokens": 3275049 }5050 }5051 },5052 {5053 "model": "gpt-5.6-terra",5054 "provider": "openai",5055 "runs": 22,5056 "errors": 0,5057 "pass_rate": 0.9545454545454546,5058 "mean_score": 0.9545454545454546,5059 "judged_mean": 5,5060 "ttft_median_ms": 869,5061 "total_median_ms": 1802.5,5062 "tps_median": 125.65,5063 "cost_total": 0.1691192,5064 "cost_mean": 0.007687236363636363,5065 "in_tokens": 113620,5066 "out_tokens": 3436,5067 "reasoning_tokens": 881,5068 "by_category": {5069 "document_analysis": {5070 "runs": 2,5071 "errors": 0,5072 "pass_rate": 1,5073 "score": 1,5074 "judged": null,5075 "ttft_ms": 666.5,5076 "total_ms": 1278,5077 "tps": 139.55,5078 "cost_mean": 0.00202,5079 "out_tokens": 855080 },5081 "json_generation": {5082 "runs": 2,5083 "errors": 0,5084 "pass_rate": 1,5085 "score": 1,5086 "judged": null,5087 "ttft_ms": 586,5088 "total_ms": 1090,5089 "tps": 132.95,5090 "cost_mean": 0.001108,5091 "out_tokens": 675092 },5093 "complex_reasoning": {5094 "runs": 2,5095 "errors": 0,5096 "pass_rate": 1,5097 "score": 1,5098 "judged": null,5099 "ttft_ms": 3022,5100 "total_ms": 4210,5101 "tps": 239.35,5102 "cost_mean": 0.003628,5103 "out_tokens": 2795104 },5105 "long_context": {5106 "runs": 2,5107 "errors": 0,5108 "pass_rate": 0.5,5109 "score": 0.5,5110 "judged": null,5111 "ttft_ms": 1204,5112 "total_ms": 1456,5113 "tps": 208.85000000000002,5114 "cost_mean": 0.0613936,5115 "out_tokens": 565116 },5117 "english_writing": {5118 "runs": 2,5119 "errors": 0,5120 "pass_rate": 1,5121 "score": 1,5122 "judged": 5,5123 "ttft_ms": 1036,5124 "total_ms": 4212.5,5125 "tps": 74.6,5126 "cost_mean": 0.002954,5127 "out_tokens": 235.55128 },5129 "french_writing": {5130 "runs": 2,5131 "errors": 0,5132 "pass_rate": 1,5133 "score": 1,5134 "judged": 5,5135 "ttft_ms": 1075,5136 "total_ms": 4789,5137 "tps": 80.5,5138 "cost_mean": 0.00376,5139 "out_tokens": 2985140 },5141 "programming": {5142 "runs": 2,5143 "errors": 0,5144 "pass_rate": 1,5145 "score": 1,5146 "judged": null,5147 "ttft_ms": 788,5148 "total_ms": 1484,5149 "tps": 153.8,5150 "cost_mean": 0.001434,5151 "out_tokens": 1075152 },5153 "simple_reasoning": {5154 "runs": 2,5155 "errors": 0,5156 "pass_rate": 1,5157 "score": 1,5158 "judged": null,5159 "ttft_ms": 693,5160 "total_ms": 1029.5,5161 "tps": 56.5,5162 "cost_mean": 0.000478,5163 "out_tokens": 195164 },5165 "knowledge_synthesis": {5166 "runs": 2,5167 "errors": 0,5168 "pass_rate": 1,5169 "score": 1,5170 "judged": 5,5171 "ttft_ms": 869,5172 "total_ms": 3308,5173 "tps": 103.05000000000001,5174 "cost_mean": 0.003474,5175 "out_tokens": 2515176 },5177 "tool_calling": {5178 "runs": 2,5179 "errors": 0,5180 "pass_rate": 1,5181 "score": 1,5182 "judged": null,5183 "ttft_ms": 1017.5,5184 "total_ms": 1369.5,5185 "tps": 117.4,5186 "cost_mean": 0.000794,5187 "out_tokens": 415188 },5189 "mathematics": {5190 "runs": 2,5191 "errors": 0,5192 "pass_rate": 1,5193 "score": 1,5194 "judged": null,5195 "ttft_ms": 2400,5196 "total_ms": 3314.5,5197 "tps": 136.3,5198 "cost_mean": 0.003516,5199 "out_tokens": 279.55200 }5201 }5202 },5203 {5204 "model": "gpt-6-luna",5205 "provider": "openai",5206 "runs": 22,5207 "errors": 0,5208 "pass_rate": 0.9545454545454546,5209 "mean_score": 0.9545454545454546,5210 "judged_mean": 4.666666666666667,5211 "ttft_median_ms": 2045,5212 "total_median_ms": 3000,5213 "tps_median": 194.35,5214 "cost_total": 0.014567000000000002,5215 "cost_mean": 0.0006621363636363638,5216 "in_tokens": 113620,5217 "out_tokens": 6410,5218 "reasoning_tokens": 3720,5219 "by_category": {5220 "document_analysis": {5221 "runs": 2,5222 "errors": 0,5223 "pass_rate": 1,5224 "score": 1,5225 "judged": null,5226 "ttft_ms": 1500,5227 "total_ms": 2418.5,5228 "tps": 140.2,5229 "cost_mean": 0.00011275,5230 "out_tokens": 125.55231 },5232 "json_generation": {5233 "runs": 2,5234 "errors": 0,5235 "pass_rate": 1,5236 "score": 1,5237 "judged": null,5238 "ttft_ms": 1590.5,5239 "total_ms": 2244,5240 "tps": 142.3,5241 "cost_mean": 0.00006170000000000001,5242 "out_tokens": 935243 },5244 "complex_reasoning": {5245 "runs": 2,5246 "errors": 0,5247 "pass_rate": 1,5248 "score": 1,5249 "judged": null,5250 "ttft_ms": 3736,5251 "total_ms": 5099,5252 "tps": 287.04999999999995,5253 "cost_mean": 0.00020775,5254 "out_tokens": 387.55255 },5256 "long_context": {5257 "runs": 2,5258 "errors": 0,5259 "pass_rate": 0.5,5260 "score": 0.5,5261 "judged": null,5262 "ttft_ms": 2542,5263 "total_ms": 2861.5,5264 "tps": 545.05,5265 "cost_mean": 0.00560715,5266 "out_tokens": 174.55267 },5268 "english_writing": {5269 "runs": 2,5270 "errors": 0,5271 "pass_rate": 1,5272 "score": 1,5273 "judged": 5,5274 "ttft_ms": 5620.5,5275 "total_ms": 7089.5,5276 "tps": 471.7,5277 "cost_mean": 0.0003494,5278 "out_tokens": 6865279 },5280 "french_writing": {5281 "runs": 2,5282 "errors": 0,5283 "pass_rate": 1,5284 "score": 1,5285 "judged": 5,5286 "ttft_ms": 4196.5,5287 "total_ms": 6311,5288 "tps": 340.25,5289 "cost_mean": 0.0003382,5290 "out_tokens": 6585291 },5292 "programming": {5293 "runs": 2,5294 "errors": 0,5295 "pass_rate": 1,5296 "score": 1,5297 "judged": null,5298 "ttft_ms": 1928.5,5299 "total_ms": 3055.5,5300 "tps": 189.05,5301 "cost_mean": 0.00011325,5302 "out_tokens": 211.55303 },5304 "simple_reasoning": {5305 "runs": 2,5306 "errors": 0,5307 "pass_rate": 1,5308 "score": 1,5309 "judged": null,5310 "ttft_ms": 1060.5,5311 "total_ms": 1400.5,5312 "tps": 196.95,5313 "cost_mean": 0.00004575,5314 "out_tokens": 66.55315 },5316 "knowledge_synthesis": {5317 "runs": 2,5318 "errors": 0,5319 "pass_rate": 1,5320 "score": 1,5321 "judged": 4,5322 "ttft_ms": 1515,5323 "total_ms": 2763.5,5324 "tps": 323.65,5325 "cost_mean": 0.00022285,5326 "out_tokens": 399.55327 },5328 "tool_calling": {5329 "runs": 2,5330 "errors": 0,5331 "pass_rate": 1,5332 "score": 1,5333 "judged": null,5334 "ttft_ms": 747,5335 "total_ms": 1100.5,5336 "tps": 116.15,5337 "cost_mean": 0.000035600000000000005,5338 "out_tokens": 415339 },5340 "mathematics": {5341 "runs": 2,5342 "errors": 0,5343 "pass_rate": 1,5344 "score": 1,5345 "judged": null,5346 "ttft_ms": 19505,5347 "total_ms": 20422.5,5348 "tps": 144.85,5349 "cost_mean": 0.0001891,5350 "out_tokens": 3625351 }5352 }5353 },5354 {5355 "model": "gpt-4o-mini",5356 "provider": "openai",5357 "runs": 22,5358 "errors": 0,5359 "pass_rate": 0.8181818181818182,5360 "mean_score": 0.9432272727272728,5361 "judged_mean": 4,5362 "ttft_median_ms": 561,5363 "total_median_ms": 1966,5364 "tps_median": 116.4,5365 "cost_total": 0.020267100000000003,5366 "cost_mean": 0.0009212318181818183,5367 "in_tokens": 113634,5368 "out_tokens": 5370,5369 "reasoning_tokens": 0,5370 "by_category": {5371 "document_analysis": {5372 "runs": 2,5373 "errors": 0,5374 "pass_rate": 1,5375 "score": 1,5376 "judged": null,5377 "ttft_ms": 1210.5,5378 "total_ms": 2798.5,5379 "tps": 86.45,5380 "cost_mean": 0.0001389,5381 "out_tokens": 1065382 },5383 "json_generation": {5384 "runs": 2,5385 "errors": 0,5386 "pass_rate": 1,5387 "score": 1,5388 "judged": null,5389 "ttft_ms": 643,5390 "total_ms": 1274.5,5391 "tps": 93.5,5392 "cost_mean": 0.0000585,5393 "out_tokens": 595394 },5395 "complex_reasoning": {5396 "runs": 2,5397 "errors": 0,5398 "pass_rate": 1,5399 "score": 1,5400 "judged": null,5401 "ttft_ms": 602.5,5402 "total_ms": 5739.5,5403 "tps": 147.10000000000002,5404 "cost_mean": 0.00047444999999999996,5405 "out_tokens": 755.55406 },5407 "long_context": {5408 "runs": 2,5409 "errors": 0,5410 "pass_rate": 1,5411 "score": 1,5412 "judged": null,5413 "ttft_ms": 1889,5414 "total_ms": 2082,5415 "tps": 50.95,5416 "cost_mean": 0.008284799999999998,5417 "out_tokens": 85418 },5419 "english_writing": {5420 "runs": 2,5421 "errors": 0,5422 "pass_rate": 0.5,5423 "score": 0.9585,5424 "judged": 4,5425 "ttft_ms": 533,5426 "total_ms": 1926.5,5427 "tps": 133.1,5428 "cost_mean": 0.00012075,5429 "out_tokens": 1855430 },5431 "french_writing": {5432 "runs": 2,5433 "errors": 0,5434 "pass_rate": 1,5435 "score": 1,5436 "judged": 5,5437 "ttft_ms": 541.5,5438 "total_ms": 2547,5439 "tps": 125.89999999999999,5440 "cost_mean": 0.00016455,5441 "out_tokens": 2515442 },5443 "programming": {5444 "runs": 2,5445 "errors": 0,5446 "pass_rate": 1,5447 "score": 1,5448 "judged": null,5449 "ttft_ms": 611,5450 "total_ms": 1588,5451 "tps": 119.80000000000001,5452 "cost_mean": 0.0000816,5453 "out_tokens": 1175454 },5455 "simple_reasoning": {5456 "runs": 2,5457 "errors": 0,5458 "pass_rate": 0,5459 "score": 0.667,5460 "judged": null,5461 "ttft_ms": 694,5462 "total_ms": 974.5,5463 "tps": 61.4,5464 "cost_mean": 0.000028800000000000002,5465 "out_tokens": 16.55466 },5467 "knowledge_synthesis": {5468 "runs": 2,5469 "errors": 0,5470 "pass_rate": 1,5471 "score": 1,5472 "judged": 3,5473 "ttft_ms": 527.5,5474 "total_ms": 2795,5475 "tps": 105.5,5476 "cost_mean": 0.00017189999999999998,5477 "out_tokens": 228.55478 },5479 "tool_calling": {5480 "runs": 2,5481 "errors": 0,5482 "pass_rate": 1,5483 "score": 1,5484 "judged": null,5485 "ttft_ms": 532.5,5486 "total_ms": 916.5,5487 "tps": 99,5488 "cost_mean": 0.0000447,5489 "out_tokens": 385490 },5491 "mathematics": {5492 "runs": 2,5493 "errors": 0,5494 "pass_rate": 0.5,5495 "score": 0.75,5496 "judged": null,5497 "ttft_ms": 478,5498 "total_ms": 7312,5499 "tps": 135.25,5500 "cost_mean": 0.0005646,5501 "out_tokens": 920.55502 }5503 }5504 },5505 {5506 "model": "gpt-5.4-nano",5507 "provider": "openai",5508 "runs": 22,5509 "errors": 0,5510 "pass_rate": 0.8636363636363636,5511 "mean_score": 0.9166363636363637,5512 "judged_mean": 4,5513 "ttft_median_ms": 548.5,5514 "total_median_ms": 1727,5515 "tps_median": 134.85000000000002,5516 "cost_total": 0.007665850000000001,5517 "cost_mean": 0.00034844772727272733,5518 "in_tokens": 113620,5519 "out_tokens": 3805,5520 "reasoning_tokens": 0,5521 "by_category": {5522 "document_analysis": {5523 "runs": 2,5524 "errors": 0,5525 "pass_rate": 1,5526 "score": 1,5527 "judged": null,5528 "ttft_ms": 587.5,5529 "total_ms": 1476.5,5530 "tps": 106.9,5531 "cost_mean": 0.00020625,5532 "out_tokens": 855533 },5534 "json_generation": {5535 "runs": 2,5536 "errors": 0,5537 "pass_rate": 1,5538 "score": 1,5539 "judged": null,5540 "ttft_ms": 587.5,5541 "total_ms": 1126.5,5542 "tps": 127.95,5543 "cost_mean": 0.00011415000000000001,5544 "out_tokens": 675545 },5546 "complex_reasoning": {5547 "runs": 2,5548 "errors": 0,5549 "pass_rate": 1,5550 "score": 1,5551 "judged": null,5552 "ttft_ms": 539,5553 "total_ms": 3506,5554 "tps": 199,5555 "cost_mean": 0.000781125,5556 "out_tokens": 602.55557 },5558 "long_context": {5559 "runs": 2,5560 "errors": 0,5561 "pass_rate": 1,5562 "score": 1,5563 "judged": null,5564 "ttft_ms": 500.5,5565 "total_ms": 776,5566 "tps": 40.2,5567 "cost_mean": 0.00114635,5568 "out_tokens": 115569 },5570 "english_writing": {5571 "runs": 2,5572 "errors": 0,5573 "pass_rate": 1,5574 "score": 1,5575 "judged": 5,5576 "ttft_ms": 658.5,5577 "total_ms": 1926,5578 "tps": 162.25,5579 "cost_mean": 0.00026967500000000004,5580 "out_tokens": 205.55581 },5582 "french_writing": {5583 "runs": 2,5584 "errors": 0,5585 "pass_rate": 1,5586 "score": 1,5587 "judged": 4.5,5588 "ttft_ms": 510,5589 "total_ms": 2368,5590 "tps": 134.85000000000002,5591 "cost_mean": 0.000331525,5592 "out_tokens": 250.55593 },5594 "programming": {5595 "runs": 2,5596 "errors": 0,5597 "pass_rate": 1,5598 "score": 1,5599 "judged": null,5600 "ttft_ms": 609,5601 "total_ms": 1925.5,5602 "tps": 123.5,5603 "cost_mean": 0.000213125,5604 "out_tokens": 158.55605 },5606 "simple_reasoning": {5607 "runs": 2,5608 "errors": 0,5609 "pass_rate": 0,5610 "score": 0.333,5611 "judged": null,5612 "ttft_ms": 511,5613 "total_ms": 824,5614 "tps": 63,5615 "cost_mean": 0.00004875,5616 "out_tokens": 195617 },5618 "knowledge_synthesis": {5619 "runs": 2,5620 "errors": 0,5621 "pass_rate": 1,5622 "score": 1,5623 "judged": 2.5,5624 "ttft_ms": 541,5625 "total_ms": 2299.5,5626 "tps": 151.64999999999998,5627 "cost_mean": 0.00037995,5628 "out_tokens": 2675629 },5630 "tool_calling": {5631 "runs": 2,5632 "errors": 0,5633 "pass_rate": 1,5634 "score": 1,5635 "judged": null,5636 "ttft_ms": 595,5637 "total_ms": 928,5638 "tps": 123.1,5639 "cost_mean": 0.00008145,5640 "out_tokens": 415641 },5642 "mathematics": {5643 "runs": 2,5644 "errors": 0,5645 "pass_rate": 0.5,5646 "score": 0.75,5647 "judged": null,5648 "ttft_ms": 643.5,5649 "total_ms": 1831,5650 "tps": 127.95,5651 "cost_mean": 0.00026057500000000003,5652 "out_tokens": 195.55653 }5654 }5655 },5656 {5657 "model": "claude-fable-5",5658 "provider": "anthropic",5659 "runs": 22,5660 "errors": 0,5661 "pass_rate": 0.9090909090909091,5662 "mean_score": 0.9090909090909091,5663 "judged_mean": 5,5664 "ttft_median_ms": 1763.5,5665 "total_median_ms": 2839.5,5666 "tps_median": 111.85,5667 "cost_total": 2.0221299999999993,5668 "cost_mean": 0.09191499999999997,5669 "in_tokens": 168738,5670 "out_tokens": 6695,5671 "reasoning_tokens": 1105,5672 "by_category": {5673 "document_analysis": {5674 "runs": 2,5675 "errors": 0,5676 "pass_rate": 1,5677 "score": 1,5678 "judged": null,5679 "ttft_ms": 955.5,5680 "total_ms": 1992.5,5681 "tps": 125.5,5682 "cost_mean": 0.01668,5683 "out_tokens": 1305684 },5685 "json_generation": {5686 "runs": 2,5687 "errors": 0,5688 "pass_rate": 1,5689 "score": 1,5690 "judged": null,5691 "ttft_ms": 1653.5,5692 "total_ms": 2755,5693 "tps": 104.45,5694 "cost_mean": 0.01069,5695 "out_tokens": 1155696 },5697 "complex_reasoning": {5698 "runs": 2,5699 "errors": 0,5700 "pass_rate": 1,5701 "score": 1,5702 "judged": null,5703 "ttft_ms": 4880,5704 "total_ms": 8848.5,5705 "tps": 194.8,5706 "cost_mean": 0.04102,5707 "out_tokens": 7745708 },5709 "long_context": {5710 "runs": 2,5711 "errors": 0,5712 "pass_rate": 0,5713 "score": 0,5714 "judged": null,5715 "ttft_ms": null,5716 "total_ms": 1710,5717 "tps": null,5718 "cost_mean": 0.80911,5719 "out_tokens": 05720 },5721 "english_writing": {5722 "runs": 2,5723 "errors": 0,5724 "pass_rate": 1,5725 "score": 1,5726 "judged": 5,5727 "ttft_ms": 1876.5,5728 "total_ms": 8400,5729 "tps": 77.25,5730 "cost_mean": 0.02622,5731 "out_tokens": 5045732 },5733 "french_writing": {5734 "runs": 2,5735 "errors": 0,5736 "pass_rate": 1,5737 "score": 1,5738 "judged": 5,5739 "ttft_ms": 1846.5,5740 "total_ms": 9302.5,5741 "tps": 76.55,5742 "cost_mean": 0.03007,5743 "out_tokens": 5695744 },5745 "programming": {5746 "runs": 2,5747 "errors": 0,5748 "pass_rate": 1,5749 "score": 1,5750 "judged": null,5751 "ttft_ms": 1488.5,5752 "total_ms": 2813,5753 "tps": 121.25,5754 "cost_mean": 0.009225,5755 "out_tokens": 160.55756 },5757 "simple_reasoning": {5758 "runs": 2,5759 "errors": 0,5760 "pass_rate": 1,5761 "score": 1,5762 "judged": null,5763 "ttft_ms": 1527.5,5764 "total_ms": 2116,5765 "tps": 32.349999999999994,5766 "cost_mean": 0.00274,5767 "out_tokens": 195768 },5769 "knowledge_synthesis": {5770 "runs": 2,5771 "errors": 0,5772 "pass_rate": 1,5773 "score": 1,5774 "judged": 5,5775 "ttft_ms": 3075,5776 "total_ms": 8038.5,5777 "tps": 114.44999999999999,5778 "cost_mean": 0.031915,5779 "out_tokens": 568.55780 },5781 "tool_calling": {5782 "runs": 2,5783 "errors": 0,5784 "pass_rate": 1,5785 "score": 1,5786 "judged": null,5787 "ttft_ms": 1831.5,5788 "total_ms": 2653.5,5789 "tps": 171.9,5790 "cost_mean": 0.013615,5791 "out_tokens": 134.55792 },5793 "mathematics": {5794 "runs": 2,5795 "errors": 0,5796 "pass_rate": 1,5797 "score": 1,5798 "judged": null,5799 "ttft_ms": 2058.5,5800 "total_ms": 4882.5,5801 "tps": 109.55,5802 "cost_mean": 0.01978,5803 "out_tokens": 3735804 }5805 }5806 }5807 ],5808 "discovery": [5809 {5810 "provider": "openai",5811 "id": "babbage-002",5812 "category": "legacy-completion",5813 "aliasOf": null,5814 "verdict": "untested",5815 "pricing": null,5816 "probes": {},5817 "notes": [5818 "category legacy-completion: not a general text model, not probed (inappropriate request type)"5819 ],5820 "created": "2023-08-21"5821 },5822 {5823 "provider": "openai",5824 "id": "chat-latest",5825 "category": "text",5826 "aliasOf": null,5827 "verdict": "tested",5828 "pricing": {5829 "input": 5,5830 "cached": 0.5,5831 "output": 305832 },5833 "probes": {5834 "text": {5835 "status": "ok",5836 "ms": 8025837 },5838 "streaming": {5839 "status": "ok",5840 "ms": 9165841 },5842 "tool_use": {5843 "status": "ok",5844 "ms": 8295845 },5846 "structured_output": {5847 "status": "ok",5848 "ms": 9935849 },5850 "vision": {5851 "status": "ok",5852 "ms": 9485853 },5854 "reasoning_control": {5855 "status": "ok",5856 "ms": 8965857 }5858 },5859 "notes": [],5860 "created": "2026-05-02"5861 },5862 {5863 "provider": "openai",5864 "id": "chatgpt-image-latest",5865 "category": "image",5866 "aliasOf": null,5867 "verdict": "untested",5868 "pricing": null,5869 "probes": {},5870 "notes": [5871 "category image: not a general text model, not probed (inappropriate request type)"5872 ],5873 "created": "2025-12-16"5874 },5875 {5876 "provider": "openai",5877 "id": "davinci-002",5878 "category": "legacy-completion",5879 "aliasOf": null,5880 "verdict": "untested",5881 "pricing": null,5882 "probes": {},5883 "notes": [5884 "category legacy-completion: not a general text model, not probed (inappropriate request type)"5885 ],5886 "created": "2023-08-21"5887 },5888 {5889 "provider": "openai",5890 "id": "gpt-3.5-turbo",5891 "category": "text",5892 "aliasOf": null,5893 "verdict": "unavailable",5894 "pricing": null,5895 "probes": {5896 "text": {5897 "status": "unavailable",5898 "ms": 201,5899 "detail": "404 404 The model `gpt-3.5-turbo` does not exist or you do not have access to it."5900 }5901 },5902 "notes": [],5903 "created": "2023-02-28"5904 },5905 {5906 "provider": "openai",5907 "id": "gpt-3.5-turbo-0125",5908 "category": "text",5909 "aliasOf": null,5910 "verdict": "unavailable",5911 "pricing": null,5912 "probes": {5913 "text": {5914 "status": "unavailable",5915 "ms": 224,5916 "detail": "404 404 The model `gpt-3.5-turbo-0125` does not exist or you do not have access to it."5917 }5918 },5919 "notes": [],5920 "created": "2024-01-23"5921 },5922 {5923 "provider": "openai",5924 "id": "gpt-3.5-turbo-1106",5925 "category": "text",5926 "aliasOf": null,5927 "verdict": "unavailable",5928 "pricing": null,5929 "probes": {5930 "text": {5931 "status": "unavailable",5932 "ms": 198,5933 "detail": "404 404 The model `gpt-3.5-turbo-1106` does not exist or you do not have access to it."5934 }5935 },5936 "notes": [],5937 "created": "2023-11-02"5938 },5939 {5940 "provider": "openai",5941 "id": "gpt-3.5-turbo-16k",5942 "category": "text",5943 "aliasOf": null,5944 "verdict": "unavailable",5945 "pricing": null,5946 "probes": {5947 "text": {5948 "status": "unavailable",5949 "ms": 267,5950 "detail": "404 404 The model `gpt-3.5-turbo-16k` does not exist or you do not have access to it."5951 }5952 },5953 "notes": [],5954 "created": "2023-05-10"5955 },5956 {5957 "provider": "openai",5958 "id": "gpt-3.5-turbo-instruct",5959 "category": "legacy-completion",5960 "aliasOf": null,5961 "verdict": "untested",5962 "pricing": null,5963 "probes": {},5964 "notes": [5965 "category legacy-completion: not a general text model, not probed (inappropriate request type)"5966 ],5967 "created": "2023-08-24"5968 },5969 {5970 "provider": "openai",5971 "id": "gpt-3.5-turbo-instruct-0914",5972 "category": "legacy-completion",5973 "aliasOf": null,5974 "verdict": "untested",5975 "pricing": null,5976 "probes": {},5977 "notes": [5978 "category legacy-completion: not a general text model, not probed (inappropriate request type)"5979 ],5980 "created": "2023-09-07"5981 },5982 {5983 "provider": "openai",5984 "id": "gpt-4",5985 "category": "text",5986 "aliasOf": null,5987 "verdict": "unavailable",5988 "pricing": null,5989 "probes": {5990 "text": {5991 "status": "unavailable",5992 "ms": 195,5993 "detail": "404 404 The model `gpt-4` does not exist or you do not have access to it."5994 }5995 },5996 "notes": [],5997 "created": "2023-06-27"5998 },5999 {6000 "provider": "openai",6001 "id": "gpt-4-0613",6002 "category": "text",6003 "aliasOf": null,6004 "verdict": "unavailable",6005 "pricing": null,6006 "probes": {6007 "text": {6008 "status": "unavailable",6009 "ms": 155,6010 "detail": "404 404 The model `gpt-4-0613` does not exist or you do not have access to it."6011 }6012 },6013 "notes": [],6014 "created": "2023-06-12"6015 },6016 {6017 "provider": "openai",6018 "id": "gpt-4-turbo",6019 "category": "text",6020 "aliasOf": null,6021 "verdict": "unavailable",6022 "pricing": null,6023 "probes": {6024 "text": {6025 "status": "unavailable",6026 "ms": 143,6027 "detail": "404 404 The model `gpt-4-turbo` does not exist or you do not have access to it."6028 }6029 },6030 "notes": [],6031 "created": "2024-04-05"6032 },6033 {6034 "provider": "openai",6035 "id": "gpt-4-turbo-2024-04-09",6036 "category": "text",6037 "aliasOf": "gpt-4-turbo",6038 "verdict": "untested",6039 "pricing": null,6040 "probes": {},6041 "notes": [6042 "dated snapshot of gpt-4-turbo; not probed separately"6043 ],6044 "created": "2024-04-08"6045 },6046 {6047 "provider": "openai",6048 "id": "gpt-4.1",6049 "category": "text",6050 "aliasOf": null,6051 "verdict": "tested",6052 "pricing": {6053 "input": 2,6054 "cached": 0.5,6055 "output": 86056 },6057 "probes": {6058 "text": {6059 "status": "ok",6060 "ms": 6506061 },6062 "streaming": {6063 "status": "ok",6064 "ms": 5476065 },6066 "tool_use": {6067 "status": "ok",6068 "ms": 6076069 },6070 "structured_output": {6071 "status": "ok",6072 "ms": 6126073 },6074 "vision": {6075 "status": "ok",6076 "ms": 5686077 },6078 "reasoning_control": {6079 "status": "ok",6080 "ms": 5416081 }6082 },6083 "notes": [],6084 "created": "2025-04-10"6085 },6086 {6087 "provider": "openai",6088 "id": "gpt-4.1-2025-04-14",6089 "category": "text",6090 "aliasOf": "gpt-4.1",6091 "verdict": "untested",6092 "pricing": {6093 "input": 2,6094 "cached": 0.5,6095 "output": 86096 },6097 "probes": {},6098 "notes": [6099 "dated snapshot of gpt-4.1; not probed separately"6100 ],6101 "created": "2025-04-10"6102 },6103 {6104 "provider": "openai",6105 "id": "gpt-4.1-mini",6106 "category": "text",6107 "aliasOf": null,6108 "verdict": "tested",6109 "pricing": {6110 "input": 0.4,6111 "cached": 0.1,6112 "output": 1.66113 },6114 "probes": {6115 "text": {6116 "status": "ok",6117 "ms": 5656118 },6119 "streaming": {6120 "status": "ok",6121 "ms": 8246122 },6123 "tool_use": {6124 "status": "ok",6125 "ms": 6346126 },6127 "structured_output": {6128 "status": "ok",6129 "ms": 5886130 },6131 "vision": {6132 "status": "unsupported",6133 "ms": 686,6134 "detail": "answer: Pink"6135 },6136 "reasoning_control": {6137 "status": "ok",6138 "ms": 4306139 }6140 },6141 "notes": [],6142 "created": "2025-04-10"6143 },6144 {6145 "provider": "openai",6146 "id": "gpt-4.1-mini-2025-04-14",6147 "category": "text",6148 "aliasOf": "gpt-4.1-mini",6149 "verdict": "untested",6150 "pricing": {6151 "input": 0.4,6152 "cached": 0.1,6153 "output": 1.66154 },6155 "probes": {},6156 "notes": [6157 "dated snapshot of gpt-4.1-mini; not probed separately"6158 ],6159 "created": "2025-04-10"6160 },6161 {6162 "provider": "openai",6163 "id": "gpt-4.1-nano",6164 "category": "text",6165 "aliasOf": null,6166 "verdict": "unavailable",6167 "pricing": {6168 "input": 0.1,6169 "cached": 0.025,6170 "output": 0.46171 },6172 "probes": {6173 "text": {6174 "status": "unavailable",6175 "ms": 568,6176 "detail": "404 404 The model `gpt-4.1-nano` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"6177 }6178 },6179 "notes": [],6180 "created": "2025-04-10"6181 },6182 {6183 "provider": "openai",6184 "id": "gpt-4.1-nano-2025-04-14",6185 "category": "text",6186 "aliasOf": "gpt-4.1-nano",6187 "verdict": "untested",6188 "pricing": {6189 "input": 0.1,6190 "cached": 0.025,6191 "output": 0.46192 },6193 "probes": {},6194 "notes": [6195 "dated snapshot of gpt-4.1-nano; not probed separately"6196 ],6197 "created": "2025-04-10"6198 },6199 {6200 "provider": "openai",6201 "id": "gpt-4o",6202 "category": "text",6203 "aliasOf": null,6204 "verdict": "tested",6205 "pricing": {6206 "input": 2.5,6207 "cached": 1.25,6208 "output": 106209 },6210 "probes": {6211 "text": {6212 "status": "ok",6213 "ms": 6506214 },6215 "streaming": {6216 "status": "ok",6217 "ms": 7106218 },6219 "tool_use": {6220 "status": "ok",6221 "ms": 6066222 },6223 "structured_output": {6224 "status": "ok",6225 "ms": 7276226 },6227 "vision": {6228 "status": "ok",6229 "ms": 6666230 },6231 "reasoning_control": {6232 "status": "ok",6233 "ms": 5666234 }6235 },6236 "notes": [],6237 "created": "2024-05-10"6238 },6239 {6240 "provider": "openai",6241 "id": "gpt-4o-2024-05-13",6242 "category": "text",6243 "aliasOf": "gpt-4o",6244 "verdict": "untested",6245 "pricing": {6246 "input": 2.5,6247 "cached": 1.25,6248 "output": 106249 },6250 "probes": {},6251 "notes": [6252 "dated snapshot of gpt-4o; not probed separately"6253 ],6254 "created": "2024-05-10"6255 },6256 {6257 "provider": "openai",6258 "id": "gpt-4o-2024-08-06",6259 "category": "text",6260 "aliasOf": "gpt-4o",6261 "verdict": "untested",6262 "pricing": {6263 "input": 2.5,6264 "cached": 1.25,6265 "output": 106266 },6267 "probes": {},6268 "notes": [6269 "dated snapshot of gpt-4o; not probed separately"6270 ],6271 "created": "2024-08-04"6272 },6273 {6274 "provider": "openai",6275 "id": "gpt-4o-2024-11-20",6276 "category": "text",6277 "aliasOf": "gpt-4o",6278 "verdict": "untested",6279 "pricing": {6280 "input": 2.5,6281 "cached": 1.25,6282 "output": 106283 },6284 "probes": {},6285 "notes": [6286 "dated snapshot of gpt-4o; not probed separately"6287 ],6288 "created": "2025-02-12"6289 },6290 {6291 "provider": "openai",6292 "id": "gpt-4o-mini",6293 "category": "text",6294 "aliasOf": null,6295 "verdict": "tested",6296 "pricing": {6297 "input": 0.15,6298 "cached": 0.075,6299 "output": 0.66300 },6301 "probes": {6302 "text": {6303 "status": "ok",6304 "ms": 4956305 },6306 "streaming": {6307 "status": "ok",6308 "ms": 5896309 },6310 "tool_use": {6311 "status": "ok",6312 "ms": 6206313 },6314 "structured_output": {6315 "status": "ok",6316 "ms": 5996317 },6318 "vision": {6319 "status": "ok",6320 "ms": 6626321 },6322 "reasoning_control": {6323 "status": "ok",6324 "ms": 17476325 }6326 },6327 "notes": [],6328 "created": "2024-07-16"6329 },6330 {6331 "provider": "openai",6332 "id": "gpt-4o-mini-2024-07-18",6333 "category": "text",6334 "aliasOf": "gpt-4o-mini",6335 "verdict": "untested",6336 "pricing": {6337 "input": 0.15,6338 "cached": 0.075,6339 "output": 0.66340 },6341 "probes": {},6342 "notes": [6343 "dated snapshot of gpt-4o-mini; not probed separately"6344 ],6345 "created": "2024-07-16"6346 },6347 {6348 "provider": "openai",6349 "id": "gpt-4o-mini-search-preview",6350 "category": "search",6351 "aliasOf": null,6352 "verdict": "untested",6353 "pricing": null,6354 "probes": {},6355 "notes": [6356 "category search: not a general text model, not probed (inappropriate request type)"6357 ],6358 "created": "2025-03-07"6359 },6360 {6361 "provider": "openai",6362 "id": "gpt-4o-mini-search-preview-2025-03-11",6363 "category": "search",6364 "aliasOf": "gpt-4o-mini-search-preview",6365 "verdict": "untested",6366 "pricing": null,6367 "probes": {},6368 "notes": [6369 "dated snapshot of gpt-4o-mini-search-preview; not probed separately"6370 ],6371 "created": "2025-03-07"6372 },6373 {6374 "provider": "openai",6375 "id": "gpt-4o-mini-transcribe",6376 "category": "audio",6377 "aliasOf": null,6378 "verdict": "untested",6379 "pricing": null,6380 "probes": {},6381 "notes": [6382 "category audio: not a general text model, not probed (inappropriate request type)"6383 ],6384 "created": "2025-03-15"6385 },6386 {6387 "provider": "openai",6388 "id": "gpt-4o-mini-transcribe-2025-03-20",6389 "category": "audio",6390 "aliasOf": "gpt-4o-mini-transcribe",6391 "verdict": "untested",6392 "pricing": null,6393 "probes": {},6394 "notes": [6395 "dated snapshot of gpt-4o-mini-transcribe; not probed separately"6396 ],6397 "created": "2025-12-13"6398 },6399 {6400 "provider": "openai",6401 "id": "gpt-4o-mini-transcribe-2025-12-15",6402 "category": "audio",6403 "aliasOf": "gpt-4o-mini-transcribe",6404 "verdict": "untested",6405 "pricing": null,6406 "probes": {},6407 "notes": [6408 "dated snapshot of gpt-4o-mini-transcribe; not probed separately"6409 ],6410 "created": "2025-12-13"6411 },6412 {6413 "provider": "openai",6414 "id": "gpt-4o-mini-tts",6415 "category": "audio",6416 "aliasOf": null,6417 "verdict": "untested",6418 "pricing": null,6419 "probes": {},6420 "notes": [6421 "category audio: not a general text model, not probed (inappropriate request type)"6422 ],6423 "created": "2025-03-19"6424 },6425 {6426 "provider": "openai",6427 "id": "gpt-4o-mini-tts-2025-03-20",6428 "category": "audio",6429 "aliasOf": "gpt-4o-mini-tts",6430 "verdict": "untested",6431 "pricing": null,6432 "probes": {},6433 "notes": [6434 "dated snapshot of gpt-4o-mini-tts; not probed separately"6435 ],6436 "created": "2025-12-13"6437 },6438 {6439 "provider": "openai",6440 "id": "gpt-4o-mini-tts-2025-12-15",6441 "category": "audio",6442 "aliasOf": "gpt-4o-mini-tts",6443 "verdict": "untested",6444 "pricing": null,6445 "probes": {},6446 "notes": [6447 "dated snapshot of gpt-4o-mini-tts; not probed separately"6448 ],6449 "created": "2025-12-13"6450 },6451 {6452 "provider": "openai",6453 "id": "gpt-4o-search-preview",6454 "category": "search",6455 "aliasOf": null,6456 "verdict": "untested",6457 "pricing": null,6458 "probes": {},6459 "notes": [6460 "category search: not a general text model, not probed (inappropriate request type)"6461 ],6462 "created": "2026-02-24"6463 },6464 {6465 "provider": "openai",6466 "id": "gpt-4o-search-preview-2025-03-11",6467 "category": "search",6468 "aliasOf": "gpt-4o-search-preview",6469 "verdict": "untested",6470 "pricing": null,6471 "probes": {},6472 "notes": [6473 "dated snapshot of gpt-4o-search-preview; not probed separately"6474 ],6475 "created": "2026-02-24"6476 },6477 {6478 "provider": "openai",6479 "id": "gpt-4o-transcribe",6480 "category": "audio",6481 "aliasOf": null,6482 "verdict": "untested",6483 "pricing": null,6484 "probes": {},6485 "notes": [6486 "category audio: not a general text model, not probed (inappropriate request type)"6487 ],6488 "created": "2025-03-15"6489 },6490 {6491 "provider": "openai",6492 "id": "gpt-4o-transcribe-diarize",6493 "category": "audio",6494 "aliasOf": null,6495 "verdict": "untested",6496 "pricing": null,6497 "probes": {},6498 "notes": [6499 "category audio: not a general text model, not probed (inappropriate request type)"6500 ],6501 "created": "2025-06-24"6502 },6503 {6504 "provider": "openai",6505 "id": "gpt-5",6506 "category": "text",6507 "aliasOf": null,6508 "verdict": "tested",6509 "pricing": {6510 "input": 1.25,6511 "cached": 0.125,6512 "output": 106513 },6514 "probes": {6515 "text": {6516 "status": "ok",6517 "ms": 20306518 },6519 "streaming": {6520 "status": "ok",6521 "ms": 12106522 },6523 "tool_use": {6524 "status": "ok",6525 "ms": 9866526 },6527 "structured_output": {6528 "status": "ok",6529 "ms": 83946530 },6531 "vision": {6532 "status": "ok",6533 "ms": 11056534 },6535 "reasoning_control": {6536 "status": "ok",6537 "ms": 9076538 }6539 },6540 "notes": [],6541 "created": "2025-08-05"6542 },6543 {6544 "provider": "openai",6545 "id": "gpt-5-2025-08-07",6546 "category": "text",6547 "aliasOf": "gpt-5",6548 "verdict": "untested",6549 "pricing": {6550 "input": 1.25,6551 "cached": 0.125,6552 "output": 106553 },6554 "probes": {},6555 "notes": [6556 "dated snapshot of gpt-5; not probed separately"6557 ],6558 "created": "2025-08-01"6559 },6560 {6561 "provider": "openai",6562 "id": "gpt-5-chat-latest",6563 "category": "text",6564 "aliasOf": null,6565 "verdict": "unavailable",6566 "pricing": {6567 "input": 1.25,6568 "cached": 0.125,6569 "output": 106570 },6571 "probes": {6572 "text": {6573 "status": "unavailable",6574 "ms": 163,6575 "detail": "404 404 The model `gpt-5-chat-latest` does not exist or you do not have access to it."6576 }6577 },6578 "notes": [],6579 "created": "2025-08-01"6580 },6581 {6582 "provider": "openai",6583 "id": "gpt-5-codex",6584 "category": "text-specialized",6585 "aliasOf": null,6586 "verdict": "unavailable",6587 "pricing": {6588 "input": 1.25,6589 "cached": 0.125,6590 "output": 106591 },6592 "probes": {6593 "text": {6594 "status": "unavailable",6595 "ms": 147,6596 "detail": "404 404 The model `gpt-5-codex` does not exist or you do not have access to it."6597 }6598 },6599 "notes": [],6600 "created": "2025-09-10"6601 },6602 {6603 "provider": "openai",6604 "id": "gpt-5-mini",6605 "category": "text",6606 "aliasOf": null,6607 "verdict": "tested",6608 "pricing": {6609 "input": 0.25,6610 "cached": 0.025,6611 "output": 26612 },6613 "probes": {6614 "text": {6615 "status": "ok",6616 "ms": 14846617 },6618 "streaming": {6619 "status": "ok",6620 "ms": 14896621 },6622 "tool_use": {6623 "status": "ok",6624 "ms": 15566625 },6626 "structured_output": {6627 "status": "ok",6628 "ms": 50956629 },6630 "vision": {6631 "status": "ok",6632 "ms": 14426633 },6634 "reasoning_control": {6635 "status": "ok",6636 "ms": 10926637 }6638 },6639 "notes": [],6640 "created": "2025-08-05"6641 },6642 {6643 "provider": "openai",6644 "id": "gpt-5-mini-2025-08-07",6645 "category": "text",6646 "aliasOf": "gpt-5-mini",6647 "verdict": "untested",6648 "pricing": {6649 "input": 0.25,6650 "cached": 0.025,6651 "output": 26652 },6653 "probes": {},6654 "notes": [6655 "dated snapshot of gpt-5-mini; not probed separately"6656 ],6657 "created": "2025-08-05"6658 },6659 {6660 "provider": "openai",6661 "id": "gpt-5-nano",6662 "category": "text",6663 "aliasOf": null,6664 "verdict": "tested",6665 "pricing": {6666 "input": 0.05,6667 "cached": 0.005,6668 "output": 0.46669 },6670 "probes": {6671 "text": {6672 "status": "ok",6673 "ms": 12736674 },6675 "streaming": {6676 "status": "ok",6677 "ms": 11036678 },6679 "tool_use": {6680 "status": "ok",6681 "ms": 9666682 },6683 "structured_output": {6684 "status": "ok",6685 "ms": 41846686 },6687 "vision": {6688 "status": "ok",6689 "ms": 8986690 },6691 "reasoning_control": {6692 "status": "ok",6693 "ms": 9816694 }6695 },6696 "notes": [],6697 "created": "2025-08-05"6698 },6699 {6700 "provider": "openai",6701 "id": "gpt-5-nano-2025-08-07",6702 "category": "text",6703 "aliasOf": "gpt-5-nano",6704 "verdict": "untested",6705 "pricing": {6706 "input": 0.05,6707 "cached": 0.005,6708 "output": 0.46709 },6710 "probes": {},6711 "notes": [6712 "dated snapshot of gpt-5-nano; not probed separately"6713 ],6714 "created": "2025-08-05"6715 },6716 {6717 "provider": "openai",6718 "id": "gpt-5-pro",6719 "category": "text",6720 "aliasOf": null,6721 "verdict": "tested",6722 "pricing": {6723 "input": 15,6724 "cached": null,6725 "output": 1206726 },6727 "probes": {6728 "text": {6729 "status": "ok",6730 "ms": 122696731 },6732 "streaming": {6733 "status": "ok",6734 "ms": 82816735 },6736 "tool_use": {6737 "status": "ok",6738 "ms": 150456739 },6740 "structured_output": {6741 "status": "ok",6742 "ms": 703156743 },6744 "vision": {6745 "status": "ok",6746 "ms": 1577546747 },6748 "reasoning_control": {6749 "status": "unsupported",6750 "ms": 135,6751 "detail": "400 400 Unsupported value: 'low' is not supported with the 'gpt-5-pro' model. Supported values are: 'high'."6752 }6753 },6754 "notes": [6755 "pro/expensive tier: only text + structured_output probed; benchmark skipped for cost ($30-$150/M input)"6756 ],6757 "created": "2025-10-03"6758 },6759 {6760 "provider": "openai",6761 "id": "gpt-5-pro-2025-10-06",6762 "category": "text",6763 "aliasOf": "gpt-5-pro",6764 "verdict": "untested",6765 "pricing": {6766 "input": 15,6767 "cached": null,6768 "output": 1206769 },6770 "probes": {},6771 "notes": [6772 "dated snapshot of gpt-5-pro; not probed separately"6773 ],6774 "created": "2025-10-03"6775 },6776 {6777 "provider": "openai",6778 "id": "gpt-5-search-api",6779 "category": "search",6780 "aliasOf": null,6781 "verdict": "untested",6782 "pricing": null,6783 "probes": {},6784 "notes": [6785 "category search: not a general text model, not probed (inappropriate request type)"6786 ],6787 "created": "2025-10-03"6788 },6789 {6790 "provider": "openai",6791 "id": "gpt-5-search-api-2025-10-14",6792 "category": "search",6793 "aliasOf": "gpt-5-search-api",6794 "verdict": "untested",6795 "pricing": null,6796 "probes": {},6797 "notes": [6798 "dated snapshot of gpt-5-search-api; not probed separately"6799 ],6800 "created": "2025-10-09"6801 },6802 {6803 "provider": "openai",6804 "id": "gpt-5.1",6805 "category": "text",6806 "aliasOf": null,6807 "verdict": "tested",6808 "pricing": {6809 "input": 1.25,6810 "cached": 0.125,6811 "output": 106812 },6813 "probes": {6814 "text": {6815 "status": "ok",6816 "ms": 6676817 },6818 "streaming": {6819 "status": "ok",6820 "ms": 7086821 },6822 "tool_use": {6823 "status": "ok",6824 "ms": 7036825 },6826 "structured_output": {6827 "status": "ok",6828 "ms": 7576829 },6830 "vision": {6831 "status": "ok",6832 "ms": 8946833 },6834 "reasoning_control": {6835 "status": "ok",6836 "ms": 6726837 }6838 },6839 "notes": [],6840 "created": "2025-11-10"6841 },6842 {6843 "provider": "openai",6844 "id": "gpt-5.1-2025-11-13",6845 "category": "text",6846 "aliasOf": "gpt-5.1",6847 "verdict": "untested",6848 "pricing": {6849 "input": 1.25,6850 "cached": 0.125,6851 "output": 106852 },6853 "probes": {},6854 "notes": [6855 "dated snapshot of gpt-5.1; not probed separately"6856 ],6857 "created": "2025-11-10"6858 },6859 {6860 "provider": "openai",6861 "id": "gpt-5.1-chat-latest",6862 "category": "text",6863 "aliasOf": null,6864 "verdict": "unavailable",6865 "pricing": {6866 "input": 1.25,6867 "cached": 0.125,6868 "output": 106869 },6870 "probes": {6871 "text": {6872 "status": "unavailable",6873 "ms": 153,6874 "detail": "404 404 The model `gpt-5.1-chat-latest` does not exist or you do not have access to it."6875 }6876 },6877 "notes": [],6878 "created": "2025-11-07"6879 },6880 {6881 "provider": "openai",6882 "id": "gpt-5.1-codex",6883 "category": "text-specialized",6884 "aliasOf": null,6885 "verdict": "unavailable",6886 "pricing": {6887 "input": 1.25,6888 "cached": 0.125,6889 "output": 106890 },6891 "probes": {6892 "text": {6893 "status": "unavailable",6894 "ms": 151,6895 "detail": "404 404 The model `gpt-5.1-codex` does not exist or you do not have access to it."6896 }6897 },6898 "notes": [],6899 "created": "2025-11-12"6900 },6901 {6902 "provider": "openai",6903 "id": "gpt-5.1-codex-max",6904 "category": "text-specialized",6905 "aliasOf": null,6906 "verdict": "unavailable",6907 "pricing": {6908 "input": 1.25,6909 "cached": 0.125,6910 "output": 106911 },6912 "probes": {6913 "text": {6914 "status": "unavailable",6915 "ms": 135,6916 "detail": "404 404 The model `gpt-5.1-codex-max` does not exist or you do not have access to it."6917 }6918 },6919 "notes": [],6920 "created": "2025-11-20"6921 },6922 {6923 "provider": "openai",6924 "id": "gpt-5.1-codex-mini",6925 "category": "text-specialized",6926 "aliasOf": null,6927 "verdict": "unavailable",6928 "pricing": {6929 "input": 0.25,6930 "cached": 0.025,6931 "output": 26932 },6933 "probes": {6934 "text": {6935 "status": "unavailable",6936 "ms": 134,6937 "detail": "404 404 The model `gpt-5.1-codex-mini` does not exist or you do not have access to it."6938 }6939 },6940 "notes": [],6941 "created": "2025-11-13"6942 },6943 {6944 "provider": "openai",6945 "id": "gpt-5.2",6946 "category": "text",6947 "aliasOf": null,6948 "verdict": "tested",6949 "pricing": {6950 "input": 1.75,6951 "cached": 0.175,6952 "output": 146953 },6954 "probes": {6955 "text": {6956 "status": "ok",6957 "ms": 7026958 },6959 "streaming": {6960 "status": "ok",6961 "ms": 9186962 },6963 "tool_use": {6964 "status": "ok",6965 "ms": 7426966 },6967 "structured_output": {6968 "status": "ok",6969 "ms": 9426970 },6971 "vision": {6972 "status": "ok",6973 "ms": 7666974 },6975 "reasoning_control": {6976 "status": "ok",6977 "ms": 7526978 }6979 },6980 "notes": [],6981 "created": "2025-12-09"6982 },6983 {6984 "provider": "openai",6985 "id": "gpt-5.2-2025-12-11",6986 "category": "text",6987 "aliasOf": "gpt-5.2",6988 "verdict": "untested",6989 "pricing": {6990 "input": 1.75,6991 "cached": 0.175,6992 "output": 146993 },6994 "probes": {},6995 "notes": [6996 "dated snapshot of gpt-5.2; not probed separately"6997 ],6998 "created": "2025-12-09"6999 },7000 {7001 "provider": "openai",7002 "id": "gpt-5.2-chat-latest",7003 "category": "text",7004 "aliasOf": null,7005 "verdict": "unavailable",7006 "pricing": {7007 "input": 1.75,7008 "cached": 0.175,7009 "output": 147010 },7011 "probes": {7012 "text": {7013 "status": "unavailable",7014 "ms": 213,7015 "detail": "404 404 The model `gpt-5.2-chat-latest` does not exist or you do not have access to it."7016 }7017 },7018 "notes": [],7019 "created": "2025-12-10"7020 },7021 {7022 "provider": "openai",7023 "id": "gpt-5.2-codex",7024 "category": "text-specialized",7025 "aliasOf": null,7026 "verdict": "unavailable",7027 "pricing": {7028 "input": 1.75,7029 "cached": 0.175,7030 "output": 147031 },7032 "probes": {7033 "text": {7034 "status": "unavailable",7035 "ms": 166,7036 "detail": "404 404 The model `gpt-5.2-codex` does not exist or you do not have access to it."7037 }7038 },7039 "notes": [],7040 "created": "2025-12-19"7041 },7042 {7043 "provider": "openai",7044 "id": "gpt-5.2-pro",7045 "category": "text",7046 "aliasOf": null,7047 "verdict": "tested",7048 "pricing": {7049 "input": 21,7050 "cached": null,7051 "output": 1687052 },7053 "probes": {7054 "text": {7055 "status": "ok",7056 "ms": 43357057 },7058 "streaming": {7059 "status": "ok",7060 "ms": 42767061 },7062 "tool_use": {7063 "status": "ok",7064 "ms": 76437065 },7066 "structured_output": {7067 "status": "ok",7068 "ms": 53387069 },7070 "vision": {7071 "status": "ok",7072 "ms": 57257073 },7074 "reasoning_control": {7075 "status": "unsupported",7076 "ms": 141,7077 "detail": "400 400 Unsupported value: 'low' is not supported with the 'gpt-5.2-pro' model. Supported values are: 'medium', 'high', and 'xhigh'."7078 }7079 },7080 "notes": [7081 "pro/expensive tier: only text + structured_output probed; benchmark skipped for cost ($30-$150/M input)"7082 ],7083 "created": "2025-12-10"7084 },7085 {7086 "provider": "openai",7087 "id": "gpt-5.2-pro-2025-12-11",7088 "category": "text",7089 "aliasOf": "gpt-5.2-pro",7090 "verdict": "untested",7091 "pricing": {7092 "input": 21,7093 "cached": null,7094 "output": 1687095 },7096 "probes": {},7097 "notes": [7098 "dated snapshot of gpt-5.2-pro; not probed separately"7099 ],7100 "created": "2025-12-10"7101 },7102 {7103 "provider": "openai",7104 "id": "gpt-5.3-chat-latest",7105 "category": "text",7106 "aliasOf": null,7107 "verdict": "unavailable",7108 "pricing": {7109 "input": 1.75,7110 "cached": 0.175,7111 "output": 147112 },7113 "probes": {7114 "text": {7115 "status": "unavailable",7116 "ms": 199,7117 "detail": "404 404 The model `gpt-5.3-chat-latest` does not exist or you do not have access to it."7118 }7119 },7120 "notes": [],7121 "created": "2026-02-27"7122 },7123 {7124 "provider": "openai",7125 "id": "gpt-5.3-codex",7126 "category": "text-specialized",7127 "aliasOf": null,7128 "verdict": "tested",7129 "pricing": {7130 "input": 1.75,7131 "cached": 0.175,7132 "output": 147133 },7134 "probes": {7135 "text": {7136 "status": "ok",7137 "ms": 9047138 },7139 "streaming": {7140 "status": "ok",7141 "ms": 8097142 },7143 "tool_use": {7144 "status": "ok",7145 "ms": 8857146 },7147 "structured_output": {7148 "status": "ok",7149 "ms": 9427150 },7151 "vision": {7152 "status": "ok",7153 "ms": 9257154 },7155 "reasoning_control": {7156 "status": "ok",7157 "ms": 8447158 }7159 },7160 "notes": [],7161 "created": "2026-02-08"7162 },7163 {7164 "provider": "openai",7165 "id": "gpt-5.4",7166 "category": "text",7167 "aliasOf": null,7168 "verdict": "tested",7169 "pricing": {7170 "input": 2.5,7171 "cached": 0.25,7172 "output": 157173 },7174 "probes": {7175 "text": {7176 "status": "ok",7177 "ms": 7207178 },7179 "streaming": {7180 "status": "ok",7181 "ms": 9557182 },7183 "tool_use": {7184 "status": "ok",7185 "ms": 10797186 },7187 "structured_output": {7188 "status": "ok",7189 "ms": 9867190 },7191 "vision": {7192 "status": "ok",7193 "ms": 7677194 },7195 "reasoning_control": {7196 "status": "ok",7197 "ms": 8837198 }7199 },7200 "notes": [],7201 "created": "2026-03-05"7202 },7203 {7204 "provider": "openai",7205 "id": "gpt-5.4-2026-03-05",7206 "category": "text",7207 "aliasOf": "gpt-5.4",7208 "verdict": "untested",7209 "pricing": {7210 "input": 2.5,7211 "cached": 0.25,7212 "output": 157213 },7214 "probes": {},7215 "notes": [7216 "dated snapshot of gpt-5.4; not probed separately"7217 ],7218 "created": "2026-03-04"7219 },7220 {7221 "provider": "openai",7222 "id": "gpt-5.4-mini",7223 "category": "text",7224 "aliasOf": null,7225 "verdict": "tested",7226 "pricing": {7227 "input": 0.75,7228 "cached": 0.075,7229 "output": 4.57230 },7231 "probes": {7232 "text": {7233 "status": "ok",7234 "ms": 7067235 },7236 "streaming": {7237 "status": "ok",7238 "ms": 5847239 },7240 "tool_use": {7241 "status": "ok",7242 "ms": 8517243 },7244 "structured_output": {7245 "status": "ok",7246 "ms": 8247247 },7248 "vision": {7249 "status": "ok",7250 "ms": 6237251 },7252 "reasoning_control": {7253 "status": "ok",7254 "ms": 5467255 }7256 },7257 "notes": [],7258 "created": "2026-03-14"7259 },7260 {7261 "provider": "openai",7262 "id": "gpt-5.4-mini-2026-03-17",7263 "category": "text",7264 "aliasOf": "gpt-5.4-mini",7265 "verdict": "untested",7266 "pricing": {7267 "input": 0.75,7268 "cached": 0.075,7269 "output": 4.57270 },7271 "probes": {},7272 "notes": [7273 "dated snapshot of gpt-5.4-mini; not probed separately"7274 ],7275 "created": "2026-03-14"7276 },7277 {7278 "provider": "openai",7279 "id": "gpt-5.4-nano",7280 "category": "text",7281 "aliasOf": null,7282 "verdict": "tested",7283 "pricing": {7284 "input": 0.2,7285 "cached": 0.02,7286 "output": 1.257287 },7288 "probes": {7289 "text": {7290 "status": "ok",7291 "ms": 7227292 },7293 "streaming": {7294 "status": "ok",7295 "ms": 7047296 },7297 "tool_use": {7298 "status": "ok",7299 "ms": 5967300 },7301 "structured_output": {7302 "status": "ok",7303 "ms": 6527304 },7305 "vision": {7306 "status": "ok",7307 "ms": 5217308 },7309 "reasoning_control": {7310 "status": "ok",7311 "ms": 5667312 }7313 },7314 "notes": [],7315 "created": "2026-03-14"7316 },7317 {7318 "provider": "openai",7319 "id": "gpt-5.4-nano-2026-03-17",7320 "category": "text",7321 "aliasOf": "gpt-5.4-nano",7322 "verdict": "untested",7323 "pricing": {7324 "input": 0.2,7325 "cached": 0.02,7326 "output": 1.257327 },7328 "probes": {},7329 "notes": [7330 "dated snapshot of gpt-5.4-nano; not probed separately"7331 ],7332 "created": "2026-03-14"7333 },7334 {7335 "provider": "openai",7336 "id": "gpt-5.4-pro",7337 "category": "text",7338 "aliasOf": null,7339 "verdict": "tested",7340 "pricing": {7341 "input": 30,7342 "cached": null,7343 "output": 1807344 },7345 "probes": {7346 "text": {7347 "status": "ok",7348 "ms": 50547349 },7350 "streaming": {7351 "status": "ok",7352 "ms": 66207353 },7354 "tool_use": {7355 "status": "ok",7356 "ms": 130177357 },7358 "structured_output": {7359 "status": "ok",7360 "ms": 222857361 },7362 "vision": {7363 "status": "ok",7364 "ms": 79707365 },7366 "reasoning_control": {7367 "status": "unsupported",7368 "ms": 149,7369 "detail": "400 400 Unsupported value: 'low' is not supported with the 'gpt-5.4-pro' model. Supported values are: 'medium', 'high', and 'xhigh'."7370 }7371 },7372 "notes": [7373 "pro/expensive tier: only text + structured_output probed; benchmark skipped for cost ($30-$150/M input)"7374 ],7375 "created": "2026-03-04"7376 },7377 {7378 "provider": "openai",7379 "id": "gpt-5.4-pro-2026-03-05",7380 "category": "text",7381 "aliasOf": "gpt-5.4-pro",7382 "verdict": "untested",7383 "pricing": {7384 "input": 30,7385 "cached": null,7386 "output": 1807387 },7388 "probes": {},7389 "notes": [7390 "dated snapshot of gpt-5.4-pro; not probed separately"7391 ],7392 "created": "2026-03-04"7393 },7394 {7395 "provider": "openai",7396 "id": "gpt-5.5",7397 "category": "text",7398 "aliasOf": null,7399 "verdict": "tested",7400 "pricing": {7401 "input": 5,7402 "cached": 0.5,7403 "output": 307404 },7405 "probes": {7406 "text": {7407 "status": "ok",7408 "ms": 9077409 },7410 "streaming": {7411 "status": "ok",7412 "ms": 10857413 },7414 "tool_use": {7415 "status": "ok",7416 "ms": 9727417 },7418 "structured_output": {7419 "status": "ok",7420 "ms": 16717421 },7422 "vision": {7423 "status": "ok",7424 "ms": 12837425 },7426 "reasoning_control": {7427 "status": "ok",7428 "ms": 10747429 }7430 },7431 "notes": [],7432 "created": "2026-04-22"7433 },7434 {7435 "provider": "openai",7436 "id": "gpt-5.5-2026-04-23",7437 "category": "text",7438 "aliasOf": "gpt-5.5",7439 "verdict": "untested",7440 "pricing": {7441 "input": 5,7442 "cached": 0.5,7443 "output": 307444 },7445 "probes": {},7446 "notes": [7447 "dated snapshot of gpt-5.5; not probed separately"7448 ],7449 "created": "2026-04-22"7450 },7451 {7452 "provider": "openai",7453 "id": "gpt-5.5-pro",7454 "category": "text",7455 "aliasOf": null,7456 "verdict": "tested",7457 "pricing": {7458 "input": 30,7459 "cached": null,7460 "output": 1807461 },7462 "probes": {7463 "text": {7464 "status": "ok",7465 "ms": 356467466 },7467 "streaming": {7468 "status": "ok",7469 "ms": 39867470 },7471 "tool_use": {7472 "status": "ok",7473 "ms": 56437474 },7475 "structured_output": {7476 "status": "ok",7477 "ms": 107067478 },7479 "vision": {7480 "status": "ok",7481 "ms": 42327482 },7483 "reasoning_control": {7484 "status": "unsupported",7485 "ms": 264,7486 "detail": "400 400 Unsupported value: 'low' is not supported with the 'gpt-5.5-pro' model. Supported values are: 'medium', 'high', and 'xhigh'."7487 }7488 },7489 "notes": [7490 "pro/expensive tier: only text + structured_output probed; benchmark skipped for cost ($30-$150/M input)"7491 ],7492 "created": "2026-04-22"7493 },7494 {7495 "provider": "openai",7496 "id": "gpt-5.5-pro-2026-04-23",7497 "category": "text",7498 "aliasOf": "gpt-5.5-pro",7499 "verdict": "untested",7500 "pricing": {7501 "input": 30,7502 "cached": null,7503 "output": 1807504 },7505 "probes": {},7506 "notes": [7507 "dated snapshot of gpt-5.5-pro; not probed separately"7508 ],7509 "created": "2026-04-22"7510 },7511 {7512 "provider": "openai",7513 "id": "gpt-5.6-luna",7514 "category": "text",7515 "aliasOf": null,7516 "verdict": "tested",7517 "pricing": {7518 "input": 0.2,7519 "cached": 0.02,7520 "output": 1.27521 },7522 "probes": {7523 "text": {7524 "status": "ok",7525 "ms": 7097526 },7527 "streaming": {7528 "status": "ok",7529 "ms": 8467530 },7531 "tool_use": {7532 "status": "ok",7533 "ms": 7917534 },7535 "structured_output": {7536 "status": "ok",7537 "ms": 16067538 },7539 "vision": {7540 "status": "ok",7541 "ms": 7657542 },7543 "reasoning_control": {7544 "status": "ok",7545 "ms": 6267546 }7547 },7548 "notes": [],7549 "created": "2026-06-23"7550 },7551 {7552 "provider": "openai",7553 "id": "gpt-5.6-sol",7554 "category": "text",7555 "aliasOf": null,7556 "verdict": "tested",7557 "pricing": {7558 "input": 4,7559 "cached": 0.4,7560 "output": 207561 },7562 "probes": {7563 "text": {7564 "status": "ok",7565 "ms": 8247566 },7567 "streaming": {7568 "status": "ok",7569 "ms": 7617570 },7571 "tool_use": {7572 "status": "ok",7573 "ms": 7947574 },7575 "structured_output": {7576 "status": "ok",7577 "ms": 8017578 },7579 "vision": {7580 "status": "ok",7581 "ms": 14677582 },7583 "reasoning_control": {7584 "status": "ok",7585 "ms": 10117586 }7587 },7588 "notes": [],7589 "created": "2026-06-23"7590 },7591 {7592 "provider": "openai",7593 "id": "gpt-5.6-terra",7594 "category": "text",7595 "aliasOf": null,7596 "verdict": "tested",7597 "pricing": {7598 "input": 2,7599 "cached": 0.2,7600 "output": 127601 },7602 "probes": {7603 "text": {7604 "status": "ok",7605 "ms": 6457606 },7607 "streaming": {7608 "status": "ok",7609 "ms": 8097610 },7611 "tool_use": {7612 "status": "ok",7613 "ms": 8767614 },7615 "structured_output": {7616 "status": "ok",7617 "ms": 9177618 },7619 "vision": {7620 "status": "ok",7621 "ms": 6987622 },7623 "reasoning_control": {7624 "status": "ok",7625 "ms": 7537626 }7627 },7628 "notes": [],7629 "created": "2026-06-23"7630 },7631 {7632 "provider": "openai",7633 "id": "gpt-6-astra",7634 "category": "text",7635 "aliasOf": null,7636 "verdict": "tested",7637 "pricing": {7638 "input": 10,7639 "cached": 1,7640 "output": 507641 },7642 "probes": {7643 "text": {7644 "status": "ok",7645 "ms": 20087646 },7647 "streaming": {7648 "status": "ok",7649 "ms": 11867650 },7651 "tool_use": {7652 "status": "ok",7653 "ms": 15137654 },7655 "structured_output": {7656 "status": "ok",7657 "ms": 22397658 },7659 "vision": {7660 "status": "ok",7661 "ms": 10717662 },7663 "reasoning_control": {7664 "status": "ok",7665 "ms": 9807666 }7667 },7668 "notes": [],7669 "created": "2026-08-27"7670 },7671 {7672 "provider": "openai",7673 "id": "gpt-6-luna",7674 "category": "text",7675 "aliasOf": null,7676 "verdict": "tested",7677 "pricing": {7678 "input": 0.1,7679 "cached": 0.01,7680 "output": 0.57681 },7682 "probes": {7683 "text": {7684 "status": "ok",7685 "ms": 8627686 },7687 "streaming": {7688 "status": "ok",7689 "ms": 8817690 },7691 "tool_use": {7692 "status": "ok",7693 "ms": 7597694 },7695 "structured_output": {7696 "status": "ok",7697 "ms": 14537698 },7699 "vision": {7700 "status": "ok",7701 "ms": 12817702 },7703 "reasoning_control": {7704 "status": "ok",7705 "ms": 5967706 }7707 },7708 "notes": [],7709 "created": "2026-09-14"7710 },7711 {7712 "provider": "openai",7713 "id": "gpt-6-sol",7714 "category": "text",7715 "aliasOf": null,7716 "verdict": "tested",7717 "pricing": {7718 "input": 2,7719 "cached": 0.2,7720 "output": 107721 },7722 "probes": {7723 "text": {7724 "status": "ok",7725 "ms": 7567726 },7727 "streaming": {7728 "status": "ok",7729 "ms": 8257730 },7731 "tool_use": {7732 "status": "ok",7733 "ms": 7897734 },7735 "structured_output": {7736 "status": "ok",7737 "ms": 14227738 },7739 "vision": {7740 "status": "ok",7741 "ms": 12287742 },7743 "reasoning_control": {7744 "status": "ok",7745 "ms": 7037746 }7747 },7748 "notes": [],7749 "created": "2026-09-14"7750 },7751 {7752 "provider": "openai",7753 "id": "gpt-6.1-sol",7754 "category": "text",7755 "aliasOf": null,7756 "verdict": "tested",7757 "pricing": {7758 "input": 2,7759 "cached": 0.1,7760 "output": 107761 },7762 "probes": {7763 "text": {7764 "status": "ok",7765 "ms": 10347766 },7767 "streaming": {7768 "status": "ok",7769 "ms": 10337770 },7771 "tool_use": {7772 "status": "ok",7773 "ms": 11407774 },7775 "structured_output": {7776 "status": "ok",7777 "ms": 24527778 },7779 "vision": {7780 "status": "ok",7781 "ms": 9237782 },7783 "reasoning_control": {7784 "status": "ok",7785 "ms": 10567786 }7787 },7788 "notes": [],7789 "created": "2026-09-27"7790 },7791 {7792 "provider": "openai",7793 "id": "gpt-6.1-terra",7794 "category": "text",7795 "aliasOf": null,7796 "verdict": "tested",7797 "pricing": {7798 "input": 1,7799 "cached": 0.05,7800 "output": 57801 },7802 "probes": {7803 "text": {7804 "status": "ok",7805 "ms": 8007806 },7807 "streaming": {7808 "status": "ok",7809 "ms": 9627810 },7811 "tool_use": {7812 "status": "ok",7813 "ms": 10547814 },7815 "structured_output": {7816 "status": "ok",7817 "ms": 13737818 },7819 "vision": {7820 "status": "ok",7821 "ms": 13797822 },7823 "reasoning_control": {7824 "status": "ok",7825 "ms": 9777826 }7827 },7828 "notes": [],7829 "created": "2026-10-13"7830 },7831 {7832 "provider": "openai",7833 "id": "gpt-audio",7834 "category": "audio",7835 "aliasOf": null,7836 "verdict": "untested",7837 "pricing": null,7838 "probes": {},7839 "notes": [7840 "category audio: not a general text model, not probed (inappropriate request type)"7841 ],7842 "created": "2025-08-28"7843 },7844 {7845 "provider": "openai",7846 "id": "gpt-audio-1.5",7847 "category": "audio",7848 "aliasOf": null,7849 "verdict": "untested",7850 "pricing": null,7851 "probes": {},7852 "notes": [7853 "category audio: not a general text model, not probed (inappropriate request type)"7854 ],7855 "created": "2026-02-20"7856 },7857 {7858 "provider": "openai",7859 "id": "gpt-audio-2025-08-28",7860 "category": "audio",7861 "aliasOf": "gpt-audio",7862 "verdict": "untested",7863 "pricing": null,7864 "probes": {},7865 "notes": [7866 "dated snapshot of gpt-audio; not probed separately"7867 ],7868 "created": "2025-08-27"7869 },7870 {7871 "provider": "openai",7872 "id": "gpt-audio-mini",7873 "category": "audio",7874 "aliasOf": null,7875 "verdict": "untested",7876 "pricing": null,7877 "probes": {},7878 "notes": [7879 "category audio: not a general text model, not probed (inappropriate request type)"7880 ],7881 "created": "2025-10-03"7882 },7883 {7884 "provider": "openai",7885 "id": "gpt-audio-mini-2025-10-06",7886 "category": "audio",7887 "aliasOf": "gpt-audio-mini",7888 "verdict": "untested",7889 "pricing": null,7890 "probes": {},7891 "notes": [7892 "dated snapshot of gpt-audio-mini; not probed separately"7893 ],7894 "created": "2025-10-03"7895 },7896 {7897 "provider": "openai",7898 "id": "gpt-audio-mini-2025-12-15",7899 "category": "audio",7900 "aliasOf": "gpt-audio-mini",7901 "verdict": "untested",7902 "pricing": null,7903 "probes": {},7904 "notes": [7905 "dated snapshot of gpt-audio-mini; not probed separately"7906 ],7907 "created": "2025-12-15"7908 },7909 {7910 "provider": "openai",7911 "id": "gpt-image-1",7912 "category": "image",7913 "aliasOf": null,7914 "verdict": "untested",7915 "pricing": null,7916 "probes": {},7917 "notes": [7918 "category image: not a general text model, not probed (inappropriate request type)"7919 ],7920 "created": "2025-04-24"7921 },7922 {7923 "provider": "openai",7924 "id": "gpt-image-1-mini",7925 "category": "image",7926 "aliasOf": null,7927 "verdict": "untested",7928 "pricing": null,7929 "probes": {},7930 "notes": [7931 "category image: not a general text model, not probed (inappropriate request type)"7932 ],7933 "created": "2025-09-26"7934 },7935 {7936 "provider": "openai",7937 "id": "gpt-image-1.5",7938 "category": "image",7939 "aliasOf": null,7940 "verdict": "untested",7941 "pricing": null,7942 "probes": {},7943 "notes": [7944 "category image: not a general text model, not probed (inappropriate request type)"7945 ],7946 "created": "2025-11-25"7947 },7948 {7949 "provider": "openai",7950 "id": "gpt-image-2",7951 "category": "image",7952 "aliasOf": null,7953 "verdict": "untested",7954 "pricing": null,7955 "probes": {},7956 "notes": [7957 "category image: not a general text model, not probed (inappropriate request type)"7958 ],7959 "created": "2026-04-17"7960 },7961 {7962 "provider": "openai",7963 "id": "gpt-image-2-2026-04-21",7964 "category": "image",7965 "aliasOf": "gpt-image-2",7966 "verdict": "untested",7967 "pricing": null,7968 "probes": {},7969 "notes": [7970 "dated snapshot of gpt-image-2; not probed separately"7971 ],7972 "created": "2026-04-17"7973 },7974 {7975 "provider": "openai",7976 "id": "gpt-image-2.5-flare",7977 "category": "image",7978 "aliasOf": null,7979 "verdict": "untested",7980 "pricing": null,7981 "probes": {},7982 "notes": [7983 "category image: not a general text model, not probed (inappropriate request type)"7984 ],7985 "created": "2026-09-04"7986 },7987 {7988 "provider": "openai",7989 "id": "gpt-image-2.5-flare-2026-09-08",7990 "category": "image",7991 "aliasOf": "gpt-image-2.5-flare",7992 "verdict": "untested",7993 "pricing": null,7994 "probes": {},7995 "notes": [7996 "dated snapshot of gpt-image-2.5-flare; not probed separately"7997 ],7998 "created": "2026-09-08"7999 },8000 {8001 "provider": "openai",8002 "id": "gpt-image-2.5-sunburst",8003 "category": "image",8004 "aliasOf": null,8005 "verdict": "untested",8006 "pricing": null,8007 "probes": {},8008 "notes": [8009 "category image: not a general text model, not probed (inappropriate request type)"8010 ],8011 "created": "2026-09-04"8012 },8013 {8014 "provider": "openai",8015 "id": "gpt-image-2.5-sunburst-2026-09-08",8016 "category": "image",8017 "aliasOf": "gpt-image-2.5-sunburst",8018 "verdict": "untested",8019 "pricing": null,8020 "probes": {},8021 "notes": [8022 "dated snapshot of gpt-image-2.5-sunburst; not probed separately"8023 ],8024 "created": "2026-09-08"8025 },8026 {8027 "provider": "openai",8028 "id": "gpt-live-1",8029 "category": "realtime",8030 "aliasOf": null,8031 "verdict": "untested",8032 "pricing": null,8033 "probes": {},8034 "notes": [8035 "category realtime: not a general text model, not probed (inappropriate request type)"8036 ],8037 "created": "2026-09-08"8038 },8039 {8040 "provider": "openai",8041 "id": "gpt-live-transcribe",8042 "category": "realtime",8043 "aliasOf": null,8044 "verdict": "untested",8045 "pricing": null,8046 "probes": {},8047 "notes": [8048 "category realtime: not a general text model, not probed (inappropriate request type)"8049 ],8050 "created": "2026-07-27"8051 },8052 {8053 "provider": "openai",8054 "id": "gpt-realtime",8055 "category": "realtime",8056 "aliasOf": null,8057 "verdict": "untested",8058 "pricing": null,8059 "probes": {},8060 "notes": [8061 "category realtime: not a general text model, not probed (inappropriate request type)"8062 ],8063 "created": "2025-08-27"8064 },8065 {8066 "provider": "openai",8067 "id": "gpt-realtime-1.5",8068 "category": "realtime",8069 "aliasOf": null,8070 "verdict": "untested",8071 "pricing": null,8072 "probes": {},8073 "notes": [8074 "category realtime: not a general text model, not probed (inappropriate request type)"8075 ],8076 "created": "2026-02-19"8077 },8078 {8079 "provider": "openai",8080 "id": "gpt-realtime-2",8081 "category": "realtime",8082 "aliasOf": null,8083 "verdict": "untested",8084 "pricing": null,8085 "probes": {},8086 "notes": [8087 "category realtime: not a general text model, not probed (inappropriate request type)"8088 ],8089 "created": "2026-05-05"8090 },8091 {8092 "provider": "openai",8093 "id": "gpt-realtime-2.1",8094 "category": "realtime",8095 "aliasOf": null,8096 "verdict": "untested",8097 "pricing": null,8098 "probes": {},8099 "notes": [8100 "category realtime: not a general text model, not probed (inappropriate request type)"8101 ],8102 "created": "2026-06-23"8103 },8104 {8105 "provider": "openai",8106 "id": "gpt-realtime-2.1-mini",8107 "category": "realtime",8108 "aliasOf": null,8109 "verdict": "untested",8110 "pricing": null,8111 "probes": {},8112 "notes": [8113 "category realtime: not a general text model, not probed (inappropriate request type)"8114 ],8115 "created": "2026-06-23"8116 },8117 {8118 "provider": "openai",8119 "id": "gpt-realtime-2025-08-28",8120 "category": "realtime",8121 "aliasOf": "gpt-realtime",8122 "verdict": "untested",8123 "pricing": null,8124 "probes": {},8125 "notes": [8126 "dated snapshot of gpt-realtime; not probed separately"8127 ],8128 "created": "2025-08-27"8129 },8130 {8131 "provider": "openai",8132 "id": "gpt-realtime-mini",8133 "category": "realtime",8134 "aliasOf": null,8135 "verdict": "untested",8136 "pricing": null,8137 "probes": {},8138 "notes": [8139 "category realtime: not a general text model, not probed (inappropriate request type)"8140 ],8141 "created": "2025-10-03"8142 },8143 {8144 "provider": "openai",8145 "id": "gpt-realtime-mini-2025-12-15",8146 "category": "realtime",8147 "aliasOf": "gpt-realtime-mini",8148 "verdict": "untested",8149 "pricing": null,8150 "probes": {},8151 "notes": [8152 "dated snapshot of gpt-realtime-mini; not probed separately"8153 ],8154 "created": "2025-12-13"8155 },8156 {8157 "provider": "openai",8158 "id": "gpt-realtime-translate",8159 "category": "realtime",8160 "aliasOf": null,8161 "verdict": "untested",8162 "pricing": null,8163 "probes": {},8164 "notes": [8165 "category realtime: not a general text model, not probed (inappropriate request type)"8166 ],8167 "created": "2026-05-05"8168 },8169 {8170 "provider": "openai",8171 "id": "gpt-realtime-whisper",8172 "category": "realtime",8173 "aliasOf": null,8174 "verdict": "untested",8175 "pricing": null,8176 "probes": {},8177 "notes": [8178 "category realtime: not a general text model, not probed (inappropriate request type)"8179 ],8180 "created": "2026-05-05"8181 },8182 {8183 "provider": "openai",8184 "id": "gpt-transcribe",8185 "category": "audio",8186 "aliasOf": null,8187 "verdict": "untested",8188 "pricing": null,8189 "probes": {},8190 "notes": [8191 "category audio: not a general text model, not probed (inappropriate request type)"8192 ],8193 "created": "2026-07-27"8194 },8195 {8196 "provider": "openai",8197 "id": "o1",8198 "category": "text",8199 "aliasOf": null,8200 "verdict": "unavailable",8201 "pricing": {8202 "input": 15,8203 "cached": 7.5,8204 "output": 608205 },8206 "probes": {8207 "text": {8208 "status": "unavailable",8209 "ms": 429,8210 "detail": "404 404 The model `o1` does not exist or you do not have access to it."8211 }8212 },8213 "notes": [],8214 "created": "2024-12-16"8215 },8216 {8217 "provider": "openai",8218 "id": "o1-2024-12-17",8219 "category": "text",8220 "aliasOf": "o1",8221 "verdict": "untested",8222 "pricing": {8223 "input": 15,8224 "cached": 7.5,8225 "output": 608226 },8227 "probes": {},8228 "notes": [8229 "dated snapshot of o1; not probed separately"8230 ],8231 "created": "2024-12-16"8232 },8233 {8234 "provider": "openai",8235 "id": "o1-pro",8236 "category": "text",8237 "aliasOf": null,8238 "verdict": "unavailable",8239 "pricing": {8240 "input": 150,8241 "cached": null,8242 "output": 6008243 },8244 "probes": {8245 "text": {8246 "status": "unavailable",8247 "ms": 207,8248 "detail": "404 404 The model `o1-pro` does not exist or you do not have access to it."8249 }8250 },8251 "notes": [8252 "pro/expensive tier: only text + structured_output probed; benchmark skipped for cost ($30-$150/M input)"8253 ],8254 "created": "2025-03-17"8255 },8256 {8257 "provider": "openai",8258 "id": "o1-pro-2025-03-19",8259 "category": "text",8260 "aliasOf": "o1-pro",8261 "verdict": "untested",8262 "pricing": {8263 "input": 150,8264 "cached": null,8265 "output": 6008266 },8267 "probes": {},8268 "notes": [8269 "dated snapshot of o1-pro; not probed separately"8270 ],8271 "created": "2025-03-17"8272 },8273 {8274 "provider": "openai",8275 "id": "o3",8276 "category": "text",8277 "aliasOf": null,8278 "verdict": "tested",8279 "pricing": {8280 "input": 2,8281 "cached": 0.5,8282 "output": 88283 },8284 "probes": {8285 "text": {8286 "status": "ok",8287 "ms": 8238288 },8289 "streaming": {8290 "status": "ok",8291 "ms": 12998292 },8293 "tool_use": {8294 "status": "ok",8295 "ms": 13778296 },8297 "structured_output": {8298 "status": "ok",8299 "ms": 12998300 },8301 "vision": {8302 "status": "ok",8303 "ms": 10988304 },8305 "reasoning_control": {8306 "status": "ok",8307 "ms": 6798308 }8309 },8310 "notes": [],8311 "created": "2025-04-09"8312 },8313 {8314 "provider": "openai",8315 "id": "o3-2025-04-16",8316 "category": "text",8317 "aliasOf": "o3",8318 "verdict": "untested",8319 "pricing": {8320 "input": 2,8321 "cached": 0.5,8322 "output": 88323 },8324 "probes": {},8325 "notes": [8326 "dated snapshot of o3; not probed separately"8327 ],8328 "created": "2025-04-08"8329 },8330 {8331 "provider": "openai",8332 "id": "o3-deep-research",8333 "category": "deep-research",8334 "aliasOf": null,8335 "verdict": "untested",8336 "pricing": null,8337 "probes": {},8338 "notes": [8339 "category deep-research: not a general text model, not probed (inappropriate request type)"8340 ],8341 "created": "2025-06-13"8342 },8343 {8344 "provider": "openai",8345 "id": "o3-deep-research-2025-06-26",8346 "category": "deep-research",8347 "aliasOf": "o3-deep-research",8348 "verdict": "untested",8349 "pricing": null,8350 "probes": {},8351 "notes": [8352 "dated snapshot of o3-deep-research; not probed separately"8353 ],8354 "created": "2025-06-25"8355 },8356 {8357 "provider": "openai",8358 "id": "o3-mini",8359 "category": "text",8360 "aliasOf": null,8361 "verdict": "unavailable",8362 "pricing": {8363 "input": 1.1,8364 "cached": 0.55,8365 "output": 4.48366 },8367 "probes": {8368 "text": {8369 "status": "unavailable",8370 "ms": 186,8371 "detail": "404 404 The model `o3-mini` does not exist or you do not have access to it."8372 }8373 },8374 "notes": [],8375 "created": "2025-01-17"8376 },8377 {8378 "provider": "openai",8379 "id": "o3-mini-2025-01-31",8380 "category": "text",8381 "aliasOf": "o3-mini",8382 "verdict": "untested",8383 "pricing": {8384 "input": 1.1,8385 "cached": 0.55,8386 "output": 4.48387 },8388 "probes": {},8389 "notes": [8390 "dated snapshot of o3-mini; not probed separately"8391 ],8392 "created": "2025-01-27"8393 },8394 {8395 "provider": "openai",8396 "id": "o3-pro",8397 "category": "text",8398 "aliasOf": null,8399 "verdict": "tested",8400 "pricing": {8401 "input": 20,8402 "cached": null,8403 "output": 808404 },8405 "probes": {8406 "text": {8407 "status": "ok",8408 "ms": 44148409 },8410 "streaming": {8411 "status": "ok",8412 "ms": 45398413 },8414 "tool_use": {8415 "status": "ok",8416 "ms": 66478417 },8418 "structured_output": {8419 "status": "ok",8420 "ms": 59278421 },8422 "vision": {8423 "status": "ok",8424 "ms": 61218425 },8426 "reasoning_control": {8427 "status": "ok",8428 "ms": 48578429 }8430 },8431 "notes": [8432 "pro/expensive tier: only text + structured_output probed; benchmark skipped for cost ($30-$150/M input)"8433 ],8434 "created": "2025-05-28"8435 },8436 {8437 "provider": "openai",8438 "id": "o3-pro-2025-06-10",8439 "category": "text",8440 "aliasOf": "o3-pro",8441 "verdict": "untested",8442 "pricing": {8443 "input": 20,8444 "cached": null,8445 "output": 808446 },8447 "probes": {},8448 "notes": [8449 "dated snapshot of o3-pro; not probed separately"8450 ],8451 "created": "2025-06-05"8452 },8453 {8454 "provider": "openai",8455 "id": "o4-mini",8456 "category": "text",8457 "aliasOf": null,8458 "verdict": "unavailable",8459 "pricing": {8460 "input": 1.1,8461 "cached": 0.275,8462 "output": 4.48463 },8464 "probes": {8465 "text": {8466 "status": "unavailable",8467 "ms": 147,8468 "detail": "404 404 The model `o4-mini` does not exist or you do not have access to it."8469 }8470 },8471 "notes": [],8472 "created": "2025-04-09"8473 },8474 {8475 "provider": "openai",8476 "id": "o4-mini-2025-04-16",8477 "category": "text",8478 "aliasOf": "o4-mini",8479 "verdict": "untested",8480 "pricing": {8481 "input": 1.1,8482 "cached": 0.275,8483 "output": 4.48484 },8485 "probes": {},8486 "notes": [8487 "dated snapshot of o4-mini; not probed separately"8488 ],8489 "created": "2025-04-08"8490 },8491 {8492 "provider": "openai",8493 "id": "o4-mini-deep-research",8494 "category": "deep-research",8495 "aliasOf": null,8496 "verdict": "untested",8497 "pricing": null,8498 "probes": {},8499 "notes": [8500 "category deep-research: not a general text model, not probed (inappropriate request type)"8501 ],8502 "created": "2025-06-11"8503 },8504 {8505 "provider": "openai",8506 "id": "o4-mini-deep-research-2025-06-26",8507 "category": "deep-research",8508 "aliasOf": "o4-mini-deep-research",8509 "verdict": "untested",8510 "pricing": null,8511 "probes": {},8512 "notes": [8513 "dated snapshot of o4-mini-deep-research; not probed separately"8514 ],8515 "created": "2025-06-25"8516 },8517 {8518 "provider": "openai",8519 "id": "omni-moderation-2024-09-26",8520 "category": "moderation",8521 "aliasOf": null,8522 "verdict": "untested",8523 "pricing": null,8524 "probes": {},8525 "notes": [8526 "category moderation: not a general text model, not probed (inappropriate request type)"8527 ],8528 "created": "2024-11-27"8529 },8530 {8531 "provider": "openai",8532 "id": "omni-moderation-latest",8533 "category": "moderation",8534 "aliasOf": null,8535 "verdict": "untested",8536 "pricing": null,8537 "probes": {},8538 "notes": [8539 "category moderation: not a general text model, not probed (inappropriate request type)"8540 ],8541 "created": "2024-11-15"8542 },8543 {8544 "provider": "openai",8545 "id": "sora-2",8546 "category": "video",8547 "aliasOf": null,8548 "verdict": "untested",8549 "pricing": null,8550 "probes": {},8551 "notes": [8552 "category video: not a general text model, not probed (inappropriate request type)"8553 ],8554 "created": "2025-10-05"8555 },8556 {8557 "provider": "openai",8558 "id": "sora-2-pro",8559 "category": "video",8560 "aliasOf": null,8561 "verdict": "untested",8562 "pricing": null,8563 "probes": {},8564 "notes": [8565 "category video: not a general text model, not probed (inappropriate request type)"8566 ],8567 "created": "2025-10-05"8568 },8569 {8570 "provider": "openai",8571 "id": "text-embedding-3-large",8572 "category": "embedding",8573 "aliasOf": null,8574 "verdict": "untested",8575 "pricing": {8576 "input": 0.13,8577 "cached": null,8578 "output": 08579 },8580 "probes": {},8581 "notes": [8582 "category embedding: not a general text model, not probed (inappropriate request type)"8583 ],8584 "created": "2024-01-22"8585 },8586 {8587 "provider": "openai",8588 "id": "text-embedding-3-small",8589 "category": "embedding",8590 "aliasOf": null,8591 "verdict": "untested",8592 "pricing": {8593 "input": 0.02,8594 "cached": null,8595 "output": 08596 },8597 "probes": {},8598 "notes": [8599 "category embedding: not a general text model, not probed (inappropriate request type)"8600 ],8601 "created": "2024-01-22"8602 },8603 {8604 "provider": "openai",8605 "id": "text-embedding-ada-002",8606 "category": "embedding",8607 "aliasOf": null,8608 "verdict": "untested",8609 "pricing": null,8610 "probes": {},8611 "notes": [8612 "category embedding: not a general text model, not probed (inappropriate request type)"8613 ],8614 "created": "2022-12-16"8615 },8616 {8617 "provider": "openai",8618 "id": "tts-1",8619 "category": "audio",8620 "aliasOf": null,8621 "verdict": "untested",8622 "pricing": null,8623 "probes": {},8624 "notes": [8625 "category audio: not a general text model, not probed (inappropriate request type)"8626 ],8627 "created": "2023-04-19"8628 },8629 {8630 "provider": "openai",8631 "id": "tts-1-1106",8632 "category": "audio",8633 "aliasOf": null,8634 "verdict": "untested",8635 "pricing": null,8636 "probes": {},8637 "notes": [8638 "category audio: not a general text model, not probed (inappropriate request type)"8639 ],8640 "created": "2023-11-03"8641 },8642 {8643 "provider": "openai",8644 "id": "tts-1-hd",8645 "category": "audio",8646 "aliasOf": null,8647 "verdict": "untested",8648 "pricing": null,8649 "probes": {},8650 "notes": [8651 "category audio: not a general text model, not probed (inappropriate request type)"8652 ],8653 "created": "2023-11-03"8654 },8655 {8656 "provider": "openai",8657 "id": "tts-1-hd-1106",8658 "category": "audio",8659 "aliasOf": null,8660 "verdict": "untested",8661 "pricing": null,8662 "probes": {},8663 "notes": [8664 "category audio: not a general text model, not probed (inappropriate request type)"8665 ],8666 "created": "2023-11-03"8667 },8668 {8669 "provider": "openai",8670 "id": "whisper-1",8671 "category": "audio",8672 "aliasOf": null,8673 "verdict": "untested",8674 "pricing": null,8675 "probes": {},8676 "notes": [8677 "category audio: not a general text model, not probed (inappropriate request type)"8678 ],8679 "created": "2023-02-27"8680 },8681 {8682 "provider": "anthropic",8683 "id": "claude-haiku-5-5",8684 "display": "Claude Haiku 5.5",8685 "category": "text",8686 "aliasOf": null,8687 "verdict": "tested",8688 "context": 1000000,8689 "maxOutput": 128000,8690 "pricing": {8691 "input": 0.1,8692 "cached": 0.01,8693 "output": 0.58694 },8695 "probes": {8696 "text": {8697 "status": "ok",8698 "ms": 5928699 },8700 "streaming": {8701 "status": "ok",8702 "ms": 6048703 },8704 "tool_use": {8705 "status": "ok",8706 "ms": 6208707 },8708 "structured_output": {8709 "status": "ok",8710 "ms": 10898711 },8712 "vision": {8713 "status": "ok",8714 "ms": 9328715 },8716 "reasoning_control": {8717 "status": "ok",8718 "ms": 5678719 }8720 },8721 "notes": [],8722 "created": "2026-10-07"8723 },8724 {8725 "provider": "anthropic",8726 "id": "claude-sonnet-5-5",8727 "display": "Claude Sonnet 5.5",8728 "category": "text",8729 "aliasOf": null,8730 "verdict": "tested",8731 "context": 1000000,8732 "maxOutput": 128000,8733 "pricing": {8734 "input": 2,8735 "cached": 0.1,8736 "output": 108737 },8738 "probes": {8739 "text": {8740 "status": "ok",8741 "ms": 12658742 },8743 "streaming": {8744 "status": "ok",8745 "ms": 10378746 },8747 "tool_use": {8748 "status": "ok",8749 "ms": 13858750 },8751 "structured_output": {8752 "status": "ok",8753 "ms": 7728754 },8755 "vision": {8756 "status": "ok",8757 "ms": 10198758 },8759 "reasoning_control": {8760 "status": "ok",8761 "ms": 11678762 }8763 },8764 "notes": [],8765 "created": "2026-09-28"8766 },8767 {8768 "provider": "anthropic",8769 "id": "claude-opus-5-5",8770 "display": "Claude Opus 5.5",8771 "category": "text",8772 "aliasOf": null,8773 "verdict": "tested",8774 "context": 1000000,8775 "maxOutput": 128000,8776 "pricing": {8777 "input": 4,8778 "cached": 0.2,8779 "output": 208780 },8781 "probes": {8782 "text": {8783 "status": "ok",8784 "ms": 8598785 },8786 "streaming": {8787 "status": "ok",8788 "ms": 9058789 },8790 "tool_use": {8791 "status": "ok",8792 "ms": 10208793 },8794 "structured_output": {8795 "status": "ok",8796 "ms": 16298797 },8798 "vision": {8799 "status": "ok",8800 "ms": 9528801 },8802 "reasoning_control": {8803 "status": "ok",8804 "ms": 31518805 }8806 },8807 "notes": [],8808 "created": "2026-09-21"8809 },8810 {8811 "provider": "anthropic",8812 "id": "claude-fable-5-1",8813 "display": "Claude Fable 5.1",8814 "category": "text",8815 "aliasOf": null,8816 "verdict": "tested",8817 "context": 1000000,8818 "maxOutput": 128000,8819 "pricing": {8820 "input": 10,8821 "cached": 0.25,8822 "output": 508823 },8824 "probes": {8825 "text": {8826 "status": "ok",8827 "ms": 7348828 },8829 "streaming": {8830 "status": "ok",8831 "ms": 7408832 },8833 "tool_use": {8834 "status": "ok",8835 "ms": 10948836 },8837 "structured_output": {8838 "status": "ok",8839 "ms": 11628840 },8841 "vision": {8842 "status": "ok",8843 "ms": 7518844 },8845 "reasoning_control": {8846 "status": "ok",8847 "ms": 7188848 }8849 },8850 "notes": [],8851 "created": "2026-08-28"8852 },8853 {8854 "provider": "anthropic",8855 "id": "claude-opus-5",8856 "display": "Claude Opus 5",8857 "category": "text",8858 "aliasOf": null,8859 "verdict": "tested",8860 "context": 1000000,8861 "maxOutput": 128000,8862 "pricing": {8863 "input": 5,8864 "cached": 0.5,8865 "output": 258866 },8867 "probes": {8868 "text": {8869 "status": "ok",8870 "ms": 15098871 },8872 "streaming": {8873 "status": "ok",8874 "ms": 7718875 },8876 "tool_use": {8877 "status": "ok",8878 "ms": 17478879 },8880 "structured_output": {8881 "status": "ok",8882 "ms": 13418883 },8884 "vision": {8885 "status": "ok",8886 "ms": 9408887 },8888 "reasoning_control": {8889 "status": "ok",8890 "ms": 14278891 }8892 },8893 "notes": [],8894 "created": "2026-07-24"8895 },8896 {8897 "provider": "anthropic",8898 "id": "claude-sonnet-5",8899 "display": "Claude Sonnet 5",8900 "category": "text",8901 "aliasOf": null,8902 "verdict": "tested",8903 "context": 1000000,8904 "maxOutput": 128000,8905 "pricing": {8906 "input": 2,8907 "cached": 0.2,8908 "output": 108909 },8910 "probes": {8911 "text": {8912 "status": "ok",8913 "ms": 9378914 },8915 "streaming": {8916 "status": "ok",8917 "ms": 10248918 },8919 "tool_use": {8920 "status": "ok",8921 "ms": 10038922 },8923 "structured_output": {8924 "status": "ok",8925 "ms": 10558926 },8927 "vision": {8928 "status": "ok",8929 "ms": 8918930 },8931 "reasoning_control": {8932 "status": "ok",8933 "ms": 8758934 }8935 },8936 "notes": [],8937 "created": "2026-06-29"8938 },8939 {8940 "provider": "anthropic",8941 "id": "claude-fable-5",8942 "display": "Claude Fable 5",8943 "category": "text",8944 "aliasOf": null,8945 "verdict": "tested",8946 "context": 1000000,8947 "maxOutput": 128000,8948 "pricing": {8949 "input": 10,8950 "cached": 1,8951 "output": 508952 },8953 "probes": {8954 "text": {8955 "status": "ok",8956 "ms": 18828957 },8958 "streaming": {8959 "status": "ok",8960 "ms": 20638961 },8962 "tool_use": {8963 "status": "ok",8964 "ms": 23948965 },8966 "structured_output": {8967 "status": "ok",8968 "ms": 20968969 },8970 "vision": {8971 "status": "ok",8972 "ms": 21198973 },8974 "reasoning_control": {8975 "status": "ok",8976 "ms": 20158977 }8978 },8979 "notes": [],8980 "created": "2026-06-07"8981 },8982 {8983 "provider": "anthropic",8984 "id": "claude-opus-4-8",8985 "display": "Claude Opus 4.8",8986 "category": "text",8987 "aliasOf": null,8988 "verdict": "tested",8989 "context": 1000000,8990 "maxOutput": 128000,8991 "pricing": {8992 "input": 5,8993 "cached": 0.5,8994 "output": 258995 },8996 "probes": {8997 "text": {8998 "status": "ok",8999 "ms": 5609000 },9001 "streaming": {9002 "status": "ok",9003 "ms": 6149004 },9005 "tool_use": {9006 "status": "ok",9007 "ms": 13549008 },9009 "structured_output": {9010 "status": "ok",9011 "ms": 16209012 },9013 "vision": {9014 "status": "ok",9015 "ms": 8369016 },9017 "reasoning_control": {9018 "status": "ok",9019 "ms": 6539020 }9021 },9022 "notes": [],9023 "created": "2026-05-28"9024 },9025 {9026 "provider": "anthropic",9027 "id": "claude-opus-4-7",9028 "display": "Claude Opus 4.7",9029 "category": "text",9030 "aliasOf": null,9031 "verdict": "tested",9032 "context": 1000000,9033 "maxOutput": 128000,9034 "pricing": {9035 "input": 5,9036 "cached": 0.5,9037 "output": 259038 },9039 "probes": {9040 "text": {9041 "status": "ok",9042 "ms": 4149043 },9044 "streaming": {9045 "status": "ok",9046 "ms": 6709047 },9048 "tool_use": {9049 "status": "ok",9050 "ms": 11729051 },9052 "structured_output": {9053 "status": "ok",9054 "ms": 12479055 },9056 "vision": {9057 "status": "ok",9058 "ms": 6649059 },9060 "reasoning_control": {9061 "status": "ok",9062 "ms": 7489063 }9064 },9065 "notes": [],9066 "created": "2026-04-14"9067 },9068 {9069 "provider": "anthropic",9070 "id": "claude-sonnet-4-6",9071 "display": "Claude Sonnet 4.6",9072 "category": "text",9073 "aliasOf": null,9074 "verdict": "tested",9075 "context": 1000000,9076 "maxOutput": 128000,9077 "pricing": {9078 "input": 3,9079 "cached": 0.3,9080 "output": 159081 },9082 "probes": {9083 "text": {9084 "status": "ok",9085 "ms": 7219086 },9087 "streaming": {9088 "status": "ok",9089 "ms": 10639090 },9091 "tool_use": {9092 "status": "ok",9093 "ms": 10649094 },9095 "structured_output": {9096 "status": "ok",9097 "ms": 9199098 },9099 "vision": {9100 "status": "ok",9101 "ms": 13939102 },9103 "reasoning_control": {9104 "status": "ok",9105 "ms": 6669106 }9107 },9108 "notes": [],9109 "created": "2026-02-17"9110 },9111 {9112 "provider": "anthropic",9113 "id": "claude-opus-4-6",9114 "display": "Claude Opus 4.6",9115 "category": "text",9116 "aliasOf": null,9117 "verdict": "tested",9118 "context": 1000000,9119 "maxOutput": 128000,9120 "pricing": {9121 "input": 5,9122 "cached": 0.5,9123 "output": 259124 },9125 "probes": {9126 "text": {9127 "status": "ok",9128 "ms": 15309129 },9130 "streaming": {9131 "status": "ok",9132 "ms": 15739133 },9134 "tool_use": {9135 "status": "ok",9136 "ms": 17729137 },9138 "structured_output": {9139 "status": "ok",9140 "ms": 18649141 },9142 "vision": {9143 "status": "ok",9144 "ms": 12809145 },9146 "reasoning_control": {9147 "status": "ok",9148 "ms": 15159149 }9150 },9151 "notes": [],9152 "created": "2026-02-04"9153 },9154 {9155 "provider": "anthropic",9156 "id": "claude-opus-4-5-20251101",9157 "display": "Claude Opus 4.5",9158 "category": "text",9159 "aliasOf": null,9160 "verdict": "tested",9161 "context": 200000,9162 "maxOutput": 64000,9163 "pricing": {9164 "input": 5,9165 "cached": 0.5,9166 "output": 259167 },9168 "probes": {9169 "text": {9170 "status": "ok",9171 "ms": 6109172 },9173 "streaming": {9174 "status": "ok",9175 "ms": 11999176 },9177 "tool_use": {9178 "status": "ok",9179 "ms": 13019180 },9181 "structured_output": {9182 "status": "ok",9183 "ms": 11579184 },9185 "vision": {9186 "status": "ok",9187 "ms": 6399188 },9189 "reasoning_control": {9190 "status": "ok",9191 "ms": 14549192 }9193 },9194 "notes": [],9195 "created": "2025-11-24"9196 },9197 {9198 "provider": "anthropic",9199 "id": "claude-haiku-4-5-20251001",9200 "display": "Claude Haiku 4.5",9201 "category": "text",9202 "aliasOf": null,9203 "verdict": "tested",9204 "context": 200000,9205 "maxOutput": 64000,9206 "pricing": {9207 "input": 1,9208 "cached": 0.1,9209 "output": 59210 },9211 "probes": {9212 "text": {9213 "status": "ok",9214 "ms": 4289215 },9216 "streaming": {9217 "status": "ok",9218 "ms": 5639219 },9220 "tool_use": {9221 "status": "ok",9222 "ms": 7779223 },9224 "structured_output": {9225 "status": "ok",9226 "ms": 11789227 },9228 "vision": {9229 "status": "ok",9230 "ms": 4279231 },9232 "reasoning_control": {9233 "status": "ok",9234 "ms": 7829235 }9236 },9237 "notes": [],9238 "created": "2025-10-15"9239 },9240 {9241 "provider": "anthropic",9242 "id": "claude-sonnet-4-5-20250929",9243 "display": "Claude Sonnet 4.5",9244 "category": "text",9245 "aliasOf": null,9246 "verdict": "tested",9247 "context": 200000,9248 "maxOutput": 64000,9249 "pricing": {9250 "input": 3,9251 "cached": 0.3,9252 "output": 159253 },9254 "probes": {9255 "text": {9256 "status": "ok",9257 "ms": 5939258 },9259 "streaming": {9260 "status": "ok",9261 "ms": 6729262 },9263 "tool_use": {9264 "status": "ok",9265 "ms": 12599266 },9267 "structured_output": {9268 "status": "ok",9269 "ms": 10759270 },9271 "vision": {9272 "status": "ok",9273 "ms": 6489274 },9275 "reasoning_control": {9276 "status": "ok",9277 "ms": 11869278 }9279 },9280 "notes": [],9281 "created": "2025-09-29"9282 }9283 ]9284}