{ "generated": "2026-11-01T07:18:30.317Z", "runs": 836, "trials_per_cell": 2, "total_estimated_cost_usd": 17.243599999999997, "categories": [ { "id": "document_analysis", "label": "Document analysis" }, { "id": "json_generation", "label": "JSON generation" }, { "id": "complex_reasoning", "label": "Complex reasoning" }, { "id": "long_context", "label": "Long-context retrieval" }, { "id": "english_writing", "label": "English writing" }, { "id": "french_writing", "label": "French writing" }, { "id": "programming", "label": "Programming" }, { "id": "simple_reasoning", "label": "Simple reasoning" }, { "id": "knowledge_synthesis", "label": "Knowledge synthesis" }, { "id": "tool_calling", "label": "Tool calling" }, { "id": "mathematics", "label": "Mathematics" } ], "pricing_sources": { "openai": { "url": "https://developers.openai.com/api/docs/pricing", "fetched": "2026-11-01", "note": "Standard tier, short context (≤272K input)" }, "anthropic": { "url": "https://platform.claude.com/docs/en/about-claude/pricing", "fetched": "2026-11-01", "note": "First-party API rates" } }, "judges": [ "claude-sonnet-5-5", "gpt-5.4" ], "models": [ { "model": "chat-latest", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 5, "ttft_median_ms": 740.5, "total_median_ms": 1392.5, "tps_median": 157.05, "cost_total": 0.4256, "cost_mean": 0.019345454545454544, "in_tokens": 113620, "out_tokens": 3506, "reasoning_tokens": 128, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 824, "total_ms": 1258, "tps": 195.95, "cost_mean": 0.00505, "out_tokens": 85 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 706.5, "total_ms": 1091, "tps": 174.65, "cost_mean": 0.00277, "out_tokens": 67 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1582, "total_ms": 2846.5, "tps": 269.25, "cost_mean": 0.010794999999999999, "out_tokens": 336.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1056.5, "total_ms": 1250.5, "tps": 58.1, "cost_mean": 0.15248499999999998, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 766, "total_ms": 2856.5, "tps": 115.5, "cost_mean": 0.007549999999999999, "out_tokens": 241 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 735.5, "total_ms": 2906, "tps": 138.4, "cost_mean": 0.009475, "out_tokens": 300.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 786, "total_ms": 1392, "tps": 203.9, "cost_mean": 0.00408, "out_tokens": 123.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 692.5, "total_ms": 900.5, "tps": 92.4, "cost_mean": 0.001195, "out_tokens": 19 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 687.5, "total_ms": 2518.5, "tps": 157.05, "cost_mean": 0.00978, "out_tokens": 287.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 622, "total_ms": 899.5, "tps": 147.7, "cost_mean": 0.001985, "out_tokens": 41 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 726, "total_ms": 1969.5, "tps": 193.7, "cost_mean": 0.007634999999999999, "out_tokens": 241 } } }, { "model": "gpt-5.3-codex", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.666666666666667, "ttft_median_ms": 601.5, "total_median_ms": 1417.5, "tps_median": 93.95, "cost_total": 0.253197, "cost_mean": 0.011508954545454546, "in_tokens": 113620, "out_tokens": 3883, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 602, "total_ms": 1296.5, "tps": 122.44999999999999, "cost_mean": 0.002065, "out_tokens": 85 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 702, "total_ms": 1284, "tps": 115.25, "cost_mean": 0.001204, "out_tokens": 67 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 700.5, "total_ms": 4894.5, "tps": 111.75, "cost_mean": 0.006810999999999999, "out_tokens": 469 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 706.5, "total_ms": 974.5, "tps": 42.150000000000006, "cost_mean": 0.09675225, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 526, "total_ms": 3880.5, "tps": 67.9, "cost_mean": 0.00329, "out_tokens": 227 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 728, "total_ms": 4085, "tps": 80.9, "cost_mean": 0.003962, "out_tokens": 271.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 548, "total_ms": 1417.5, "tps": 140.45, "cost_mean": 0.0018392500000000002, "out_tokens": 122 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 542, "total_ms": 831, "tps": 65.9, "cost_mean": 0.00048475, "out_tokens": 19 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4, "ttft_ms": 609, "total_ms": 3382.5, "tps": 93.55000000000001, "cost_mean": 0.00402325, "out_tokens": 258.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 623, "total_ms": 1083, "tps": 89.05000000000001, "cost_mean": 0.00083825, "out_tokens": 41 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 538.5, "total_ms": 3267.5, "tps": 136, "cost_mean": 0.00532875, "out_tokens": 370.5 } } }, { "model": "gpt-4.1", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.166666666666667, "ttft_median_ms": 449.5, "total_median_ms": 1722, "tps_median": 162.95, "cost_total": 0.2591960000000001, "cost_mean": 0.011781636363636367, "in_tokens": 113634, "out_tokens": 5479, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 477.5, "total_ms": 1064.5, "tps": 174, "cost_mean": 0.001812, "out_tokens": 101 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 522.5, "total_ms": 890.5, "tps": 160.7, "cost_mean": 0.00078, "out_tokens": 59 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 421.5, "total_ms": 4315, "tps": 253.25, "cost_mean": 0.00817, "out_tokens": 986 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2847.5, "total_ms": 3115, "tps": 33.2, "cost_mean": 0.104512, "out_tokens": 8 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 420.5, "total_ms": 1776, "tps": 161.7, "cost_mean": 0.001882, "out_tokens": 219 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 345.5, "total_ms": 1943, "tps": 179.25, "cost_mean": 0.00245, "out_tokens": 283 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 371, "total_ms": 1115, "tps": 184.5, "cost_mean": 0.001192, "out_tokens": 130 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 392.5, "total_ms": 631, "tps": 67.05000000000001, "cost_mean": 0.00038, "out_tokens": 16 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 2.5, "ttft_ms": 545.5, "total_ms": 1812.5, "tps": 216.05, "cost_mean": 0.002652, "out_tokens": 273.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 501.5, "total_ms": 789, "tps": 132.2, "cost_mean": 0.000596, "out_tokens": 38 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 637, "total_ms": 2967.5, "tps": 266.9, "cost_mean": 0.0051719999999999995, "out_tokens": 626 } } }, { "model": "gpt-6-sol", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 5, "ttft_median_ms": 1106, "total_median_ms": 1732, "tps_median": 149.39999999999998, "cost_total": 0.16421719999999998, "cost_mean": 0.007464418181818181, "in_tokens": 113620, "out_tokens": 3633, "reasoning_tokens": 1358, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 955, "total_ms": 1538, "tps": 185.8, "cost_mean": 0.0020800000000000003, "out_tokens": 108 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1200.5, "total_ms": 1732, "tps": 127.64999999999999, "cost_mean": 0.000974, "out_tokens": 67 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1983, "total_ms": 2787, "tps": 254.15, "cost_mean": 0.002325, "out_tokens": 204.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1154.5, "total_ms": 1452.5, "tps": 38.35, "cost_mean": 0.0608316, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 2360, "total_ms": 4195, "tps": 241.05, "cost_mean": 0.003848, "out_tokens": 372 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 2382, "total_ms": 4593.5, "tps": 213.2, "cost_mean": 0.004373999999999999, "out_tokens": 419 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1837, "total_ms": 2589.5, "tps": 311.70000000000005, "cost_mean": 0.002495, "out_tokens": 234.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 849, "total_ms": 1187, "tps": 104.94999999999999, "cost_mean": 0.000595, "out_tokens": 34.5 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 933.5, "total_ms": 2514.5, "tps": 164.5, "cost_mean": 0.003062, "out_tokens": 260 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 828.5, "total_ms": 1251, "tps": 97.25, "cost_mean": 0.000712, "out_tokens": 41 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 940, "total_ms": 1281, "tps": 81.6, "cost_mean": 0.000812, "out_tokens": 65 } } }, { "model": "gpt-5.4", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 5, "ttft_median_ms": 715.5, "total_median_ms": 1817.5, "tps_median": 85.69999999999999, "cost_total": 0.3470499999999999, "cost_mean": 0.015774999999999997, "in_tokens": 113620, "out_tokens": 4200, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 878, "total_ms": 1630.5, "tps": 114.30000000000001, "cost_mean": 0.002525, "out_tokens": 85 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 787, "total_ms": 1453.5, "tps": 100.65, "cost_mean": 0.001385, "out_tokens": 67 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 740, "total_ms": 8292, "tps": 85.69999999999999, "cost_mean": 0.010055000000000001, "out_tokens": 647 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1132, "total_ms": 1526.5, "tps": 32.7, "cost_mean": 0.1381625, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 589, "total_ms": 4781.5, "tps": 54, "cost_mean": 0.0035575, "out_tokens": 226.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 783, "total_ms": 7653.5, "tps": 40.75, "cost_mean": 0.0044225, "out_tokens": 279.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 796.5, "total_ms": 1764.5, "tps": 112.25, "cost_mean": 0.001815, "out_tokens": 108.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 628, "total_ms": 982.5, "tps": 53.95, "cost_mean": 0.0005975, "out_tokens": 19 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 734, "total_ms": 4693.5, "tps": 72.8, "cost_mean": 0.0048975, "out_tokens": 288 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 652.5, "total_ms": 1011.5, "tps": 114.1, "cost_mean": 0.0009925, "out_tokens": 41 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 645, "total_ms": 3608, "tps": 110.5, "cost_mean": 0.005115, "out_tokens": 327.5 } } }, { "model": "claude-fable-5-1", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 5, "ttft_median_ms": 389, "total_median_ms": 2098.5, "tps_median": 118.3, "cost_total": 1.9785900000000003, "cost_mean": 0.08993590909090911, "in_tokens": 168774, "out_tokens": 5817, "reasoning_tokens": 709, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 727.5, "total_ms": 1647, "tps": 128.3, "cost_mean": 0.0161, "out_tokens": 118 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 737, "total_ms": 1587.5, "tps": 123.4, "cost_mean": 0.01021, "out_tokens": 105 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1766, "total_ms": 5206, "tps": 140.2, "cost_mean": 0.026165, "out_tokens": 476.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1388, "total_ms": 1444, "tps": 220.55, "cost_mean": 0.80968, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 340.5, "total_ms": 7003, "tps": 64.19999999999999, "cost_mean": 0.022289999999999997, "out_tokens": 425 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 342, "total_ms": 7573, "tps": 74.80000000000001, "cost_mean": 0.02844, "out_tokens": 536 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 360.5, "total_ms": 2098.5, "tps": 118.3, "cost_mean": 0.011495, "out_tokens": 205.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 331.5, "total_ms": 740.5, "tps": 46.4, "cost_mean": 0.00276, "out_tokens": 19 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 2914, "total_ms": 8164.5, "tps": 125.94999999999999, "cost_mean": 0.036559999999999995, "out_tokens": 661 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 422.5, "total_ms": 1559, "tps": 121.1, "cost_mean": 0.013645000000000001, "out_tokens": 135.5 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 350.5, "total_ms": 2466, "tps": 102.2, "cost_mean": 0.01195, "out_tokens": 216 } } }, { "model": "gpt-5.2", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.333333333333333, "ttft_median_ms": 675.5, "total_median_ms": 2144.5, "tps_median": 85.35, "cost_total": 0.16192260000000006, "cost_mean": 0.007360118181818184, "in_tokens": 113620, "out_tokens": 3541, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 792, "total_ms": 1564.5, "tps": 110, "cost_mean": 0.002065, "out_tokens": 85 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 704.5, "total_ms": 1407.5, "tps": 97.6, "cost_mean": 0.001204, "out_tokens": 67 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 571.5, "total_ms": 4959, "tps": 87.9, "cost_mean": 0.005656, "out_tokens": 386.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 779.5, "total_ms": 1019.5, "tps": 46, "cost_mean": 0.053509049999999996, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 570.5, "total_ms": 4333.5, "tps": 57.55, "cost_mean": 0.003143, "out_tokens": 216.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 717.5, "total_ms": 3830.5, "tps": 81.25, "cost_mean": 0.003689, "out_tokens": 252 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 801.5, "total_ms": 2144.5, "tps": 102.80000000000001, "cost_mean": 0.00206325, "out_tokens": 138 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 629, "total_ms": 999.5, "tps": 52.55, "cost_mean": 0.00048475, "out_tokens": 19 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 3, "ttft_ms": 584.5, "total_ms": 3930, "tps": 72.7, "cost_mean": 0.0037992499999999997, "out_tokens": 242.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 710.5, "total_ms": 1148, "tps": 93.7, "cost_mean": 0.00083825, "out_tokens": 41 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 708.5, "total_ms": 4149.5, "tps": 89.94999999999999, "cost_mean": 0.00450975, "out_tokens": 312 } } }, { "model": "claude-sonnet-5", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.333333333333333, "ttft_median_ms": 949.5, "total_median_ms": 2168.5, "tps_median": 145.05, "cost_total": 0.415752, "cost_mean": 0.018897818181818182, "in_tokens": 168866, "out_tokens": 7802, "reasoning_tokens": 2116, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 971, "total_ms": 1678.5, "tps": 196.15, "cost_mean": 0.0034260000000000002, "out_tokens": 139 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1258.5, "total_ms": 1904.5, "tps": 162.6, "cost_mean": 0.002038, "out_tokens": 105 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 6056.5, "total_ms": 11139, "tps": 252.3, "cost_mean": 0.012919, "out_tokens": 1245.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1202, "total_ms": 1569.5, "tps": 30.200000000000003, "cost_mean": 0.161932, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 949.5, "total_ms": 6332.5, "tps": 88.05000000000001, "cost_mean": 0.0049440000000000005, "out_tokens": 474 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1487.5, "total_ms": 7222, "tps": 110.45, "cost_mean": 0.006659, "out_tokens": 633.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 494, "total_ms": 1814, "tps": 145.05, "cost_mean": 0.002155, "out_tokens": 191.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 926, "total_ms": 1158.5, "tps": 343.1, "cost_mean": 0.001118, "out_tokens": 76 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 3, "ttft_ms": 479.5, "total_ms": 4911, "tps": 101.65, "cost_mean": 0.005202999999999999, "out_tokens": 450.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 751.5, "total_ms": 1455, "tps": 200.45, "cost_mean": 0.002896, "out_tokens": 139 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2783.5, "total_ms": 4106, "tps": 117.85, "cost_mean": 0.004586, "out_tokens": 436 } } }, { "model": "gpt-5.1", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.5, "ttft_median_ms": 649.5, "total_median_ms": 2181, "tps_median": 138.85000000000002, "cost_total": 0.12493499999999999, "cost_mean": 0.0056788636363636355, "in_tokens": 113620, "out_tokens": 4483, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 826.5, "total_ms": 1363, "tps": 171.60000000000002, "cost_mean": 0.001535, "out_tokens": 91 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 653.5, "total_ms": 1112.5, "tps": 159.45, "cost_mean": 0.00092, "out_tokens": 73 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 614, "total_ms": 4113, "tps": 189.2, "cost_mean": 0.00679, "out_tokens": 661.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1506.5, "total_ms": 1775, "tps": 64.05, "cost_mean": 0.03820875, "out_tokens": 17 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 559.5, "total_ms": 4746.5, "tps": 52.8, "cost_mean": 0.0022849999999999997, "out_tokens": 220.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 987, "total_ms": 3910, "tps": 101.05, "cost_mean": 0.00307, "out_tokens": 295.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 808.5, "total_ms": 1711.5, "tps": 161.65, "cost_mean": 0.00155375, "out_tokens": 146 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 584.5, "total_ms": 833, "tps": 100.69999999999999, "cost_mean": 0.00040625, "out_tokens": 25 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 3.5, "ttft_ms": 594.5, "total_ms": 2606, "tps": 132.95, "cost_mean": 0.0029537499999999998, "out_tokens": 266.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 672.5, "total_ms": 1023.5, "tps": 134.10000000000002, "cost_mean": 0.00065875, "out_tokens": 47 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 680, "total_ms": 2868.5, "tps": 183, "cost_mean": 0.00408625, "out_tokens": 398.5 } } }, { "model": "gpt-6.1-sol", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.666666666666667, "ttft_median_ms": 1444, "total_median_ms": 2183.5, "tps_median": 88.75, "cost_total": 0.15463759999999996, "cost_mean": 0.007028981818181817, "in_tokens": 113620, "out_tokens": 3227, "reasoning_tokens": 619, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1136.5, "total_ms": 1875.5, "tps": 151.64999999999998, "cost_mean": 0.00212, "out_tokens": 112 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1059.5, "total_ms": 1674, "tps": 115.75, "cost_mean": 0.000974, "out_tokens": 67 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2701, "total_ms": 4874, "tps": 126.4, "cost_mean": 0.00298, "out_tokens": 270 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1422, "total_ms": 1687.5, "tps": 42.2, "cost_mean": 0.0580718, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1979, "total_ms": 7634.5, "tps": 44, "cost_mean": 0.002618, "out_tokens": 249 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 2749.5, "total_ms": 7974, "tps": 60.2, "cost_mean": 0.0033290000000000004, "out_tokens": 314.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 3534, "total_ms": 4413, "tps": 247.70000000000002, "cost_mean": 0.0023150000000000002, "out_tokens": 216.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1110.5, "total_ms": 1417.5, "tps": 93.05000000000001, "cost_mean": 0.00052, "out_tokens": 27 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4, "ttft_ms": 846, "total_ms": 4112, "tps": 72.75, "cost_mean": 0.002832, "out_tokens": 237 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 794.5, "total_ms": 1173.5, "tps": 108.69999999999999, "cost_mean": 0.000712, "out_tokens": 41 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1805, "total_ms": 2183.5, "tps": 74.44999999999999, "cost_mean": 0.000847, "out_tokens": 68.5 } } }, { "model": "claude-sonnet-4-5-20250929", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.333333333333333, "ttft_median_ms": 774.5, "total_median_ms": 2337, "tps_median": 89.85, "cost_total": 0.474456, "cost_mean": 0.021566181818181818, "in_tokens": 129532, "out_tokens": 5724, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1042, "total_ms": 1914.5, "tps": 115.80000000000001, "cost_mean": 0.004035, "out_tokens": 101 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1074, "total_ms": 1832, "tps": 105.55000000000001, "cost_mean": 0.002394, "out_tokens": 80 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 591, "total_ms": 10630.5, "tps": 83.80000000000001, "cost_mean": 0.01311, "out_tokens": 842 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1624, "total_ms": 2080, "tps": 28.8, "cost_mean": 0.18549, "out_tokens": 13 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 725.5, "total_ms": 6388, "tps": 43.150000000000006, "cost_mean": 0.003882, "out_tokens": 244 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 561.5, "total_ms": 8078.5, "tps": 49.5, "cost_mean": 0.005955, "out_tokens": 372 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 589.5, "total_ms": 2337, "tps": 99.6, "cost_mean": 0.002853, "out_tokens": 174 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 985.5, "total_ms": 1149.5, "tps": 122.4, "cost_mean": 0.000741, "out_tokens": 20 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 3, "ttft_ms": 630, "total_ms": 7236, "tps": 41.55, "cost_mean": 0.004907999999999999, "out_tokens": 274 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1197.5, "total_ms": 1718.5, "tps": 230.05, "cost_mean": 0.004242, "out_tokens": 120 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 774.5, "total_ms": 7696, "tps": 89.85, "cost_mean": 0.009618, "out_tokens": 622 } } }, { "model": "claude-haiku-5-5", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.833333333333333, "ttft_median_ms": 1301, "total_median_ms": 2404, "tps_median": 486.8, "cost_total": 0.023120900000000003, "cost_mean": 0.0010509500000000001, "in_tokens": 168774, "out_tokens": 12487, "reasoning_tokens": 6979, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2934, "total_ms": 3251, "tps": 636.95, "cost_mean": 0.00020275000000000002, "out_tokens": 201.5 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1049, "total_ms": 1319, "tps": 389.35, "cost_mean": 0.00010209999999999999, "out_tokens": 105 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2643, "total_ms": 4890, "tps": 530.05, "cost_mean": 0.00061815, "out_tokens": 1189.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2369.5, "total_ms": 2404, "tps": 4265.85, "cost_mean": 0.008162550000000001, "out_tokens": 142.5 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 442.5, "total_ms": 2749, "tps": 180.55, "cost_mean": 0.00021715, "out_tokens": 413.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4.5, "ttft_ms": 6442.5, "total_ms": 7764, "tps": 1654.15, "cost_mean": 0.0011124, "out_tokens": 2192 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 345.5, "total_ms": 888.5, "tps": 351.25, "cost_mean": 0.0001067, "out_tokens": 189 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 603, "total_ms": 728, "tps": 710.9, "cost_mean": 0.0000566, "out_tokens": 77 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 4008, "total_ms": 5274, "tps": 890.55, "cost_mean": 0.0005986, "out_tokens": 1127 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 532.5, "total_ms": 868, "tps": 387.3, "cost_mean": 0.0001337, "out_tokens": 130 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1145.5, "total_ms": 2074.5, "tps": 297.8, "cost_mean": 0.00024974999999999997, "out_tokens": 476.5 } } }, { "model": "gpt-5.6-sol", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 5, "ttft_median_ms": 1107.5, "total_median_ms": 2466, "tps_median": 132.5, "cost_total": 0.3350544, "cost_mean": 0.015229745454545453, "in_tokens": 113620, "out_tokens": 3964, "reasoning_tokens": 1123, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 774.5, "total_ms": 1327.5, "tps": 153.85, "cost_mean": 0.0037, "out_tokens": 85 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 743.5, "total_ms": 1333, "tps": 113.75, "cost_mean": 0.001948, "out_tokens": 67 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 3244, "total_ms": 4249.5, "tps": 329.20000000000005, "cost_mean": 0.00718, "out_tokens": 331 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1641, "total_ms": 1933, "tps": 37.849999999999994, "cost_mean": 0.1216632, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1089.5, "total_ms": 3794, "tps": 108.85, "cost_mean": 0.006136, "out_tokens": 294 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1252, "total_ms": 4038, "tps": 116.95, "cost_mean": 0.006888, "out_tokens": 326 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1839.5, "total_ms": 2626, "tps": 271.20000000000005, "cost_mean": 0.0044399999999999995, "out_tokens": 207 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 787, "total_ms": 1087, "tps": 63.3, "cost_mean": 0.00088, "out_tokens": 19 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1906, "total_ms": 4141.5, "tps": 152.5, "cost_mean": 0.007743999999999999, "out_tokens": 341 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 860.5, "total_ms": 1244, "tps": 144.85000000000002, "cost_mean": 0.001714, "out_tokens": 55.5 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1170, "total_ms": 2600, "tps": 132.5, "cost_mean": 0.005234, "out_tokens": 245.5 } } }, { "model": "gpt-5.5", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.833333333333333, "ttft_median_ms": 1755, "total_median_ms": 2472, "tps_median": 146.55, "cost_total": 0.4671800000000001, "cost_mean": 0.02123545454545455, "in_tokens": 113620, "out_tokens": 4892, "reasoning_tokens": 2460, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 890, "total_ms": 1613, "tps": 118.1, "cost_mean": 0.00505, "out_tokens": 85 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1008.5, "total_ms": 1639, "tps": 139.89999999999998, "cost_mean": 0.003385, "out_tokens": 87.5 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 4300.5, "total_ms": 5810.5, "tps": 255.85000000000002, "cost_mean": 0.01225, "out_tokens": 385 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2015.5, "total_ms": 2255, "tps": 403, "cost_mean": 0.15496000000000001, "out_tokens": 93.5 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1137, "total_ms": 4611, "tps": 71.25, "cost_mean": 0.007730000000000001, "out_tokens": 247 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 9677, "total_ms": 13390.5, "tps": 213.05, "cost_mean": 0.02245, "out_tokens": 733 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2119, "total_ms": 3176, "tps": 199.85000000000002, "cost_mean": 0.00669, "out_tokens": 210.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1398, "total_ms": 1776.5, "tps": 191.60000000000002, "cost_mean": 0.002785, "out_tokens": 72 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4.5, "ttft_ms": 2466.5, "total_ms": 4838.5, "tps": 160.9, "cost_mean": 0.012209999999999999, "out_tokens": 368.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 894.5, "total_ms": 1262, "tps": 111.7, "cost_mean": 0.001985, "out_tokens": 41 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1952.5, "total_ms": 2295.5, "tps": 79.65, "cost_mean": 0.004095, "out_tokens": 123 } } }, { "model": "claude-opus-4-5-20251101", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.666666666666667, "ttft_median_ms": 861.5, "total_median_ms": 2475.5, "tps_median": 97.8, "cost_total": 0.78326, "cost_mean": 0.03560272727272727, "in_tokens": 129532, "out_tokens": 5424, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 861.5, "total_ms": 1650.5, "tps": 128, "cost_mean": 0.006725, "out_tokens": 101 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 864.5, "total_ms": 1384.5, "tps": 132.7, "cost_mean": 0.003715, "out_tokens": 69 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 816, "total_ms": 9210.5, "tps": 83.25, "cost_mean": 0.01825, "out_tokens": 698 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1994.5, "total_ms": 2146, "tps": 85.9, "cost_mean": 0.30915, "out_tokens": 13 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 646.5, "total_ms": 6864, "tps": 42.2, "cost_mean": 0.006932499999999999, "out_tokens": 262.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 656.5, "total_ms": 7292.5, "tps": 57.65, "cost_mean": 0.010187499999999999, "out_tokens": 382.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 820, "total_ms": 2475.5, "tps": 104.55000000000001, "cost_mean": 0.00473, "out_tokens": 173 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1088, "total_ms": 1182.5, "tps": 210.9, "cost_mean": 0.001235, "out_tokens": 20 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4, "ttft_ms": 990.5, "total_ms": 7481.5, "tps": 48.150000000000006, "cost_mean": 0.009142500000000001, "out_tokens": 312.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1352.5, "total_ms": 1748.5, "tps": 303, "cost_mean": 0.00707, "out_tokens": 120 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 571.5, "total_ms": 6305.5, "tps": 97.8, "cost_mean": 0.014492499999999998, "out_tokens": 560.5 } } }, { "model": "claude-opus-5-5", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.833333333333333, "ttft_median_ms": 1703, "total_median_ms": 2782.5, "tps_median": 160.6, "cost_total": 0.9427159999999999, "cost_mean": 0.04285072727272727, "in_tokens": 168774, "out_tokens": 13381, "reasoning_tokens": 7584, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1019.5, "total_ms": 1820.5, "tps": 147.95, "cost_mean": 0.00644, "out_tokens": 118 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1703, "total_ms": 2361.5, "tps": 159.35, "cost_mean": 0.004084, "out_tokens": 105 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2424, "total_ms": 6117, "tps": 184.45, "cost_mean": 0.014536, "out_tokens": 680 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2306.5, "total_ms": 2336, "tps": 375.15, "cost_mean": 0.323872, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 12624.5, "total_ms": 15484, "tps": 667.1500000000001, "cost_mean": 0.038606, "out_tokens": 1909.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 14963, "total_ms": 18435.5, "tps": 698.8, "cost_mean": 0.049145999999999995, "out_tokens": 2424.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1170, "total_ms": 2647.5, "tps": 185.3, "cost_mean": 0.005938000000000001, "out_tokens": 272.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 835.5, "total_ms": 1282.5, "tps": 132.1, "cost_mean": 0.001904, "out_tokens": 59 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4.5, "ttft_ms": 2205, "total_ms": 5931.5, "tps": 145.45, "cost_mean": 0.012244, "out_tokens": 542 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 946, "total_ms": 1873, "tps": 140.14999999999998, "cost_mean": 0.005348, "out_tokens": 130 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 901.5, "total_ms": 3753, "tps": 134.05, "cost_mean": 0.00924, "out_tokens": 439 } } }, { "model": "claude-sonnet-5-5", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.666666666666667, "ttft_median_ms": 1019.5, "total_median_ms": 2853, "tps_median": 201.95, "cost_total": 0.432488, "cost_mean": 0.019658545454545453, "in_tokens": 168774, "out_tokens": 9494, "reasoning_tokens": 2881, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1609.5, "total_ms": 2040.5, "tps": 275.05, "cost_mean": 0.00322, "out_tokens": 118 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1719, "total_ms": 2151.5, "tps": 242.89999999999998, "cost_mean": 0.002042, "out_tokens": 105 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2570.5, "total_ms": 5096, "tps": 297.4, "cost_mean": 0.007977999999999999, "out_tokens": 751 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2243, "total_ms": 2270, "tps": 415, "cost_mean": 0.161936, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 463, "total_ms": 4608.5, "tps": 115.1, "cost_mean": 0.004968, "out_tokens": 476 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 7827.5, "total_ms": 10002, "tps": 768.95, "cost_mean": 0.016973000000000002, "out_tokens": 1664.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 798, "total_ms": 4436.5, "tps": 201.95, "cost_mean": 0.007589, "out_tokens": 734.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 688.5, "total_ms": 1222, "tps": 35.599999999999994, "cost_mean": 0.000552, "out_tokens": 19 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4, "ttft_ms": 853, "total_ms": 3735.5, "tps": 146.15, "cost_mean": 0.004912, "out_tokens": 421 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 766.5, "total_ms": 1619, "tps": 152.5, "cost_mean": 0.002674, "out_tokens": 130 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 808.5, "total_ms": 2853, "tps": 155.05, "cost_mean": 0.0034000000000000002, "out_tokens": 317 } } }, { "model": "claude-opus-4-8", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.666666666666667, "ttft_median_ms": 487.5, "total_median_ms": 2865, "tps_median": 103.8, "cost_total": 0.9819399999999999, "cost_mean": 0.04463363636363636, "in_tokens": 168738, "out_tokens": 5530, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2111, "total_ms": 3173.5, "tps": 110.15, "cost_mean": 0.008015, "out_tokens": 117 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1108.5, "total_ms": 2021, "tps": 114, "cost_mean": 0.00507, "out_tokens": 104 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 455, "total_ms": 5179, "tps": 97.75, "cost_mean": 0.0126975, "out_tokens": 461.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1920, "total_ms": 2182, "tps": 41.95, "cost_mean": 0.40483, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 454.5, "total_ms": 7459, "tps": 64.65, "cost_mean": 0.0117725, "out_tokens": 450.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 447.5, "total_ms": 9605, "tps": 59.7, "cost_mean": 0.014447499999999999, "out_tokens": 545.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 553.5, "total_ms": 1918.5, "tps": 125.75, "cost_mean": 0.0048874999999999995, "out_tokens": 171.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 486, "total_ms": 662, "tps": 110.19999999999999, "cost_mean": 0.00137, "out_tokens": 19 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4, "ttft_ms": 495.5, "total_ms": 5562.5, "tps": 87.4, "cost_mean": 0.012782499999999999, "out_tokens": 441.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 508.5, "total_ms": 1669, "tps": 124.19999999999999, "cost_mean": 0.007045, "out_tokens": 144 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 435.5, "total_ms": 3139.5, "tps": 110.95, "cost_mean": 0.0080525, "out_tokens": 299.5 } } }, { "model": "o3", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.333333333333333, "ttft_median_ms": 1762.5, "total_median_ms": 2943.5, "tps_median": 533, "cost_total": 0.32102, "cost_mean": 0.014591818181818183, "in_tokens": 113614, "out_tokens": 11724, "reasoning_tokens": 8512, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1638, "total_ms": 2220.5, "tps": 592.3499999999999, "cost_mean": 0.003548, "out_tokens": 318.5 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1224.5, "total_ms": 1664.5, "tps": 454.9, "cost_mean": 0.00186, "out_tokens": 194.5 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 5316, "total_ms": 5720, "tps": 2221.35, "cost_mean": 0.007516, "out_tokens": 904.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 3176, "total_ms": 3456.5, "tps": 423.2, "cost_mean": 0.111322, "out_tokens": 115.5 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 2516, "total_ms": 4191.5, "tps": 468.55, "cost_mean": 0.005572, "out_tokens": 680.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4.5, "ttft_ms": 6182.5, "total_ms": 7472, "tps": 1105.05, "cost_mean": 0.011652, "out_tokens": 1433.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1925.5, "total_ms": 2572, "tps": 701.0999999999999, "cost_mean": 0.003778, "out_tokens": 453.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1455.5, "total_ms": 1748.5, "tps": 1004.5, "cost_mean": 0.002498, "out_tokens": 281 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 3.5, "ttft_ms": 1336, "total_ms": 3022, "tps": 261.75, "cost_mean": 0.00395, "out_tokens": 436 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 742, "total_ms": 1044, "tps": 317.95000000000005, "cost_mean": 0.00106, "out_tokens": 95.5 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 3940.5, "total_ms": 4663.5, "tps": 231.4, "cost_mean": 0.0077540000000000005, "out_tokens": 949 } } }, { "model": "gpt-6-astra", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 5, "ttft_median_ms": 2376, "total_median_ms": 3056.5, "tps_median": 77.85, "cost_total": 0.809736, "cost_mean": 0.03680618181818182, "in_tokens": 113620, "out_tokens": 3406, "reasoning_tokens": 776, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1307.5, "total_ms": 2154.5, "tps": 126.35000000000001, "cost_mean": 0.01035, "out_tokens": 107 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1995, "total_ms": 2804.5, "tps": 93.5, "cost_mean": 0.00487, "out_tokens": 67 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 3496, "total_ms": 6345, "tps": 96.95, "cost_mean": 0.015224999999999999, "out_tokens": 276.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2465.5, "total_ms": 2827.5, "tps": 30.4, "cost_mean": 0.30415800000000004, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 2419.5, "total_ms": 8367.5, "tps": 43.45, "cost_mean": 0.013565, "out_tokens": 258.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 2570, "total_ms": 9359, "tps": 45.150000000000006, "cost_mean": 0.016245, "out_tokens": 306.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 3521, "total_ms": 4526, "tps": 208.55, "cost_mean": 0.011300000000000001, "out_tokens": 211 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1421.5, "total_ms": 1873.5, "tps": 93.1, "cost_mean": 0.00305, "out_tokens": 36 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 2987.5, "total_ms": 7010.5, "tps": 76, "cost_mean": 0.01761, "out_tokens": 306 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1530.5, "total_ms": 1973, "tps": 93.94999999999999, "cost_mean": 0.00356, "out_tokens": 41 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2493, "total_ms": 2858.5, "tps": 75.15, "cost_mean": 0.004935, "out_tokens": 82.5 } } }, { "model": "gpt-6.1-terra", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.666666666666667, "ttft_median_ms": 1514.5, "total_median_ms": 3151, "tps_median": 92.6, "cost_total": 0.0809488, "cost_mean": 0.003679490909090909, "in_tokens": 113620, "out_tokens": 3953, "reasoning_tokens": 1303, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 888.5, "total_ms": 1951, "tps": 80, "cost_mean": 0.000925, "out_tokens": 85 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1085.5, "total_ms": 2026.5, "tps": 84.05000000000001, "cost_mean": 0.0005395, "out_tokens": 77.5 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2656.5, "total_ms": 4411, "tps": 174.5, "cost_mean": 0.0016675000000000001, "out_tokens": 305.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 921.5, "total_ms": 1217, "tps": 37.45, "cost_mean": 0.0290359, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 2098.5, "total_ms": 5373.5, "tps": 94.35, "cost_mean": 0.0015965, "out_tokens": 306.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1514.5, "total_ms": 4905, "tps": 95.7, "cost_mean": 0.0017144999999999999, "out_tokens": 324.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1833.5, "total_ms": 3197, "tps": 142.4, "cost_mean": 0.0010400000000000001, "out_tokens": 193 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1024.5, "total_ms": 1447, "tps": 104.19999999999999, "cost_mean": 0.00034250000000000003, "out_tokens": 43.5 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4, "ttft_ms": 2715.5, "total_ms": 5120.5, "tps": 157.15, "cost_mean": 0.0021260000000000003, "out_tokens": 379 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 821, "total_ms": 1412, "tps": 69.4, "cost_mean": 0.000356, "out_tokens": 41 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1954, "total_ms": 3430, "tps": 80.6, "cost_mean": 0.001131, "out_tokens": 210 } } }, { "model": "claude-opus-4-7", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.333333333333333, "ttft_median_ms": 686.5, "total_median_ms": 3320.5, "tps_median": 91.35, "cost_total": 0.9817649999999998, "cost_mean": 0.04462568181818181, "in_tokens": 169618, "out_tokens": 5347, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2202, "total_ms": 3266.5, "tps": 112.80000000000001, "cost_mean": 0.008115, "out_tokens": 120 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1585, "total_ms": 2583.5, "tps": 107.1, "cost_mean": 0.00517, "out_tokens": 107 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 613, "total_ms": 4529.5, "tps": 91.35, "cost_mean": 0.010110000000000001, "out_tokens": 357 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2775, "total_ms": 2984.5, "tps": 67.3, "cost_mean": 0.40493, "out_tokens": 14 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 531, "total_ms": 8492.5, "tps": 59.25, "cost_mean": 0.0123225, "out_tokens": 471.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1160, "total_ms": 11929.5, "tps": 52.95, "cost_mean": 0.015085000000000001, "out_tokens": 570 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 479.5, "total_ms": 1789.5, "tps": 125.19999999999999, "cost_mean": 0.004725, "out_tokens": 164 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 734.5, "total_ms": 894.5, "tps": 159.8, "cost_mean": 0.00147, "out_tokens": 22 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 3, "ttft_ms": 574.5, "total_ms": 6505, "tps": 71.55000000000001, "cost_mean": 0.012344999999999998, "out_tokens": 423 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 992, "total_ms": 1948.5, "tps": 152.64999999999998, "cost_mean": 0.009045, "out_tokens": 146 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 458.5, "total_ms": 4039, "tps": 76.3, "cost_mean": 0.007565, "out_tokens": 279 } } }, { "model": "claude-opus-5", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.666666666666667, "ttft_median_ms": 2530, "total_median_ms": 4043, "tps_median": 160.89999999999998, "cost_total": 1.1158750000000002, "cost_mean": 0.05072159090909092, "in_tokens": 168730, "out_tokens": 10889, "reasoning_tokens": 5458, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2203, "total_ms": 3265.5, "tps": 111.1, "cost_mean": 0.00804, "out_tokens": 118 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1354, "total_ms": 2252.5, "tps": 116.85, "cost_mean": 0.005095, "out_tokens": 105 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 4756.5, "total_ms": 8775.5, "tps": 168.89999999999998, "cost_mean": 0.0179975, "out_tokens": 673.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2696, "total_ms": 2739, "tps": 263.54999999999995, "cost_mean": 0.40483, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 9286.5, "total_ms": 14506, "tps": 245.64999999999998, "cost_mean": 0.0260725, "out_tokens": 1022.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 17383.5, "total_ms": 22242, "tps": 390.75, "cost_mean": 0.047385, "out_tokens": 1863 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 4119.5, "total_ms": 7068.5, "tps": 156.60000000000002, "cost_mean": 0.011574999999999998, "out_tokens": 439 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2304.5, "total_ms": 2482.5, "tps": 444.34999999999997, "cost_mean": 0.00272, "out_tokens": 73 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4, "ttft_ms": 4503, "total_ms": 9943, "tps": 121.35, "cost_mean": 0.01812, "out_tokens": 655 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1556, "total_ms": 2426, "tps": 182.3, "cost_mean": 0.007375, "out_tokens": 158 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1415.5, "total_ms": 4043, "tps": 99.9, "cost_mean": 0.0087275, "out_tokens": 326.5 } } }, { "model": "claude-opus-4-6", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.833333333333333, "ttft_median_ms": 1621.5, "total_median_ms": 4197.5, "tps_median": 74.5, "cost_total": 0.77559, "cost_mean": 0.03525409090909091, "in_tokens": 129538, "out_tokens": 5116, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2807, "total_ms": 3685, "tps": 115, "cost_mean": 0.00673, "out_tokens": 101 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 3382.5, "total_ms": 3993, "tps": 113.19999999999999, "cost_mean": 0.00372, "out_tokens": 69 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1414, "total_ms": 12800, "tps": 61.4, "cost_mean": 0.018250000000000002, "out_tokens": 698 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2326, "total_ms": 2823, "tps": 26.4, "cost_mean": 0.30915, "out_tokens": 13 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1475.5, "total_ms": 8633, "tps": 36.7, "cost_mean": 0.0069325, "out_tokens": 262.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1509.5, "total_ms": 9183.5, "tps": 50.55, "cost_mean": 0.0103, "out_tokens": 387 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2020.5, "total_ms": 3737.5, "tps": 96.19999999999999, "cost_mean": 0.00453, "out_tokens": 165 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1441, "total_ms": 1718, "tps": 76.05, "cost_mean": 0.001235, "out_tokens": 20 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4.5, "ttft_ms": 1718.5, "total_ms": 8823, "tps": 42.2, "cost_mean": 0.008817499999999999, "out_tokens": 299.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1791, "total_ms": 2906.5, "tps": 118.5, "cost_mean": 0.007375, "out_tokens": 132 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1582.5, "total_ms": 6329.5, "tps": 86.55, "cost_mean": 0.010755, "out_tokens": 411 } } }, { "model": "claude-sonnet-4-6", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 5, "ttft_median_ms": 735, "total_median_ms": 4707.5, "tps_median": 89.6, "cost_total": 0.4661640000000001, "cost_mean": 0.02118927272727273, "in_tokens": 129538, "out_tokens": 5170, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 3623, "total_ms": 4525.5, "tps": 111.94999999999999, "cost_mean": 0.004038, "out_tokens": 101 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2725.5, "total_ms": 3381, "tps": 105.4, "cost_mean": 0.002232, "out_tokens": 69 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 659.5, "total_ms": 10339, "tps": 69.85, "cost_mean": 0.01062, "out_tokens": 676 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2302.5, "total_ms": 2669, "tps": 35.5, "cost_mean": 0.18549, "out_tokens": 13 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 646, "total_ms": 7520.5, "tps": 40.05, "cost_mean": 0.0043395, "out_tokens": 274.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 714, "total_ms": 8037.5, "tps": 50.3, "cost_mean": 0.005895, "out_tokens": 368 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 673.5, "total_ms": 2598.5, "tps": 91.5, "cost_mean": 0.002883, "out_tokens": 176 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1070.5, "total_ms": 1231, "tps": 124.9, "cost_mean": 0.000741, "out_tokens": 20 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 872.5, "total_ms": 7648, "tps": 45.75, "cost_mean": 0.005448, "out_tokens": 310 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 681, "total_ms": 1792.5, "tps": 117.9, "cost_mean": 0.00441, "out_tokens": 131 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 611, "total_ms": 5533.5, "tps": 90.69999999999999, "cost_mean": 0.0069855, "out_tokens": 446.5 } } }, { "model": "gpt-5", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.333333333333333, "ttft_median_ms": 4121, "total_median_ms": 4969.5, "tps_median": 962.45, "cost_total": 0.26522899999999994, "cost_mean": 0.012055863636363634, "in_tokens": 113620, "out_tokens": 17346, "reasoning_tokens": 13696, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 5090.5, "total_ms": 5656, "tps": 1595.9499999999998, "cost_mean": 0.0093, "out_tokens": 867.5 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2910, "total_ms": 3430, "tps": 1050.35, "cost_mean": 0.00564, "out_tokens": 545 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 4953, "total_ms": 5845, "tps": 1215.55, "cost_mean": 0.010825000000000001, "out_tokens": 1065 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 3088.5, "total_ms": 3382, "tps": 796.45, "cost_mean": 0.04594575, "out_tokens": 207.5 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 9221.5, "total_ms": 10391.5, "tps": 1243.15, "cost_mean": 0.0146, "out_tokens": 1452 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 7258, "total_ms": 9252.5, "tps": 763.4, "cost_mean": 0.01303, "out_tokens": 1291.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 3734, "total_ms": 4640.5, "tps": 960.75, "cost_mean": 0.00878375, "out_tokens": 869 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1297, "total_ms": 1580.5, "tps": 830.9, "cost_mean": 0.0024462499999999996, "out_tokens": 229 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 3, "ttft_ms": 5380.5, "total_ms": 7232, "tps": 566.55, "cost_mean": 0.01069875, "out_tokens": 1041 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1965, "total_ms": 2430, "tps": 574.1, "cost_mean": 0.00260875, "out_tokens": 242 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 7273.5, "total_ms": 7600, "tps": 197, "cost_mean": 0.008736250000000001, "out_tokens": 863.5 } } }, { "model": "gpt-5-mini", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 1, "mean_score": 1, "judged_mean": 4.833333333333333, "ttft_median_ms": 4395, "total_median_ms": 5673, "tps_median": 374.55, "cost_total": 0.056407, "cost_mean": 0.0025639545454545455, "in_tokens": 113620, "out_tokens": 14001, "reasoning_tokens": 10176, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 4189.5, "total_ms": 5282.5, "tps": 378.8, "cost_mean": 0.000953, "out_tokens": 414 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2894, "total_ms": 3835.5, "tps": 241.25, "cost_mean": 0.000492, "out_tokens": 227 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 9508.5, "total_ms": 12444, "tps": 349.75, "cost_mean": 0.0020740000000000003, "out_tokens": 1019.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2807.5, "total_ms": 3102.5, "tps": 471.75, "cost_mean": 0.01406575, "out_tokens": 133 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 8008, "total_ms": 10396.5, "tps": 364.35, "cost_mean": 0.001692, "out_tokens": 838 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 13945.5, "total_ms": 16488.5, "tps": 558.05, "cost_mean": 0.00286, "out_tokens": 1418.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 4148.5, "total_ms": 5509, "tps": 401.75, "cost_mean": 0.00111375, "out_tokens": 547.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2540.5, "total_ms": 2858, "tps": 829.3, "cost_mean": 0.00055525, "out_tokens": 262 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4.5, "ttft_ms": 6918, "total_ms": 9179.5, "tps": 393.45000000000005, "cost_mean": 0.00183175, "out_tokens": 887 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1149, "total_ms": 1626.5, "tps": 205.14999999999998, "cost_mean": 0.00023075, "out_tokens": 96.5 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 6625, "total_ms": 8280.5, "tps": 156.05, "cost_mean": 0.00233525, "out_tokens": 1157.5 } } }, { "model": "gpt-4o", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 0.9545454545454546, "mean_score": 0.9962272727272729, "judged_mean": 3.8333333333333335, "ttft_median_ms": 656, "total_median_ms": 2401.5, "tps_median": 106.25, "cost_total": 0.3303150000000001, "cost_mean": 0.015014318181818186, "in_tokens": 113634, "out_tokens": 4623, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 709.5, "total_ms": 1207, "tps": 160.05, "cost_mean": 0.002045, "out_tokens": 79 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 507, "total_ms": 977, "tps": 134.05, "cost_mean": 0.000985, "out_tokens": 60 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 670.5, "total_ms": 3947, "tps": 131.45, "cost_mean": 0.0046675, "out_tokens": 431.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 3213.5, "total_ms": 3409, "tps": 42.45, "cost_mean": 0.13808, "out_tokens": 8 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 767.5, "total_ms": 2344.5, "tps": 117.9, "cost_mean": 0.0020175, "out_tokens": 185.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 0.5, "score": 0.9585, "judged": 4, "ttft_ms": 702.5, "total_ms": 3305, "tps": 106.25, "cost_mean": 0.0029975, "out_tokens": 276.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1073, "total_ms": 2237, "tps": 116.85, "cost_mean": 0.001415, "out_tokens": 122.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 622, "total_ms": 832, "tps": 79.25, "cost_mean": 0.000475, "out_tokens": 16 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 2.5, "ttft_ms": 692, "total_ms": 2541.5, "tps": 108.8, "cost_mean": 0.002585, "out_tokens": 200.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 610.5, "total_ms": 993, "tps": 99.5, "cost_mean": 0.000745, "out_tokens": 38 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 656, "total_ms": 8235.5, "tps": 123.5, "cost_mean": 0.009145, "out_tokens": 894 } } }, { "model": "gpt-5.6-luna", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 0.9545454545454546, "mean_score": 0.9772727272727273, "judged_mean": 4.666666666666667, "ttft_median_ms": 1398, "total_median_ms": 3014, "tps_median": 136.45, "cost_total": 0.018583520000000006, "cost_mean": 0.0008447054545454548, "in_tokens": 113620, "out_tokens": 4829, "reasoning_tokens": 1858, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 911.5, "total_ms": 1586.5, "tps": 184.9, "cost_mean": 0.0002494, "out_tokens": 124.5 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 689.5, "total_ms": 1329, "tps": 105.4, "cost_mean": 0.0001108, "out_tokens": 67 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2832.5, "total_ms": 4203.5, "tps": 326.4, "cost_mean": 0.0005625999999999999, "out_tokens": 445.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1057.5, "total_ms": 1326.5, "tps": 41.900000000000006, "cost_mean": 0.0060853600000000015, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1268, "total_ms": 4736, "tps": 95.25, "cost_mean": 0.00040340000000000005, "out_tokens": 325.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1906, "total_ms": 4849, "tps": 145.6, "cost_mean": 0.0005325999999999999, "out_tokens": 428.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1963.5, "total_ms": 3067, "tps": 238.75, "cost_mean": 0.00033120000000000003, "out_tokens": 263.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1299, "total_ms": 1657, "tps": 162.8, "cost_mean": 0.0000946, "out_tokens": 58 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4, "ttft_ms": 1662, "total_ms": 4621, "tps": 112.8, "cost_mean": 0.0004464, "out_tokens": 333.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 994.5, "total_ms": 1377, "tps": 138.95, "cost_mean": 0.0000938, "out_tokens": 53 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 0.5, "score": 0.75, "judged": null, "ttft_ms": 2430.5, "total_ms": 3678.5, "tps": 105.05, "cost_mean": 0.00038159999999999995, "out_tokens": 304.5 } } }, { "model": "gpt-4.1-mini", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 0.9090909090909091, "mean_score": 0.9735, "judged_mean": 3.8333333333333335, "ttft_median_ms": 449, "total_median_ms": 1683.5, "tps_median": 153.3, "cost_total": 0.053280799999999996, "cost_mean": 0.0024218545454545454, "in_tokens": 113634, "out_tokens": 5228, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 391.5, "total_ms": 940.5, "tps": 145, "cost_mean": 0.00032720000000000004, "out_tokens": 79 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 421.5, "total_ms": 839.5, "tps": 141.2, "cost_mean": 0.000156, "out_tokens": 59 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 365, "total_ms": 5553.5, "tps": 189.45, "cost_mean": 0.0016308000000000002, "out_tokens": 984 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1922.5, "total_ms": 2153.5, "tps": 35.4, "cost_mean": 0.021824, "out_tokens": 8 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 416, "total_ms": 1670, "tps": 174.6, "cost_mean": 0.00037640000000000004, "out_tokens": 219 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 0.5, "score": 0.9585, "judged": 4, "ttft_ms": 501, "total_ms": 2025.5, "tps": 176.8, "cost_mean": 0.0004676, "out_tokens": 269 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 444, "total_ms": 1209, "tps": 156.25, "cost_mean": 0.00022160000000000002, "out_tokens": 119.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 983, "total_ms": 1233.5, "tps": 64.15, "cost_mean": 0.000076, "out_tokens": 16 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 2.5, "ttft_ms": 445, "total_ms": 2108.5, "tps": 149.55, "cost_mean": 0.0004904, "out_tokens": 248.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 548.5, "total_ms": 832.5, "tps": 135.85, "cost_mean": 0.00011920000000000001, "out_tokens": 38 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 0.5, "score": 0.75, "judged": null, "ttft_ms": 708.5, "total_ms": 4104, "tps": 173.75, "cost_mean": 0.0009512000000000001, "out_tokens": 574 } } }, { "model": "claude-haiku-4-5-20251001", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 0.9090909090909091, "mean_score": 0.9697272727272727, "judged_mean": 3.3333333333333335, "ttft_median_ms": 502, "total_median_ms": 1966.5, "tps_median": 134.55, "cost_total": 0.15852199999999997, "cost_mean": 0.007205545454545453, "in_tokens": 129532, "out_tokens": 5798, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1132.5, "total_ms": 1880.5, "tps": 152.39999999999998, "cost_mean": 0.00141, "out_tokens": 114 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1039.5, "total_ms": 1524.5, "tps": 142.55, "cost_mean": 0.000743, "out_tokens": 69 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 404.5, "total_ms": 7524, "tps": 131.8, "cost_mean": 0.0048425, "out_tokens": 936.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 865.5, "total_ms": 973, "tps": 121.5, "cost_mean": 0.06183, "out_tokens": 13 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4, "ttft_ms": 373, "total_ms": 3227, "tps": 89.9, "cost_mean": 0.0013565, "out_tokens": 256.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 3.5, "ttft_ms": 384.5, "total_ms": 3746, "tps": 102.35, "cost_mean": 0.0018449999999999999, "out_tokens": 344 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 553, "total_ms": 1930.5, "tps": 151.3, "cost_mean": 0.0011235, "out_tokens": 208.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 0, "score": 0.667, "judged": null, "ttft_ms": 508, "total_ms": 598, "tps": 221.55, "cost_mean": 0.000247, "out_tokens": 20 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 2.5, "ttft_ms": 368, "total_ms": 3060.5, "tps": 94.6, "cost_mean": 0.0015184999999999999, "out_tokens": 250.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 536.5, "total_ms": 1165, "tps": 192.55, "cost_mean": 0.001419, "out_tokens": 121 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 372.5, "total_ms": 4615, "tps": 133.35000000000002, "cost_mean": 0.0029259999999999998, "out_tokens": 566 } } }, { "model": "gpt-5-nano", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 0.9545454545454546, "mean_score": 0.9659090909090909, "judged_mean": 3.6666666666666665, "ttft_median_ms": 6133, "total_median_ms": 6849.5, "tps_median": 1362.1, "cost_total": 0.0140714, "cost_mean": 0.0006396090909090909, "in_tokens": 113620, "out_tokens": 27168, "reasoning_tokens": 24384, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 9956, "total_ms": 10721, "tps": 1359.9, "cost_mean": 0.00044120000000000004, "out_tokens": 1040.5 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2280.5, "total_ms": 2955, "tps": 952, "cost_mean": 0.0002278, "out_tokens": 550.5 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 5995, "total_ms": 6223, "tps": 5903.200000000001, "cost_mean": 0.000546, "out_tokens": 1347.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1907.5, "total_ms": 2222, "tps": 705.3, "cost_mean": 0.0015985500000000002, "out_tokens": 192.5 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 11278.5, "total_ms": 12370, "tps": 2324.3, "cost_mean": 0.0009774, "out_tokens": 2435.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 0.5, "score": 0.625, "judged": 3, "ttft_ms": 9189, "total_ms": 11226.5, "tps": 2000.2, "cost_mean": 0.0010896, "out_tokens": 2712.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 6243, "total_ms": 7041, "tps": 2127.05, "cost_mean": 0.00068495, "out_tokens": 1703 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1874, "total_ms": 2087.5, "tps": 2235.9, "cost_mean": 0.00019645000000000003, "out_tokens": 475.5 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 3, "ttft_ms": 6880, "total_ms": 8291.5, "tps": 1267.7, "cost_mean": 0.00072255, "out_tokens": 1777.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1292.5, "total_ms": 1613.5, "tps": 741.5999999999999, "cost_mean": 0.00010255, "out_tokens": 237.5 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 7906, "total_ms": 8184.5, "tps": 322.9, "cost_mean": 0.00044865000000000004, "out_tokens": 1111.5 } } }, { "model": "gpt-5.4-mini", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 0.9545454545454546, "mean_score": 0.9545454545454546, "judged_mean": 4.666666666666667, "ttft_median_ms": 623.5, "total_median_ms": 1202.5, "tps_median": 166.65, "cost_total": 0.06353849999999998, "cost_mean": 0.0028881136363636357, "in_tokens": 113620, "out_tokens": 3439, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 733.5, "total_ms": 1127, "tps": 216, "cost_mean": 0.0007575, "out_tokens": 85 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 705.5, "total_ms": 1126, "tps": 159.4, "cost_mean": 0.0004155, "out_tokens": 67 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 0.5, "score": 0.5, "judged": null, "ttft_ms": 485.5, "total_ms": 1996, "tps": 206.85, "cost_mean": 0.00151125, "out_tokens": 312.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 869, "total_ms": 1068, "tps": 56.1, "cost_mean": 0.02287275, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 501, "total_ms": 1988.5, "tps": 152.2, "cost_mean": 0.00106725, "out_tokens": 226.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 605, "total_ms": 2124, "tps": 169.85, "cost_mean": 0.00123, "out_tokens": 258 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 555, "total_ms": 1096, "tps": 246.75, "cost_mean": 0.000657, "out_tokens": 133.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 722.5, "total_ms": 1016.5, "tps": 64.7, "cost_mean": 0.00017925, "out_tokens": 19 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4, "ttft_ms": 623, "total_ms": 1963, "tps": 180.35000000000002, "cost_mean": 0.0012487499999999999, "out_tokens": 239 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 600.5, "total_ms": 935, "tps": 123.3, "cost_mean": 0.00029775, "out_tokens": 41 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 604.5, "total_ms": 1753, "tps": 284.5, "cost_mean": 0.0015322500000000002, "out_tokens": 327 } } }, { "model": "gpt-5.6-terra", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 0.9545454545454546, "mean_score": 0.9545454545454546, "judged_mean": 5, "ttft_median_ms": 869, "total_median_ms": 1802.5, "tps_median": 125.65, "cost_total": 0.1691192, "cost_mean": 0.007687236363636363, "in_tokens": 113620, "out_tokens": 3436, "reasoning_tokens": 881, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 666.5, "total_ms": 1278, "tps": 139.55, "cost_mean": 0.00202, "out_tokens": 85 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 586, "total_ms": 1090, "tps": 132.95, "cost_mean": 0.001108, "out_tokens": 67 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 3022, "total_ms": 4210, "tps": 239.35, "cost_mean": 0.003628, "out_tokens": 279 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 0.5, "score": 0.5, "judged": null, "ttft_ms": 1204, "total_ms": 1456, "tps": 208.85000000000002, "cost_mean": 0.0613936, "out_tokens": 56 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1036, "total_ms": 4212.5, "tps": 74.6, "cost_mean": 0.002954, "out_tokens": 235.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1075, "total_ms": 4789, "tps": 80.5, "cost_mean": 0.00376, "out_tokens": 298 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 788, "total_ms": 1484, "tps": 153.8, "cost_mean": 0.001434, "out_tokens": 107 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 693, "total_ms": 1029.5, "tps": 56.5, "cost_mean": 0.000478, "out_tokens": 19 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 869, "total_ms": 3308, "tps": 103.05000000000001, "cost_mean": 0.003474, "out_tokens": 251 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1017.5, "total_ms": 1369.5, "tps": 117.4, "cost_mean": 0.000794, "out_tokens": 41 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2400, "total_ms": 3314.5, "tps": 136.3, "cost_mean": 0.003516, "out_tokens": 279.5 } } }, { "model": "gpt-6-luna", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 0.9545454545454546, "mean_score": 0.9545454545454546, "judged_mean": 4.666666666666667, "ttft_median_ms": 2045, "total_median_ms": 3000, "tps_median": 194.35, "cost_total": 0.014567000000000002, "cost_mean": 0.0006621363636363638, "in_tokens": 113620, "out_tokens": 6410, "reasoning_tokens": 3720, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1500, "total_ms": 2418.5, "tps": 140.2, "cost_mean": 0.00011275, "out_tokens": 125.5 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1590.5, "total_ms": 2244, "tps": 142.3, "cost_mean": 0.00006170000000000001, "out_tokens": 93 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 3736, "total_ms": 5099, "tps": 287.04999999999995, "cost_mean": 0.00020775, "out_tokens": 387.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 0.5, "score": 0.5, "judged": null, "ttft_ms": 2542, "total_ms": 2861.5, "tps": 545.05, "cost_mean": 0.00560715, "out_tokens": 174.5 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 5620.5, "total_ms": 7089.5, "tps": 471.7, "cost_mean": 0.0003494, "out_tokens": 686 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 4196.5, "total_ms": 6311, "tps": 340.25, "cost_mean": 0.0003382, "out_tokens": 658 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1928.5, "total_ms": 3055.5, "tps": 189.05, "cost_mean": 0.00011325, "out_tokens": 211.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1060.5, "total_ms": 1400.5, "tps": 196.95, "cost_mean": 0.00004575, "out_tokens": 66.5 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4, "ttft_ms": 1515, "total_ms": 2763.5, "tps": 323.65, "cost_mean": 0.00022285, "out_tokens": 399.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 747, "total_ms": 1100.5, "tps": 116.15, "cost_mean": 0.000035600000000000005, "out_tokens": 41 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 19505, "total_ms": 20422.5, "tps": 144.85, "cost_mean": 0.0001891, "out_tokens": 362 } } }, { "model": "gpt-4o-mini", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 0.8181818181818182, "mean_score": 0.9432272727272728, "judged_mean": 4, "ttft_median_ms": 561, "total_median_ms": 1966, "tps_median": 116.4, "cost_total": 0.020267100000000003, "cost_mean": 0.0009212318181818183, "in_tokens": 113634, "out_tokens": 5370, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1210.5, "total_ms": 2798.5, "tps": 86.45, "cost_mean": 0.0001389, "out_tokens": 106 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 643, "total_ms": 1274.5, "tps": 93.5, "cost_mean": 0.0000585, "out_tokens": 59 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 602.5, "total_ms": 5739.5, "tps": 147.10000000000002, "cost_mean": 0.00047444999999999996, "out_tokens": 755.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1889, "total_ms": 2082, "tps": 50.95, "cost_mean": 0.008284799999999998, "out_tokens": 8 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 0.5, "score": 0.9585, "judged": 4, "ttft_ms": 533, "total_ms": 1926.5, "tps": 133.1, "cost_mean": 0.00012075, "out_tokens": 185 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 541.5, "total_ms": 2547, "tps": 125.89999999999999, "cost_mean": 0.00016455, "out_tokens": 251 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 611, "total_ms": 1588, "tps": 119.80000000000001, "cost_mean": 0.0000816, "out_tokens": 117 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 0, "score": 0.667, "judged": null, "ttft_ms": 694, "total_ms": 974.5, "tps": 61.4, "cost_mean": 0.000028800000000000002, "out_tokens": 16.5 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 3, "ttft_ms": 527.5, "total_ms": 2795, "tps": 105.5, "cost_mean": 0.00017189999999999998, "out_tokens": 228.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 532.5, "total_ms": 916.5, "tps": 99, "cost_mean": 0.0000447, "out_tokens": 38 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 0.5, "score": 0.75, "judged": null, "ttft_ms": 478, "total_ms": 7312, "tps": 135.25, "cost_mean": 0.0005646, "out_tokens": 920.5 } } }, { "model": "gpt-5.4-nano", "provider": "openai", "runs": 22, "errors": 0, "pass_rate": 0.8636363636363636, "mean_score": 0.9166363636363637, "judged_mean": 4, "ttft_median_ms": 548.5, "total_median_ms": 1727, "tps_median": 134.85000000000002, "cost_total": 0.007665850000000001, "cost_mean": 0.00034844772727272733, "in_tokens": 113620, "out_tokens": 3805, "reasoning_tokens": 0, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 587.5, "total_ms": 1476.5, "tps": 106.9, "cost_mean": 0.00020625, "out_tokens": 85 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 587.5, "total_ms": 1126.5, "tps": 127.95, "cost_mean": 0.00011415000000000001, "out_tokens": 67 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 539, "total_ms": 3506, "tps": 199, "cost_mean": 0.000781125, "out_tokens": 602.5 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 500.5, "total_ms": 776, "tps": 40.2, "cost_mean": 0.00114635, "out_tokens": 11 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 658.5, "total_ms": 1926, "tps": 162.25, "cost_mean": 0.00026967500000000004, "out_tokens": 205.5 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 4.5, "ttft_ms": 510, "total_ms": 2368, "tps": 134.85000000000002, "cost_mean": 0.000331525, "out_tokens": 250.5 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 609, "total_ms": 1925.5, "tps": 123.5, "cost_mean": 0.000213125, "out_tokens": 158.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 0, "score": 0.333, "judged": null, "ttft_ms": 511, "total_ms": 824, "tps": 63, "cost_mean": 0.00004875, "out_tokens": 19 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 2.5, "ttft_ms": 541, "total_ms": 2299.5, "tps": 151.64999999999998, "cost_mean": 0.00037995, "out_tokens": 267 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 595, "total_ms": 928, "tps": 123.1, "cost_mean": 0.00008145, "out_tokens": 41 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 0.5, "score": 0.75, "judged": null, "ttft_ms": 643.5, "total_ms": 1831, "tps": 127.95, "cost_mean": 0.00026057500000000003, "out_tokens": 195.5 } } }, { "model": "claude-fable-5", "provider": "anthropic", "runs": 22, "errors": 0, "pass_rate": 0.9090909090909091, "mean_score": 0.9090909090909091, "judged_mean": 5, "ttft_median_ms": 1763.5, "total_median_ms": 2839.5, "tps_median": 111.85, "cost_total": 2.0221299999999993, "cost_mean": 0.09191499999999997, "in_tokens": 168738, "out_tokens": 6695, "reasoning_tokens": 1105, "by_category": { "document_analysis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 955.5, "total_ms": 1992.5, "tps": 125.5, "cost_mean": 0.01668, "out_tokens": 130 }, "json_generation": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1653.5, "total_ms": 2755, "tps": 104.45, "cost_mean": 0.01069, "out_tokens": 115 }, "complex_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 4880, "total_ms": 8848.5, "tps": 194.8, "cost_mean": 0.04102, "out_tokens": 774 }, "long_context": { "runs": 2, "errors": 0, "pass_rate": 0, "score": 0, "judged": null, "ttft_ms": null, "total_ms": 1710, "tps": null, "cost_mean": 0.80911, "out_tokens": 0 }, "english_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1876.5, "total_ms": 8400, "tps": 77.25, "cost_mean": 0.02622, "out_tokens": 504 }, "french_writing": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 1846.5, "total_ms": 9302.5, "tps": 76.55, "cost_mean": 0.03007, "out_tokens": 569 }, "programming": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1488.5, "total_ms": 2813, "tps": 121.25, "cost_mean": 0.009225, "out_tokens": 160.5 }, "simple_reasoning": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1527.5, "total_ms": 2116, "tps": 32.349999999999994, "cost_mean": 0.00274, "out_tokens": 19 }, "knowledge_synthesis": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": 5, "ttft_ms": 3075, "total_ms": 8038.5, "tps": 114.44999999999999, "cost_mean": 0.031915, "out_tokens": 568.5 }, "tool_calling": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 1831.5, "total_ms": 2653.5, "tps": 171.9, "cost_mean": 0.013615, "out_tokens": 134.5 }, "mathematics": { "runs": 2, "errors": 0, "pass_rate": 1, "score": 1, "judged": null, "ttft_ms": 2058.5, "total_ms": 4882.5, "tps": 109.55, "cost_mean": 0.01978, "out_tokens": 373 } } } ], "discovery": [ { "provider": "openai", "id": "babbage-002", "category": "legacy-completion", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category legacy-completion: not a general text model, not probed (inappropriate request type)" ], "created": "2023-08-21" }, { "provider": "openai", "id": "chat-latest", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 5, "cached": 0.5, "output": 30 }, "probes": { "text": { "status": "ok", "ms": 802 }, "streaming": { "status": "ok", "ms": 916 }, "tool_use": { "status": "ok", "ms": 829 }, "structured_output": { "status": "ok", "ms": 993 }, "vision": { "status": "ok", "ms": 948 }, "reasoning_control": { "status": "ok", "ms": 896 } }, "notes": [], "created": "2026-05-02" }, { "provider": "openai", "id": "chatgpt-image-latest", "category": "image", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category image: not a general text model, not probed (inappropriate request type)" ], "created": "2025-12-16" }, { "provider": "openai", "id": "davinci-002", "category": "legacy-completion", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category legacy-completion: not a general text model, not probed (inappropriate request type)" ], "created": "2023-08-21" }, { "provider": "openai", "id": "gpt-3.5-turbo", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": null, "probes": { "text": { "status": "unavailable", "ms": 201, "detail": "404 404 The model `gpt-3.5-turbo` does not exist or you do not have access to it." } }, "notes": [], "created": "2023-02-28" }, { "provider": "openai", "id": "gpt-3.5-turbo-0125", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": null, "probes": { "text": { "status": "unavailable", "ms": 224, "detail": "404 404 The model `gpt-3.5-turbo-0125` does not exist or you do not have access to it." } }, "notes": [], "created": "2024-01-23" }, { "provider": "openai", "id": "gpt-3.5-turbo-1106", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": null, "probes": { "text": { "status": "unavailable", "ms": 198, "detail": "404 404 The model `gpt-3.5-turbo-1106` does not exist or you do not have access to it." } }, "notes": [], "created": "2023-11-02" }, { "provider": "openai", "id": "gpt-3.5-turbo-16k", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": null, "probes": { "text": { "status": "unavailable", "ms": 267, "detail": "404 404 The model `gpt-3.5-turbo-16k` does not exist or you do not have access to it." } }, "notes": [], "created": "2023-05-10" }, { "provider": "openai", "id": "gpt-3.5-turbo-instruct", "category": "legacy-completion", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category legacy-completion: not a general text model, not probed (inappropriate request type)" ], "created": "2023-08-24" }, { "provider": "openai", "id": "gpt-3.5-turbo-instruct-0914", "category": "legacy-completion", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category legacy-completion: not a general text model, not probed (inappropriate request type)" ], "created": "2023-09-07" }, { "provider": "openai", "id": "gpt-4", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": null, "probes": { "text": { "status": "unavailable", "ms": 195, "detail": "404 404 The model `gpt-4` does not exist or you do not have access to it." } }, "notes": [], "created": "2023-06-27" }, { "provider": "openai", "id": "gpt-4-0613", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": null, "probes": { "text": { "status": "unavailable", "ms": 155, "detail": "404 404 The model `gpt-4-0613` does not exist or you do not have access to it." } }, "notes": [], "created": "2023-06-12" }, { "provider": "openai", "id": "gpt-4-turbo", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": null, "probes": { "text": { "status": "unavailable", "ms": 143, "detail": "404 404 The model `gpt-4-turbo` does not exist or you do not have access to it." } }, "notes": [], "created": "2024-04-05" }, { "provider": "openai", "id": "gpt-4-turbo-2024-04-09", "category": "text", "aliasOf": "gpt-4-turbo", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-4-turbo; not probed separately" ], "created": "2024-04-08" }, { "provider": "openai", "id": "gpt-4.1", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 2, "cached": 0.5, "output": 8 }, "probes": { "text": { "status": "ok", "ms": 650 }, "streaming": { "status": "ok", "ms": 547 }, "tool_use": { "status": "ok", "ms": 607 }, "structured_output": { "status": "ok", "ms": 612 }, "vision": { "status": "ok", "ms": 568 }, "reasoning_control": { "status": "ok", "ms": 541 } }, "notes": [], "created": "2025-04-10" }, { "provider": "openai", "id": "gpt-4.1-2025-04-14", "category": "text", "aliasOf": "gpt-4.1", "verdict": "untested", "pricing": { "input": 2, "cached": 0.5, "output": 8 }, "probes": {}, "notes": [ "dated snapshot of gpt-4.1; not probed separately" ], "created": "2025-04-10" }, { "provider": "openai", "id": "gpt-4.1-mini", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 0.4, "cached": 0.1, "output": 1.6 }, "probes": { "text": { "status": "ok", "ms": 565 }, "streaming": { "status": "ok", "ms": 824 }, "tool_use": { "status": "ok", "ms": 634 }, "structured_output": { "status": "ok", "ms": 588 }, "vision": { "status": "unsupported", "ms": 686, "detail": "answer: Pink" }, "reasoning_control": { "status": "ok", "ms": 430 } }, "notes": [], "created": "2025-04-10" }, { "provider": "openai", "id": "gpt-4.1-mini-2025-04-14", "category": "text", "aliasOf": "gpt-4.1-mini", "verdict": "untested", "pricing": { "input": 0.4, "cached": 0.1, "output": 1.6 }, "probes": {}, "notes": [ "dated snapshot of gpt-4.1-mini; not probed separately" ], "created": "2025-04-10" }, { "provider": "openai", "id": "gpt-4.1-nano", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 0.1, "cached": 0.025, "output": 0.4 }, "probes": { "text": { "status": "unavailable", "ms": 568, "detail": "404 404 The model `gpt-4.1-nano` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations" } }, "notes": [], "created": "2025-04-10" }, { "provider": "openai", "id": "gpt-4.1-nano-2025-04-14", "category": "text", "aliasOf": "gpt-4.1-nano", "verdict": "untested", "pricing": { "input": 0.1, "cached": 0.025, "output": 0.4 }, "probes": {}, "notes": [ "dated snapshot of gpt-4.1-nano; not probed separately" ], "created": "2025-04-10" }, { "provider": "openai", "id": "gpt-4o", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 2.5, "cached": 1.25, "output": 10 }, "probes": { "text": { "status": "ok", "ms": 650 }, "streaming": { "status": "ok", "ms": 710 }, "tool_use": { "status": "ok", "ms": 606 }, "structured_output": { "status": "ok", "ms": 727 }, "vision": { "status": "ok", "ms": 666 }, "reasoning_control": { "status": "ok", "ms": 566 } }, "notes": [], "created": "2024-05-10" }, { "provider": "openai", "id": "gpt-4o-2024-05-13", "category": "text", "aliasOf": "gpt-4o", "verdict": "untested", "pricing": { "input": 2.5, "cached": 1.25, "output": 10 }, "probes": {}, "notes": [ "dated snapshot of gpt-4o; not probed separately" ], "created": "2024-05-10" }, { "provider": "openai", "id": "gpt-4o-2024-08-06", "category": "text", "aliasOf": "gpt-4o", "verdict": "untested", "pricing": { "input": 2.5, "cached": 1.25, "output": 10 }, "probes": {}, "notes": [ "dated snapshot of gpt-4o; not probed separately" ], "created": "2024-08-04" }, { "provider": "openai", "id": "gpt-4o-2024-11-20", "category": "text", "aliasOf": "gpt-4o", "verdict": "untested", "pricing": { "input": 2.5, "cached": 1.25, "output": 10 }, "probes": {}, "notes": [ "dated snapshot of gpt-4o; not probed separately" ], "created": "2025-02-12" }, { "provider": "openai", "id": "gpt-4o-mini", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 0.15, "cached": 0.075, "output": 0.6 }, "probes": { "text": { "status": "ok", "ms": 495 }, "streaming": { "status": "ok", "ms": 589 }, "tool_use": { "status": "ok", "ms": 620 }, "structured_output": { "status": "ok", "ms": 599 }, "vision": { "status": "ok", "ms": 662 }, "reasoning_control": { "status": "ok", "ms": 1747 } }, "notes": [], "created": "2024-07-16" }, { "provider": "openai", "id": "gpt-4o-mini-2024-07-18", "category": "text", "aliasOf": "gpt-4o-mini", "verdict": "untested", "pricing": { "input": 0.15, "cached": 0.075, "output": 0.6 }, "probes": {}, "notes": [ "dated snapshot of gpt-4o-mini; not probed separately" ], "created": "2024-07-16" }, { "provider": "openai", "id": "gpt-4o-mini-search-preview", "category": "search", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category search: not a general text model, not probed (inappropriate request type)" ], "created": "2025-03-07" }, { "provider": "openai", "id": "gpt-4o-mini-search-preview-2025-03-11", "category": "search", "aliasOf": "gpt-4o-mini-search-preview", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-4o-mini-search-preview; not probed separately" ], "created": "2025-03-07" }, { "provider": "openai", "id": "gpt-4o-mini-transcribe", "category": "audio", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category audio: not a general text model, not probed (inappropriate request type)" ], "created": "2025-03-15" }, { "provider": "openai", "id": "gpt-4o-mini-transcribe-2025-03-20", "category": "audio", "aliasOf": "gpt-4o-mini-transcribe", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-4o-mini-transcribe; not probed separately" ], "created": "2025-12-13" }, { "provider": "openai", "id": "gpt-4o-mini-transcribe-2025-12-15", "category": "audio", "aliasOf": "gpt-4o-mini-transcribe", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-4o-mini-transcribe; not probed separately" ], "created": "2025-12-13" }, { "provider": "openai", "id": "gpt-4o-mini-tts", "category": "audio", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category audio: not a general text model, not probed (inappropriate request type)" ], "created": "2025-03-19" }, { "provider": "openai", "id": "gpt-4o-mini-tts-2025-03-20", "category": "audio", "aliasOf": "gpt-4o-mini-tts", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-4o-mini-tts; not probed separately" ], "created": "2025-12-13" }, { "provider": "openai", "id": "gpt-4o-mini-tts-2025-12-15", "category": "audio", "aliasOf": "gpt-4o-mini-tts", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-4o-mini-tts; not probed separately" ], "created": "2025-12-13" }, { "provider": "openai", "id": "gpt-4o-search-preview", "category": "search", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category search: not a general text model, not probed (inappropriate request type)" ], "created": "2026-02-24" }, { "provider": "openai", "id": "gpt-4o-search-preview-2025-03-11", "category": "search", "aliasOf": "gpt-4o-search-preview", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-4o-search-preview; not probed separately" ], "created": "2026-02-24" }, { "provider": "openai", "id": "gpt-4o-transcribe", "category": "audio", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category audio: not a general text model, not probed (inappropriate request type)" ], "created": "2025-03-15" }, { "provider": "openai", "id": "gpt-4o-transcribe-diarize", "category": "audio", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category audio: not a general text model, not probed (inappropriate request type)" ], "created": "2025-06-24" }, { "provider": "openai", "id": "gpt-5", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 1.25, "cached": 0.125, "output": 10 }, "probes": { "text": { "status": "ok", "ms": 2030 }, "streaming": { "status": "ok", "ms": 1210 }, "tool_use": { "status": "ok", "ms": 986 }, "structured_output": { "status": "ok", "ms": 8394 }, "vision": { "status": "ok", "ms": 1105 }, "reasoning_control": { "status": "ok", "ms": 907 } }, "notes": [], "created": "2025-08-05" }, { "provider": "openai", "id": "gpt-5-2025-08-07", "category": "text", "aliasOf": "gpt-5", "verdict": "untested", "pricing": { "input": 1.25, "cached": 0.125, "output": 10 }, "probes": {}, "notes": [ "dated snapshot of gpt-5; not probed separately" ], "created": "2025-08-01" }, { "provider": "openai", "id": "gpt-5-chat-latest", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 1.25, "cached": 0.125, "output": 10 }, "probes": { "text": { "status": "unavailable", "ms": 163, "detail": "404 404 The model `gpt-5-chat-latest` does not exist or you do not have access to it." } }, "notes": [], "created": "2025-08-01" }, { "provider": "openai", "id": "gpt-5-codex", "category": "text-specialized", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 1.25, "cached": 0.125, "output": 10 }, "probes": { "text": { "status": "unavailable", "ms": 147, "detail": "404 404 The model `gpt-5-codex` does not exist or you do not have access to it." } }, "notes": [], "created": "2025-09-10" }, { "provider": "openai", "id": "gpt-5-mini", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 0.25, "cached": 0.025, "output": 2 }, "probes": { "text": { "status": "ok", "ms": 1484 }, "streaming": { "status": "ok", "ms": 1489 }, "tool_use": { "status": "ok", "ms": 1556 }, "structured_output": { "status": "ok", "ms": 5095 }, "vision": { "status": "ok", "ms": 1442 }, "reasoning_control": { "status": "ok", "ms": 1092 } }, "notes": [], "created": "2025-08-05" }, { "provider": "openai", "id": "gpt-5-mini-2025-08-07", "category": "text", "aliasOf": "gpt-5-mini", "verdict": "untested", "pricing": { "input": 0.25, "cached": 0.025, "output": 2 }, "probes": {}, "notes": [ "dated snapshot of gpt-5-mini; not probed separately" ], "created": "2025-08-05" }, { "provider": "openai", "id": "gpt-5-nano", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 0.05, "cached": 0.005, "output": 0.4 }, "probes": { "text": { "status": "ok", "ms": 1273 }, "streaming": { "status": "ok", "ms": 1103 }, "tool_use": { "status": "ok", "ms": 966 }, "structured_output": { "status": "ok", "ms": 4184 }, "vision": { "status": "ok", "ms": 898 }, "reasoning_control": { "status": "ok", "ms": 981 } }, "notes": [], "created": "2025-08-05" }, { "provider": "openai", "id": "gpt-5-nano-2025-08-07", "category": "text", "aliasOf": "gpt-5-nano", "verdict": "untested", "pricing": { "input": 0.05, "cached": 0.005, "output": 0.4 }, "probes": {}, "notes": [ "dated snapshot of gpt-5-nano; not probed separately" ], "created": "2025-08-05" }, { "provider": "openai", "id": "gpt-5-pro", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 15, "cached": null, "output": 120 }, "probes": { "text": { "status": "ok", "ms": 12269 }, "streaming": { "status": "ok", "ms": 8281 }, "tool_use": { "status": "ok", "ms": 15045 }, "structured_output": { "status": "ok", "ms": 70315 }, "vision": { "status": "ok", "ms": 157754 }, "reasoning_control": { "status": "unsupported", "ms": 135, "detail": "400 400 Unsupported value: 'low' is not supported with the 'gpt-5-pro' model. Supported values are: 'high'." } }, "notes": [ "pro/expensive tier: only text + structured_output probed; benchmark skipped for cost ($30-$150/M input)" ], "created": "2025-10-03" }, { "provider": "openai", "id": "gpt-5-pro-2025-10-06", "category": "text", "aliasOf": "gpt-5-pro", "verdict": "untested", "pricing": { "input": 15, "cached": null, "output": 120 }, "probes": {}, "notes": [ "dated snapshot of gpt-5-pro; not probed separately" ], "created": "2025-10-03" }, { "provider": "openai", "id": "gpt-5-search-api", "category": "search", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category search: not a general text model, not probed (inappropriate request type)" ], "created": "2025-10-03" }, { "provider": "openai", "id": "gpt-5-search-api-2025-10-14", "category": "search", "aliasOf": "gpt-5-search-api", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-5-search-api; not probed separately" ], "created": "2025-10-09" }, { "provider": "openai", "id": "gpt-5.1", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 1.25, "cached": 0.125, "output": 10 }, "probes": { "text": { "status": "ok", "ms": 667 }, "streaming": { "status": "ok", "ms": 708 }, "tool_use": { "status": "ok", "ms": 703 }, "structured_output": { "status": "ok", "ms": 757 }, "vision": { "status": "ok", "ms": 894 }, "reasoning_control": { "status": "ok", "ms": 672 } }, "notes": [], "created": "2025-11-10" }, { "provider": "openai", "id": "gpt-5.1-2025-11-13", "category": "text", "aliasOf": "gpt-5.1", "verdict": "untested", "pricing": { "input": 1.25, "cached": 0.125, "output": 10 }, "probes": {}, "notes": [ "dated snapshot of gpt-5.1; not probed separately" ], "created": "2025-11-10" }, { "provider": "openai", "id": "gpt-5.1-chat-latest", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 1.25, "cached": 0.125, "output": 10 }, "probes": { "text": { "status": "unavailable", "ms": 153, "detail": "404 404 The model `gpt-5.1-chat-latest` does not exist or you do not have access to it." } }, "notes": [], "created": "2025-11-07" }, { "provider": "openai", "id": "gpt-5.1-codex", "category": "text-specialized", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 1.25, "cached": 0.125, "output": 10 }, "probes": { "text": { "status": "unavailable", "ms": 151, "detail": "404 404 The model `gpt-5.1-codex` does not exist or you do not have access to it." } }, "notes": [], "created": "2025-11-12" }, { "provider": "openai", "id": "gpt-5.1-codex-max", "category": "text-specialized", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 1.25, "cached": 0.125, "output": 10 }, "probes": { "text": { "status": "unavailable", "ms": 135, "detail": "404 404 The model `gpt-5.1-codex-max` does not exist or you do not have access to it." } }, "notes": [], "created": "2025-11-20" }, { "provider": "openai", "id": "gpt-5.1-codex-mini", "category": "text-specialized", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 0.25, "cached": 0.025, "output": 2 }, "probes": { "text": { "status": "unavailable", "ms": 134, "detail": "404 404 The model `gpt-5.1-codex-mini` does not exist or you do not have access to it." } }, "notes": [], "created": "2025-11-13" }, { "provider": "openai", "id": "gpt-5.2", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 1.75, "cached": 0.175, "output": 14 }, "probes": { "text": { "status": "ok", "ms": 702 }, "streaming": { "status": "ok", "ms": 918 }, "tool_use": { "status": "ok", "ms": 742 }, "structured_output": { "status": "ok", "ms": 942 }, "vision": { "status": "ok", "ms": 766 }, "reasoning_control": { "status": "ok", "ms": 752 } }, "notes": [], "created": "2025-12-09" }, { "provider": "openai", "id": "gpt-5.2-2025-12-11", "category": "text", "aliasOf": "gpt-5.2", "verdict": "untested", "pricing": { "input": 1.75, "cached": 0.175, "output": 14 }, "probes": {}, "notes": [ "dated snapshot of gpt-5.2; not probed separately" ], "created": "2025-12-09" }, { "provider": "openai", "id": "gpt-5.2-chat-latest", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 1.75, "cached": 0.175, "output": 14 }, "probes": { "text": { "status": "unavailable", "ms": 213, "detail": "404 404 The model `gpt-5.2-chat-latest` does not exist or you do not have access to it." } }, "notes": [], "created": "2025-12-10" }, { "provider": "openai", "id": "gpt-5.2-codex", "category": "text-specialized", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 1.75, "cached": 0.175, "output": 14 }, "probes": { "text": { "status": "unavailable", "ms": 166, "detail": "404 404 The model `gpt-5.2-codex` does not exist or you do not have access to it." } }, "notes": [], "created": "2025-12-19" }, { "provider": "openai", "id": "gpt-5.2-pro", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 21, "cached": null, "output": 168 }, "probes": { "text": { "status": "ok", "ms": 4335 }, "streaming": { "status": "ok", "ms": 4276 }, "tool_use": { "status": "ok", "ms": 7643 }, "structured_output": { "status": "ok", "ms": 5338 }, "vision": { "status": "ok", "ms": 5725 }, "reasoning_control": { "status": "unsupported", "ms": 141, "detail": "400 400 Unsupported value: 'low' is not supported with the 'gpt-5.2-pro' model. Supported values are: 'medium', 'high', and 'xhigh'." } }, "notes": [ "pro/expensive tier: only text + structured_output probed; benchmark skipped for cost ($30-$150/M input)" ], "created": "2025-12-10" }, { "provider": "openai", "id": "gpt-5.2-pro-2025-12-11", "category": "text", "aliasOf": "gpt-5.2-pro", "verdict": "untested", "pricing": { "input": 21, "cached": null, "output": 168 }, "probes": {}, "notes": [ "dated snapshot of gpt-5.2-pro; not probed separately" ], "created": "2025-12-10" }, { "provider": "openai", "id": "gpt-5.3-chat-latest", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 1.75, "cached": 0.175, "output": 14 }, "probes": { "text": { "status": "unavailable", "ms": 199, "detail": "404 404 The model `gpt-5.3-chat-latest` does not exist or you do not have access to it." } }, "notes": [], "created": "2026-02-27" }, { "provider": "openai", "id": "gpt-5.3-codex", "category": "text-specialized", "aliasOf": null, "verdict": "tested", "pricing": { "input": 1.75, "cached": 0.175, "output": 14 }, "probes": { "text": { "status": "ok", "ms": 904 }, "streaming": { "status": "ok", "ms": 809 }, "tool_use": { "status": "ok", "ms": 885 }, "structured_output": { "status": "ok", "ms": 942 }, "vision": { "status": "ok", "ms": 925 }, "reasoning_control": { "status": "ok", "ms": 844 } }, "notes": [], "created": "2026-02-08" }, { "provider": "openai", "id": "gpt-5.4", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 2.5, "cached": 0.25, "output": 15 }, "probes": { "text": { "status": "ok", "ms": 720 }, "streaming": { "status": "ok", "ms": 955 }, "tool_use": { "status": "ok", "ms": 1079 }, "structured_output": { "status": "ok", "ms": 986 }, "vision": { "status": "ok", "ms": 767 }, "reasoning_control": { "status": "ok", "ms": 883 } }, "notes": [], "created": "2026-03-05" }, { "provider": "openai", "id": "gpt-5.4-2026-03-05", "category": "text", "aliasOf": "gpt-5.4", "verdict": "untested", "pricing": { "input": 2.5, "cached": 0.25, "output": 15 }, "probes": {}, "notes": [ "dated snapshot of gpt-5.4; not probed separately" ], "created": "2026-03-04" }, { "provider": "openai", "id": "gpt-5.4-mini", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 0.75, "cached": 0.075, "output": 4.5 }, "probes": { "text": { "status": "ok", "ms": 706 }, "streaming": { "status": "ok", "ms": 584 }, "tool_use": { "status": "ok", "ms": 851 }, "structured_output": { "status": "ok", "ms": 824 }, "vision": { "status": "ok", "ms": 623 }, "reasoning_control": { "status": "ok", "ms": 546 } }, "notes": [], "created": "2026-03-14" }, { "provider": "openai", "id": "gpt-5.4-mini-2026-03-17", "category": "text", "aliasOf": "gpt-5.4-mini", "verdict": "untested", "pricing": { "input": 0.75, "cached": 0.075, "output": 4.5 }, "probes": {}, "notes": [ "dated snapshot of gpt-5.4-mini; not probed separately" ], "created": "2026-03-14" }, { "provider": "openai", "id": "gpt-5.4-nano", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 0.2, "cached": 0.02, "output": 1.25 }, "probes": { "text": { "status": "ok", "ms": 722 }, "streaming": { "status": "ok", "ms": 704 }, "tool_use": { "status": "ok", "ms": 596 }, "structured_output": { "status": "ok", "ms": 652 }, "vision": { "status": "ok", "ms": 521 }, "reasoning_control": { "status": "ok", "ms": 566 } }, "notes": [], "created": "2026-03-14" }, { "provider": "openai", "id": "gpt-5.4-nano-2026-03-17", "category": "text", "aliasOf": "gpt-5.4-nano", "verdict": "untested", "pricing": { "input": 0.2, "cached": 0.02, "output": 1.25 }, "probes": {}, "notes": [ "dated snapshot of gpt-5.4-nano; not probed separately" ], "created": "2026-03-14" }, { "provider": "openai", "id": "gpt-5.4-pro", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 30, "cached": null, "output": 180 }, "probes": { "text": { "status": "ok", "ms": 5054 }, "streaming": { "status": "ok", "ms": 6620 }, "tool_use": { "status": "ok", "ms": 13017 }, "structured_output": { "status": "ok", "ms": 22285 }, "vision": { "status": "ok", "ms": 7970 }, "reasoning_control": { "status": "unsupported", "ms": 149, "detail": "400 400 Unsupported value: 'low' is not supported with the 'gpt-5.4-pro' model. Supported values are: 'medium', 'high', and 'xhigh'." } }, "notes": [ "pro/expensive tier: only text + structured_output probed; benchmark skipped for cost ($30-$150/M input)" ], "created": "2026-03-04" }, { "provider": "openai", "id": "gpt-5.4-pro-2026-03-05", "category": "text", "aliasOf": "gpt-5.4-pro", "verdict": "untested", "pricing": { "input": 30, "cached": null, "output": 180 }, "probes": {}, "notes": [ "dated snapshot of gpt-5.4-pro; not probed separately" ], "created": "2026-03-04" }, { "provider": "openai", "id": "gpt-5.5", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 5, "cached": 0.5, "output": 30 }, "probes": { "text": { "status": "ok", "ms": 907 }, "streaming": { "status": "ok", "ms": 1085 }, "tool_use": { "status": "ok", "ms": 972 }, "structured_output": { "status": "ok", "ms": 1671 }, "vision": { "status": "ok", "ms": 1283 }, "reasoning_control": { "status": "ok", "ms": 1074 } }, "notes": [], "created": "2026-04-22" }, { "provider": "openai", "id": "gpt-5.5-2026-04-23", "category": "text", "aliasOf": "gpt-5.5", "verdict": "untested", "pricing": { "input": 5, "cached": 0.5, "output": 30 }, "probes": {}, "notes": [ "dated snapshot of gpt-5.5; not probed separately" ], "created": "2026-04-22" }, { "provider": "openai", "id": "gpt-5.5-pro", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 30, "cached": null, "output": 180 }, "probes": { "text": { "status": "ok", "ms": 35646 }, "streaming": { "status": "ok", "ms": 3986 }, "tool_use": { "status": "ok", "ms": 5643 }, "structured_output": { "status": "ok", "ms": 10706 }, "vision": { "status": "ok", "ms": 4232 }, "reasoning_control": { "status": "unsupported", "ms": 264, "detail": "400 400 Unsupported value: 'low' is not supported with the 'gpt-5.5-pro' model. Supported values are: 'medium', 'high', and 'xhigh'." } }, "notes": [ "pro/expensive tier: only text + structured_output probed; benchmark skipped for cost ($30-$150/M input)" ], "created": "2026-04-22" }, { "provider": "openai", "id": "gpt-5.5-pro-2026-04-23", "category": "text", "aliasOf": "gpt-5.5-pro", "verdict": "untested", "pricing": { "input": 30, "cached": null, "output": 180 }, "probes": {}, "notes": [ "dated snapshot of gpt-5.5-pro; not probed separately" ], "created": "2026-04-22" }, { "provider": "openai", "id": "gpt-5.6-luna", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 0.2, "cached": 0.02, "output": 1.2 }, "probes": { "text": { "status": "ok", "ms": 709 }, "streaming": { "status": "ok", "ms": 846 }, "tool_use": { "status": "ok", "ms": 791 }, "structured_output": { "status": "ok", "ms": 1606 }, "vision": { "status": "ok", "ms": 765 }, "reasoning_control": { "status": "ok", "ms": 626 } }, "notes": [], "created": "2026-06-23" }, { "provider": "openai", "id": "gpt-5.6-sol", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 4, "cached": 0.4, "output": 20 }, "probes": { "text": { "status": "ok", "ms": 824 }, "streaming": { "status": "ok", "ms": 761 }, "tool_use": { "status": "ok", "ms": 794 }, "structured_output": { "status": "ok", "ms": 801 }, "vision": { "status": "ok", "ms": 1467 }, "reasoning_control": { "status": "ok", "ms": 1011 } }, "notes": [], "created": "2026-06-23" }, { "provider": "openai", "id": "gpt-5.6-terra", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 2, "cached": 0.2, "output": 12 }, "probes": { "text": { "status": "ok", "ms": 645 }, "streaming": { "status": "ok", "ms": 809 }, "tool_use": { "status": "ok", "ms": 876 }, "structured_output": { "status": "ok", "ms": 917 }, "vision": { "status": "ok", "ms": 698 }, "reasoning_control": { "status": "ok", "ms": 753 } }, "notes": [], "created": "2026-06-23" }, { "provider": "openai", "id": "gpt-6-astra", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 10, "cached": 1, "output": 50 }, "probes": { "text": { "status": "ok", "ms": 2008 }, "streaming": { "status": "ok", "ms": 1186 }, "tool_use": { "status": "ok", "ms": 1513 }, "structured_output": { "status": "ok", "ms": 2239 }, "vision": { "status": "ok", "ms": 1071 }, "reasoning_control": { "status": "ok", "ms": 980 } }, "notes": [], "created": "2026-08-27" }, { "provider": "openai", "id": "gpt-6-luna", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 0.1, "cached": 0.01, "output": 0.5 }, "probes": { "text": { "status": "ok", "ms": 862 }, "streaming": { "status": "ok", "ms": 881 }, "tool_use": { "status": "ok", "ms": 759 }, "structured_output": { "status": "ok", "ms": 1453 }, "vision": { "status": "ok", "ms": 1281 }, "reasoning_control": { "status": "ok", "ms": 596 } }, "notes": [], "created": "2026-09-14" }, { "provider": "openai", "id": "gpt-6-sol", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 2, "cached": 0.2, "output": 10 }, "probes": { "text": { "status": "ok", "ms": 756 }, "streaming": { "status": "ok", "ms": 825 }, "tool_use": { "status": "ok", "ms": 789 }, "structured_output": { "status": "ok", "ms": 1422 }, "vision": { "status": "ok", "ms": 1228 }, "reasoning_control": { "status": "ok", "ms": 703 } }, "notes": [], "created": "2026-09-14" }, { "provider": "openai", "id": "gpt-6.1-sol", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 2, "cached": 0.1, "output": 10 }, "probes": { "text": { "status": "ok", "ms": 1034 }, "streaming": { "status": "ok", "ms": 1033 }, "tool_use": { "status": "ok", "ms": 1140 }, "structured_output": { "status": "ok", "ms": 2452 }, "vision": { "status": "ok", "ms": 923 }, "reasoning_control": { "status": "ok", "ms": 1056 } }, "notes": [], "created": "2026-09-27" }, { "provider": "openai", "id": "gpt-6.1-terra", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 1, "cached": 0.05, "output": 5 }, "probes": { "text": { "status": "ok", "ms": 800 }, "streaming": { "status": "ok", "ms": 962 }, "tool_use": { "status": "ok", "ms": 1054 }, "structured_output": { "status": "ok", "ms": 1373 }, "vision": { "status": "ok", "ms": 1379 }, "reasoning_control": { "status": "ok", "ms": 977 } }, "notes": [], "created": "2026-10-13" }, { "provider": "openai", "id": "gpt-audio", "category": "audio", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category audio: not a general text model, not probed (inappropriate request type)" ], "created": "2025-08-28" }, { "provider": "openai", "id": "gpt-audio-1.5", "category": "audio", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category audio: not a general text model, not probed (inappropriate request type)" ], "created": "2026-02-20" }, { "provider": "openai", "id": "gpt-audio-2025-08-28", "category": "audio", "aliasOf": "gpt-audio", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-audio; not probed separately" ], "created": "2025-08-27" }, { "provider": "openai", "id": "gpt-audio-mini", "category": "audio", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category audio: not a general text model, not probed (inappropriate request type)" ], "created": "2025-10-03" }, { "provider": "openai", "id": "gpt-audio-mini-2025-10-06", "category": "audio", "aliasOf": "gpt-audio-mini", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-audio-mini; not probed separately" ], "created": "2025-10-03" }, { "provider": "openai", "id": "gpt-audio-mini-2025-12-15", "category": "audio", "aliasOf": "gpt-audio-mini", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-audio-mini; not probed separately" ], "created": "2025-12-15" }, { "provider": "openai", "id": "gpt-image-1", "category": "image", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category image: not a general text model, not probed (inappropriate request type)" ], "created": "2025-04-24" }, { "provider": "openai", "id": "gpt-image-1-mini", "category": "image", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category image: not a general text model, not probed (inappropriate request type)" ], "created": "2025-09-26" }, { "provider": "openai", "id": "gpt-image-1.5", "category": "image", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category image: not a general text model, not probed (inappropriate request type)" ], "created": "2025-11-25" }, { "provider": "openai", "id": "gpt-image-2", "category": "image", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category image: not a general text model, not probed (inappropriate request type)" ], "created": "2026-04-17" }, { "provider": "openai", "id": "gpt-image-2-2026-04-21", "category": "image", "aliasOf": "gpt-image-2", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-image-2; not probed separately" ], "created": "2026-04-17" }, { "provider": "openai", "id": "gpt-image-2.5-flare", "category": "image", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category image: not a general text model, not probed (inappropriate request type)" ], "created": "2026-09-04" }, { "provider": "openai", "id": "gpt-image-2.5-flare-2026-09-08", "category": "image", "aliasOf": "gpt-image-2.5-flare", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-image-2.5-flare; not probed separately" ], "created": "2026-09-08" }, { "provider": "openai", "id": "gpt-image-2.5-sunburst", "category": "image", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category image: not a general text model, not probed (inappropriate request type)" ], "created": "2026-09-04" }, { "provider": "openai", "id": "gpt-image-2.5-sunburst-2026-09-08", "category": "image", "aliasOf": "gpt-image-2.5-sunburst", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-image-2.5-sunburst; not probed separately" ], "created": "2026-09-08" }, { "provider": "openai", "id": "gpt-live-1", "category": "realtime", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category realtime: not a general text model, not probed (inappropriate request type)" ], "created": "2026-09-08" }, { "provider": "openai", "id": "gpt-live-transcribe", "category": "realtime", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category realtime: not a general text model, not probed (inappropriate request type)" ], "created": "2026-07-27" }, { "provider": "openai", "id": "gpt-realtime", "category": "realtime", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category realtime: not a general text model, not probed (inappropriate request type)" ], "created": "2025-08-27" }, { "provider": "openai", "id": "gpt-realtime-1.5", "category": "realtime", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category realtime: not a general text model, not probed (inappropriate request type)" ], "created": "2026-02-19" }, { "provider": "openai", "id": "gpt-realtime-2", "category": "realtime", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category realtime: not a general text model, not probed (inappropriate request type)" ], "created": "2026-05-05" }, { "provider": "openai", "id": "gpt-realtime-2.1", "category": "realtime", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category realtime: not a general text model, not probed (inappropriate request type)" ], "created": "2026-06-23" }, { "provider": "openai", "id": "gpt-realtime-2.1-mini", "category": "realtime", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category realtime: not a general text model, not probed (inappropriate request type)" ], "created": "2026-06-23" }, { "provider": "openai", "id": "gpt-realtime-2025-08-28", "category": "realtime", "aliasOf": "gpt-realtime", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-realtime; not probed separately" ], "created": "2025-08-27" }, { "provider": "openai", "id": "gpt-realtime-mini", "category": "realtime", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category realtime: not a general text model, not probed (inappropriate request type)" ], "created": "2025-10-03" }, { "provider": "openai", "id": "gpt-realtime-mini-2025-12-15", "category": "realtime", "aliasOf": "gpt-realtime-mini", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of gpt-realtime-mini; not probed separately" ], "created": "2025-12-13" }, { "provider": "openai", "id": "gpt-realtime-translate", "category": "realtime", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category realtime: not a general text model, not probed (inappropriate request type)" ], "created": "2026-05-05" }, { "provider": "openai", "id": "gpt-realtime-whisper", "category": "realtime", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category realtime: not a general text model, not probed (inappropriate request type)" ], "created": "2026-05-05" }, { "provider": "openai", "id": "gpt-transcribe", "category": "audio", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category audio: not a general text model, not probed (inappropriate request type)" ], "created": "2026-07-27" }, { "provider": "openai", "id": "o1", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 15, "cached": 7.5, "output": 60 }, "probes": { "text": { "status": "unavailable", "ms": 429, "detail": "404 404 The model `o1` does not exist or you do not have access to it." } }, "notes": [], "created": "2024-12-16" }, { "provider": "openai", "id": "o1-2024-12-17", "category": "text", "aliasOf": "o1", "verdict": "untested", "pricing": { "input": 15, "cached": 7.5, "output": 60 }, "probes": {}, "notes": [ "dated snapshot of o1; not probed separately" ], "created": "2024-12-16" }, { "provider": "openai", "id": "o1-pro", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 150, "cached": null, "output": 600 }, "probes": { "text": { "status": "unavailable", "ms": 207, "detail": "404 404 The model `o1-pro` does not exist or you do not have access to it." } }, "notes": [ "pro/expensive tier: only text + structured_output probed; benchmark skipped for cost ($30-$150/M input)" ], "created": "2025-03-17" }, { "provider": "openai", "id": "o1-pro-2025-03-19", "category": "text", "aliasOf": "o1-pro", "verdict": "untested", "pricing": { "input": 150, "cached": null, "output": 600 }, "probes": {}, "notes": [ "dated snapshot of o1-pro; not probed separately" ], "created": "2025-03-17" }, { "provider": "openai", "id": "o3", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 2, "cached": 0.5, "output": 8 }, "probes": { "text": { "status": "ok", "ms": 823 }, "streaming": { "status": "ok", "ms": 1299 }, "tool_use": { "status": "ok", "ms": 1377 }, "structured_output": { "status": "ok", "ms": 1299 }, "vision": { "status": "ok", "ms": 1098 }, "reasoning_control": { "status": "ok", "ms": 679 } }, "notes": [], "created": "2025-04-09" }, { "provider": "openai", "id": "o3-2025-04-16", "category": "text", "aliasOf": "o3", "verdict": "untested", "pricing": { "input": 2, "cached": 0.5, "output": 8 }, "probes": {}, "notes": [ "dated snapshot of o3; not probed separately" ], "created": "2025-04-08" }, { "provider": "openai", "id": "o3-deep-research", "category": "deep-research", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category deep-research: not a general text model, not probed (inappropriate request type)" ], "created": "2025-06-13" }, { "provider": "openai", "id": "o3-deep-research-2025-06-26", "category": "deep-research", "aliasOf": "o3-deep-research", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of o3-deep-research; not probed separately" ], "created": "2025-06-25" }, { "provider": "openai", "id": "o3-mini", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 1.1, "cached": 0.55, "output": 4.4 }, "probes": { "text": { "status": "unavailable", "ms": 186, "detail": "404 404 The model `o3-mini` does not exist or you do not have access to it." } }, "notes": [], "created": "2025-01-17" }, { "provider": "openai", "id": "o3-mini-2025-01-31", "category": "text", "aliasOf": "o3-mini", "verdict": "untested", "pricing": { "input": 1.1, "cached": 0.55, "output": 4.4 }, "probes": {}, "notes": [ "dated snapshot of o3-mini; not probed separately" ], "created": "2025-01-27" }, { "provider": "openai", "id": "o3-pro", "category": "text", "aliasOf": null, "verdict": "tested", "pricing": { "input": 20, "cached": null, "output": 80 }, "probes": { "text": { "status": "ok", "ms": 4414 }, "streaming": { "status": "ok", "ms": 4539 }, "tool_use": { "status": "ok", "ms": 6647 }, "structured_output": { "status": "ok", "ms": 5927 }, "vision": { "status": "ok", "ms": 6121 }, "reasoning_control": { "status": "ok", "ms": 4857 } }, "notes": [ "pro/expensive tier: only text + structured_output probed; benchmark skipped for cost ($30-$150/M input)" ], "created": "2025-05-28" }, { "provider": "openai", "id": "o3-pro-2025-06-10", "category": "text", "aliasOf": "o3-pro", "verdict": "untested", "pricing": { "input": 20, "cached": null, "output": 80 }, "probes": {}, "notes": [ "dated snapshot of o3-pro; not probed separately" ], "created": "2025-06-05" }, { "provider": "openai", "id": "o4-mini", "category": "text", "aliasOf": null, "verdict": "unavailable", "pricing": { "input": 1.1, "cached": 0.275, "output": 4.4 }, "probes": { "text": { "status": "unavailable", "ms": 147, "detail": "404 404 The model `o4-mini` does not exist or you do not have access to it." } }, "notes": [], "created": "2025-04-09" }, { "provider": "openai", "id": "o4-mini-2025-04-16", "category": "text", "aliasOf": "o4-mini", "verdict": "untested", "pricing": { "input": 1.1, "cached": 0.275, "output": 4.4 }, "probes": {}, "notes": [ "dated snapshot of o4-mini; not probed separately" ], "created": "2025-04-08" }, { "provider": "openai", "id": "o4-mini-deep-research", "category": "deep-research", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category deep-research: not a general text model, not probed (inappropriate request type)" ], "created": "2025-06-11" }, { "provider": "openai", "id": "o4-mini-deep-research-2025-06-26", "category": "deep-research", "aliasOf": "o4-mini-deep-research", "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "dated snapshot of o4-mini-deep-research; not probed separately" ], "created": "2025-06-25" }, { "provider": "openai", "id": "omni-moderation-2024-09-26", "category": "moderation", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category moderation: not a general text model, not probed (inappropriate request type)" ], "created": "2024-11-27" }, { "provider": "openai", "id": "omni-moderation-latest", "category": "moderation", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category moderation: not a general text model, not probed (inappropriate request type)" ], "created": "2024-11-15" }, { "provider": "openai", "id": "sora-2", "category": "video", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category video: not a general text model, not probed (inappropriate request type)" ], "created": "2025-10-05" }, { "provider": "openai", "id": "sora-2-pro", "category": "video", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category video: not a general text model, not probed (inappropriate request type)" ], "created": "2025-10-05" }, { "provider": "openai", "id": "text-embedding-3-large", "category": "embedding", "aliasOf": null, "verdict": "untested", "pricing": { "input": 0.13, "cached": null, "output": 0 }, "probes": {}, "notes": [ "category embedding: not a general text model, not probed (inappropriate request type)" ], "created": "2024-01-22" }, { "provider": "openai", "id": "text-embedding-3-small", "category": "embedding", "aliasOf": null, "verdict": "untested", "pricing": { "input": 0.02, "cached": null, "output": 0 }, "probes": {}, "notes": [ "category embedding: not a general text model, not probed (inappropriate request type)" ], "created": "2024-01-22" }, { "provider": "openai", "id": "text-embedding-ada-002", "category": "embedding", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category embedding: not a general text model, not probed (inappropriate request type)" ], "created": "2022-12-16" }, { "provider": "openai", "id": "tts-1", "category": "audio", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category audio: not a general text model, not probed (inappropriate request type)" ], "created": "2023-04-19" }, { "provider": "openai", "id": "tts-1-1106", "category": "audio", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category audio: not a general text model, not probed (inappropriate request type)" ], "created": "2023-11-03" }, { "provider": "openai", "id": "tts-1-hd", "category": "audio", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category audio: not a general text model, not probed (inappropriate request type)" ], "created": "2023-11-03" }, { "provider": "openai", "id": "tts-1-hd-1106", "category": "audio", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category audio: not a general text model, not probed (inappropriate request type)" ], "created": "2023-11-03" }, { "provider": "openai", "id": "whisper-1", "category": "audio", "aliasOf": null, "verdict": "untested", "pricing": null, "probes": {}, "notes": [ "category audio: not a general text model, not probed (inappropriate request type)" ], "created": "2023-02-27" }, { "provider": "anthropic", "id": "claude-haiku-5-5", "display": "Claude Haiku 5.5", "category": "text", "aliasOf": null, "verdict": "tested", "context": 1000000, "maxOutput": 128000, "pricing": { "input": 0.1, "cached": 0.01, "output": 0.5 }, "probes": { "text": { "status": "ok", "ms": 592 }, "streaming": { "status": "ok", "ms": 604 }, "tool_use": { "status": "ok", "ms": 620 }, "structured_output": { "status": "ok", "ms": 1089 }, "vision": { "status": "ok", "ms": 932 }, "reasoning_control": { "status": "ok", "ms": 567 } }, "notes": [], "created": "2026-10-07" }, { "provider": "anthropic", "id": "claude-sonnet-5-5", "display": "Claude Sonnet 5.5", "category": "text", "aliasOf": null, "verdict": "tested", "context": 1000000, "maxOutput": 128000, "pricing": { "input": 2, "cached": 0.1, "output": 10 }, "probes": { "text": { "status": "ok", "ms": 1265 }, "streaming": { "status": "ok", "ms": 1037 }, "tool_use": { "status": "ok", "ms": 1385 }, "structured_output": { "status": "ok", "ms": 772 }, "vision": { "status": "ok", "ms": 1019 }, "reasoning_control": { "status": "ok", "ms": 1167 } }, "notes": [], "created": "2026-09-28" }, { "provider": "anthropic", "id": "claude-opus-5-5", "display": "Claude Opus 5.5", "category": "text", "aliasOf": null, "verdict": "tested", "context": 1000000, "maxOutput": 128000, "pricing": { "input": 4, "cached": 0.2, "output": 20 }, "probes": { "text": { "status": "ok", "ms": 859 }, "streaming": { "status": "ok", "ms": 905 }, "tool_use": { "status": "ok", "ms": 1020 }, "structured_output": { "status": "ok", "ms": 1629 }, "vision": { "status": "ok", "ms": 952 }, "reasoning_control": { "status": "ok", "ms": 3151 } }, "notes": [], "created": "2026-09-21" }, { "provider": "anthropic", "id": "claude-fable-5-1", "display": "Claude Fable 5.1", "category": "text", "aliasOf": null, "verdict": "tested", "context": 1000000, "maxOutput": 128000, "pricing": { "input": 10, "cached": 0.25, "output": 50 }, "probes": { "text": { "status": "ok", "ms": 734 }, "streaming": { "status": "ok", "ms": 740 }, "tool_use": { "status": "ok", "ms": 1094 }, "structured_output": { "status": "ok", "ms": 1162 }, "vision": { "status": "ok", "ms": 751 }, "reasoning_control": { "status": "ok", "ms": 718 } }, "notes": [], "created": "2026-08-28" }, { "provider": "anthropic", "id": "claude-opus-5", "display": "Claude Opus 5", "category": "text", "aliasOf": null, "verdict": "tested", "context": 1000000, "maxOutput": 128000, "pricing": { "input": 5, "cached": 0.5, "output": 25 }, "probes": { "text": { "status": "ok", "ms": 1509 }, "streaming": { "status": "ok", "ms": 771 }, "tool_use": { "status": "ok", "ms": 1747 }, "structured_output": { "status": "ok", "ms": 1341 }, "vision": { "status": "ok", "ms": 940 }, "reasoning_control": { "status": "ok", "ms": 1427 } }, "notes": [], "created": "2026-07-24" }, { "provider": "anthropic", "id": "claude-sonnet-5", "display": "Claude Sonnet 5", "category": "text", "aliasOf": null, "verdict": "tested", "context": 1000000, "maxOutput": 128000, "pricing": { "input": 2, "cached": 0.2, "output": 10 }, "probes": { "text": { "status": "ok", "ms": 937 }, "streaming": { "status": "ok", "ms": 1024 }, "tool_use": { "status": "ok", "ms": 1003 }, "structured_output": { "status": "ok", "ms": 1055 }, "vision": { "status": "ok", "ms": 891 }, "reasoning_control": { "status": "ok", "ms": 875 } }, "notes": [], "created": "2026-06-29" }, { "provider": "anthropic", "id": "claude-fable-5", "display": "Claude Fable 5", "category": "text", "aliasOf": null, "verdict": "tested", "context": 1000000, "maxOutput": 128000, "pricing": { "input": 10, "cached": 1, "output": 50 }, "probes": { "text": { "status": "ok", "ms": 1882 }, "streaming": { "status": "ok", "ms": 2063 }, "tool_use": { "status": "ok", "ms": 2394 }, "structured_output": { "status": "ok", "ms": 2096 }, "vision": { "status": "ok", "ms": 2119 }, "reasoning_control": { "status": "ok", "ms": 2015 } }, "notes": [], "created": "2026-06-07" }, { "provider": "anthropic", "id": "claude-opus-4-8", "display": "Claude Opus 4.8", "category": "text", "aliasOf": null, "verdict": "tested", "context": 1000000, "maxOutput": 128000, "pricing": { "input": 5, "cached": 0.5, "output": 25 }, "probes": { "text": { "status": "ok", "ms": 560 }, "streaming": { "status": "ok", "ms": 614 }, "tool_use": { "status": "ok", "ms": 1354 }, "structured_output": { "status": "ok", "ms": 1620 }, "vision": { "status": "ok", "ms": 836 }, "reasoning_control": { "status": "ok", "ms": 653 } }, "notes": [], "created": "2026-05-28" }, { "provider": "anthropic", "id": "claude-opus-4-7", "display": "Claude Opus 4.7", "category": "text", "aliasOf": null, "verdict": "tested", "context": 1000000, "maxOutput": 128000, "pricing": { "input": 5, "cached": 0.5, "output": 25 }, "probes": { "text": { "status": "ok", "ms": 414 }, "streaming": { "status": "ok", "ms": 670 }, "tool_use": { "status": "ok", "ms": 1172 }, "structured_output": { "status": "ok", "ms": 1247 }, "vision": { "status": "ok", "ms": 664 }, "reasoning_control": { "status": "ok", "ms": 748 } }, "notes": [], "created": "2026-04-14" }, { "provider": "anthropic", "id": "claude-sonnet-4-6", "display": "Claude Sonnet 4.6", "category": "text", "aliasOf": null, "verdict": "tested", "context": 1000000, "maxOutput": 128000, "pricing": { "input": 3, "cached": 0.3, "output": 15 }, "probes": { "text": { "status": "ok", "ms": 721 }, "streaming": { "status": "ok", "ms": 1063 }, "tool_use": { "status": "ok", "ms": 1064 }, "structured_output": { "status": "ok", "ms": 919 }, "vision": { "status": "ok", "ms": 1393 }, "reasoning_control": { "status": "ok", "ms": 666 } }, "notes": [], "created": "2026-02-17" }, { "provider": "anthropic", "id": "claude-opus-4-6", "display": "Claude Opus 4.6", "category": "text", "aliasOf": null, "verdict": "tested", "context": 1000000, "maxOutput": 128000, "pricing": { "input": 5, "cached": 0.5, "output": 25 }, "probes": { "text": { "status": "ok", "ms": 1530 }, "streaming": { "status": "ok", "ms": 1573 }, "tool_use": { "status": "ok", "ms": 1772 }, "structured_output": { "status": "ok", "ms": 1864 }, "vision": { "status": "ok", "ms": 1280 }, "reasoning_control": { "status": "ok", "ms": 1515 } }, "notes": [], "created": "2026-02-04" }, { "provider": "anthropic", "id": "claude-opus-4-5-20251101", "display": "Claude Opus 4.5", "category": "text", "aliasOf": null, "verdict": "tested", "context": 200000, "maxOutput": 64000, "pricing": { "input": 5, "cached": 0.5, "output": 25 }, "probes": { "text": { "status": "ok", "ms": 610 }, "streaming": { "status": "ok", "ms": 1199 }, "tool_use": { "status": "ok", "ms": 1301 }, "structured_output": { "status": "ok", "ms": 1157 }, "vision": { "status": "ok", "ms": 639 }, "reasoning_control": { "status": "ok", "ms": 1454 } }, "notes": [], "created": "2025-11-24" }, { "provider": "anthropic", "id": "claude-haiku-4-5-20251001", "display": "Claude Haiku 4.5", "category": "text", "aliasOf": null, "verdict": "tested", "context": 200000, "maxOutput": 64000, "pricing": { "input": 1, "cached": 0.1, "output": 5 }, "probes": { "text": { "status": "ok", "ms": 428 }, "streaming": { "status": "ok", "ms": 563 }, "tool_use": { "status": "ok", "ms": 777 }, "structured_output": { "status": "ok", "ms": 1178 }, "vision": { "status": "ok", "ms": 427 }, "reasoning_control": { "status": "ok", "ms": 782 } }, "notes": [], "created": "2025-10-15" }, { "provider": "anthropic", "id": "claude-sonnet-4-5-20250929", "display": "Claude Sonnet 4.5", "category": "text", "aliasOf": null, "verdict": "tested", "context": 200000, "maxOutput": 64000, "pricing": { "input": 3, "cached": 0.3, "output": 15 }, "probes": { "text": { "status": "ok", "ms": 593 }, "streaming": { "status": "ok", "ms": 672 }, "tool_use": { "status": "ok", "ms": 1259 }, "structured_output": { "status": "ok", "ms": 1075 }, "vision": { "status": "ok", "ms": 648 }, "reasoning_control": { "status": "ok", "ms": 1186 } }, "notes": [], "created": "2025-09-29" } ] }