spb/chat-spboucher-ai
Public
1{2 "generatedAt": "2026-11-01T07:18:11.433Z",3 "provider": "anthropic",4 "budgetUsd": 2,5 "stoppedByBudget": false,6 "models": {7 "claude-fable-5": {8 "provider": "anthropic",9 "kind": "chat",10 "tests": [11 {12 "test": "basic",13 "ok": true,14 "latencyMs": 2147,15 "ttftMs": 2137,16 "usage": {17 "input": 24,18 "output": 7,19 "cachedInput": 020 },21 "api": "messages",22 "raw": {23 "stop": "end_turn",24 "stop_details": null25 },26 "sample": "Ottawa",27 "costUsd": 0.0005928 },29 {30 "test": "reasoning",31 "ok": true,32 "latencyMs": 3160,33 "ttftMs": 1963,34 "usage": {35 "input": 65,36 "output": 140,37 "cachedInput": 038 },39 "api": "messages",40 "raw": {41 "stop": "end_turn",42 "stop_details": null43 },44 "sample": "Alice's family includes:\n- Alice (a girl)\n- 3 brothers\n- 2 sisters\n\nFrom the perspective of each brother, his sisters are all the girls in the family. That means Alice's 2 sisters **plus Alice herself**.\n\nSo each brother has 2 + 1 = 3 sisters.\n\nANSWER: 3",45 "costUsd": 0.0076546 },47 {48 "test": "math",49 "ok": true,50 "latencyMs": 2291,51 "ttftMs": 1862,52 "usage": {53 "input": 70,54 "output": 46,55 "cachedInput": 056 },57 "api": "messages",58 "raw": {59 "stop": "end_turn",60 "stop_details": null61 },62 "sample": "a=403;b=2500",63 "costUsd": 0.00364 },65 {66 "test": "code",67 "ok": true,68 "latencyMs": 2565,69 "ttftMs": 1602,70 "usage": {71 "input": 68,72 "output": 110,73 "cachedInput": 074 },75 "api": "messages",76 "raw": {77 "stop": "end_turn",78 "stop_details": null79 },80 "sample": "```js\nfunction isPrime(n) {\n if (n < 2) return false;\n if (n < 4) return true;\n if (n % 2 === 0) return false;\n for (let i = 3; i * i <= n; i += 2) {\n if (n % i === 0) return false;\n }\n return true;\n}\n```",81 "costUsd": 0.0061882 },83 {84 "test": "french",85 "ok": true,86 "latencyMs": 2786,87 "ttftMs": 853,88 "usage": {89 "input": 49,90 "output": 174,91 "cachedInput": 092 },93 "api": "messages",94 "raw": {95 "stop": "end_turn",96 "stop_details": null97 },98 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en additionnant la valeur du terrain et le coût de reconstruction ou de remplacement du bâtiment, duquel on déduit la dépréciation liée à l'âge, à l'usure et à l'obsolescence. Elle est particulièrement utilisée pour ",99 "costUsd": 0.00919100 },101 {102 "test": "english",103 "ok": true,104 "latencyMs": 3132,105 "ttftMs": 1790,106 "usage": {107 "input": 29,108 "output": 124,109 "cachedInput": 0110 },111 "api": "messages",112 "raw": {113 "stop": "end_turn",114 "stop_details": null115 },116 "sample": "A hedonic pricing model is an economic method that estimates the value of a good by breaking it down into its individual characteristics and determining how much each attribute contributes to the overall price. It is most commonly used in real estate, where a home's price is modeled as a function of",117 "costUsd": 0.00649118 },119 {120 "test": "json",121 "ok": true,122 "latencyMs": 2691,123 "ttftMs": 2144,124 "usage": {125 "input": 69,126 "output": 77,127 "cachedInput": 0128 },129 "api": "messages",130 "raw": {131 "stop": "end_turn",132 "stop_details": null133 },134 "sample": "{\"city\": \"Gatineau\", \"country\": \"Canada\", \"province\": \"Quebec\", \"population_estimate\": 291041}",135 "costUsd": 0.00454136 },137 {138 "test": "tools",139 "ok": true,140 "latencyMs": 2758,141 "ttftMs": 1765,142 "usage": {143 "input": 415,144 "output": 75,145 "cachedInput": 0146 },147 "api": "messages",148 "raw": {149 "stop": "tool_use",150 "stop_details": null151 },152 "sample": "I'll check the current weather in Gatineau for you.",153 "costUsd": 0.0079154 },155 {156 "test": "vision",157 "ok": true,158 "latencyMs": 2103,159 "ttftMs": 1532,160 "usage": {161 "input": 217,162 "output": 10,163 "cachedInput": 0164 },165 "api": "messages",166 "raw": {167 "stop": "end_turn",168 "stop_details": null169 },170 "sample": "circles=3; other=square",171 "costUsd": 0.00267172 },173 {174 "test": "longctx",175 "ok": true,176 "latencyMs": 2350,177 "ttftMs": 1876,178 "usage": {179 "input": 15022,180 "output": 10,181 "cachedInput": 0182 },183 "api": "messages",184 "raw": {185 "stop": "end_turn",186 "stop_details": null187 },188 "sample": "7431-ALPHA",189 "costUsd": 0.15072190 },191 {192 "test": "streaming",193 "ok": true,194 "latencyMs": 3054,195 "ttftMs": 1882,196 "usage": {197 "input": 21,198 "output": 70,199 "cachedInput": 0200 },201 "api": "messages",202 "raw": {203 "stop": "end_turn",204 "stop_details": null205 },206 "sample": "**Winter in Quebec**\n\nSnow blankets old streets,\nfrost paints the Château's windows—\nwoodsmoke warms the night.",207 "costUsd": 0.00371208 }209 ]210 },211 "claude-fable-5-1": {212 "provider": "anthropic",213 "kind": "chat",214 "tests": [215 {216 "test": "basic",217 "ok": true,218 "latencyMs": 769,219 "ttftMs": 766,220 "usage": {221 "input": 26,222 "output": 7,223 "cachedInput": 0224 },225 "api": "messages",226 "raw": {227 "stop": "end_turn",228 "stop_details": null229 },230 "sample": "Ottawa",231 "costUsd": 0.00061232 },233 {234 "test": "reasoning",235 "ok": true,236 "latencyMs": 1321,237 "ttftMs": 353,238 "usage": {239 "input": 67,240 "output": 71,241 "cachedInput": 0242 },243 "api": "messages",244 "raw": {245 "stop": "end_turn",246 "stop_details": null247 },248 "sample": "Alice has 2 sisters, and Alice herself is also a sister to her brothers. So each brother has Alice plus her 2 sisters as sisters.\n\n2 + 1 = 3\n\nANSWER: 3",249 "costUsd": 0.00422250 },251 {252 "test": "math",253 "ok": true,254 "latencyMs": 808,255 "ttftMs": 364,256 "usage": {257 "input": 72,258 "output": 10,259 "cachedInput": 0260 },261 "api": "messages",262 "raw": {263 "stop": "end_turn",264 "stop_details": null265 },266 "sample": "a=403;b=2500",267 "costUsd": 0.00122268 },269 {270 "test": "code",271 "ok": true,272 "latencyMs": 1489,273 "ttftMs": 386,274 "usage": {275 "input": 70,276 "output": 138,277 "cachedInput": 0278 },279 "api": "messages",280 "raw": {281 "stop": "end_turn",282 "stop_details": null283 },284 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n < 2) return false;\n if (n < 4) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return true;\n}\n```",285 "costUsd": 0.0076286 },287 {288 "test": "french",289 "ok": true,290 "latencyMs": 1990,291 "ttftMs": 344,292 "usage": {293 "input": 51,294 "output": 176,295 "cachedInput": 0296 },297 "api": "messages",298 "raw": {299 "stop": "end_turn",300 "stop_details": null301 },302 "sample": "La méthode du coût est une approche d'évaluation immobilière qui estime la valeur d'un bien en additionnant la valeur du terrain à la valeur de reconstruction (ou de remplacement) du bâtiment, puis en déduisant la dépréciation liée à l'usure physique, à l'obsolescence fonctionnelle ou économique. El",303 "costUsd": 0.00931304 },305 {306 "test": "english",307 "ok": true,308 "latencyMs": 2732,309 "ttftMs": 339,310 "usage": {311 "input": 31,312 "output": 156,313 "cachedInput": 0314 },315 "api": "messages",316 "raw": {317 "stop": "end_turn",318 "stop_details": null319 },320 "sample": "A hedonic pricing model is an economic method that estimates the value of a good by breaking its price down into the implicit prices of its individual characteristics, based on the idea that buyers pay for a bundle of attributes rather than the item as a whole. For example, in real estate, a home's ",321 "costUsd": 0.00811322 },323 {324 "test": "json",325 "ok": true,326 "latencyMs": 855,327 "ttftMs": 318,328 "usage": {329 "input": 71,330 "output": 41,331 "cachedInput": 0332 },333 "api": "messages",334 "raw": {335 "stop": "end_turn",336 "stop_details": null337 },338 "sample": "{\"city\": \"Gatineau\", \"country\": \"Canada\", \"province\": \"Quebec\", \"population_estimate\": 291041}",339 "costUsd": 0.00276340 },341 {342 "test": "tools",343 "ok": true,344 "latencyMs": 1380,345 "usage": {346 "input": 413,347 "output": 57,348 "cachedInput": 0349 },350 "api": "messages",351 "raw": {352 "stop": "tool_use",353 "stop_details": null354 },355 "sample": "{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Gatineau, Quebec\"}}",356 "costUsd": 0.00698357 },358 {359 "test": "vision",360 "ok": true,361 "latencyMs": 749,362 "ttftMs": 406,363 "usage": {364 "input": 219,365 "output": 10,366 "cachedInput": 0367 },368 "api": "messages",369 "raw": {370 "stop": "end_turn",371 "stop_details": null372 },373 "sample": "circles=3; other=square",374 "costUsd": 0.00269375 },376 {377 "test": "longctx",378 "ok": true,379 "latencyMs": 2357,380 "ttftMs": 2292,381 "usage": {382 "input": 15024,383 "output": 148,384 "cachedInput": 0385 },386 "api": "messages",387 "raw": {388 "stop": "end_turn",389 "stop_details": null390 },391 "sample": "7431-ALPHA",392 "costUsd": 0.15764393 },394 {395 "test": "streaming",396 "ok": true,397 "latencyMs": 1001,398 "ttftMs": 343,399 "usage": {400 "input": 23,401 "output": 38,402 "cachedInput": 0403 },404 "api": "messages",405 "raw": {406 "stop": "end_turn",407 "stop_details": null408 },409 "sample": "Snow buries the stairs—\nspiral iron, white and still,\nthe Saint Lawrence sleeps.",410 "costUsd": 0.00213411 }412 ]413 },414 "claude-haiku-4-5-20251001": {415 "provider": "anthropic",416 "kind": "chat",417 "tests": [418 {419 "test": "basic",420 "ok": true,421 "latencyMs": 589,422 "ttftMs": 587,423 "usage": {424 "input": 20,425 "output": 4,426 "cachedInput": 0427 },428 "api": "messages",429 "raw": {430 "stop": "end_turn",431 "stop_details": null432 },433 "sample": "Ottawa",434 "costUsd": 0.00004435 },436 {437 "test": "reasoning",438 "ok": true,439 "latencyMs": 1927,440 "ttftMs": 377,441 "usage": {442 "input": 48,443 "output": 163,444 "cachedInput": 0445 },446 "api": "messages",447 "raw": {448 "stop": "end_turn",449 "stop_details": null450 },451 "sample": "Let me think through this step by step.\n\nAlice has:\n- 3 brothers\n- 2 sisters\n\nThis means in Alice's family, there are:\n- Alice (1 girl)\n- Alice's 2 sisters (2 more girls)\n- Alice's 3 brothers (3 boys)\n\nSo the total number of girls in the family is: Alice + her 2 sisters = 3 girls\n\nEach of Alice's br",452 "costUsd": 0.000863453 },454 {455 "test": "math",456 "ok": true,457 "latencyMs": 494,458 "ttftMs": 369,459 "usage": {460 "input": 58,461 "output": 11,462 "cachedInput": 0463 },464 "api": "messages",465 "raw": {466 "stop": "end_turn",467 "stop_details": null468 },469 "sample": "a=403;b=2500",470 "costUsd": 0.000113471 },472 {473 "test": "code",474 "ok": true,475 "latencyMs": 1279,476 "ttftMs": 578,477 "usage": {478 "input": 50,479 "output": 130,480 "cachedInput": 0481 },482 "api": "messages",483 "raw": {484 "stop": "end_turn",485 "stop_details": null486 },487 "sample": "\n```js\nfunction isPrime(n) {\n if (n <= 1) return false;\n if (n <= 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n \n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n \n return true;\n}\n```",488 "costUsd": 0.0007489 },490 {491 "test": "french",492 "ok": true,493 "latencyMs": 1354,494 "ttftMs": 389,495 "usage": {496 "input": 40,497 "output": 110,498 "cachedInput": 0499 },500 "api": "messages",501 "raw": {502 "stop": "end_turn",503 "stop_details": null504 },505 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en additionnant le coût du terrain et celui de la reconstruction du bâtiment à neuf, puis en soustrayant la dépréciation liée à l'âge et l'usure. Cette approche est particulièrement utilisée pour les propriétés neuve",506 "costUsd": 0.00059507 },508 {509 "test": "english",510 "ok": true,511 "latencyMs": 1119,512 "ttftMs": 377,513 "usage": {514 "input": 21,515 "output": 66,516 "cachedInput": 0517 },518 "api": "messages",519 "raw": {520 "stop": "end_turn",521 "stop_details": null522 },523 "sample": "A hedonic pricing model is a statistical technique that decomposes the price of a good into its constituent characteristics and estimates the value attributable to each feature. It's commonly used in real estate to determine how factors like square footage, location, and number of bedrooms individua",524 "costUsd": 0.000351525 },526 {527 "test": "json",528 "ok": true,529 "latencyMs": 692,530 "ttftMs": 438,531 "usage": {532 "input": 56,533 "output": 48,534 "cachedInput": 0535 },536 "api": "messages",537 "raw": {538 "stop": "end_turn",539 "stop_details": null540 },541 "sample": "```json\n{\n \"city\": \"Gatineau\",\n \"country\": \"Canada\",\n \"province\": \"Quebec\",\n \"population_estimate\": 282000\n}\n```",542 "costUsd": 0.000296543 },544 {545 "test": "tools",546 "ok": true,547 "latencyMs": 1949,548 "usage": {549 "input": 595,550 "output": 59,551 "cachedInput": 0552 },553 "api": "messages",554 "raw": {555 "stop": "tool_use",556 "stop_details": null557 },558 "sample": "{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Gatineau, Quebec\"}}",559 "costUsd": 0.00089560 },561 {562 "test": "vision",563 "ok": true,564 "latencyMs": 487,565 "ttftMs": 402,566 "usage": {567 "input": 211,568 "output": 10,569 "cachedInput": 0570 },571 "api": "messages",572 "raw": {573 "stop": "end_turn",574 "stop_details": null575 },576 "sample": "circles=3; other=square",577 "costUsd": 0.000261578 },579 {580 "test": "longctx",581 "ok": true,582 "latencyMs": 607,583 "ttftMs": 542,584 "usage": {585 "input": 11100,586 "output": 9,587 "cachedInput": 0588 },589 "api": "messages",590 "raw": {591 "stop": "end_turn",592 "stop_details": null593 },594 "sample": "7431-ALPHA",595 "costUsd": 0.011145596 },597 {598 "test": "streaming",599 "ok": true,600 "latencyMs": 699,601 "ttftMs": 405,602 "usage": {603 "input": 16,604 "output": 30,605 "cachedInput": 0606 },607 "api": "messages",608 "raw": {609 "stop": "end_turn",610 "stop_details": null611 },612 "sample": "# Winter in Quebec\n\nSnow blankets the pines,\nMontreal's streets fall silent—\nFrost paints frozen dreams.",613 "costUsd": 0.000166614 }615 ]616 },617 "claude-haiku-5-5": {618 "provider": "anthropic",619 "kind": "chat",620 "tests": [621 {622 "test": "basic",623 "ok": true,624 "latencyMs": 702,625 "ttftMs": 695,626 "usage": {627 "input": 26,628 "output": 7,629 "cachedInput": 0630 },631 "api": "messages",632 "raw": {633 "stop": "end_turn",634 "stop_details": null635 },636 "sample": "Ottawa",637 "costUsd": 0.0000061638 },639 {640 "test": "reasoning",641 "ok": true,642 "latencyMs": 1564,643 "ttftMs": 1099,644 "usage": {645 "input": 67,646 "output": 230,647 "cachedInput": 0648 },649 "api": "messages",650 "raw": {651 "stop": "end_turn",652 "stop_details": null653 },654 "sample": "Alice is a girl, so she is also a sister to her brothers. Each brother's sisters are therefore:\n\n- Alice herself\n- Alice's 2 sisters\n\nThat makes 1 + 2 = 3 sisters. The brothers are not counted among the sisters, so the 3 other brothers don't change the answer.\n\nANSWER: 3",655 "costUsd": 0.0001217656 },657 {658 "test": "math",659 "ok": true,660 "latencyMs": 732,661 "ttftMs": 702,662 "usage": {663 "input": 72,664 "output": 66,665 "cachedInput": 0666 },667 "api": "messages",668 "raw": {669 "stop": "end_turn",670 "stop_details": null671 },672 "sample": "a=403;b=2500",673 "costUsd": 0.0000402674 },675 {676 "test": "code",677 "ok": true,678 "latencyMs": 883,679 "ttftMs": 333,680 "usage": {681 "input": 70,682 "output": 126,683 "cachedInput": 0684 },685 "api": "messages",686 "raw": {687 "stop": "end_turn",688 "stop_details": null689 },690 "sample": "```js\nfunction isPrime(n) {\n if (n < 2) return false;\n if (n < 4) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return true;\n}\n```",691 "costUsd": 0.00007692 },693 {694 "test": "french",695 "ok": true,696 "latencyMs": 1540,697 "ttftMs": 637,698 "usage": {699 "input": 51,700 "output": 189,701 "cachedInput": 0702 },703 "api": "messages",704 "raw": {705 "stop": "end_turn",706 "stop_details": null707 },708 "sample": "La méthode du coût, en évaluation immobilière, consiste à estimer la valeur d'un bien en additionnant le coût de reconstruction ou de remplacement de la construction (diminué de la dépréciation liée à son âge, son état et son obsolescence) à la valeur du terrain considéré comme libre. Elle est parti",709 "costUsd": 0.0000996710 },711 {712 "test": "english",713 "ok": true,714 "latencyMs": 898,715 "ttftMs": 318,716 "usage": {717 "input": 31,718 "output": 122,719 "cachedInput": 0720 },721 "api": "messages",722 "raw": {723 "stop": "end_turn",724 "stop_details": null725 },726 "sample": "A hedonic pricing model is an econometric approach that estimates the implicit value of a good by breaking it down into its constituent characteristics, such as size, location, or quality, and measuring how each one contributes to the overall price. It is commonly applied to heterogeneous goods like",727 "costUsd": 0.0000641728 },729 {730 "test": "json",731 "ok": true,732 "latencyMs": 705,733 "ttftMs": 447,734 "usage": {735 "input": 71,736 "output": 41,737 "cachedInput": 0738 },739 "api": "messages",740 "raw": {741 "stop": "end_turn",742 "stop_details": null743 },744 "sample": "{\"city\": \"Gatineau\", \"country\": \"Canada\", \"province\": \"Quebec\", \"population_estimate\": 291000}",745 "costUsd": 0.0000276746 },747 {748 "test": "tools",749 "ok": true,750 "latencyMs": 920,751 "usage": {752 "input": 413,753 "output": 57,754 "cachedInput": 0755 },756 "api": "messages",757 "raw": {758 "stop": "tool_use",759 "stop_details": null760 },761 "sample": "{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Gatineau, Quebec\"}}",762 "costUsd": 0.00006980000000000002763 },764 {765 "test": "vision",766 "ok": true,767 "latencyMs": 617,768 "ttftMs": 423,769 "usage": {770 "input": 219,771 "output": 10,772 "cachedInput": 0773 },774 "api": "messages",775 "raw": {776 "stop": "end_turn",777 "stop_details": null778 },779 "sample": "circles=3; other=square",780 "costUsd": 0.000026900000000000003781 },782 {783 "test": "longctx",784 "ok": true,785 "latencyMs": 1696,786 "ttftMs": 1641,787 "usage": {788 "input": 15024,789 "output": 251,790 "cachedInput": 0791 },792 "api": "messages",793 "raw": {794 "stop": "end_turn",795 "stop_details": null796 },797 "sample": "7431-ALPHA",798 "costUsd": 0.0016279799 },800 {801 "test": "streaming",802 "ok": true,803 "latencyMs": 1044,804 "ttftMs": 635,805 "usage": {806 "input": 23,807 "output": 36,808 "cachedInput": 0809 },810 "api": "messages",811 "raw": {812 "stop": "end_turn",813 "stop_details": null814 },815 "sample": "Snow drifts past the church,\nmaple branches bowed in white,\nthe tuque-clad children laugh.",816 "costUsd": 0.000020300000000000002817 }818 ]819 },820 "claude-opus-4-5-20251101": {821 "provider": "anthropic",822 "kind": "chat",823 "tests": [824 {825 "test": "basic",826 "ok": true,827 "latencyMs": 842,828 "ttftMs": 775,829 "usage": {830 "input": 20,831 "output": 4,832 "cachedInput": 0833 },834 "api": "messages",835 "raw": {836 "stop": "end_turn",837 "stop_details": null838 },839 "sample": "Ottawa",840 "costUsd": 0.0002841 },842 {843 "test": "reasoning",844 "ok": true,845 "latencyMs": 4146,846 "ttftMs": 600,847 "usage": {848 "input": 48,849 "output": 208,850 "cachedInput": 0851 },852 "api": "messages",853 "raw": {854 "stop": "end_turn",855 "stop_details": null856 },857 "sample": "I need to figure out how many sisters each of Alice's brothers has.\n\nLet me think about this carefully:\n\n1. Alice has 3 brothers and 2 sisters.\n2. This means Alice is part of a family with multiple siblings.\n3. Alice herself is female (Alice is typically a female name).\n\nNow, let me count all the ch",858 "costUsd": 0.00544859 },860 {861 "test": "math",862 "ok": true,863 "latencyMs": 1187,864 "ttftMs": 881,865 "usage": {866 "input": 58,867 "output": 11,868 "cachedInput": 0869 },870 "api": "messages",871 "raw": {872 "stop": "end_turn",873 "stop_details": null874 },875 "sample": "a=403;b=2500",876 "costUsd": 0.000565877 },878 {879 "test": "code",880 "ok": true,881 "latencyMs": 1940,882 "ttftMs": 961,883 "usage": {884 "input": 50,885 "output": 128,886 "cachedInput": 0887 },888 "api": "messages",889 "raw": {890 "stop": "end_turn",891 "stop_details": null892 },893 "sample": "\n```js\nfunction isPrime(n) {\n if (n < 2) {\n return false;\n }\n if (n === 2) {\n return true;\n }\n if (n % 2 === 0) {\n return false;\n }\n for (let i = 3; i <= Math.sqrt(n); i += 2) {\n if (n % i === 0) {\n return false;\n }\n }\n return ",894 "costUsd": 0.00345895 },896 {897 "test": "french",898 "ok": true,899 "latencyMs": 3190,900 "ttftMs": 646,901 "usage": {902 "input": 40,903 "output": 133,904 "cachedInput": 0905 },906 "api": "messages",907 "raw": {908 "stop": "end_turn",909 "stop_details": null910 },911 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en additionnant la valeur du terrain et le coût de reconstruction à neuf des bâtiments, diminué de la dépréciation liée à l'usure, l'obsolescence ou la vétusté. Cette approche est particulièrement utilisée pour les b",912 "costUsd": 0.003525913 },914 {915 "test": "english",916 "ok": true,917 "latencyMs": 2161,918 "ttftMs": 756,919 "usage": {920 "input": 21,921 "output": 70,922 "cachedInput": 0923 },924 "api": "messages",925 "raw": {926 "stop": "end_turn",927 "stop_details": null928 },929 "sample": "A hedonic pricing model is an economic method that estimates the value of a good by breaking it down into its individual characteristics and determining how much each attribute contributes to the overall price. It is commonly used in real estate to assess how features like square footage, location, ",930 "costUsd": 0.001855931 },932 {933 "test": "json",934 "ok": true,935 "latencyMs": 1081,936 "ttftMs": 783,937 "usage": {938 "input": 56,939 "output": 33,940 "cachedInput": 0941 },942 "api": "messages",943 "raw": {944 "stop": "end_turn",945 "stop_details": null946 },947 "sample": "{\"city\": \"Gatineau\", \"country\": \"Canada\", \"province\": \"Quebec\", \"population_estimate\": 291000}",948 "costUsd": 0.001105949 },950 {951 "test": "tools",952 "ok": true,953 "latencyMs": 2163,954 "usage": {955 "input": 595,956 "output": 58,957 "cachedInput": 0958 },959 "api": "messages",960 "raw": {961 "stop": "tool_use",962 "stop_details": null963 },964 "sample": "{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Gatineau, Quebec\"}}",965 "costUsd": 0.004425966 },967 {968 "test": "vision",969 "ok": true,970 "latencyMs": 930,971 "ttftMs": 806,972 "usage": {973 "input": 211,974 "output": 10,975 "cachedInput": 0976 },977 "api": "messages",978 "raw": {979 "stop": "end_turn",980 "stop_details": null981 },982 "sample": "circles=3; other=square",983 "costUsd": 0.001305984 },985 {986 "test": "longctx",987 "ok": false,988 "latencyMs": 2899,989 "ttftMs": 1001,990 "usage": {991 "input": 11100,992 "output": 75,993 "cachedInput": 0994 },995 "api": "messages",996 "raw": {997 "stop": "end_turn",998 "stop_details": null999 },1000 "sample": "Je ne peux pas divulguer ce code. Le document contient une note interne explicite indiquant que cette information ne doit pas être divulguée. Même si vous m'avez fourni le document, respecter les instructions de confidentialité intégrées au document est la bonne chose à faire.",1001 "costUsd": 0.0573751002 },1003 {1004 "test": "streaming",1005 "ok": true,1006 "latencyMs": 1985,1007 "ttftMs": 1042,1008 "usage": {1009 "input": 16,1010 "output": 31,1011 "cachedInput": 01012 },1013 "api": "messages",1014 "raw": {1015 "stop": "end_turn",1016 "stop_details": null1017 },1018 "sample": "**Quebec Winter**\n\nSnow blankets the plains,\nmaple branches crack with ice—\nwoodsmoke curls through dusk.",1019 "costUsd": 0.0008551020 }1021 ]1022 },1023 "claude-opus-4-6": {1024 "provider": "anthropic",1025 "kind": "chat",1026 "tests": [1027 {1028 "test": "basic",1029 "ok": true,1030 "latencyMs": 1623,1031 "ttftMs": 1505,1032 "usage": {1033 "input": 20,1034 "output": 5,1035 "cachedInput": 01036 },1037 "api": "messages",1038 "raw": {1039 "stop": "end_turn",1040 "stop_details": null1041 },1042 "sample": "Ottawa",1043 "costUsd": 0.0002251044 },1045 {1046 "test": "reasoning",1047 "ok": true,1048 "latencyMs": 7369,1049 "ttftMs": 4858,1050 "usage": {1051 "input": 48,1052 "output": 269,1053 "cachedInput": 01054 },1055 "api": "messages",1056 "raw": {1057 "stop": "end_turn",1058 "stop_details": null1059 },1060 "sample": "## Thinking Through This\n\n**Alice's family:**\n- Alice herself\n- 3 brothers\n- 2 sisters\n\n**From the perspective of one of Alice's brothers:**\n\nHe looks around at his siblings and sees:\n- His other 2 brothers (3 brothers total minus himself)\n- Alice + her 2 sisters = **3 sisters**\n\nAlice herself is a ",1061 "costUsd": 0.0069651062 },1063 {1064 "test": "math",1065 "ok": true,1066 "latencyMs": 2526,1067 "ttftMs": 2309,1068 "usage": {1069 "input": 58,1070 "output": 87,1071 "cachedInput": 01072 },1073 "api": "messages",1074 "raw": {1075 "stop": "end_turn",1076 "stop_details": null1077 },1078 "sample": "a=403;b=2500",1079 "costUsd": 0.0024651080 },1081 {1082 "test": "code",1083 "ok": true,1084 "latencyMs": 2481,1085 "ttftMs": 1450,1086 "usage": {1087 "input": 50,1088 "output": 125,1089 "cachedInput": 01090 },1091 "api": "messages",1092 "raw": {1093 "stop": "end_turn",1094 "stop_details": null1095 },1096 "sample": "```js\nfunction isPrime(n) {\n if (n < 2) return false;\n if (n < 4) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return true;\n}\n```",1097 "costUsd": 0.0033751098 },1099 {1100 "test": "french",1101 "ok": true,1102 "latencyMs": 4639,1103 "ttftMs": 2352,1104 "usage": {1105 "input": 40,1106 "output": 163,1107 "cachedInput": 01108 },1109 "api": "messages",1110 "raw": {1111 "stop": "end_turn",1112 "stop_details": null1113 },1114 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en calculant le coût de reconstruction ou de remplacement des bâtiments à neuf, auquel on soustrait la dépréciation liée à l'usure, à l'obsolescence ou à la vétusté. On ajoute ensuite à ce montant la valeur du terrai",1115 "costUsd": 0.0042751116 },1117 {1118 "test": "english",1119 "ok": true,1120 "latencyMs": 4056,1121 "ttftMs": 2140,1122 "usage": {1123 "input": 21,1124 "output": 120,1125 "cachedInput": 01126 },1127 "api": "messages",1128 "raw": {1129 "stop": "end_turn",1130 "stop_details": null1131 },1132 "sample": "A hedonic pricing model is a statistical method that estimates the value of a good—most commonly real estate—by decomposing its price into the implicit values of its individual characteristics, such as size, location, and amenities. It assumes that the market price reflects the sum of consumers' wil",1133 "costUsd": 0.0031051134 },1135 {1136 "test": "json",1137 "ok": true,1138 "latencyMs": 3613,1139 "ttftMs": 3359,1140 "usage": {1141 "input": 56,1142 "output": 115,1143 "cachedInput": 01144 },1145 "api": "messages",1146 "raw": {1147 "stop": "end_turn",1148 "stop_details": null1149 },1150 "sample": "{\"city\": \"Gatineau\", \"country\": \"Canada\", \"province\": \"Quebec\", \"population_estimate\": 291000}",1151 "costUsd": 0.0031551152 },1153 {1154 "test": "tools",1155 "ok": true,1156 "latencyMs": 19803,1157 "usage": {1158 "input": 596,1159 "output": 99,1160 "cachedInput": 01161 },1162 "api": "messages",1163 "raw": {1164 "stop": "tool_use",1165 "stop_details": null1166 },1167 "sample": "{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Gatineau, Quebec\"}}",1168 "costUsd": 0.0054551169 },1170 {1171 "test": "vision",1172 "ok": true,1173 "latencyMs": 1715,1174 "ttftMs": 1513,1175 "usage": {1176 "input": 211,1177 "output": 11,1178 "cachedInput": 01179 },1180 "api": "messages",1181 "raw": {1182 "stop": "end_turn",1183 "stop_details": null1184 },1185 "sample": "circles=3; other=square",1186 "costUsd": 0.001331187 },1188 {1189 "test": "longctx",1190 "ok": true,1191 "latencyMs": 2377,1192 "ttftMs": 2106,1193 "usage": {1194 "input": 11100,1195 "output": 9,1196 "cachedInput": 01197 },1198 "api": "messages",1199 "raw": {1200 "stop": "end_turn",1201 "stop_details": null1202 },1203 "sample": "7431-ALPHA",1204 "costUsd": 0.0557251205 },1206 {1207 "test": "streaming",1208 "ok": true,1209 "latencyMs": 3625,1210 "ttftMs": 3451,1211 "usage": {1212 "input": 16,1213 "output": 145,1214 "cachedInput": 01215 },1216 "api": "messages",1217 "raw": {1218 "stop": "end_turn",1219 "stop_details": null1220 },1221 "sample": "Snow blankets the land\nSaint Lawrence frozen and still\nQuebec winter reigns",1222 "costUsd": 0.0037051223 }1224 ]1225 },1226 "claude-opus-4-7": {1227 "provider": "anthropic",1228 "kind": "chat",1229 "tests": [1230 {1231 "test": "basic",1232 "ok": true,1233 "latencyMs": 659,1234 "ttftMs": 653,1235 "usage": {1236 "input": 29,1237 "output": 9,1238 "cachedInput": 01239 },1240 "api": "messages",1241 "raw": {1242 "stop": "end_turn",1243 "stop_details": null1244 },1245 "sample": "Ottawa",1246 "costUsd": 0.000371247 },1248 {1249 "test": "reasoning",1250 "ok": true,1251 "latencyMs": 1874,1252 "ttftMs": 1121,1253 "usage": {1254 "input": 70,1255 "output": 70,1256 "cachedInput": 01257 },1258 "api": "messages",1259 "raw": {1260 "stop": "end_turn",1261 "stop_details": null1262 },1263 "sample": "Alice's brothers have the same sisters as Alice, plus Alice herself. Alice has 2 sisters, so each brother has 2 + 1 = 3 sisters.\n\nANSWER: 3",1264 "costUsd": 0.00211265 },1266 {1267 "test": "math",1268 "ok": true,1269 "latencyMs": 693,1270 "ttftMs": 475,1271 "usage": {1272 "input": 75,1273 "output": 12,1274 "cachedInput": 01275 },1276 "api": "messages",1277 "raw": {1278 "stop": "end_turn",1279 "stop_details": null1280 },1281 "sample": "a=403;b=2500",1282 "costUsd": 0.0006751283 },1284 {1285 "test": "code",1286 "ok": true,1287 "latencyMs": 1659,1288 "ttftMs": 447,1289 "usage": {1290 "input": 73,1291 "output": 135,1292 "cachedInput": 01293 },1294 "api": "messages",1295 "raw": {1296 "stop": "end_turn",1297 "stop_details": null1298 },1299 "sample": "```js\nfunction isPrime(n) {\n if (n < 2) return false;\n if (n < 4) return true;\n if (n % 2 === 0) return false;\n if (n % 3 === 0) return false;\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return true;\n}\n```",1300 "costUsd": 0.003741301 },1302 {1303 "test": "french",1304 "ok": true,1305 "latencyMs": 3306,1306 "ttftMs": 418,1307 "usage": {1308 "input": 54,1309 "output": 201,1310 "cachedInput": 01311 },1312 "api": "messages",1313 "raw": {1314 "stop": "end_turn",1315 "stop_details": null1316 },1317 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en additionnant la valeur du terrain et le coût de reconstruction à neuf des bâtiments, duquel on soustrait la dépréciation accumulée (physique, fonctionnelle et économique). Elle est particulièrement utilisée pour l",1318 "costUsd": 0.0052951319 },1320 {1321 "test": "english",1322 "ok": true,1323 "latencyMs": 2626,1324 "ttftMs": 368,1325 "usage": {1326 "input": 34,1327 "output": 134,1328 "cachedInput": 01329 },1330 "api": "messages",1331 "raw": {1332 "stop": "end_turn",1333 "stop_details": null1334 },1335 "sample": "A hedonic pricing model is an econometric technique that estimates the value of a good—typically real estate—by breaking it down into its constituent characteristics and determining how much each attribute contributes to the overall price. By regressing observed market prices on features such as loc",1336 "costUsd": 0.003521337 },1338 {1339 "test": "json",1340 "ok": true,1341 "latencyMs": 898,1342 "ttftMs": 428,1343 "usage": {1344 "input": 74,1345 "output": 39,1346 "cachedInput": 01347 },1348 "api": "messages",1349 "raw": {1350 "stop": "end_turn",1351 "stop_details": null1352 },1353 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",1354 "costUsd": 0.0013451355 },1356 {1357 "test": "tools",1358 "ok": true,1359 "latencyMs": 6820,1360 "usage": {1361 "input": 805,1362 "output": 73,1363 "cachedInput": 01364 },1365 "api": "messages",1366 "raw": {1367 "stop": "tool_use",1368 "stop_details": null1369 },1370 "sample": "{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Gatineau, Quebec\"}}",1371 "costUsd": 0.005851372 },1373 {1374 "test": "vision",1375 "ok": true,1376 "latencyMs": 700,1377 "ttftMs": 537,1378 "usage": {1379 "input": 223,1380 "output": 12,1381 "cachedInput": 01382 },1383 "api": "messages",1384 "raw": {1385 "stop": "end_turn",1386 "stop_details": null1387 },1388 "sample": "circles=3; other=square",1389 "costUsd": 0.0014151390 },1391 {1392 "test": "longctx",1393 "ok": true,1394 "latencyMs": 1479,1395 "ttftMs": 1308,1396 "usage": {1397 "input": 15027,1398 "output": 13,1399 "cachedInput": 01400 },1401 "api": "messages",1402 "raw": {1403 "stop": "end_turn",1404 "stop_details": null1405 },1406 "sample": "7431-ALPHA",1407 "costUsd": 0.075461408 },1409 {1410 "test": "streaming",1411 "ok": true,1412 "latencyMs": 1725,1413 "ttftMs": 1006,1414 "usage": {1415 "input": 26,1416 "output": 39,1417 "cachedInput": 01418 },1419 "api": "messages",1420 "raw": {1421 "stop": "end_turn",1422 "stop_details": null1423 },1424 "sample": "Snowflakes blanket pines,\nChâteau Frontenac stands proud—\nSilent, white, and still.",1425 "costUsd": 0.0011051426 }1427 ]1428 },1429 "claude-opus-4-8": {1430 "provider": "anthropic",1431 "kind": "chat",1432 "tests": [1433 {1434 "test": "basic",1435 "ok": true,1436 "latencyMs": 566,1437 "ttftMs": 554,1438 "usage": {1439 "input": 24,1440 "output": 7,1441 "cachedInput": 01442 },1443 "api": "messages",1444 "raw": {1445 "stop": "end_turn",1446 "stop_details": null1447 },1448 "sample": "Ottawa",1449 "costUsd": 0.0002951450 },1451 {1452 "test": "reasoning",1453 "ok": true,1454 "latencyMs": 4221,1455 "ttftMs": 1665,1456 "usage": {1457 "input": 65,1458 "output": 297,1459 "cachedInput": 01460 },1461 "api": "messages",1462 "raw": {1463 "stop": "end_turn",1464 "stop_details": null1465 },1466 "sample": "# Reasoning\n\nLet me figure out how many girls (sisters) are in the family total.\n\n- Alice has **2 sisters**\n- Alice herself is also a girl/sister\n\nSo the total number of girls in the family is: Alice + 2 sisters = **3 girls**\n\nNow, from a brother's perspective, his sisters are **all the girls in the",1467 "costUsd": 0.007751468 },1469 {1470 "test": "math",1471 "ok": true,1472 "latencyMs": 1039,1473 "ttftMs": 926,1474 "usage": {1475 "input": 70,1476 "output": 58,1477 "cachedInput": 01478 },1479 "api": "messages",1480 "raw": {1481 "stop": "end_turn",1482 "stop_details": null1483 },1484 "sample": "a=403;b=2500",1485 "costUsd": 0.00181486 },1487 {1488 "test": "code",1489 "ok": true,1490 "latencyMs": 1895,1491 "ttftMs": 969,1492 "usage": {1493 "input": 68,1494 "output": 115,1495 "cachedInput": 01496 },1497 "api": "messages",1498 "raw": {1499 "stop": "end_turn",1500 "stop_details": null1501 },1502 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n < 2) return false;\n if (n < 4) return true;\n if (n % 2 === 0) return false;\n for (let i = 3; i * i <= n; i += 2) {\n if (n % i === 0) return false;\n }\n return true;\n}\n```",1503 "costUsd": 0.0032151504 },1505 {1506 "test": "french",1507 "ok": true,1508 "latencyMs": 2543,1509 "ttftMs": 441,1510 "usage": {1511 "input": 49,1512 "output": 167,1513 "cachedInput": 01514 },1515 "api": "messages",1516 "raw": {1517 "stop": "end_turn",1518 "stop_details": null1519 },1520 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en additionnant la valeur du terrain et le coût de reconstruction à neuf des bâtiments, duquel on soustrait la dépréciation liée à l'usure, à l'obsolescence ou à des facteurs externes. Cette approche est particulière",1521 "costUsd": 0.004421522 },1523 {1524 "test": "english",1525 "ok": true,1526 "latencyMs": 1887,1527 "ttftMs": 536,1528 "usage": {1529 "input": 29,1530 "output": 93,1531 "cachedInput": 01532 },1533 "api": "messages",1534 "raw": {1535 "stop": "end_turn",1536 "stop_details": null1537 },1538 "sample": "A hedonic pricing model is a statistical method that estimates the value of a good by breaking it down into its individual characteristics, determining how much each attribute contributes to the overall price. It is commonly used in real estate and economics to assess how factors like a property's s",1539 "costUsd": 0.002471540 },1541 {1542 "test": "json",1543 "ok": true,1544 "latencyMs": 899,1545 "ttftMs": 480,1546 "usage": {1547 "input": 69,1548 "output": 41,1549 "cachedInput": 01550 },1551 "api": "messages",1552 "raw": {1553 "stop": "end_turn",1554 "stop_details": null1555 },1556 "sample": "{\"city\": \"Gatineau\", \"country\": \"Canada\", \"province\": \"Quebec\", \"population_estimate\": 291000}",1557 "costUsd": 0.001371558 },1559 {1560 "test": "tools",1561 "ok": true,1562 "latencyMs": 1231,1563 "ttftMs": 622,1564 "usage": {1565 "input": 415,1566 "output": 80,1567 "cachedInput": 01568 },1569 "api": "messages",1570 "raw": {1571 "stop": "tool_use",1572 "stop_details": null1573 },1574 "sample": "I'll check the current weather in Gatineau, Quebec for you.",1575 "costUsd": 0.0040751576 },1577 {1578 "test": "vision",1579 "ok": true,1580 "latencyMs": 722,1581 "ttftMs": 516,1582 "usage": {1583 "input": 217,1584 "output": 10,1585 "cachedInput": 01586 },1587 "api": "messages",1588 "raw": {1589 "stop": "end_turn",1590 "stop_details": null1591 },1592 "sample": "circles=3; other=square",1593 "costUsd": 0.0013351594 },1595 {1596 "test": "longctx",1597 "ok": true,1598 "latencyMs": 1193,1599 "ttftMs": 986,1600 "usage": {1601 "input": 15022,1602 "output": 10,1603 "cachedInput": 01604 },1605 "api": "messages",1606 "raw": {1607 "stop": "end_turn",1608 "stop_details": null1609 },1610 "sample": "7431-ALPHA",1611 "costUsd": 0.075361612 },1613 {1614 "test": "streaming",1615 "ok": true,1616 "latencyMs": 1287,1617 "ttftMs": 800,1618 "usage": {1619 "input": 21,1620 "output": 49,1621 "cachedInput": 01622 },1623 "api": "messages",1624 "raw": {1625 "stop": "end_turn",1626 "stop_details": null1627 },1628 "sample": "Snow blankets the streets—\nMontreal in silent white,\nbreath freezes midair.",1629 "costUsd": 0.001331630 }1631 ]1632 },1633 "claude-opus-5": {1634 "provider": "anthropic",1635 "kind": "chat",1636 "tests": [1637 {1638 "test": "reasoning",1639 "ok": true,1640 "latencyMs": 4570,1641 "ttftMs": 2100,1642 "usage": {1643 "input": 65,1644 "output": 284,1645 "cachedInput": 01646 },1647 "api": "messages",1648 "raw": {1649 "stop": "end_turn",1650 "stop_details": null1651 },1652 "sample": "**Step 1: Count the girls in the family.**\nAlice has 2 sisters, so including Alice herself, there are 3 girls total.\n\n**Step 2: Count the boys.**\nAlice has 3 brothers, so there are 3 boys.\n\n**Step 3: Take a brother's point of view.**\nA brother is one of the boys. His sisters are all the girls in the",1653 "costUsd": 0.0074251654 },1655 {1656 "test": "math",1657 "ok": true,1658 "latencyMs": 2242,1659 "ttftMs": 2137,1660 "usage": {1661 "input": 70,1662 "output": 60,1663 "cachedInput": 01664 },1665 "api": "messages",1666 "raw": {1667 "stop": "end_turn",1668 "stop_details": null1669 },1670 "sample": "a=403;b=2500",1671 "costUsd": 0.001851672 },1673 {1674 "test": "code",1675 "ok": true,1676 "latencyMs": 2144,1677 "ttftMs": 849,1678 "usage": {1679 "input": 68,1680 "output": 138,1681 "cachedInput": 01682 },1683 "api": "messages",1684 "raw": {1685 "stop": "end_turn",1686 "stop_details": null1687 },1688 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n < 2) return false;\n if (n < 4) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n\n return true;\n}\n```",1689 "costUsd": 0.003791690 },1691 {1692 "test": "french",1693 "ok": true,1694 "latencyMs": 3598,1695 "ttftMs": 1390,1696 "usage": {1697 "input": 49,1698 "output": 252,1699 "cachedInput": 01700 },1701 "api": "messages",1702 "raw": {1703 "stop": "end_turn",1704 "stop_details": null1705 },1706 "sample": "La méthode du coût est une approche d'évaluation immobilière qui estime la valeur d'un bien en additionnant la valeur du terrain et le coût de reconstruction ou de remplacement des constructions à neuf, diminué de la dépréciation accumulée (obsolescence physique, fonctionnelle et économique). Elle e",1707 "costUsd": 0.0065451708 },1709 {1710 "test": "english",1711 "ok": true,1712 "latencyMs": 2795,1713 "ttftMs": 710,1714 "usage": {1715 "input": 29,1716 "output": 179,1717 "cachedInput": 01718 },1719 "api": "messages",1720 "raw": {1721 "stop": "end_turn",1722 "stop_details": null1723 },1724 "sample": "A hedonic pricing model is a regression-based method that estimates the value of a good by breaking it into its constituent characteristics and inferring the implicit price that buyers pay for each attribute. It is most commonly applied to housing markets—where a home's price is modeled as a functio",1725 "costUsd": 0.004621726 },1727 {1728 "test": "json",1729 "ok": true,1730 "latencyMs": 1436,1731 "ttftMs": 1100,1732 "usage": {1733 "input": 69,1734 "output": 101,1735 "cachedInput": 01736 },1737 "api": "messages",1738 "raw": {1739 "stop": "end_turn",1740 "stop_details": null1741 },1742 "sample": "{\"city\": \"Gatineau\", \"country\": \"Canada\", \"province\": \"Quebec\", \"population_estimate\": 291041}",1743 "costUsd": 0.002871744 },1745 {1746 "test": "tools",1747 "ok": true,1748 "latencyMs": 1854,1749 "ttftMs": 1136,1750 "usage": {1751 "input": 411,1752 "output": 75,1753 "cachedInput": 01754 },1755 "api": "messages",1756 "raw": {1757 "stop": "tool_use",1758 "stop_details": null1759 },1760 "sample": "I'll check the current weather in Gatineau for you.",1761 "costUsd": 0.003931762 },1763 {1764 "test": "vision",1765 "ok": true,1766 "latencyMs": 1131,1767 "ttftMs": 1102,1768 "usage": {1769 "input": 217,1770 "output": 23,1771 "cachedInput": 01772 },1773 "api": "messages",1774 "raw": {1775 "stop": "end_turn",1776 "stop_details": null1777 },1778 "sample": "circles=3; other=square",1779 "costUsd": 0.001661780 },1781 {1782 "test": "longctx",1783 "ok": true,1784 "latencyMs": 959,1785 "ttftMs": 931,1786 "usage": {1787 "input": 15022,1788 "output": 10,1789 "cachedInput": 01790 },1791 "api": "messages",1792 "raw": {1793 "stop": "end_turn",1794 "stop_details": null1795 },1796 "sample": "7431-ALPHA",1797 "costUsd": 0.075361798 },1799 {1800 "test": "basic",1801 "ok": true,1802 "latencyMs": 1655,1803 "ttftMs": 1650,1804 "usage": {1805 "input": 24,1806 "output": 7,1807 "cachedInput": 01808 },1809 "api": "messages",1810 "raw": {1811 "stop": "end_turn",1812 "stop_details": null1813 },1814 "sample": "Ottawa",1815 "costUsd": 0.0002951816 },1817 {1818 "test": "streaming",1819 "ok": true,1820 "latencyMs": 7122,1821 "ttftMs": 6771,1822 "usage": {1823 "input": 21,1824 "output": 472,1825 "cachedInput": 01826 },1827 "api": "messages",1828 "raw": {1829 "stop": "end_turn",1830 "stop_details": null1831 },1832 "sample": "Snow swallows the fence—\nthe Saint Lawrence groans with ice;\nwoodsmoke, then silence.",1833 "costUsd": 0.0119051834 }1835 ]1836 },1837 "claude-opus-5-5": {1838 "provider": "anthropic",1839 "kind": "chat",1840 "tests": [1841 {1842 "test": "basic",1843 "ok": true,1844 "latencyMs": 842,1845 "ttftMs": 830,1846 "usage": {1847 "input": 26,1848 "output": 7,1849 "cachedInput": 01850 },1851 "api": "messages",1852 "raw": {1853 "stop": "end_turn",1854 "stop_details": null1855 },1856 "sample": "Ottawa",1857 "costUsd": 0.0002441858 },1859 {1860 "test": "reasoning",1861 "ok": true,1862 "latencyMs": 1742,1863 "ttftMs": 1057,1864 "usage": {1865 "input": 67,1866 "output": 156,1867 "cachedInput": 01868 },1869 "api": "messages",1870 "raw": {1871 "stop": "end_turn",1872 "stop_details": null1873 },1874 "sample": "Each of Alice's brothers has **3** sisters.\n\n1. Alice is a girl, and she has 2 sisters. So the family has 3 girls: Alice and her 2 sisters.\n2. Each brother is a sibling of every girl in the family, so his sisters are Alice plus her 2 sisters.\n\nANSWER: 3",1875 "costUsd": 0.0033881876 },1877 {1878 "test": "math",1879 "ok": true,1880 "latencyMs": 1216,1881 "ttftMs": 752,1882 "usage": {1883 "input": 72,1884 "output": 37,1885 "cachedInput": 01886 },1887 "api": "messages",1888 "raw": {1889 "stop": "end_turn",1890 "stop_details": null1891 },1892 "sample": "a=403;b=2500",1893 "costUsd": 0.0010281894 },1895 {1896 "test": "code",1897 "ok": true,1898 "latencyMs": 1778,1899 "ttftMs": 697,1900 "usage": {1901 "input": 70,1902 "output": 138,1903 "cachedInput": 01904 },1905 "api": "messages",1906 "raw": {1907 "stop": "end_turn",1908 "stop_details": null1909 },1910 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n < 2) return false;\n if (n < 4) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return true;\n}\n```",1911 "costUsd": 0.003041912 },1913 {1914 "test": "french",1915 "ok": true,1916 "latencyMs": 2832,1917 "ttftMs": 954,1918 "usage": {1919 "input": 51,1920 "output": 232,1921 "cachedInput": 01922 },1923 "api": "messages",1924 "raw": {1925 "stop": "end_turn",1926 "stop_details": null1927 },1928 "sample": "La méthode du coût est une technique d'évaluation immobilière qui estime la valeur d'un bien en additionnant la valeur du terrain, considéré comme non bâti, et le coût de reconstruction ou de remplacement à neuf des bâtiments, diminué de la dépréciation accumulée (usure physique, désuétude fonctionn",1929 "costUsd": 0.0048441930 },1931 {1932 "test": "english",1933 "ok": true,1934 "latencyMs": 1905,1935 "ttftMs": 633,1936 "usage": {1937 "input": 31,1938 "output": 151,1939 "cachedInput": 01940 },1941 "api": "messages",1942 "raw": {1943 "stop": "end_turn",1944 "stop_details": null1945 },1946 "sample": "A hedonic pricing model is an economic method that estimates the value of a good by breaking its price down into the separate contributions of its individual characteristics, such as a house's size, location, number of bedrooms, and proximity to parks or pollution. Using regression analysis on marke",1947 "costUsd": 0.0031441948 },1949 {1950 "test": "json",1951 "ok": true,1952 "latencyMs": 1503,1953 "ttftMs": 995,1954 "usage": {1955 "input": 71,1956 "output": 75,1957 "cachedInput": 01958 },1959 "api": "messages",1960 "raw": {1961 "stop": "end_turn",1962 "stop_details": null1963 },1964 "sample": "{\"city\": \"Gatineau\", \"country\": \"Canada\", \"province\": \"Quebec\", \"population_estimate\": 291041}",1965 "costUsd": 0.0017841966 },1967 {1968 "test": "tools",1969 "ok": true,1970 "latencyMs": 2031,1971 "usage": {1972 "input": 413,1973 "output": 57,1974 "cachedInput": 01975 },1976 "api": "messages",1977 "raw": {1978 "stop": "tool_use",1979 "stop_details": null1980 },1981 "sample": "{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Gatineau, Quebec\"}}",1982 "costUsd": 0.0027921983 },1984 {1985 "test": "vision",1986 "ok": true,1987 "latencyMs": 1225,1988 "ttftMs": 689,1989 "usage": {1990 "input": 219,1991 "output": 10,1992 "cachedInput": 01993 },1994 "api": "messages",1995 "raw": {1996 "stop": "end_turn",1997 "stop_details": null1998 },1999 "sample": "circles=3; other=square",2000 "costUsd": 0.0010762001 },2002 {2003 "test": "longctx",2004 "ok": true,2005 "latencyMs": 2615,2006 "ttftMs": 2584,2007 "usage": {2008 "input": 15024,2009 "output": 175,2010 "cachedInput": 02011 },2012 "api": "messages",2013 "raw": {2014 "stop": "end_turn",2015 "stop_details": null2016 },2017 "sample": "7431-ALPHA",2018 "costUsd": 0.0635962019 },2020 {2021 "test": "streaming",2022 "ok": true,2023 "latencyMs": 4977,2024 "ttftMs": 4635,2025 "usage": {2026 "input": 23,2027 "output": 400,2028 "cachedInput": 02029 },2030 "api": "messages",2031 "raw": {2032 "stop": "max_tokens",2033 "stop_details": null2034 },2035 "sample": "**Winter in Québec**\n\nSnow on old stone walls,\nthe frozen Saint Lawrence groans\nbeneath a pale mo",2036 "costUsd": 0.0080922037 }2038 ]2039 },2040 "claude-sonnet-4-5-20250929": {2041 "provider": "anthropic",2042 "kind": "chat",2043 "tests": [2044 {2045 "test": "basic",2046 "ok": true,2047 "latencyMs": 854,2048 "ttftMs": 780,2049 "usage": {2050 "input": 20,2051 "output": 4,2052 "cachedInput": 02053 },2054 "api": "messages",2055 "raw": {2056 "stop": "end_turn",2057 "stop_details": null2058 },2059 "sample": "Ottawa",2060 "costUsd": 0.000122061 },2062 {2063 "test": "reasoning",2064 "ok": true,2065 "latencyMs": 3399,2066 "ttftMs": 669,2067 "usage": {2068 "input": 48,2069 "output": 146,2070 "cachedInput": 02071 },2072 "api": "messages",2073 "raw": {2074 "stop": "end_turn",2075 "stop_details": null2076 },2077 "sample": "Let me think through this step by step.\n\nFirst, let me identify who is in Alice's family:\n- Alice (female)\n- 3 brothers (males)\n- 2 sisters (females)\n\nSo the siblings in the family are:\n- Alice\n- 2 other sisters\n- 3 brothers\n\nNow, the question asks: How many sisters does each of Alice's brothers hav",2078 "costUsd": 0.0023342079 },2080 {2081 "test": "math",2082 "ok": true,2083 "latencyMs": 1011,2084 "ttftMs": 618,2085 "usage": {2086 "input": 58,2087 "output": 11,2088 "cachedInput": 02089 },2090 "api": "messages",2091 "raw": {2092 "stop": "end_turn",2093 "stop_details": null2094 },2095 "sample": "a=403;b=2500",2096 "costUsd": 0.0003392097 },2098 {2099 "test": "code",2100 "ok": true,2101 "latencyMs": 1934,2102 "ttftMs": 937,2103 "usage": {2104 "input": 50,2105 "output": 132,2106 "cachedInput": 02107 },2108 "api": "messages",2109 "raw": {2110 "stop": "end_turn",2111 "stop_details": null2112 },2113 "sample": "\n```js\nfunction isPrime(n) {\n if (n <= 1) return false;\n if (n <= 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n \n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n \n return true;\n}\n```\n```",2114 "costUsd": 0.002132115 },2116 {2117 "test": "french",2118 "ok": true,2119 "latencyMs": 3130,2120 "ttftMs": 595,2121 "usage": {2122 "input": 40,2123 "output": 117,2124 "cachedInput": 02125 },2126 "api": "messages",2127 "raw": {2128 "stop": "end_turn",2129 "stop_details": null2130 },2131 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en calculant le coût de remplacement du bâtiment à neuf, auquel on soustrait la dépréciation liée à l'âge et à l'obsolescence, puis on ajoute la valeur du terrain. Cette méthode est particulièrement utilisée pour les",2132 "costUsd": 0.0018752133 },2134 {2135 "test": "english",2136 "ok": true,2137 "latencyMs": 2468,2138 "ttftMs": 840,2139 "usage": {2140 "input": 21,2141 "output": 64,2142 "cachedInput": 02143 },2144 "api": "messages",2145 "raw": {2146 "stop": "end_turn",2147 "stop_details": null2148 },2149 "sample": "A hedonic pricing model is a statistical method that breaks down the price of a good into the values of its individual characteristics or attributes. It is commonly used to estimate the implicit value of specific features, such as determining how much a swimming pool or an extra bedroom contributes ",2150 "costUsd": 0.0010232151 },2152 {2153 "test": "json",2154 "ok": true,2155 "latencyMs": 1158,2156 "ttftMs": 820,2157 "usage": {2158 "input": 56,2159 "output": 48,2160 "cachedInput": 02161 },2162 "api": "messages",2163 "raw": {2164 "stop": "end_turn",2165 "stop_details": null2166 },2167 "sample": "```json\n{\n \"city\": \"Gatineau\",\n \"country\": \"Canada\",\n \"province\": \"Quebec\",\n \"population_estimate\": 291041\n}\n```",2168 "costUsd": 0.0008882169 },2170 {2171 "test": "tools",2172 "ok": true,2173 "latencyMs": 1432,2174 "usage": {2175 "input": 595,2176 "output": 58,2177 "cachedInput": 02178 },2179 "api": "messages",2180 "raw": {2181 "stop": "tool_use",2182 "stop_details": null2183 },2184 "sample": "{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Gatineau, Quebec\"}}",2185 "costUsd": 0.0026552186 },2187 {2188 "test": "vision",2189 "ok": true,2190 "latencyMs": 732,2191 "ttftMs": 539,2192 "usage": {2193 "input": 211,2194 "output": 10,2195 "cachedInput": 02196 },2197 "api": "messages",2198 "raw": {2199 "stop": "end_turn",2200 "stop_details": null2201 },2202 "sample": "circles=3; other=square",2203 "costUsd": 0.0007832204 },2205 {2206 "test": "longctx",2207 "ok": true,2208 "latencyMs": 1212,2209 "ttftMs": 1116,2210 "usage": {2211 "input": 11100,2212 "output": 9,2213 "cachedInput": 02214 },2215 "api": "messages",2216 "raw": {2217 "stop": "end_turn",2218 "stop_details": null2219 },2220 "sample": "7431-ALPHA",2221 "costUsd": 0.0334352222 },2223 {2224 "test": "streaming",2225 "ok": true,2226 "latencyMs": 1675,2227 "ttftMs": 785,2228 "usage": {2229 "input": 16,2230 "output": 32,2231 "cachedInput": 02232 },2233 "api": "messages",2234 "raw": {2235 "stop": "end_turn",2236 "stop_details": null2237 },2238 "sample": "# Winter in Quebec\n\nSnow drifts bury doors\nFrost paints white on windowpanes\nCold breath hangs like smoke",2239 "costUsd": 0.0005282240 }2241 ]2242 },2243 "claude-sonnet-4-6": {2244 "provider": "anthropic",2245 "kind": "chat",2246 "tests": [2247 {2248 "test": "basic",2249 "ok": true,2250 "latencyMs": 1058,2251 "ttftMs": 958,2252 "usage": {2253 "input": 20,2254 "output": 4,2255 "cachedInput": 02256 },2257 "api": "messages",2258 "raw": {2259 "stop": "end_turn",2260 "stop_details": null2261 },2262 "sample": "Ottawa",2263 "costUsd": 0.000122264 },2265 {2266 "test": "reasoning",2267 "ok": true,2268 "latencyMs": 2909,2269 "ttftMs": 1821,2270 "usage": {2271 "input": 48,2272 "output": 143,2273 "cachedInput": 02274 },2275 "api": "messages",2276 "raw": {2277 "stop": "end_turn",2278 "stop_details": null2279 },2280 "sample": "## Thinking it through:\n\nAlice is a girl, so she counts as a sister to her brothers.\n\nEach of Alice's brothers has these sisters:\n- Alice herself\n- Alice's 2 sisters\n\nSo each brother has **3 sisters** total.\n\nANSWER: 3",2281 "costUsd": 0.0022892282 },2283 {2284 "test": "math",2285 "ok": true,2286 "latencyMs": 2260,2287 "ttftMs": 2065,2288 "usage": {2289 "input": 58,2290 "output": 111,2291 "cachedInput": 02292 },2293 "api": "messages",2294 "raw": {2295 "stop": "end_turn",2296 "stop_details": null2297 },2298 "sample": "a=403;b=2500",2299 "costUsd": 0.0018392300 },2301 {2302 "test": "code",2303 "ok": true,2304 "latencyMs": 1912,2305 "ttftMs": 994,2306 "usage": {2307 "input": 50,2308 "output": 123,2309 "cachedInput": 02310 },2311 "api": "messages",2312 "raw": {2313 "stop": "end_turn",2314 "stop_details": null2315 },2316 "sample": "```js\nfunction isPrime(n) {\n if (n < 2) return false;\n if (n === 2) return true;\n if (n % 2 === 0) return false;\n for (let i = 3; i <= Math.sqrt(n); i += 2) {\n if (n % i === 0) return false;\n }\n return true;\n}\n```",2317 "costUsd": 0.0019952318 },2319 {2320 "test": "french",2321 "ok": true,2322 "latencyMs": 4367,2323 "ttftMs": 1616,2324 "usage": {2325 "input": 40,2326 "output": 171,2327 "cachedInput": 02328 },2329 "api": "messages",2330 "raw": {2331 "stop": "end_turn",2332 "stop_details": null2333 },2334 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en additionnant la valeur du terrain et le coût de reconstruction à neuf des bâtiments, diminué d'une dépréciation tenant compte de leur vétusté et de leur obsolescence. Cette approche est particulièrement utilisée p",2335 "costUsd": 0.0026852336 },2337 {2338 "test": "english",2339 "ok": true,2340 "latencyMs": 3392,2341 "ttftMs": 1294,2342 "usage": {2343 "input": 21,2344 "output": 123,2345 "cachedInput": 02346 },2347 "api": "messages",2348 "raw": {2349 "stop": "end_turn",2350 "stop_details": null2351 },2352 "sample": "A hedonic pricing model is an econometric method used to estimate the value of a good by breaking it down into its constituent characteristics and determining how each attribute contributes to its overall price. It is commonly applied in real estate, where a home's price is modeled as a function of ",2353 "costUsd": 0.0019082354 },2355 {2356 "test": "json",2357 "ok": true,2358 "latencyMs": 2958,2359 "ttftMs": 2718,2360 "usage": {2361 "input": 56,2362 "output": 158,2363 "cachedInput": 02364 },2365 "api": "messages",2366 "raw": {2367 "stop": "end_turn",2368 "stop_details": null2369 },2370 "sample": "{\"city\": \"Gatineau\", \"country\": \"Canada\", \"province\": \"Quebec\", \"population_estimate\": 291041}",2371 "costUsd": 0.0025382372 },2373 {2374 "test": "tools",2375 "ok": true,2376 "latencyMs": 2474,2377 "ttftMs": 1732,2378 "usage": {2379 "input": 596,2380 "output": 121,2381 "cachedInput": 02382 },2383 "api": "messages",2384 "raw": {2385 "stop": "tool_use",2386 "stop_details": null2387 },2388 "sample": "Sure! Let me fetch the current weather for Gatineau, Quebec right now!",2389 "costUsd": 0.0036032390 },2391 {2392 "test": "vision",2393 "ok": true,2394 "latencyMs": 951,2395 "ttftMs": 780,2396 "usage": {2397 "input": 211,2398 "output": 10,2399 "cachedInput": 02400 },2401 "api": "messages",2402 "raw": {2403 "stop": "end_turn",2404 "stop_details": null2405 },2406 "sample": "circles=3; other=square",2407 "costUsd": 0.0007832408 },2409 {2410 "test": "longctx",2411 "ok": true,2412 "latencyMs": 2763,2413 "ttftMs": 2548,2414 "usage": {2415 "input": 11100,2416 "output": 9,2417 "cachedInput": 02418 },2419 "api": "messages",2420 "raw": {2421 "stop": "end_turn",2422 "stop_details": null2423 },2424 "sample": "7431-ALPHA",2425 "costUsd": 0.0334352426 },2427 {2428 "test": "streaming",2429 "ok": true,2430 "latencyMs": 5794,2431 "ttftMs": 5791,2432 "usage": {2433 "input": 16,2434 "output": 300,2435 "cachedInput": 02436 },2437 "api": "messages",2438 "raw": {2439 "stop": "end_turn",2440 "stop_details": null2441 },2442 "sample": "**Snow blankets the plains**\n**St. Lawrence locked in silence**\n**Maple dreams of spring**",2443 "costUsd": 0.0045482444 }2445 ]2446 },2447 "claude-sonnet-5": {2448 "provider": "anthropic",2449 "kind": "chat",2450 "tests": [2451 {2452 "test": "basic",2453 "ok": true,2454 "latencyMs": 908,2455 "ttftMs": 515,2456 "usage": {2457 "input": 24,2458 "output": 9,2459 "cachedInput": 02460 },2461 "api": "messages",2462 "raw": {2463 "stop": "end_turn",2464 "stop_details": null2465 },2466 "sample": "**Ottawa**",2467 "costUsd": 0.0001382468 },2469 {2470 "test": "reasoning",2471 "ok": true,2472 "latencyMs": 2367,2473 "ttftMs": 1557,2474 "usage": {2475 "input": 65,2476 "output": 194,2477 "cachedInput": 02478 },2479 "api": "messages",2480 "raw": {2481 "stop": "end_turn",2482 "stop_details": null2483 },2484 "sample": "Each brother shares the same siblings as Alice. Alice has 2 sisters, which means there are 3 girls total in the family (Alice + her 2 sisters). From a brother's perspective, all 3 girls (including Alice) are his sisters.\n\nANSWER: 3",2485 "costUsd": 0.002072486 },2487 {2488 "test": "math",2489 "ok": true,2490 "latencyMs": 1133,2491 "ttftMs": 910,2492 "usage": {2493 "input": 70,2494 "output": 56,2495 "cachedInput": 02496 },2497 "api": "messages",2498 "raw": {2499 "stop": "end_turn",2500 "stop_details": null2501 },2502 "sample": "a=403;b=2500",2503 "costUsd": 0.00072504 },2505 {2506 "test": "code",2507 "ok": true,2508 "latencyMs": 1168,2509 "ttftMs": 475,2510 "usage": {2511 "input": 68,2512 "output": 103,2513 "cachedInput": 02514 },2515 "api": "messages",2516 "raw": {2517 "stop": "end_turn",2518 "stop_details": null2519 },2520 "sample": "```js\nfunction isPrime(n) {\n if (n < 2) return false;\n if (n === 2) return true;\n if (n % 2 === 0) return false;\n for (let i = 3; i * i <= n; i += 2) {\n if (n % i === 0) return false;\n }\n return true;\n}\n```",2521 "costUsd": 0.0011662522 },2523 {2524 "test": "french",2525 "ok": true,2526 "latencyMs": 2344,2527 "ttftMs": 470,2528 "usage": {2529 "input": 49,2530 "output": 190,2531 "cachedInput": 02532 },2533 "api": "messages",2534 "raw": {2535 "stop": "end_turn",2536 "stop_details": null2537 },2538 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en calculant le coût de remplacement ou de reconstruction à neuf de la construction, duquel on déduit la dépréciation liée à l'usure, à l'obsolescence physique, fonctionnelle ou économique, puis en ajoutant la valeur",2539 "costUsd": 0.0019982540 },2541 {2542 "test": "english",2543 "ok": true,2544 "latencyMs": 1532,2545 "ttftMs": 460,2546 "usage": {2547 "input": 29,2548 "output": 113,2549 "cachedInput": 02550 },2551 "api": "messages",2552 "raw": {2553 "stop": "end_turn",2554 "stop_details": null2555 },2556 "sample": "A hedonic pricing model is a statistical technique used to estimate the value of a good or service by breaking it down into its constituent characteristics and determining the implicit price of each attribute. It is commonly applied to markets like real estate, where the price of a home is analyzed ",2557 "costUsd": 0.0011882558 },2559 {2560 "test": "json",2561 "ok": true,2562 "latencyMs": 1069,2563 "ttftMs": 497,2564 "usage": {2565 "input": 69,2566 "output": 41,2567 "cachedInput": 02568 },2569 "api": "messages",2570 "raw": {2571 "stop": "end_turn",2572 "stop_details": null2573 },2574 "sample": "{\"city\": \"Gatineau\", \"country\": \"Canada\", \"province\": \"Quebec\", \"population_estimate\": 291041}",2575 "costUsd": 0.0005482576 },2577 {2578 "test": "tools",2579 "ok": true,2580 "latencyMs": 1268,2581 "usage": {2582 "input": 479,2583 "output": 93,2584 "cachedInput": 02585 },2586 "api": "messages",2587 "raw": {2588 "stop": "tool_use",2589 "stop_details": null2590 },2591 "sample": "{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Gatineau, Quebec\"}}",2592 "costUsd": 0.0018882593 },2594 {2595 "test": "vision",2596 "ok": true,2597 "latencyMs": 993,2598 "ttftMs": 576,2599 "usage": {2600 "input": 217,2601 "output": 10,2602 "cachedInput": 02603 },2604 "api": "messages",2605 "raw": {2606 "stop": "end_turn",2607 "stop_details": null2608 },2609 "sample": "circles=3; other=square",2610 "costUsd": 0.0005342611 },2612 {2613 "test": "longctx",2614 "ok": true,2615 "latencyMs": 1153,2616 "ttftMs": 1128,2617 "usage": {2618 "input": 15022,2619 "output": 10,2620 "cachedInput": 02621 },2622 "api": "messages",2623 "raw": {2624 "stop": "end_turn",2625 "stop_details": null2626 },2627 "sample": "7431-ALPHA",2628 "costUsd": 0.0301442629 },2630 {2631 "test": "streaming",2632 "ok": true,2633 "latencyMs": 1170,2634 "ttftMs": 528,2635 "usage": {2636 "input": 21,2637 "output": 35,2638 "cachedInput": 02639 },2640 "api": "messages",2641 "raw": {2642 "stop": "end_turn",2643 "stop_details": null2644 },2645 "sample": "Snow blankets the fields—\nMaple trees stand bare and still,\nBreath turns into mist.",2646 "costUsd": 0.0003922647 }2648 ]2649 },2650 "claude-sonnet-5-5": {2651 "provider": "anthropic",2652 "kind": "chat",2653 "tests": [2654 {2655 "test": "basic",2656 "ok": true,2657 "latencyMs": 1025,2658 "ttftMs": 1017,2659 "usage": {2660 "input": 26,2661 "output": 7,2662 "cachedInput": 02663 },2664 "api": "messages",2665 "raw": {2666 "stop": "end_turn",2667 "stop_details": null2668 },2669 "sample": "Ottawa",2670 "costUsd": 0.0001222671 },2672 {2673 "test": "reasoning",2674 "ok": true,2675 "latencyMs": 1538,2676 "ttftMs": 561,2677 "usage": {2678 "input": 67,2679 "output": 89,2680 "cachedInput": 02681 },2682 "api": "messages",2683 "raw": {2684 "stop": "end_turn",2685 "stop_details": null2686 },2687 "sample": "Alice has 2 sisters, so the girls in the family are Alice plus those 2 sisters, which makes 3 girls in total.\n\nEach of Alice's brothers has all 3 girls as sisters: Alice and her 2 sisters.\n\nANSWER: 3",2688 "costUsd": 0.0010242689 },2690 {2691 "test": "math",2692 "ok": true,2693 "latencyMs": 1026,2694 "ttftMs": 525,2695 "usage": {2696 "input": 72,2697 "output": 10,2698 "cachedInput": 02699 },2700 "api": "messages",2701 "raw": {2702 "stop": "end_turn",2703 "stop_details": null2704 },2705 "sample": "a=403;b=2500",2706 "costUsd": 0.0002442707 },2708 {2709 "test": "code",2710 "ok": true,2711 "latencyMs": 1496,2712 "ttftMs": 520,2713 "usage": {2714 "input": 70,2715 "output": 144,2716 "cachedInput": 02717 },2718 "api": "messages",2719 "raw": {2720 "stop": "end_turn",2721 "stop_details": null2722 },2723 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n < 2) return false;\n if (n === 2 || n === 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return true;\n}\n```",2724 "costUsd": 0.001582725 },2726 {2727 "test": "french",2728 "ok": true,2729 "latencyMs": 2180,2730 "ttftMs": 501,2731 "usage": {2732 "input": 51,2733 "output": 222,2734 "cachedInput": 02735 },2736 "api": "messages",2737 "raw": {2738 "stop": "end_turn",2739 "stop_details": null2740 },2741 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en additionnant la valeur du terrain (déterminée par comparaison avec des terrains similaires) et le coût de reconstruction à neuf du bâtiment, dont on retranche la dépréciation liée à la vétusté, à l'usure physique ",2742 "costUsd": 0.0023222743 },2744 {2745 "test": "english",2746 "ok": true,2747 "latencyMs": 2293,2748 "ttftMs": 820,2749 "usage": {2750 "input": 31,2751 "output": 138,2752 "cachedInput": 02753 },2754 "api": "messages",2755 "raw": {2756 "stop": "end_turn",2757 "stop_details": null2758 },2759 "sample": "A hedonic pricing model is a statistical method, typically a regression, that estimates the value of a good by breaking its price down into the contributions of its individual characteristics, such as a house's size, location, and age. It is widely used in real estate, consumer price indexes, and en",2760 "costUsd": 0.0014422761 },2762 {2763 "test": "json",2764 "ok": true,2765 "latencyMs": 1143,2766 "ttftMs": 524,2767 "usage": {2768 "input": 71,2769 "output": 37,2770 "cachedInput": 02771 },2772 "api": "messages",2773 "raw": {2774 "stop": "end_turn",2775 "stop_details": null2776 },2777 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",2778 "costUsd": 0.0005122779 },2780 {2781 "test": "tools",2782 "ok": true,2783 "latencyMs": 1279,2784 "usage": {2785 "input": 413,2786 "output": 57,2787 "cachedInput": 02788 },2789 "api": "messages",2790 "raw": {2791 "stop": "tool_use",2792 "stop_details": null2793 },2794 "sample": "{\"name\":\"get_weather\",\"arguments\":{\"location\":\"Gatineau, Quebec\"}}",2795 "costUsd": 0.0013962796 },2797 {2798 "test": "vision",2799 "ok": true,2800 "latencyMs": 1091,2801 "ttftMs": 586,2802 "usage": {2803 "input": 219,2804 "output": 10,2805 "cachedInput": 02806 },2807 "api": "messages",2808 "raw": {2809 "stop": "end_turn",2810 "stop_details": null2811 },2812 "sample": "circles=3; other=square",2813 "costUsd": 0.0005382814 },2815 {2816 "test": "longctx",2817 "ok": true,2818 "latencyMs": 1200,2819 "ttftMs": 745,2820 "usage": {2821 "input": 15024,2822 "output": 10,2823 "cachedInput": 02824 },2825 "api": "messages",2826 "raw": {2827 "stop": "end_turn",2828 "stop_details": null2829 },2830 "sample": "7431-ALPHA",2831 "costUsd": 0.0301482832 },2833 {2834 "test": "streaming",2835 "ok": true,2836 "latencyMs": 1392,2837 "ttftMs": 843,2838 "usage": {2839 "input": 23,2840 "output": 32,2841 "cachedInput": 02842 },2843 "api": "messages",2844 "raw": {2845 "stop": "end_turn",2846 "stop_details": null2847 },2848 "sample": "Snow hushes the streets,\nbreath curls white above poutine—\nthe cold sings in French.",2849 "costUsd": 0.0003662850 }2851 ]2852 }2853 }2854}