spb/chat-spboucher-ai
Public
1{2 "generatedAt": "2026-11-01T07:18:11.428Z",3 "provider": "openai",4 "budgetUsd": 2,5 "stoppedByBudget": false,6 "models": {7 "babbage-002": {8 "provider": "openai",9 "kind": "legacy-completion",10 "tests": []11 },12 "chatgpt-image-latest": {13 "provider": "openai",14 "kind": "image",15 "tests": []16 },17 "davinci-002": {18 "provider": "openai",19 "kind": "legacy-completion",20 "tests": []21 },22 "gpt-3.5-turbo-instruct": {23 "provider": "openai",24 "kind": "legacy-completion",25 "tests": []26 },27 "gpt-3.5-turbo-instruct-0914": {28 "provider": "openai",29 "kind": "legacy-completion",30 "tests": []31 },32 "gpt-4o-mini-transcribe": {33 "provider": "openai",34 "kind": "stt",35 "tests": []36 },37 "gpt-4o-mini-tts": {38 "provider": "openai",39 "kind": "tts",40 "tests": []41 },42 "gpt-4o-transcribe": {43 "provider": "openai",44 "kind": "stt",45 "tests": []46 },47 "gpt-4o-transcribe-diarize": {48 "provider": "openai",49 "kind": "stt",50 "tests": []51 },52 "gpt-audio": {53 "provider": "openai",54 "kind": "audio-chat",55 "tests": []56 },57 "gpt-audio-1.5": {58 "provider": "openai",59 "kind": "audio-chat",60 "tests": []61 },62 "gpt-audio-mini": {63 "provider": "openai",64 "kind": "audio-chat",65 "tests": []66 },67 "gpt-image-1": {68 "provider": "openai",69 "kind": "image",70 "tests": []71 },72 "gpt-image-1-mini": {73 "provider": "openai",74 "kind": "image",75 "tests": []76 },77 "gpt-image-1.5": {78 "provider": "openai",79 "kind": "image",80 "tests": []81 },82 "gpt-image-2": {83 "provider": "openai",84 "kind": "image",85 "tests": []86 },87 "gpt-image-2.5-flare": {88 "provider": "openai",89 "kind": "image",90 "tests": []91 },92 "gpt-image-2.5-sunburst": {93 "provider": "openai",94 "kind": "image",95 "tests": []96 },97 "gpt-live-1": {98 "provider": "openai",99 "kind": "realtime",100 "tests": []101 },102 "gpt-live-transcribe": {103 "provider": "openai",104 "kind": "realtime",105 "tests": []106 },107 "gpt-realtime": {108 "provider": "openai",109 "kind": "realtime",110 "tests": []111 },112 "gpt-realtime-1.5": {113 "provider": "openai",114 "kind": "realtime",115 "tests": []116 },117 "gpt-realtime-2": {118 "provider": "openai",119 "kind": "realtime",120 "tests": []121 },122 "gpt-realtime-2.1": {123 "provider": "openai",124 "kind": "realtime",125 "tests": []126 },127 "gpt-realtime-2.1-mini": {128 "provider": "openai",129 "kind": "realtime",130 "tests": []131 },132 "gpt-realtime-mini": {133 "provider": "openai",134 "kind": "realtime",135 "tests": []136 },137 "gpt-realtime-translate": {138 "provider": "openai",139 "kind": "realtime",140 "tests": []141 },142 "gpt-realtime-whisper": {143 "provider": "openai",144 "kind": "realtime",145 "tests": []146 },147 "gpt-transcribe": {148 "provider": "openai",149 "kind": "stt",150 "tests": []151 },152 "o3-deep-research": {153 "provider": "openai",154 "kind": "deep-research",155 "tests": []156 },157 "o4-mini-deep-research": {158 "provider": "openai",159 "kind": "deep-research",160 "tests": []161 },162 "omni-moderation-latest": {163 "provider": "openai",164 "kind": "moderation",165 "tests": []166 },167 "sora-2": {168 "provider": "openai",169 "kind": "video",170 "tests": []171 },172 "sora-2-pro": {173 "provider": "openai",174 "kind": "video",175 "tests": []176 },177 "text-embedding-3-large": {178 "provider": "openai",179 "kind": "embedding",180 "tests": []181 },182 "text-embedding-3-small": {183 "provider": "openai",184 "kind": "embedding",185 "tests": []186 },187 "text-embedding-ada-002": {188 "provider": "openai",189 "kind": "embedding",190 "tests": []191 },192 "tts-1": {193 "provider": "openai",194 "kind": "tts",195 "tests": []196 },197 "tts-1-1106": {198 "provider": "openai",199 "kind": "tts",200 "tests": []201 },202 "tts-1-hd": {203 "provider": "openai",204 "kind": "tts",205 "tests": []206 },207 "tts-1-hd-1106": {208 "provider": "openai",209 "kind": "tts",210 "tests": []211 },212 "whisper-1": {213 "provider": "openai",214 "kind": "stt",215 "tests": []216 },217 "chat-latest": {218 "provider": "openai",219 "kind": "chat",220 "tests": [221 {222 "test": "basic",223 "ok": true,224 "latencyMs": 1279,225 "usage": {226 "input": 19,227 "output": 6,228 "cachedInput": 0,229 "reasoning": 0230 },231 "api": "responses",232 "raw": {233 "status": "completed",234 "incomplete": null235 },236 "sample": "Ottawa",237 "costUsd": 0.000275238 },239 {240 "test": "reasoning",241 "ok": true,242 "latencyMs": 1086,243 "usage": {244 "input": 42,245 "output": 39,246 "cachedInput": 0,247 "reasoning": 0248 },249 "api": "responses",250 "raw": {251 "status": "completed",252 "incomplete": null253 },254 "sample": "Alice has 2 sisters, so there are 3 girls in the family, including Alice. Each of her brothers has all 3 girls as sisters.\n\nANSWER: 3",255 "costUsd": 0.00138256 },257 {258 "test": "math",259 "ok": true,260 "latencyMs": 1215,261 "usage": {262 "input": 50,263 "output": 11,264 "cachedInput": 0,265 "reasoning": 0266 },267 "api": "responses",268 "raw": {269 "status": "completed",270 "incomplete": null271 },272 "sample": "a=403;b=2500",273 "costUsd": 0.00058274 },275 {276 "test": "code",277 "ok": true,278 "latencyMs": 1387,279 "usage": {280 "input": 49,281 "output": 97,282 "cachedInput": 0,283 "reasoning": 0284 },285 "api": "responses",286 "raw": {287 "status": "completed",288 "incomplete": null289 },290 "sample": "```js\nfunction isPrime(n) {\n if (n <= 1 || !Number.isInteger(n)) return false;\n if (n === 2) return true;\n if (n % 2 === 0) return false;\n\n for (let i = 3; i <= Math.sqrt(n); i += 2) {\n if (n % i === 0) return false;\n }\n\n return true;\n}\n```",291 "costUsd": 0.003155292 },293 {294 "test": "french",295 "ok": true,296 "latencyMs": 1340,297 "usage": {298 "input": 31,299 "output": 77,300 "cachedInput": 0,301 "reasoning": 0302 },303 "api": "responses",304 "raw": {305 "status": "completed",306 "incomplete": null307 },308 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d’un bien en additionnant la valeur du terrain et le coût de reconstruction ou de remplacement du bâtiment, après déduction de sa dépréciation. Elle est principalement utilisée pour évaluer des biens récents, spécialisés ou po",309 "costUsd": 0.002465310 },311 {312 "test": "english",313 "ok": true,314 "latencyMs": 1209,315 "usage": {316 "input": 20,317 "output": 57,318 "cachedInput": 0,319 "reasoning": 0320 },321 "api": "responses",322 "raw": {323 "status": "completed",324 "incomplete": null325 },326 "sample": "A hedonic pricing model is an economic method used to estimate the value of a product or asset by breaking it down into its individual characteristics or features. It is commonly used in real estate to determine how factors such as location, size, and amenities influence property prices.",327 "costUsd": 0.00181328 },329 {330 "test": "json",331 "ok": true,332 "latencyMs": 1039,333 "usage": {334 "input": 52,335 "output": 27,336 "cachedInput": 0,337 "reasoning": 0338 },339 "api": "responses",340 "raw": {341 "status": "completed",342 "incomplete": null343 },344 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",345 "costUsd": 0.00107346 },347 {348 "test": "tools",349 "ok": true,350 "latencyMs": 1079,351 "usage": {352 "input": 69,353 "output": 22,354 "cachedInput": 0,355 "reasoning": 0356 },357 "api": "responses",358 "raw": {359 "status": "completed",360 "incomplete": null361 },362 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",363 "costUsd": 0.001005364 },365 {366 "test": "vision",367 "ok": true,368 "latencyMs": 973,369 "usage": {370 "input": 194,371 "output": 12,372 "cachedInput": 0,373 "reasoning": 0374 },375 "api": "responses",376 "raw": {377 "status": "completed",378 "incomplete": null379 },380 "sample": "circles=3; other=square",381 "costUsd": 0.00133382 },383 {384 "test": "longctx",385 "ok": true,386 "latencyMs": 995,387 "usage": {388 "input": 9134,389 "output": 9,390 "cachedInput": 0,391 "reasoning": 0392 },393 "api": "responses",394 "raw": {395 "status": "completed",396 "incomplete": null397 },398 "sample": "7431-ALPHA",399 "costUsd": 0.04594400 },401 {402 "test": "streaming",403 "ok": true,404 "latencyMs": 980,405 "ttftMs": 547,406 "usage": {407 "input": 15,408 "output": 23,409 "cachedInput": 0,410 "reasoning": 0411 },412 "api": "responses",413 "sample": "Snow blankets Québec, \nSt. Lawrence sleeps beneath ice, \nWarm lights bloom at dusk.",414 "costUsd": 0.000765415 }416 ],417 "api": "responses"418 },419 "gpt-3.5-turbo": {420 "provider": "openai",421 "kind": "chat",422 "tests": [423 {424 "test": "basic",425 "ok": false,426 "latencyMs": 288,427 "error": "404 404 The model `gpt-3.5-turbo` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"428 }429 ]430 },431 "gpt-4": {432 "provider": "openai",433 "kind": "chat",434 "tests": [435 {436 "test": "basic",437 "ok": false,438 "latencyMs": 277,439 "error": "404 404 The model `gpt-4` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"440 }441 ]442 },443 "gpt-4-turbo": {444 "provider": "openai",445 "kind": "chat",446 "tests": [447 {448 "test": "basic",449 "ok": false,450 "latencyMs": 312,451 "error": "404 404 The model `gpt-4-turbo` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"452 }453 ]454 },455 "gpt-4.1": {456 "provider": "openai",457 "kind": "chat",458 "tests": [459 {460 "test": "basic",461 "ok": true,462 "latencyMs": 608,463 "usage": {464 "input": 20,465 "output": 3,466 "cachedInput": 0,467 "reasoning": 0468 },469 "api": "responses",470 "raw": {471 "status": "completed",472 "incomplete": null473 },474 "sample": "Ottawa",475 "costUsd": 0.000064476 },477 {478 "test": "reasoning",479 "ok": true,480 "latencyMs": 983,481 "usage": {482 "input": 43,483 "output": 104,484 "cachedInput": 0,485 "reasoning": 0486 },487 "api": "responses",488 "raw": {489 "status": "completed",490 "incomplete": null491 },492 "sample": "Let's break down the information:\n\n- Alice has **3 brothers** and **2 sisters**.\n- Alice herself is a girl, so she is also a sister.\n\nSo, in total, there are:\n- **Sisters:** Alice + 2 sisters = **3 sisters**\n- **Brothers:** 3 brothers\n\nThe question: \"How many sisters does each of Alice's brothers ha",493 "costUsd": 0.000918494 },495 {496 "test": "math",497 "ok": false,498 "latencyMs": 546,499 "usage": {500 "input": 51,501 "output": 8,502 "cachedInput": 0,503 "reasoning": 0504 },505 "api": "responses",506 "raw": {507 "status": "completed",508 "incomplete": null509 },510 "sample": "a=411;b=2500",511 "costUsd": 0.000166512 },513 {514 "test": "code",515 "ok": true,516 "latencyMs": 886,517 "usage": {518 "input": 50,519 "output": 88,520 "cachedInput": 0,521 "reasoning": 0522 },523 "api": "responses",524 "raw": {525 "status": "completed",526 "incomplete": null527 },528 "sample": "```js\nfunction isPrime(n) {\n if (n <= 1) return false;\n if (n === 2) return true;\n if (n % 2 === 0) return false;\n for (let i = 3; i <= Math.sqrt(n); i += 2) {\n if (n % i === 0) return false;\n }\n return true;\n}\n```",529 "costUsd": 0.000804530 },531 {532 "test": "french",533 "ok": true,534 "latencyMs": 1039,535 "usage": {536 "input": 32,537 "output": 70,538 "cachedInput": 0,539 "reasoning": 0540 },541 "api": "responses",542 "raw": {543 "status": "completed",544 "incomplete": null545 },546 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en additionnant la valeur du terrain à celle du coût de reconstruction ou de remplacement du bâtiment, après déduction de la dépréciation. Elle est particulièrement utilisée pour les biens atypiques ou lorsque le mar",547 "costUsd": 0.000624548 },549 {550 "test": "english",551 "ok": true,552 "latencyMs": 655,553 "usage": {554 "input": 21,555 "output": 54,556 "cachedInput": 0,557 "reasoning": 0558 },559 "api": "responses",560 "raw": {561 "status": "completed",562 "incomplete": null563 },564 "sample": "A hedonic pricing model is an economic valuation method that estimates the market value of a good by analyzing how its individual characteristics or features influence its price. It is commonly used in real estate, where factors such as location, size, and amenities help determine property prices.",565 "costUsd": 0.000474566 },567 {568 "test": "json",569 "ok": true,570 "latencyMs": 704,571 "usage": {572 "input": 53,573 "output": 37,574 "cachedInput": 0,575 "reasoning": 0576 },577 "api": "responses",578 "raw": {579 "status": "completed",580 "incomplete": null581 },582 "sample": "{\n \"city\": \"Gatineau\",\n \"country\": \"Canada\",\n \"province\": \"Quebec\",\n \"population_estimate\": 297000\n}",583 "costUsd": 0.000402584 },585 {586 "test": "tools",587 "ok": true,588 "latencyMs": 487,589 "usage": {590 "input": 64,591 "output": 19,592 "cachedInput": 0,593 "reasoning": 0594 },595 "api": "responses",596 "raw": {597 "status": "completed",598 "incomplete": null599 },600 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",601 "costUsd": 0.00028602 },603 {604 "test": "vision",605 "ok": true,606 "latencyMs": 598,607 "usage": {608 "input": 123,609 "output": 9,610 "cachedInput": 0,611 "reasoning": 0612 },613 "api": "responses",614 "raw": {615 "status": "completed",616 "incomplete": null617 },618 "sample": "circles=3; other=square",619 "costUsd": 0.000318620 },621 {622 "test": "longctx",623 "ok": true,624 "latencyMs": 1203,625 "usage": {626 "input": 9135,627 "output": 6,628 "cachedInput": 0,629 "reasoning": 0630 },631 "api": "responses",632 "raw": {633 "status": "completed",634 "incomplete": null635 },636 "sample": "7431-ALPHA",637 "costUsd": 0.018318638 },639 {640 "test": "streaming",641 "ok": true,642 "latencyMs": 736,643 "ttftMs": 413,644 "usage": {645 "input": 16,646 "output": 21,647 "cachedInput": 0,648 "reasoning": 0649 },650 "api": "responses",651 "sample": "Snow cloaks the old town, \nRiver breathes icy silence— \nStars shimmer on ice.",652 "costUsd": 0.0002653 }654 ],655 "api": "responses"656 },657 "gpt-4.1-mini": {658 "provider": "openai",659 "kind": "chat",660 "tests": [661 {662 "test": "basic",663 "ok": true,664 "latencyMs": 604,665 "usage": {666 "input": 20,667 "output": 3,668 "cachedInput": 0,669 "reasoning": 0670 },671 "api": "responses",672 "raw": {673 "status": "completed",674 "incomplete": null675 },676 "sample": "Ottawa",677 "costUsd": 0.000012800000000000001678 },679 {680 "test": "reasoning",681 "ok": true,682 "latencyMs": 1021,683 "usage": {684 "input": 43,685 "output": 88,686 "cachedInput": 0,687 "reasoning": 0688 },689 "api": "responses",690 "raw": {691 "status": "completed",692 "incomplete": null693 },694 "sample": "Alice has 2 sisters in total, including herself. Since Alice's brothers are her siblings, they share the same sisters. The brothers see Alice and the 2 sisters as their sisters.\n\nAlice is not a sister to herself, so from the brothers' perspective, the sisters are Alice and the 2 other sisters, makin",695 "costUsd": 0.000158696 },697 {698 "test": "math",699 "ok": false,700 "latencyMs": 582,701 "usage": {702 "input": 51,703 "output": 8,704 "cachedInput": 0,705 "reasoning": 0706 },707 "api": "responses",708 "raw": {709 "status": "completed",710 "incomplete": null711 },712 "sample": "a=417;b=2500",713 "costUsd": 0.0000332714 },715 {716 "test": "code",717 "ok": true,718 "latencyMs": 1086,719 "usage": {720 "input": 50,721 "output": 108,722 "cachedInput": 0,723 "reasoning": 0724 },725 "api": "responses",726 "raw": {727 "status": "completed",728 "incomplete": null729 },730 "sample": "```js\nfunction isPrime(n) {\n if (n <= 1) return false;\n if (n <= 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return true;\n}\n```",731 "costUsd": 0.00019280000000000002732 },733 {734 "test": "french",735 "ok": true,736 "latencyMs": 897,737 "usage": {738 "input": 32,739 "output": 69,740 "cachedInput": 0,741 "reasoning": 0742 },743 "api": "responses",744 "raw": {745 "status": "completed",746 "incomplete": null747 },748 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en calculant la somme du coût de reconstruction à neuf, diminuée de la dépréciation, ajoutée à la valeur du terrain. Elle est particulièrement utilisée pour les biens ne générant pas de revenus ou sans comparaison di",749 "costUsd": 0.0001232750 },751 {752 "test": "english",753 "ok": true,754 "latencyMs": 1009,755 "usage": {756 "input": 21,757 "output": 57,758 "cachedInput": 0,759 "reasoning": 0760 },761 "api": "responses",762 "raw": {763 "status": "completed",764 "incomplete": null765 },766 "sample": "A hedonic pricing model is an economic method used to estimate the value of a good or property by breaking down its price into the individual contributions of its characteristics or features. It is commonly applied in real estate to assess how factors like location, size, and amenities affect proper",767 "costUsd": 0.00009960000000000001768 },769 {770 "test": "json",771 "ok": true,772 "latencyMs": 705,773 "usage": {774 "input": 53,775 "output": 37,776 "cachedInput": 0,777 "reasoning": 0778 },779 "api": "responses",780 "raw": {781 "status": "completed",782 "incomplete": null783 },784 "sample": "{\n \"city\": \"Gatineau\",\n \"country\": \"Canada\",\n \"province\": \"Quebec\",\n \"population_estimate\": 291041\n}",785 "costUsd": 0.0000804786 },787 {788 "test": "tools",789 "ok": true,790 "latencyMs": 834,791 "usage": {792 "input": 64,793 "output": 19,794 "cachedInput": 0,795 "reasoning": 0796 },797 "api": "responses",798 "raw": {799 "status": "completed",800 "incomplete": null801 },802 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",803 "costUsd": 0.000056804 },805 {806 "test": "vision",807 "ok": true,808 "latencyMs": 596,809 "usage": {810 "input": 250,811 "output": 9,812 "cachedInput": 0,813 "reasoning": 0814 },815 "api": "responses",816 "raw": {817 "status": "completed",818 "incomplete": null819 },820 "sample": "circles=3; other=square",821 "costUsd": 0.0001144822 },823 {824 "test": "longctx",825 "ok": true,826 "latencyMs": 919,827 "usage": {828 "input": 9135,829 "output": 6,830 "cachedInput": 0,831 "reasoning": 0832 },833 "api": "responses",834 "raw": {835 "status": "completed",836 "incomplete": null837 },838 "sample": "7431-ALPHA",839 "costUsd": 0.0036636840 },841 {842 "test": "streaming",843 "ok": true,844 "latencyMs": 552,845 "ttftMs": 343,846 "usage": {847 "input": 16,848 "output": 23,849 "cachedInput": 0,850 "reasoning": 0851 },852 "api": "responses",853 "sample": "Snow drapes quiet streets, \nMaple scent in frosty air, \nLights dance on still lakes.",854 "costUsd": 0.0000432855 }856 ],857 "api": "responses"858 },859 "gpt-4.1-nano": {860 "provider": "openai",861 "kind": "chat",862 "tests": [863 {864 "test": "basic",865 "ok": false,866 "latencyMs": 424,867 "error": "404 404 The model `gpt-4.1-nano` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"868 }869 ]870 },871 "gpt-4o": {872 "provider": "openai",873 "kind": "chat",874 "tests": [875 {876 "test": "basic",877 "ok": true,878 "latencyMs": 704,879 "usage": {880 "input": 20,881 "output": 3,882 "cachedInput": 0,883 "reasoning": 0884 },885 "api": "responses",886 "raw": {887 "status": "completed",888 "incomplete": null889 },890 "sample": "Ottawa",891 "costUsd": 0.00008892 },893 {894 "test": "reasoning",895 "ok": false,896 "latencyMs": 1507,897 "usage": {898 "input": 43,899 "output": 32,900 "cachedInput": 0,901 "reasoning": 0902 },903 "api": "responses",904 "raw": {905 "status": "completed",906 "incomplete": null907 },908 "sample": "Alice has 2 sisters (including herself). Each of Alice's brothers has the same sisters: Alice and her 2 sisters. \n\nANSWER: 2",909 "costUsd": 0.0004275910 },911 {912 "test": "math",913 "ok": false,914 "latencyMs": 644,915 "usage": {916 "input": 51,917 "output": 8,918 "cachedInput": 0,919 "reasoning": 0920 },921 "api": "responses",922 "raw": {923 "status": "completed",924 "incomplete": null925 },926 "sample": "a=427;b=2500",927 "costUsd": 0.0002075928 },929 {930 "test": "code",931 "ok": true,932 "latencyMs": 1111,933 "usage": {934 "input": 50,935 "output": 108,936 "cachedInput": 0,937 "reasoning": 0938 },939 "api": "responses",940 "raw": {941 "status": "completed",942 "incomplete": null943 },944 "sample": "```js\nfunction isPrime(n) {\n if (n <= 1) return false;\n if (n <= 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return true;\n}\n```",945 "costUsd": 0.001205946 },947 {948 "test": "french",949 "ok": true,950 "latencyMs": 1049,951 "usage": {952 "input": 32,953 "output": 75,954 "cachedInput": 0,955 "reasoning": 0956 },957 "api": "responses",958 "raw": {959 "status": "completed",960 "incomplete": null961 },962 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en calculant le coût de remplacement ou de reproduction de la construction, moins la dépréciation accumulée, puis en ajoutant la valeur du terrain. Cette approche est souvent utilisée lorsqu'il n'existe pas de compar",963 "costUsd": 0.00083964 },965 {966 "test": "english",967 "ok": true,968 "latencyMs": 5002,969 "usage": {970 "input": 21,971 "output": 56,972 "cachedInput": 0,973 "reasoning": 0974 },975 "api": "responses",976 "raw": {977 "status": "completed",978 "incomplete": null979 },980 "sample": "A hedonic pricing model estimates the value of a good or service by breaking it down into its component attributes and analyzing how each characteristic contributes to the overall price. This approach is often used in real estate to assess how features like location, size, and amenities impact prope",981 "costUsd": 0.0006125982 },983 {984 "test": "json",985 "ok": true,986 "latencyMs": 882,987 "usage": {988 "input": 53,989 "output": 37,990 "cachedInput": 0,991 "reasoning": 0992 },993 "api": "responses",994 "raw": {995 "status": "completed",996 "incomplete": null997 },998 "sample": "{\n \"city\": \"Gatineau\",\n \"country\": \"Canada\",\n \"province\": \"Quebec\",\n \"population_estimate\": 290000\n}",999 "costUsd": 0.00050251000 },1001 {1002 "test": "tools",1003 "ok": true,1004 "latencyMs": 851,1005 "usage": {1006 "input": 64,1007 "output": 19,1008 "cachedInput": 0,1009 "reasoning": 01010 },1011 "api": "responses",1012 "raw": {1013 "status": "completed",1014 "incomplete": null1015 },1016 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",1017 "costUsd": 0.000351018 },1019 {1020 "test": "vision",1021 "ok": true,1022 "latencyMs": 853,1023 "usage": {1024 "input": 123,1025 "output": 9,1026 "cachedInput": 0,1027 "reasoning": 01028 },1029 "api": "responses",1030 "raw": {1031 "status": "completed",1032 "incomplete": null1033 },1034 "sample": "circles=3; other=square",1035 "costUsd": 0.00039751036 },1037 {1038 "test": "longctx",1039 "ok": true,1040 "latencyMs": 1088,1041 "usage": {1042 "input": 9135,1043 "output": 6,1044 "cachedInput": 0,1045 "reasoning": 01046 },1047 "api": "responses",1048 "raw": {1049 "status": "completed",1050 "incomplete": null1051 },1052 "sample": "7431-ALPHA",1053 "costUsd": 0.02289751054 },1055 {1056 "test": "streaming",1057 "ok": true,1058 "latencyMs": 887,1059 "ttftMs": 457,1060 "usage": {1061 "input": 16,1062 "output": 22,1063 "cachedInput": 0,1064 "reasoning": 01065 },1066 "api": "responses",1067 "sample": "Silent snow descends, \nFrosty breath whispers of peace— \nQuébec dreams in white.",1068 "costUsd": 0.000261069 }1070 ],1071 "api": "responses"1072 },1073 "gpt-4o-mini": {1074 "provider": "openai",1075 "kind": "chat",1076 "tests": [1077 {1078 "test": "basic",1079 "ok": true,1080 "latencyMs": 510,1081 "usage": {1082 "input": 20,1083 "output": 4,1084 "cachedInput": 0,1085 "reasoning": 01086 },1087 "api": "responses",1088 "raw": {1089 "status": "completed",1090 "incomplete": null1091 },1092 "sample": "Ottawa.",1093 "costUsd": 0.00000541094 },1095 {1096 "test": "reasoning",1097 "ok": false,1098 "latencyMs": 1209,1099 "usage": {1100 "input": 43,1101 "output": 61,1102 "cachedInput": 0,1103 "reasoning": 01104 },1105 "api": "responses",1106 "raw": {1107 "status": "completed",1108 "incomplete": null1109 },1110 "sample": "Alice has 2 sisters and 3 brothers. Each of Alice's brothers will have the same siblings as Alice, meaning they will also have the 2 sisters that Alice has. Therefore, each of Alice's brothers has 2 sisters. \n\nSo, the answer is: \n\nANSWER: 2",1111 "costUsd": 0.000043051112 },1113 {1114 "test": "math",1115 "ok": false,1116 "latencyMs": 589,1117 "usage": {1118 "input": 51,1119 "output": 9,1120 "cachedInput": 0,1121 "reasoning": 01122 },1123 "api": "responses",1124 "raw": {1125 "status": "completed",1126 "incomplete": null1127 },1128 "sample": "a=397; b=2500",1129 "costUsd": 0.0000130499999999999991130 },1131 {1132 "test": "code",1133 "ok": true,1134 "latencyMs": 893,1135 "usage": {1136 "input": 50,1137 "output": 60,1138 "cachedInput": 0,1139 "reasoning": 01140 },1141 "api": "responses",1142 "raw": {1143 "status": "completed",1144 "incomplete": null1145 },1146 "sample": "```js\nfunction isPrime(n) {\n if (n <= 1) return false;\n for (let i = 2; i <= Math.sqrt(n); i++) {\n if (n % i === 0) return false;\n }\n return true;\n}\n```",1147 "costUsd": 0.00004351148 },1149 {1150 "test": "french",1151 "ok": true,1152 "latencyMs": 997,1153 "usage": {1154 "input": 32,1155 "output": 68,1156 "cachedInput": 0,1157 "reasoning": 01158 },1159 "api": "responses",1160 "raw": {1161 "status": "completed",1162 "incomplete": null1163 },1164 "sample": "La méthode du coût en évaluation immobilière consiste à déterminer la valeur d'un bien en additionnant le coût des matériaux et de la main-d'œuvre nécessaires à sa construction, en tenant compte de l'amortissement. Cette approche est souvent utilisée pour les propriétés nouvelles ou spécialisées, où",1165 "costUsd": 0.00004561166 },1167 {1168 "test": "english",1169 "ok": true,1170 "latencyMs": 1106,1171 "usage": {1172 "input": 21,1173 "output": 61,1174 "cachedInput": 0,1175 "reasoning": 01176 },1177 "api": "responses",1178 "raw": {1179 "status": "completed",1180 "incomplete": null1181 },1182 "sample": "A hedonic pricing model is an analytical approach used to estimate the value of a good or service by breaking it down into its individual characteristics and attributes. This model often applies to real estate and consumer products, where prices reflect the perceived utility derived from various fea",1183 "costUsd": 0.000039751184 },1185 {1186 "test": "json",1187 "ok": true,1188 "latencyMs": 876,1189 "usage": {1190 "input": 53,1191 "output": 37,1192 "cachedInput": 0,1193 "reasoning": 01194 },1195 "api": "responses",1196 "raw": {1197 "status": "completed",1198 "incomplete": null1199 },1200 "sample": "{\n \"city\": \"Gatineau\",\n \"country\": \"Canada\",\n \"province\": \"Quebec\",\n \"population_estimate\": 300000\n}",1201 "costUsd": 0.0000301499999999999981202 },1203 {1204 "test": "tools",1205 "ok": true,1206 "latencyMs": 634,1207 "usage": {1208 "input": 64,1209 "output": 19,1210 "cachedInput": 0,1211 "reasoning": 01212 },1213 "api": "responses",1214 "raw": {1215 "status": "completed",1216 "incomplete": null1217 },1218 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",1219 "costUsd": 0.0000211220 },1221 {1222 "test": "vision",1223 "ok": true,1224 "latencyMs": 718,1225 "usage": {1226 "input": 2871,1227 "output": 9,1228 "cachedInput": 0,1229 "reasoning": 01230 },1231 "api": "responses",1232 "raw": {1233 "status": "completed",1234 "incomplete": null1235 },1236 "sample": "circles=3; other=square",1237 "costUsd": 0.000436049999999999951238 },1239 {1240 "test": "longctx",1241 "ok": true,1242 "latencyMs": 750,1243 "usage": {1244 "input": 9135,1245 "output": 6,1246 "cachedInput": 0,1247 "reasoning": 01248 },1249 "api": "responses",1250 "raw": {1251 "status": "completed",1252 "incomplete": null1253 },1254 "sample": "7431-ALPHA",1255 "costUsd": 0.00137384999999999981256 },1257 {1258 "test": "streaming",1259 "ok": true,1260 "latencyMs": 684,1261 "ttftMs": 482,1262 "usage": {1263 "input": 16,1264 "output": 21,1265 "cachedInput": 0,1266 "reasoning": 01267 },1268 "api": "responses",1269 "sample": "Snow whispers softly, \nPines cloaked in crystal beauty— \nQuebec's cold embrace.",1270 "costUsd": 0.0000151271 }1272 ],1273 "api": "responses"1274 },1275 "gpt-4o-mini-search-preview": {1276 "provider": "openai",1277 "kind": "search",1278 "tests": [1279 {1280 "test": "basic",1281 "ok": false,1282 "latencyMs": 277,1283 "error": "404 404 The model `gpt-4o-mini-search-preview` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"1284 }1285 ]1286 },1287 "gpt-4o-search-preview": {1288 "provider": "openai",1289 "kind": "search",1290 "tests": [1291 {1292 "test": "basic",1293 "ok": false,1294 "latencyMs": 275,1295 "error": "404 404 The model `gpt-4o-search-preview` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"1296 }1297 ]1298 },1299 "gpt-5": {1300 "provider": "openai",1301 "kind": "chat",1302 "tests": [1303 {1304 "test": "reasoning",1305 "ok": true,1306 "latencyMs": 5702,1307 "usage": {1308 "input": 42,1309 "output": 399,1310 "cachedInput": 0,1311 "reasoning": 3201312 },1313 "api": "responses",1314 "raw": {1315 "status": "completed",1316 "incomplete": null1317 },1318 "sample": "Alice is one of the girls. If she has 2 sisters, then there are 3 girls total (Alice + 2). Each brother’s sisters are all three girls. \nANSWER: 3",1319 "costUsd": 0.00404251320 },1321 {1322 "test": "math",1323 "ok": true,1324 "latencyMs": 2439,1325 "usage": {1326 "input": 50,1327 "output": 230,1328 "cachedInput": 0,1329 "reasoning": 1921330 },1331 "api": "responses",1332 "raw": {1333 "status": "completed",1334 "incomplete": null1335 },1336 "sample": "a=403;b=2500",1337 "costUsd": 0.00236251338 },1339 {1340 "test": "code",1341 "ok": true,1342 "latencyMs": 7366,1343 "usage": {1344 "input": 49,1345 "output": 639,1346 "cachedInput": 0,1347 "reasoning": 5121348 },1349 "api": "responses",1350 "raw": {1351 "status": "completed",1352 "incomplete": null1353 },1354 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n < 2) return false;\n if (n === 2 || n === 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return true;\n}\n```",1355 "costUsd": 0.006451251356 },1357 {1358 "test": "french",1359 "ok": true,1360 "latencyMs": 2603,1361 "usage": {1362 "input": 31,1363 "output": 312,1364 "cachedInput": 0,1365 "reasoning": 1921366 },1367 "api": "responses",1368 "raw": {1369 "status": "completed",1370 "incomplete": null1371 },1372 "sample": "La méthode du coût évalue un bien en additionnant la valeur du terrain au coût de remplacement ou de reproduction de l’immeuble, puis en retranchant la dépréciation (physique, fonctionnelle et externe). Elle est surtout utilisée pour les biens spéciaux ou neufs et lorsque les données de marché sont ",1373 "costUsd": 0.003158751374 },1375 {1376 "test": "english",1377 "ok": true,1378 "latencyMs": 4497,1379 "usage": {1380 "input": 20,1381 "output": 311,1382 "cachedInput": 0,1383 "reasoning": 1921384 },1385 "api": "responses",1386 "raw": {1387 "status": "completed",1388 "incomplete": null1389 },1390 "sample": "A hedonic pricing model decomposes an observed market price into the implicit values of its characteristics, typically using regression analysis to estimate each attribute's marginal contribution. It is widely used in housing and other differentiated-good markets to infer willingness to pay for feat",1391 "costUsd": 0.0031351392 },1393 {1394 "test": "json",1395 "ok": true,1396 "latencyMs": 3814,1397 "usage": {1398 "input": 52,1399 "output": 484,1400 "cachedInput": 0,1401 "reasoning": 4481402 },1403 "api": "responses",1404 "raw": {1405 "status": "completed",1406 "incomplete": null1407 },1408 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",1409 "costUsd": 0.0049051410 },1411 {1412 "test": "tools",1413 "ok": true,1414 "latencyMs": 1448,1415 "usage": {1416 "input": 69,1417 "output": 152,1418 "cachedInput": 0,1419 "reasoning": 1281420 },1421 "api": "responses",1422 "raw": {1423 "status": "completed",1424 "incomplete": null1425 },1426 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",1427 "costUsd": 0.001606251428 },1429 {1430 "test": "vision",1431 "ok": true,1432 "latencyMs": 1798,1433 "usage": {1434 "input": 107,1435 "output": 196,1436 "cachedInput": 0,1437 "reasoning": 1281438 },1439 "api": "responses",1440 "raw": {1441 "status": "completed",1442 "incomplete": null1443 },1444 "sample": "circles=3; other=square",1445 "costUsd": 0.002093751446 },1447 {1448 "test": "longctx",1449 "ok": true,1450 "latencyMs": 3685,1451 "usage": {1452 "input": 9134,1453 "output": 323,1454 "cachedInput": 0,1455 "reasoning": 2561456 },1457 "api": "responses",1458 "raw": {1459 "status": "completed",1460 "incomplete": null1461 },1462 "sample": "7431-ALPHA",1463 "costUsd": 0.01464751464 },1465 {1466 "test": "basic",1467 "ok": true,1468 "latencyMs": 1813,1469 "usage": {1470 "input": 19,1471 "output": 101,1472 "cachedInput": 0,1473 "reasoning": 641474 },1475 "api": "responses",1476 "raw": {1477 "status": "completed",1478 "incomplete": null1479 },1480 "sample": "Ottawa",1481 "costUsd": 0.001033751482 },1483 {1484 "test": "streaming",1485 "ok": true,1486 "latencyMs": 8238,1487 "ttftMs": 7922,1488 "usage": {1489 "input": 15,1490 "output": 950,1491 "cachedInput": 0,1492 "reasoning": 8961493 },1494 "api": "responses",1495 "sample": "Saint Lawrence locked fast\nice moans under old bridges—\nBonhomme’s red sash glows",1496 "costUsd": 0.009518751497 }1498 ],1499 "api": "responses"1500 },1501 "gpt-5-chat-latest": {1502 "provider": "openai",1503 "kind": "chat",1504 "tests": [1505 {1506 "test": "basic",1507 "ok": false,1508 "latencyMs": 279,1509 "error": "404 404 The model `gpt-5-chat-latest` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"1510 }1511 ]1512 },1513 "gpt-5-codex": {1514 "provider": "openai",1515 "kind": "chat",1516 "tests": [1517 {1518 "test": "basic",1519 "ok": false,1520 "latencyMs": 287,1521 "error": "404 404 The model `gpt-5-codex` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"1522 }1523 ]1524 },1525 "gpt-5-mini": {1526 "provider": "openai",1527 "kind": "chat",1528 "tests": [1529 {1530 "test": "reasoning",1531 "ok": true,1532 "latencyMs": 3711,1533 "usage": {1534 "input": 42,1535 "output": 327,1536 "cachedInput": 0,1537 "reasoning": 2561538 },1539 "api": "responses",1540 "raw": {1541 "status": "completed",1542 "incomplete": null1543 },1544 "sample": "Alice is one of the sisters, and she has 2 other sisters, so there are 3 sisters in total. Each of her brothers has those 3 sisters. ANSWER: 3",1545 "costUsd": 0.00066451546 },1547 {1548 "test": "math",1549 "ok": true,1550 "latencyMs": 3159,1551 "usage": {1552 "input": 50,1553 "output": 298,1554 "cachedInput": 0,1555 "reasoning": 2561556 },1557 "api": "responses",1558 "raw": {1559 "status": "completed",1560 "incomplete": null1561 },1562 "sample": "a=403;b=2500",1563 "costUsd": 0.00060851564 },1565 {1566 "test": "code",1567 "ok": true,1568 "latencyMs": 4263,1569 "usage": {1570 "input": 49,1571 "output": 455,1572 "cachedInput": 0,1573 "reasoning": 3201574 },1575 "api": "responses",1576 "raw": {1577 "status": "completed",1578 "incomplete": null1579 },1580 "sample": "```js\nfunction isPrime(n) {\n if (typeof n !== 'number' || !Number.isInteger(n) || n < 2) return false;\n if (n === 2 || n === 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return ",1581 "costUsd": 0.000922251582 },1583 {1584 "test": "french",1585 "ok": true,1586 "latencyMs": 3761,1587 "usage": {1588 "input": 31,1589 "output": 337,1590 "cachedInput": 0,1591 "reasoning": 1921592 },1593 "api": "responses",1594 "raw": {1595 "status": "completed",1596 "incomplete": null1597 },1598 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en additionnant la valeur du terrain et le coût de reproduction ou de remplacement des constructions, puis en déduisant l'obsolescence et la dépréciation accumulées. Elle est particulièrement utile pour les immeubles",1599 "costUsd": 0.000681751600 },1601 {1602 "test": "english",1603 "ok": true,1604 "latencyMs": 2936,1605 "usage": {1606 "input": 20,1607 "output": 305,1608 "cachedInput": 0,1609 "reasoning": 1921610 },1611 "api": "responses",1612 "raw": {1613 "status": "completed",1614 "incomplete": null1615 },1616 "sample": "A hedonic pricing model decomposes the price of a differentiated good into the implicit prices of its measurable attributes by regressing observed transaction prices on those characteristics. It is commonly used in housing, labor, and product markets to estimate how much each feature (e.g., square f",1617 "costUsd": 0.0006151618 },1619 {1620 "test": "json",1621 "ok": true,1622 "latencyMs": 4528,1623 "usage": {1624 "input": 52,1625 "output": 418,1626 "cachedInput": 0,1627 "reasoning": 3841628 },1629 "api": "responses",1630 "raw": {1631 "status": "completed",1632 "incomplete": null1633 },1634 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",1635 "costUsd": 0.0008491636 },1637 {1638 "test": "tools",1639 "ok": true,1640 "latencyMs": 1728,1641 "usage": {1642 "input": 69,1643 "output": 97,1644 "cachedInput": 0,1645 "reasoning": 641646 },1647 "api": "responses",1648 "raw": {1649 "status": "completed",1650 "incomplete": null1651 },1652 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",1653 "costUsd": 0.000211251654 },1655 {1656 "test": "vision",1657 "ok": true,1658 "latencyMs": 1422,1659 "usage": {1660 "input": 194,1661 "output": 96,1662 "cachedInput": 0,1663 "reasoning": 641664 },1665 "api": "responses",1666 "raw": {1667 "status": "completed",1668 "incomplete": null1669 },1670 "sample": "circles=3; other=square",1671 "costUsd": 0.00024051672 },1673 {1674 "test": "longctx",1675 "ok": true,1676 "latencyMs": 2108,1677 "usage": {1678 "input": 9134,1679 "output": 186,1680 "cachedInput": 0,1681 "reasoning": 1281682 },1683 "api": "responses",1684 "raw": {1685 "status": "completed",1686 "incomplete": null1687 },1688 "sample": "7431-ALPHA",1689 "costUsd": 0.00265551690 },1691 {1692 "test": "basic",1693 "ok": true,1694 "latencyMs": 1572,1695 "usage": {1696 "input": 19,1697 "output": 65,1698 "cachedInput": 0,1699 "reasoning": 01700 },1701 "api": "responses",1702 "raw": {1703 "status": "completed",1704 "incomplete": null1705 },1706 "sample": "Ottawa",1707 "costUsd": 0.000134751708 },1709 {1710 "test": "streaming",1711 "ok": true,1712 "latencyMs": 10441,1713 "ttftMs": 10075,1714 "usage": {1715 "input": 15,1716 "output": 856,1717 "cachedInput": 0,1718 "reasoning": 8321719 },1720 "api": "responses",1721 "sample": "Gray roofs under snow\nStreet lamps halo the soft falling\nMaple smoke in air",1722 "costUsd": 0.001715751723 }1724 ],1725 "api": "responses"1726 },1727 "gpt-5-nano": {1728 "provider": "openai",1729 "kind": "chat",1730 "tests": [1731 {1732 "test": "reasoning",1733 "ok": true,1734 "latencyMs": 4900,1735 "usage": {1736 "input": 42,1737 "output": 1051,1738 "cachedInput": 0,1739 "reasoning": 9601740 },1741 "api": "responses",1742 "raw": {1743 "status": "completed",1744 "incomplete": null1745 },1746 "sample": "There are 2 other sisters besides Alice, and Alice is also his sister, so each brother has 3 sisters in total.\n\nANSWER: 3",1747 "costUsd": 0.00042250000000000011748 },1749 {1750 "test": "math",1751 "ok": true,1752 "latencyMs": 1239,1753 "usage": {1754 "input": 50,1755 "output": 235,1756 "cachedInput": 0,1757 "reasoning": 1921758 },1759 "api": "responses",1760 "raw": {1761 "status": "completed",1762 "incomplete": null1763 },1764 "sample": "a=403;b=2500",1765 "costUsd": 0.00009651766 },1767 {1768 "test": "code",1769 "ok": true,1770 "latencyMs": 4674,1771 "usage": {1772 "input": 49,1773 "output": 1037,1774 "cachedInput": 0,1775 "reasoning": 8321776 },1777 "api": "responses",1778 "raw": {1779 "status": "completed",1780 "incomplete": null1781 },1782 "sample": "```js\nfunction isPrime(n) {\n if (typeof n !== 'number' || !Number.isFinite(n)) return false;\n if (n !== Math.floor(n)) return false;\n if (n < 2) return false;\n if (n === 2 || n === 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n let i = 5;\n while (i * i <= n) {\n if (n % i =",1783 "costUsd": 0.000417251784 },1785 {1786 "test": "french",1787 "ok": true,1788 "latencyMs": 3034,1789 "usage": {1790 "input": 31,1791 "output": 853,1792 "cachedInput": 0,1793 "reasoning": 7681794 },1795 "api": "responses",1796 "raw": {1797 "status": "completed",1798 "incomplete": null1799 },1800 "sample": "La méthode du coût en évaluation immobilière estime la valeur d'un bien à partir du coût de reconstruction ou de remplacement du bâtiment, estimé à jour. On y ajoute la valeur du terrain et on retranche les dépréciations (physiques, fonctionnelles et économiques) pour obtenir la valeur finale.",1801 "costUsd": 0.000342750000000000061802 },1803 {1804 "test": "english",1805 "ok": true,1806 "latencyMs": 2844,1807 "usage": {1808 "input": 20,1809 "output": 592,1810 "cachedInput": 0,1811 "reasoning": 5121812 },1813 "api": "responses",1814 "raw": {1815 "status": "completed",1816 "incomplete": null1817 },1818 "sample": "A hedonic pricing model estimates the value of a good or service by decomposing its price into the value of its observable attributes or characteristics. It uses regression analysis to relate price to these attributes, yielding implicit prices for each characteristic and enabling analysis of how fea",1819 "costUsd": 0.00023781820 },1821 {1822 "test": "json",1823 "ok": true,1824 "latencyMs": 6989,1825 "usage": {1826 "input": 52,1827 "output": 1433,1828 "cachedInput": 0,1829 "reasoning": 13441830 },1831 "api": "responses",1832 "raw": {1833 "status": "completed",1834 "incomplete": null1835 },1836 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":330000}",1837 "costUsd": 0.00057580000000000011838 },1839 {1840 "test": "tools",1841 "ok": true,1842 "latencyMs": 1511,1843 "usage": {1844 "input": 69,1845 "output": 180,1846 "cachedInput": 0,1847 "reasoning": 1281848 },1849 "api": "responses",1850 "raw": {1851 "status": "completed",1852 "incomplete": null1853 },1854 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",1855 "costUsd": 0.000075451856 },1857 {1858 "test": "vision",1859 "ok": true,1860 "latencyMs": 1219,1861 "usage": {1862 "input": 232,1863 "output": 187,1864 "cachedInput": 0,1865 "reasoning": 1281866 },1867 "api": "responses",1868 "raw": {1869 "status": "completed",1870 "incomplete": null1871 },1872 "sample": "circles=3; other=square",1873 "costUsd": 0.00008641874 },1875 {1876 "test": "longctx",1877 "ok": true,1878 "latencyMs": 1924,1879 "usage": {1880 "input": 9134,1881 "output": 273,1882 "cachedInput": 0,1883 "reasoning": 2561884 },1885 "api": "responses",1886 "raw": {1887 "status": "completed",1888 "incomplete": null1889 },1890 "sample": "7431-ALPHA",1891 "costUsd": 0.00056591892 },1893 {1894 "test": "basic",1895 "ok": true,1896 "latencyMs": 1779,1897 "usage": {1898 "input": 19,1899 "output": 119,1900 "cachedInput": 0,1901 "reasoning": 641902 },1903 "api": "responses",1904 "raw": {1905 "status": "completed",1906 "incomplete": null1907 },1908 "sample": "Ottawa",1909 "costUsd": 0.000048551910 },1911 {1912 "test": "streaming",1913 "ok": true,1914 "latencyMs": 10863,1915 "ttftMs": 10356,1916 "usage": {1917 "input": 15,1918 "output": 2091,1919 "cachedInput": 0,1920 "reasoning": 20481921 },1922 "api": "responses",1923 "sample": "Snow sighs on the pines\nThe old town wears a white shawl\nIce threads the river",1924 "costUsd": 0.000837151925 }1926 ],1927 "api": "responses"1928 },1929 "gpt-5-pro": {1930 "provider": "openai",1931 "kind": "chat-pro",1932 "tests": [1933 {1934 "test": "basic",1935 "ok": true,1936 "latencyMs": 13713,1937 "usage": {1938 "input": 19,1939 "output": 72,1940 "cachedInput": 0,1941 "reasoning": 641942 },1943 "api": "responses",1944 "raw": {1945 "status": "completed",1946 "incomplete": null1947 },1948 "sample": "Ottawa",1949 "costUsd": 0.0089251950 },1951 {1952 "test": "reasoning",1953 "ok": true,1954 "latencyMs": 49039,1955 "usage": {1956 "input": 42,1957 "output": 813,1958 "cachedInput": 0,1959 "reasoning": 7681960 },1961 "api": "responses",1962 "raw": {1963 "status": "completed",1964 "incomplete": null1965 },1966 "sample": "Alice has 2 sisters, and Alice herself is also a sister to her brothers. So each brother has 3 sisters. ANSWER: 3",1967 "costUsd": 0.098191968 },1969 {1970 "test": "json",1971 "ok": true,1972 "latencyMs": 47132,1973 "usage": {1974 "input": 52,1975 "output": 976,1976 "cachedInput": 0,1977 "reasoning": 8961978 },1979 "api": "responses",1980 "raw": {1981 "status": "completed",1982 "incomplete": null1983 },1984 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",1985 "costUsd": 0.11791986 },1987 {1988 "test": "tools",1989 "ok": true,1990 "latencyMs": 15890,1991 "usage": {1992 "input": 69,1993 "output": 231,1994 "cachedInput": 0,1995 "reasoning": 1921996 },1997 "api": "responses",1998 "raw": {1999 "status": "completed",2000 "incomplete": null2001 },2002 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",2003 "costUsd": 0.0287552004 }2005 ],2006 "api": "responses"2007 },2008 "gpt-5-search-api": {2009 "provider": "openai",2010 "kind": "search",2011 "tests": [2012 {2013 "test": "basic",2014 "ok": true,2015 "latencyMs": 1262,2016 "usage": {2017 "input": 1576,2018 "output": 6,2019 "cachedInput": 0,2020 "reasoning": 02021 },2022 "api": "chat",2023 "raw": {2024 "finish": "stop"2025 },2026 "sample": "Ottawa",2027 "costUsd": 0.002032028 },2029 {2030 "test": "reasoning",2031 "ok": true,2032 "latencyMs": 1707,2033 "usage": {2034 "input": 1599,2035 "output": 133,2036 "cachedInput": 0,2037 "reasoning": 02038 },2039 "api": "chat",2040 "raw": {2041 "finish": "stop"2042 },2043 "sample": "Each of Alice’s brothers has 3 sisters. Because Alice has 2 sisters in addition to herself, each brother’s sisters are: Alice + those 2 sisters = 3. \nANSWER: 3",2044 "costUsd": 0.003328752045 },2046 {2047 "test": "math",2048 "ok": false,2049 "latencyMs": 1218,2050 "usage": {2051 "input": 1607,2052 "output": 11,2053 "cachedInput": 0,2054 "reasoning": 02055 },2056 "api": "chat",2057 "raw": {2058 "finish": "stop"2059 },2060 "sample": "a=411;b=2500",2061 "costUsd": 0.002118752062 },2063 {2064 "test": "code",2065 "ok": true,2066 "latencyMs": 2032,2067 "usage": {2068 "input": 1606,2069 "output": 159,2070 "cachedInput": 0,2071 "reasoning": 02072 },2073 "api": "chat",2074 "raw": {2075 "finish": "stop"2076 },2077 "sample": "```js\nfunction isPrime(n) {\n if (n <= 1 || n !== Math.floor(n)) return false;\n if (n === 2 || n === 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n const limit = Math.floor(Math.sqrt(n));\n for (let i = 5; i <= limit; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return fal",2078 "costUsd": 0.00359752079 },2080 {2081 "test": "french",2082 "ok": true,2083 "latencyMs": 5782,2084 "usage": {2085 "input": 16623,2086 "output": 171,2087 "cachedInput": 0,2088 "reasoning": 02089 },2090 "api": "chat",2091 "raw": {2092 "finish": "stop"2093 },2094 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d’un bien en additionnant la valeur du terrain (si vacant) et le coût à neuf de construction d’un bâtiment équivalent, puis en soustrayant la dépréciation due à l’usure, à l’obsolescence ou à des facteurs externes ([pmea.ca](h",2095 "costUsd": 0.022488752096 },2097 {2098 "test": "english",2099 "ok": true,2100 "latencyMs": 1621,2101 "usage": {2102 "input": 1577,2103 "output": 93,2104 "cachedInput": 0,2105 "reasoning": 02106 },2107 "api": "chat",2108 "raw": {2109 "finish": "stop"2110 },2111 "sample": "A hedonic pricing model is an economic method used to estimate the value of a good or service by breaking it down into its component attributes and determining how much each contributes to overall price. It relies on observed market data to statistically isolate how individual characteristics (such ",2112 "costUsd": 0.002901252113 },2114 {2115 "test": "json",2116 "ok": true,2117 "latencyMs": 4414,2118 "usage": {2119 "input": 16436,2120 "output": 27,2121 "cachedInput": 1408,2122 "reasoning": 02123 },2124 "api": "chat",2125 "raw": {2126 "finish": "stop"2127 },2128 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":305437}",2129 "costUsd": 0.0192312130 },2131 {2132 "test": "tools",2133 "ok": false,2134 "latencyMs": 103,2135 "error": "404 404 tools is not supported in this model. For a list of supported models, refer to https://platform.openai.com/docs/guides/function-calling#models-supporting-function-calling."2136 },2137 {2138 "test": "vision",2139 "ok": false,2140 "latencyMs": 1643,2141 "usage": {2142 "input": 1673,2143 "output": 12,2144 "cachedInput": 0,2145 "reasoning": 02146 },2147 "api": "chat",2148 "raw": {2149 "finish": "stop"2150 },2151 "sample": "circles=3; other=triangle",2152 "costUsd": 0.002211252153 },2154 {2155 "test": "longctx",2156 "ok": true,2157 "latencyMs": 1313,2158 "usage": {2159 "input": 10691,2160 "output": 9,2161 "cachedInput": 1408,2162 "reasoning": 02163 },2164 "api": "chat",2165 "raw": {2166 "finish": "stop"2167 },2168 "sample": "7431-ALPHA",2169 "costUsd": 0.011869752170 },2171 {2172 "test": "streaming",2173 "ok": true,2174 "latencyMs": 1582,2175 "ttftMs": 1073,2176 "usage": {2177 "input": 1572,2178 "output": 24,2179 "cachedInput": 11522180 },2181 "api": "chat",2182 "sample": "Québec snowfall drifts— \nRiver whispers under ice, \nMaple’s scarlet glow.",2183 "costUsd": 0.0009092184 }2185 ],2186 "api": "chat"2187 },2188 "gpt-5.1": {2189 "provider": "openai",2190 "kind": "chat",2191 "tests": [2192 {2193 "test": "basic",2194 "ok": true,2195 "latencyMs": 808,2196 "usage": {2197 "input": 19,2198 "output": 12,2199 "cachedInput": 0,2200 "reasoning": 02201 },2202 "api": "responses",2203 "raw": {2204 "status": "completed",2205 "incomplete": null2206 },2207 "sample": "Ottawa",2208 "costUsd": 0.000143752209 },2210 {2211 "test": "reasoning",2212 "ok": true,2213 "latencyMs": 1431,2214 "usage": {2215 "input": 42,2216 "output": 71,2217 "cachedInput": 0,2218 "reasoning": 02219 },2220 "api": "responses",2221 "raw": {2222 "status": "completed",2223 "incomplete": null2224 },2225 "sample": "Alice has 3 brothers and 2 sisters.\n\nTotal girls in the family: Alice + 2 sisters = 3 sisters.\n\nEach brother’s sisters are all the girls in the family (since each brother is not counted as his own sister). So each brother has 3 sisters.\n\nANSWER: 3",2226 "costUsd": 0.00076252227 },2228 {2229 "test": "math",2230 "ok": false,2231 "latencyMs": 743,2232 "usage": {2233 "input": 50,2234 "output": 17,2235 "cachedInput": 0,2236 "reasoning": 02237 },2238 "api": "responses",2239 "raw": {2240 "status": "completed",2241 "incomplete": null2242 },2243 "sample": "a=419;b=2500",2244 "costUsd": 0.00023252245 },2246 {2247 "test": "code",2248 "ok": true,2249 "latencyMs": 1084,2250 "usage": {2251 "input": 49,2252 "output": 117,2253 "cachedInput": 0,2254 "reasoning": 02255 },2256 "api": "responses",2257 "raw": {2258 "status": "completed",2259 "incomplete": null2260 },2261 "sample": "```js\nfunction isPrime(n) {\n if (n <= 1) return false;\n if (n <= 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return true;\n}\n```",2262 "costUsd": 0.001231252263 },2264 {2265 "test": "french",2266 "ok": true,2267 "latencyMs": 1419,2268 "usage": {2269 "input": 31,2270 "output": 85,2271 "cachedInput": 0,2272 "reasoning": 02273 },2274 "api": "responses",2275 "raw": {2276 "status": "completed",2277 "incomplete": null2278 },2279 "sample": "La méthode du coût, en évaluation immobilière, consiste à estimer la valeur d’un bien en additionnant la valeur du terrain à celle des constructions, calculée sur la base du coût de remplacement ou de reproduction. On déduit ensuite la dépréciation (vieillissement, obsolescence, vétusté) pour obteni",2280 "costUsd": 0.000888752281 },2282 {2283 "test": "english",2284 "ok": true,2285 "latencyMs": 1116,2286 "usage": {2287 "input": 20,2288 "output": 77,2289 "cachedInput": 0,2290 "reasoning": 02291 },2292 "api": "responses",2293 "raw": {2294 "status": "completed",2295 "incomplete": null2296 },2297 "sample": "A hedonic pricing model is an econometric method that estimates the value of a good or service by decomposing its price into the implicit prices of its attributes or characteristics. It is commonly used to infer the value of non-market attributes—such as environmental quality or neighborhood safety—",2298 "costUsd": 0.0007952299 },2300 {2301 "test": "json",2302 "ok": true,2303 "latencyMs": 815,2304 "usage": {2305 "input": 52,2306 "output": 33,2307 "cachedInput": 0,2308 "reasoning": 02309 },2310 "api": "responses",2311 "raw": {2312 "status": "completed",2313 "incomplete": null2314 },2315 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":300000}",2316 "costUsd": 0.0003952317 },2318 {2319 "test": "tools",2320 "ok": true,2321 "latencyMs": 738,2322 "usage": {2323 "input": 69,2324 "output": 28,2325 "cachedInput": 0,2326 "reasoning": 02327 },2328 "api": "responses",2329 "raw": {2330 "status": "completed",2331 "incomplete": null2332 },2333 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",2334 "costUsd": 0.000366252335 },2336 {2337 "test": "vision",2338 "ok": true,2339 "latencyMs": 871,2340 "usage": {2341 "input": 107,2342 "output": 18,2343 "cachedInput": 0,2344 "reasoning": 02345 },2346 "api": "responses",2347 "raw": {2348 "status": "completed",2349 "incomplete": null2350 },2351 "sample": "circles=3; other=square",2352 "costUsd": 0.000313752353 },2354 {2355 "test": "longctx",2356 "ok": true,2357 "latencyMs": 812,2358 "usage": {2359 "input": 9134,2360 "output": 15,2361 "cachedInput": 0,2362 "reasoning": 02363 },2364 "api": "responses",2365 "raw": {2366 "status": "completed",2367 "incomplete": null2368 },2369 "sample": "7431-ALPHA",2370 "costUsd": 0.01156752371 },2372 {2373 "test": "streaming",2374 "ok": true,2375 "latencyMs": 1035,2376 "ttftMs": 610,2377 "usage": {2378 "input": 15,2379 "output": 34,2380 "cachedInput": 0,2381 "reasoning": 02382 },2383 "api": "responses",2384 "sample": "Snowstorm-veiled steeples, \nMaple smoke threads frozen streets— \nNorth wind hymns in French.",2385 "costUsd": 0.000358752386 }2387 ],2388 "api": "responses"2389 },2390 "gpt-5.1-chat-latest": {2391 "provider": "openai",2392 "kind": "chat",2393 "tests": [2394 {2395 "test": "basic",2396 "ok": false,2397 "latencyMs": 275,2398 "error": "404 404 The model `gpt-5.1-chat-latest` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"2399 }2400 ]2401 },2402 "gpt-5.1-codex": {2403 "provider": "openai",2404 "kind": "chat",2405 "tests": [2406 {2407 "test": "basic",2408 "ok": false,2409 "latencyMs": 232,2410 "error": "404 404 The model `gpt-5.1-codex` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"2411 }2412 ]2413 },2414 "gpt-5.1-codex-max": {2415 "provider": "openai",2416 "kind": "chat",2417 "tests": [2418 {2419 "test": "basic",2420 "ok": false,2421 "latencyMs": 257,2422 "error": "404 404 The model `gpt-5.1-codex-max` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"2423 }2424 ]2425 },2426 "gpt-5.1-codex-mini": {2427 "provider": "openai",2428 "kind": "chat",2429 "tests": [2430 {2431 "test": "basic",2432 "ok": false,2433 "latencyMs": 327,2434 "error": "404 404 The model `gpt-5.1-codex-mini` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"2435 }2436 ]2437 },2438 "gpt-5.2": {2439 "provider": "openai",2440 "kind": "chat",2441 "tests": [2442 {2443 "test": "basic",2444 "ok": true,2445 "latencyMs": 701,2446 "usage": {2447 "input": 19,2448 "output": 6,2449 "cachedInput": 0,2450 "reasoning": 02451 },2452 "api": "responses",2453 "raw": {2454 "status": "completed",2455 "incomplete": null2456 },2457 "sample": "Ottawa",2458 "costUsd": 0.000117252459 },2460 {2461 "test": "reasoning",2462 "ok": false,2463 "latencyMs": 1496,2464 "usage": {2465 "input": 42,2466 "output": 47,2467 "cachedInput": 0,2468 "reasoning": 02469 },2470 "api": "responses",2471 "raw": {2472 "status": "completed",2473 "incomplete": null2474 },2475 "sample": "Alice is one of the sisters. The family has 2 sisters total (Alice plus one more) and 3 brothers. Each brother has the same set of sisters: the 2 sisters.\n\nANSWER: 2",2476 "costUsd": 0.00073152477 },2478 {2479 "test": "math",2480 "ok": true,2481 "latencyMs": 2918,2482 "usage": {2483 "input": 50,2484 "output": 11,2485 "cachedInput": 0,2486 "reasoning": 02487 },2488 "api": "responses",2489 "raw": {2490 "status": "completed",2491 "incomplete": null2492 },2493 "sample": "a=403;b=2500",2494 "costUsd": 0.00024152495 },2496 {2497 "test": "code",2498 "ok": true,2499 "latencyMs": 1766,2500 "usage": {2501 "input": 49,2502 "output": 122,2503 "cachedInput": 0,2504 "reasoning": 02505 },2506 "api": "responses",2507 "raw": {2508 "status": "completed",2509 "incomplete": null2510 },2511 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n <= 1) return false;\n if (n === 2 || n === 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return true;\n}\n```",2512 "costUsd": 0.001793752513 },2514 {2515 "test": "french",2516 "ok": true,2517 "latencyMs": 1864,2518 "usage": {2519 "input": 31,2520 "output": 89,2521 "cachedInput": 0,2522 "reasoning": 02523 },2524 "api": "responses",2525 "raw": {2526 "status": "completed",2527 "incomplete": null2528 },2529 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d’un bien en additionnant la valeur du terrain et le coût de reconstruction à neuf des améliorations (bâtiments), puis en retranchant la dépréciation (usure, obsolescence, etc.). Elle est surtout utilisée pour des biens récent",2530 "costUsd": 0.001300252531 },2532 {2533 "test": "english",2534 "ok": true,2535 "latencyMs": 1506,2536 "usage": {2537 "input": 20,2538 "output": 55,2539 "cachedInput": 0,2540 "reasoning": 02541 },2542 "api": "responses",2543 "raw": {2544 "status": "completed",2545 "incomplete": null2546 },2547 "sample": "A hedonic pricing model estimates the implicit value of an item’s individual characteristics by relating observed market prices to measurable attributes. It is commonly used for heterogeneous goods like housing to infer how features (e.g., size, location, amenities) contribute to price.",2548 "costUsd": 0.0008052549 },2550 {2551 "test": "json",2552 "ok": true,2553 "latencyMs": 936,2554 "usage": {2555 "input": 52,2556 "output": 27,2557 "cachedInput": 0,2558 "reasoning": 02559 },2560 "api": "responses",2561 "raw": {2562 "status": "completed",2563 "incomplete": null2564 },2565 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",2566 "costUsd": 0.0004692567 },2568 {2569 "test": "tools",2570 "ok": true,2571 "latencyMs": 928,2572 "usage": {2573 "input": 69,2574 "output": 22,2575 "cachedInput": 0,2576 "reasoning": 02577 },2578 "api": "responses",2579 "raw": {2580 "status": "completed",2581 "incomplete": null2582 },2583 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",2584 "costUsd": 0.000428752585 },2586 {2587 "test": "vision",2588 "ok": true,2589 "latencyMs": 721,2590 "usage": {2591 "input": 194,2592 "output": 12,2593 "cachedInput": 0,2594 "reasoning": 02595 },2596 "api": "responses",2597 "raw": {2598 "status": "completed",2599 "incomplete": null2600 },2601 "sample": "circles=3; other=square",2602 "costUsd": 0.00050752603 },2604 {2605 "test": "longctx",2606 "ok": false,2607 "latencyMs": 959,2608 "usage": {2609 "input": 9134,2610 "output": 26,2611 "cachedInput": 0,2612 "reasoning": 02613 },2614 "api": "responses",2615 "raw": {2616 "status": "completed",2617 "incomplete": null2618 },2619 "sample": "Je ne peux pas divulguer un code secret extrait d’une note interne marquée « Ne pas divulguer ».",2620 "costUsd": 0.01634852621 },2622 {2623 "test": "streaming",2624 "ok": true,2625 "latencyMs": 1220,2626 "ttftMs": 529,2627 "usage": {2628 "input": 15,2629 "output": 26,2630 "cachedInput": 0,2631 "reasoning": 02632 },2633 "api": "responses",2634 "sample": "St. Lawrence locked in ice, \nMaple smoke warms dusk-blue streets— \nSnow hushes Québec.",2635 "costUsd": 0.000390252636 }2637 ],2638 "api": "responses"2639 },2640 "gpt-5.2-chat-latest": {2641 "provider": "openai",2642 "kind": "chat",2643 "tests": [2644 {2645 "test": "basic",2646 "ok": false,2647 "latencyMs": 243,2648 "error": "404 404 The model `gpt-5.2-chat-latest` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"2649 }2650 ]2651 },2652 "gpt-5.2-codex": {2653 "provider": "openai",2654 "kind": "chat",2655 "tests": [2656 {2657 "test": "basic",2658 "ok": false,2659 "latencyMs": 246,2660 "error": "404 404 The model `gpt-5.2-codex` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"2661 }2662 ]2663 },2664 "gpt-5.2-pro": {2665 "provider": "openai",2666 "kind": "chat-pro",2667 "tests": [2668 {2669 "test": "basic",2670 "ok": true,2671 "latencyMs": 4095,2672 "usage": {2673 "input": 19,2674 "output": 6,2675 "cachedInput": 0,2676 "reasoning": 02677 },2678 "api": "responses",2679 "raw": {2680 "status": "completed",2681 "incomplete": null2682 },2683 "sample": "Ottawa",2684 "costUsd": 0.0014072685 },2686 {2687 "test": "reasoning",2688 "ok": true,2689 "latencyMs": 6396,2690 "usage": {2691 "input": 42,2692 "output": 91,2693 "cachedInput": 0,2694 "reasoning": 362695 },2696 "api": "responses",2697 "raw": {2698 "status": "completed",2699 "incomplete": null2700 },2701 "sample": "Alice is one of the sisters. Since she has 2 sisters, that means there are **3 sisters total** in the family (Alice + 2 others). Each of Alice’s brothers therefore has **3 sisters**.\n\nANSWER: 3",2702 "costUsd": 0.016172703 },2704 {2705 "test": "json",2706 "ok": true,2707 "latencyMs": 6297,2708 "usage": {2709 "input": 52,2710 "output": 93,2711 "cachedInput": 0,2712 "reasoning": 642713 },2714 "api": "responses",2715 "raw": {2716 "status": "completed",2717 "incomplete": null2718 },2719 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",2720 "costUsd": 0.0167162721 },2722 {2723 "test": "tools",2724 "ok": true,2725 "latencyMs": 9220,2726 "usage": {2727 "input": 69,2728 "output": 22,2729 "cachedInput": 0,2730 "reasoning": 02731 },2732 "api": "responses",2733 "raw": {2734 "status": "completed",2735 "incomplete": null2736 },2737 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",2738 "costUsd": 0.0051452739 }2740 ],2741 "api": "responses"2742 },2743 "gpt-5.3-chat-latest": {2744 "provider": "openai",2745 "kind": "chat",2746 "tests": [2747 {2748 "test": "basic",2749 "ok": false,2750 "latencyMs": 320,2751 "error": "404 404 The model `gpt-5.3-chat-latest` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"2752 }2753 ]2754 },2755 "gpt-5.3-codex": {2756 "provider": "openai",2757 "kind": "chat",2758 "tests": [2759 {2760 "test": "basic",2761 "ok": true,2762 "latencyMs": 656,2763 "usage": {2764 "input": 19,2765 "output": 6,2766 "cachedInput": 0,2767 "reasoning": 02768 },2769 "api": "responses",2770 "raw": {2771 "status": "completed",2772 "incomplete": null2773 },2774 "sample": "Ottawa",2775 "costUsd": 0.000117252776 },2777 {2778 "test": "reasoning",2779 "ok": false,2780 "latencyMs": 1209,2781 "usage": {2782 "input": 42,2783 "output": 46,2784 "cachedInput": 0,2785 "reasoning": 02786 },2787 "api": "responses",2788 "raw": {2789 "status": "completed",2790 "incomplete": null2791 },2792 "sample": "Alice has 2 sisters total, and Alice is one of them.\n\nSo each of Alice’s brothers has the same sisters: **Alice + Alice’s other sister = 2 sisters**.\n\nANSWER: 2",2793 "costUsd": 0.00071752794 },2795 {2796 "test": "math",2797 "ok": true,2798 "latencyMs": 828,2799 "usage": {2800 "input": 50,2801 "output": 11,2802 "cachedInput": 0,2803 "reasoning": 02804 },2805 "api": "responses",2806 "raw": {2807 "status": "completed",2808 "incomplete": null2809 },2810 "sample": "a=403;b=2500",2811 "costUsd": 0.00024152812 },2813 {2814 "test": "code",2815 "ok": true,2816 "latencyMs": 1511,2817 "usage": {2818 "input": 49,2819 "output": 117,2820 "cachedInput": 0,2821 "reasoning": 02822 },2823 "api": "responses",2824 "raw": {2825 "status": "completed",2826 "incomplete": null2827 },2828 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n <= 1) return false;\n if (n <= 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n\n return true;\n}\n```",2829 "costUsd": 0.001723752830 },2831 {2832 "test": "french",2833 "ok": true,2834 "latencyMs": 1635,2835 "usage": {2836 "input": 31,2837 "output": 87,2838 "cachedInput": 0,2839 "reasoning": 02840 },2841 "api": "responses",2842 "raw": {2843 "status": "completed",2844 "incomplete": null2845 },2846 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d’un bien en additionnant la valeur du terrain et le coût de reconstruction à neuf du bâtiment, puis en retranchant les dépréciations (usure, obsolescence, etc.). Elle est surtout utilisée pour les biens récents, spécialisés o",2847 "costUsd": 0.001272252848 },2849 {2850 "test": "english",2851 "ok": true,2852 "latencyMs": 1682,2853 "usage": {2854 "input": 20,2855 "output": 64,2856 "cachedInput": 0,2857 "reasoning": 02858 },2859 "api": "responses",2860 "raw": {2861 "status": "completed",2862 "incomplete": null2863 },2864 "sample": "A hedonic pricing model is a statistical method that estimates how different characteristics of a good—such as size, location, or quality—contribute to its overall price. It is commonly used in markets like housing to isolate the implicit value of each attribute by comparing prices across similar it",2865 "costUsd": 0.0009312866 },2867 {2868 "test": "json",2869 "ok": true,2870 "latencyMs": 992,2871 "usage": {2872 "input": 52,2873 "output": 27,2874 "cachedInput": 0,2875 "reasoning": 02876 },2877 "api": "responses",2878 "raw": {2879 "status": "completed",2880 "incomplete": null2881 },2882 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",2883 "costUsd": 0.0004692884 },2885 {2886 "test": "tools",2887 "ok": true,2888 "latencyMs": 791,2889 "usage": {2890 "input": 69,2891 "output": 22,2892 "cachedInput": 0,2893 "reasoning": 02894 },2895 "api": "responses",2896 "raw": {2897 "status": "completed",2898 "incomplete": null2899 },2900 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",2901 "costUsd": 0.000428752902 },2903 {2904 "test": "vision",2905 "ok": true,2906 "latencyMs": 804,2907 "usage": {2908 "input": 194,2909 "output": 12,2910 "cachedInput": 0,2911 "reasoning": 02912 },2913 "api": "responses",2914 "raw": {2915 "status": "completed",2916 "incomplete": null2917 },2918 "sample": "circles=3; other=square",2919 "costUsd": 0.00050752920 },2921 {2922 "test": "longctx",2923 "ok": true,2924 "latencyMs": 779,2925 "usage": {2926 "input": 9134,2927 "output": 9,2928 "cachedInput": 0,2929 "reasoning": 02930 },2931 "api": "responses",2932 "raw": {2933 "status": "completed",2934 "incomplete": null2935 },2936 "sample": "7431-ALPHA",2937 "costUsd": 0.01611052938 },2939 {2940 "test": "streaming",2941 "ok": true,2942 "latencyMs": 1049,2943 "ttftMs": 545,2944 "usage": {2945 "input": 15,2946 "output": 27,2947 "cachedInput": 0,2948 "reasoning": 02949 },2950 "api": "responses",2951 "sample": "Snow hushes old pines, \nSt. Lawrence breathes silver mist— \nQuebec dreams in white.",2952 "costUsd": 0.000404252953 }2954 ],2955 "api": "responses"2956 },2957 "gpt-5.4": {2958 "provider": "openai",2959 "kind": "chat",2960 "tests": [2961 {2962 "test": "basic",2963 "ok": true,2964 "latencyMs": 940,2965 "usage": {2966 "input": 19,2967 "output": 6,2968 "cachedInput": 0,2969 "reasoning": 02970 },2971 "api": "responses",2972 "raw": {2973 "status": "completed",2974 "incomplete": null2975 },2976 "sample": "Ottawa",2977 "costUsd": 0.00013752978 },2979 {2980 "test": "reasoning",2981 "ok": false,2982 "latencyMs": 1559,2983 "usage": {2984 "input": 42,2985 "output": 38,2986 "cachedInput": 0,2987 "reasoning": 02988 },2989 "api": "responses",2990 "raw": {2991 "status": "completed",2992 "incomplete": null2993 },2994 "sample": "Alice is one of the sisters.\n\nSo the family has:\n- 3 brothers\n- 2 sisters total\n\nEach brother has 2 sisters.\n\nANSWER: 2",2995 "costUsd": 0.0006752996 },2997 {2998 "test": "math",2999 "ok": true,3000 "latencyMs": 840,3001 "usage": {3002 "input": 50,3003 "output": 11,3004 "cachedInput": 0,3005 "reasoning": 03006 },3007 "api": "responses",3008 "raw": {3009 "status": "completed",3010 "incomplete": null3011 },3012 "sample": "a=403;b=2500",3013 "costUsd": 0.000293014 },3015 {3016 "test": "code",3017 "ok": true,3018 "latencyMs": 1466,3019 "usage": {3020 "input": 49,3021 "output": 105,3022 "cachedInput": 0,3023 "reasoning": 03024 },3025 "api": "responses",3026 "raw": {3027 "status": "completed",3028 "incomplete": null3029 },3030 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n <= 1) return false;\n if (n === 2) return true;\n if (n % 2 === 0) return false;\n\n const limit = Math.floor(Math.sqrt(n));\n for (let i = 3; i <= limit; i += 2) {\n if (n % i === 0) return false;\n }\n\n return true;\n}\n```",3031 "costUsd": 0.00169753032 },3033 {3034 "test": "french",3035 "ok": true,3036 "latencyMs": 2605,3037 "usage": {3038 "input": 31,3039 "output": 102,3040 "cachedInput": 0,3041 "reasoning": 03042 },3043 "api": "responses",3044 "raw": {3045 "status": "completed",3046 "incomplete": null3047 },3048 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d’un bien en additionnant la valeur du terrain et le coût de reconstruction à neuf du bâtiment, puis en retranchant la dépréciation due à l’usure, à l’obsolescence ou à des facteurs externes. Elle est surtout utilisée pour les",3049 "costUsd": 0.00160753050 },3051 {3052 "test": "english",3053 "ok": true,3054 "latencyMs": 1766,3055 "usage": {3056 "input": 20,3057 "output": 63,3058 "cachedInput": 0,3059 "reasoning": 03060 },3061 "api": "responses",3062 "raw": {3063 "status": "completed",3064 "incomplete": null3065 },3066 "sample": "A hedonic pricing model estimates how the price of a good or asset is determined by the value of its individual characteristics, such as a house’s size, location, or number of bedrooms. It is commonly used to isolate the implicit price of each attribute by statistically relating observed prices to o",3067 "costUsd": 0.0009953068 },3069 {3070 "test": "json",3071 "ok": true,3072 "latencyMs": 1020,3073 "usage": {3074 "input": 52,3075 "output": 27,3076 "cachedInput": 0,3077 "reasoning": 03078 },3079 "api": "responses",3080 "raw": {3081 "status": "completed",3082 "incomplete": null3083 },3084 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",3085 "costUsd": 0.0005353086 },3087 {3088 "test": "tools",3089 "ok": true,3090 "latencyMs": 1196,3091 "usage": {3092 "input": 69,3093 "output": 22,3094 "cachedInput": 0,3095 "reasoning": 03096 },3097 "api": "responses",3098 "raw": {3099 "status": "completed",3100 "incomplete": null3101 },3102 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",3103 "costUsd": 0.00050253104 },3105 {3106 "test": "vision",3107 "ok": true,3108 "latencyMs": 772,3109 "usage": {3110 "input": 194,3111 "output": 12,3112 "cachedInput": 0,3113 "reasoning": 03114 },3115 "api": "responses",3116 "raw": {3117 "status": "completed",3118 "incomplete": null3119 },3120 "sample": "circles=3; other=square",3121 "costUsd": 0.0006653122 },3123 {3124 "test": "longctx",3125 "ok": true,3126 "latencyMs": 1010,3127 "usage": {3128 "input": 9134,3129 "output": 9,3130 "cachedInput": 0,3131 "reasoning": 03132 },3133 "api": "responses",3134 "raw": {3135 "status": "completed",3136 "incomplete": null3137 },3138 "sample": "7431-ALPHA",3139 "costUsd": 0.022973140 },3141 {3142 "test": "streaming",3143 "ok": true,3144 "latencyMs": 1486,3145 "ttftMs": 651,3146 "usage": {3147 "input": 15,3148 "output": 24,3149 "cachedInput": 0,3150 "reasoning": 03151 },3152 "api": "responses",3153 "sample": "Snow hushes old stones \nSt. Lawrence breathes silver and cold \nPines cradle moonlight",3154 "costUsd": 0.00039753155 }3156 ],3157 "api": "responses"3158 },3159 "gpt-5.4-mini": {3160 "provider": "openai",3161 "kind": "chat",3162 "tests": [3163 {3164 "test": "basic",3165 "ok": true,3166 "latencyMs": 645,3167 "usage": {3168 "input": 19,3169 "output": 6,3170 "cachedInput": 0,3171 "reasoning": 03172 },3173 "api": "responses",3174 "raw": {3175 "status": "completed",3176 "incomplete": null3177 },3178 "sample": "Ottawa",3179 "costUsd": 0.000041253180 },3181 {3182 "test": "reasoning",3183 "ok": true,3184 "latencyMs": 764,3185 "usage": {3186 "input": 42,3187 "output": 42,3188 "cachedInput": 0,3189 "reasoning": 03190 },3191 "api": "responses",3192 "raw": {3193 "status": "completed",3194 "incomplete": null3195 },3196 "sample": "Alice has 2 sisters, and all of Alice’s brothers share the same sisters. That means each brother has 3 sisters total: Alice plus her 2 sisters.\n\nANSWER: 3",3197 "costUsd": 0.00022053198 },3199 {3200 "test": "math",3201 "ok": false,3202 "latencyMs": 935,3203 "usage": {3204 "input": 50,3205 "output": 11,3206 "cachedInput": 0,3207 "reasoning": 03208 },3209 "api": "responses",3210 "raw": {3211 "status": "completed",3212 "incomplete": null3213 },3214 "sample": "a=427;b=2500",3215 "costUsd": 0.0000873216 },3217 {3218 "test": "code",3219 "ok": true,3220 "latencyMs": 1192,3221 "usage": {3222 "input": 49,3223 "output": 119,3224 "cachedInput": 0,3225 "reasoning": 03226 },3227 "api": "responses",3228 "raw": {3229 "status": "completed",3230 "incomplete": null3231 },3232 "sample": "```js\nfunction isPrime(n) {\n if (n <= 1 || n % 1 !== 0) return false;\n if (n <= 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return true;\n}\n```",3233 "costUsd": 0.000572253234 },3235 {3236 "test": "french",3237 "ok": true,3238 "latencyMs": 944,3239 "usage": {3240 "input": 31,3241 "output": 100,3242 "cachedInput": 0,3243 "reasoning": 03244 },3245 "api": "responses",3246 "raw": {3247 "status": "completed",3248 "incomplete": null3249 },3250 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d’un bien en additionnant la valeur du terrain et le coût de reconstruction à neuf du bâtiment, puis en retranchant la dépréciation liée à l’usure, à l’obsolescence ou à d’autres pertes de valeur. Elle est surtout utilisée pou",3251 "costUsd": 0.000473253252 },3253 {3254 "test": "english",3255 "ok": true,3256 "latencyMs": 888,3257 "usage": {3258 "input": 20,3259 "output": 66,3260 "cachedInput": 0,3261 "reasoning": 03262 },3263 "api": "responses",3264 "raw": {3265 "status": "completed",3266 "incomplete": null3267 },3268 "sample": "A hedonic pricing model is an economic method used to estimate the value of a good by breaking its price into the implicit values of its individual characteristics, such as location, size, quality, or amenities. It is often used in real estate and environmental economics to measure how much differen",3269 "costUsd": 0.0003123270 },3271 {3272 "test": "json",3273 "ok": true,3274 "latencyMs": 600,3275 "usage": {3276 "input": 52,3277 "output": 27,3278 "cachedInput": 0,3279 "reasoning": 03280 },3281 "api": "responses",3282 "raw": {3283 "status": "completed",3284 "incomplete": null3285 },3286 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":290000}",3287 "costUsd": 0.00016053288 },3289 {3290 "test": "tools",3291 "ok": true,3292 "latencyMs": 644,3293 "usage": {3294 "input": 69,3295 "output": 22,3296 "cachedInput": 0,3297 "reasoning": 03298 },3299 "api": "responses",3300 "raw": {3301 "status": "completed",3302 "incomplete": null3303 },3304 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",3305 "costUsd": 0.000150753306 },3307 {3308 "test": "vision",3309 "ok": true,3310 "latencyMs": 647,3311 "usage": {3312 "input": 194,3313 "output": 12,3314 "cachedInput": 0,3315 "reasoning": 03316 },3317 "api": "responses",3318 "raw": {3319 "status": "completed",3320 "incomplete": null3321 },3322 "sample": "circles=3; other=square",3323 "costUsd": 0.00019953324 },3325 {3326 "test": "longctx",3327 "ok": true,3328 "latencyMs": 671,3329 "usage": {3330 "input": 9134,3331 "output": 9,3332 "cachedInput": 0,3333 "reasoning": 03334 },3335 "api": "responses",3336 "raw": {3337 "status": "completed",3338 "incomplete": null3339 },3340 "sample": "7431-ALPHA",3341 "costUsd": 0.0068913342 },3343 {3344 "test": "streaming",3345 "ok": true,3346 "latencyMs": 739,3347 "ttftMs": 458,3348 "usage": {3349 "input": 15,3350 "output": 21,3351 "cachedInput": 0,3352 "reasoning": 03353 },3354 "api": "responses",3355 "sample": "Snow hushes Québec \nMaples sleep beneath ice light \nWind sings through stone streets",3356 "costUsd": 0.000105753357 }3358 ],3359 "api": "responses"3360 },3361 "gpt-5.4-nano": {3362 "provider": "openai",3363 "kind": "chat",3364 "tests": [3365 {3366 "test": "basic",3367 "ok": true,3368 "latencyMs": 570,3369 "usage": {3370 "input": 19,3371 "output": 6,3372 "cachedInput": 0,3373 "reasoning": 03374 },3375 "api": "responses",3376 "raw": {3377 "status": "completed",3378 "incomplete": null3379 },3380 "sample": "Ottawa",3381 "costUsd": 0.00001133382 },3383 {3384 "test": "reasoning",3385 "ok": false,3386 "latencyMs": 979,3387 "usage": {3388 "input": 42,3389 "output": 90,3390 "cachedInput": 0,3391 "reasoning": 03392 },3393 "api": "responses",3394 "raw": {3395 "status": "completed",3396 "incomplete": null3397 },3398 "sample": "Alice’s family has 3 brothers and 2 sisters total. Alice is one of the sisters, so the other sister count is 1. Each of Alice’s brothers therefore has that same other sister (Alice’s sister), plus Alice herself—but the question asks specifically “How many sisters does each brother have?” For a broth",3399 "costUsd": 0.000120900000000000013400 },3401 {3402 "test": "math",3403 "ok": false,3404 "latencyMs": 969,3405 "usage": {3406 "input": 50,3407 "output": 11,3408 "cachedInput": 0,3409 "reasoning": 03410 },3411 "api": "responses",3412 "raw": {3413 "status": "completed",3414 "incomplete": null3415 },3416 "sample": "a=491;b=2500",3417 "costUsd": 0.000023753418 },3419 {3420 "test": "code",3421 "ok": true,3422 "latencyMs": 1069,3423 "usage": {3424 "input": 49,3425 "output": 117,3426 "cachedInput": 0,3427 "reasoning": 03428 },3429 "api": "responses",3430 "raw": {3431 "status": "completed",3432 "incomplete": null3433 },3434 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n <= 1) return false;\n if (n <= 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n for (let i = 5; i * i <= n; i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n return true;\n}\n```",3435 "costUsd": 0.000156053436 },3437 {3438 "test": "french",3439 "ok": true,3440 "latencyMs": 1074,3441 "usage": {3442 "input": 31,3443 "output": 101,3444 "cachedInput": 0,3445 "reasoning": 03446 },3447 "api": "responses",3448 "raw": {3449 "status": "completed",3450 "incomplete": null3451 },3452 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d’un bien en additionnant le coût de reconstruction à neuf (ou de remplacement) du bien et la valeur de la dépréciation liée à son âge et à son état, puis en tenant compte des coûts éventuels. Elle aboutit à une estimation bas",3453 "costUsd": 0.000132453454 },3455 {3456 "test": "english",3457 "ok": true,3458 "latencyMs": 779,3459 "usage": {3460 "input": 20,3461 "output": 55,3462 "cachedInput": 0,3463 "reasoning": 03464 },3465 "api": "responses",3466 "raw": {3467 "status": "completed",3468 "incomplete": null3469 },3470 "sample": "A hedonic pricing model explains how the price of a good or service is determined by its underlying attributes. It estimates the contribution (implicit price) of each attribute—such as size, location, or quality—typically using statistical regression on observed market prices.",3471 "costUsd": 0.000072753472 },3473 {3474 "test": "json",3475 "ok": true,3476 "latencyMs": 524,3477 "usage": {3478 "input": 52,3479 "output": 27,3480 "cachedInput": 0,3481 "reasoning": 03482 },3483 "api": "responses",3484 "raw": {3485 "status": "completed",3486 "incomplete": null3487 },3488 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":317000}",3489 "costUsd": 0.0000441499999999999963490 },3491 {3492 "test": "tools",3493 "ok": true,3494 "latencyMs": 647,3495 "usage": {3496 "input": 69,3497 "output": 22,3498 "cachedInput": 0,3499 "reasoning": 03500 },3501 "api": "responses",3502 "raw": {3503 "status": "completed",3504 "incomplete": null3505 },3506 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",3507 "costUsd": 0.0000412999999999999943508 },3509 {3510 "test": "vision",3511 "ok": true,3512 "latencyMs": 590,3513 "usage": {3514 "input": 194,3515 "output": 12,3516 "cachedInput": 0,3517 "reasoning": 03518 },3519 "api": "responses",3520 "raw": {3521 "status": "completed",3522 "incomplete": null3523 },3524 "sample": "circles=3; other=square",3525 "costUsd": 0.000053800000000000013526 },3527 {3528 "test": "longctx",3529 "ok": true,3530 "latencyMs": 557,3531 "usage": {3532 "input": 9134,3533 "output": 9,3534 "cachedInput": 0,3535 "reasoning": 03536 },3537 "api": "responses",3538 "raw": {3539 "status": "completed",3540 "incomplete": null3541 },3542 "sample": "7431-ALPHA",3543 "costUsd": 0.00183805000000000013544 },3545 {3546 "test": "streaming",3547 "ok": true,3548 "latencyMs": 757,3549 "ttftMs": 541,3550 "usage": {3551 "input": 15,3552 "output": 25,3553 "cachedInput": 0,3554 "reasoning": 03555 },3556 "api": "responses",3557 "sample": "Quebec in winter— \nsnow hushes street and maple, \nblue dusk, wind sighs.",3558 "costUsd": 0.000034253559 }3560 ],3561 "api": "responses"3562 },3563 "gpt-5.4-pro": {3564 "provider": "openai",3565 "kind": "chat-pro",3566 "tests": [3567 {3568 "test": "basic",3569 "ok": true,3570 "latencyMs": 6194,3571 "usage": {3572 "input": 19,3573 "output": 25,3574 "cachedInput": 0,3575 "reasoning": 173576 },3577 "api": "responses",3578 "raw": {3579 "status": "completed",3580 "incomplete": null3581 },3582 "sample": "Ottawa",3583 "costUsd": 0.005073584 },3585 {3586 "test": "reasoning",3587 "ok": true,3588 "latencyMs": 14838,3589 "usage": {3590 "input": 42,3591 "output": 132,3592 "cachedInput": 0,3593 "reasoning": 643594 },3595 "api": "responses",3596 "raw": {3597 "status": "completed",3598 "incomplete": null3599 },3600 "sample": "Alice is one of the sisters.\n\nSo if Alice has **2 sisters**, that means there are **3 girls total** in the family:\n- Alice\n- Sister 1\n- Sister 2\n\nEach of Alice’s **3 brothers** has all **3 girls** as sisters.\n\nANSWER: 3",3601 "costUsd": 0.025023602 },3603 {3604 "test": "json",3605 "ok": true,3606 "latencyMs": 22867,3607 "usage": {3608 "input": 52,3609 "output": 199,3610 "cachedInput": 0,3611 "reasoning": 1703612 },3613 "api": "responses",3614 "raw": {3615 "status": "completed",3616 "incomplete": null3617 },3618 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",3619 "costUsd": 0.037383620 },3621 {3622 "test": "tools",3623 "ok": true,3624 "latencyMs": 9786,3625 "usage": {3626 "input": 69,3627 "output": 22,3628 "cachedInput": 0,3629 "reasoning": 03630 },3631 "api": "responses",3632 "raw": {3633 "status": "completed",3634 "incomplete": null3635 },3636 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",3637 "costUsd": 0.006033638 }3639 ],3640 "api": "responses"3641 },3642 "gpt-5.5": {3643 "provider": "openai",3644 "kind": "chat",3645 "tests": [3646 {3647 "test": "basic",3648 "ok": true,3649 "latencyMs": 1199,3650 "usage": {3651 "input": 19,3652 "output": 23,3653 "cachedInput": 0,3654 "reasoning": 153655 },3656 "api": "responses",3657 "raw": {3658 "status": "completed",3659 "incomplete": null3660 },3661 "sample": "Ottawa",3662 "costUsd": 0.0007853663 },3664 {3665 "test": "reasoning",3666 "ok": true,3667 "latencyMs": 2058,3668 "usage": {3669 "input": 42,3670 "output": 75,3671 "cachedInput": 0,3672 "reasoning": 483673 },3674 "api": "responses",3675 "raw": {3676 "status": "completed",3677 "incomplete": null3678 },3679 "sample": "Each of Alice’s brothers has 3 sisters: Alice and her 2 sisters.\n\nANSWER: 3",3680 "costUsd": 0.002463681 },3682 {3683 "test": "math",3684 "ok": true,3685 "latencyMs": 1478,3686 "usage": {3687 "input": 50,3688 "output": 54,3689 "cachedInput": 0,3690 "reasoning": 413691 },3692 "api": "responses",3693 "raw": {3694 "status": "completed",3695 "incomplete": null3696 },3697 "sample": "a=403;b=2500",3698 "costUsd": 0.001873699 },3700 {3701 "test": "code",3702 "ok": true,3703 "latencyMs": 2089,3704 "usage": {3705 "input": 49,3706 "output": 116,3707 "cachedInput": 0,3708 "reasoning": 233709 },3710 "api": "responses",3711 "raw": {3712 "status": "completed",3713 "incomplete": null3714 },3715 "sample": "```js\nfunction isPrime(n) {\n if (n <= 1) return false;\n if (n === 2) return true;\n if (n % 2 === 0) return false;\n\n for (let i = 3; i * i <= n; i += 2) {\n if (n % i === 0) return false;\n }\n\n return true;\n}\n```",3716 "costUsd": 0.0037253717 },3718 {3719 "test": "french",3720 "ok": true,3721 "latencyMs": 2150,3722 "usage": {3723 "input": 31,3724 "output": 85,3725 "cachedInput": 0,3726 "reasoning": 163727 },3728 "api": "responses",3729 "raw": {3730 "status": "completed",3731 "incomplete": null3732 },3733 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d’un bien en calculant le coût de remplacement ou de reconstruction du bâtiment, puis en retranchant la dépréciation éventuelle. On y ajoute généralement la valeur du terrain pour obtenir la valeur totale estimée de la proprié",3734 "costUsd": 0.0027053735 },3736 {3737 "test": "english",3738 "ok": true,3739 "latencyMs": 2207,3740 "usage": {3741 "input": 20,3742 "output": 85,3743 "cachedInput": 0,3744 "reasoning": 183745 },3746 "api": "responses",3747 "raw": {3748 "status": "completed",3749 "incomplete": null3750 },3751 "sample": "A hedonic pricing model estimates the value of a good by breaking it into its measurable characteristics, such as size, location, quality, or features. It is often used in real estate and environmental economics to infer how much people are willing to pay for specific attributes, like an extra bedro",3752 "costUsd": 0.002653753 },3754 {3755 "test": "json",3756 "ok": true,3757 "latencyMs": 2290,3758 "usage": {3759 "input": 52,3760 "output": 101,3761 "cachedInput": 0,3762 "reasoning": 723763 },3764 "api": "responses",3765 "raw": {3766 "status": "completed",3767 "incomplete": null3768 },3769 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",3770 "costUsd": 0.003293771 },3772 {3773 "test": "tools",3774 "ok": true,3775 "latencyMs": 1006,3776 "usage": {3777 "input": 69,3778 "output": 22,3779 "cachedInput": 0,3780 "reasoning": 03781 },3782 "api": "responses",3783 "raw": {3784 "status": "completed",3785 "incomplete": null3786 },3787 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",3788 "costUsd": 0.0010053789 },3790 {3791 "test": "vision",3792 "ok": true,3793 "latencyMs": 1236,3794 "usage": {3795 "input": 194,3796 "output": 35,3797 "cachedInput": 0,3798 "reasoning": 213799 },3800 "api": "responses",3801 "raw": {3802 "status": "completed",3803 "incomplete": null3804 },3805 "sample": "circles=3; other=square",3806 "costUsd": 0.002023807 },3808 {3809 "test": "longctx",3810 "ok": true,3811 "latencyMs": 1891,3812 "usage": {3813 "input": 9134,3814 "output": 67,3815 "cachedInput": 0,3816 "reasoning": 563817 },3818 "api": "responses",3819 "raw": {3820 "status": "completed",3821 "incomplete": null3822 },3823 "sample": "7431-ALPHA",3824 "costUsd": 0.047683825 },3826 {3827 "test": "streaming",3828 "ok": true,3829 "latencyMs": 1572,3830 "ttftMs": 730,3831 "usage": {3832 "input": 15,3833 "output": 22,3834 "cachedInput": 0,3835 "reasoning": 03836 },3837 "api": "responses",3838 "sample": "Snow veils old Quebec \nPines whisper in frozen light \nMaple dreams of spring",3839 "costUsd": 0.0007353840 }3841 ],3842 "api": "responses"3843 },3844 "gpt-5.5-pro": {3845 "provider": "openai",3846 "kind": "chat-pro",3847 "tests": [3848 {3849 "test": "basic",3850 "ok": true,3851 "latencyMs": 4383,3852 "usage": {3853 "input": 19,3854 "output": 34,3855 "cachedInput": 0,3856 "reasoning": 263857 },3858 "api": "responses",3859 "raw": {3860 "status": "completed",3861 "incomplete": null3862 },3863 "sample": "Ottawa",3864 "costUsd": 0.006693865 },3866 {3867 "test": "reasoning",3868 "ok": true,3869 "latencyMs": 5531,3870 "usage": {3871 "input": 42,3872 "output": 123,3873 "cachedInput": 0,3874 "reasoning": 923875 },3876 "api": "responses",3877 "raw": {3878 "status": "completed",3879 "incomplete": null3880 },3881 "sample": "Alice’s brothers have Alice plus her 2 sisters as sisters, so each brother has 3 sisters.\n\nANSWER: 3",3882 "costUsd": 0.02343883 },3884 {3885 "test": "json",3886 "ok": true,3887 "latencyMs": 10439,3888 "usage": {3889 "input": 52,3890 "output": 149,3891 "cachedInput": 0,3892 "reasoning": 1203893 },3894 "api": "responses",3895 "raw": {3896 "status": "completed",3897 "incomplete": null3898 },3899 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",3900 "costUsd": 0.028383901 },3902 {3903 "test": "tools",3904 "ok": true,3905 "latencyMs": 11055,3906 "usage": {3907 "input": 69,3908 "output": 22,3909 "cachedInput": 0,3910 "reasoning": 03911 },3912 "api": "responses",3913 "raw": {3914 "status": "completed",3915 "incomplete": null3916 },3917 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",3918 "costUsd": 0.006033919 }3920 ],3921 "api": "responses"3922 },3923 "gpt-5.6-luna": {3924 "provider": "openai",3925 "kind": "chat",3926 "tests": [3927 {3928 "test": "reasoning",3929 "ok": true,3930 "latencyMs": 1453,3931 "usage": {3932 "input": 42,3933 "output": 61,3934 "cachedInput": 0,3935 "reasoning": 293936 },3937 "api": "responses",3938 "raw": {3939 "status": "completed",3940 "incomplete": null3941 },3942 "sample": "Each of Alice’s brothers has Alice and her 2 sisters as sisters, so each has 3 sisters.\n\nANSWER: 3",3943 "costUsd": 0.00008163944 },3945 {3946 "test": "math",3947 "ok": true,3948 "latencyMs": 1256,3949 "usage": {3950 "input": 50,3951 "output": 46,3952 "cachedInput": 0,3953 "reasoning": 333954 },3955 "api": "responses",3956 "raw": {3957 "status": "completed",3958 "incomplete": null3959 },3960 "sample": "a=403;b=2500",3961 "costUsd": 0.000065199999999999993962 },3963 {3964 "test": "code",3965 "ok": true,3966 "latencyMs": 2650,3967 "usage": {3968 "input": 49,3969 "output": 172,3970 "cachedInput": 0,3971 "reasoning": 733972 },3973 "api": "responses",3974 "raw": {3975 "status": "completed",3976 "incomplete": null3977 },3978 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n < 2) return false;\n if (n === 2) return true;\n if (n % 2 === 0) return false;\n\n for (let i = 3; i * i <= n; i += 2) {\n if (n % i === 0) return false;\n }\n\n return true;\n}\n```",3979 "costUsd": 0.000216200000000000023980 },3981 {3982 "test": "french",3983 "ok": true,3984 "latencyMs": 1716,3985 "usage": {3986 "input": 31,3987 "output": 69,3988 "cachedInput": 0,3989 "reasoning": 03990 },3991 "api": "responses",3992 "raw": {3993 "status": "completed",3994 "incomplete": null3995 },3996 "sample": "La méthode du coût en évaluation immobilière estime la valeur d’un bien en additionnant le coût actuel de reconstruction ou de remplacement du bâtiment et la valeur du terrain. Elle déduit ensuite la dépréciation liée à l’âge, à l’usure, à l’obsolescence ou aux défauts du bien.",3997 "costUsd": 0.0000893998 },3999 {4000 "test": "english",4001 "ok": true,4002 "latencyMs": 1581,4003 "usage": {4004 "input": 20,4005 "output": 54,4006 "cachedInput": 0,4007 "reasoning": 04008 },4009 "api": "responses",4010 "raw": {4011 "status": "completed",4012 "incomplete": null4013 },4014 "sample": "A hedonic pricing model estimates how the individual characteristics of a differentiated good or service contribute to its overall price. It is commonly used to value nonmarket attributes—such as housing location, environmental quality, or product features—by analyzing observed price differences.",4015 "costUsd": 0.000068799999999999994016 },4017 {4018 "test": "json",4019 "ok": true,4020 "latencyMs": 2404,4021 "usage": {4022 "input": 52,4023 "output": 158,4024 "cachedInput": 0,4025 "reasoning": 1294026 },4027 "api": "responses",4028 "raw": {4029 "status": "completed",4030 "incomplete": null4031 },4032 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",4033 "costUsd": 0.00024034 },4035 {4036 "test": "tools",4037 "ok": true,4038 "latencyMs": 964,4039 "usage": {4040 "input": 69,4041 "output": 22,4042 "cachedInput": 0,4043 "reasoning": 04044 },4045 "api": "responses",4046 "raw": {4047 "status": "completed",4048 "incomplete": null4049 },4050 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",4051 "costUsd": 0.00004024052 },4053 {4054 "test": "vision",4055 "ok": true,4056 "latencyMs": 1008,4057 "usage": {4058 "input": 194,4059 "output": 12,4060 "cachedInput": 0,4061 "reasoning": 04062 },4063 "api": "responses",4064 "raw": {4065 "status": "completed",4066 "incomplete": null4067 },4068 "sample": "circles=3; other=square",4069 "costUsd": 0.0000532000000000000064070 },4071 {4072 "test": "longctx",4073 "ok": true,4074 "latencyMs": 1069,4075 "usage": {4076 "input": 9134,4077 "output": 37,4078 "cachedInput": 0,4079 "reasoning": 264080 },4081 "api": "responses",4082 "raw": {4083 "status": "completed",4084 "incomplete": null4085 },4086 "sample": "7431-ALPHA",4087 "costUsd": 0.00187120000000000044088 },4089 {4090 "test": "basic",4091 "ok": true,4092 "latencyMs": 1399,4093 "usage": {4094 "input": 19,4095 "output": 6,4096 "cachedInput": 0,4097 "reasoning": 04098 },4099 "api": "responses",4100 "raw": {4101 "status": "completed",4102 "incomplete": null4103 },4104 "sample": "Ottawa",4105 "costUsd": 0.0000114106 },4107 {4108 "test": "streaming",4109 "ok": true,4110 "latencyMs": 3609,4111 "ttftMs": 3271,4112 "usage": {4113 "input": 15,4114 "output": 336,4115 "cachedInput": 0,4116 "reasoning": 3114117 },4118 "api": "responses",4119 "sample": "Quebec snowflakes fall \nThrough the pine forests, moonlight \nChurch bells fade in snow",4120 "costUsd": 0.00040624121 }4122 ],4123 "api": "responses"4124 },4125 "gpt-5.6-sol": {4126 "provider": "openai",4127 "kind": "chat",4128 "tests": [4129 {4130 "test": "basic",4131 "ok": true,4132 "latencyMs": 839,4133 "usage": {4134 "input": 19,4135 "output": 6,4136 "cachedInput": 0,4137 "reasoning": 04138 },4139 "api": "responses",4140 "raw": {4141 "status": "completed",4142 "incomplete": null4143 },4144 "sample": "Ottawa",4145 "costUsd": 0.0001964146 },4147 {4148 "test": "reasoning",4149 "ok": true,4150 "latencyMs": 1594,4151 "usage": {4152 "input": 42,4153 "output": 60,4154 "cachedInput": 0,4155 "reasoning": 244156 },4157 "api": "responses",4158 "raw": {4159 "status": "completed",4160 "incomplete": null4161 },4162 "sample": "Alice’s brothers share the same siblings. Each brother has Alice plus her 2 sisters, for a total of 3 sisters.\n\nANSWER: 3",4163 "costUsd": 0.0013684164 },4165 {4166 "test": "math",4167 "ok": true,4168 "latencyMs": 776,4169 "usage": {4170 "input": 50,4171 "output": 11,4172 "cachedInput": 0,4173 "reasoning": 04174 },4175 "api": "responses",4176 "raw": {4177 "status": "completed",4178 "incomplete": null4179 },4180 "sample": "a=403;b=2500",4181 "costUsd": 0.000424182 },4183 {4184 "test": "code",4185 "ok": true,4186 "latencyMs": 1554,4187 "usage": {4188 "input": 49,4189 "output": 97,4190 "cachedInput": 0,4191 "reasoning": 04192 },4193 "api": "responses",4194 "raw": {4195 "status": "completed",4196 "incomplete": null4197 },4198 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n < 2) return false;\n if (n === 2) return true;\n if (n % 2 === 0) return false;\n\n for (let i = 3; i <= Math.sqrt(n); i += 2) {\n if (n % i === 0) return false;\n }\n\n return true;\n}\n```",4199 "costUsd": 0.0021364200 },4201 {4202 "test": "french",4203 "ok": true,4204 "latencyMs": 1922,4205 "usage": {4206 "input": 31,4207 "output": 112,4208 "cachedInput": 0,4209 "reasoning": 404210 },4211 "api": "responses",4212 "raw": {4213 "status": "completed",4214 "incomplete": null4215 },4216 "sample": "La méthode du coût estime la valeur d’un bien immobilier en additionnant la valeur du terrain au coût de reconstruction ou de remplacement des bâtiments, puis en déduisant leur dépréciation. Elle est surtout utilisée pour les biens récents, spécialisés ou rarement vendus, lorsque les comparaisons de",4217 "costUsd": 0.0023644218 },4219 {4220 "test": "english",4221 "ok": true,4222 "latencyMs": 1307,4223 "usage": {4224 "input": 20,4225 "output": 64,4226 "cachedInput": 0,4227 "reasoning": 134228 },4229 "api": "responses",4230 "raw": {4231 "status": "completed",4232 "incomplete": null4233 },4234 "sample": "A hedonic pricing model estimates an asset’s value by analyzing how its individual characteristics affect its price. It is commonly used in real estate to measure the value of features such as location, size, amenities, and environmental quality.",4235 "costUsd": 0.001364236 },4237 {4238 "test": "json",4239 "ok": true,4240 "latencyMs": 1527,4241 "usage": {4242 "input": 52,4243 "output": 77,4244 "cachedInput": 0,4245 "reasoning": 484246 },4247 "api": "responses",4248 "raw": {4249 "status": "completed",4250 "incomplete": null4251 },4252 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",4253 "costUsd": 0.0017484254 },4255 {4256 "test": "tools",4257 "ok": true,4258 "latencyMs": 691,4259 "usage": {4260 "input": 69,4261 "output": 22,4262 "cachedInput": 0,4263 "reasoning": 04264 },4265 "api": "responses",4266 "raw": {4267 "status": "completed",4268 "incomplete": null4269 },4270 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",4271 "costUsd": 0.0007164272 },4273 {4274 "test": "vision",4275 "ok": true,4276 "latencyMs": 777,4277 "usage": {4278 "input": 194,4279 "output": 12,4280 "cachedInput": 0,4281 "reasoning": 04282 },4283 "api": "responses",4284 "raw": {4285 "status": "completed",4286 "incomplete": null4287 },4288 "sample": "circles=3; other=square",4289 "costUsd": 0.0010164290 },4291 {4292 "test": "longctx",4293 "ok": true,4294 "latencyMs": 1025,4295 "usage": {4296 "input": 9134,4297 "output": 9,4298 "cachedInput": 0,4299 "reasoning": 04300 },4301 "api": "responses",4302 "raw": {4303 "status": "completed",4304 "incomplete": null4305 },4306 "sample": "7431-ALPHA",4307 "costUsd": 0.0367164308 },4309 {4310 "test": "streaming",4311 "ok": true,4312 "latencyMs": 2541,4313 "ttftMs": 2268,4314 "usage": {4315 "input": 15,4316 "output": 110,4317 "cachedInput": 0,4318 "reasoning": 864319 },4320 "api": "responses",4321 "sample": "Snow veils old Quebec \nMaples sleep by frozen streams \nBlue dusk warms the roofs",4322 "costUsd": 0.002264323 }4324 ],4325 "api": "responses"4326 },4327 "gpt-5.6-terra": {4328 "provider": "openai",4329 "kind": "chat",4330 "tests": [4331 {4332 "test": "basic",4333 "ok": true,4334 "latencyMs": 859,4335 "usage": {4336 "input": 19,4337 "output": 6,4338 "cachedInput": 0,4339 "reasoning": 04340 },4341 "api": "responses",4342 "raw": {4343 "status": "completed",4344 "incomplete": null4345 },4346 "sample": "Ottawa",4347 "costUsd": 0.000114348 },4349 {4350 "test": "reasoning",4351 "ok": true,4352 "latencyMs": 1206,4353 "usage": {4354 "input": 42,4355 "output": 52,4356 "cachedInput": 0,4357 "reasoning": 234358 },4359 "api": "responses",4360 "raw": {4361 "status": "completed",4362 "incomplete": null4363 },4364 "sample": "Alice and her 2 sisters make 3 sisters total. Each brother has 3 sisters.\n\nANSWER: 3",4365 "costUsd": 0.0007084366 },4367 {4368 "test": "math",4369 "ok": true,4370 "latencyMs": 1006,4371 "usage": {4372 "input": 50,4373 "output": 45,4374 "cachedInput": 0,4375 "reasoning": 324376 },4377 "api": "responses",4378 "raw": {4379 "status": "completed",4380 "incomplete": null4381 },4382 "sample": "a=403;b=2500",4383 "costUsd": 0.000644384 },4385 {4386 "test": "code",4387 "ok": true,4388 "latencyMs": 1223,4389 "usage": {4390 "input": 49,4391 "output": 97,4392 "cachedInput": 0,4393 "reasoning": 04394 },4395 "api": "responses",4396 "raw": {4397 "status": "completed",4398 "incomplete": null4399 },4400 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n < 2) return false;\n if (n === 2) return true;\n if (n % 2 === 0) return false;\n\n for (let i = 3; i * i <= n; i += 2) {\n if (n % i === 0) return false;\n }\n\n return true;\n}\n```",4401 "costUsd": 0.0012624402 },4403 {4404 "test": "french",4405 "ok": true,4406 "latencyMs": 1999,4407 "usage": {4408 "input": 31,4409 "output": 86,4410 "cachedInput": 0,4411 "reasoning": 04412 },4413 "api": "responses",4414 "raw": {4415 "status": "completed",4416 "incomplete": null4417 },4418 "sample": "La méthode du coût estime la valeur d’un bien immobilier en additionnant la valeur du terrain et le coût de reconstruction à neuf du bâtiment, puis en déduisant la dépréciation liée notamment à l’âge, à l’usure ou à l’obsolescence. Elle est particulièrement utilisée pour les immeubles récents, atypi",4419 "costUsd": 0.0010944420 },4421 {4422 "test": "english",4423 "ok": true,4424 "latencyMs": 1414,4425 "usage": {4426 "input": 20,4427 "output": 62,4428 "cachedInput": 0,4429 "reasoning": 04430 },4431 "api": "responses",4432 "raw": {4433 "status": "completed",4434 "incomplete": null4435 },4436 "sample": "A hedonic pricing model estimates how the price of a good reflects the value of its individual characteristics, such as a house’s size, location, number of bedrooms, and school quality. By statistically relating observed prices to these attributes, it can estimate the implicit price buyers place on ",4437 "costUsd": 0.0007844438 },4439 {4440 "test": "json",4441 "ok": true,4442 "latencyMs": 1214,4443 "usage": {4444 "input": 52,4445 "output": 63,4446 "cachedInput": 0,4447 "reasoning": 344448 },4449 "api": "responses",4450 "raw": {4451 "status": "completed",4452 "incomplete": null4453 },4454 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",4455 "costUsd": 0.000864456 },4457 {4458 "test": "tools",4459 "ok": true,4460 "latencyMs": 716,4461 "usage": {4462 "input": 69,4463 "output": 22,4464 "cachedInput": 0,4465 "reasoning": 04466 },4467 "api": "responses",4468 "raw": {4469 "status": "completed",4470 "incomplete": null4471 },4472 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",4473 "costUsd": 0.0004024474 },4475 {4476 "test": "vision",4477 "ok": true,4478 "latencyMs": 806,4479 "usage": {4480 "input": 194,4481 "output": 12,4482 "cachedInput": 0,4483 "reasoning": 04484 },4485 "api": "responses",4486 "raw": {4487 "status": "completed",4488 "incomplete": null4489 },4490 "sample": "circles=3; other=square",4491 "costUsd": 0.0005324492 },4493 {4494 "test": "longctx",4495 "ok": false,4496 "latencyMs": 1322,4497 "usage": {4498 "input": 9134,4499 "output": 51,4500 "cachedInput": 0,4501 "reasoning": 354502 },4503 "api": "responses",4504 "raw": {4505 "status": "completed",4506 "incomplete": null4507 },4508 "sample": "Je ne peux pas divulguer de code secret.",4509 "costUsd": 0.018884510 },4511 {4512 "test": "streaming",4513 "ok": true,4514 "latencyMs": 1116,4515 "ttftMs": 611,4516 "usage": {4517 "input": 15,4518 "output": 25,4519 "cachedInput": 0,4520 "reasoning": 04521 },4522 "api": "responses",4523 "sample": "Snow hushes pine boughs \nSt. Lawrence breathes silver mist \nQuébec dreams in white",4524 "costUsd": 0.000334525 }4526 ],4527 "api": "responses"4528 },4529 "gpt-6-astra": {4530 "provider": "openai",4531 "kind": "chat",4532 "tests": [4533 {4534 "test": "basic",4535 "ok": true,4536 "latencyMs": 1616,4537 "usage": {4538 "input": 19,4539 "output": 6,4540 "cachedInput": 0,4541 "reasoning": 04542 },4543 "api": "responses",4544 "raw": {4545 "status": "completed",4546 "incomplete": null4547 },4548 "sample": "Ottawa",4549 "costUsd": 0.000494550 },4551 {4552 "test": "reasoning",4553 "ok": true,4554 "latencyMs": 2021,4555 "usage": {4556 "input": 42,4557 "output": 43,4558 "cachedInput": 0,4559 "reasoning": 144560 },4561 "api": "responses",4562 "raw": {4563 "status": "completed",4564 "incomplete": null4565 },4566 "sample": "Each brother has Alice and her 2 sisters as sisters, for a total of 3.\n\nANSWER: 3",4567 "costUsd": 0.002574568 },4569 {4570 "test": "math",4571 "ok": true,4572 "latencyMs": 1203,4573 "usage": {4574 "input": 50,4575 "output": 28,4576 "cachedInput": 0,4577 "reasoning": 154578 },4579 "api": "responses",4580 "raw": {4581 "status": "completed",4582 "incomplete": null4583 },4584 "sample": "a=403;b=2500",4585 "costUsd": 0.00194586 },4587 {4588 "test": "code",4589 "ok": true,4590 "latencyMs": 2190,4591 "usage": {4592 "input": 49,4593 "output": 122,4594 "cachedInput": 0,4595 "reasoning": 04596 },4597 "api": "responses",4598 "raw": {4599 "status": "completed",4600 "incomplete": null4601 },4602 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n < 2) return false;\n if (n === 2 || n === 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n\n for (let i = 5; i <= Math.sqrt(n); i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n\n return true;\n}\n```",4603 "costUsd": 0.006594604 },4605 {4606 "test": "french",4607 "ok": true,4608 "latencyMs": 3449,4609 "usage": {4610 "input": 31,4611 "output": 107,4612 "cachedInput": 0,4613 "reasoning": 214614 },4615 "api": "responses",4616 "raw": {4617 "status": "completed",4618 "incomplete": null4619 },4620 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d’un bien en additionnant la valeur du terrain et le coût actuel de reconstruction ou de remplacement du bâtiment, puis en déduisant sa dépréciation liée à l’usure et à l’obsolescence. Elle est particulièrement utile pour les ",4621 "costUsd": 0.005664622 },4623 {4624 "test": "english",4625 "ok": true,4626 "latencyMs": 3273,4627 "usage": {4628 "input": 20,4629 "output": 72,4630 "cachedInput": 0,4631 "reasoning": 194632 },4633 "api": "responses",4634 "raw": {4635 "status": "completed",4636 "incomplete": null4637 },4638 "sample": "A hedonic pricing model estimates how individual characteristics of a product or asset contribute to its market price. For example, it can estimate how a home’s size, location, and proximity to parks affect its value while holding other factors constant.",4639 "costUsd": 0.00384640 },4641 {4642 "test": "json",4643 "ok": true,4644 "latencyMs": 2595,4645 "usage": {4646 "input": 52,4647 "output": 56,4648 "cachedInput": 0,4649 "reasoning": 274650 },4651 "api": "responses",4652 "raw": {4653 "status": "completed",4654 "incomplete": null4655 },4656 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",4657 "costUsd": 0.003324658 },4659 {4660 "test": "tools",4661 "ok": true,4662 "latencyMs": 1414,4663 "usage": {4664 "input": 69,4665 "output": 22,4666 "cachedInput": 0,4667 "reasoning": 04668 },4669 "api": "responses",4670 "raw": {4671 "status": "completed",4672 "incomplete": null4673 },4674 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",4675 "costUsd": 0.001794676 },4677 {4678 "test": "vision",4679 "ok": true,4680 "latencyMs": 1797,4681 "usage": {4682 "input": 194,4683 "output": 12,4684 "cachedInput": 0,4685 "reasoning": 04686 },4687 "api": "responses",4688 "raw": {4689 "status": "completed",4690 "incomplete": null4691 },4692 "sample": "circles=3; other=square",4693 "costUsd": 0.002544694 },4695 {4696 "test": "longctx",4697 "ok": true,4698 "latencyMs": 1398,4699 "usage": {4700 "input": 9134,4701 "output": 9,4702 "cachedInput": 0,4703 "reasoning": 04704 },4705 "api": "responses",4706 "raw": {4707 "status": "completed",4708 "incomplete": null4709 },4710 "sample": "7431-ALPHA",4711 "costUsd": 0.091794712 },4713 {4714 "test": "streaming",4715 "ok": true,4716 "latencyMs": 4253,4717 "ttftMs": 3799,4718 "usage": {4719 "input": 15,4720 "output": 185,4721 "cachedInput": 0,4722 "reasoning": 1634723 },4724 "api": "responses",4725 "sample": "Snow blankets Quebec \nCold winds sweep the frozen streets \nMaples dream of spring",4726 "costUsd": 0.00944727 }4728 ],4729 "api": "responses"4730 },4731 "gpt-6-luna": {4732 "provider": "openai",4733 "kind": "chat",4734 "tests": [4735 {4736 "test": "reasoning",4737 "ok": true,4738 "latencyMs": 1809,4739 "usage": {4740 "input": 42,4741 "output": 95,4742 "cachedInput": 0,4743 "reasoning": 684744 },4745 "api": "responses",4746 "raw": {4747 "status": "completed",4748 "incomplete": null4749 },4750 "sample": "Each brother has Alice and her 2 sisters, so each has 3 sisters.\n\nANSWER: 3",4751 "costUsd": 0.00005174752 },4753 {4754 "test": "math",4755 "ok": true,4756 "latencyMs": 1019,4757 "usage": {4758 "input": 50,4759 "output": 53,4760 "cachedInput": 0,4761 "reasoning": 404762 },4763 "api": "responses",4764 "raw": {4765 "status": "completed",4766 "incomplete": null4767 },4768 "sample": "a=403;b=2500",4769 "costUsd": 0.00003154770 },4771 {4772 "test": "code",4773 "ok": true,4774 "latencyMs": 2834,4775 "usage": {4776 "input": 49,4777 "output": 219,4778 "cachedInput": 0,4779 "reasoning": 1204780 },4781 "api": "responses",4782 "raw": {4783 "status": "completed",4784 "incomplete": null4785 },4786 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n < 2) return false;\n if (n === 2) return true;\n if (n % 2 === 0) return false;\n\n for (let i = 3; i <= Math.sqrt(n); i += 2) {\n if (n % i === 0) return false;\n }\n\n return true;\n}\n```",4787 "costUsd": 0.00011444788 },4789 {4790 "test": "french",4791 "ok": true,4792 "latencyMs": 2005,4793 "usage": {4794 "input": 31,4795 "output": 126,4796 "cachedInput": 0,4797 "reasoning": 434798 },4799 "api": "responses",4800 "raw": {4801 "status": "completed",4802 "incomplete": null4803 },4804 "sample": "La méthode du coût estime la valeur d’un bien en additionnant la valeur du terrain et le coût de reconstruction ou de remplacement du bâtiment, puis en déduisant la dépréciation et l’obsolescence. Elle est particulièrement utile pour les biens récents ou atypiques, mais peut être moins fiable lorsqu",4805 "costUsd": 0.00006614806 },4807 {4808 "test": "english",4809 "ok": true,4810 "latencyMs": 2286,4811 "usage": {4812 "input": 20,4813 "output": 94,4814 "cachedInput": 0,4815 "reasoning": 434816 },4817 "api": "responses",4818 "raw": {4819 "status": "completed",4820 "incomplete": null4821 },4822 "sample": "A hedonic pricing model estimates how much individual features of a product or service contribute to its market price. It typically uses statistical analysis to separate the value associated with characteristics such as a home’s size, location, and amenities.",4823 "costUsd": 0.0000494824 },4825 {4826 "test": "json",4827 "ok": true,4828 "latencyMs": 2719,4829 "usage": {4830 "input": 52,4831 "output": 190,4832 "cachedInput": 0,4833 "reasoning": 1614834 },4835 "api": "responses",4836 "raw": {4837 "status": "completed",4838 "incomplete": null4839 },4840 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":300000}",4841 "costUsd": 0.00010024842 },4843 {4844 "test": "tools",4845 "ok": true,4846 "latencyMs": 1070,4847 "usage": {4848 "input": 69,4849 "output": 22,4850 "cachedInput": 0,4851 "reasoning": 04852 },4853 "api": "responses",4854 "raw": {4855 "status": "completed",4856 "incomplete": null4857 },4858 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",4859 "costUsd": 0.0000178999999999999984860 },4861 {4862 "test": "vision",4863 "ok": true,4864 "latencyMs": 1181,4865 "usage": {4866 "input": 194,4867 "output": 30,4868 "cachedInput": 0,4869 "reasoning": 164870 },4871 "api": "responses",4872 "raw": {4873 "status": "completed",4874 "incomplete": null4875 },4876 "sample": "circles=3; other=square",4877 "costUsd": 0.00003444878 },4879 {4880 "test": "longctx",4881 "ok": true,4882 "latencyMs": 1216,4883 "usage": {4884 "input": 9134,4885 "output": 37,4886 "cachedInput": 0,4887 "reasoning": 264888 },4889 "api": "responses",4890 "raw": {4891 "status": "completed",4892 "incomplete": null4893 },4894 "sample": "7431-ALPHA",4895 "costUsd": 0.00093194896 },4897 {4898 "test": "basic",4899 "ok": true,4900 "latencyMs": 1368,4901 "usage": {4902 "input": 19,4903 "output": 20,4904 "cachedInput": 0,4905 "reasoning": 124906 },4907 "api": "responses",4908 "raw": {4909 "status": "completed",4910 "incomplete": null4911 },4912 "sample": "Ottawa",4913 "costUsd": 0.00001194914 },4915 {4916 "test": "streaming",4917 "ok": true,4918 "latencyMs": 3344,4919 "ttftMs": 2952,4920 "usage": {4921 "input": 15,4922 "output": 253,4923 "cachedInput": 0,4924 "reasoning": 2294925 },4926 "api": "responses",4927 "sample": "Snow crowns old Quebec \nCold north winds cross the river \nLanterns glow through snow",4928 "costUsd": 0.0001284929 }4930 ],4931 "api": "responses"4932 },4933 "gpt-6-sol": {4934 "provider": "openai",4935 "kind": "chat",4936 "tests": [4937 {4938 "test": "basic",4939 "ok": true,4940 "latencyMs": 835,4941 "usage": {4942 "input": 19,4943 "output": 6,4944 "cachedInput": 0,4945 "reasoning": 04946 },4947 "api": "responses",4948 "raw": {4949 "status": "completed",4950 "incomplete": null4951 },4952 "sample": "Ottawa",4953 "costUsd": 0.0000984954 },4955 {4956 "test": "reasoning",4957 "ok": true,4958 "latencyMs": 1516,4959 "usage": {4960 "input": 42,4961 "output": 49,4962 "cachedInput": 0,4963 "reasoning": 184964 },4965 "api": "responses",4966 "raw": {4967 "status": "completed",4968 "incomplete": null4969 },4970 "sample": "Alice’s brothers each have Alice and her two sisters as sisters, so each has 3 sisters. ANSWER: 3",4971 "costUsd": 0.0005744972 },4973 {4974 "test": "math",4975 "ok": true,4976 "latencyMs": 989,4977 "usage": {4978 "input": 50,4979 "output": 34,4980 "cachedInput": 0,4981 "reasoning": 214982 },4983 "api": "responses",4984 "raw": {4985 "status": "completed",4986 "incomplete": null4987 },4988 "sample": "a=403;b=2500",4989 "costUsd": 0.000444990 },4991 {4992 "test": "code",4993 "ok": true,4994 "latencyMs": 2111,4995 "usage": {4996 "input": 49,4997 "output": 214,4998 "cachedInput": 0,4999 "reasoning": 1155000 },5001 "api": "responses",5002 "raw": {5003 "status": "completed",5004 "incomplete": null5005 },5006 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isInteger(n) || n < 2) return false;\n if (n === 2) return true;\n if (n % 2 === 0) return false;\n\n for (let i = 3; i * i <= n; i += 2) {\n if (n % i === 0) return false;\n }\n\n return true;\n}\n```",5007 "costUsd": 0.0022385008 },5009 {5010 "test": "french",5011 "ok": true,5012 "latencyMs": 1738,5013 "usage": {5014 "input": 31,5015 "output": 91,5016 "cachedInput": 0,5017 "reasoning": 305018 },5019 "api": "responses",5020 "raw": {5021 "status": "completed",5022 "incomplete": null5023 },5024 "sample": "La méthode du coût estime la valeur d’un bien immobilier en additionnant la valeur du terrain au coût de reconstruction du bâtiment, puis en déduisant sa dépréciation. Elle est surtout utilisée pour les biens atypiques ou lorsqu’il existe peu de ventes comparables.",5025 "costUsd": 0.0009725026 },5027 {5028 "test": "english",5029 "ok": true,5030 "latencyMs": 1107,5031 "usage": {5032 "input": 20,5033 "output": 51,5034 "cachedInput": 0,5035 "reasoning": 05036 },5037 "api": "responses",5038 "raw": {5039 "status": "completed",5040 "incomplete": null5041 },5042 "sample": "A hedonic pricing model estimates the value of a product or asset by examining how its features affect its price. For example, it can estimate how much a house’s location, size, or number of bedrooms contributes to its sale price.",5043 "costUsd": 0.000555044 },5045 {5046 "test": "json",5047 "ok": true,5048 "latencyMs": 1636,5049 "usage": {5050 "input": 52,5051 "output": 74,5052 "cachedInput": 0,5053 "reasoning": 455054 },5055 "api": "responses",5056 "raw": {5057 "status": "completed",5058 "incomplete": null5059 },5060 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",5061 "costUsd": 0.0008445062 },5063 {5064 "test": "tools",5065 "ok": true,5066 "latencyMs": 951,5067 "usage": {5068 "input": 69,5069 "output": 22,5070 "cachedInput": 0,5071 "reasoning": 05072 },5073 "api": "responses",5074 "raw": {5075 "status": "completed",5076 "incomplete": null5077 },5078 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",5079 "costUsd": 0.0003585080 },5081 {5082 "test": "vision",5083 "ok": true,5084 "latencyMs": 963,5085 "usage": {5086 "input": 194,5087 "output": 12,5088 "cachedInput": 0,5089 "reasoning": 05090 },5091 "api": "responses",5092 "raw": {5093 "status": "completed",5094 "incomplete": null5095 },5096 "sample": "circles=3; other=square",5097 "costUsd": 0.0005085098 },5099 {5100 "test": "longctx",5101 "ok": true,5102 "latencyMs": 1092,5103 "usage": {5104 "input": 9134,5105 "output": 37,5106 "cachedInput": 0,5107 "reasoning": 265108 },5109 "api": "responses",5110 "raw": {5111 "status": "completed",5112 "incomplete": null5113 },5114 "sample": "7431-ALPHA",5115 "costUsd": 0.0186385116 },5117 {5118 "test": "streaming",5119 "ok": true,5120 "latencyMs": 1865,5121 "ttftMs": 1608,5122 "usage": {5123 "input": 15,5124 "output": 149,5125 "cachedInput": 0,5126 "reasoning": 1245127 },5128 "api": "responses",5129 "sample": "Snow veils old Quebec \nIce drifts down the Saint Lawrence \nMaples dream of spring",5130 "costUsd": 0.001525131 }5132 ],5133 "api": "responses"5134 },5135 "gpt-6.1-sol": {5136 "provider": "openai",5137 "kind": "chat",5138 "tests": [5139 {5140 "test": "basic",5141 "ok": true,5142 "latencyMs": 1397,5143 "usage": {5144 "input": 19,5145 "output": 6,5146 "cachedInput": 0,5147 "reasoning": 05148 },5149 "api": "responses",5150 "raw": {5151 "status": "completed",5152 "incomplete": null5153 },5154 "sample": "Ottawa",5155 "costUsd": 0.0000985156 },5157 {5158 "test": "reasoning",5159 "ok": true,5160 "latencyMs": 1848,5161 "usage": {5162 "input": 42,5163 "output": 45,5164 "cachedInput": 0,5165 "reasoning": 165166 },5167 "api": "responses",5168 "raw": {5169 "status": "completed",5170 "incomplete": null5171 },5172 "sample": "Each brother has Alice and her 2 sisters as sisters, for a total of 3.\n\nANSWER: 3",5173 "costUsd": 0.0005345174 },5175 {5176 "test": "math",5177 "ok": true,5178 "latencyMs": 1424,5179 "usage": {5180 "input": 50,5181 "output": 29,5182 "cachedInput": 0,5183 "reasoning": 165184 },5185 "api": "responses",5186 "raw": {5187 "status": "completed",5188 "incomplete": null5189 },5190 "sample": "a=403;b=2500",5191 "costUsd": 0.000395192 },5193 {5194 "test": "code",5195 "ok": true,5196 "latencyMs": 2020,5197 "usage": {5198 "input": 49,5199 "output": 123,5200 "cachedInput": 0,5201 "reasoning": 05202 },5203 "api": "responses",5204 "raw": {5205 "status": "completed",5206 "incomplete": null5207 },5208 "sample": "```js\nfunction isPrime(n) {\n if (!Number.isSafeInteger(n) || n < 2) return false;\n if (n === 2 || n === 3) return true;\n if (n % 2 === 0 || n % 3 === 0) return false;\n\n for (let i = 5; i <= Math.sqrt(n); i += 6) {\n if (n % i === 0 || n % (i + 2) === 0) return false;\n }\n\n return true;\n}\n```",5209 "costUsd": 0.0013285210 },5211 {5212 "test": "french",5213 "ok": true,5214 "latencyMs": 3055,5215 "usage": {5216 "input": 31,5217 "output": 87,5218 "cachedInput": 0,5219 "reasoning": 05220 },5221 "api": "responses",5222 "raw": {5223 "status": "completed",5224 "incomplete": null5225 },5226 "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d’un bien en additionnant la valeur du terrain et le coût actuel de reconstruction du bâtiment, diminué de sa dépréciation liée à l’usure, à l’obsolescence ou à des facteurs externes. Elle est particulièrement utile pour les b",5227 "costUsd": 0.0009325228 },5229 {5230 "test": "english",5231 "ok": true,5232 "latencyMs": 2410,5233 "usage": {5234 "input": 20,5235 "output": 73,5236 "cachedInput": 0,5237 "reasoning": 115238 },5239 "api": "responses",5240 "raw": {5241 "status": "completed",5242 "incomplete": null5243 },5244 "sample": "A hedonic pricing model estimates a product’s value by separating the contributions of its individual characteristics, such as a home’s size, location, and number of bedrooms. It uses observed market prices to estimate how much buyers are willing to pay for each characteristic, holding other factors",5245 "costUsd": 0.000775246 },5247 {5248 "test": "json",5249 "ok": true,5250 "latencyMs": 2327,5251 "usage": {5252 "input": 52,5253 "output": 65,5254 "cachedInput": 0,5255 "reasoning": 365256 },5257 "api": "responses",5258 "raw": {5259 "status": "completed",5260 "incomplete": null5261 },5262 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",5263 "costUsd": 0.0007545264 },5265 {5266 "test": "tools",5267 "ok": true,5268 "latencyMs": 1557,5269 "usage": {5270 "input": 69,5271 "output": 22,5272 "cachedInput": 0,5273 "reasoning": 05274 },5275 "api": "responses",5276 "raw": {5277 "status": "completed",5278 "incomplete": null5279 },5280 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",5281 "costUsd": 0.0003585282 },5283 {5284 "test": "vision",5285 "ok": true,5286 "latencyMs": 1188,5287 "usage": {5288 "input": 194,5289 "output": 12,5290 "cachedInput": 0,5291 "reasoning": 05292 },5293 "api": "responses",5294 "raw": {5295 "status": "completed",5296 "incomplete": null5297 },5298 "sample": "circles=3; other=square",5299 "costUsd": 0.0005085300 },5301 {5302 "test": "longctx",5303 "ok": true,5304 "latencyMs": 1358,5305 "usage": {5306 "input": 9134,5307 "output": 9,5308 "cachedInput": 0,5309 "reasoning": 05310 },5311 "api": "responses",5312 "raw": {5313 "status": "completed",5314 "incomplete": null5315 },5316 "sample": "7431-ALPHA",5317 "costUsd": 0.0183585318 },5319 {5320 "test": "streaming",5321 "ok": true,5322 "latencyMs": 3362,5323 "ttftMs": 3051,5324 "usage": {5325 "input": 15,5326 "output": 142,5327 "cachedInput": 0,5328 "reasoning": 1205329 },5330 "api": "responses",5331 "sample": "Snow blankets Québec \nCold winds sweep the frozen streets \nChurch bells warm the night",5332 "costUsd": 0.001455333 }5334 ],5335 "api": "responses"5336 },5337 "gpt-6.1-terra": {5338 "provider": "openai",5339 "kind": "chat",5340 "tests": [5341 {5342 "test": "basic",5343 "ok": true,5344 "latencyMs": 1679,5345 "usage": {5346 "input": 19,5347 "output": 6,5348 "cachedInput": 0,5349 "reasoning": 05350 },5351 "api": "responses",5352 "raw": {5353 "status": "completed",5354 "incomplete": null5355 },5356 "sample": "Ottawa",5357 "costUsd": 0.0000495358 },5359 {5360 "test": "reasoning",5361 "ok": true,5362 "latencyMs": 1913,5363 "usage": {5364 "input": 42,5365 "output": 41,5366 "cachedInput": 0,5367 "reasoning": 195368 },5369 "api": "responses",5370 "raw": {5371 "status": "completed",5372 "incomplete": null5373 },5374 "sample": "Each brother has Alice and her 2 sisters as sisters.\n\nANSWER: 3",5375 "costUsd": 0.0002475376 },5377 {5378 "test": "math",5379 "ok": true,5380 "latencyMs": 2414,5381 "usage": {5382 "input": 50,5383 "output": 37,5384 "cachedInput": 0,5385 "reasoning": 245386 },5387 "api": "responses",5388 "raw": {5389 "status": "completed",5390 "incomplete": null5391 },5392 "sample": "a=403;b=2500",5393 "costUsd": 0.0002355394 },5395 {5396 "test": "code",5397 "ok": true,5398 "latencyMs": 2772,5399 "usage": {5400 "input": 49,5401 "output": 139,5402 "cachedInput": 0,5403 "reasoning": 745404 },5405 "api": "responses",5406 "raw": {5407 "status": "completed",5408 "incomplete": null5409 },5410 "sample": "```js\nfunction isPrime(n) {\n if (n < 2) return false;\n for (let i = 2; i * i <= n; i++) {\n if (n % i === 0) return false;\n }\n return true;\n}\n```",5411 "costUsd": 0.0007445412 },5413 {5414 "test": "french",5415 "ok": true,5416 "latencyMs": 2289,5417 "usage": {5418 "input": 31,5419 "output": 67,5420 "cachedInput": 0,5421 "reasoning": 05422 },5423 "api": "responses",5424 "raw": {5425 "status": "completed",5426 "incomplete": null5427 },5428 "sample": "La méthode du coût consiste à estimer la valeur d’un bien immobilier en additionnant la valeur du terrain et le coût actuel de construction d’un bâtiment équivalent. On déduit ensuite la dépréciation liée notamment à l’usure, à la vétusté et aux éventuelles obsolescences.",5429 "costUsd": 0.0003665430 },5431 {5432 "test": "english",5433 "ok": true,5434 "latencyMs": 1832,5435 "usage": {5436 "input": 20,5437 "output": 51,5438 "cachedInput": 0,5439 "reasoning": 05440 },5441 "api": "responses",5442 "raw": {5443 "status": "completed",5444 "incomplete": null5445 },5446 "sample": "A hedonic pricing model estimates the value of a good by examining how its characteristics affect its price. For example, it can estimate how much features such as location, size, and number of bedrooms contribute to a home’s sale price.",5447 "costUsd": 0.0002755448 },5449 {5450 "test": "json",5451 "ok": true,5452 "latencyMs": 2208,5453 "usage": {5454 "input": 52,5455 "output": 105,5456 "cachedInput": 0,5457 "reasoning": 765458 },5459 "api": "responses",5460 "raw": {5461 "status": "completed",5462 "incomplete": null5463 },5464 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":300000}",5465 "costUsd": 0.0005775466 },5467 {5468 "test": "tools",5469 "ok": true,5470 "latencyMs": 1797,5471 "usage": {5472 "input": 69,5473 "output": 22,5474 "cachedInput": 0,5475 "reasoning": 05476 },5477 "api": "responses",5478 "raw": {5479 "status": "completed",5480 "incomplete": null5481 },5482 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",5483 "costUsd": 0.0001795484 },5485 {5486 "test": "vision",5487 "ok": true,5488 "latencyMs": 1076,5489 "usage": {5490 "input": 194,5491 "output": 12,5492 "cachedInput": 0,5493 "reasoning": 05494 },5495 "api": "responses",5496 "raw": {5497 "status": "completed",5498 "incomplete": null5499 },5500 "sample": "circles=3; other=square",5501 "costUsd": 0.0002545502 },5503 {5504 "test": "longctx",5505 "ok": true,5506 "latencyMs": 1390,5507 "usage": {5508 "input": 9134,5509 "output": 9,5510 "cachedInput": 0,5511 "reasoning": 05512 },5513 "api": "responses",5514 "raw": {5515 "status": "completed",5516 "incomplete": null5517 },5518 "sample": "7431-ALPHA",5519 "costUsd": 0.0091795520 },5521 {5522 "test": "streaming",5523 "ok": true,5524 "latencyMs": 2287,5525 "ttftMs": 1841,5526 "usage": {5527 "input": 15,5528 "output": 107,5529 "cachedInput": 0,5530 "reasoning": 835531 },5532 "api": "responses",5533 "sample": "Snow crowns Quebec roofs \nThe Saint Lawrence sleeps in ice \nCold stars gild the pines",5534 "costUsd": 0.000555535 }5536 ],5537 "api": "responses"5538 },5539 "o1": {5540 "provider": "openai",5541 "kind": "chat",5542 "tests": [5543 {5544 "test": "basic",5545 "ok": false,5546 "latencyMs": 434,5547 "error": "404 404 The model `o1` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"5548 }5549 ]5550 },5551 "o1-pro": {5552 "provider": "openai",5553 "kind": "chat-pro",5554 "tests": [5555 {5556 "test": "basic",5557 "ok": false,5558 "latencyMs": 241,5559 "error": "404 404 The model `o1-pro` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"5560 }5561 ]5562 },5563 "o3": {5564 "provider": "openai",5565 "kind": "chat",5566 "tests": [5567 {5568 "test": "basic",5569 "ok": true,5570 "latencyMs": 1135,5571 "usage": {5572 "input": 19,5573 "output": 35,5574 "cachedInput": 0,5575 "reasoning": 05576 },5577 "api": "responses",5578 "raw": {5579 "status": "completed",5580 "incomplete": null5581 },5582 "sample": "Ottawa",5583 "costUsd": 0.0003185584 },5585 {5586 "test": "reasoning",5587 "ok": true,5588 "latencyMs": 1643,5589 "usage": {5590 "input": 42,5591 "output": 225,5592 "cachedInput": 0,5593 "reasoning": 1285594 },5595 "api": "responses",5596 "raw": {5597 "status": "completed",5598 "incomplete": null5599 },5600 "sample": "Alice is one of the sisters herself. \nFor any one of her brothers, the female siblings are: \n1. Alice \n2. Sister #1 \n3. Sister #2 \n\nSo each brother has 3 sisters. \nANSWER: 3",5601 "costUsd": 0.0018845602 },5603 {5604 "test": "math",5605 "ok": true,5606 "latencyMs": 1925,5607 "usage": {5608 "input": 50,5609 "output": 214,5610 "cachedInput": 0,5611 "reasoning": 1925612 },5613 "api": "responses",5614 "raw": {5615 "status": "completed",5616 "incomplete": null5617 },5618 "sample": "a=403;b=2500",5619 "costUsd": 0.0018125620 },5621 {5622 "test": "code",5623 "ok": true,5624 "latencyMs": 1148,5625 "usage": {5626 "input": 49,5627 "output": 141,5628 "cachedInput": 0,5629 "reasoning": 05630 },5631 "api": "responses",5632 "raw": {5633 "status": "completed",5634 "incomplete": null5635 },5636 "sample": "```js\nfunction isPrime(n) {\n if (n <= 1 || n % 1 !== 0) return false;\n if (n === 2) return true;\n if (n % 2 === 0) return false;\n\n const limit = Math.sqrt(n);\n for (let i = 3; i <= limit; i += 2) {\n if (n % i === 0) return false;\n }\n return true;\n}\n```",5637 "costUsd": 0.0012265638 },5639 {5640 "test": "french",5641 "ok": true,5642 "latencyMs": 2235,5643 "usage": {5644 "input": 31,5645 "output": 290,5646 "cachedInput": 0,5647 "reasoning": 1925648 },5649 "api": "responses",5650 "raw": {5651 "status": "completed",5652 "incomplete": null5653 },5654 "sample": "La méthode du coût, ou approche par le coût de remplacement, consiste à estimer la valeur d’un bien immobilier en additionnant la valeur du terrain à celle du coût actuel de reconstruction ou de reconstitution du bâtiment, puis en retranchant la dépréciation physique, fonctionnelle et économique. El",5655 "costUsd": 0.0023825656 },5657 {5658 "test": "english",5659 "ok": true,5660 "latencyMs": 1757,5661 "usage": {5662 "input": 20,5663 "output": 234,5664 "cachedInput": 0,5665 "reasoning": 1285666 },5667 "api": "responses",5668 "raw": {5669 "status": "completed",5670 "incomplete": null5671 },5672 "sample": "A hedonic pricing model is an econometric technique that decomposes the market price of a differentiated good into the implicit prices of its observable attributes. By regressing transaction prices on characteristics such as size, quality, location, and environmental factors, it quantifies how much ",5673 "costUsd": 0.0019125674 },5675 {5676 "test": "json",5677 "ok": true,5678 "latencyMs": 1596,5679 "usage": {5680 "input": 52,5681 "output": 207,5682 "cachedInput": 0,5683 "reasoning": 1285684 },5685 "api": "responses",5686 "raw": {5687 "status": "completed",5688 "incomplete": null5689 },5690 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",5691 "costUsd": 0.001765692 },5693 {5694 "test": "tools",5695 "ok": true,5696 "latencyMs": 1182,5697 "usage": {5698 "input": 66,5699 "output": 135,5700 "cachedInput": 0,5701 "reasoning": 645702 },5703 "api": "responses",5704 "raw": {5705 "status": "completed",5706 "incomplete": null5707 },5708 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",5709 "costUsd": 0.0012125710 },5711 {5712 "test": "vision",5713 "ok": true,5714 "latencyMs": 1256,5715 "usage": {5716 "input": 112,5717 "output": 89,5718 "cachedInput": 0,5719 "reasoning": 645720 },5721 "api": "responses",5722 "raw": {5723 "status": "completed",5724 "incomplete": null5725 },5726 "sample": "circles=3; other=square",5727 "costUsd": 0.0009365728 },5729 {5730 "test": "longctx",5731 "ok": true,5732 "latencyMs": 1620,5733 "usage": {5734 "input": 9134,5735 "output": 192,5736 "cachedInput": 0,5737 "reasoning": 1285738 },5739 "api": "responses",5740 "raw": {5741 "status": "completed",5742 "incomplete": null5743 },5744 "sample": "7431-ALPHA",5745 "costUsd": 0.0198045746 },5747 {5748 "test": "streaming",5749 "ok": true,5750 "latencyMs": 1039,5751 "ttftMs": 660,5752 "usage": {5753 "input": 15,5754 "output": 39,5755 "cachedInput": 0,5756 "reasoning": 05757 },5758 "api": "responses",5759 "sample": "Snow cloaks silent pines, \nSaint-Lawrence exhales cold mist— \nStars crackle like ice.",5760 "costUsd": 0.0003425761 }5762 ],5763 "api": "responses"5764 },5765 "o3-mini": {5766 "provider": "openai",5767 "kind": "chat",5768 "tests": [5769 {5770 "test": "basic",5771 "ok": false,5772 "latencyMs": 275,5773 "error": "404 404 The model `o3-mini` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"5774 }5775 ]5776 },5777 "o3-pro": {5778 "provider": "openai",5779 "kind": "chat-pro",5780 "tests": [5781 {5782 "test": "basic",5783 "ok": true,5784 "latencyMs": 4543,5785 "usage": {5786 "input": 19,5787 "output": 35,5788 "cachedInput": 0,5789 "reasoning": 05790 },5791 "api": "responses",5792 "raw": {5793 "status": "completed",5794 "incomplete": null5795 },5796 "sample": "Ottawa",5797 "costUsd": 0.003185798 },5799 {5800 "test": "reasoning",5801 "ok": true,5802 "latencyMs": 9652,5803 "usage": {5804 "input": 42,5805 "output": 168,5806 "cachedInput": 0,5807 "reasoning": 645808 },5809 "api": "responses",5810 "raw": {5811 "status": "completed",5812 "incomplete": null5813 },5814 "sample": "A brother of Alice counts all his female siblings as sisters. \nAlice herself is female, and there are 2 additional sisters. \nThus, each brother has 1 (Alice) + 2 (other sisters) = 3 sisters.\n\nANSWER: 3",5815 "costUsd": 0.014285816 },5817 {5818 "test": "json",5819 "ok": true,5820 "latencyMs": 6627,5821 "usage": {5822 "input": 52,5823 "output": 218,5824 "cachedInput": 0,5825 "reasoning": 1285826 },5827 "api": "responses",5828 "raw": {5829 "status": "completed",5830 "incomplete": null5831 },5832 "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",5833 "costUsd": 0.018485834 },5835 {5836 "test": "tools",5837 "ok": true,5838 "latencyMs": 6422,5839 "usage": {5840 "input": 66,5841 "output": 117,5842 "cachedInput": 0,5843 "reasoning": 645844 },5845 "api": "responses",5846 "raw": {5847 "status": "completed",5848 "incomplete": null5849 },5850 "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",5851 "costUsd": 0.010685852 }5853 ],5854 "api": "responses"5855 },5856 "o4-mini": {5857 "provider": "openai",5858 "kind": "chat",5859 "tests": [5860 {5861 "test": "basic",5862 "ok": false,5863 "latencyMs": 243,5864 "error": "404 404 The model `o4-mini` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations"5865 }5866 ]5867 }5868 }5869}