SPB Git

Repository Public

xai-results.json 51.7 KB 1,699 lines json

1{2  "generatedAt": "2026-11-17T16:57:32.758Z",3  "provider": "xai",4  "budgetUsd": 6,5  "stoppedByBudget": false,6  "models": {7    "grok-4.20-0309-non-reasoning": {8      "provider": "xai",9      "kind": "chat",10      "tests": [11        {12          "test": "basic",13          "ok": true,14          "latencyMs": 2160,15          "usage": {16            "input": 197,17            "output": 2,18            "cachedInput": 128,19            "reasoning": 0,20            "costUsd": 0.0001168521          },22          "api": "responses",23          "raw": {24            "status": "completed",25            "incomplete": null26          },27          "sample": "Ottawa",28          "costUsd": 0.0001168529        },30        {31          "test": "reasoning",32          "ok": true,33          "latencyMs": 1542,34          "usage": {35            "input": 220,36            "output": 136,37            "cachedInput": 128,38            "reasoning": 0,39            "costUsd": 0.000480640          },41          "api": "responses",42          "raw": {43            "status": "completed",44            "incomplete": null45          },46          "sample": "Alice has 2 sisters (including herself, from the perspective of her brothers). Each of her brothers therefore has 3 sisters in total: Alice + the 2 other sisters.\n\nThe question is \"how many sisters does each of Alice's brothers have?\" The answer is not 2, because that would be omitting Alice herself",47          "costUsd": 0.000480648        },49        {50          "test": "math",51          "ok": false,52          "latencyMs": 528,53          "usage": {54            "input": 228,55            "output": 7,56            "cachedInput": 128,57            "reasoning": 0,58            "costUsd": 0.000168159          },60          "api": "responses",61          "raw": {62            "status": "completed",63            "incomplete": null64          },65          "sample": "a=475;b=2500",66          "costUsd": 0.000168167        },68        {69          "test": "code",70          "ok": true,71          "latencyMs": 735,72          "usage": {73            "input": 227,74            "output": 59,75            "cachedInput": 128,76            "reasoning": 0,77            "costUsd": 0.0002968578          },79          "api": "responses",80          "raw": {81            "status": "completed",82            "incomplete": null83          },84          "sample": "```js\nfunction isPrime(n) {\n  if (n < 2) return false;\n  for (let i = 2; i * i <= n; i++) {\n    if (n % i === 0) return false;\n  }\n  return true;\n}\n```",85          "costUsd": 0.0002968586        },87        {88          "test": "french",89          "ok": true,90          "latencyMs": 1086,91          "usage": {92            "input": 209,93            "output": 66,94            "cachedInput": 128,95            "reasoning": 0,96            "costUsd": 0.0002918597          },98          "api": "responses",99          "raw": {100            "status": "completed",101            "incomplete": null102          },103          "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d’un bien en additionnant le coût de reconstruction du bâtiment neuf, auquel on soustrait la dépréciation due à l’usure et à l’obsolescence. On ajoute ensuite la valeur du terrain pour obtenir la valeur totale du bien.",104          "costUsd": 0.00029185105        },106        {107          "test": "english",108          "ok": true,109          "latencyMs": 1444,110          "usage": {111            "input": 198,112            "output": 66,113            "cachedInput": 128,114            "reasoning": 0,115            "costUsd": 0.0002781116          },117          "api": "responses",118          "raw": {119            "status": "completed",120            "incomplete": null121          },122          "sample": "A hedonic pricing model is an econometric method that estimates the implicit prices of individual product attributes by regressing the observed market price of a good on its measurable characteristics. It is most commonly used in real estate and environmental economics to value features such as loca",123          "costUsd": 0.0002781124        },125        {126          "test": "json",127          "ok": true,128          "latencyMs": 1392,129          "usage": {130            "input": 229,131            "output": 35,132            "cachedInput": 128,133            "reasoning": 0,134            "costUsd": 0.00023935135          },136          "api": "responses",137          "raw": {138            "status": "completed",139            "incomplete": null140          },141          "sample": "{\n  \"city\": \"Gatineau\",\n  \"country\": \"Canada\",\n  \"province\": \"Quebec\",\n  \"population_estimate\": 299000\n}",142          "costUsd": 0.00023935143        },144        {145          "test": "tools",146          "ok": true,147          "latencyMs": 482,148          "usage": {149            "input": 295,150            "output": 14,151            "cachedInput": 128,152            "reasoning": 0,153            "costUsd": 0.00026935154          },155          "api": "responses",156          "raw": {157            "status": "completed",158            "incomplete": null159          },160          "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",161          "costUsd": 0.00026935162        },163        {164          "test": "vision",165          "ok": true,166          "latencyMs": 699,167          "usage": {168            "input": 375,169            "output": 7,170            "cachedInput": 128,171            "reasoning": 0,172            "costUsd": 0.00035185173          },174          "api": "responses",175          "raw": {176            "status": "completed",177            "incomplete": null178          },179          "sample": "circles=3; other=square",180          "costUsd": 0.00035185181        },182        {183          "test": "longctx",184          "ok": true,185          "latencyMs": 961,186          "usage": {187            "input": 9161,188            "output": 6,189            "cachedInput": 128,190            "reasoning": 0,191            "costUsd": 0.01133185192          },193          "api": "responses",194          "raw": {195            "status": "completed",196            "incomplete": null197          },198          "sample": "**7431-ALPHA**",199          "costUsd": 0.01133185200        },201        {202          "test": "streaming",203          "ok": true,204          "latencyMs": 949,205          "ttftMs": 625,206          "usage": {207            "input": 193,208            "output": 27,209            "cachedInput": 128,210            "reasoning": 0,211            "costUsd": 0.00017435212          },213          "api": "responses",214          "sample": "A haiku about winter in Quebec:\n\nSilent snow descends  \nQuébec’s spires wear white mantles  \nBreath turns into ghosts",215          "costUsd": 0.00017435216        }217      ],218      "api": "responses"219    },220    "grok-4.20-0309-reasoning": {221      "provider": "xai",222      "kind": "chat",223      "tests": [224        {225          "test": "basic",226          "ok": true,227          "latencyMs": 2422,228          "usage": {229            "input": 199,230            "output": 91,231            "cachedInput": 128,232            "reasoning": 89,233            "costUsd": 0.00034185234          },235          "api": "responses",236          "raw": {237            "status": "completed",238            "incomplete": null239          },240          "sample": "Ottawa",241          "costUsd": 0.00034185242        },243        {244          "test": "reasoning",245          "ok": true,246          "latencyMs": 4279,247          "usage": {248            "input": 222,249            "output": 464,250            "cachedInput": 128,251            "reasoning": 442,252            "costUsd": 0.0013031253          },254          "api": "responses",255          "raw": {256            "status": "completed",257            "incomplete": null258          },259          "sample": "Each of Alice's brothers has Alice herself + the 2 other girls as sisters.  \nANSWER: 3",260          "costUsd": 0.0013031261        },262        {263          "test": "math",264          "ok": true,265          "latencyMs": 3660,266          "usage": {267            "input": 230,268            "output": 431,269            "cachedInput": 128,270            "reasoning": 424,271            "costUsd": 0.0012306272          },273          "api": "responses",274          "raw": {275            "status": "completed",276            "incomplete": null277          },278          "sample": "a=403;b=2500",279          "costUsd": 0.0012306280        },281        {282          "test": "code",283          "ok": true,284          "latencyMs": 4260,285          "usage": {286            "input": 229,287            "output": 618,288            "cachedInput": 128,289            "reasoning": 511,290            "costUsd": 0.00169685291          },292          "api": "responses",293          "raw": {294            "status": "completed",295            "incomplete": null296          },297          "sample": "```js\nfunction isPrime(n) {\n  if (n <= 1) return false;\n  if (n <= 3) return true;\n  if (n % 2 === 0 || n % 3 === 0) return false;\n  for (let i = 5; i * i <= n; i += 6) {\n    if (n % i === 0 || n % (i + 2) === 0) return false;\n  }\n  return true;\n}\n```",298          "costUsd": 0.00169685299        },300        {301          "test": "french",302          "ok": true,303          "latencyMs": 5126,304          "usage": {305            "input": 211,306            "output": 539,307            "cachedInput": 128,308            "reasoning": 469,309            "costUsd": 0.00147685310          },311          "api": "responses",312          "raw": {313            "status": "completed",314            "incomplete": null315          },316          "sample": "La méthode du coût en évaluation immobilière estime la valeur d’un bien en additionnant la valeur du terrain et le coût de reconstruction de l’immeuble neuf, duquel on déduit la dépréciation physique, fonctionnelle et économique. Elle est surtout utilisée pour les propriétés spéciales ou lorsque les",317          "costUsd": 0.00147685318        },319        {320          "test": "english",321          "ok": true,322          "latencyMs": 3729,323          "usage": {324            "input": 200,325            "output": 452,326            "cachedInput": 128,327            "reasoning": 385,328            "costUsd": 0.0012456329          },330          "api": "responses",331          "raw": {332            "status": "completed",333            "incomplete": null334          },335          "sample": "A hedonic pricing model is an econometric method that estimates the implicit value of individual characteristics or attributes that make up the price of a good or service. It is most commonly applied to housing markets, where regression analysis breaks down a property’s sale price into contributions",336          "costUsd": 0.0012456337        },338        {339          "test": "json",340          "ok": true,341          "latencyMs": 4013,342          "usage": {343            "input": 231,344            "output": 528,345            "cachedInput": 128,346            "reasoning": 493,347            "costUsd": 0.00147435348          },349          "api": "responses",350          "raw": {351            "status": "completed",352            "incomplete": null353          },354          "sample": "{\n  \"city\": \"Gatineau\",\n  \"country\": \"Canada\",\n  \"province\": \"Quebec\",\n  \"population_estimate\": 292000\n}",355          "costUsd": 0.00147435356        },357        {358          "test": "tools",359          "ok": true,360          "latencyMs": 1366,361          "usage": {362            "input": 297,363            "output": 158,364            "cachedInput": 128,365            "reasoning": 144,366            "costUsd": 0.00063185367          },368          "api": "responses",369          "raw": {370            "status": "completed",371            "incomplete": null372          },373          "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",374          "costUsd": 0.00063185375        },376        {377          "test": "vision",378          "ok": true,379          "latencyMs": 2211,380          "usage": {381            "input": 377,382            "output": 250,383            "cachedInput": 128,384            "reasoning": 243,385            "costUsd": 0.00096185386          },387          "api": "responses",388          "raw": {389            "status": "completed",390            "incomplete": null391          },392          "sample": "circles=3; other=square",393          "costUsd": 0.00096185394        },395        {396          "test": "longctx",397          "ok": true,398          "latencyMs": 3324,399          "usage": {400            "input": 9163,401            "output": 384,402            "cachedInput": 128,403            "reasoning": 380,404            "costUsd": 0.01227935405          },406          "api": "responses",407          "raw": {408            "status": "completed",409            "incomplete": null410          },411          "sample": "7431-ALPHA",412          "costUsd": 0.01227935413        },414        {415          "test": "streaming",416          "ok": true,417          "latencyMs": 12393,418          "ttftMs": 12116,419          "usage": {420            "input": 195,421            "output": 1491,422            "cachedInput": 128,423            "reasoning": 1472,424            "costUsd": 0.00383685425          },426          "api": "responses",427          "sample": "**Haiku:**\n\nFrozen St. Lawrence  \nwhispers secrets of New France  \nunder silent snow",428          "costUsd": 0.00383685429        }430      ],431      "api": "responses"432    },433    "grok-4.20-multi-agent-0309": {434      "provider": "xai",435      "kind": "chat",436      "tests": [437        {438          "test": "basic",439          "ok": true,440          "latencyMs": 5048,441          "usage": {442            "input": 7494,443            "output": 1594,444            "cachedInput": 5760,445            "reasoning": 1455,446            "costUsd": 0.0073045447          },448          "api": "responses",449          "raw": {450            "status": "completed",451            "incomplete": null452          },453          "sample": "Ottawa\n\n\nThe capital of Canada is Ottawa. This is a basic geography fact that all team members confirmed independently.",454          "costUsd": 0.0073045455        },456        {457          "test": "reasoning",458          "ok": true,459          "latencyMs": 3648,460          "usage": {461            "input": 6056,462            "output": 1572,463            "cachedInput": 3840,464            "reasoning": 1395,465            "costUsd": 0.007468466          },467          "api": "responses",468          "raw": {469            "status": "completed",470            "incomplete": null471          },472          "sample": "3\n\n\nANSWER: 3",473          "costUsd": 0.007468474        },475        {476          "test": "math",477          "ok": true,478          "latencyMs": 3801,479          "usage": {480            "input": 6071,481            "output": 1733,482            "cachedInput": 4480,483            "reasoning": 1447,484            "costUsd": 0.00721725485          },486          "api": "responses",487          "raw": {488            "status": "completed",489            "incomplete": null490          },491          "sample": "a=403;b=2500\n\n\nThe sum of the first n odd positives is n² (provable by induction or by noting it's an arithmetic series with first term 1, last term 2n-1, n terms: n/2 * (first+last) = n/2 * (2n) = n²). For n=50 this is exactly 2500. The arithmetic for part (a) is elementary: 17*23 expands as (20-3)",492          "costUsd": 0.00721725493        },494        {495          "test": "code",496          "ok": true,497          "latencyMs": 6121,498          "usage": {499            "input": 7004,500            "output": 3252,501            "cachedInput": 4480,502            "reasoning": 2721,503            "costUsd": 0.012181504          },505          "api": "responses",506          "raw": {507            "status": "completed",508            "incomplete": null509          },510          "sample": "```js\nfunction isPrime(n) {\n  if (n <= 1) return false;\n  if (n <= 3) return true;\n  if (n % 2 === 0 || n % 3 === 0) return false;\n  for (let i = 5; i * i <= n; i += 6) {\n    if (n % i === 0 || n % (i + 2) === 0) return false;\n  }\n  return true;\n}\n```",511          "costUsd": 0.012181512        },513        {514          "test": "french",515          "ok": true,516          "latencyMs": 4429,517          "usage": {518            "input": 3951,519            "output": 2024,520            "cachedInput": 3200,521            "reasoning": 1858,522            "costUsd": 0.00663875523          },524          "api": "responses",525          "raw": {526            "status": "completed",527            "incomplete": null528          },529          "sample": "La méthode du coût en évaluation immobilière estime la valeur d’un bien en calculant le coût de reconstruction à neuf de la construction, auquel on ajoute la valeur du terrain. On en déduit ensuite la dépréciation liée à l’usure, à l’obsolescence fonctionnelle ou économique afin d’obtenir la valeur ",530          "costUsd": 0.00663875531        },532        {533          "test": "english",534          "ok": true,535          "latencyMs": 3515,536          "usage": {537            "input": 2695,538            "output": 1508,539            "cachedInput": 2560,540            "reasoning": 1320,541            "costUsd": 0.00445075542          },543          "api": "responses",544          "raw": {545            "status": "completed",546            "incomplete": null547          },548          "sample": "A hedonic pricing model is an econometric method that decomposes the observed market price of a good or service into implicit prices for its individual characteristics or attributes, typically using regression analysis. It is commonly applied in fields like real estate and environmental economics to",549          "costUsd": 0.00445075550        },551        {552          "test": "json",553          "ok": true,554          "latencyMs": 6934,555          "usage": {556            "input": 10066,557            "output": 3354,558            "cachedInput": 6400,559            "reasoning": 3070,560            "costUsd": 0.0142475561          },562          "api": "responses",563          "raw": {564            "status": "completed",565            "incomplete": null566          },567          "sample": "{\n  \"city\": \"Gatineau\",\n  \"country\": \"Canada\",\n  \"province\": \"Quebec\",\n  \"population_estimate\": 292000\n}",568          "costUsd": 0.0142475569        },570        {571          "test": "tools",572          "ok": false,573          "latencyMs": 122,574          "error": "400 400 \"Client-side tools for multi-agent models require beta access\""575        },576        {577          "test": "vision",578          "ok": true,579          "latencyMs": 4876,580          "usage": {581            "input": 8562,582            "output": 2092,583            "cachedInput": 4992,584            "reasoning": 1861,585            "costUsd": 0.0106909586          },587          "api": "responses",588          "raw": {589            "status": "completed",590            "incomplete": null591          },592          "sample": "circles=3; other=square\n\n\nThe image clearly depicts three red circles arranged horizontally at the top, with a single blue square positioned below them in the center. There are no other shapes or circles present. This directly maps to the required output format.",593          "costUsd": 0.0106909594        },595        {596          "test": "longctx",597          "ok": true,598          "latencyMs": 3428,599          "usage": {600            "input": 78630,601            "output": 1404,602            "cachedInput": 40960,603            "reasoning": 1268,604            "costUsd": 0.0587895605          },606          "api": "responses",607          "raw": {608            "status": "completed",609            "incomplete": null610          },611          "sample": "7431-ALPHA",612          "costUsd": 0.0587895613        },614        {615          "test": "streaming",616          "ok": true,617          "latencyMs": 18298,618          "ttftMs": 18176,619          "usage": {620            "input": 17011,621            "output": 8523,622            "cachedInput": 7168,623            "reasoning": 8277,624            "costUsd": 0.03504485625          },626          "api": "responses",627          "sample": "Snow whispers softly  \nOver Quebec's ancient stones  \nWinter's cold embrace",628          "costUsd": 0.03504485629        }630      ],631      "api": "responses"632    },633    "grok-4.3": {634      "provider": "xai",635      "kind": "chat",636      "tests": [637        {638          "test": "basic",639          "ok": true,640          "latencyMs": 1462,641          "usage": {642            "input": 205,643            "output": 82,644            "cachedInput": 128,645            "reasoning": 80,646            "costUsd": 0.00032685647          },648          "api": "responses",649          "raw": {650            "status": "completed",651            "incomplete": null652          },653          "sample": "Ottawa",654          "costUsd": 0.00032685655        },656        {657          "test": "reasoning",658          "ok": true,659          "latencyMs": 5200,660          "usage": {661            "input": 228,662            "output": 332,663            "cachedInput": 128,664            "reasoning": 284,665            "costUsd": 0.0009806666          },667          "api": "responses",668          "raw": {669            "status": "completed",670            "incomplete": null671          },672          "sample": "Alice is one of 3 girls in the family (herself + 2 sisters). There are also 3 boys (her brothers).  \n\nEach of those brothers therefore has all 3 girls as his sisters.  \n\nANSWER: 3",673          "costUsd": 0.0009806674        },675        {676          "test": "math",677          "ok": true,678          "latencyMs": 3210,679          "usage": {680            "input": 236,681            "output": 235,682            "cachedInput": 128,683            "reasoning": 228,684            "costUsd": 0.0007481685          },686          "api": "responses",687          "raw": {688            "status": "completed",689            "incomplete": null690          },691          "sample": "a=403;b=2500",692          "costUsd": 0.0007481693        },694        {695          "test": "code",696          "ok": true,697          "latencyMs": 6303,698          "usage": {699            "input": 235,700            "output": 483,701            "cachedInput": 128,702            "reasoning": 396,703            "costUsd": 0.00136685704          },705          "api": "responses",706          "raw": {707            "status": "completed",708            "incomplete": null709          },710          "sample": "```js\nfunction isPrime(n) {\n    if (n <= 1) return false;\n    if (n === 2) return true;\n    if (n % 2 === 0) return false;\n    for (let i = 3; i * i <= n; i += 2) {\n        if (n % i === 0) return false;\n    }\n    return true;\n}\n```",711          "costUsd": 0.00136685712        },713        {714          "test": "french",715          "ok": true,716          "latencyMs": 4852,717          "usage": {718            "input": 217,719            "output": 302,720            "cachedInput": 128,721            "reasoning": 248,722            "costUsd": 0.00089185723          },724          "api": "responses",725          "raw": {726            "status": "completed",727            "incomplete": null728          },729          "sample": "La méthode du coût en évaluation immobilière estime la valeur d’un bien en calculant le coût de reconstruction ou de remplacement à neuf, diminué de la dépréciation accumulée. On ajoute ensuite la valeur du terrain pour déterminer la valeur totale du bien.",730          "costUsd": 0.00089185731        },732        {733          "test": "english",734          "ok": true,735          "latencyMs": 5115,736          "usage": {737            "input": 206,738            "output": 286,739            "cachedInput": 128,740            "reasoning": 216,741            "costUsd": 0.0008381742          },743          "api": "responses",744          "raw": {745            "status": "completed",746            "incomplete": null747          },748          "sample": "A hedonic pricing model is an econometric method that decomposes the observed market price of a differentiated good (such as housing) into the implicit marginal values of its constituent attributes by regressing price on those characteristics while controlling for other factors. This revealed-prefer",749          "costUsd": 0.0008381750        },751        {752          "test": "json",753          "ok": true,754          "latencyMs": 6450,755          "usage": {756            "input": 237,757            "output": 447,758            "cachedInput": 128,759            "reasoning": 416,760            "costUsd": 0.00127935761          },762          "api": "responses",763          "raw": {764            "status": "completed",765            "incomplete": null766          },767          "sample": "{\n\"city\": \"Gatineau\",\n\"country\": \"Canada\",\n\"province\": \"Quebec\",\n\"population_estimate\": 291000\n}",768          "costUsd": 0.00127935769        },770        {771          "test": "tools",772          "ok": true,773          "latencyMs": 2142,774          "usage": {775            "input": 303,776            "output": 123,777            "cachedInput": 128,778            "reasoning": 109,779            "costUsd": 0.00055185780          },781          "api": "responses",782          "raw": {783            "status": "completed",784            "incomplete": null785          },786          "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",787          "costUsd": 0.00055185788        },789        {790          "test": "vision",791          "ok": true,792          "latencyMs": 2885,793          "usage": {794            "input": 385,795            "output": 182,796            "cachedInput": 128,797            "reasoning": 175,798            "costUsd": 0.00080185799          },800          "api": "responses",801          "raw": {802            "status": "completed",803            "incomplete": null804          },805          "sample": "circles=3; other=square",806          "costUsd": 0.00080185807        },808        {809          "test": "longctx",810          "ok": true,811          "latencyMs": 5407,812          "usage": {813            "input": 9169,814            "output": 396,815            "cachedInput": 128,816            "reasoning": 392,817            "costUsd": 0.01231685818          },819          "api": "responses",820          "raw": {821            "status": "completed",822            "incomplete": null823          },824          "sample": "7431-ALPHA",825          "costUsd": 0.01231685826        },827        {828          "test": "streaming",829          "ok": true,830          "latencyMs": 11708,831          "ttftMs": 11392,832          "usage": {833            "input": 201,834            "output": 745,835            "cachedInput": 128,836            "reasoning": 723,837            "costUsd": 0.00197935838          },839          "api": "responses",840          "sample": "Snow veils Quebec's streets,  \nIcy winds howl through the night,  \nSpring waits in silence.",841          "costUsd": 0.00197935842        }843      ],844      "api": "responses"845    },846    "grok-4.5": {847      "provider": "xai",848      "kind": "chat",849      "tests": [850        {851          "test": "basic",852          "ok": true,853          "latencyMs": 1133,854          "usage": {855            "input": 507,856            "output": 54,857            "cachedInput": 384,858            "reasoning": 52,859            "costUsd": 0.0006852860          },861          "api": "responses",862          "raw": {863            "status": "completed",864            "incomplete": null865          },866          "sample": "Ottawa",867          "costUsd": 0.0006852868        },869        {870          "test": "reasoning",871          "ok": true,872          "latencyMs": 3429,873          "usage": {874            "input": 531,875            "output": 243,876            "cachedInput": 384,877            "reasoning": 193,878            "costUsd": 0.0018672879          },880          "api": "responses",881          "raw": {882            "status": "completed",883            "incomplete": null884          },885          "sample": "Alice has 3 brothers and 2 sisters, so the family has 3 boys and 3 girls total (Alice + her 2 sisters).\n\nEach of Alice's brothers therefore has the same 3 girls as sisters.\n\nANSWER: 3",886          "costUsd": 0.0018672887        },888        {889          "test": "math",890          "ok": true,891          "latencyMs": 2106,892          "usage": {893            "input": 538,894            "output": 151,895            "cachedInput": 384,896            "reasoning": 143,897            "costUsd": 0.0013292898          },899          "api": "responses",900          "raw": {901            "status": "completed",902            "incomplete": null903          },904          "sample": "a=403;b=2500",905          "costUsd": 0.0013292906        },907        {908          "test": "code",909          "ok": true,910          "latencyMs": 3276,911          "usage": {912            "input": 538,913            "output": 385,914            "cachedInput": 384,915            "reasoning": 277,916            "costUsd": 0.0027332917          },918          "api": "responses",919          "raw": {920            "status": "completed",921            "incomplete": null922          },923          "sample": "```js\nfunction isPrime(n) {\n  if (n <= 1) return false;\n  if (n <= 3) return true;\n  if (n % 2 === 0 || n % 3 === 0) return false;\n  for (let i = 5; i * i <= n; i += 6) {\n    if (n % i === 0 || n % (i + 2) === 0) return false;\n  }\n  return true;\n}\n```",924          "costUsd": 0.0027332925        },926        {927          "test": "french",928          "ok": true,929          "latencyMs": 3964,930          "usage": {931            "input": 520,932            "output": 242,933            "cachedInput": 384,934            "reasoning": 176,935            "costUsd": 0.0018392936          },937          "api": "responses",938          "raw": {939            "status": "completed",940            "incomplete": null941          },942          "sample": "La méthode du coût en évaluation immobilière estime la valeur d’un bien en calculant le coût de reconstruction ou de remplacement des bâtiments, diminué de la dépréciation, puis en y ajoutant la valeur du terrain. Elle est surtout employée pour des biens atypiques ou lorsque les comparables de march",943          "costUsd": 0.0018392944        },945        {946          "test": "english",947          "ok": true,948          "latencyMs": 2652,949          "usage": {950            "input": 508,951            "output": 117,952            "cachedInput": 384,953            "reasoning": 42,954            "costUsd": 0.0010652955          },956          "api": "responses",957          "raw": {958            "status": "completed",959            "incomplete": null960          },961          "sample": "A hedonic pricing model is a statistical method used in economics to estimate the value of a good or service by breaking it down into its constituent characteristics and measuring how each attribute contributes to the overall market price. It is commonly applied to heterogeneous goods such as housin",962          "costUsd": 0.0010652963        },964        {965          "test": "json",966          "ok": true,967          "latencyMs": 2260,968          "usage": {969            "input": 539,970            "output": 180,971            "cachedInput": 384,972            "reasoning": 154,973            "costUsd": 0.0015052974          },975          "api": "responses",976          "raw": {977            "status": "completed",978            "incomplete": null979          },980          "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291000}",981          "costUsd": 0.0015052982        },983        {984          "test": "tools",985          "ok": true,986          "latencyMs": 836,987          "usage": {988            "input": 606,989            "output": 55,990            "cachedInput": 512,991            "reasoning": 40,992            "costUsd": 0.0006716993          },994          "api": "responses",995          "raw": {996            "status": "completed",997            "incomplete": null998          },999          "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",1000          "costUsd": 0.00067161001        },1002        {1003          "test": "vision",1004          "ok": true,1005          "latencyMs": 1798,1006          "usage": {1007            "input": 694,1008            "output": 110,1009            "cachedInput": 512,1010            "reasoning": 103,1011            "costUsd": 0.00117761012          },1013          "api": "responses",1014          "raw": {1015            "status": "completed",1016            "incomplete": null1017          },1018          "sample": "circles=3; other=square",1019          "costUsd": 0.00117761020        },1021        {1022          "test": "longctx",1023          "ok": true,1024          "latencyMs": 1516,1025          "usage": {1026            "input": 9397,1027            "output": 78,1028            "cachedInput": 384,1029            "reasoning": 74,1030            "costUsd": 0.01860921031          },1032          "api": "responses",1033          "raw": {1034            "status": "completed",1035            "incomplete": null1036          },1037          "sample": "7431-ALPHA",1038          "costUsd": 0.01860921039        },1040        {1041          "test": "streaming",1042          "ok": true,1043          "latencyMs": 3253,1044          "ttftMs": 2869,1045          "usage": {1046            "input": 503,1047            "output": 173,1048            "cachedInput": 384,1049            "reasoning": 154,1050            "costUsd": 0.00139121051          },1052          "api": "responses",1053          "sample": "Snow blankets the pines  \nQuebec winds howl through the night  \nFrost claims the silent streets",1054          "costUsd": 0.00139121055        }1056      ],1057      "api": "responses"1058    },1059    "grok-4.6": {1060      "provider": "xai",1061      "kind": "chat",1062      "tests": [1063        {1064          "test": "basic",1065          "ok": true,1066          "latencyMs": 2986,1067          "usage": {1068            "input": 649,1069            "output": 186,1070            "cachedInput": 512,1071            "reasoning": 184,1072            "costUsd": 0.0016461073          },1074          "api": "responses",1075          "raw": {1076            "status": "completed",1077            "incomplete": null1078          },1079          "sample": "Ottawa",1080          "costUsd": 0.0016461081        },1082        {1083          "test": "reasoning",1084          "ok": true,1085          "latencyMs": 6609,1086          "usage": {1087            "input": 673,1088            "output": 342,1089            "cachedInput": 512,1090            "reasoning": 287,1091            "costUsd": 0.002631092          },1093          "api": "responses",1094          "raw": {1095            "status": "completed",1096            "incomplete": null1097          },1098          "sample": "Alice is one of the girls in the family and has 2 sisters, so there are 3 girls in total. She also has 3 brothers, so there are 3 boys. Each of those brothers therefore has all 3 girls as sisters.\n\nANSWER: 3",1099          "costUsd": 0.002631100        },1101        {1102          "test": "math",1103          "ok": true,1104          "latencyMs": 3126,1105          "usage": {1106            "input": 680,1107            "output": 230,1108            "cachedInput": 512,1109            "reasoning": 222,1110            "costUsd": 0.0019721111          },1112          "api": "responses",1113          "raw": {1114            "status": "completed",1115            "incomplete": null1116          },1117          "sample": "a=403;b=2500",1118          "costUsd": 0.0019721119        },1120        {1121          "test": "code",1122          "ok": true,1123          "latencyMs": 4029,1124          "usage": {1125            "input": 680,1126            "output": 351,1127            "cachedInput": 512,1128            "reasoning": 243,1129            "costUsd": 0.0026981130          },1131          "api": "responses",1132          "raw": {1133            "status": "completed",1134            "incomplete": null1135          },1136          "sample": "```js\nfunction isPrime(n) {\n  if (n <= 1) return false;\n  if (n <= 3) return true;\n  if (n % 2 === 0 || n % 3 === 0) return false;\n  for (let i = 5; i * i <= n; i += 6) {\n    if (n % i === 0 || n % (i + 2) === 0) return false;\n  }\n  return true;\n}\n```",1137          "costUsd": 0.0026981138        },1139        {1140          "test": "french",1141          "ok": true,1142          "latencyMs": 11152,1143          "usage": {1144            "input": 662,1145            "output": 678,1146            "cachedInput": 512,1147            "reasoning": 612,1148            "costUsd": 0.0046241149          },1150          "api": "responses",1151          "raw": {1152            "status": "completed",1153            "incomplete": null1154          },1155          "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en additionnant le coût de reconstruction du bâtiment, après déduction de la dépréciation, à la valeur du terrain. Cette méthode est particulièrement adaptée aux immeubles récents ou aux propriétés pour lesquelles il",1156          "costUsd": 0.0046241157        },1158        {1159          "test": "english",1160          "ok": true,1161          "latencyMs": 7004,1162          "usage": {1163            "input": 650,1164            "output": 314,1165            "cachedInput": 512,1166            "reasoning": 255,1167            "costUsd": 0.0024161168          },1169          "api": "responses",1170          "raw": {1171            "status": "completed",1172            "incomplete": null1173          },1174          "sample": "A hedonic pricing model is an econometric technique that infers the implicit prices of a good’s individual attributes by regressing observed market prices on those characteristics. It is widely used in housing and environmental economics to isolate how features such as location, size, or air quality",1175          "costUsd": 0.0024161176        },1177        {1178          "test": "json",1179          "ok": true,1180          "latencyMs": 5269,1181          "usage": {1182            "input": 681,1183            "output": 368,1184            "cachedInput": 512,1185            "reasoning": 342,1186            "costUsd": 0.0028021187          },1188          "api": "responses",1189          "raw": {1190            "status": "completed",1191            "incomplete": null1192          },1193          "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291000}",1194          "costUsd": 0.0028021195        },1196        {1197          "test": "tools",1198          "ok": true,1199          "latencyMs": 1686,1200          "usage": {1201            "input": 748,1202            "output": 96,1203            "cachedInput": 640,1204            "reasoning": 81,1205            "costUsd": 0.0011121206          },1207          "api": "responses",1208          "raw": {1209            "status": "completed",1210            "incomplete": null1211          },1212          "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",1213          "costUsd": 0.0011121214        },1215        {1216          "test": "vision",1217          "ok": true,1218          "latencyMs": 1924,1219          "usage": {1220            "input": 836,1221            "output": 114,1222            "cachedInput": 640,1223            "reasoning": 107,1224            "costUsd": 0.0013961225          },1226          "api": "responses",1227          "raw": {1228            "status": "completed",1229            "incomplete": null1230          },1231          "sample": "circles=3; other=square",1232          "costUsd": 0.0013961233        },1234        {1235          "test": "longctx",1236          "ok": true,1237          "latencyMs": 8405,1238          "usage": {1239            "input": 9539,1240            "output": 568,1241            "cachedInput": 512,1242            "reasoning": 564,1243            "costUsd": 0.0217181244          },1245          "api": "responses",1246          "raw": {1247            "status": "completed",1248            "incomplete": null1249          },1250          "sample": "7431-ALPHA",1251          "costUsd": 0.0217181252        },1253        {1254          "test": "streaming",1255          "ok": true,1256          "latencyMs": 16593,1257          "ttftMs": 16518,1258          "usage": {1259            "input": 645,1260            "output": 764,1261            "cachedInput": 512,1262            "reasoning": 745,1263            "costUsd": 0.0051061264          },1265          "api": "responses",1266          "sample": "Snow drapes the rooftops  \nQuebec's winter silence deep  \nFrosted breath in air",1267          "costUsd": 0.0051061268        }1269      ],1270      "api": "responses"1271    },1272    "grok-4.7": {1273      "provider": "xai",1274      "kind": "chat",1275      "tests": [1276        {1277          "test": "basic",1278          "ok": true,1279          "latencyMs": 969,1280          "usage": {1281            "input": 1255,1282            "output": 63,1283            "cachedInput": 1152,1284            "reasoning": 61,1285            "costUsd": 0.001161286          },1287          "api": "responses",1288          "raw": {1289            "status": "completed",1290            "incomplete": null1291          },1292          "sample": "Ottawa",1293          "costUsd": 0.001161294        },1295        {1296          "test": "reasoning",1297          "ok": true,1298          "latencyMs": 4640,1299          "usage": {1300            "input": 1279,1301            "output": 345,1302            "cachedInput": 1152,1303            "reasoning": 304,1304            "costUsd": 0.00291305          },1306          "api": "responses",1307          "raw": {1308            "status": "completed",1309            "incomplete": null1310          },1311          "sample": "Alice has 2 sisters, so there are 3 girls in the family in total (Alice plus her 2 sisters). Each of her brothers therefore has those same 3 sisters.\n\nANSWER: 3",1312          "costUsd": 0.00291313        },1314        {1315          "test": "math",1316          "ok": true,1317          "latencyMs": 1990,1318          "usage": {1319            "input": 1286,1320            "output": 133,1321            "cachedInput": 1152,1322            "reasoning": 125,1323            "costUsd": 0.0016421324          },1325          "api": "responses",1326          "raw": {1327            "status": "completed",1328            "incomplete": null1329          },1330          "sample": "a=403;b=2500",1331          "costUsd": 0.0016421332        },1333        {1334          "test": "code",1335          "ok": true,1336          "latencyMs": 3059,1337          "usage": {1338            "input": 1286,1339            "output": 307,1340            "cachedInput": 1152,1341            "reasoning": 199,1342            "costUsd": 0.0026861343          },1344          "api": "responses",1345          "raw": {1346            "status": "completed",1347            "incomplete": null1348          },1349          "sample": "```js\nfunction isPrime(n) {\n  if (n <= 1) return false;\n  if (n <= 3) return true;\n  if (n % 2 === 0 || n % 3 === 0) return false;\n  for (let i = 5; i * i <= n; i += 6) {\n    if (n % i === 0 || n % (i + 2) === 0) return false;\n  }\n  return true;\n}\n```",1350          "costUsd": 0.0026861351        },1352        {1353          "test": "french",1354          "ok": true,1355          "latencyMs": 6716,1356          "usage": {1357            "input": 1268,1358            "output": 240,1359            "cachedInput": 1152,1360            "reasoning": 177,1361            "costUsd": 0.0022481362          },1363          "api": "responses",1364          "raw": {1365            "status": "completed",1366            "incomplete": null1367          },1368          "sample": "La méthode du coût en évaluation immobilière estime la valeur d’un bien en additionnant la valeur du terrain au coût de reconstruction ou de remplacement des bâtiments, diminué de la dépréciation. Elle sert surtout lorsque le bien est atypique ou que les méthodes par comparaison et par le revenu son",1369          "costUsd": 0.0022481370        },1371        {1372          "test": "english",1373          "ok": true,1374          "latencyMs": 4758,1375          "usage": {1376            "input": 1256,1377            "output": 215,1378            "cachedInput": 1152,1379            "reasoning": 161,1380            "costUsd": 0.0020741381          },1382          "api": "responses",1383          "raw": {1384            "status": "completed",1385            "incomplete": null1386          },1387          "sample": "A hedonic pricing model estimates the value of a good by relating its market price to the implicit prices of its individual characteristics. It is widely used in areas such as real estate and environmental economics to measure how attributes like size, location, or air quality affect observed prices",1388          "costUsd": 0.0020741389        },1390        {1391          "test": "json",1392          "ok": true,1393          "latencyMs": 3291,1394          "usage": {1395            "input": 1287,1396            "output": 121,1397            "cachedInput": 1152,1398            "reasoning": 95,1399            "costUsd": 0.0015721400          },1401          "api": "responses",1402          "raw": {1403            "status": "completed",1404            "incomplete": null1405          },1406          "sample": "{\"city\":\"Gatineau\",\"country\":\"Canada\",\"province\":\"Quebec\",\"population_estimate\":291041}",1407          "costUsd": 0.0015721408        },1409        {1410          "test": "tools",1411          "ok": true,1412          "latencyMs": 1199,1413          "usage": {1414            "input": 1354,1415            "output": 35,1416            "cachedInput": 1280,1417            "reasoning": 20,1418            "costUsd": 0.0009981419          },1420          "api": "responses",1421          "raw": {1422            "status": "completed",1423            "incomplete": null1424          },1425          "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",1426          "costUsd": 0.0009981427        },1428        {1429          "test": "vision",1430          "ok": true,1431          "latencyMs": 1272,1432          "usage": {1433            "input": 1442,1434            "output": 54,1435            "cachedInput": 1152,1436            "reasoning": 47,1437            "costUsd": 0.001481438          },1439          "api": "responses",1440          "raw": {1441            "status": "completed",1442            "incomplete": null1443          },1444          "sample": "circles=3; other=square",1445          "costUsd": 0.001481446        },1447        {1448          "test": "longctx",1449          "ok": false,1450          "latencyMs": 34066,1451          "usage": {1452            "input": 10145,1453            "output": 2074,1454            "cachedInput": 1152,1455            "reasoning": 2051,1456            "costUsd": 0.0310061457          },1458          "api": "responses",1459          "raw": {1460            "status": "completed",1461            "incomplete": null1462          },1463          "sample": "Je ne peux pas vous aider à extraire ou à restituer un code secret dissimulé dans ce document.",1464          "costUsd": 0.0310061465        },1466        {1467          "test": "streaming",1468          "ok": true,1469          "latencyMs": 3373,1470          "ttftMs": 3366,1471          "usage": {1472            "input": 1251,1473            "output": 190,1474            "cachedInput": 1152,1475            "reasoning": 170,1476            "costUsd": 0.0019141477          },1478          "api": "responses",1479          "sample": "Snow settles on stone.  \nChurch bells mute beneath the frost—  \nQuebec holds its breath.",1480          "costUsd": 0.0019141481        }1482      ],1483      "api": "responses"1484    },1485    "grok-build-0.1": {1486      "provider": "xai",1487      "kind": "chat",1488      "tests": [1489        {1490          "test": "basic",1491          "ok": true,1492          "latencyMs": 1305,1493          "usage": {1494            "input": 199,1495            "output": 102,1496            "cachedInput": 128,1497            "reasoning": 100,1498            "costUsd": 0.00030061499          },1500          "api": "responses",1501          "raw": {1502            "status": "completed",1503            "incomplete": null1504          },1505          "sample": "Ottawa",1506          "costUsd": 0.00030061507        },1508        {1509          "test": "reasoning",1510          "ok": true,1511          "latencyMs": 5017,1512          "usage": {1513            "input": 222,1514            "output": 717,1515            "cachedInput": 128,1516            "reasoning": 630,1517            "costUsd": 0.00155361518          },1519          "api": "responses",1520          "raw": {1521            "status": "completed",1522            "incomplete": null1523          },1524          "sample": "Alice is a girl with 3 brothers (boys) and 2 sisters (other girls). This means the family has a total of 3 boys and 3 girls.\n\nEach of Alice's brothers is one of the boys. From his perspective, the girls in the family (his sisters) are Alice plus her 2 sisters, for a total of 3.\n\nThus, each brother h",1525          "costUsd": 0.00155361526        },1527        {1528          "test": "math",1529          "ok": true,1530          "latencyMs": 2564,1531          "usage": {1532            "input": 230,1533            "output": 374,1534            "cachedInput": 128,1535            "reasoning": 367,1536            "costUsd": 0.00087561537          },1538          "api": "responses",1539          "raw": {1540            "status": "completed",1541            "incomplete": null1542          },1543          "sample": "a=403;b=2500",1544          "costUsd": 0.00087561545        },1546        {1547          "test": "code",1548          "ok": true,1549          "latencyMs": 2821,1550          "usage": {1551            "input": 229,1552            "output": 552,1553            "cachedInput": 128,1554            "reasoning": 465,1555            "costUsd": 0.00123061556          },1557          "api": "responses",1558          "raw": {1559            "status": "completed",1560            "incomplete": null1561          },1562          "sample": "```js\nfunction isPrime(n) {\n  if (n <= 1) return false;\n  if (n === 2) return true;\n  if (n % 2 === 0) return false;\n  for (let i = 3; i * i <= n; i += 2) {\n    if (n % i === 0) return false;\n  }\n  return true;\n}\n```",1563          "costUsd": 0.00123061564        },1565        {1566          "test": "french",1567          "ok": true,1568          "latencyMs": 3913,1569          "usage": {1570            "input": 211,1571            "output": 743,1572            "cachedInput": 128,1573            "reasoning": 665,1574            "costUsd": 0.00159461575          },1576          "api": "responses",1577          "raw": {1578            "status": "completed",1579            "incomplete": null1580          },1581          "sample": "La méthode du coût en évaluation immobilière consiste à estimer la valeur d'un bien en calculant le coût de reconstruction ou de remplacement du bâtiment, déduction faite de la dépréciation accumulée, puis en ajoutant la valeur du terrain. \n\nElle s'applique principalement aux immeubles neufs, spécia",1582          "costUsd": 0.00159461583        },1584        {1585          "test": "english",1586          "ok": true,1587          "latencyMs": 3997,1588          "usage": {1589            "input": 200,1590            "output": 473,1591            "cachedInput": 128,1592            "reasoning": 406,1593            "costUsd": 0.00104361594          },1595          "api": "responses",1596          "raw": {1597            "status": "completed",1598            "incomplete": null1599          },1600          "sample": "A hedonic pricing model is an econometric technique that estimates the value of a good or service by decomposing its price into the implicit values of its individual attributes or characteristics. It assumes that the price of a heterogeneous product, such as a house, reflects the sum of the marginal",1601          "costUsd": 0.00104361602        },1603        {1604          "test": "json",1605          "ok": true,1606          "latencyMs": 10696,1607          "usage": {1608            "input": 231,1609            "output": 879,1610            "cachedInput": 128,1611            "reasoning": 848,1612            "costUsd": 0.00188661613          },1614          "api": "responses",1615          "raw": {1616            "status": "completed",1617            "incomplete": null1618          },1619          "sample": "{\n\"city\": \"Gatineau\",\n\"country\": \"Canada\",\n\"province\": \"Quebec\",\n\"population_estimate\": 291041\n}",1620          "costUsd": 0.00188661621        },1622        {1623          "test": "tools",1624          "ok": true,1625          "latencyMs": 1755,1626          "usage": {1627            "input": 297,1628            "output": 237,1629            "cachedInput": 128,1630            "reasoning": 223,1631            "costUsd": 0.00066861632          },1633          "api": "responses",1634          "raw": {1635            "status": "completed",1636            "incomplete": null1637          },1638          "sample": "{\"name\":\"get_weather\",\"arguments\":\"{\\\"location\\\":\\\"Gatineau, Quebec\\\"}\"}",1639          "costUsd": 0.00066861640        },1641        {1642          "test": "vision",1643          "ok": true,1644          "latencyMs": 2403,1645          "usage": {1646            "input": 379,1647            "output": 380,1648            "cachedInput": 128,1649            "reasoning": 373,1650            "costUsd": 0.00103661651          },1652          "api": "responses",1653          "raw": {1654            "status": "completed",1655            "incomplete": null1656          },1657          "sample": "circles=3; other=square",1658          "costUsd": 0.00103661659        },1660        {1661          "test": "longctx",1662          "ok": true,1663          "latencyMs": 3572,1664          "usage": {1665            "input": 9163,1666            "output": 651,1667            "cachedInput": 128,1668            "reasoning": 647,1669            "costUsd": 0.01036261670          },1671          "api": "responses",1672          "raw": {1673            "status": "completed",1674            "incomplete": null1675          },1676          "sample": "7431-ALPHA",1677          "costUsd": 0.01036261678        },1679        {1680          "test": "streaming",1681          "ok": true,1682          "latencyMs": 9441,1683          "ttftMs": 9172,1684          "usage": {1685            "input": 195,1686            "output": 1828,1687            "cachedInput": 128,1688            "reasoning": 1810,1689            "costUsd": 0.00374861690          },1691          "api": "responses",1692          "sample": "Snow blankets Quebec,  \nMaple forests dream in white—  \nWinter holds its breath.",1693          "costUsd": 0.00374861694        }1695      ],1696      "api": "responses"1697    }1698  }1699}
Files main
Menu

Theme