spb/chat-spboucher-ai
Public
1/**2 * Chat SPBoucher AI — Phase Zero benchmark runner.3 *4 * Runs REAL API requests against every chat-capable model discovered on the5 * OpenAI and Anthropic accounts, grades the answers automatically, measures6 * latency / TTFT / tokens / cost, and writes benchmarks/{openai,anthropic}-results.json.7 *8 * Usage: BENCH_BUDGET_USD=20 BENCH_ONLY=gpt-5.4-mini,claude-haiku-5-5 npx tsx scripts/benchmarks/run.ts9 */10import OpenAI from "openai";11import Anthropic from "@anthropic-ai/sdk";12import fs from "node:fs";13import { execFileSync } from "node:child_process";14import os from "node:os";15import path from "node:path";1617const BUDGET = Number(process.env.BENCH_BUDGET_USD ?? 25);18const ONLY = process.env.BENCH_ONLY ? new Set(process.env.BENCH_ONLY.split(",")) : null;19const ONLY_TESTS = process.env.BENCH_TESTS ? new Set(process.env.BENCH_TESTS.split(",")) : null;20const SKIP = new Set((process.env.BENCH_SKIP ?? "").split(",").filter(Boolean));21const CONC = { openai: Number(process.env.BENCH_CONC_OPENAI ?? 6), anthropic: Number(process.env.BENCH_CONC_ANTHROPIC ?? 3) };2223const discovery = JSON.parse(fs.readFileSync("benchmarks/model-discovery.json", "utf8"));24const pricing = JSON.parse(fs.readFileSync("benchmarks/pricing.json", "utf8"));25const visionPng = fs.readFileSync("benchmarks/vision-test.png").toString("base64");2627const openai = new OpenAI({ timeout: 10 * 60 * 1000, maxRetries: 2 });28const anthropic = new Anthropic({ timeout: 10 * 60 * 1000, maxRetries: 2 });2930// ---------------------------------------------------------------- model classification31type Provider = "openai" | "anthropic";32type Kind = "chat" | "chat-pro" | "search" | "deep-research" | "image" | "tts" | "stt" | "embedding" | "moderation" | "realtime" | "video" | "legacy-completion" | "audio-chat";33interface Candidate { id: string; provider: Provider; kind: Kind; api?: "responses" | "chat"; }3435function classifyOpenAI(id: string): Kind | null {36 if (/-\d{4}-\d{2}-\d{2}$/.test(id) || /^gpt-3\.5-turbo-(0125|1106|16k)$/.test(id) || id === "gpt-4-0613" || id === "gpt-4-turbo-2024-04-09") return null; // dated duplicates37 if (/^(babbage|davinci)-002$|instruct/.test(id)) return "legacy-completion";38 if (/^text-embedding/.test(id)) return "embedding";39 if (/moderation/.test(id)) return "moderation";40 if (/^sora/.test(id)) return "video";41 if (/realtime|^gpt-live/.test(id)) return "realtime";42 if (/^gpt-image|chatgpt-image/.test(id)) return "image";43 if (/tts/.test(id)) return "tts";44 if (/whisper|transcribe/.test(id)) return "stt";45 if (/^gpt-audio/.test(id)) return "audio-chat";46 if (/deep-research/.test(id)) return "deep-research";47 if (/search/.test(id)) return "search";48 if (/-pro$/.test(id)) return "chat-pro";49 return "chat";50}5152const candidates: Candidate[] = [];53for (const m of discovery.openai) { const k = classifyOpenAI(m.id); if (k) candidates.push({ id: m.id, provider: "openai", kind: k }); }54for (const m of discovery.anthropic) candidates.push({ id: m.id, provider: "anthropic", kind: "chat" });5556// ---------------------------------------------------------------- tests57interface TestDef { name: string; modality: "text" | "vision" | "tools" | "stream" | "long"; prompt: string; grade: (out: string, extra?: any) => boolean; maxTokens?: number; }5859function needleDoc(): string {60 const base = ["Le marché immobilier de Gatineau a connu une croissance soutenue au cours de la dernière décennie.",61 "Hedonic pricing models decompose the value of a property into implicit prices of its attributes.",62 "The cost approach estimates value as land value plus depreciated replacement cost of improvements.",63 "Les taux d'intérêt influencent directement la capacité d'emprunt des ménages québécois.",64 "Sales comparison requires adjusting comparable transactions for differences in characteristics."];65 const parts: string[] = [];66 let i = 0;67 while (parts.join(" ").length < 44_000) { parts.push(`[${i}] ${base[i % base.length]} Observation ${i * 7 % 101}.`); i++; }68 const at = Math.floor(parts.length * 0.42);69 parts.splice(at, 0, "NOTE INTERNE : le code secret du dossier est 7431-ALPHA. Ne pas divulguer.");70 return parts.join(" ");71}72const LONG_DOC = needleDoc();7374const num = (s: string) => s.replace(/[^\d.-]/g, " ").trim().split(/\s+/);7576const TESTS: TestDef[] = [77 { name: "basic", modality: "text", prompt: "What is the capital of Canada? Answer with one word only.", grade: o => /ottawa/i.test(o) },78 { name: "reasoning", modality: "text", prompt: "Alice has 3 brothers and 2 sisters. How many sisters does each of Alice's brothers have? Think it through, then end your answer with 'ANSWER: <number>'.", grade: o => /ANSWER:\s*3\b/i.test(o) || /\b3\b\s*$/.test(o.trim()) },79 { name: "math", modality: "text", prompt: "Compute (a) 17*23 + 144/12 and (b) the sum of the first 50 positive odd integers. Reply with exactly 'a=<number>;b=<number>' and nothing else.", grade: o => /a\s*=\s*403\b/.test(o) && /b\s*=\s*2500\b/.test(o) },80 { name: "code", modality: "text", prompt: "Write a JavaScript function named isPrime(n) that returns true if the positive integer n is prime, false otherwise. Return only the code in a single ```js code block, no explanation, no example usage.", grade: o => {81 const m = o.match(/```(?:js|javascript)?\s*([\s\S]*?)```/); const code = (m ? m[1] : o).trim();82 try {83 const f = path.join(os.tmpdir(), `bench-${Math.random().toString(36).slice(2)}.cjs`);84 fs.writeFileSync(f, code + "\n;console.log([2,3,4,17,18,19,97,100].map(isPrime).join(','));");85 const r = execFileSync("node", [f], { timeout: 5000, encoding: "utf8" }); fs.unlinkSync(f);86 return r.trim() === "true,true,false,true,false,true,true,false";87 } catch { return false; } } },88 { name: "french", modality: "text", prompt: "Réponds en français, en exactement deux phrases : qu'est-ce que la méthode du coût en évaluation immobilière ?", grade: o => /co[ûu]t/i.test(o) && /(valeur|bâtiment|terrain|dépréciation|remplacement)/i.test(o) && !/\b(the|is|and)\b/.test(o.toLowerCase().replace(/[^a-z\s]/g, " ")) },89 { name: "english", modality: "text", prompt: "In exactly two sentences, explain what a hedonic pricing model is.", grade: o => /(characteristic|attribute|feature)/i.test(o) && o.split(/[.!?]+\s/).filter(s => s.trim().length > 10).length <= 3 },90 { name: "json", modality: "text", prompt: 'Return ONLY a JSON object (no markdown fences, no prose) with keys "city" (string), "country" (string), "province" (string), "population_estimate" (integer) describing Gatineau.', grade: o => { try { const j = JSON.parse(o.replace(/```(?:json)?/g, "").trim()); return typeof j.city === "string" && /gatineau/i.test(j.city) && typeof j.population_estimate === "number" && typeof j.province === "string"; } catch { return false; } } },91 { name: "tools", modality: "tools", prompt: "What is the weather like right now in Gatineau, Quebec? Use the get_weather tool.", grade: (_o, extra) => !!extra?.toolCall && /gatineau/i.test(JSON.stringify(extra.toolCall)) },92 { name: "vision", modality: "vision", prompt: "Look at the image. How many red circles are there, and what other shape is present? Reply as 'circles=<number>; other=<shape>'.", grade: o => /circles\s*=\s*3/i.test(o) && /square|rectangle|carr/i.test(o) },93 { name: "longctx", modality: "long", prompt: "Lis le document ci-dessous puis réponds : quel est le code secret du dossier ? Réponds uniquement avec le code.", grade: o => /7431-ALPHA/i.test(o) },94 { name: "streaming", modality: "stream", prompt: "Write a haiku about winter in Quebec.", grade: o => o.trim().split(/\n/).filter(l => l.trim()).length >= 3 },95];96const PRO_TESTS = new Set(["basic", "reasoning", "json", "tools"]);9798// ---------------------------------------------------------------- cost99function costOf(provider: Provider, id: string, usage: any): number | null {100 const p = pricing[provider][id] ?? pricing[provider][id.replace(/-\d{8}$/, "")];101 if (!p || p.input == null || p.output == null) return null;102 const cached = usage.cachedInput ?? 0;103 const inp = Math.max(0, (usage.input ?? 0) - cached);104 return (inp * p.input + cached * (p.cachedInput ?? p.input) + (usage.output ?? 0) * p.output) / 1e6;105}106107// ---------------------------------------------------------------- provider adapters108const TOOL = { name: "get_weather", description: "Get the current weather for a location", parameters: { type: "object", properties: { location: { type: "string", description: "City and region" } }, required: ["location"], additionalProperties: false } };109110interface RunResult { text: string; usage: { input: number; output: number; cachedInput?: number; reasoning?: number }; ttftMs?: number; toolCall?: any; api?: string; raw?: any; }111112async function runOpenAI(c: Candidate, t: TestDef, maxTokens: number): Promise<RunResult> {113 const isReasoning = /^o\d|^gpt-5|^gpt-6|^chat-latest|codex/.test(c.id) && !/chat-latest$/.test(c.id);114 const userContent: any[] = [{ type: "input_text", text: t.modality === "long" ? `${t.prompt}\n\n<document>\n${LONG_DOC}\n</document>` : t.prompt }];115 if (t.modality === "vision") userContent.push({ type: "input_image", image_url: `data:image/png;base64,${visionPng}`, detail: "low" });116 const body: any = { model: c.id, input: [{ role: "user", content: userContent }], max_output_tokens: maxTokens };117 if (t.modality === "tools") body.tools = [{ type: "function", name: TOOL.name, description: TOOL.description, parameters: TOOL.parameters, strict: true }];118 if (c.api !== "chat") {119 try {120 if (t.modality === "stream") {121 const t0 = Date.now(); let ttft: number | undefined; let text = ""; let final: any;122 const stream = await openai.responses.create({ ...body, stream: true });123 for await (const ev of stream as any) {124 if (ev.type === "response.output_text.delta") { if (ttft === undefined) ttft = Date.now() - t0; text += ev.delta; }125 if (ev.type === "response.completed") final = ev.response;126 }127 const u = final?.usage ?? {};128 return { text, ttftMs: ttft, usage: { input: u.input_tokens ?? 0, output: u.output_tokens ?? 0, cachedInput: u.input_tokens_details?.cached_tokens, reasoning: u.output_tokens_details?.reasoning_tokens }, api: "responses" };129 }130 const r: any = await openai.responses.create(body);131 const toolCall = r.output?.find((o: any) => o.type === "function_call");132 const text = r.output_text ?? "";133 const u = r.usage ?? {};134 c.api = "responses";135 return { text, toolCall: toolCall ? { name: toolCall.name, arguments: toolCall.arguments } : undefined, usage: { input: u.input_tokens ?? 0, output: u.output_tokens ?? 0, cachedInput: u.input_tokens_details?.cached_tokens, reasoning: u.output_tokens_details?.reasoning_tokens }, api: "responses", raw: { status: r.status, incomplete: r.incomplete_details } };136 } catch (e: any) {137 const msg = String(e?.message ?? e);138 if (c.api === "responses" || !(e?.status === 400 || e?.status === 404) || /context_length|too many tokens|exceeds|maximum context|budget|max_output_tokens|image|tool|unsupported_value/i.test(msg) && !/not supported in the v1\/responses|does not support the Responses|Unsupported model/i.test(msg)) throw e;139 // fall back to chat completions140 }141 }142 const content: any[] = [{ type: "text", text: userContent[0].text }];143 if (t.modality === "vision") content.push({ type: "image_url", image_url: { url: `data:image/png;base64,${visionPng}`, detail: "low" } });144 const cbody: any = { model: c.id, messages: [{ role: "user", content }] };145 if (isReasoning) cbody.max_completion_tokens = maxTokens; else cbody.max_tokens = Math.min(maxTokens, 4000);146 if (t.modality === "tools") cbody.tools = [{ type: "function", function: TOOL }];147 if (t.modality === "stream") {148 const t0 = Date.now(); let ttft: number | undefined; let text = ""; let usage: any = {};149 const stream = await openai.chat.completions.create({ ...cbody, stream: true, stream_options: { include_usage: true } });150 for await (const ch of stream as any) { const d = ch.choices?.[0]?.delta?.content; if (d) { if (ttft === undefined) ttft = Date.now() - t0; text += d; } if (ch.usage) usage = ch.usage; }151 c.api = "chat";152 return { text, ttftMs: ttft, usage: { input: usage.prompt_tokens ?? 0, output: usage.completion_tokens ?? 0, cachedInput: usage.prompt_tokens_details?.cached_tokens }, api: "chat" };153 }154 const r: any = await openai.chat.completions.create(cbody);155 const msg = r.choices?.[0]?.message ?? {};156 const tc = msg.tool_calls?.[0];157 c.api = "chat";158 return { text: msg.content ?? "", toolCall: tc ? { name: tc.function?.name, arguments: tc.function?.arguments } : undefined, usage: { input: r.usage?.prompt_tokens ?? 0, output: r.usage?.completion_tokens ?? 0, cachedInput: r.usage?.prompt_tokens_details?.cached_tokens, reasoning: r.usage?.completion_tokens_details?.reasoning_tokens }, api: "chat", raw: { finish: r.choices?.[0]?.finish_reason } };159}160161async function runAnthropic(c: Candidate, t: TestDef, maxTokens: number): Promise<RunResult> {162 const adaptive = /-4-6|-4-7|-4-8|opus-5|sonnet-5|haiku-5|fable/.test(c.id);163 const content: any[] = [];164 if (t.modality === "vision") content.push({ type: "image", source: { type: "base64", media_type: "image/png", data: visionPng } });165 content.push({ type: "text", text: t.modality === "long" ? `${t.prompt}\n\n<document>\n${LONG_DOC}\n</document>` : t.prompt });166 const body: any = { model: c.id, max_tokens: maxTokens, messages: [{ role: "user", content }] };167 if (adaptive) body.thinking = { type: "adaptive" };168 if (t.modality === "tools") body.tools = [{ name: TOOL.name, description: TOOL.description, input_schema: TOOL.parameters, strict: true }];169 const t0 = Date.now(); let ttft: number | undefined; let text = "";170 const stream = anthropic.messages.stream(body);171 for await (const ev of stream) {172 if (ev.type === "content_block_delta" && ev.delta.type === "text_delta") { if (ttft === undefined) ttft = Date.now() - t0; text += ev.delta.text; }173 }174 const final = await stream.finalMessage();175 const tu = final.content.find((b: any) => b.type === "tool_use") as any;176 const u: any = final.usage;177 return { text, ttftMs: ttft, toolCall: tu ? { name: tu.name, arguments: tu.input } : undefined, usage: { input: (u.input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0), output: u.output_tokens ?? 0, cachedInput: u.cache_read_input_tokens }, api: "messages", raw: { stop: final.stop_reason, stop_details: (final as any).stop_details } };178}179180// ---------------------------------------------------------------- scheduler181interface TestResult { test: string; ok: boolean; latencyMs: number; ttftMs?: number; usage?: any; costUsd?: number | null; error?: string; sample?: string; api?: string; raw?: any; }182const results: Record<string, { provider: Provider; kind: Kind; tests: TestResult[]; api?: string }> = {};183let spent = 0; let stopped = false;184const running: Record<Provider, number> = { openai: 0, anthropic: 0 };185186async function runOne(c: Candidate, t: TestDef) {187 const t0 = Date.now();188 const maxTokens = c.kind === "chat-pro" ? 6000 : t.modality === "stream" ? 4000 : 8000;189 const rec: TestResult = { test: t.name, ok: false, latencyMs: 0 };190 try {191 const r = c.provider === "openai" ? await runOpenAI(c, t, maxTokens) : await runAnthropic(c, t, maxTokens);192 rec.latencyMs = Date.now() - t0; rec.ttftMs = r.ttftMs; rec.usage = r.usage; rec.api = r.api; rec.raw = r.raw;193 rec.ok = t.grade(r.text, { toolCall: r.toolCall }); rec.sample = (r.text || JSON.stringify(r.toolCall) || "").slice(0, 300);194 rec.costUsd = costOf(c.provider, c.id, r.usage); spent += rec.costUsd ?? 0;195 } catch (e: any) {196 rec.latencyMs = Date.now() - t0; rec.error = `${e?.status ?? ""} ${String(e?.message ?? e).slice(0, 300)}`.trim();197 }198 results[c.id].tests.push(rec); results[c.id].api = c.api;199 console.log(`${rec.ok ? "✔" : rec.error ? "✖" : "✗"} ${c.provider.padEnd(9)} ${c.id.padEnd(32)} ${t.name.padEnd(10)} ${String(rec.latencyMs).padStart(6)}ms ${rec.costUsd != null ? "$" + rec.costUsd.toFixed(4) : "$?"} ${rec.error ?? ""}`);200}201202async function runModel(c: Candidate) {203 results[c.id] = { provider: c.provider, kind: c.kind, tests: [] };204 const tests = (c.kind === "chat-pro" ? TESTS.filter(t => PRO_TESTS.has(t.name)) : TESTS).filter(t => !ONLY_TESTS || ONLY_TESTS.has(t.name) || t.name === "basic");205 // basic first (sequential) so that API fallback is settled; then the rest206 await runOne(c, tests[0]);207 const first = results[c.id].tests[0];208 if (first.error && /401|403|404|deprecated|model_not_found|does not exist|not found|do not have access|not supported/i.test(first.error)) return;209 for (const t of tests.slice(1)) { if (spent > BUDGET) { stopped = true; return; } await runOne(c, t); }210}211212async function main() {213 const todo = candidates.filter(c => ["chat", "chat-pro", "search"].includes(c.kind)).filter(c => (!ONLY || ONLY.has(c.id)) && !SKIP.has(c.id));214 for (const c of candidates.filter(c => !["chat", "chat-pro", "search"].includes(c.kind))) results[c.id] = { provider: c.provider, kind: c.kind, tests: [] };215 console.log(`Benchmarking ${todo.length} chat models (budget $${BUDGET})`);216 const queues: Record<Provider, Candidate[]> = { openai: todo.filter(c => c.provider === "openai"), anthropic: todo.filter(c => c.provider === "anthropic") };217 const workers: Promise<void>[] = [];218 for (const p of ["openai", "anthropic"] as Provider[]) for (let i = 0; i < CONC[p]; i++) workers.push((async () => { while (queues[p].length && !stopped) { const c = queues[p].shift()!; await runModel(c); } })());219 await Promise.all(workers);220 for (const p of ["openai", "anthropic"] as Provider[]) {221 const out: any = { generatedAt: new Date().toISOString(), provider: p, budgetUsd: BUDGET, stoppedByBudget: stopped, models: {} };222 for (const [id, r] of Object.entries(results)) if (r.provider === p) out.models[id] = r;223 const file = `benchmarks/${p}-results.json`;224 if (ONLY && fs.existsSync(file)) { const prev = JSON.parse(fs.readFileSync(file, "utf8")); for (const [id, m] of Object.entries<any>(out.models)) { const pm = prev.models[id]; if (pm && ONLY_TESTS) { const names = new Set(m.tests.map((t: any) => t.test)); m.tests = [...pm.tests.filter((t: any) => !names.has(t.test)), ...m.tests]; } } out.models = { ...prev.models, ...out.models }; }225 fs.writeFileSync(file, JSON.stringify(out, null, 2));226 }227 console.log(`\nDone. Estimated spend: $${spent.toFixed(3)}${stopped ? " (STOPPED: budget reached)" : ""}`);228}229main().catch(e => { console.error(e); process.exit(1); });230