spb/chat-spboucher-ai

1/**2 * Builds capability-matrix.json, benchmark-report.md and routing-recommendations.md3 * from the raw results produced by run.ts / modalities.ts. No numbers are invented:4 * every figure comes from a recorded API response.5 */6import fs from "node:fs";78const discovery = JSON.parse(fs.readFileSync("benchmarks/model-discovery.json", "utf8"));9const pricing = JSON.parse(fs.readFileSync("benchmarks/pricing.json", "utf8"));10const oa = JSON.parse(fs.readFileSync("benchmarks/openai-results.json", "utf8"));11const an = JSON.parse(fs.readFileSync("benchmarks/anthropic-results.json", "utf8"));12const mod = fs.existsSync("benchmarks/modality-results.json") ? JSON.parse(fs.readFileSync("benchmarks/modality-results.json", "utf8")).results : {};1314const QUALITY_TESTS = ["reasoning", "math", "code", "french", "english", "json", "tools", "vision", "longctx"];15const TEXT_TESTS = ["basic", "reasoning", "math", "code", "french", "english", "json"];1617type Cap = true | false | "unknown";18interface Entry {19  id: string; provider: "openai" | "anthropic"; kind: string; api?: string;20  available: boolean; tested: boolean; status: "verified" | "unavailable" | "unsupported" | "untested";21  unavailableReason?: string;22  capabilities: Record<string, Cap>;23  contextWindow?: number; maxOutputTokens?: number;24  pricing?: any;25  benchmark?: { tests: number; passed: number; qualityScore: number | null; p50LatencyMs: number | null; p50TtftMs: number | null; totalCostUsd: number; avgOutputTokens: number | null; perTest: Record<string, "pass" | "fail" | "error">; errors: string[] };26  lastVerifiedAt: string;27}2829const median = (xs: number[]) => { if (!xs.length) return null; const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length / 2)]; };30const now = new Date().toISOString();31const entries: Entry[] = [];3233function priceFor(provider: string, id: string) { return pricing[provider][id] ?? pricing[provider][id.replace(/-\d{8}$/, "")] ?? undefined; }3435for (const [provider, res] of [["openai", oa], ["anthropic", an]] as const) {36  for (const [id, r] of Object.entries<any>(res.models)) {37    const e: Entry = { id, provider, kind: r.kind, api: r.api, available: false, tested: false, status: "untested", capabilities: {}, pricing: priceFor(provider, id), lastVerifiedAt: now };38    const disc = discovery[provider].find((m: any) => m.id === id);39    if (provider === "anthropic" && disc) { e.contextWindow = disc.max_input_tokens; e.maxOutputTokens = disc.max_tokens; }40    const tests: any[] = r.tests ?? [];41    const by: Record<string, any> = Object.fromEntries(tests.map(t => [t.test, t]));42    const m = mod[id];43    if (["chat", "chat-pro", "search"].includes(r.kind)) {44      if (!tests.length) { e.status = "untested"; }45      else if (tests.length === 1 && tests[0].error && /404|deprecated|not found|does not exist|401|403/i.test(tests[0].error)) {46        e.status = "unavailable"; e.unavailableReason = tests[0].error.replace(/^\d+\s+\d+\s+/, "").slice(0, 160);47      } else {48        e.available = true; e.tested = true; e.status = "verified";49        const cap = (t: string): Cap => by[t] ? (by[t].ok ? true : by[t].error ? (/not support|unsupported|invalid|image|vision|tool/i.test(by[t].error) ? false : "unknown") : false) : "unknown";50        e.capabilities = {51          text: by.basic?.ok ?? "unknown", vision: cap("vision"), tools: cap("tools"), structuredOutput: cap("json"),52          streaming: by.streaming ? (by.streaming.error ? false : true) : "unknown", reasoning: cap("reasoning"), longContext: cap("longctx"),53          audioInput: false, audioOutput: false, imageGeneration: false,54        };55        const q = QUALITY_TESTS.filter(t => by[t]);56        const passed = tests.filter(t => t.ok).length;57        const lat = tests.filter(t => TEXT_TESTS.includes(t.test) && !t.error).map(t => t.latencyMs);58        const ttft = tests.filter(t => t.ttftMs != null).map(t => t.ttftMs);59        const outs = tests.filter(t => t.usage?.output).map(t => t.usage.output);60        e.benchmark = {61          tests: tests.length, passed, qualityScore: q.length ? Math.round(100 * q.filter(t => by[t].ok).length / q.length) : null,62          p50LatencyMs: median(lat), p50TtftMs: median(ttft), totalCostUsd: tests.reduce((s, t) => s + (t.costUsd ?? 0), 0),63          avgOutputTokens: outs.length ? Math.round(outs.reduce((a, b) => a + b, 0) / outs.length) : null,64          perTest: Object.fromEntries(tests.map(t => [t.test, t.ok ? "pass" : t.error ? "error" : "fail"])),65          errors: [...new Set(tests.filter(t => t.error).map(t => `${t.test}: ${t.error.slice(0, 140)}`))],66        };67      }68    } else if (m) {69      e.tested = true; e.available = m.ok; e.status = m.ok ? "verified" : "unavailable"; if (!m.ok) e.unavailableReason = m.error;70      e.capabilities = { text: false, vision: false, tools: false, structuredOutput: false, streaming: false, reasoning: false, longContext: false,71        imageGeneration: m.kind === "image" ? m.ok : false, audioOutput: m.kind === "tts" ? m.ok : false, audioInput: m.kind === "stt" ? m.ok : m.kind === "audio-chat" ? "unknown" : false };72      e.benchmark = { tests: 1, passed: m.ok ? 1 : 0, qualityScore: null, p50LatencyMs: m.latencyMs, p50TtftMs: null, totalCostUsd: 0, avgOutputTokens: null, perTest: { [m.kind]: m.ok ? "pass" : "error" }, errors: m.error ? [m.error] : [] };73      if (m.kind === "stt" && m.correct === false) e.benchmark.perTest.stt = "fail";74    } else {75      e.status = ["realtime", "video", "legacy-completion", "deep-research"].includes(r.kind) ? "unsupported" : "untested";76      e.unavailableReason = r.kind === "realtime" ? "WebSocket/WebRTC realtime model — not a chat-completion model; not exercised" : r.kind === "video" ? "Video generation — not tested (cost)" : r.kind === "deep-research" ? "Requires hosted web-search tools and long runs — not benchmarked without explicit budget approval" : r.kind === "legacy-completion" ? "Legacy completions endpoint — not used by this app" : "not tested";77    }78    entries.push(e);79  }80}81entries.sort((a, b) => a.provider.localeCompare(b.provider) || (b.benchmark?.qualityScore ?? -1) - (a.benchmark?.qualityScore ?? -1) || a.id.localeCompare(b.id));82fs.writeFileSync("benchmarks/capability-matrix.json", JSON.stringify({ generatedAt: now, entries }, null, 2));8384// ------------------------------------------------------------------ report85const chat = entries.filter(e => e.status === "verified" && e.benchmark?.qualityScore != null);86const fmt$ = (p: any) => p && p.input != null && p.output != null ? `$${p.input} / $${p.output}` : "—";87const blended = (p: any) => p && p.input != null && p.output != null ? (3 * p.input + p.output) / 4 : null;88let md = `# Chat SPBoucher AI — Benchmark report\n\nGenerated ${now}. Every figure below comes from a real API response recorded in \`openai-results.json\`, \`anthropic-results.json\` or \`modality-results.json\`. Prices come from the official pricing pages (see \`pricing.json\`). Nothing is estimated from memory.\n\n`;89md += `## Summary\n\n| | OpenAI | Anthropic |\n|---|---|---|\n`;90for (const [label, f] of [["Models listed by the API", (p: string) => discovery[p].length], ["Chat models benchmarked (verified)", (p: string) => chat.filter(e => e.provider === p).length], ["Listed but unavailable (404/deprecated)", (p: string) => entries.filter(e => e.provider === p && e.status === "unavailable").length], ["Specialized models verified (image/tts/stt/embedding/moderation)", (p: string) => entries.filter(e => e.provider === p && e.status === "verified" && !["chat", "chat-pro", "search"].includes(e.kind)).length], ["Not exercised (realtime/video/legacy/deep-research)", (p: string) => entries.filter(e => e.provider === p && (e.status === "unsupported" || e.status === "untested")).length]] as const) md += `| ${label} | ${f("openai")} | ${f("anthropic")} |\n`;91md += `\nTotal benchmark spend (estimated from usage × verified prices): **$${(oa.models ? Object.values<any>(oa.models).concat(Object.values<any>(an.models)).flatMap(m => m.tests).reduce((s: number, t: any) => s + (t.costUsd ?? 0), 0) : 0).toFixed(2)}**.\n\n`;92md += `## Test suite\n\n| Test | What is measured | Grading |\n|---|---|---|\n| basic | capital of Canada | contains "Ottawa" |\n| reasoning | sisters/brothers riddle | final answer 3 |\n| math | 17·23+144/12 and Σ first 50 odd | a=403;b=2500 |\n| code | isPrime(n) in JS | code executed with node against 8 cases |\n| french | 2 phrases sur la méthode du coût | vocabulaire FR, aucun mot anglais courant |\n| english | hedonic pricing in 2 sentences | key terms, ≤3 sentences |\n| json | strict JSON object about Gatineau | JSON.parse + typed keys |\n| tools | get_weather function call | tool call emitted with location=Gatineau |\n| vision | PNG with 3 red circles + blue square | circles=3 and square named |\n| longctx | ~12k-token document, needle at 42% | code 7431-ALPHA returned |\n| streaming | haiku, streamed | time-to-first-token measured |\n\nSettings: provider defaults (OpenAI Responses API; Anthropic adaptive thinking where supported), max 8 000 output tokens (6 000 for *-pro models, which ran a reduced 4-test suite), one run per test. Single-shot results are indicative, not statistically robust.\n\n`;9394md += `## Verified chat models — ranked by quality score\n\nQuality score = share of the 9 graded tests passed (reasoning, math, code, french, english, json, tools, vision, longctx). Latency = median over text tests. Prices in $/1M tokens (input / output).\n\n| Model | Provider | Quality | p50 latency | p50 TTFT | Price in/out | Blended $/1M* | Failures / errors |\n|---|---|---|---|---|---|---|---|\n`;95for (const e of [...chat].sort((a, b) => (b.benchmark!.qualityScore! - a.benchmark!.qualityScore!) || (a.benchmark!.p50LatencyMs ?? 1e9) - (b.benchmark!.p50LatencyMs ?? 1e9))) {96  const b = e.benchmark!;97  const fails = Object.entries(b.perTest).filter(([, v]) => v !== "pass").map(([k, v]) => `${k}${v === "error" ? "⚠" : ""}`).join(", ") || "—";98  const bl = blended(e.pricing);99  md += `| ${e.id} | ${e.provider} | ${b.qualityScore}% | ${b.p50LatencyMs ?? "—"} ms | ${b.p50TtftMs ?? "—"} ms | ${fmt$(e.pricing)} | ${bl != null ? "$" + bl.toFixed(2) : "—"} | ${fails} |\n`;100}101md += `\n\\* Blended = (3 × input + output) / 4, a simple comparison heuristic, not a billing figure.\n\n`;102103md += `## Per-test results\n\n| Model | ${QUALITY_TESTS.concat(["basic", "streaming"]).join(" | ")} |\n|---|${QUALITY_TESTS.concat(["basic", "streaming"]).map(() => "---").join("|")}|\n`;104const sym = (v?: string) => v === "pass" ? "✅" : v === "fail" ? "❌" : v === "error" ? "⚠️" : "·";105for (const e of chat) md += `| ${e.id} | ${QUALITY_TESTS.concat(["basic", "streaming"]).map(t => sym(e.benchmark!.perTest[t])).join(" | ")} |\n`;106107md += `\n## Listed but unavailable\n\nThese ids are returned by the provider's model listing but reject inference. The app marks them **unavailable** and hides them by default.\n\n| Model | Provider | Reason |\n|---|---|---|\n`;108for (const e of entries.filter(e => e.status === "unavailable")) md += `| ${e.id} | ${e.provider} | ${e.unavailableReason ?? ""} |\n`;109110md += `\n## Specialized models (modality tests)\n\n| Model | Kind | Result | Latency | Notes |\n|---|---|---|---|---|\n`;111for (const e of entries.filter(e => !["chat", "chat-pro", "search"].includes(e.kind) && e.tested)) { const m = mod[e.id] ?? {}; md += `| ${e.id} | ${e.kind} | ${e.available ? "✅ verified" : "❌ " + (e.unavailableReason ?? "")} | ${m.latencyMs ?? "—"} ms | ${m.text ? `"${String(m.text).slice(0, 60)}"` : m.dims ? `${m.dims} dims` : m.usage ? `tokens ${JSON.stringify(m.usage).slice(0, 80)}` : ""} |\n`; }112113md += `\n## Not exercised\n\n| Model | Kind | Why |\n|---|---|---|\n`;114for (const e of entries.filter(e => e.status === "unsupported" || e.status === "untested")) md += `| ${e.id} | ${e.kind} | ${e.unavailableReason ?? ""} |\n`;115116md += `\n## Errors encountered on verified models\n\n`;117for (const e of chat.filter(e => e.benchmark!.errors.length)) md += `- **${e.id}**: ${e.benchmark!.errors.join("; ")}\n`;118md += `\n## Limitations\n\n- One run per test: latency and pass/fail are single observations.\n- Reasoning models spend hidden reasoning tokens; their latency and cost reflect default effort.\n- Pricing was read from the public pricing pages on ${pricing._meta.verifiedAt}; models missing from those pages have no cost estimate.\n- The long-context test uses ~12k tokens, far below the advertised windows; it checks retrieval, not window size.\n`;119fs.writeFileSync("benchmarks/benchmark-report.md", md);120121// ------------------------------------------------------------------ routing recommendations122const withPrice = chat.filter(e => blended(e.pricing) != null);123// Quality: highest score; ties (the suite saturates for frontier models) broken by HIGHER list price — the vendor's124// higher tier is assumed stronger on tasks harder than this suite — then by lower latency. *-pro models excluded (latency).125const byQ = [...chat].sort((a, b) => (b.benchmark!.qualityScore! - a.benchmark!.qualityScore!) || ((blended(b.pricing) ?? -1) - (blended(a.pricing) ?? -1)) || ((a.benchmark!.p50LatencyMs ?? 1e9) - (b.benchmark!.p50LatencyMs ?? 1e9)));126const quality = byQ.filter(e => e.kind !== "chat-pro").slice(0, 3);127const fast = [...chat].filter(e => e.benchmark!.qualityScore! >= 89 && e.benchmark!.p50LatencyMs != null).sort((a, b) => a.benchmark!.p50LatencyMs! - b.benchmark!.p50LatencyMs!).slice(0, 3);128const economy = withPrice.filter(e => e.benchmark!.qualityScore! >= 100).sort((a, b) => blended(a.pricing)! - blended(b.pricing)!).slice(0, 3);129const best = (t: string) => [...chat].filter(e => e.benchmark!.perTest[t] === "pass").sort((a, b) => (b.benchmark!.qualityScore! - a.benchmark!.qualityScore!) || ((blended(a.pricing) ?? 1e9) - (blended(b.pricing) ?? 1e9)) || ((a.benchmark!.p50LatencyMs ?? 1e9) - (b.benchmark!.p50LatencyMs ?? 1e9))).slice(0, 3);130const rec = {131  generatedAt: now,132  modes: { quality: quality.map(e => e.id), fast: fast.map(e => e.id), economy: economy.map(e => e.id) },133  byTask: Object.fromEntries(["code", "math", "reasoning", "french", "vision", "tools", "json", "longctx"].map(t => [t, best(t).map(e => e.id)])),134  specialized: {135    imageGeneration: entries.filter(e => e.kind === "image" && e.available).map(e => e.id),136    tts: entries.filter(e => e.kind === "tts" && e.available).map(e => e.id),137    stt: entries.filter(e => e.kind === "stt" && e.available).map(e => e.id),138    embedding: entries.filter(e => e.kind === "embedding" && e.available).map(e => e.id),139  },140};141fs.writeFileSync("benchmarks/routing-recommendations.json", JSON.stringify(rec, null, 2));142let rm = `# Routing recommendations\n\nDerived mechanically from \`capability-matrix.json\` (${now}). The in-app router uses the same data at runtime; the owner can override any choice.\n\n`;143const line = (es: Entry[]) => es.map(e => `\`${e.id}\` (${e.benchmark!.qualityScore}%, ${e.benchmark!.p50LatencyMs} ms, ${fmt$(e.pricing)})`).join(", ");144rm += `## Modes\n\n- **Quality** — highest quality score; ties broken by higher list price (assumption: the vendor's higher tier is stronger on tasks harder than this suite), then latency; *-pro models excluded (latency): ${line(quality)}\n- **Fast** — lowest median latency among models scoring ≥ 89 %: ${line(fast)}\n- **Economy** — cheapest blended price among models scoring 100 %: ${line(economy)}\n- **Automatic** — classify the task (code / math / vision / long document / general / French), pick the first candidate of the matching task list below that is available, fall back to the Quality list.\n\n## By task\n\n`;145for (const [t, ids] of Object.entries(rec.byTask)) rm += `- **${t}**: ${(ids as string[]).map(i => `\`${i}\``).join(", ") || "none passed"}\n`;146rm += `\n## Specialized\n\n- Image generation: ${rec.specialized.imageGeneration.map(i => `\`${i}\``).join(", ")}\n- Text-to-speech: ${rec.specialized.tts.map(i => `\`${i}\``).join(", ")}\n- Speech-to-text: ${rec.specialized.stt.map(i => `\`${i}\``).join(", ")}\n- Embeddings (memory & semantic search): ${rec.specialized.embedding.map(i => `\`${i}\``).join(", ")}\n\n## Defaults proposed for the app\n\n- Default chat model: \`${quality[0]?.id ?? "n/a"}\`\n- Fast mode: \`${fast[0]?.id ?? "n/a"}\`\n- Economy mode: \`${economy[0]?.id ?? "n/a"}\`\n- Title generation / memory extraction / classification (cheap, fast): \`${economy[0]?.id ?? "n/a"}\`\n- Embeddings: \`text-embedding-3-small\` (1536 dims, verified)\n- Image generation: \`${rec.specialized.imageGeneration.includes("gpt-image-1.5") ? "gpt-image-1.5" : rec.specialized.imageGeneration[0] ?? "n/a"}\`\n- STT: \`gpt-4o-mini-transcribe\` · TTS: \`gpt-4o-mini-tts\`\n`;147fs.writeFileSync("benchmarks/routing-recommendations.md", rm);148console.log(`entries=${entries.length} verified chat=${chat.length} unavailable=${entries.filter(e => e.status === "unavailable").length}`);149console.log("quality:", rec.modes.quality, "fast:", rec.modes.fast, "economy:", rec.modes.economy);150