spb/note-spboucher-ai

1/**2 * Phase Zero §5.3 — real inference benchmarks.3 * Usage: npx tsx benchmarks/scripts/bench.ts [--models a,b] [--prompts x,y] [--trials N] [--concurrency N]4 * Writes benchmarks/results/raw/bench-runs.jsonl (one line per trial) — resumable (skips done trials).5 */6import fs from "node:fs";7import path from "node:path";8import { ROOT, RESULTS, RAW, appendJSONL, estimateCost, pLimit } from "./lib.ts";9import { call } from "./providers.ts";10import { gradeRegex, gradeCode, gradeJSON, gradeTool, gradeWritingDeterministic, gradeMath, llmJudge, type Grade } from "./graders.ts";1112const args = Object.fromEntries(process.argv.slice(2).map((a, i, arr) => a.startsWith("--") ? [a.slice(2), arr[i + 1]] : []).filter((x) => x.length));13const trials = Number(args.trials ?? 2);14const concurrency = Number(args.concurrency ?? 8);1516const discovery = JSON.parse(fs.readFileSync(path.join(RESULTS, "discovery.json"), "utf8"));17const EXPENSIVE = /-pro$|^o1|^o3-pro/;18let models: { provider: "openai" | "anthropic"; id: string }[] = discovery.models19  .filter((m: any) => m.verdict === "tested" && !EXPENSIVE.test(m.id))20  .map((m: any) => ({ provider: m.provider, id: m.id }));21if (args.models) { const want = args.models.split(","); models = models.filter((m) => want.includes(m.id)); }2223const promptFiles = fs.readdirSync(path.join(ROOT, "benchmarks/prompts")).filter((f) => f.endsWith(".json"));24let prompts = promptFiles.map((f) => JSON.parse(fs.readFileSync(path.join(ROOT, "benchmarks/prompts", f), "utf8")));25if (args.prompts) { const want = args.prompts.split(","); prompts = prompts.filter((p) => want.includes(p.id)); }26const haystack = fs.readFileSync(path.join(ROOT, "benchmarks/prompts/long-context.txt"), "utf8");2728const RUNS = path.join(RAW, "bench-runs.jsonl");29const done = new Set<string>();30if (fs.existsSync(RUNS)) for (const l of fs.readFileSync(RUNS, "utf8").split("\n").filter(Boolean)) { const j = JSON.parse(l); done.add(`${j.model}|${j.prompt}|${j.trial}`); }3132const RUBRICS: Record<string, string> = {33  french_writing: "Quality of formal academic French (grammar, register, fluency), relevance, respect of all constraints (length 150-220 words, the three required expressions present, no title/list, no English words).",34  english_writing: "Quality of formal academic English (grammar, register, fluency), relevance, respect of all constraints (length 150-220 words, the three required expressions, no title/list).",35  knowledge_synthesis: "Accuracy to the three sources, explicit identification of the 41 vs 58 tokens/s contradiction and which is more recent (S2), correct inline citations [S1][S2][S3], concise (120-200 words), no invented facts.",36};3738async function runOne(m: { provider: "openai" | "anthropic"; id: string }, p: any, trial: number) {39  const key = `${m.id}|${p.id}|${trial}`;40  if (done.has(key)) return;41  const prompt = p.haystack ? p.prompt.replace("{{HAYSTACK}}", haystack) : p.prompt;42  const r = await call(m.provider, { model: m.id, prompt, maxTokens: p.max_tokens ?? 4000, jsonSchema: p.schema, tools: p.tools, stream: true, timeoutMs: 300_000 });43  let grade: Grade = { pass: false, score: 0, detail: r.error ? `request failed: ${r.error}` : "ungraded" };44  if (r.ok) {45    switch (p.grader) {46      case "regex": grade = gradeRegex(r.text, p.expect); break;47      case "code": grade = gradeCode(r.text); break;48      case "math": grade = gradeMath(r.text); break;49      case "json": grade = gradeJSON(r.text, p.schema.schema, p.expect_json); break;50      case "tool": grade = gradeTool(r.toolCalls, p.expect_tool); break;51      case "writing": case "synthesis": {52        grade = gradeWritingDeterministic(r.text, p.lang ?? "en", p.must_contain, p.words);53        if (p.grader === "synthesis") { // language check is not meaningful here; keep terms+length only54          const terms = p.must_contain.filter((t: string) => r.text.includes(t)).length / p.must_contain.length;55          const wc = r.text.trim().split(/\s+/).length; const inRange = wc >= p.words[0] && wc <= p.words[1] + 30;56          grade = { pass: terms === 1 && inRange, score: (terms + (inRange ? 1 : 0)) / 2, detail: `terms ${Math.round(terms * p.must_contain.length)}/${p.must_contain.length}; words ${wc}` };57        }58        try { grade.judged = await llmJudge(p.prompt, r.text, RUBRICS[p.id]); } catch (e: any) { grade.detail += `; judge error ${e?.message}`; }59        break;60      }61    }62  }63  const u = r.usage;64  const rec = {65    ts: new Date().toISOString(), provider: m.provider, model: m.id, prompt: p.id, category: p.category, trial,66    ok: r.ok, error: r.error, status: r.status, stop: r.stop,67    ttft_ms: r.ttft_ms === null ? null : Math.round(r.ttft_ms), total_ms: Math.round(r.total_ms),68    input_tokens: u?.input_tokens ?? null, output_tokens: u?.output_tokens ?? null, reasoning_tokens: u?.reasoning_tokens ?? null, cached_tokens: u?.cached_tokens ?? null,69    usage_source: u ? "provider" : null,70    output_tps: u && r.ttft_ms !== null && r.total_ms > r.ttft_ms ? +(Math.max(0, u.output_tokens - (u.reasoning_tokens ?? 0)) / ((r.total_ms - r.ttft_ms) / 1000)).toFixed(1) : null, output_tps_note: "visible (non-reasoning) output tokens ÷ (total − TTFT)",71    cost_usd: u ? estimateCost(m.id, u.input_tokens, u.output_tokens, u.cached_tokens) : null, cost_source: "estimated from provider usage × published list price",72    pass: grade.pass, score: +grade.score.toFixed(3), grade_detail: grade.detail, judged: grade.judged ?? null,73    response_excerpt: r.text.slice(0, 1200), tool_calls: r.toolCalls,74  };75  appendJSONL(RUNS, rec);76  console.log(`${m.id.padEnd(28)} ${p.id.padEnd(20)} t${trial} ${r.ok ? (grade.pass ? "PASS" : "fail") : "ERR "} score=${rec.score} ttft=${rec.ttft_ms}ms total=${rec.total_ms}ms out=${rec.output_tokens} $${rec.cost_usd?.toFixed(4) ?? "?"} ${r.error ? r.error.slice(0, 80) : ""}`);77}7879const tasks: (() => Promise<void>)[] = [];80for (const m of models) for (const p of prompts) for (let t = 1; t <= trials; t++) tasks.push(() => runOne(m, p, t));81console.log(`${models.length} models × ${prompts.length} prompts × ${trials} trials = ${tasks.length} runs (${done.size} already done)`);82await pLimit(concurrency, tasks);83console.log("bench complete");84