spb/note-spboucher-ai

1/** Deterministic graders + LLM judge for writing tasks. */2import vm from "node:vm";3import { call } from "./providers.ts";45export interface Grade { pass: boolean; score: number; detail: string; judged?: { judge: string; score: number; rationale: string }[] }67export function gradeRegex(text: string, expect: string[]): Grade {8  const hits = expect.map((re) => new RegExp(re, "i").test(text));9  const n = hits.filter(Boolean).length;10  return { pass: n === expect.length, score: n / expect.length, detail: `${n}/${expect.length} expected patterns matched` };11}1213export function gradeCode(text: string): Grade {14  const m = text.match(/```(?:javascript|js)?\s*([\s\S]*?)```/);15  const code = m ? m[1] : text;16  const tests: [string[], number, string[]][] = [17    [["i", "love", "leetcode", "i", "love", "coding"], 2, ["i", "love"]],18    [["the", "day", "is", "sunny", "the", "the", "the", "sunny", "is", "is"], 4, ["the", "is", "sunny", "day"]],19    [["b", "a", "c", "a", "b"], 3, ["a", "b", "c"]],20    [["x"], 1, ["x"]],21  ];22  let passed = 0; const errs: string[] = [];23  for (const [words, k, want] of tests) {24    try {25      const ctx = vm.createContext({});26      vm.runInContext(code + `\n;globalThis.__out = topKFrequent(${JSON.stringify(words)}, ${k});`, ctx, { timeout: 2000 });27      const got = (ctx as any).__out;28      if (JSON.stringify(got) === JSON.stringify(want)) passed++; else errs.push(`got ${JSON.stringify(got)} want ${JSON.stringify(want)}`);29    } catch (e: any) { errs.push(String(e?.message).slice(0, 80)); }30  }31  return { pass: passed === tests.length, score: passed / tests.length, detail: errs.length ? errs.join(" | ").slice(0, 300) : "all tests passed" };32}3334function validateSchema(val: any, schema: any, path = "$"): string[] {35  const errs: string[] = [];36  if (!schema) return errs;37  const t = schema.type;38  if (t === "object") {39    if (typeof val !== "object" || val === null || Array.isArray(val)) return [`${path}: not object`];40    for (const r of schema.required ?? []) if (!(r in val)) errs.push(`${path}.${r}: missing`);41    for (const [k, v] of Object.entries(val)) {42      if (schema.properties?.[k]) errs.push(...validateSchema(v, schema.properties[k], `${path}.${k}`));43      else if (schema.additionalProperties === false) errs.push(`${path}.${k}: unexpected`);44    }45  } else if (t === "array") {46    if (!Array.isArray(val)) return [`${path}: not array`];47    val.forEach((x, i) => errs.push(...validateSchema(x, schema.items, `${path}[${i}]`)));48  } else if (t === "string") { if (typeof val !== "string") errs.push(`${path}: not string`); if (schema.enum && !schema.enum.includes(val)) errs.push(`${path}: not in enum`); }49  else if (t === "integer") { if (!Number.isInteger(val)) errs.push(`${path}: not integer`); }50  else if (t === "number") { if (typeof val !== "number") errs.push(`${path}: not number`); }51  return errs;52}5354export function gradeJSON(text: string, schema: any, expect: Record<string, any>): Grade {55  let j: any;56  const cleaned = text.trim().replace(/^```(?:json)?\s*/i, "").replace(/```\s*$/, "");57  try { j = JSON.parse(cleaned); } catch { return { pass: false, score: 0, detail: "invalid JSON" }; }58  const errs = validateSchema(j, schema);59  let checks = 0, ok = 0;60  for (const [k, v] of Object.entries(expect)) {61    checks++;62    if (k === "authors_count") { if (Array.isArray(j.authors) && j.authors.length === v) ok++; continue; }63    const got = j[k];64    if (Array.isArray(v)) { if (JSON.stringify((got ?? []).map(String).sort()) === JSON.stringify(v.map(String).sort())) ok++; }65    else if (typeof v === "number") { if (typeof got === "number" && Math.abs(got - v) < 0.01) ok++; }66    else if (String(got).trim() === String(v)) ok++;67  }68  const schemaOk = errs.length === 0;69  const score = (schemaOk ? 0.3 : 0) + 0.7 * (ok / Math.max(1, checks));70  return { pass: schemaOk && ok === checks, score, detail: `${schemaOk ? "schema ok" : "schema errors: " + errs.slice(0, 3).join(", ")}; fields ${ok}/${checks}` };71}7273export function gradeTool(toolCalls: { name: string; args: any }[], expect: { name: string; args: Record<string, string>; forbid: string[] }): Grade {74  if (toolCalls.some((c) => expect.forbid.includes(c.name))) return { pass: false, score: 0, detail: "forbidden tool called" };75  const c = toolCalls.find((x) => x.name === expect.name);76  if (!c) return { pass: false, score: 0, detail: "expected tool not called" };77  const keys = Object.keys(expect.args);78  const ok = keys.filter((k) => new RegExp(expect.args[k], "i").test(String(c.args?.[k] ?? ""))).length;79  return { pass: ok === keys.length, score: 0.4 + 0.6 * (ok / keys.length), detail: `tool called; ${ok}/${keys.length} args correct` };80}8182const FR_STOP = ["le", "la", "les", "des", "une", "est", "pour", "dans", "que", "qui", "et", "du", "aux", "sont", "en"];83const EN_STOP = ["the", "and", "of", "to", "for", "that", "with", "are", "is", "in", "which", "this", "as", "by", "on"];84function langRatio(text: string, lang: "fr" | "en") {85  const words = text.toLowerCase().match(/[a-zàâçéèêëîïôûùüÿœ'-]+/g) ?? [];86  const stop = new Set(lang === "fr" ? FR_STOP : EN_STOP);87  const other = new Set(lang === "fr" ? EN_STOP : FR_STOP);88  let s = 0, o = 0; for (const w of words) { if (stop.has(w)) s++; if (other.has(w)) o++; }89  return { words: words.length, stopHits: s, otherHits: o };90}9192export function gradeWritingDeterministic(text: string, lang: "fr" | "en", mustContain: string[], words: [number, number]): Grade {93  const lr = langRatio(text, lang);94  const contains = mustContain.filter((m) => text.toLowerCase().includes(m.toLowerCase())).length;95  const wc = text.trim().split(/\s+/).length;96  const inRange = wc >= words[0] - 10 && wc <= words[1] + 25;97  const langOk = lr.stopHits >= 5 && lr.otherHits <= Math.max(2, lr.stopHits * 0.15);98  const noList = !/^\s*[-*•]\s|^\s*\d+\.\s/m.test(text);99  const parts = [contains / mustContain.length, inRange ? 1 : 0, langOk ? 1 : 0, noList ? 1 : 0];100  const score = parts.reduce((a, b) => a + b, 0) / parts.length;101  return { pass: score === 1, score, detail: `terms ${contains}/${mustContain.length}; words ${wc} (${inRange ? "ok" : "out of range"}); language ${langOk ? "ok" : "suspect"}; ${noList ? "no list" : "list detected"}` };102}103104const JUDGES: { provider: "openai" | "anthropic"; model: string }[] = [105  { provider: "anthropic", model: "claude-sonnet-5-5" },106  { provider: "openai", model: "gpt-5.4" },107];108const JUDGE_SCHEMA = { type: "object", properties: { score: { type: "integer", minimum: 1, maximum: 5 }, rationale: { type: "string" } }, required: ["score", "rationale"], additionalProperties: false };109110/** Two-judge LLM rubric (1-5). Clearly labelled as LLM-judged in reports. */111export async function llmJudge(task: string, response: string, rubric: string) {112  const prompt = `You are grading a model response for a benchmark.\n\nTASK GIVEN TO THE MODEL:\n<task>\n${task}\n</task>\n\nMODEL RESPONSE:\n<response>\n${response}\n</response>\n\nRUBRIC: ${rubric}\nScore 1 (poor) to 5 (excellent). Be strict about constraint violations. Return JSON {"score": int, "rationale": string (<= 40 words)}.`;113  const out: { judge: string; score: number; rationale: string }[] = [];114  for (const j of JUDGES) {115    const r = await call(j.provider, { model: j.model, prompt, maxTokens: 2000, jsonSchema: { name: "grade", schema: JUDGE_SCHEMA }, stream: false, reasoning: "low", timeoutMs: 120_000 });116    try { const g = JSON.parse(r.text); out.push({ judge: j.model, score: Number(g.score), rationale: String(g.rationale) }); } catch { out.push({ judge: j.model, score: NaN, rationale: `judge failed: ${r.error ?? r.text.slice(0, 80)}` }); }117  }118  return out;119}120121/** Mathematics task: A = 233168 ; B = x^2 e^{2x} (2x+3) in any order/notation. */122export function gradeMath(text: string): Grade {123  const a = /233[,\s]?168/.test(text);124  const bLine = (text.match(/B:[^\n]*/g) ?? []).pop() ?? text.slice(-300);125  const n = bLine.replace(/²/g, "^2").replace(/\\[()\[\]]/g, "").replace(/\\(cdot|times|left|right)/g, "").replace(/[\s{}$*·]/g, "").replace(/\\?/g, "");126  const hasX2 = /x\^2/.test(n);127  const hasExp = /e\^\(?2x\)?/.test(n);128  const hasFactor = /\(?(2x\+3|3\+2x)\)?/.test(n);129  const b = hasX2 && hasExp && hasFactor;130  return { pass: a && b, score: ((a ? 1 : 0) + (b ? 1 : 0)) / 2, detail: `A ${a ? "ok" : "wrong"}; B ${b ? "ok" : "wrong"} (${bLine.slice(0, 60)})` };131}132