/**
 * Chat SPBoucher AI — Phase Zero benchmark runner.
 *
 * Runs REAL API requests against every chat-capable model discovered on the
 * OpenAI and Anthropic accounts, grades the answers automatically, measures
 * latency / TTFT / tokens / cost, and writes benchmarks/{openai,anthropic}-results.json.
 *
 * Usage:  BENCH_BUDGET_USD=20 BENCH_ONLY=gpt-5.4-mini,claude-haiku-5-5 npx tsx scripts/benchmarks/run.ts
 */
import OpenAI from "openai";
import Anthropic from "@anthropic-ai/sdk";
import fs from "node:fs";
import { execFileSync } from "node:child_process";
import os from "node:os";
import path from "node:path";

const BUDGET = Number(process.env.BENCH_BUDGET_USD ?? 25);
const ONLY = process.env.BENCH_ONLY ? new Set(process.env.BENCH_ONLY.split(",")) : null;
const ONLY_TESTS = process.env.BENCH_TESTS ? new Set(process.env.BENCH_TESTS.split(",")) : null;
const SKIP = new Set((process.env.BENCH_SKIP ?? "").split(",").filter(Boolean));
const CONC = { openai: Number(process.env.BENCH_CONC_OPENAI ?? 6), anthropic: Number(process.env.BENCH_CONC_ANTHROPIC ?? 3) };

const discovery = JSON.parse(fs.readFileSync("benchmarks/model-discovery.json", "utf8"));
const pricing = JSON.parse(fs.readFileSync("benchmarks/pricing.json", "utf8"));
const visionPng = fs.readFileSync("benchmarks/vision-test.png").toString("base64");

const openai = new OpenAI({ timeout: 10 * 60 * 1000, maxRetries: 2 });
const anthropic = new Anthropic({ timeout: 10 * 60 * 1000, maxRetries: 2 });

// ---------------------------------------------------------------- model classification
type Provider = "openai" | "anthropic";
type Kind = "chat" | "chat-pro" | "search" | "deep-research" | "image" | "tts" | "stt" | "embedding" | "moderation" | "realtime" | "video" | "legacy-completion" | "audio-chat";
interface Candidate { id: string; provider: Provider; kind: Kind; api?: "responses" | "chat"; }

function classifyOpenAI(id: string): Kind | null {
  if (/-\d{4}-\d{2}-\d{2}$/.test(id) || /^gpt-3\.5-turbo-(0125|1106|16k)$/.test(id) || id === "gpt-4-0613" || id === "gpt-4-turbo-2024-04-09") return null; // dated duplicates
  if (/^(babbage|davinci)-002$|instruct/.test(id)) return "legacy-completion";
  if (/^text-embedding/.test(id)) return "embedding";
  if (/moderation/.test(id)) return "moderation";
  if (/^sora/.test(id)) return "video";
  if (/realtime|^gpt-live/.test(id)) return "realtime";
  if (/^gpt-image|chatgpt-image/.test(id)) return "image";
  if (/tts/.test(id)) return "tts";
  if (/whisper|transcribe/.test(id)) return "stt";
  if (/^gpt-audio/.test(id)) return "audio-chat";
  if (/deep-research/.test(id)) return "deep-research";
  if (/search/.test(id)) return "search";
  if (/-pro$/.test(id)) return "chat-pro";
  return "chat";
}

const candidates: Candidate[] = [];
for (const m of discovery.openai) { const k = classifyOpenAI(m.id); if (k) candidates.push({ id: m.id, provider: "openai", kind: k }); }
for (const m of discovery.anthropic) candidates.push({ id: m.id, provider: "anthropic", kind: "chat" });

// ---------------------------------------------------------------- tests
interface TestDef { name: string; modality: "text" | "vision" | "tools" | "stream" | "long"; prompt: string; grade: (out: string, extra?: any) => boolean; maxTokens?: number; }

function needleDoc(): string {
  const base = ["Le marché immobilier de Gatineau a connu une croissance soutenue au cours de la dernière décennie.",
    "Hedonic pricing models decompose the value of a property into implicit prices of its attributes.",
    "The cost approach estimates value as land value plus depreciated replacement cost of improvements.",
    "Les taux d'intérêt influencent directement la capacité d'emprunt des ménages québécois.",
    "Sales comparison requires adjusting comparable transactions for differences in characteristics."];
  const parts: string[] = [];
  let i = 0;
  while (parts.join(" ").length < 44_000) { parts.push(`[${i}] ${base[i % base.length]} Observation ${i * 7 % 101}.`); i++; }
  const at = Math.floor(parts.length * 0.42);
  parts.splice(at, 0, "NOTE INTERNE : le code secret du dossier est 7431-ALPHA. Ne pas divulguer.");
  return parts.join(" ");
}
const LONG_DOC = needleDoc();

const num = (s: string) => s.replace(/[^\d.-]/g, " ").trim().split(/\s+/);

const TESTS: TestDef[] = [
  { name: "basic", modality: "text", prompt: "What is the capital of Canada? Answer with one word only.", grade: o => /ottawa/i.test(o) },
  { name: "reasoning", modality: "text", prompt: "Alice has 3 brothers and 2 sisters. How many sisters does each of Alice's brothers have? Think it through, then end your answer with 'ANSWER: <number>'.", grade: o => /ANSWER:\s*3\b/i.test(o) || /\b3\b\s*$/.test(o.trim()) },
  { name: "math", modality: "text", prompt: "Compute (a) 17*23 + 144/12 and (b) the sum of the first 50 positive odd integers. Reply with exactly 'a=<number>;b=<number>' and nothing else.", grade: o => /a\s*=\s*403\b/.test(o) && /b\s*=\s*2500\b/.test(o) },
  { name: "code", modality: "text", prompt: "Write a JavaScript function named isPrime(n) that returns true if the positive integer n is prime, false otherwise. Return only the code in a single ```js code block, no explanation, no example usage.", grade: o => {
      const m = o.match(/```(?:js|javascript)?\s*([\s\S]*?)```/); const code = (m ? m[1] : o).trim();
      try {
        const f = path.join(os.tmpdir(), `bench-${Math.random().toString(36).slice(2)}.cjs`);
        fs.writeFileSync(f, code + "\n;console.log([2,3,4,17,18,19,97,100].map(isPrime).join(','));");
        const r = execFileSync("node", [f], { timeout: 5000, encoding: "utf8" }); fs.unlinkSync(f);
        return r.trim() === "true,true,false,true,false,true,true,false";
      } catch { return false; } } },
  { name: "french", modality: "text", prompt: "Réponds en français, en exactement deux phrases : qu'est-ce que la méthode du coût en évaluation immobilière ?", grade: o => /co[ûu]t/i.test(o) && /(valeur|bâtiment|terrain|dépréciation|remplacement)/i.test(o) && !/\b(the|is|and)\b/.test(o.toLowerCase().replace(/[^a-z\s]/g, " ")) },
  { name: "english", modality: "text", prompt: "In exactly two sentences, explain what a hedonic pricing model is.", grade: o => /(characteristic|attribute|feature)/i.test(o) && o.split(/[.!?]+\s/).filter(s => s.trim().length > 10).length <= 3 },
  { name: "json", modality: "text", prompt: 'Return ONLY a JSON object (no markdown fences, no prose) with keys "city" (string), "country" (string), "province" (string), "population_estimate" (integer) describing Gatineau.', grade: o => { try { const j = JSON.parse(o.replace(/```(?:json)?/g, "").trim()); return typeof j.city === "string" && /gatineau/i.test(j.city) && typeof j.population_estimate === "number" && typeof j.province === "string"; } catch { return false; } } },
  { name: "tools", modality: "tools", prompt: "What is the weather like right now in Gatineau, Quebec? Use the get_weather tool.", grade: (_o, extra) => !!extra?.toolCall && /gatineau/i.test(JSON.stringify(extra.toolCall)) },
  { name: "vision", modality: "vision", prompt: "Look at the image. How many red circles are there, and what other shape is present? Reply as 'circles=<number>; other=<shape>'.", grade: o => /circles\s*=\s*3/i.test(o) && /square|rectangle|carr/i.test(o) },
  { name: "longctx", modality: "long", prompt: "Lis le document ci-dessous puis réponds : quel est le code secret du dossier ? Réponds uniquement avec le code.", grade: o => /7431-ALPHA/i.test(o) },
  { name: "streaming", modality: "stream", prompt: "Write a haiku about winter in Quebec.", grade: o => o.trim().split(/\n/).filter(l => l.trim()).length >= 3 },
];
const PRO_TESTS = new Set(["basic", "reasoning", "json", "tools"]);

// ---------------------------------------------------------------- cost
function costOf(provider: Provider, id: string, usage: any): number | null {
  const p = pricing[provider][id] ?? pricing[provider][id.replace(/-\d{8}$/, "")];
  if (!p || p.input == null || p.output == null) return null;
  const cached = usage.cachedInput ?? 0;
  const inp = Math.max(0, (usage.input ?? 0) - cached);
  return (inp * p.input + cached * (p.cachedInput ?? p.input) + (usage.output ?? 0) * p.output) / 1e6;
}

// ---------------------------------------------------------------- provider adapters
const TOOL = { name: "get_weather", description: "Get the current weather for a location", parameters: { type: "object", properties: { location: { type: "string", description: "City and region" } }, required: ["location"], additionalProperties: false } };

interface RunResult { text: string; usage: { input: number; output: number; cachedInput?: number; reasoning?: number }; ttftMs?: number; toolCall?: any; api?: string; raw?: any; }

async function runOpenAI(c: Candidate, t: TestDef, maxTokens: number): Promise<RunResult> {
  const isReasoning = /^o\d|^gpt-5|^gpt-6|^chat-latest|codex/.test(c.id) && !/chat-latest$/.test(c.id);
  const userContent: any[] = [{ type: "input_text", text: t.modality === "long" ? `${t.prompt}\n\n<document>\n${LONG_DOC}\n</document>` : t.prompt }];
  if (t.modality === "vision") userContent.push({ type: "input_image", image_url: `data:image/png;base64,${visionPng}`, detail: "low" });
  const body: any = { model: c.id, input: [{ role: "user", content: userContent }], max_output_tokens: maxTokens };
  if (t.modality === "tools") body.tools = [{ type: "function", name: TOOL.name, description: TOOL.description, parameters: TOOL.parameters, strict: true }];
  if (c.api !== "chat") {
    try {
      if (t.modality === "stream") {
        const t0 = Date.now(); let ttft: number | undefined; let text = ""; let final: any;
        const stream = await openai.responses.create({ ...body, stream: true });
        for await (const ev of stream as any) {
          if (ev.type === "response.output_text.delta") { if (ttft === undefined) ttft = Date.now() - t0; text += ev.delta; }
          if (ev.type === "response.completed") final = ev.response;
        }
        const u = final?.usage ?? {};
        return { text, ttftMs: ttft, usage: { input: u.input_tokens ?? 0, output: u.output_tokens ?? 0, cachedInput: u.input_tokens_details?.cached_tokens, reasoning: u.output_tokens_details?.reasoning_tokens }, api: "responses" };
      }
      const r: any = await openai.responses.create(body);
      const toolCall = r.output?.find((o: any) => o.type === "function_call");
      const text = r.output_text ?? "";
      const u = r.usage ?? {};
      c.api = "responses";
      return { text, toolCall: toolCall ? { name: toolCall.name, arguments: toolCall.arguments } : undefined, usage: { input: u.input_tokens ?? 0, output: u.output_tokens ?? 0, cachedInput: u.input_tokens_details?.cached_tokens, reasoning: u.output_tokens_details?.reasoning_tokens }, api: "responses", raw: { status: r.status, incomplete: r.incomplete_details } };
    } catch (e: any) {
      const msg = String(e?.message ?? e);
      if (c.api === "responses" || !(e?.status === 400 || e?.status === 404) || /context_length|too many tokens|exceeds|maximum context|budget|max_output_tokens|image|tool|unsupported_value/i.test(msg) && !/not supported in the v1\/responses|does not support the Responses|Unsupported model/i.test(msg)) throw e;
      // fall back to chat completions
    }
  }
  const content: any[] = [{ type: "text", text: userContent[0].text }];
  if (t.modality === "vision") content.push({ type: "image_url", image_url: { url: `data:image/png;base64,${visionPng}`, detail: "low" } });
  const cbody: any = { model: c.id, messages: [{ role: "user", content }] };
  if (isReasoning) cbody.max_completion_tokens = maxTokens; else cbody.max_tokens = Math.min(maxTokens, 4000);
  if (t.modality === "tools") cbody.tools = [{ type: "function", function: TOOL }];
  if (t.modality === "stream") {
    const t0 = Date.now(); let ttft: number | undefined; let text = ""; let usage: any = {};
    const stream = await openai.chat.completions.create({ ...cbody, stream: true, stream_options: { include_usage: true } });
    for await (const ch of stream as any) { const d = ch.choices?.[0]?.delta?.content; if (d) { if (ttft === undefined) ttft = Date.now() - t0; text += d; } if (ch.usage) usage = ch.usage; }
    c.api = "chat";
    return { text, ttftMs: ttft, usage: { input: usage.prompt_tokens ?? 0, output: usage.completion_tokens ?? 0, cachedInput: usage.prompt_tokens_details?.cached_tokens }, api: "chat" };
  }
  const r: any = await openai.chat.completions.create(cbody);
  const msg = r.choices?.[0]?.message ?? {};
  const tc = msg.tool_calls?.[0];
  c.api = "chat";
  return { text: msg.content ?? "", toolCall: tc ? { name: tc.function?.name, arguments: tc.function?.arguments } : undefined, usage: { input: r.usage?.prompt_tokens ?? 0, output: r.usage?.completion_tokens ?? 0, cachedInput: r.usage?.prompt_tokens_details?.cached_tokens, reasoning: r.usage?.completion_tokens_details?.reasoning_tokens }, api: "chat", raw: { finish: r.choices?.[0]?.finish_reason } };
}

async function runAnthropic(c: Candidate, t: TestDef, maxTokens: number): Promise<RunResult> {
  const adaptive = /-4-6|-4-7|-4-8|opus-5|sonnet-5|haiku-5|fable/.test(c.id);
  const content: any[] = [];
  if (t.modality === "vision") content.push({ type: "image", source: { type: "base64", media_type: "image/png", data: visionPng } });
  content.push({ type: "text", text: t.modality === "long" ? `${t.prompt}\n\n<document>\n${LONG_DOC}\n</document>` : t.prompt });
  const body: any = { model: c.id, max_tokens: maxTokens, messages: [{ role: "user", content }] };
  if (adaptive) body.thinking = { type: "adaptive" };
  if (t.modality === "tools") body.tools = [{ name: TOOL.name, description: TOOL.description, input_schema: TOOL.parameters, strict: true }];
  const t0 = Date.now(); let ttft: number | undefined; let text = "";
  const stream = anthropic.messages.stream(body);
  for await (const ev of stream) {
    if (ev.type === "content_block_delta" && ev.delta.type === "text_delta") { if (ttft === undefined) ttft = Date.now() - t0; text += ev.delta.text; }
  }
  const final = await stream.finalMessage();
  const tu = final.content.find((b: any) => b.type === "tool_use") as any;
  const u: any = final.usage;
  return { text, ttftMs: ttft, toolCall: tu ? { name: tu.name, arguments: tu.input } : undefined, usage: { input: (u.input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0), output: u.output_tokens ?? 0, cachedInput: u.cache_read_input_tokens }, api: "messages", raw: { stop: final.stop_reason, stop_details: (final as any).stop_details } };
}

// ---------------------------------------------------------------- scheduler
interface TestResult { test: string; ok: boolean; latencyMs: number; ttftMs?: number; usage?: any; costUsd?: number | null; error?: string; sample?: string; api?: string; raw?: any; }
const results: Record<string, { provider: Provider; kind: Kind; tests: TestResult[]; api?: string }> = {};
let spent = 0; let stopped = false;
const running: Record<Provider, number> = { openai: 0, anthropic: 0 };

async function runOne(c: Candidate, t: TestDef) {
  const t0 = Date.now();
  const maxTokens = c.kind === "chat-pro" ? 6000 : t.modality === "stream" ? 4000 : 8000;
  const rec: TestResult = { test: t.name, ok: false, latencyMs: 0 };
  try {
    const r = c.provider === "openai" ? await runOpenAI(c, t, maxTokens) : await runAnthropic(c, t, maxTokens);
    rec.latencyMs = Date.now() - t0; rec.ttftMs = r.ttftMs; rec.usage = r.usage; rec.api = r.api; rec.raw = r.raw;
    rec.ok = t.grade(r.text, { toolCall: r.toolCall }); rec.sample = (r.text || JSON.stringify(r.toolCall) || "").slice(0, 300);
    rec.costUsd = costOf(c.provider, c.id, r.usage); spent += rec.costUsd ?? 0;
  } catch (e: any) {
    rec.latencyMs = Date.now() - t0; rec.error = `${e?.status ?? ""} ${String(e?.message ?? e).slice(0, 300)}`.trim();
  }
  results[c.id].tests.push(rec); results[c.id].api = c.api;
  console.log(`${rec.ok ? "✔" : rec.error ? "✖" : "✗"} ${c.provider.padEnd(9)} ${c.id.padEnd(32)} ${t.name.padEnd(10)} ${String(rec.latencyMs).padStart(6)}ms ${rec.costUsd != null ? "$" + rec.costUsd.toFixed(4) : "$?"} ${rec.error ?? ""}`);
}

async function runModel(c: Candidate) {
  results[c.id] = { provider: c.provider, kind: c.kind, tests: [] };
  const tests = (c.kind === "chat-pro" ? TESTS.filter(t => PRO_TESTS.has(t.name)) : TESTS).filter(t => !ONLY_TESTS || ONLY_TESTS.has(t.name) || t.name === "basic");
  // basic first (sequential) so that API fallback is settled; then the rest
  await runOne(c, tests[0]);
  const first = results[c.id].tests[0];
  if (first.error && /401|403|404|deprecated|model_not_found|does not exist|not found|do not have access|not supported/i.test(first.error)) return;
  for (const t of tests.slice(1)) { if (spent > BUDGET) { stopped = true; return; } await runOne(c, t); }
}

async function main() {
  const todo = candidates.filter(c => ["chat", "chat-pro", "search"].includes(c.kind)).filter(c => (!ONLY || ONLY.has(c.id)) && !SKIP.has(c.id));
  for (const c of candidates.filter(c => !["chat", "chat-pro", "search"].includes(c.kind))) results[c.id] = { provider: c.provider, kind: c.kind, tests: [] };
  console.log(`Benchmarking ${todo.length} chat models (budget $${BUDGET})`);
  const queues: Record<Provider, Candidate[]> = { openai: todo.filter(c => c.provider === "openai"), anthropic: todo.filter(c => c.provider === "anthropic") };
  const workers: Promise<void>[] = [];
  for (const p of ["openai", "anthropic"] as Provider[]) for (let i = 0; i < CONC[p]; i++) workers.push((async () => { while (queues[p].length && !stopped) { const c = queues[p].shift()!; await runModel(c); } })());
  await Promise.all(workers);
  for (const p of ["openai", "anthropic"] as Provider[]) {
    const out: any = { generatedAt: new Date().toISOString(), provider: p, budgetUsd: BUDGET, stoppedByBudget: stopped, models: {} };
    for (const [id, r] of Object.entries(results)) if (r.provider === p) out.models[id] = r;
    const file = `benchmarks/${p}-results.json`;
    if (ONLY && fs.existsSync(file)) { const prev = JSON.parse(fs.readFileSync(file, "utf8")); for (const [id, m] of Object.entries<any>(out.models)) { const pm = prev.models[id]; if (pm && ONLY_TESTS) { const names = new Set(m.tests.map((t: any) => t.test)); m.tests = [...pm.tests.filter((t: any) => !names.has(t.test)), ...m.tests]; } } out.models = { ...prev.models, ...out.models }; }
    fs.writeFileSync(file, JSON.stringify(out, null, 2));
  }
  console.log(`\nDone. Estimated spend: $${spent.toFixed(3)}${stopped ? " (STOPPED: budget reached)" : ""}`);
}
main().catch(e => { console.error(e); process.exit(1); });