spb/note-spboucher-ai

1/**2 * Phase Zero §5.2 — real model discovery.3 * Lists models from both providers, classifies them, and PROBES each candidate text model with4 * real requests: text, streaming, tool use, structured output, vision, reasoning control.5 * Output: benchmarks/results/discovery.json (+ raw per-probe log).6 */7import fs from "node:fs";8import path from "node:path";9import { openai, anthropic, RESULTS, RAW, writeJSON, appendJSONL, priceFor, pLimit, estimateCost } from "./lib.ts";10import { call, type CallOpts } from "./providers.ts";1112type ProbeStatus = "ok" | "unsupported" | "error" | "untested" | "unavailable";13interface Probe { status: ProbeStatus; ms?: number; detail?: string; }14interface ModelRecord {15  provider: "openai" | "anthropic"; id: string; display?: string; created?: string; family: string;16  category: "text" | "text-specialized" | "embedding" | "audio" | "image" | "video" | "moderation" | "legacy-completion" | "search" | "deep-research" | "realtime" | "unknown";17  aliasOf?: string | null;18  context?: number | null; maxOutput?: number | null; capabilities?: unknown;19  pricing: ReturnType<typeof priceFor>;20  probes: Record<string, Probe>;21  verdict: "tested" | "unsupported" | "untested" | "unavailable";22  notes: string[];23}2425const SCHEMA = { type: "object", properties: { city: { type: "string" }, population_millions: { type: "number" } }, required: ["city", "population_millions"], additionalProperties: false };26const TOOL = [{ name: "get_weather", description: "Get current weather for a city", parameters: { type: "object", properties: { city: { type: "string" } }, required: ["city"], additionalProperties: false } }];2728function classifyOpenAI(id: string): ModelRecord["category"] {29  if (/embedding/.test(id)) return "embedding";30  if (/moderation/.test(id)) return "moderation";31  if (/sora/.test(id)) return "video";32  if (/image|dall-e/.test(id)) return "image";33  if (/realtime|live/.test(id)) return "realtime";34  if (/audio|tts|transcribe|whisper/.test(id)) return "audio";35  if (/deep-research/.test(id)) return "deep-research";36  if (/search/.test(id)) return "search";37  if (/babbage|davinci|instruct/.test(id)) return "legacy-completion";38  if (/codex/.test(id)) return "text-specialized";39  if (/^(gpt|o\d|chat-latest)/.test(id)) return "text";40  return "unknown";41}42const family = (id: string) => id.replace(/-\d{4}-\d{2}-\d{2}$/, "").replace(/-(mini|nano|pro|chat-latest|codex.*|astra|luna|sol|terra)$/, "");4344async function probeModel(rec: ModelRecord, log: string) {45  const p = rec.provider;46  const base: Partial<CallOpts> = { model: rec.id, maxTokens: 4000, timeoutMs: 180_000 };47  const run = async (name: string, o: CallOpts, check: (r: any) => string | null) => {48    const r = await call(p, o);49    appendJSONL(log, { model: rec.id, probe: name, ok: r.ok, ms: Math.round(r.total_ms), error: r.error, status: r.status, usage: r.usage, text: r.text.slice(0, 200), tools: r.toolCalls });50    if (!r.ok) {51      const unsupported = r.status === 400 || r.status === 404 || /not supported|does not support|unsupported|invalid model|not found|does not exist|Unsupported/i.test(r.error ?? "");52      rec.probes[name] = { status: r.status === 404 || /not found|does not exist|no access/i.test(r.error ?? "") ? "unavailable" : unsupported ? "unsupported" : "error", ms: Math.round(r.total_ms), detail: `${r.status ?? ""} ${r.error}`.trim() };53      return r;54    }55    const fail = check(r);56    rec.probes[name] = fail ? { status: "unsupported", ms: Math.round(r.total_ms), detail: fail } : { status: "ok", ms: Math.round(r.total_ms) };57    return r;58  };59  // 1. text (non-stream)60  const t = await run("text", { ...base, prompt: "Reply with exactly the word PONG.", stream: false } as CallOpts, (r) => (/pong/i.test(r.text) ? null : `unexpected: ${r.text.slice(0, 80)}`));61  if (!t.ok) { rec.verdict = rec.probes.text.status === "unavailable" ? "unavailable" : "unsupported"; return; }62  // 2. streaming63  await run("streaming", { ...base, prompt: "Count from 1 to 10 separated by spaces.", stream: true } as CallOpts, (r) => (r.ttft_ms !== null ? null : "no streamed delta"));64  // 3. tool use65  await run("tool_use", { ...base, prompt: "What is the weather in Gatineau right now? Use the get_weather tool.", tools: TOOL, stream: false } as CallOpts, (r) => (r.toolCalls.some((c: any) => c.name === "get_weather") ? null : "no tool call emitted"));66  // 4. structured output67  await run("structured_output", { ...base, prompt: "Give the city Montreal and its population in millions as JSON.", jsonSchema: { name: "city", schema: SCHEMA }, stream: false } as CallOpts, (r) => { try { const j = JSON.parse(r.text); return typeof j.city === "string" && typeof j.population_millions === "number" ? null : "schema mismatch"; } catch { return "invalid JSON"; } });68  // 5. vision69  await run("vision", { ...base, prompt: "What single colour is this image? Answer with one word.", image: true, stream: false } as CallOpts, (r) => (/red/i.test(r.text) ? null : `answer: ${r.text.slice(0, 60)}`));70  // 6. reasoning control71  await run("reasoning_control", { ...base, prompt: "What is 17*23? Answer with the number only.", reasoning: "low", stream: false } as CallOpts, (r) => (/391/.test(r.text) ? null : `answer: ${r.text.slice(0, 60)}`));72  rec.verdict = "tested";73}7475async function main() {76  const log = path.join(RAW, "discovery-probes.jsonl");77  fs.rmSync(log, { force: true });78  const records: ModelRecord[] = [];7980  // ---- OpenAI81  const oa = await openai.models.list();82  const oaIds: { id: string; created: number; owned_by: string }[] = [];83  for await (const m of oa as any) oaIds.push(m);84  oaIds.sort((a, b) => a.id.localeCompare(b.id));85  const aliasSet = new Set(oaIds.map((m) => m.id));86  for (const m of oaIds) {87    const cat = classifyOpenAI(m.id);88    const aliasOf = /-\d{4}-\d{2}-\d{2}$/.test(m.id) && aliasSet.has(m.id.replace(/-\d{4}-\d{2}-\d{2}$/, "")) ? m.id.replace(/-\d{4}-\d{2}-\d{2}$/, "") : null;89    records.push({ provider: "openai", id: m.id, created: new Date(m.created * 1000).toISOString().slice(0, 10), family: family(m.id), category: cat, aliasOf, pricing: priceFor(m.id), probes: {}, verdict: "untested", notes: [] });90  }91  // ---- Anthropic92  const an = await anthropic.models.list({ limit: 100 });93  for await (const m of an as any) {94    let detail: any = null;95    try { detail = await (anthropic.models as any).retrieve(m.id, null, { headers: { "anthropic-version": "2026-10-21" } }); } catch (e: any) { }96    records.push({ provider: "anthropic", id: m.id, display: m.display_name, created: String(m.created_at).slice(0, 10), family: m.id.replace(/-\d{8}$/, ""), category: "text", aliasOf: null, context: detail?.max_input_tokens ?? null, maxOutput: detail?.max_tokens ?? null, capabilities: detail?.capabilities ?? null, pricing: priceFor(m.id), probes: {}, verdict: "untested", notes: [] });97  }9899  // Which to probe: all text / text-specialized aliases (no dated snapshots), all Anthropic models.100  // Pro models are probed for "text" only via the same path (they are slow/expensive) → still real.101  const toProbe = records.filter((r) => (r.provider === "anthropic") || ((r.category === "text" || r.category === "text-specialized") && !r.aliasOf));102  for (const r of records) if (r.aliasOf) r.notes.push(`dated snapshot of ${r.aliasOf}; not probed separately`);103  for (const r of records) if (!toProbe.includes(r) && !r.aliasOf) r.notes.push(`category ${r.category}: not a general text model, not probed (inappropriate request type)`);104  const expensive = (id: string) => /-pro$|^o1-pro|^o3-pro/.test(id);105  console.log(`Probing ${toProbe.length} models…`);106  await pLimit(6, toProbe.map((r) => async () => {107    if (expensive(r.id)) {108      // only text + structured output, small budgets; marks the rest untested for cost reasons.109      r.notes.push("pro/expensive tier: only text + structured_output probed; benchmark skipped for cost ($30-$150/M input)");110    }111    const t0 = Date.now();112    try { await probeModel(r, log); } catch (e: any) { r.notes.push(`probe crashed: ${e?.message}`); r.verdict = "untested"; }113    console.log(`${r.id.padEnd(30)} ${r.verdict.padEnd(12)} ${Object.entries(r.probes).map(([k, v]) => `${k}=${v.status}`).join(" ")}  (${((Date.now() - t0) / 1000).toFixed(1)}s)`);114  }));115  // Estimated cost of this discovery run116  let cost = 0;117  for (const line of fs.readFileSync(log, "utf8").split("\n").filter(Boolean)) { const j = JSON.parse(line); if (j.usage) cost += estimateCost(j.model, j.usage.input_tokens, j.usage.output_tokens, j.usage.cached_tokens) ?? 0; }118  writeJSON(path.join(RESULTS, "discovery.json"), { generated: new Date().toISOString(), estimated_cost_usd: +cost.toFixed(4), models: records });119  console.log(`\nDone. ${records.length} models recorded, estimated probe cost $${cost.toFixed(4)}`);120}121main().catch((e) => { console.error(e); process.exit(1); });122