1/**2 * Real-API provider test matrix. Runs each capability against every provider with the3 * owner keys from .env and writes docs/provider-test-matrix.md. Each cell is ✅ only when4 * the real request succeeded. Cheap prompts, small max tokens.5 *6 * pnpm providers:matrix → all providers7 * pnpm providers:matrix anthropic → one provider8 */9import "./load-env";10import fs from "node:fs";11import zlib from "node:zlib";12import { getAdapter } from "@/lib/ai/providers";13import { PROVIDER_IDS, type PolyModel, type ProviderId, type UnifiedChatRequest, type UnifiedStreamEvent } from "@/lib/ai/core/types";1415const KEYS: Record<ProviderId, string | undefined> = {16 openai: process.env.OPENAI_API_KEY,17 anthropic: process.env.ANTHROPIC_API_KEY,18 gemini: process.env.GOOGLE_GEMINI_API_KEY,19 xai: process.env.XAI_API_KEY,20};2122/** Cheap, current models per provider used for the matrix. */23const TEST_MODELS: Record<ProviderId, { text: string; reasoning: string; vision: string }> = {24 openai: { text: "gpt-5.4-mini", reasoning: "gpt-5.4-mini", vision: "gpt-5.4-mini" },25 anthropic: { text: "claude-haiku-4-5", reasoning: "claude-sonnet-5", vision: "claude-haiku-4-5" },26 gemini: { text: "gemini-3.5-flash-lite", reasoning: "gemini-3.5-flash-lite", vision: "gemini-3.5-flash-lite" },27 xai: { text: "grok-4.20-0309-non-reasoning", reasoning: "grok-4.3", vision: "grok-4.20-0309-non-reasoning" },28};2930// 2x2 PNG (red, green / blue, white) — 32x32 needed for xAI minimums, so we scale via a bigger canvas.31function makePng(size: number): string {32 // Simple uncompressed PNG writer (RGB) for a solid red square with a white diagonal.33 const crcTable: number[] = [];34 for (let n = 0; n < 256; n++) {35 let c = n;36 for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;37 crcTable[n] = c >>> 0;38 }39 const crc = (buf: Buffer) => {40 let c = 0xffffffff;41 for (const b of buf) c = crcTable[(c ^ b) & 0xff] ^ (c >>> 8);42 return (c ^ 0xffffffff) >>> 0;43 };44 const chunk = (type: string, data: Buffer) => {45 const len = Buffer.alloc(4);46 len.writeUInt32BE(data.length);47 const td = Buffer.concat([Buffer.from(type), data]);48 const c = Buffer.alloc(4);49 c.writeUInt32BE(crc(td));50 return Buffer.concat([len, td, c]);51 };52 const ihdr = Buffer.alloc(13);53 ihdr.writeUInt32BE(size, 0);54 ihdr.writeUInt32BE(size, 4);55 ihdr[8] = 8;56 ihdr[9] = 2;57 const raw = Buffer.alloc((size * 3 + 1) * size);58 for (let y = 0; y < size; y++) {59 raw[y * (size * 3 + 1)] = 0;60 for (let x = 0; x < size; x++) {61 const o = y * (size * 3 + 1) + 1 + x * 3;62 const diag = Math.abs(x - y) < 2;63 raw[o] = diag ? 255 : 220;64 raw[o + 1] = diag ? 255 : 30;65 raw[o + 2] = diag ? 255 : 30;66 }67 }68 return Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), chunk("IHDR", ihdr), chunk("IDAT", zlib.deflateSync(raw)), chunk("IEND", Buffer.alloc(0))]).toString("base64");69}7071type Cell = "✅" | "❌" | "—";72const ROWS = ["Auth", "Model list", "Text", "Streaming", "System", "Vision", "Tools", "Structured output", "Reasoning", "Token usage", "Invalid key", "Error normalization", "Long response"] as const;73type Row = (typeof ROWS)[number];7475async function collect(req: UnifiedChatRequest, attempt = 1): Promise<Awaited<ReturnType<typeof collectOnce>>> {76 try {77 return await collectOnce(req);78 } catch (e) {79 const poly = (e as { poly?: { code: string; retryAfterMs?: number } }).poly;80 if (poly && (poly.code === "RATE_LIMITED" || poly.code === "PROVIDER_UNAVAILABLE") && attempt <= 3) {81 await new Promise((r) => setTimeout(r, Math.max(3000, poly.retryAfterMs ?? 0) * attempt));82 return collect(req, attempt + 1);83 }84 throw e;85 }86}8788async function collectOnce(req: UnifiedChatRequest) {89 const adapter = getAdapter(req.provider);90 const events: UnifiedStreamEvent[] = [];91 for await (const ev of adapter.streamChat(req)) events.push(ev);92 const err = events.find((e) => e.type === "error");93 if (err && err.type === "error") throw Object.assign(new Error(err.error.message), { poly: err.error });94 const text = events.filter((e) => e.type === "text-delta").map((e) => (e as { text: string }).text).join("");95 const reasoning = events.filter((e) => e.type === "reasoning-delta").map((e) => (e as { text: string }).text).join("");96 const usage = events.find((e) => e.type === "usage") as { usage: { inputTokens: number; outputTokens: number; reasoningTokens?: number } } | undefined;97 const tools = events.filter((e) => e.type === "tool-end") as { name: string; arguments: Record<string, unknown> }[];98 const deltas = events.filter((e) => e.type === "text-delta" || e.type === "reasoning-delta").length;99 return { text, reasoning, usage: usage?.usage, tools, deltas, finish: (events.find((e) => e.type === "finish") as { reason?: string } | undefined)?.reason };100}101102async function runProvider(p: ProviderId, results: Record<Row, Record<ProviderId, Cell>>, notes: string[]) {103 const key = KEYS[p];104 const adapter = getAdapter(p);105 const set = (r: Row, v: Cell) => (results[r][p] = v);106 if (!key) {107 for (const r of ROWS) set(r, "—");108 notes.push(`${p}: no key in env`);109 return;110 }111 const note = (s: string) => notes.push(`${p}: ${s}`);112 const m = TEST_MODELS[p];113 let models: PolyModel[] = [];114 try {115 const v = await adapter.validateApiKey(key);116 set("Auth", v.ok ? "✅" : "❌");117 if (!v.ok) note(`auth: ${v.error?.code} ${v.error?.message}`);118 } catch (e) {119 set("Auth", "❌");120 note(`auth threw: ${(e as Error).message}`);121 }122 try {123 models = await adapter.listModels(key);124 set("Model list", models.length ? "✅" : "❌");125 note(`models: ${models.length} (${models.slice(0, 5).map((x) => x.id).join(", ")}…)`);126 } catch (e) {127 set("Model list", "❌");128 note(`list: ${(e as Error).message}`);129 }130 // Aliases such as `claude-haiku-4-5` resolve to the dated snapshot listed by the provider.131 const info = (id: string) => models.find((x) => x.id === id) ?? models.find((x) => x.id.startsWith(`${id}-`));132 const base = (model: string): UnifiedChatRequest => ({ provider: p, model, apiKey: key, messages: [{ role: "user", content: [{ type: "text", text: "Reply with exactly: OK" }] }], settings: { maxTokens: 2000 }, modelInfo: info(model) });133134 try {135 const r = await collect(base(m.text));136 set("Text", r.text.trim().length ? "✅" : "❌");137 set("Streaming", r.deltas >= 1 ? "✅" : "❌");138 set("Token usage", r.usage && r.usage.inputTokens > 0 && r.usage.outputTokens > 0 ? "✅" : "❌");139 note(`text(${m.text}): "${r.text.trim().slice(0, 40)}" deltas=${r.deltas} usage=${JSON.stringify(r.usage)} finish=${r.finish}`);140 } catch (e) {141 set("Text", "❌");142 set("Streaming", "❌");143 set("Token usage", "❌");144 note(`text: ${(e as Error).message}`);145 }146 try {147 const r = await collect({ ...base(m.text), system: "You are a pirate. Always start your reply with 'Arr'.", messages: [{ role: "user", content: [{ type: "text", text: "Say hello in five words." }] }] });148 set("System", /arr/i.test(r.text) ? "✅" : "❌");149 note(`system: "${r.text.trim().slice(0, 60)}"`);150 } catch (e) {151 set("System", "❌");152 note(`system: ${(e as Error).message}`);153 }154 try {155 const r = await collect({ ...base(m.vision), messages: [{ role: "user", content: [{ type: "image", mimeType: "image/png", data: makePng(64) }, { type: "text", text: "What is the dominant color of this image? One word." }] }] });156 set("Vision", /red/i.test(r.text) ? "✅" : "❌");157 note(`vision: "${r.text.trim().slice(0, 40)}"`);158 } catch (e) {159 set("Vision", "❌");160 note(`vision: ${(e as Error).message}`);161 }162 try {163 const r = await collect({ ...base(m.text), messages: [{ role: "user", content: [{ type: "text", text: "What is 1234 * 5678? Use the calculator tool." }] }], tools: [{ name: "calculator", description: "Evaluate an arithmetic expression", parameters: { type: "object", properties: { expression: { type: "string" } }, required: ["expression"], additionalProperties: false }, strict: true }] });164 set("Tools", r.tools.length && r.tools[0].name === "calculator" ? "✅" : "❌");165 note(`tools: ${JSON.stringify(r.tools.map((t) => ({ n: t.name, a: t.arguments })))} finish=${r.finish}`);166 } catch (e) {167 set("Tools", "❌");168 note(`tools: ${(e as Error).message}`);169 }170 try {171 const r = await collect({ ...base(m.text), settings: { maxTokens: 2000, responseFormat: { type: "json_schema", schema: { type: "object", properties: { city: { type: "string" }, country: { type: "string" } }, required: ["city", "country"], additionalProperties: false }, schemaName: "city" } }, messages: [{ role: "user", content: [{ type: "text", text: "Give the capital of Canada as JSON." }] }] });172 let ok = false;173 try {174 const j = JSON.parse(r.text);175 ok = typeof j.city === "string" && typeof j.country === "string";176 } catch {177 ok = false;178 }179 set("Structured output", ok ? "✅" : "❌");180 note(`structured: ${r.text.trim().slice(0, 80)}`);181 } catch (e) {182 set("Structured output", "❌");183 note(`structured: ${(e as Error).message}`);184 }185 try {186 const r = await collect({ ...base(m.reasoning), settings: { maxTokens: 6000, reasoningEffort: p === "anthropic" ? "high" : "low", includeReasoning: true }, messages: [{ role: "user", content: [{ type: "text", text: "How many prime numbers are there strictly between 1000 and 1100? Think it through carefully, then answer with just the number." }] }] });187 const hasReasoning = r.reasoning.length > 0 || (r.usage?.reasoningTokens ?? 0) > 0;188 set("Reasoning", hasReasoning ? "✅" : "❌");189 note(`reasoning(${m.reasoning}): text="${r.text.trim().slice(0, 30)}" reasoningChars=${r.reasoning.length} reasoningTokens=${r.usage?.reasoningTokens}`);190 } catch (e) {191 set("Reasoning", "❌");192 note(`reasoning: ${(e as Error).message}`);193 }194 try {195 const bad = await adapter.validateApiKey(p === "gemini" ? "AIzaSyInvalidKey0000000000000000000000" : p === "anthropic" ? "sk-ant-api03-invalid" : p === "xai" ? "xai-invalid" : "sk-invalid");196 set("Invalid key", !bad.ok && bad.error?.code === "INVALID_API_KEY" ? "✅" : "❌");197 note(`invalid key → ${bad.error?.code} (${bad.error?.status})`);198 } catch (e) {199 set("Invalid key", "❌");200 note(`invalid key threw: ${(e as Error).message}`);201 }202 try {203 await collect(base("definitely-not-a-model-xyz"));204 set("Error normalization", "❌");205 } catch (e) {206 const poly = (e as { poly?: { code: string } }).poly;207 set("Error normalization", poly?.code === "MODEL_NOT_FOUND" || poly?.code === "INVALID_PARAMETER" ? "✅" : "❌");208 note(`unknown model → ${poly?.code}: ${(e as Error).message.slice(0, 80)}`);209 }210 try {211 const r = await collect({ ...base(m.text), settings: { maxTokens: 3000 }, messages: [{ role: "user", content: [{ type: "text", text: "Write a 400-word story about a lighthouse keeper. Plain prose." }] }] });212 set("Long response", r.text.split(/\s+/).length > 250 ? "✅" : "❌");213 note(`long: ${r.text.split(/\s+/).length} words, ${r.deltas} deltas`);214 } catch (e) {215 set("Long response", "❌");216 note(`long: ${(e as Error).message}`);217 }218}219220async function main() {221 const only = process.argv[2] as ProviderId | undefined;222 const providers = only ? [only] : [...PROVIDER_IDS];223 const results = Object.fromEntries(ROWS.map((r) => [r, Object.fromEntries(PROVIDER_IDS.map((p) => [p, "—" as Cell]))])) as Record<Row, Record<ProviderId, Cell>>;224 const notes: string[] = [];225 await Promise.all(providers.map((p) => runProvider(p, results, notes)));226 // Merge with a previous full matrix so single-provider runs do not blank the others.227 if (only && fs.existsSync("docs/provider-test-matrix.md")) {228 const prev = fs.readFileSync("docs/provider-test-matrix.md", "utf8");229 for (const r of ROWS) {230 const m = prev.match(new RegExp(`^\\| ${r.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} \\| (.+) \\|$`, "m"));231 if (!m) continue;232 const cells = m[1].split(" | ") as Cell[];233 PROVIDER_IDS.forEach((pp, i) => {234 if (pp !== only && cells[i]) results[r][pp] = cells[i];235 });236 }237 }238 const date = new Date().toISOString().slice(0, 10);239 const header = `| Capability | OpenAI | Anthropic | Gemini | xAI |\n| --- | :-: | :-: | :-: | :-: |`;240 const lines = ROWS.map((r) => `| ${r} | ${PROVIDER_IDS.map((p) => results[r][p]).join(" | ")} |`);241 const md = `# Provider test matrix\n\nGenerated by \`pnpm providers:matrix\` against the REAL provider APIs on ${date}. A cell is ✅ only when the live request succeeded and the assertion passed. “—” = not run (no key).\n\nTest models: ${PROVIDER_IDS.map((p) => `${p}: ${TEST_MODELS[p].text} / reasoning ${TEST_MODELS[p].reasoning}`).join("; ")}.\n\n${header}\n${lines.join("\n")}\n\n## Notes\n\n${notes.map((n) => `- ${n.replace(/\n/g, " ")}`).join("\n")}\n`;242 fs.mkdirSync("docs", { recursive: true });243 fs.writeFileSync("docs/provider-test-matrix.md", md);244 console.log(md);245}246247main().catch((e) => {248 console.error(e);249 process.exit(1);250});251