1/**2 * Real-API provider test matrix. Runs each capability against every provider with the3 * owner keys from .env and writes docs/provider-test-matrix.md. Each cell is ✅ only when4 * the real request succeeded. Cheap prompts, small max tokens.5 *6 * pnpm providers:matrix → all providers7 * pnpm providers:matrix anthropic → one provider8 */9import "./load-env";10import fs from "node:fs";11import zlib from "node:zlib";12import { getAdapter } from "@/lib/ai/providers";13import { ownerKey } from "@/lib/ai/providers/env-keys";14import { PROVIDER_IDS, type PolyModel, type ProviderId, type UnifiedChatRequest, type UnifiedStreamEvent } from "@/lib/ai/core/types";1516const KEYS = Object.fromEntries(PROVIDER_IDS.map((p) => [p, ownerKey(p)])) as Record<ProviderId, string | undefined>;1718/** Cheap, current models per provider used for the matrix. */19const TEST_MODELS: Record<ProviderId, { text: string; reasoning: string; vision: string }> = {20 openai: { text: "gpt-5.4-mini", reasoning: "gpt-5.4-mini", vision: "gpt-5.4-mini" },21 anthropic: { text: "claude-haiku-4-5", reasoning: "claude-sonnet-5", vision: "claude-haiku-4-5" },22 gemini: { text: "gemini-3.5-flash-lite", reasoning: "gemini-3.5-flash-lite", vision: "gemini-3.5-flash-lite" },23 xai: { text: "grok-4.20-0309-non-reasoning", reasoning: "grok-4.3", vision: "grok-4.20-0309-non-reasoning" },24 mistral: { text: "mistral-small-latest", reasoning: "magistral-medium-latest", vision: "mistral-small-latest" },25 deepseek: { text: "deepseek-v4-flash", reasoning: "deepseek-v4-flash", vision: "deepseek-v4-flash-vision-exp" },26 kimi: { text: "kimi-k2.6", reasoning: "kimi-k3", vision: "kimi-k2.6" },27 openrouter: { text: "openai/gpt-5.4-nano", reasoning: "openai/gpt-5.4-nano", vision: "openai/gpt-5.4-nano" },28 cerebras: { text: "gemma-4-31b", reasoning: "gpt-oss-120b", vision: "gemma-4-31b" },29};3031// 2x2 PNG (red, green / blue, white) — 32x32 needed for xAI minimums, so we scale via a bigger canvas.32function makePng(size: number): string {33 // Simple uncompressed PNG writer (RGB) for a solid red square with a white diagonal.34 const crcTable: number[] = [];35 for (let n = 0; n < 256; n++) {36 let c = n;37 for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;38 crcTable[n] = c >>> 0;39 }40 const crc = (buf: Buffer) => {41 let c = 0xffffffff;42 for (const b of buf) c = crcTable[(c ^ b) & 0xff] ^ (c >>> 8);43 return (c ^ 0xffffffff) >>> 0;44 };45 const chunk = (type: string, data: Buffer) => {46 const len = Buffer.alloc(4);47 len.writeUInt32BE(data.length);48 const td = Buffer.concat([Buffer.from(type), data]);49 const c = Buffer.alloc(4);50 c.writeUInt32BE(crc(td));51 return Buffer.concat([len, td, c]);52 };53 const ihdr = Buffer.alloc(13);54 ihdr.writeUInt32BE(size, 0);55 ihdr.writeUInt32BE(size, 4);56 ihdr[8] = 8;57 ihdr[9] = 2;58 const raw = Buffer.alloc((size * 3 + 1) * size);59 for (let y = 0; y < size; y++) {60 raw[y * (size * 3 + 1)] = 0;61 for (let x = 0; x < size; x++) {62 const o = y * (size * 3 + 1) + 1 + x * 3;63 const diag = Math.abs(x - y) < 2;64 raw[o] = diag ? 255 : 220;65 raw[o + 1] = diag ? 255 : 30;66 raw[o + 2] = diag ? 255 : 30;67 }68 }69 return Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), chunk("IHDR", ihdr), chunk("IDAT", zlib.deflateSync(raw)), chunk("IEND", Buffer.alloc(0))]).toString("base64");70}7172type Cell = "✅" | "❌" | "—";73const ROWS = ["Auth", "Model list", "Text", "Streaming", "System", "Vision", "Tools", "Structured output", "Reasoning", "Token usage", "Invalid key", "Error normalization", "Long response"] as const;74type Row = (typeof ROWS)[number];7576async function collect(req: UnifiedChatRequest, attempt = 1): Promise<Awaited<ReturnType<typeof collectOnce>>> {77 try {78 return await collectOnce(req);79 } catch (e) {80 const poly = (e as { poly?: { code: string; retryAfterMs?: number } }).poly;81 if (poly && (poly.code === "RATE_LIMITED" || poly.code === "PROVIDER_UNAVAILABLE") && attempt <= 3) {82 await new Promise((r) => setTimeout(r, Math.max(3000, poly.retryAfterMs ?? 0) * attempt));83 return collect(req, attempt + 1);84 }85 throw e;86 }87}8889async function collectOnce(req: UnifiedChatRequest) {90 const adapter = getAdapter(req.provider);91 const events: UnifiedStreamEvent[] = [];92 for await (const ev of adapter.streamChat(req)) events.push(ev);93 const err = events.find((e) => e.type === "error");94 if (err && err.type === "error") throw Object.assign(new Error(err.error.message), { poly: err.error });95 const text = events.filter((e) => e.type === "text-delta").map((e) => (e as { text: string }).text).join("");96 const reasoning = events.filter((e) => e.type === "reasoning-delta").map((e) => (e as { text: string }).text).join("");97 const usage = events.find((e) => e.type === "usage") as { usage: { inputTokens: number; outputTokens: number; reasoningTokens?: number } } | undefined;98 const tools = events.filter((e) => e.type === "tool-end") as { name: string; arguments: Record<string, unknown> }[];99 const deltas = events.filter((e) => e.type === "text-delta" || e.type === "reasoning-delta").length;100 return { text, reasoning, usage: usage?.usage, tools, deltas, finish: (events.find((e) => e.type === "finish") as { reason?: string } | undefined)?.reason };101}102103async function runProvider(p: ProviderId, results: Record<Row, Record<ProviderId, Cell>>, notes: string[]) {104 const key = KEYS[p];105 const adapter = getAdapter(p);106 const set = (r: Row, v: Cell) => (results[r][p] = v);107 if (!key) {108 for (const r of ROWS) set(r, "—");109 notes.push(`${p}: no key in env`);110 return;111 }112 const note = (s: string) => notes.push(`${p}: ${s}`);113 const m = TEST_MODELS[p];114 let models: PolyModel[] = [];115 try {116 const v = await adapter.validateApiKey(key);117 set("Auth", v.ok ? "✅" : "❌");118 if (!v.ok) note(`auth: ${v.error?.code} ${v.error?.message}`);119 } catch (e) {120 set("Auth", "❌");121 note(`auth threw: ${(e as Error).message}`);122 }123 try {124 models = await adapter.listModels(key);125 set("Model list", models.length ? "✅" : "❌");126 note(`models: ${models.length} (${models.slice(0, 5).map((x) => x.id).join(", ")}…)`);127 } catch (e) {128 set("Model list", "❌");129 note(`list: ${(e as Error).message}`);130 }131 // Aliases such as `claude-haiku-4-5` resolve to the dated snapshot listed by the provider.132 const info = (id: string) => models.find((x) => x.id === id) ?? models.find((x) => x.id.startsWith(`${id}-`));133 const base = (model: string): UnifiedChatRequest => ({ provider: p, model, apiKey: key, messages: [{ role: "user", content: [{ type: "text", text: "Reply with exactly: OK" }] }], settings: { maxTokens: 2000 }, modelInfo: info(model) });134135 try {136 const r = await collect(base(m.text));137 set("Text", r.text.trim().length ? "✅" : "❌");138 set("Streaming", r.deltas >= 1 ? "✅" : "❌");139 set("Token usage", r.usage && r.usage.inputTokens > 0 && r.usage.outputTokens > 0 ? "✅" : "❌");140 note(`text(${m.text}): "${r.text.trim().slice(0, 40)}" deltas=${r.deltas} usage=${JSON.stringify(r.usage)} finish=${r.finish}`);141 } catch (e) {142 set("Text", "❌");143 set("Streaming", "❌");144 set("Token usage", "❌");145 note(`text: ${(e as Error).message}`);146 }147 try {148 const r = await collect({ ...base(m.text), system: "You are a pirate. Always start your reply with 'Arr'.", messages: [{ role: "user", content: [{ type: "text", text: "Say hello in five words." }] }] });149 set("System", /arr/i.test(r.text) ? "✅" : "❌");150 note(`system: "${r.text.trim().slice(0, 60)}"`);151 } catch (e) {152 set("System", "❌");153 note(`system: ${(e as Error).message}`);154 }155 try {156 const r = await collect({ ...base(m.vision), messages: [{ role: "user", content: [{ type: "image", mimeType: "image/png", data: makePng(64) }, { type: "text", text: "What is the dominant color of this image? One word." }] }] });157 set("Vision", /red/i.test(r.text) ? "✅" : "❌");158 note(`vision: "${r.text.trim().slice(0, 40)}"`);159 } catch (e) {160 set("Vision", "❌");161 note(`vision: ${(e as Error).message}`);162 }163 try {164 const r = await collect({ ...base(m.text), messages: [{ role: "user", content: [{ type: "text", text: "What is 1234 * 5678? Use the calculator tool." }] }], tools: [{ name: "calculator", description: "Evaluate an arithmetic expression", parameters: { type: "object", properties: { expression: { type: "string" } }, required: ["expression"], additionalProperties: false }, strict: true }] });165 set("Tools", r.tools.length && r.tools[0].name === "calculator" ? "✅" : "❌");166 note(`tools: ${JSON.stringify(r.tools.map((t) => ({ n: t.name, a: t.arguments })))} finish=${r.finish}`);167 } catch (e) {168 set("Tools", "❌");169 note(`tools: ${(e as Error).message}`);170 }171 try {172 const r = await collect({ ...base(m.text), settings: { maxTokens: 2000, responseFormat: { type: "json_schema", schema: { type: "object", properties: { city: { type: "string" }, country: { type: "string" } }, required: ["city", "country"], additionalProperties: false }, schemaName: "city" } }, messages: [{ role: "user", content: [{ type: "text", text: "Give the capital of Canada as JSON." }] }] });173 let ok = false;174 try {175 const j = JSON.parse(r.text.trim().replace(/^```(?:json)?\s*|\s*```$/g, ""));176 ok = typeof j.city === "string" && typeof j.country === "string";177 } catch {178 ok = false;179 }180 set("Structured output", ok ? "✅" : "❌");181 note(`structured: ${r.text.trim().slice(0, 80)}`);182 } catch (e) {183 set("Structured output", "❌");184 note(`structured: ${(e as Error).message}`);185 }186 try {187 const r = await collect({ ...base(m.reasoning), settings: { maxTokens: 6000, reasoningEffort: p === "anthropic" ? "high" : "low", includeReasoning: true }, messages: [{ role: "user", content: [{ type: "text", text: "How many prime numbers are there strictly between 1000 and 1100? Think it through carefully, then answer with just the number." }] }] });188 const hasReasoning = r.reasoning.length > 0 || (r.usage?.reasoningTokens ?? 0) > 0;189 set("Reasoning", hasReasoning ? "✅" : "❌");190 note(`reasoning(${m.reasoning}): text="${r.text.trim().slice(0, 30)}" reasoningChars=${r.reasoning.length} reasoningTokens=${r.usage?.reasoningTokens}`);191 } catch (e) {192 set("Reasoning", "❌");193 note(`reasoning: ${(e as Error).message}`);194 }195 try {196 const badKeys: Record<string, string> = { gemini: "AIzaSyInvalidKey0000000000000000000000", anthropic: "sk-ant-api03-invalid", xai: "xai-invalid", mistral: "invalidinvalidinvalidinvalid0000", openrouter: "sk-or-v1-invalid0000000000000000", cerebras: "csk-invalid00000000000000000000" };197 const bad = await adapter.validateApiKey(badKeys[p] ?? "sk-invalid-0000000000000000");198 set("Invalid key", !bad.ok && bad.error?.code === "INVALID_API_KEY" ? "✅" : "❌");199 note(`invalid key → ${bad.error?.code} (${bad.error?.status})`);200 } catch (e) {201 set("Invalid key", "❌");202 note(`invalid key threw: ${(e as Error).message}`);203 }204 try {205 await collect(base("definitely-not-a-model-xyz"));206 set("Error normalization", "❌");207 } catch (e) {208 const poly = (e as { poly?: { code: string } }).poly;209 set("Error normalization", poly?.code === "MODEL_NOT_FOUND" || poly?.code === "INVALID_PARAMETER" ? "✅" : "❌");210 note(`unknown model → ${poly?.code}: ${(e as Error).message.slice(0, 80)}`);211 }212 try {213 const r = await collect({ ...base(m.text), settings: { maxTokens: 3000 }, messages: [{ role: "user", content: [{ type: "text", text: "Write a 400-word story about a lighthouse keeper. Plain prose." }] }] });214 set("Long response", r.text.split(/\s+/).length > 250 ? "✅" : "❌");215 note(`long: ${r.text.split(/\s+/).length} words, ${r.deltas} deltas`);216 } catch (e) {217 set("Long response", "❌");218 note(`long: ${(e as Error).message}`);219 }220}221222async function main() {223 const only = process.argv[2] as ProviderId | undefined;224 const providers = only ? [only] : [...PROVIDER_IDS];225 const results = Object.fromEntries(ROWS.map((r) => [r, Object.fromEntries(PROVIDER_IDS.map((p) => [p, "—" as Cell]))])) as Record<Row, Record<ProviderId, Cell>>;226 const notes: string[] = [];227 await Promise.all(providers.map((p) => runProvider(p, results, notes)));228 // Merge with a previous full matrix so single-provider runs do not blank the others.229 if (only && fs.existsSync("docs/provider-test-matrix.md")) {230 const prev = fs.readFileSync("docs/provider-test-matrix.md", "utf8");231 for (const r of ROWS) {232 const m = prev.match(new RegExp(`^\\| ${r.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} \\| (.+) \\|$`, "m"));233 if (!m) continue;234 const cells = m[1].split(" | ") as Cell[];235 PROVIDER_IDS.forEach((pp, i) => {236 if (pp !== only && cells[i]) results[r][pp] = cells[i];237 });238 }239 }240 const date = new Date().toISOString().slice(0, 10);241 const header = `| Capability | ${PROVIDER_IDS.map((p) => p).join(" | ")} |\n| --- | ${PROVIDER_IDS.map(() => ":-:").join(" | ")} |`;242 const lines = ROWS.map((r) => `| ${r} | ${PROVIDER_IDS.map((p) => results[r][p]).join(" | ")} |`);243 const md = `# Provider test matrix\n\nGenerated by \`pnpm providers:matrix\` against the REAL provider APIs on ${date}. A cell is ✅ only when the live request succeeded and the assertion passed. “—” = not run (no key).\n\nTest models: ${PROVIDER_IDS.map((p) => `${p}: ${TEST_MODELS[p].text} / reasoning ${TEST_MODELS[p].reasoning}`).join("; ")}.\n\n${header}\n${lines.join("\n")}\n\n## Notes\n\n${notes.map((n) => `- ${n.replace(/\n/g, " ")}`).join("\n")}\n`;244 fs.mkdirSync("docs", { recursive: true });245 fs.writeFileSync("docs/provider-test-matrix.md", md);246 console.log(md);247}248249main().catch((e) => {250 console.error(e);251 process.exit(1);252});253