TypeScript 97.4%
SQL 1%
JavaScript 0.9%
CSS 0.6%
1/**2 * Real-API provider test matrix. Runs each capability against every provider with the3 * owner keys from .env and writes docs/provider-test-matrix.md. Each cell is ✅ only when4 * the real request succeeded. Cheap prompts, small max tokens.5 *6 * pnpm providers:matrix → all providers7 * pnpm providers:matrix anthropic → one provider8 */9import "./load-env";10import fs from "node:fs";11import zlib from "node:zlib";12import { getAdapter } from "@/lib/ai/providers";13import { ownerKey } from "@/lib/ai/providers/env-keys";14import { PROVIDER_IDS, type PolyModel, type ProviderId, type UnifiedChatRequest, type UnifiedStreamEvent } from "@/lib/ai/core/types";1516const KEYS = Object.fromEntries(PROVIDER_IDS.map((p) => [p, ownerKey(p)])) as Record<ProviderId, string | undefined>;1718/** Cheap, current models per provider used for the matrix. */19const TEST_MODELS: Record<ProviderId, { text: string; reasoning: string; vision: string }> = {20 openai: { text: "gpt-5.4-mini", reasoning: "gpt-5.4-mini", vision: "gpt-5.4-mini" },21 anthropic: { text: "claude-haiku-4-5", reasoning: "claude-sonnet-5", vision: "claude-haiku-4-5" },22 gemini: { text: "gemini-3.5-flash-lite", reasoning: "gemini-3.5-flash-lite", vision: "gemini-3.5-flash-lite" },23 xai: { text: "grok-4.20-0309-non-reasoning", reasoning: "grok-4.3", vision: "grok-4.20-0309-non-reasoning" },24 mistral: { text: "mistral-small-latest", reasoning: "magistral-medium-latest", vision: "mistral-small-latest" },25 deepseek: { text: "deepseek-v4-flash", reasoning: "deepseek-v4-flash", vision: "deepseek-v4-flash-vision-exp" },26 kimi: { text: "kimi-k2.6", reasoning: "kimi-k3", vision: "kimi-k2.6" },27 openrouter: { text: "openai/gpt-5.4-nano", reasoning: "openai/gpt-5.4-nano", vision: "openai/gpt-5.4-nano" },28 cerebras: { text: "gemma-4-31b", reasoning: "gpt-oss-120b", vision: "gemma-4-31b" },29 custom: { text: "", reasoning: "", vision: "" }, // per-user endpoints — never has an owner key, always skipped30};3132// 2x2 PNG (red, green / blue, white) — 32x32 needed for xAI minimums, so we scale via a bigger canvas.33function makePng(size: number): string {34 // Simple uncompressed PNG writer (RGB) for a solid red square with a white diagonal.35 const crcTable: number[] = [];36 for (let n = 0; n < 256; n++) {37 let c = n;38 for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;39 crcTable[n] = c >>> 0;40 }41 const crc = (buf: Buffer) => {42 let c = 0xffffffff;43 for (const b of buf) c = crcTable[(c ^ b) & 0xff] ^ (c >>> 8);44 return (c ^ 0xffffffff) >>> 0;45 };46 const chunk = (type: string, data: Buffer) => {47 const len = Buffer.alloc(4);48 len.writeUInt32BE(data.length);49 const td = Buffer.concat([Buffer.from(type), data]);50 const c = Buffer.alloc(4);51 c.writeUInt32BE(crc(td));52 return Buffer.concat([len, td, c]);53 };54 const ihdr = Buffer.alloc(13);55 ihdr.writeUInt32BE(size, 0);56 ihdr.writeUInt32BE(size, 4);57 ihdr[8] = 8;58 ihdr[9] = 2;59 const raw = Buffer.alloc((size * 3 + 1) * size);60 for (let y = 0; y < size; y++) {61 raw[y * (size * 3 + 1)] = 0;62 for (let x = 0; x < size; x++) {63 const o = y * (size * 3 + 1) + 1 + x * 3;64 const diag = Math.abs(x - y) < 2;65 raw[o] = diag ? 255 : 220;66 raw[o + 1] = diag ? 255 : 30;67 raw[o + 2] = diag ? 255 : 30;68 }69 }70 return Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), chunk("IHDR", ihdr), chunk("IDAT", zlib.deflateSync(raw)), chunk("IEND", Buffer.alloc(0))]).toString("base64");71}7273type Cell = "✅" | "❌" | "—";74const ROWS = ["Auth", "Model list", "Text", "Streaming", "System", "Vision", "Tools", "Structured output", "Reasoning", "Token usage", "Invalid key", "Error normalization", "Long response"] as const;75type Row = (typeof ROWS)[number];7677async function collect(req: UnifiedChatRequest, attempt = 1): Promise<Awaited<ReturnType<typeof collectOnce>>> {78 try {79 return await collectOnce(req);80 } catch (e) {81 const poly = (e as { poly?: { code: string; retryAfterMs?: number } }).poly;82 if (poly && (poly.code === "RATE_LIMITED" || poly.code === "PROVIDER_UNAVAILABLE") && attempt <= 3) {83 await new Promise((r) => setTimeout(r, Math.max(3000, poly.retryAfterMs ?? 0) * attempt));84 return collect(req, attempt + 1);85 }86 throw e;87 }88}8990async function collectOnce(req: UnifiedChatRequest) {91 const adapter = getAdapter(req.provider);92 const events: UnifiedStreamEvent[] = [];93 for await (const ev of adapter.streamChat(req)) events.push(ev);94 const err = events.find((e) => e.type === "error");95 if (err && err.type === "error") throw Object.assign(new Error(err.error.message), { poly: err.error });96 const text = events.filter((e) => e.type === "text-delta").map((e) => (e as { text: string }).text).join("");97 const reasoning = events.filter((e) => e.type === "reasoning-delta").map((e) => (e as { text: string }).text).join("");98 const usage = events.find((e) => e.type === "usage") as { usage: { inputTokens: number; outputTokens: number; reasoningTokens?: number } } | undefined;99 const tools = events.filter((e) => e.type === "tool-end") as { name: string; arguments: Record<string, unknown> }[];100 const deltas = events.filter((e) => e.type === "text-delta" || e.type === "reasoning-delta").length;101 return { text, reasoning, usage: usage?.usage, tools, deltas, finish: (events.find((e) => e.type === "finish") as { reason?: string } | undefined)?.reason };102}103104async function runProvider(p: ProviderId, results: Record<Row, Record<ProviderId, Cell>>, notes: string[]) {105 const key = KEYS[p];106 const adapter = getAdapter(p);107 const set = (r: Row, v: Cell) => (results[r][p] = v);108 if (!key) {109 for (const r of ROWS) set(r, "—");110 notes.push(`${p}: no key in env`);111 return;112 }113 const note = (s: string) => notes.push(`${p}: ${s}`);114 const m = TEST_MODELS[p];115 let models: PolyModel[] = [];116 try {117 const v = await adapter.validateApiKey(key);118 set("Auth", v.ok ? "✅" : "❌");119 if (!v.ok) note(`auth: ${v.error?.code} ${v.error?.message}`);120 } catch (e) {121 set("Auth", "❌");122 note(`auth threw: ${(e as Error).message}`);123 }124 try {125 models = await adapter.listModels(key);126 set("Model list", models.length ? "✅" : "❌");127 note(`models: ${models.length} (${models.slice(0, 5).map((x) => x.id).join(", ")}…)`);128 } catch (e) {129 set("Model list", "❌");130 note(`list: ${(e as Error).message}`);131 }132 // Aliases such as `claude-haiku-4-5` resolve to the dated snapshot listed by the provider.133 const info = (id: string) => models.find((x) => x.id === id) ?? models.find((x) => x.id.startsWith(`${id}-`));134 const base = (model: string): UnifiedChatRequest => ({ provider: p, model, apiKey: key, messages: [{ role: "user", content: [{ type: "text", text: "Reply with exactly: OK" }] }], settings: { maxTokens: 2000 }, modelInfo: info(model) });135136 try {137 const r = await collect(base(m.text));138 set("Text", r.text.trim().length ? "✅" : "❌");139 set("Streaming", r.deltas >= 1 ? "✅" : "❌");140 set("Token usage", r.usage && r.usage.inputTokens > 0 && r.usage.outputTokens > 0 ? "✅" : "❌");141 note(`text(${m.text}): "${r.text.trim().slice(0, 40)}" deltas=${r.deltas} usage=${JSON.stringify(r.usage)} finish=${r.finish}`);142 } catch (e) {143 set("Text", "❌");144 set("Streaming", "❌");145 set("Token usage", "❌");146 note(`text: ${(e as Error).message}`);147 }148 try {149 const r = await collect({ ...base(m.text), system: "You are a pirate. Always start your reply with 'Arr'.", messages: [{ role: "user", content: [{ type: "text", text: "Say hello in five words." }] }] });150 set("System", /arr/i.test(r.text) ? "✅" : "❌");151 note(`system: "${r.text.trim().slice(0, 60)}"`);152 } catch (e) {153 set("System", "❌");154 note(`system: ${(e as Error).message}`);155 }156 try {157 const r = await collect({ ...base(m.vision), messages: [{ role: "user", content: [{ type: "image", mimeType: "image/png", data: makePng(64) }, { type: "text", text: "What is the dominant color of this image? One word." }] }] });158 set("Vision", /red/i.test(r.text) ? "✅" : "❌");159 note(`vision: "${r.text.trim().slice(0, 40)}"`);160 } catch (e) {161 set("Vision", "❌");162 note(`vision: ${(e as Error).message}`);163 }164 try {165 const r = await collect({ ...base(m.text), messages: [{ role: "user", content: [{ type: "text", text: "What is 1234 * 5678? Use the calculator tool." }] }], tools: [{ name: "calculator", description: "Evaluate an arithmetic expression", parameters: { type: "object", properties: { expression: { type: "string" } }, required: ["expression"], additionalProperties: false }, strict: true }] });166 set("Tools", r.tools.length && r.tools[0].name === "calculator" ? "✅" : "❌");167 note(`tools: ${JSON.stringify(r.tools.map((t) => ({ n: t.name, a: t.arguments })))} finish=${r.finish}`);168 } catch (e) {169 set("Tools", "❌");170 note(`tools: ${(e as Error).message}`);171 }172 try {173 const r = await collect({ ...base(m.text), settings: { maxTokens: 2000, responseFormat: { type: "json_schema", schema: { type: "object", properties: { city: { type: "string" }, country: { type: "string" } }, required: ["city", "country"], additionalProperties: false }, schemaName: "city" } }, messages: [{ role: "user", content: [{ type: "text", text: "Give the capital of Canada as JSON." }] }] });174 let ok = false;175 try {176 const j = JSON.parse(r.text.trim().replace(/^```(?:json)?\s*|\s*```$/g, ""));177 ok = typeof j.city === "string" && typeof j.country === "string";178 } catch {179 ok = false;180 }181 set("Structured output", ok ? "✅" : "❌");182 note(`structured: ${r.text.trim().slice(0, 80)}`);183 } catch (e) {184 set("Structured output", "❌");185 note(`structured: ${(e as Error).message}`);186 }187 try {188 const r = await collect({ ...base(m.reasoning), settings: { maxTokens: 6000, reasoningEffort: p === "anthropic" ? "high" : "low", includeReasoning: true }, messages: [{ role: "user", content: [{ type: "text", text: "How many prime numbers are there strictly between 1000 and 1100? Think it through carefully, then answer with just the number." }] }] });189 const hasReasoning = r.reasoning.length > 0 || (r.usage?.reasoningTokens ?? 0) > 0;190 set("Reasoning", hasReasoning ? "✅" : "❌");191 note(`reasoning(${m.reasoning}): text="${r.text.trim().slice(0, 30)}" reasoningChars=${r.reasoning.length} reasoningTokens=${r.usage?.reasoningTokens}`);192 } catch (e) {193 set("Reasoning", "❌");194 note(`reasoning: ${(e as Error).message}`);195 }196 try {197 const badKeys: Record<string, string> = { gemini: "AIzaSyInvalidKey0000000000000000000000", anthropic: "sk-ant-api03-invalid", xai: "xai-invalid", mistral: "invalidinvalidinvalidinvalid0000", openrouter: "sk-or-v1-invalid0000000000000000", cerebras: "csk-invalid00000000000000000000" };198 const bad = await adapter.validateApiKey(badKeys[p] ?? "sk-invalid-0000000000000000");199 set("Invalid key", !bad.ok && bad.error?.code === "INVALID_API_KEY" ? "✅" : "❌");200 note(`invalid key → ${bad.error?.code} (${bad.error?.status})`);201 } catch (e) {202 set("Invalid key", "❌");203 note(`invalid key threw: ${(e as Error).message}`);204 }205 try {206 await collect(base("definitely-not-a-model-xyz"));207 set("Error normalization", "❌");208 } catch (e) {209 const poly = (e as { poly?: { code: string } }).poly;210 set("Error normalization", poly?.code === "MODEL_NOT_FOUND" || poly?.code === "INVALID_PARAMETER" ? "✅" : "❌");211 note(`unknown model → ${poly?.code}: ${(e as Error).message.slice(0, 80)}`);212 }213 try {214 const r = await collect({ ...base(m.text), settings: { maxTokens: 3000 }, messages: [{ role: "user", content: [{ type: "text", text: "Write a 400-word story about a lighthouse keeper. Plain prose." }] }] });215 set("Long response", r.text.split(/\s+/).length > 250 ? "✅" : "❌");216 note(`long: ${r.text.split(/\s+/).length} words, ${r.deltas} deltas`);217 } catch (e) {218 set("Long response", "❌");219 note(`long: ${(e as Error).message}`);220 }221}222223async function main() {224 const only = process.argv[2] as ProviderId | undefined;225 const providers = only ? [only] : [...PROVIDER_IDS];226 const results = Object.fromEntries(ROWS.map((r) => [r, Object.fromEntries(PROVIDER_IDS.map((p) => [p, "—" as Cell]))])) as Record<Row, Record<ProviderId, Cell>>;227 const notes: string[] = [];228 await Promise.all(providers.map((p) => runProvider(p, results, notes)));229 // Merge with a previous full matrix so single-provider runs do not blank the others.230 if (only && fs.existsSync("docs/provider-test-matrix.md")) {231 const prev = fs.readFileSync("docs/provider-test-matrix.md", "utf8");232 for (const r of ROWS) {233 const m = prev.match(new RegExp(`^\\| ${r.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} \\| (.+) \\|$`, "m"));234 if (!m) continue;235 const cells = m[1].split(" | ") as Cell[];236 PROVIDER_IDS.forEach((pp, i) => {237 if (pp !== only && cells[i]) results[r][pp] = cells[i];238 });239 }240 }241 const date = new Date().toISOString().slice(0, 10);242 const header = `| Capability | ${PROVIDER_IDS.map((p) => p).join(" | ")} |\n| --- | ${PROVIDER_IDS.map(() => ":-:").join(" | ")} |`;243 const lines = ROWS.map((r) => `| ${r} | ${PROVIDER_IDS.map((p) => results[r][p]).join(" | ")} |`);244 const md = `# Provider test matrix\n\nGenerated by \`pnpm providers:matrix\` against the REAL provider APIs on ${date}. A cell is ✅ only when the live request succeeded and the assertion passed. “—” = not run (no key).\n\nTest models: ${PROVIDER_IDS.map((p) => `${p}: ${TEST_MODELS[p].text} / reasoning ${TEST_MODELS[p].reasoning}`).join("; ")}.\n\n${header}\n${lines.join("\n")}\n\n## Notes\n\n${notes.map((n) => `- ${n.replace(/\n/g, " ")}`).join("\n")}\n`;245 fs.mkdirSync("docs", { recursive: true });246 fs.writeFileSync("docs/provider-test-matrix.md", md);247 console.log(md);248}249250main().catch((e) => {251 console.error(e);252 process.exit(1);253});254