Repository Public Research preview
1// Second matrix: beta features and the legacy API version. Same rules as probe.mjs.2import Anthropic from "@anthropic-ai/sdk";3import { writeFileSync } from "node:fs";4const client = new Anthropic({ maxRetries: 3 });5// Account identifiers never belong in a committed result file.6const scrub = (s) => String(s).replace(/org: [0-9a-f-]{36}/g, "org: <redacted>");7const legacy = new Anthropic({ maxRetries: 3, defaultHeaders: { "anthropic-version": "2023-06-01" } });8const user = [{ role: "user", content: "Reply with the single word: ok" }];9const tool = { name: "get_time", description: "Return the current time.", input_schema: { type: "object", properties: { tz: { type: "string" } }, required: ["tz"] } };10const P = {11 legacy_temperature: (m) => [legacy.messages, { model: m, max_tokens: 64, temperature: 0.5, messages: user }],12 legacy_top_p: (m) => [legacy.messages, { model: m, max_tokens: 64, top_p: 0.9, messages: user }],13 legacy_prefill: (m) => [legacy.messages, { model: m, max_tokens: 64, messages: [...user, { role: "assistant", content: "o" }] }],14 task_budget: (m) => [client.beta.messages, { model: m, max_tokens: 256, betas: ["task-budgets-2026-03-13"], output_config: { task_budget: { type: "tokens", total: 20000 } }, messages: user }],15 compaction: (m) => [client.beta.messages, { model: m, max_tokens: 64, betas: ["compact-2026-01-12"], context_management: { edits: [{ type: "compact_20260112" }] }, messages: user }],16 context_edit_tools: (m) => [client.beta.messages, { model: m, max_tokens: 64, betas: ["context-management-2025-06-27"], context_management: { edits: [{ type: "clear_tool_uses_20250919" }] }, tools: [tool], messages: user }],17 fallbacks_default: (m) => [client.beta.messages, { model: m, max_tokens: 64, betas: ["server-side-fallback-2026-07-01"], fallbacks: "default", messages: user }],18 fast_beta: (m) => [client.beta.messages, { model: m, max_tokens: 64, betas: ["fast-mode-2026-02-01"], speed: "fast", messages: user }],19 display_updates: (m) => [client.beta.messages, { model: m, max_tokens: 256, betas: ["thinking-display-updates-2026-08-18"], thinking: { type: "adaptive", display: "updates" }, messages: user }],20 memory_tool: (m) => [client.messages, { model: m, max_tokens: 64, tools: [{ type: "memory_20250818", name: "memory" }], messages: user }],21 bash_tool: (m) => [client.messages, { model: m, max_tokens: 64, tools: [{ type: "bash_20250124", name: "bash" }], messages: user }],22 text_editor_tool: (m) => [client.messages, { model: m, max_tokens: 64, tools: [{ type: "text_editor_20250728", name: "str_replace_based_edit_tool" }], messages: user }],23 web_fetch_2026: (m) => [client.messages, { model: m, max_tokens: 64, tools: [{ type: "web_fetch_20260209", name: "web_fetch", max_uses: 1 }], messages: user }],24 tool_search_bm25: (m) => [client.messages, { model: m, max_tokens: 64, tools: [{ type: "tool_search_tool_bm25_20251119", name: "tool_search_tool_bm25" }, { ...tool, defer_loading: true }, { name: "noop", description: "Does nothing.", input_schema: { type: "object", properties: {} } }], messages: user }],25 image_input: (m) => [client.messages, { model: m, max_tokens: 64, messages: [{ role: "user", content: [{ type: "image", source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEElEQVR4nGP4z8CAFWEXHbQSACj/P8Fu7N9hAAAAAElFTkSuQmCC" } }, { type: "text", text: "One word: what colour is this square?" }] }] }],26 citations_doc: (m) => [client.messages, { model: m, max_tokens: 128, messages: [{ role: "user", content: [{ type: "document", source: { type: "text", media_type: "text/plain", data: "The registry was launched on 9 November 2026." }, citations: { enabled: true } }, { type: "text", text: "When was the registry launched? One sentence." }] }] }],27};28const models = [];29for await (const m of client.models.list({ limit: 100 })) models.push(m.id);30const out = { generated_at: new Date().toISOString(), models: {} };31await Promise.all(models.map(async (m) => {32 const res = {};33 for (const [name, f] of Object.entries(P)) {34 const [api, req] = f(m);35 try { const r = await api.create(req, { timeout: 120_000 }); res[name] = { ok: true, stop_reason: r.stop_reason, blocks: r.content.map((b) => b.type), model: r.model, text: r.content.find((b) => b.type === "text")?.text?.slice(0, 50), speed: r.usage?.speed, citations: r.content.some((b) => b.citations?.length) || undefined }; }36 catch (e) { res[name] = { ok: false, status: e.status ?? null, message: scrub(e.error?.error?.message ?? e.message ?? "").slice(0, 260) }; }37 }38 out.models[m] = res;39}));40writeFileSync(new URL("./matrix2.json", import.meta.url), JSON.stringify(out, null, 2));41const names = Object.keys(P);42for (const id of Object.keys(out.models).sort()) console.log(id.padEnd(28), names.map((n) => (out.models[id][n].ok ? "✓" : out.models[id][n].status ?? "E")).join(" "));43console.log("\ncolumns:", names.join(", "));44const seen = {};45for (const [id, r] of Object.entries(out.models)) for (const [n, v] of Object.entries(r)) if (!v.ok) (seen[`${n} | ${v.message.replace(id, "<model>").slice(0, 150)}`] ??= []).push(id.replace("claude-", ""));46for (const [k, v] of Object.entries(seen)) console.log(v.length, "|", k, "|", v.length < 7 ? v.sort().join(" ") : "(many)");47const img = Object.entries(out.models).map(([id, r]) => `${id.replace("claude-", "")}:${r.image_input.text}`).join(" · ");48console.log("\nimage:", img);49