spb/llm-api-toolkits

1// Empirical behaviour matrix: sends small requests to every model the key can see and records2// what the API accepts or rejects. The linter rules and the skill's model reference are built3// from this file, not from memory. Usage: ANTHROPIC_API_KEY=... node probes/probe.mjs4import Anthropic from "@anthropic-ai/sdk";5import { writeFileSync } from "node:fs";6import { PROBES, user } from "./requests.mjs";78const client = new Anthropic({ maxRetries: 3 });9// Account identifiers never belong in a committed result file.10const scrub = (s) => String(s).replace(/org: [0-9a-f-]{36}/g, "org: <redacted>");11async function run(model, name) {12  const req = PROBES[name](model);13  try {14    const r = await client.messages.create(req, { timeout: 120_000 });15    return { ok: true, stop_reason: r.stop_reason, blocks: r.content.map((b) => b.type), usage: { in: r.usage.input_tokens, out: r.usage.output_tokens }, text: r.content.find((b) => b.type === "text")?.text?.slice(0, 60) };16  } catch (e) {17    return { ok: false, status: e.status ?? null, type: e.error?.error?.type ?? e.name, message: scrub(e.error?.error?.message ?? e.message ?? "").slice(0, 300) };18  }19}20async function count(model) {21  try { const r = await client.messages.countTokens({ model, messages: user }); return { ok: true, input_tokens: r.input_tokens }; }22  catch (e) { return { ok: false, status: e.status, message: (e.error?.error?.message ?? e.message).slice(0, 200) }; }23}2425const only = process.argv[2]?.split(",");26const models = [];27for await (const m of client.models.list({ limit: 100 })) models.push(m);28const out = { generated_at: new Date().toISOString(), sdk: "@anthropic-ai/sdk", models: {} };29await Promise.all(models.filter((m) => !only || only.includes(m.id)).map(async (m) => {30  const res = { display_name: m.display_name, created_at: m.created_at, max_input_tokens: m.max_input_tokens, max_tokens: m.max_tokens, capabilities: m.capabilities, probes: {} };31  for (const name of Object.keys(PROBES)) res.probes[name] = await run(m.id, name);32  res.probes.count_tokens = await count(m.id);33  out.models[m.id] = res;34  process.stderr.write(`done ${m.id}\n`);35}));36writeFileSync(new URL("./matrix.json", import.meta.url), JSON.stringify(out, null, 2));37const names = [...Object.keys(PROBES), "count_tokens"];38for (const id of Object.keys(out.models).sort()) console.log(id.padEnd(28), names.map((n) => (out.models[id].probes[n].ok ? "✓" : out.models[id].probes[n].status ?? "E")).join(" "));39console.log("\ncolumns:", names.join(", "));40