spb/llm-api-toolkits

1// Empirical behaviour matrix for OpenAI text models. Usage: OPENAI_API_KEY=… node probes/probe.mjs [model,model]2import OpenAI from "openai";3import { readFileSync, writeFileSync } from "node:fs";4import { PROBES, TEXT_MODELS } from "./requests.mjs";56const client = new OpenAI({ maxRetries: 3, timeout: 180_000 });7// Account identifiers never belong in a committed result file.8const scrub = (s) => String(s ?? "").replace(/org-[A-Za-z0-9]{10,}/g, "org-<redacted>").replace(/proj_[A-Za-z0-9]{10,}/g, "proj_<redacted>").replace(/req_[a-f0-9]{16,}/g, "req_<redacted>");910async function run(model, name) {11  const [api, build] = PROBES[name];12  const req = build(model);13  try {14    if (api === "responses") {15      const r = await client.responses.create(req);16      if (req.background && r.id) client.responses.cancel(r.id).catch(() => {});17      return { ok: true, status: r.status, incomplete: r.incomplete_details?.reason ?? null, items: r.output.map((o) => o.type), text: (r.output_text ?? "").slice(0, 50), usage: r.usage ? { in: r.usage.input_tokens, out: r.usage.output_tokens, reasoning: r.usage.output_tokens_details?.reasoning_tokens ?? 0 } : null, tier: r.service_tier ?? null };18    }19    const r = await client.chat.completions.create(req);20    return { ok: true, finish: r.choices[0]?.finish_reason, choices: r.choices.length, text: (r.choices[0]?.message?.content ?? "").slice(0, 50), tool_calls: r.choices[0]?.message?.tool_calls?.length ?? 0 };21  } catch (e) {22    return { ok: false, status: e.status ?? null, code: e.code ?? null, param: e.param ?? null, type: e.type ?? e.name, message: scrub(e.error?.message ?? e.message).slice(0, 280) };23  }24}2526const only = process.argv[2] && process.argv[2] !== "all" ? process.argv[2].split(",") : undefined;27// PROBES=name,name re-runs only those probes and merges them into the existing matrix.28const PROBE_NAMES = process.env.PROBES ? process.env.PROBES.split(",") : Object.keys(PROBES);29const visible = new Set();30for await (const m of client.models.list()) visible.add(m.id);31const out = { generated_at: new Date().toISOString(), sdk: "openai (official Node SDK)", models: {} };32const todo = TEXT_MODELS.filter((m) => visible.has(m) && (!only || only.includes(m)));33let i = 0;34async function worker() {35  while (i < todo.length) {36    const m = todo[i++];37    const res = {};38    if (process.env.SERIAL) { for (const name of PROBE_NAMES) { res[name] = await run(m, name); await new Promise((r) => setTimeout(r, 1500)); } }39    else await Promise.all(PROBE_NAMES.map(async (name) => { res[name] = await run(m, name); }));40    out.models[m] = { probes: res };41    process.stderr.write(`done ${m}\n`);42  }43}44await Promise.all([worker(), worker(), worker(), worker()]);45if (process.env.PROBES) {46  const full = JSON.parse(readFileSync(new URL("./matrix.json", import.meta.url)));47  for (const [id, v] of Object.entries(out.models)) Object.assign(full.models[id].probes, v.probes);48  writeFileSync(new URL("./matrix.json", import.meta.url), JSON.stringify(full, null, 1));49} else writeFileSync(new URL(only ? "./matrix.partial.json" : "./matrix.json", import.meta.url), JSON.stringify(out, null, 1));50const names = PROBE_NAMES;51for (const id of todo) console.log(id.padEnd(15), names.map((n) => (out.models[id].probes[n].ok ? "✓" : String(out.models[id].probes[n].status ?? "E")[0])).join(""));52console.log("\ncolumns:", names.map((n, k) => `${k + 1}:${n}`).join(" "));53