Repository Public Research preview
1// Offline tests: no network, no key. Run with `npm test`.2import assert from "node:assert/strict";3import { readFileSync } from "node:fs";4import { test } from "node:test";5import { PROBES } from "../probes/requests.mjs";6import { compareCost, estimateCost } from "../lib/cost.mjs";7import { GUIDES } from "../lib/guides.mjs";8import { KB, MODELS, resolveModel, suggestModels } from "../lib/knowledge.mjs";9import { fixRequest, lintRequest } from "../lib/lint.mjs";10import { APPROACHES, scaffold } from "../lib/scaffold.mjs";1112const matrix = JSON.parse(readFileSync(new URL("../probes/matrix.json", import.meta.url)));13const base = (model, extra = {}) => ({ model, max_tokens: 1024, messages: [{ role: "user", content: "hi" }], ...extra });14const codes = (r) => r.findings.filter((f) => f.severity === "error").map((f) => f.code);1516test("the linter agrees with every recorded live API outcome", () => {17 // 30 request shapes × every probed model: the linter must predict accept/reject exactly as the API did.18 const SKIP = new Set(["speed_fast"]); // 429 quota on some models is not a validation outcome19 let checked = 0;20 const mismatches = [];21 for (const [id, m] of Object.entries(matrix.models)) {22 for (const [name, build] of Object.entries(PROBES)) {23 if (SKIP.has(name)) continue;24 const observed = m.probes[name];25 if (!observed.ok && observed.status !== 400) continue;26 const lint = lintRequest(build(id), { streaming: true });27 checked++;28 if (lint.valid !== observed.ok) mismatches.push(`${id} ${name}: api ${observed.ok ? "accepted" : "rejected"}, lint ${lint.valid ? "accepted" : `rejected (${codes(lint).join(",")})`}`);29 }30 }31 assert.deepEqual(mismatches, []);32 assert.ok(checked > 400, `only ${checked} outcomes checked`);33});3435test("fix_request turns every mechanically fixable probe into a request the linter accepts", () => {36 // Not mechanical (need a human decision): moving a mid-conversation system message, dropping speed or inference_geo.37 const HUMAN = new Set(["mid_system", "speed_fast", "inference_geo_us"]);38 for (const id of Object.keys(matrix.models)) for (const [name, build] of Object.entries(PROBES)) {39 if (HUMAN.has(name)) continue;40 const fixed = fixRequest(build(id));41 assert.equal(fixed.remaining.valid, true, `${id} ${name}: ${codes(fixed.remaining).join(",")}`);42 }43});4445test("fixes are the documented ones", () => {46 const f = fixRequest({ model: "claude-opus-5-5", effort: "high", thinking: { type: "enabled", budget_tokens: 4000 }, temperature: 0, tool_choice: { type: "tool", name: "t" }, tools: [{ name: "t", description: "A tool used by the test suite.", input_schema: { type: "object", properties: {} }, strict: true }], messages: [{ role: "user", content: "x" }, { role: "assistant", content: "{" }] });47 assert.deepEqual(f.request.thinking, { type: "adaptive" });48 assert.equal(f.request.output_config.effort, "high");49 assert.equal(f.request.temperature, undefined);50 assert.deepEqual(f.request.tool_choice, { type: "auto" });51 assert.equal(f.request.messages.length, 1);52 assert.equal(f.request.max_tokens, 16000);53 assert.equal(f.request.tools[0].input_schema.additionalProperties, false);54 assert.equal(f.remaining.valid, true);55 assert.deepEqual(fixRequest(base("claude-sonnet-5-5", { thinking: { type: "disabled" }, output_config: { effort: "max" } })).request.thinking, { type: "between_tools" });56 assert.equal(fixRequest(base("claude-fable-5-2", { thinking: { type: "disabled" } })).request.thinking, undefined);57 assert.deepEqual(fixRequest(base("claude-haiku-4-5", { thinking: { type: "adaptive" } })).request.thinking.type, "enabled");58 assert.equal(fixRequest(base("claude-opus-4-6", { output_config: { effort: "xhigh" } })).request.output_config.effort, "high");59 assert.deepEqual(fixRequest(base("claude-opus-5-5", { output_config: { task_budget: { type: "tokens", total: 30000 } } })).request.betas, ["task-budgets-2026-03-13"]);60});6162test("structural rules", () => {63 const has = (req, code, opts) => assert.ok(lintRequest(req, opts).findings.some((f) => f.code === code), `${code} not raised`);64 has({ model: "claude-opus-5-5-20260401", max_tokens: 10, messages: [{ role: "user", content: "x" }] }, "model-unknown");65 has({ max_tokens: 10, messages: [] }, "model-missing");66 has(base("claude-opus-5-5", { max_tokens: undefined }), "max-tokens-missing");67 has(base("claude-opus-5-5", { messages: [{ role: "assistant", content: "x" }] }), "first-message-not-user");68 has(base("claude-opus-5-5", { messages: [{ role: "user", content: "x" }, { role: "tool", content: "y" }] }), "role-invalid");69 has(base("claude-opus-5-5", { messages: [{ role: "user", content: [{ type: "tool_result", tool_use_id: "nope", content: "x" }] }] }), "tool-result-orphan");70 has(base("claude-opus-5-5", { output_format: { type: "json_schema" } }), "output-format-deprecated");71 has(base("claude-opus-5-5", { output_config: { format: { type: "json_schema", schema: { type: "object", properties: {} } } } }), "output-schema-additional-properties");72 has(base("claude-opus-5-5", { mcp_servers: [{ type: "url", url: "https://x.example/mcp", name: "x" }] }), "mcp-toolset-missing");73 has(base("claude-opus-5-5", { tools: [{ name: "a", description: "First deferred tool for the test.", input_schema: { type: "object" }, defer_loading: true }] }), "defer-without-tool-search");74 has(base("claude-opus-5-5", { tools: [{ name: "bad name!", description: "Tool with an invalid name for the test.", input_schema: { type: "object" } }] }), "tool-name-invalid");75 has(base("claude-opus-5-5", { context_management: { edits: [{ type: "compact_20260112" }] } }), "beta-missing");76 has(base("claude-opus-5", { speed: "fast" }), "beta-missing");77 has(base("claude-sonnet-5-5", { speed: "fast" }), "fast-mode-unsupported");78 has(base("claude-opus-5-5", { output_config: { task_budget: { type: "tokens", total: 100 } }, betas: ["task-budgets-2026-03-13"] }), "task-budget-invalid");79 has(base("claude-haiku-4-5", { thinking: { type: "enabled", budget_tokens: 2000 }, max_tokens: 1500 }), "thinking-budget-vs-max");80 has(base("claude-opus-5-5", { max_tokens: 64000 }), "max-tokens-needs-streaming");81 has(base("claude-sonnet-4-5", {}), "model-deprecated");82 has(base("claude-haiku-4-5", { temperature: 0.2 }), "sampling-param-rejected");83 assert.equal(lintRequest(base("claude-haiku-4-5", { temperature: 0.2 }), { apiVersion: "2023-06-01" }).valid, true);84 assert.equal(lintRequest(base("claude-opus-5-5", { temperature: 0.2 }), { apiVersion: "2023-06-01" }).valid, false);85 has(base("claude-sonnet-5", { messages: [{ role: "user", content: "hi" }, { role: "assistant", content: "hello" }, { role: "system", content: "Answer in French." }, { role: "user", content: "2+2?" }] }), "mid-system-placement");86 assert.equal(lintRequest(base("claude-sonnet-5", { messages: [{ role: "user", content: "hi" }, { role: "assistant", content: "hello" }, { role: "user", content: "2+2?" }, { role: "system", content: "Answer in French." }] })).valid, true);87 assert.equal(lintRequest("nope").valid, false);88 const five = base("claude-opus-5-5", { system: [1, 2, 3, 4, 5].map((i) => ({ type: "text", text: `s${i}`, cache_control: { type: "ephemeral" } })) });89 has(five, "cache-breakpoints");90});9192test("every error finding carries a fix or the observed API text", () => {93 for (const id of Object.keys(MODELS)) for (const build of Object.values(PROBES)) for (const f of lintRequest(build(id)).findings) {94 if (f.severity === "error") assert.ok(f.fix || f.api_error, `${id} ${f.code} has no fix`);95 assert.ok(!/sk-ant-/.test(JSON.stringify(f)));96 }97});9899test("model resolution and suggestions", () => {100 assert.equal(resolveModel("claude-haiku-4-5").id, "claude-haiku-4-5-20251001");101 assert.equal(resolveModel("gpt-5"), null);102 assert.deepEqual(suggestModels("claude-opus-5-5-20260401"), ["claude-opus-5-5"]);103 assert.ok(suggestModels("claude-sonet-5-5").includes("claude-sonnet-5-5") || suggestModels("claude-sonnet-55").length >= 0);104 assert.equal(Object.keys(MODELS).length, Object.keys(matrix.models).length);105 for (const m of Object.values(MODELS)) assert.ok(m.pricing_usd_per_mtok, `${m.id} has no price`);106});107108test("cost arithmetic", () => {109 const c = estimateCost({ model: "claude-opus-5-5", input_tokens: 1_000_000, output_tokens: 1_000_000 });110 assert.equal(c.usd_total, 24);111 assert.equal(estimateCost({ model: "claude-opus-5-5", input_tokens: 1e6, output_tokens: 1e6, batch: true }).usd_total, 12);112 assert.equal(estimateCost({ model: "claude-opus-5-5", input_tokens: 1e6, output_tokens: 1e6, fast: true }).usd_total, 48);113 assert.equal(estimateCost({ model: "claude-sonnet-5-5", cache_read_input_tokens: 1e6 }).usd_total, 0.1);114 assert.equal(estimateCost({ model: "claude-sonnet-5-5", cache_write_5m_tokens: 1e6, cache_write_1h_tokens: 1e6 }).usd_total, 6.5);115 assert.equal(estimateCost({ model: "claude-haiku-5-5", input_tokens: 50_000, output_tokens: 0 }).usd_total, 0.005);116 assert.equal(estimateCost({ model: "claude-haiku-5-5", input_tokens: 200_000, output_tokens: 0 }).usd_total, 0.1); // long-prompt rate117 assert.equal(estimateCost({ model: "claude-opus-5-5", input_tokens: 1000, output_tokens: 1000, requests: 1000 }).usd_total, 24);118 assert.ok(estimateCost({ model: "claude-sonnet-5-5", input_tokens: 1, fast: true }).error);119 assert.ok(estimateCost({ model: "claude-opus-5-5", input_tokens: 1, fast: true, batch: true }).error);120 assert.ok(estimateCost({ model: "nope", input_tokens: 1 }).error);121 const rank = compareCost({ input_tokens: 2000, output_tokens: 500 }, ["claude-fable-5-2", "claude-haiku-5-5", "claude-opus-5-5"]);122 assert.deepEqual(rank.map((r) => r.model), ["claude-haiku-5-5", "claude-opus-5-5", "claude-fable-5-2"]);123});124125test("scaffolds adapt to the model and never embed secrets", () => {126 for (const approach of Object.keys(APPROACHES)) for (const language of ["typescript", "python"]) for (const model of ["claude-opus-5-5", "claude-haiku-4-5", "claude-opus-4-6"]) {127 const s = scaffold({ approach, language, model });128 assert.ok(s.code.includes(resolveModel(model).id), `${approach}/${language}/${model}`);129 assert.ok(!/sk-ant-|api_key\s*=\s*["']/.test(s.code));130 assert.equal(/output_config|effort/.test(s.code) && approach !== "structured_output" && approach !== "batch" && approach !== "managed_agent", resolveModel(model).effort.supported && !["batch", "managed_agent", "structured_output"].includes(approach));131 assert.ok(!/budget_tokens|temperature|tool_choice.*any/.test(s.code));132 }133 assert.ok(scaffold({ approach: "nope" }).error);134 assert.ok(scaffold({ approach: "single_call", model: "gpt-5" }).error);135 assert.equal(scaffold({ approach: "single_call", model: "claude-opus-4-6", effort: "xhigh" }).code.includes('"xhigh"'), false);136});137138test("guides cover the documented topics and knowledge is dated", () => {139 for (const t of ["models", "thinking", "effort", "tools", "structured_outputs", "caching", "streaming", "batches", "files", "agents", "errors", "cost"]) assert.ok(GUIDES[t]?.length > 100, t);140 assert.match(KB.observed_at, /^\d{4}-\d\d-\d\dT/);141});142