Repository Public Research preview
1// Offline tests: no network, no key. Run with `npm test`.2import assert from "node:assert/strict";3import { readFileSync } from "node:fs";4import { test } from "node:test";5import { compareCost, estimateCost } from "../lib/cost.mjs";6import { GUIDES } from "../lib/guides.mjs";7import { KB, MODELS, resolveModel, suggestModels } from "../lib/knowledge.mjs";8import { chatToResponses, detectApi, fixRequest, lintRequest } from "../lib/lint.mjs";9import { APPROACHES, scaffold } from "../lib/scaffold.mjs";10import { PROBES } from "../probes/requests.mjs";1112const matrix = JSON.parse(readFileSync(new URL("../probes/matrix.json", import.meta.url)));13const errs = (r) => r.findings.filter((f) => f.severity === "error").map((f) => f.code);14const accepted = (o) => o.ok || /Could not finish the message because max_tokens/.test(o.message ?? "");1516test("the linter agrees with every recorded live API outcome", () => {17 let checked = 0;18 const mismatches = [];19 for (const [id, m] of Object.entries(matrix.models)) for (const [name, [api, build]] of Object.entries(PROBES)) {20 const o = m.probes[name];21 if (!accepted(o) && ![400, 403, 404].includes(o.status)) continue; // 429 = capacity, not validation22 const lint = lintRequest(build(id), { api });23 checked++;24 if (lint.valid !== accepted(o)) mismatches.push(`${id} ${name}: api ${accepted(o) ? "accepted" : "rejected"}, lint ${lint.valid ? "accepted" : `rejected (${errs(lint)})`}`);25 }26 assert.deepEqual(mismatches, []);27 assert.ok(checked > 1500, `only ${checked} outcomes checked`);28});2930test("fix_request turns every mechanically fixable probe into a request the linter accepts", () => {31 // Not mechanical: a model that is not served on Chat Completions needs conversion (chat_to_responses).32 for (const id of Object.keys(matrix.models)) for (const [name, [api, build]] of Object.entries(PROBES)) {33 if (api === "chat" && !MODELS[id].chat_completions.supported) continue;34 if (name === "chat_tools" && MODELS[id].chat_completions.function_tools !== "yes") continue; // needs a human choice: Responses or effort none35 if (name === "chat_tools_effort_none" && MODELS[id].chat_completions.function_tools === "no") continue;36 if (name === "web_search" || name === "web_search_preview" || name === "code_interpreter") { if (MODELS[id].responses.hosted_tools_need_auto_choice) { const f = fixRequest(build(id), { api }); assert.equal(f.remaining.valid, true, `${id} ${name}`); continue; } }37 const f = fixRequest(build(id), { api });38 assert.equal(f.remaining.valid, true, `${id} ${name}: ${errs(f.remaining)}`);39 }40});4142test("chat_to_responses yields a valid Responses request for every chat probe and model", () => {43 for (const id of Object.keys(matrix.models)) for (const [name, [api, build]] of Object.entries(PROBES)) {44 if (api !== "chat") continue;45 const c = chatToResponses(build(id));46 assert.equal(c.remaining.valid, true, `${id} ${name}: ${errs(c.remaining)}`);47 assert.equal(detectApi(c.request), "responses");48 assert.equal(c.request.messages, undefined);49 }50 const c = chatToResponses({ model: "gpt-5.5-pro", messages: [{ role: "system", content: "Be brief." }, { role: "user", content: [{ type: "text", text: "Describe" }, { type: "image_url", image_url: { url: "https://example.org/a.png" } }] }, { role: "assistant", tool_calls: [{ id: "call_1", type: "function", function: { name: "f", arguments: "{}" } }] }, { role: "tool", tool_call_id: "call_1", content: "done" }], max_tokens: 500, stop: ["x"], n: 2, tools: [{ type: "function", function: { name: "f", description: "A function used by the test suite.", parameters: { type: "object", properties: {} } } }], tool_choice: { type: "function", function: { name: "f" } }, response_format: { type: "json_schema", json_schema: { name: "o", schema: { type: "object", properties: { a: { type: "string" } }, required: ["a"], additionalProperties: false }, strict: true } } });51 assert.equal(c.request.instructions, "Be brief.");52 assert.deepEqual(c.request.input[0].content.map((x) => x.type), ["input_text", "input_image"]);53 assert.equal(c.request.input[1].type, "function_call");54 assert.deepEqual(c.request.input[2], { type: "function_call_output", call_id: "call_1", output: "done" });55 assert.equal(c.request.max_output_tokens, 500);56 assert.deepEqual(c.request.tool_choice, { type: "function", name: "f" });57 assert.equal(c.request.tools[0].name, "f");58 assert.equal(c.request.text.format.name, "o");59 assert.equal(c.notes.filter((n) => /no Responses API equivalent/.test(n)).length, 2);60 assert.equal(c.remaining.valid, true);61});6263test("documented fixes", () => {64 const f = fixRequest({ model: "gpt-6.1-sol", input: "hi", temperature: 0, max_tokens: 5, reasoning_effort: "minimal", response_format: { type: "json_schema", json_schema: { name: "x", schema: { type: "object", properties: {}, additionalProperties: false }, strict: true } }, tools: [{ type: "function", function: { name: "f", description: "A function used by the test suite.", parameters: { type: "object", properties: {} } } }], tool_choice: "any" });65 assert.deepEqual(f.request.reasoning, { effort: "low" });66 assert.equal(f.request.max_output_tokens, 16);67 assert.equal(f.request.temperature, undefined);68 assert.equal(f.request.text.format.name, "x");69 assert.equal(f.request.tools[0].name, "f");70 assert.equal(f.request.tool_choice, "required");71 assert.equal(f.remaining.valid, true);72 assert.equal(fixRequest({ model: "gpt-6-luna", input: "x", temperature: 0.2, reasoning: { effort: "none" } }).request.temperature, 0.2); // allowed with effort none73 assert.equal(fixRequest({ model: "gpt-6-luna", input: "x", temperature: 0.2 }).request.temperature, undefined);74 assert.equal(fixRequest({ model: "gpt-5.4", input: "x", temperature: 0.2 }).request.temperature, 0.2); // default effort is none75 assert.equal(fixRequest({ model: "gpt-5.4", input: "x", temperature: 0.2, reasoning: { effort: "high" } }).request.temperature, undefined);76 assert.equal(fixRequest({ model: "gpt-4o", input: "x", reasoning: { effort: "high" } }).request.reasoning, undefined);77 assert.equal(fixRequest({ model: "gpt-5-pro", input: "x", reasoning: { effort: "low" } }).request.reasoning.effort, "high");78 assert.equal(fixRequest({ model: "gpt-5", messages: [{ role: "user", content: "x" }], max_tokens: 100, stop: ["a"], temperature: 0.3 }).request.max_completion_tokens, 100);79});8081test("structural rules", () => {82 const has = (req, code, opts) => assert.ok(lintRequest(req, opts).findings.some((f) => f.code === code), `${code} not raised: ${errs(lintRequest(req, opts))}`);83 has({ model: "gpt-7-ultra", input: "x" }, "model-unknown");84 has({ input: "x" }, "model-missing");85 has({ model: "gpt-6-luna", messages: [{ role: "user", content: "x" }] }, "responses-uses-messages", { api: "responses" });86 has({ model: "gpt-6-luna", input: "x", response_format: { type: "json_object" } }, "responses-response-format");87 has({ model: "gpt-6-luna", input: "x", system: "y" }, "responses-system-field");88 has({ model: "gpt-6-luna", input: "x", text: { format: { type: "json_schema", name: "a", schema: { type: "object", properties: { a: { type: "string" } } } } } }, "strict-additional-properties");89 has({ model: "gpt-6-luna", input: "x", text: { format: { type: "json_schema", name: "a", schema: { type: "object", properties: { a: { type: "string" }, b: { type: "string" } }, required: ["a"], additionalProperties: false } } } }, "strict-required-all");90 has({ model: "gpt-6-luna", input: "x", text: { format: { type: "json_schema", schema: { type: "object", properties: {}, additionalProperties: false } } } }, "format-name-missing");91 has({ model: "gpt-6-luna", input: [{ role: "user", content: [{ type: "text", text: "x" }] }] }, "content-part-chat-shape");92 has({ model: "gpt-6-luna", input: [{ type: "function_call_output", call_id: "c1", output: "x" }] }, "tool-output-orphan");93 has({ model: "gpt-6-luna", input: "x", tools: [{ type: "function", function: { name: "f" } }] }, "tool-chat-shape");94 has({ model: "gpt-6-luna", messages: [{ role: "user", content: "x" }], tools: [{ type: "function", name: "f", description: "A function used by the test suite.", parameters: { type: "object" } }] }, "tool-responses-shape");95 has({ model: "gpt-6-luna", messages: [{ role: "user", content: "x" }], tools: [{ type: "web_search" }] }, "builtin-tool-on-chat");96 has({ model: "gpt-6-luna", input: "x", tools: [{ type: "code_interpreter" }] }, "code-interpreter-container");97 has({ model: "gpt-6-luna", input: "x", tools: [{ type: "file_search" }] }, "file-search-vector-store");98 has({ model: "gpt-6-luna", input: "x", tool_choice: "any", tools: [{ type: "function", name: "f", description: "A function used by the test suite.", parameters: { type: "object" } }] }, "tool-choice-any");99 has({ model: "gpt-6-luna", input: "x", background: true, store: false }, "background-needs-store");100 has({ model: "gpt-6-luna", input: "x", max_output_tokens: 200000 }, "max-output-over-cap");101 has({ model: "gpt-6-luna", input: "x", max_output_tokens: 100 }, "max-output-small-for-reasoning");102 has({ model: "gpt-5.5-pro", messages: [{ role: "user", content: "x" }] }, "chat-endpoint-unsupported");103 has({ model: "gpt-6.1-sol", messages: [{ role: "user", content: "x" }], tools: [{ type: "function", function: { name: "f", description: "A function used by the test suite.", parameters: { type: "object" } } }] }, "chat-tools-unsupported");104 assert.equal(lintRequest({ model: "gpt-5.4-2026-03-05", input: "x" }).model, "gpt-5.4"); // dated snapshots resolve to their alias105 assert.equal(lintRequest("nope").valid, false);106 assert.equal(detectApi({ model: "x", messages: [] }), "chat");107});108109test("every error finding carries a fix or the observed API text", () => {110 for (const id of Object.keys(MODELS)) for (const [, [api, build]] of Object.entries(PROBES)) for (const f of lintRequest(build(id), { api }).findings) {111 if (f.severity === "error") assert.ok(f.fix || f.api_error, `${id} ${f.code} has no fix`);112 assert.ok(!/sk-[A-Za-z0-9]{20}|org-[A-Za-z0-9]{12}/.test(JSON.stringify(f)));113 }114});115116test("knowledge covers every probed model with limits and prices", () => {117 assert.equal(Object.keys(MODELS).length, Object.keys(matrix.models).length);118 for (const m of Object.values(MODELS)) { assert.ok(m.context_window > 0, `${m.id} context`); assert.ok(m.max_output_tokens > 0, `${m.id} max output`); }119 const unpriced = Object.values(MODELS).filter((m) => !m.pricing_usd_per_mtok).map((m) => m.id).sort();120 assert.deepEqual(unpriced, ["chat-latest", "gpt-5.3-codex"]); // not on the pricing page: reported as unknown, never guessed121 assert.deepEqual(suggestModels("gpt-6.1"), ["gpt-6.1-terra", "gpt-6.1-sol"].sort((a, b) => b.length - a.length));122 assert.equal(resolveModel("claude-opus-5-5"), null);123 assert.match(KB.observed_at, /^\d{4}-\d\d-\d\dT/);124 assert.ok(!JSON.stringify(matrix).match(/org-[A-Za-z0-9]{12}|sk-proj/));125});126127test("cost arithmetic", () => {128 assert.equal(estimateCost({ model: "gpt-6.1-sol", input_tokens: 200000, output_tokens: 200000 }).usd_total, 2.4);129 assert.equal(estimateCost({ model: "gpt-6.1-sol", input_tokens: 200000, output_tokens: 200000, batch: true }).usd_total, 1.2);130 assert.equal(estimateCost({ model: "gpt-6.1-sol", input_tokens: 200000, output_tokens: 200000, fast: true }).usd_total, 4.8);131 assert.equal(estimateCost({ model: "gpt-6.1-sol", cached_input_tokens: 100000 }).usd_total, 0.01);132 assert.equal(estimateCost({ model: "gpt-6.1-sol", input_tokens: 1e6, output_tokens: 1e6 }).usd_total, 19); // long-context rates: $4 + $15133 assert.equal(estimateCost({ model: "gpt-6.1-sol", input_tokens: 300000, output_tokens: 0 }).usd_total, 1.2); // long-context rate above 272K134 assert.equal(estimateCost({ model: "gpt-4o-mini", input_tokens: 1e6, output_tokens: 1e6 }).usd_total, 0.75);135 assert.equal(estimateCost({ model: "text-embedding-3-small", input_tokens: 1e6 }).usd_total, 0.02);136 assert.equal(estimateCost({ model: "gpt-5.4", input_tokens: 1000, output_tokens: 1000, requests: 1000 }).usd_total, 17.5);137 assert.ok(estimateCost({ model: "gpt-5.3-codex", input_tokens: 1 }).error);138 assert.ok(estimateCost({ model: "gpt-5.4-nano", input_tokens: 1, fast: true }).error);139 assert.ok(estimateCost({ model: "gpt-6-luna", input_tokens: 1, fast: true, batch: true }).error);140 assert.deepEqual(compareCost({ input_tokens: 2000, output_tokens: 500 }, ["gpt-6-astra", "gpt-6-luna", "gpt-6.1-sol"]).map((r) => r.model), ["gpt-6-luna", "gpt-6.1-sol", "gpt-6-astra"]);141});142143test("scaffolds adapt to the model and never embed secrets", () => {144 for (const approach of Object.keys(APPROACHES)) for (const language of ["typescript", "python"]) for (const model of ["gpt-6.1-sol", "gpt-4.1-mini", "gpt-5-pro"]) {145 const s = scaffold({ approach, language, model });146 assert.ok(!/sk-[A-Za-z0-9]{10}|api_key\s*=\s*["']/.test(s.code), `${approach}/${language}`);147 if (approach !== "embeddings") assert.ok(s.code.includes(`"${model}"`), `${approach}/${language}/${model}`);148 if (model === "gpt-4.1-mini") assert.ok(!/reasoning/.test(s.code.replace(/reasoning tokens|reasoning items/g, "")), `${approach}/${language} emits reasoning for a non-reasoning model`);149 if (model === "gpt-5-pro" && !["embeddings", "agents_sdk", "batch"].includes(approach)) assert.ok(/effort"?: "high"/.test(s.code), `${approach}/${language} must use the only supported effort`);150 assert.ok(!/temperature|max_tokens\b|chat\.completions/.test(s.code));151 }152 assert.ok(scaffold({ approach: "nope" }).error);153 assert.ok(scaffold({ approach: "single_call", model: "claude-opus-5-5" }).error);154});155156test("guides exist for the documented topics", () => {157 for (const t of ["models", "responses_api", "chat_completions", "reasoning", "sampling", "structured_outputs", "function_calling", "hosted_tools", "state", "batches", "embeddings", "agents", "errors", "cost"]) assert.ok(GUIDES[t]?.length > 100, t);158});159