Repository Public Research preview
1// Tests of the skill's offline CLI and of the skill package itself. Run: node --test skill-tests/cli.test.mjs2import assert from "node:assert/strict";3import { execFileSync, spawnSync } from "node:child_process";4import { existsSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";5import { tmpdir } from "node:os";6import { join } from "node:path";7import { test } from "node:test";8import { fileURLToPath } from "node:url";910const skill = fileURLToPath(new URL("../skill/openai-api-toolkit/", import.meta.url));11const cli = join(skill, "scripts/openai-api.mjs");12const run = (args, input) => spawnSync(process.execPath, [cli, ...args], { input, encoding: "utf8", env: { PATH: process.env.PATH } });13const tmp = mkdtempSync(join(tmpdir(), "skill-cli-"));1415test("SKILL.md follows the Agent Skills specification", () => {16 const md = readFileSync(join(skill, "SKILL.md"), "utf8");17 const fm = /^---\n([\s\S]*?)\n---\n/.exec(md)[1];18 const name = /^name: (.+)$/m.exec(fm)[1];19 const description = /^description: (.+)$/m.exec(fm)[1];20 assert.equal(name, "openai-api-toolkit");21 assert.match(name, /^[a-z0-9]+(-[a-z0-9]+)*$/);22 assert.ok(description.length > 50 && description.length <= 1024);23 assert.ok(md.split("\n").length < 500);24 for (const ref of md.matchAll(/`(references\/[a-z-]+\.md)`/g)) assert.ok(existsSync(join(skill, ref[1])), `${ref[1]} is referenced but missing`);25 for (const f of readdirSync(join(skill, "references"))) assert.ok(md.includes(`references/${f}`) || readFileSync(join(skill, "README.md"), "utf8").includes(f), `${f} is not referenced`);26});2728test("the skill engine is byte-identical to the MCP package engine", () => {29 for (const f of ["lint.mjs", "cost.mjs", "scaffold.mjs", "knowledge.mjs", "guides.mjs"]) assert.equal(readFileSync(join(skill, "scripts/lib", f), "utf8"), readFileSync(new URL(`../mcp/lib/${f}`, import.meta.url), "utf8"), f);30 assert.equal(readFileSync(join(skill, "scripts/knowledge.json"), "utf8"), readFileSync(new URL("../mcp/knowledge.json", import.meta.url), "utf8"));31});3233test("scripts use no network, no dependencies and no credentials", () => {34 for (const f of ["scripts/openai-api.mjs", ...readdirSync(join(skill, "scripts/lib")).map((x) => `scripts/lib/${x}`)]) {35 const src = readFileSync(join(skill, f), "utf8");36 // Only the module's own header imports count; scaffold templates contain import lines as text.37 const header = src.split(/\n(?!import |\/\/|\/\*| \*|#!)/)[0];38 const imports = [...header.matchAll(/^import .* from "([^"]+)";$/gm)].map((m) => m[1]);39 for (const i of imports) assert.ok(i.startsWith("node:") || i.startsWith("./"), `${f} imports ${i}`);40 assert.ok(!/\bfetch\(|https\.request|net\.connect|process\.env\.OPENAI/.test(src), `${f} touches network or credentials`);41 }42});4344test("lint, fix, convert and migrate", () => {45 const bad = JSON.stringify({ model: "gpt-6.1-sol", messages: [{ role: "user", content: "hi" }], max_tokens: 50, temperature: 0 });46 const l = run(["lint", "-"], bad);47 assert.equal(l.status, 2);48 assert.deepEqual(JSON.parse(l.stdout).findings.filter((f) => f.severity === "error").map((f) => f.code).sort(), ["max-tokens-legacy", "sampling-param-rejected"]);49 const file = join(tmp, "req.json");50 writeFileSync(file, bad);51 const fixed = JSON.parse(run(["fix", file]).stdout);52 assert.equal(fixed.remaining.valid, true);53 writeFileSync(join(tmp, "fixed.json"), JSON.stringify(fixed.request));54 assert.equal(run(["lint", join(tmp, "fixed.json")]).status, 0);55 const conv = JSON.parse(run(["convert", file]).stdout);56 assert.equal(conv.request.input, "hi");57 assert.equal(conv.request.max_output_tokens, 50);58 assert.equal(conv.remaining.valid, true);59 const mig = JSON.parse(run(["migrate", file, "gpt-5.5-pro"]).stdout);60 assert.equal(mig.to, "gpt-5.5-pro");61 assert.equal(mig.converted_to, "responses"); // Responses-only target: the chat request is converted62 assert.equal(mig.remaining.valid, true);63 assert.equal(mig.request.messages, undefined);64 assert.equal(JSON.parse(run(["migrate", file, "gpt-5.4"]).stdout).remaining.valid, true);65 assert.equal(JSON.parse(run(["lint", "-", "--api", "chat"], JSON.stringify({ model: "gpt-4o", messages: [{ role: "user", content: "x" }], temperature: 0.1, max_tokens: 10 })).stdout).valid, true);66 assert.equal(run(["lint", "/nonexistent.json"]).status, 1);67 assert.equal(run(["lint", "-"], "{not json").status, 1);68});6970test("models, cost, compare, guide", () => {71 assert.equal(JSON.parse(run(["models"]).stdout).models.length, 29);72 assert.equal(JSON.parse(run(["models", "gpt-6.1-sol"]).stdout).responses.sampling_params, "never");73 assert.equal(run(["models", "claude-opus-5-5"]).status, 1);74 assert.equal(JSON.parse(run(["cost", "gpt-6.1-sol", "200000", "200000"]).stdout).usd_total, 2.4);75 assert.equal(JSON.parse(run(["cost", "gpt-6.1-sol", "200000", "200000", "--batch"]).stdout).usd_total, 1.2);76 assert.equal(run(["cost", "gpt-6.1-sol", "-1", "5"]).status, 1);77 assert.equal(run(["cost", "gpt-5.3-codex", "10", "5"]).status, 1); // no official price: refuse rather than guess78 assert.equal(JSON.parse(run(["compare", "1000", "1000"]).stdout)[0].model, "gpt-5-nano");79 assert.match(run(["guide", "sampling"]).stdout, /none/);80 assert.equal(run(["guide", "nope"]).status, 1);81 assert.match(run([]).stderr, /openai-api\.mjs lint/);82});8384test("scaffold writes a file and refuses to overwrite", () => {85 const out = join(tmp, "agent.py");86 assert.equal(run(["scaffold", "function_loop", "--language", "python", "--model", "gpt-6.1-sol", "--out", out]).status, 0);87 assert.match(readFileSync(out, "utf8"), /function_call_output/);88 assert.equal(run(["scaffold", "function_loop", "--out", out]).status, 1);89 assert.equal(run(["scaffold", "nope"]).status, 1);90 execFileSync("python3", ["-c", `import ast,sys;ast.parse(open(${JSON.stringify(out)}).read())`]);91});9293test("generated references state their source and cover every model", () => {94 const rules = readFileSync(join(skill, "references/request-rules.md"), "utf8");95 const models = readFileSync(join(skill, "references/models.md"), "utf8");96 for (const id of Object.keys(JSON.parse(readFileSync(join(skill, "scripts/knowledge.json"))).models)) { assert.ok(rules.includes(`\`${id}\``), id); assert.ok(models.includes(`\`${id}\``), id); }97 assert.match(rules, /Generated by build-skill\.mjs from live API probes/);98 assert.match(rules, /Responses only/);99});100