SPB Git

Repository Public Research preview

skill.test.mjs 16.8 KB 268 lines javascript

1/**2 * =============================================================================3 *                  SP BOUCHER | SOFTWARE & RESEARCH4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : LLM API toolkits8 * File             : protocol/skill-tests/skill.test.mjs9 * Module           : protocol10 * Repository       : https://git.spboucher.ai/llm-api-toolkits11 * Created          : 2026-11-1012 * Last Modified    : 2026-11-1013 *14 * Author           : Simon-Pierre Boucher15 * Position         : Professor of Finance16 * Institution      : Université du Québec en Outaouais (UQO)17 * Organization     : SP Boucher18 * Website          : https://spboucher.ai19 * Git              : https://git.spboucher.ai20 *21 * Preferred Contact: dev@spboucher.ai22 * Development      : dev@spboucher.ai23 * Support          : support@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142// Tests of the skill: spec compliance, engine identity with the MCP package, and the command-line43// tool end to end in a temporary project.44// Run: node --test skill-tests/skill.test.mjs45import assert from "node:assert/strict";46import { execFile } from "node:child_process";47import fs from "node:fs";48import { tmpdir } from "node:os";49import { join } from "node:path";50import { after, test } from "node:test";51import { fileURLToPath } from "node:url";5253const skill = fileURLToPath(new URL("../skill/preregistered-research/", import.meta.url));54const mcp = fileURLToPath(new URL("../mcp/", import.meta.url));55const tool = join(skill, "scripts/ledger.mjs");56const made = [];57after(() => { for (const d of made) fs.rmSync(d, { recursive: true, force: true }); });58function project() {59  const d = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), "prereg-skill-"))); made.push(d);60  fs.writeFileSync(join(d, "protocol.md"), "# Protocol\n");61  fs.writeFileSync(join(d, "analysis.py"), "print(1)\n");62  fs.writeFileSync(join(d, "sample.jsonl"), "{}\n");63  fs.writeFileSync(join(d, "out.jsonl"), "{}\n");64  return d;65}66const cliIn = (cwd, env = {}) => (...args) => new Promise((res) => execFile(process.execPath, [tool, ...args], { cwd, encoding: "utf8", env: { PATH: process.env.PATH, ...env } }, (err, stdout, stderr) => {67  let json = null, error = null;68  try { json = JSON.parse(stdout); } catch { /* text output */ }69  try { error = JSON.parse(stderr).error; } catch { /* no structured error */ }70  res({ status: err ? (typeof err.code === "number" ? err.code : 1) : 0, stdout, stderr, json, error });71}));72const walk = (dir, base = dir) => fs.readdirSync(dir, { withFileTypes: true }).flatMap((e) => (e.isDirectory() ? walk(join(dir, e.name), base) : [join(dir, e.name).slice(base.length).replace(/^\//, "")])).sort();7374/* ───────── spec compliance ───────── */7576test("SKILL.md follows the Agent Skills specification", () => {77  const md = fs.readFileSync(join(skill, "SKILL.md"), "utf8");78  const m = md.match(/^---\n([\s\S]*?)\n---\n/);79  assert.ok(m, "frontmatter present");80  const fm = Object.fromEntries(m[1].split("\n").map((l) => [l.slice(0, l.indexOf(":")), l.slice(l.indexOf(":") + 1).trim()]));81  assert.equal(fm.name, "preregistered-research");82  assert.match(fm.name, /^[a-z0-9]+(-[a-z0-9]+)*$/);83  assert.ok(fm.description.length >= 50 && fm.description.length <= 1024, `description length ${fm.description.length}`);84  assert.match(fm.description, /Use when/);85  assert.ok(!/[<>]/.test(fm.description));86  assert.ok(md.split("\n").length < 500);87  const refs = [...md.matchAll(/`((?:references|scripts)\/[A-Za-z0-9_./-]+)`/g)].map((x) => x[1]);88  assert.ok(refs.length >= 4);89  for (const r of refs) assert.ok(fs.existsSync(join(skill, r)), `${r} is referenced but missing`);90});9192test("the package has every expected file and a valid manifest", () => {93  assert.deepEqual(walk(skill), ["CHANGELOG.md", "LICENSE", "README.md", "SKILL.md", "agent-registry.json", "examples/claude-code.png", "examples/codex.png", "references/cli.md", "references/method.md", "references/templates.md", "scripts/ledger.mjs", "scripts/lib/ledger.mjs"]);94  const man = JSON.parse(fs.readFileSync(join(skill, "agent-registry.json"), "utf8"));95  assert.equal(man.name, "@spboucher/preregistered-research");96  assert.equal(man.type, "skill");97  assert.match(man.version, /^1\.0\.\d+$/);98  assert.equal(man.license, "MIT");99  assert.ok(man.description.length >= 10 && man.description.length <= 300);100  assert.ok(man.keywords.length <= 12 && man.keywords.every((k) => k === k.toLowerCase()));101  assert.ok(!man.permissions.includes("network"), "the skill does not use the network");102  const server = JSON.parse(fs.readFileSync(join(mcp, "agent-registry.json"), "utf8"));103  assert.equal(server.version, man.version);104  assert.ok(server.description.length <= 300);105});106107test("the engine is byte-identical to the one in the MCP package", () => {108  assert.deepEqual(fs.readFileSync(join(skill, "scripts/lib/ledger.mjs")), fs.readFileSync(join(mcp, "lib/ledger.mjs")), "run node build-skill.mjs");109  assert.deepEqual(fs.readFileSync(join(skill, "LICENSE")), fs.readFileSync(join(mcp, "LICENSE")));110});111112test("scripts import only Node built-ins and local files, and never reach the network or a shell", () => {113  for (const f of ["scripts/ledger.mjs", "scripts/lib/ledger.mjs"]) {114    const src = fs.readFileSync(join(skill, f), "utf8");115    const imports = [...src.matchAll(/^\s*import\s[^"']*["']([^"']+)["']/gm)].map((x) => x[1]);116    assert.ok(imports.length > 0, f);117    for (const i of imports) assert.ok(i.startsWith("node:") || i.startsWith("./"), `${f} imports ${i}`);118    for (const banned of ["node:child_process", "node:http", "node:https", "node:net", "node:dgram", "fetch(", "eval("]) assert.ok(!src.includes(banned), `${f} uses ${banned}`);119  }120});121122test("the tools named in the command reference exist in the server, and every server tool is documented", async () => {123  const ref = fs.readFileSync(join(skill, "references/cli.md"), "utf8");124  const documented = new Set([...ref.matchAll(/`([a-z_]+)`/g)].map((x) => x[1]));125  const server = JSON.parse(fs.readFileSync(join(mcp, "agent-registry.json"), "utf8")).mcp.tools.map((t) => t.name);126  for (const t of server) assert.ok(documented.has(t), `${t} is not in references/cli.md`);127  const { createServer } = await import(new URL("../mcp/server.mjs", import.meta.url));128  const registered = Object.keys(createServer({})._registeredTools).sort();129  assert.deepEqual(registered, [...server].sort(), "manifest tool list matches the server");130});131132test("documents are in plain English with no leftovers", () => {133  for (const f of walk(skill).filter((x) => x.endsWith(".md"))) {134    const txt = fs.readFileSync(join(skill, f), "utf8");135    assert.ok(!/TODO|FIXME|lorem ipsum/i.test(txt), f);136    assert.ok(!/\p{Extended_Pictographic}/u.test(txt), `${f} contains an emoji`);137  }138  for (const f of ["method.md", "templates.md"]) assert.match(fs.readFileSync(join(skill, "references", f), "utf8"), /^## Contents$/m, `${f} needs a table of contents`);139});140141/* ───────── command-line round trip ───────── */142143test("help is printed without a project, and bad usage exits 2", async () => {144  const cli = cliIn(tmpdir());145  const h = await cli("help");146  assert.equal(h.status, 0);147  for (const c of ["init", "hypothesis", "freeze", "verify", "amend", "decide", "deviate", "gate open", "run", "spend", "result", "status", "report", "check"]) assert.ok(h.stdout.includes(`  ${c}`), c);148  assert.equal((await cli()).status, 0);149  assert.equal((await cli("frobnicate")).status, 2);150  assert.equal((await cli("freeze", "--file")).status, 2);151});152153test("a study from init to report, through the command line", async () => {154  const root = project(), cli = cliIn(root);155  const none = await cli("status");156  assert.equal(none.status, 1);157  assert.equal(none.error.code, "not_initialised");158159  assert.equal((await cli("init", "--title", "CLI study", "--budget", "2", "--outcome-data", "the y column")).json.project.budget_usd, 2);160  assert.equal((await cli("hypothesis", "--statement", "x raises y in the full sample", "--test", "OLS of y on x", "--rule", "coefficient > 0, p < 0.05")).json.registered, "H1");161  assert.equal((await cli("hypothesis", "--statement", "the effect is larger in small units", "--test", "interaction term", "--rule", "interaction > 0, p < 0.05", "--role", "secondary")).json.registered, "H2");162  assert.equal((await cli("gate", "open", "--name", "A", "--criterion", "Design reviewed", "--criterion", "Code runs on synthetic data")).json.criteria.length, 2);163  assert.equal((await cli("gate", "close", "--name", "A", "--outcome", "go", "--evidence", "Review done, three objections addressed")).json.outcome, "go");164  const fz = await cli("freeze", "--file", "protocol.md", "--file", "analysis.py");165  assert.match(fz.json.root_hash, /^[0-9a-f]{64}$/);166  const again = await cli("freeze", "--file", "protocol.md");167  assert.equal(again.status, 1);168  assert.equal(again.error.code, "already_frozen");169  assert.equal((await cli("verify")).status, 0);170  assert.equal((await cli("decide", "--question", "Cluster by unit or period?", "--option", "unit", "--option", "period", "--choice", "unit", "--why", "Treatment varies at the unit level.", "--timing", "before")).json.id, "D-01");171  assert.equal((await cli("data", "--description", "First read of the outcome column")).json.outcome_data, true);172173  const spec = ["--command", "python extract.py", "--code", "analysis.py", "--input", "sample.jsonl", "--seed", "7"];174  assert.equal((await cli("run", "--check", ...spec)).status, 0);175  assert.equal((await cli("run", "--name", "Extraction", ...spec, "--output", "out.jsonl", "--expensive", "--nondeterministic")).json.id, "R-01");176  const chk = await cli("run", "--check", ...spec);177  assert.equal(chk.status, 3, "an identical expensive run is on record");178  assert.equal(chk.json.rerun_advised, false);179  const dup = await cli("run", "--name", "Extraction again", ...spec, "--expensive");180  assert.equal(dup.status, 1);181  assert.equal(dup.error.code, "expensive_rerun");182  assert.deepEqual((await cli("run", "--name", "Extraction again", ...spec, "--rerun-reason", "Output file was lost with the disk.")).json.repeats, ["R-01"]);183184  assert.equal((await cli("spend", "--provider", "api-a", "--model", "model-x", "--purpose", "extraction", "--cost", "0.1", "--in", "1000", "--out", "200", "--run", "R-01")).json.total_usd, 0.1);185  assert.equal((await cli("spend", "--provider", "api-a", "--purpose", "extraction", "--cost", "0.2")).json.total_usd, 0.3);186  assert.equal((await cli("spend")).json.remaining_usd, 1.7);187  assert.equal((await cli("spend", "--budget", "3", "--why", "Second wave approved by the user.")).json.previous_usd, 2);188  assert.equal((await cli("spend", "--provider", "api-a", "--purpose", "x", "--cost", "abc")).status, 2);189190  const r1 = await cli("result", "--hypothesis", "H1", "--verdict", "not_confirmed", "--estimate", "0.01", "--uncertainty", "95% CI [-0.04, 0.06]", "--run", "R-01");191  assert.equal(r1.json.status, "confirmatory");192  assert.equal((await cli("deviate", "--protocol-said", "Interaction with a size dummy.", "--done", "Interaction with log size.", "--why", "The dummy had almost no variation.", "--impact", "exploratory", "--hypothesis", "H2")).json.applies_to[0], "H2");193  fs.appendFileSync(join(root, "analysis.py"), "# log size\n");194  const ver = await cli("verify");195  assert.equal(ver.status, 3);196  assert.deepEqual(ver.json.changed, ["analysis.py"]);197  const am = await cli("amend", "--file", "protocol.md", "--file", "analysis.py", "--reason", "H2 uses log size instead of a dummy.", "--data-seen", "yes", "--affects", "H2");198  assert.equal(am.json.version, 2);199  assert.equal((await cli("amend", "--file", "protocol.md", "--reason", "A change with no declared timing.")).error.code, "invalid_input");200  assert.equal((await cli("result", "--hypothesis", "H2", "--verdict", "inconclusive", "--estimate", "about 0.02")).json.status, "exploratory");201  assert.equal((await cli("question", "--ask", "Is the second source worth adding?")).json.id, "Q-01");202203  const st = await cli("status");204  assert.equal(st.status, 0);205  for (const part of ["CLI study — 18 records", "Protocol: version 2", "files intact, 1 amendment(s)", "H1 (primary) not_confirmed (confirmatory)", "H2 (secondary) inconclusive (exploratory)", "A (round 1): go", "Q-01 Is the second source worth adding?", "Spending: 0.3 USD of 3 (remaining 2.7)", "1 open question(s) to resolve"]) assert.ok(st.stdout.includes(part), `status lacks "${part}"\n${st.stdout}`);206  assert.equal((await cli("status", "--json")).json.records, 18);207  assert.equal((await cli("question", "--resolve", "Q-01", "--answer", "No, coverage is already sufficient.")).json.resolved, true);208209  const rep = await cli("report", "--write");210  assert.equal(rep.status, 0);211  assert.match(rep.stdout, /^# Transparency report — CLI study/);212  assert.match(rep.stdout, /\| H1 \|.*\| not confirmed \| confirmatory \| 0\.01 \| 95% CI \[-0\.04, 0\.06\] \|/);213  assert.match(rep.stdout, /\| H2 \|.*\| inconclusive \| exploratory \| about 0\.02 \|/);214  assert.match(rep.stdout, /R-02 repeats R-01: Output file was lost with the disk\./);215  assert.match(rep.stderr, /Saved: \.research\/reports\/report-19\.md/);216  assert.equal((await cli("report", "--json")).json.results.length, 2);217  assert.deepEqual(walk(join(root, ".research")), ["HEAD", "ledger.jsonl", "reports/report-19.json", "reports/report-19.md"]);218219  const ok = await cli("check");220  assert.equal(ok.status, 0);221  assert.equal(ok.json.records_verified, 19);222});223224test("check pinpoints tampering and the tool then refuses to append", async () => {225  const root = project(), cli = cliIn(root);226  await cli("init", "--title", "Tamper");227  for (let i = 0; i < 4; i++) await cli("question", "--ask", `Open point number ${i}?`);228  const file = join(root, ".research", "ledger.jsonl");229  const lines = fs.readFileSync(file, "utf8").trimEnd().split("\n");230  fs.writeFileSync(file, [lines[0], lines[1], lines[3], lines[2], lines[4]].join("\n") + "\n");231  const c = await cli("check");232  assert.equal(c.status, 3);233  assert.deepEqual([c.json.first_problem.line, c.json.first_problem.kind], [3, "sequence_break"]);234  const refused = await cli("question", "--ask", "Can this still be appended?");235  assert.equal(refused.status, 3);236  assert.equal(refused.error.code, "ledger_corrupt");237  assert.equal(refused.error.problem.line, 3);238  const st = await cli("status");239  assert.equal(st.status, 3);240  assert.match(st.stdout, /LEDGER BROKEN at record 3/);241  fs.writeFileSync(file, lines.join("\n") + "\n");242  assert.equal((await cli("check")).status, 0);243});244245test("the root comes from --root, then the environment, then the working directory; outside paths are refused", async () => {246  const a = project(), b = project(), c = project();247  await cliIn(c)("init", "--title", "From cwd");248  await cliIn(c, { RESEARCH_LEDGER_ROOT: b })("init", "--title", "From env");249  await cliIn(c, { RESEARCH_LEDGER_ROOT: b })("init", "--title", "From flag", "--root", a);250  const title = (d) => JSON.parse(fs.readFileSync(join(d, ".research", "ledger.jsonl"), "utf8").split("\n")[0]).data.title;251  assert.deepEqual([title(a), title(b), title(c)], ["From flag", "From env", "From cwd"]);252  await cliIn(a)("hypothesis", "--statement", "x raises y in the full sample", "--test", "OLS of y on x", "--rule", "beta > 0");253  const out = await cliIn(a)("freeze", "--file", join(b, "protocol.md"));254  assert.equal(out.status, 1);255  assert.equal(out.error.code, "path_forbidden");256  fs.symlinkSync(join(b, "protocol.md"), join(a, "linked.md"));257  assert.equal((await cliIn(a)("freeze", "--file", "linked.md")).error.code, "path_forbidden");258});259260test("the skill works from a copy with nothing else around it", async () => {261  const dest = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), "prereg-copy-"))); made.push(dest);262  fs.cpSync(skill, join(dest, "skill"), { recursive: true });263  const root = project();264  const run = (...args) => new Promise((res) => execFile(process.execPath, [join(dest, "skill/scripts/ledger.mjs"), ...args], { cwd: root, encoding: "utf8", env: { PATH: process.env.PATH } }, (err, stdout) => res({ status: err ? 1 : 0, stdout })));265  assert.equal((await run("init", "--title", "Standalone")).status, 0);266  assert.match((await run("status")).stdout, /Standalone — 1 records/);267});268
Files main
Menu

Theme