Repository Public Research preview
1// Tests of the manuscript-peer-review skill: specification compliance, engine identity with the2// MCP package, generated reference, rubric arithmetic and the command-line tool.3// Run: node --test skill-tests/4import assert from "node:assert/strict";5import { spawnSync } from "node:child_process";6import { existsSync, mkdtempSync, readFileSync, readdirSync, realpathSync, symlinkSync, writeFileSync } from "node:fs";7import { tmpdir } from "node:os";8import { join } from "node:path";9import { test } from "node:test";10import { fileURLToPath } from "node:url";11import { CODES } from "../mcp/lib/codes.mjs";1213const skill = fileURLToPath(new URL("../skill/manuscript-peer-review/", import.meta.url));14const mcp = fileURLToPath(new URL("../mcp/", import.meta.url));15const FX = join(mcp, "test", "fixtures");16const cliPath = join(skill, "scripts", "manuscript.mjs");17const cli = (args, opts = {}) => {18 const r = spawnSync(process.execPath, [opts.script ?? cliPath, ...args], { encoding: "utf8", cwd: opts.cwd ?? skill, env: { PATH: process.env.PATH }, maxBuffer: 64 * 1024 * 1024 });19 return { status: r.status, stdout: r.stdout, stderr: r.stderr, json: () => JSON.parse(r.stdout), err: () => JSON.parse(r.stderr).error };20};21const docs = () => ["SKILL.md", "README.md", "CHANGELOG.md", ...readdirSync(join(skill, "references")).map((f) => `references/${f}`)];2223test("SKILL.md follows the Agent Skills specification and references exist", () => {24 const md = readFileSync(join(skill, "SKILL.md"), "utf8");25 const fm = /^---\n([\s\S]*?)\n---\n/.exec(md)[1];26 const name = /^name: (.+)$/m.exec(fm)[1];27 assert.equal(name, "manuscript-peer-review");28 assert.match(name, /^[a-z0-9]+(-[a-z0-9]+)*$/);29 assert.ok(name.length <= 64);30 const d = /^description: (.+)$/m.exec(fm)[1];31 assert.ok(d.length > 50 && d.length <= 1024, `description length ${d.length}`);32 assert.match(fm, /^license: MIT$/m);33 assert.match(fm, /^compatibility: .+$/m);34 assert.ok(md.split("\n").length < 500);35 const refs = new Set();36 for (const f of docs()) for (const m of readFileSync(join(skill, f), "utf8").matchAll(/`((?:references|scripts)\/[A-Za-z0-9_./-]+)`/g)) refs.add(m[1]);37 assert.ok(refs.size >= 6);38 for (const r of refs) assert.ok(existsSync(join(skill, r)), `${r} is referenced but missing`);39 for (const f of readdirSync(join(skill, "references"))) assert.ok(md.includes(`references/${f}`), `SKILL.md never points to references/${f}`);40});4142test("manifest and package files", () => {43 const m = JSON.parse(readFileSync(join(skill, "agent-registry.json"), "utf8"));44 assert.equal(m.name, "@spboucher/manuscript-peer-review");45 assert.equal(m.type, "skill");46 assert.equal(m.entrypoint, "SKILL.md");47 assert.match(m.version, /^\d+\.\d+\.\d+$/);48 assert.ok(m.description.length >= 10 && m.description.length <= 300);49 assert.ok(m.keywords.length <= 12 && m.keywords.every((k) => /^[a-z0-9][a-z0-9-]{0,31}$/.test(k)));50 assert.deepEqual(m.permissions, ["filesystem:read"]);51 assert.equal(/version: "(.+)"/.exec(readFileSync(join(skill, "SKILL.md"), "utf8"))[1], m.version);52 for (const f of ["LICENSE", "README.md", "CHANGELOG.md"]) assert.ok(existsSync(join(skill, f)), f);53 const mm = JSON.parse(readFileSync(join(mcp, "agent-registry.json"), "utf8"));54 assert.equal(mm.type, "mcp-server");55 assert.deepEqual(mm.mcp.args, ["server.mjs"]);56 assert.equal(mm.version, JSON.parse(readFileSync(join(mcp, "package.json"), "utf8")).version);57});5859test("the skill engine is byte-identical to the MCP engine and imports only Node.js modules", () => {60 const a = readdirSync(join(mcp, "lib")).sort(), b = readdirSync(join(skill, "scripts", "lib")).sort();61 assert.deepEqual(b, a, "run node build-skill.mjs");62 for (const f of a) assert.ok(readFileSync(join(mcp, "lib", f)).equals(readFileSync(join(skill, "scripts", "lib", f))), `${f} differs: run node build-skill.mjs`);63 for (const f of ["scripts/manuscript.mjs", ...b.map((x) => `scripts/lib/${x}`)]) {64 const src = readFileSync(join(skill, f), "utf8");65 for (const m of src.matchAll(/^\s*(?:import|export)\s[^;]*?from\s+["']([^"']+)["']/gm)) assert.ok(m[1].startsWith("node:") || m[1].startsWith("./"), `${f} imports ${m[1]}`);66 assert.ok(!/\b(?:child_process|node:net|node:http|node:https|node:dgram|fetch\()/.test(src), `${f} must not start processes or use the network`);67 }68});6970test("finding-codes.md lists every code of the engine", () => {71 const md = readFileSync(join(skill, "references", "finding-codes.md"), "utf8");72 const listed = [...md.matchAll(/^\| `([a-z-]+)` \|/gm)].map((m) => m[1]).sort();73 assert.deepEqual(listed, Object.keys(CODES).sort(), "run node build-skill.mjs");74 assert.equal(listed.length, 67);75 for (const f of ["SKILL.md", "README.md", "references/roles.md", "references/procedure.md"]) {76 for (const m of readFileSync(join(skill, f), "utf8").matchAll(/`((?:outline|cite|xref|num|stat|table|figure|claim|style|repl|integrity)-[a-z-]+[a-z])`/g)) assert.ok(CODES[m[1]], `${f} mentions unknown code ${m[1]}`);77 }78});7980test("rubric: weights sum to 100 and the worked example adds up", () => {81 const md = readFileSync(join(skill, "references", "rubric.md"), "utf8");82 const weights = [...md.matchAll(/^\| \d \| [^|]+ \| (\d+) \| [^|]+ \|$/gm)].map((m) => Number(m[1]));83 assert.equal(weights.length, 7);84 assert.equal(weights.reduce((a, b) => a + b, 0), 100);85 const rows = [...md.matchAll(/^\d [A-Za-z ]+?\s{2,}(\d)\s+(\d+)\s+([\d.]+)\s/gm)].map((m) => m.slice(1).map(Number));86 assert.equal(rows.length, 7);87 let total = 0;88 for (const [level, weight, points] of rows) { assert.equal(points, (weight * (level - 1)) / 4); total += points; }89 assert.deepEqual(rows.map((r) => r[1]), weights);90 assert.equal(Number(/^Total\s+([\d.]+)/m.exec(md)[1]), total);91});9293test("documents and the script are plain ASCII (no emoji, no invisible characters)", () => {94 for (const f of [...docs(), "scripts/manuscript.mjs"]) {95 const s = readFileSync(join(skill, f), "utf8");96 const bad = [...s].find((ch) => ch.charCodeAt(0) > 126);97 assert.equal(bad, undefined, `${f} contains a non-ASCII character: ${JSON.stringify(bad)}`);98 }99});100101test("CLI: usage, unknown command, bad options", () => {102 const none = cli([]);103 assert.equal(none.status, 1);104 assert.match(none.stdout, /node manuscript\.mjs audit <file>/);105 const help = cli(["--help"]);106 assert.equal(help.status, 0);107 assert.match(help.stdout, /--fail-on error\|warning\|never/);108 const bad = cli(["frobnicate", "x.tex"]);109 assert.equal(bad.status, 1);110 assert.equal(bad.err().code, "usage");111 assert.equal(cli(["audit"]).status, 1);112 assert.equal(cli(["audit", "a.tex", "--fail-on", "sometimes"]).err().code, "usage");113 assert.equal(cli(["audit", "a.tex", "--min-severity", "fatal"]).err().code, "usage");114 assert.equal(cli(["compare", "a.tex"]).err().code, "usage");115 assert.equal(cli(["audit", join(FX, "flawed", "nope.tex")]).err().code, "not_found");116});117118test("CLI: audit of the flawed manuscript, JSON and text, exit codes", () => {119 const r = cli(["audit", join(FX, "flawed", "paper.tex")]);120 assert.equal(r.status, 2);121 const a = r.json();122 assert.equal(a.tool, "full_audit");123 assert.equal(a.summary.total, 46);124 assert.equal(a.summary.errors, 9);125 assert.ok(a.findings.every((f) => f.code in CODES && f.location.line >= 1 && typeof f.evidence === "string"));126 const e = cli(["audit", join(FX, "flawed", "paper.tex"), "--min-severity", "error"]).json();127 assert.equal(e.findings.length, 9);128 assert.equal(e.summary.total, 46);129 const t = cli(["audit", join(FX, "flawed", "paper.tex"), "--text", "--min-severity", "error"]);130 assert.equal(t.status, 2);131 assert.equal(t.stdout.trim().split("\n").length, 11);132 assert.match(t.stdout, /^ERROR {3}paper\.tex:\d+ {2}cite-missing-entry {2}Cited key "ghostkey2019"/m);133 assert.match(t.stdout, /not a quality score/);134 assert.match(t.stdout, /never means the manuscript is correct/);135 assert.equal(cli(["audit", join(FX, "flawed", "paper.tex"), "--fail-on", "never"]).status, 0);136});137138test("CLI: clean manuscripts exit 0 at every fail level", () => {139 for (const [dir, file] of [["clean", "paper.tex"], ["md-clean", "paper.md"]]) {140 const r = cli(["audit", join(FX, dir, file), "--fail-on", "warning"]);141 assert.equal(r.status, 0, r.stdout.slice(0, 400));142 assert.equal(r.json().summary.total, 0);143 }144});145146test("CLI: each single audit returns its own tool and the planted findings", () => {147 const expect = { outline: ["outline", 3], citations: ["audit_citations", 7], refs: ["audit_cross_references", 6], numbers: ["audit_numbers", 4], stats: ["audit_statistics_reporting", 6], tables: ["audit_tables", 7], claims: ["audit_claims", 3], style: ["audit_style", 6], replication: ["audit_replication", 4] };148 let sum = 0;149 for (const [cmd, [tool, n]] of Object.entries(expect)) {150 const r = cli([cmd, join(FX, "flawed", "paper.tex"), "--fail-on", "never"]);151 assert.equal(r.status, 0, cmd);152 const j = r.json();153 assert.equal(j.tool, tool);154 assert.equal(j.findings.length, n, cmd);155 sum += n;156 }157 assert.equal(sum, 46);158 const md = cli(["citations", join(FX, "md-flawed", "paper.md")]);159 assert.equal(md.status, 2);160 assert.deepEqual(md.json().findings.map((f) => f.code).sort(), ["cite-missing-entry", "cite-unused-entry"]);161 const bib = cli(["citations", join(FX, "md-flawed", "paper.md"), "--bib", "md-clean/refs.bib", "--root", FX]);162 assert.deepEqual(bib.json().findings.map((f) => f.code), ["cite-missing-entry"]);163 assert.equal(bib.json().findings[0].location.file, join("md-flawed", "paper.md"));164});165166test("CLI: scaffold, compare and codes", () => {167 const s = cli(["scaffold", join(FX, "flawed", "paper.tex")]);168 assert.equal(s.status, 0);169 assert.match(s.stdout, /^# Referee report: Remote Work and Output/);170 assert.match(s.stdout, /\[REVIEWER\]/);171 assert.match(s.stdout, /\*\*num-percent-count-mismatch\*\*, paper\.tex:\d+\. 412 of 1250 is 33\.0%, not the stated 38\.0%\./);172 assert.equal(cli(["scaffold", join(FX, "flawed", "paper.tex"), "--json"]).json().tool, "review_scaffold");173 const c = cli(["compare", join(FX, "flawed", "paper.tex"), join(FX, "clean", "paper.tex")]);174 assert.equal(c.status, 0);175 const j = c.json();176 assert.deepEqual(j.sections.removed.map((x) => x.title), ["Controls", "Heterogeneity", "Discussion"]);177 assert.ok(j.numbers.changed.some((n) => n.old === "14.2%" && n.new === "12.4%" && n.in_abstract));178 assert.ok(j.citations.removed.some((x) => x.key === "ghostkey2019"));179 const codes = cli(["codes"]).json();180 assert.equal(codes.length, 67);181 assert.ok(codes.every((x) => x.code && x.severity && x.meaning && x.verify && typeof x.heuristic === "boolean"));182});183184test("CLI: files are read only inside the root", () => {185 const base = realpathSync(mkdtempSync(join(tmpdir(), "ms-skill-")));186 writeFileSync(join(base, "secret.tex"), "\\section{S}\nPRIVATE-MARKER\n");187 const inside = join(base, "proj");188 spawnSync("mkdir", ["-p", join(inside, "paper")]);189 writeFileSync(join(inside, "paper", "main.tex"), "\\section{A}\nText that is long enough to make a section of ten words here.\n\\input{../tables/t1}\n\\input{../../secret}\n");190 spawnSync("mkdir", ["-p", join(inside, "tables")]);191 writeFileSync(join(inside, "tables", "t1.tex"), "See Table~\\ref{tab:none}.\n");192 symlinkSync(join(base, "secret.tex"), join(inside, "paper", "link.tex"));193 const narrow = cli(["outline", join(inside, "paper", "main.tex"), "--fail-on", "never"]).json();194 assert.deepEqual(narrow.unresolved_includes.map((u) => u.reason), ["outside the allowed root", "outside the allowed root"]);195 assert.equal(narrow.findings.filter((f) => f.code === "outline-include-not-read").length, 2);196 const wide = cli(["audit", join(inside, "paper", "main.tex"), "--root", inside, "--fail-on", "never"]);197 assert.ok(!wide.stdout.includes("PRIVATE-MARKER"));198 const w = wide.json();199 assert.deepEqual(w.manuscript.unresolved_includes.map((u) => u.name), ["../../secret"]);200 assert.ok(w.findings.some((f) => f.code === "xref-undefined" && f.location.file === join("tables", "t1.tex")));201 assert.equal(cli(["audit", join(base, "secret.tex"), "--root", inside]).err().code, "path_forbidden");202 assert.equal(cli(["audit", join(inside, "paper", "link.tex")]).err().code, "path_forbidden");203 assert.equal(cli(["citations", join(inside, "paper", "main.tex"), "--bib", "../../secret.tex"]).err().code, "path_forbidden");204});205206test("CLI: works from another directory, with a relative path and through a symlink", () => {207 const rel = cli(["audit", "paper.tex", "--min-severity", "error"], { cwd: join(FX, "flawed") });208 assert.equal(rel.status, 2);209 assert.equal(rel.json().findings.length, 9);210 const dir = realpathSync(mkdtempSync(join(tmpdir(), "ms-skill-")));211 symlinkSync(skill, join(dir, "installed"));212 const r = cli(["audit", join(FX, "clean", "paper.tex")], { script: join(dir, "installed", "scripts", "manuscript.mjs"), cwd: dir });213 assert.equal(r.status, 0);214 assert.equal(r.json().summary.total, 0);215});216