SPB Git

Repository Public Research preview

skill.test.mjs 11.7 KB 179 lines javascript

1/**2 * =============================================================================3 *                  SP BOUCHER | SOFTWARE & RESEARCH4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : LLM API toolkits8 * File             : xlsx/skill-tests/skill.test.mjs9 * Module           : xlsx10 * Repository       : https://git.spboucher.ai/llm-api-toolkits11 * Created          : 2026-11-1012 * Last Modified    : 2026-11-1013 *14 * Author           : Simon-Pierre Boucher15 * Position         : Professor of Finance16 * Institution      : Université du Québec en Outaouais (UQO)17 * Organization     : SP Boucher18 * Website          : https://spboucher.ai19 * Git              : https://git.spboucher.ai20 *21 * Preferred Contact: dev@spboucher.ai22 * Development      : dev@spboucher.ai23 * Support          : support@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142// Tests of the skill: specification compliance, engine identity, and the command-line tool on the fixtures.43// Run: node --test skill-tests/skill.test.mjs44import assert from "node:assert/strict";45import { spawnSync } from "node:child_process";46import { copyFileSync, existsSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";47import { tmpdir } from "node:os";48import { basename, join } from "node:path";49import { test } from "node:test";50import { fileURLToPath } from "node:url";51import { ENGINE } from "../build-skill.mjs";5253const skill = fileURLToPath(new URL("../skill/excel-model-workbook/", import.meta.url));54const mcp = fileURLToPath(new URL("../mcp/", import.meta.url)), fx = join(mcp, "test/fixtures");55const work = mkdtempSync(join(tmpdir(), "xlsx-skill-"));56for (const f of readdirSync(fx).filter((f) => f.endsWith(".xlsx"))) copyFileSync(join(fx, f), join(work, f));57const cli = (...args) => {58  const env = { PATH: process.env.PATH };59  if (args[0] && typeof args[0] === "object") Object.assign(env, args.shift());60  const r = spawnSync(process.execPath, [join(skill, "scripts/xlsx.mjs"), ...args], { cwd: work, encoding: "utf8", env, maxBuffer: 64 * 1024 * 1024 });61  let json = null;62  try { json = JSON.parse(r.stdout); } catch { /* Markdown or usage text */ }63  return { status: r.status, out: r.stdout, err: r.stderr, json };64};6566test("SKILL.md follows the Agent Skills specification and every referenced file exists", () => {67  const md = readFileSync(join(skill, "SKILL.md"), "utf8");68  const fm = /^---\n([\s\S]*?)\n---\n/.exec(md)[1];69  const name = /^name: (.+)$/m.exec(fm)[1];70  assert.equal(name, "excel-model-workbook");71  assert.equal(name, basename(skill));72  assert.match(name, /^[a-z0-9]+(-[a-z0-9]+)*$/);73  const d = /^description: (.+)$/m.exec(fm)[1];74  assert.ok(d.length > 50 && d.length <= 1024, `description is ${d.length} characters`);75  assert.ok(/^compatibility: (.+)$/m.exec(fm)[1].length <= 500);76  assert.match(fm, /^license: MIT$/m);77  assert.ok(md.split("\n").length < 500);78  const refs = [...md.matchAll(/`((?:references|scripts|assets)\/[A-Za-z0-9_./-]+)`/g)].map((m) => m[1]);79  assert.ok(refs.length >= 6);80  for (const ref of refs) assert.ok(existsSync(join(skill, ref)), `${ref} is referenced but missing`);81  for (const f of readdirSync(join(skill, "references"))) assert.ok(md.includes(`references/${f}`), `${f} is not mentioned in SKILL.md`);82  const manifest = JSON.parse(readFileSync(join(skill, "agent-registry.json"), "utf8"));83  assert.deepEqual([manifest.name, manifest.type, manifest.entrypoint], ["@spboucher/excel-model-workbook", "skill", "SKILL.md"]);84  assert.equal(manifest.version, /version: "(.+)"/.exec(fm)[1]);85  assert.ok(existsSync(join(skill, "LICENSE")) && existsSync(join(skill, "README.md")) && existsSync(join(skill, "CHANGELOG.md")));86});8788test("the skill engine is byte-identical to the MCP engine and has no dependency", () => {89  assert.deepEqual(readdirSync(join(skill, "scripts/lib")).sort(), [...ENGINE].sort());90  for (const f of ENGINE) assert.ok(readFileSync(join(skill, "scripts/lib", f)).equals(readFileSync(join(mcp, "lib", f))), `${f} differs: run node build-skill.mjs`);91  for (const f of ["xlsx.mjs", ...ENGINE.map((e) => `lib/${e}`)]) {92    const src = readFileSync(join(skill, "scripts", f), "utf8");93    for (const m of src.matchAll(/^\s*import\s[^;]*?from\s+["']([^"']+)["']/gm)) assert.ok(m[1].startsWith("node:") || m[1].startsWith("./"), `${f} imports ${m[1]}`);94    assert.ok(!/\b(child_process|node:net|node:http|node:https|node:dgram|fetch\()/.test(src), `${f} must not spawn processes or use the network`);95    assert.ok(!/\beval\(|new Function\(/.test(src), `${f} must not evaluate code`);96  }97  assert.ok(!existsSync(join(skill, "node_modules")) && !existsSync(join(skill, "package.json")));98});99100test("documentation states facts the tool confirms", () => {101  const all = ["SKILL.md", "README.md", "references/modeling-standard.md", "references/audit-procedure.md", "references/grading.md", "references/defects.md"].map((f) => readFileSync(join(skill, f), "utf8")).join("\n");102  const help = cli("--help").out;103  const commands = [...help.matchAll(/^ {2}([a-z]+) +</gm)].map((m) => m[1]);104  assert.equal(commands.length, 14);105  for (const c of commands) assert.ok(new RegExp(`\`${c}[\` ]`).test(all), `command ${c} is not documented`);106  for (const m of all.matchAll(/node scripts\/xlsx\.mjs ([a-z]+)/g)) assert.ok(commands.includes(m[1]), `documentation uses unknown command ${m[1]}`);107  for (const v of ["correct", "correct_value_typed_as_constant", "carried_forward_error", "wrong", "missing", "unknown"]) assert.ok(all.includes(`\`${v}\``), v);108  for (const v of ["consistent", "consistent_where_checked", "stale_or_altered_values", "no_cached_values", "nothing_recomputable"]) assert.ok(all.includes(`\`${v}\``), v);109  assert.ok(!/[\u{1F300}-\u{1FAFF}]/u.test(all));110});111112test("command line: audit commands give the exact findings of the defective fixture", () => {113  assert.deepEqual(cli("summary", "defects.xlsx").json.totals, { sheets: 5, cells: 125, formulas: 60, error_cells: 3, formulas_without_cached_value: 0, circular_references: 1 });114  assert.deepEqual(cli("errors", "defects.xlsx").json.items.map((x) => [x.ref, x.root_cause]), [["Outputs!B4", true], ["Outputs!B6", true], ["Outputs!B7", false]]);115  assert.deepEqual(cli("circular", "defects.xlsx").json.items[0].loop, ["Calc!B9", "Calc!B10", "Calc!B9"]);116  const h = cli("hardcoded", "defects.xlsx", "--ignore", "0,1,2,1000").json;117  assert.deepEqual(h.constants_inside_formulas.items.map((x) => x.ref), ["Calc!E4", "Calc!B9"]);118  assert.equal(h.constants_where_a_formula_is_expected.items[0].expected_formula, "=D2*Inputs!$B$2");119  assert.equal(cli("consistency", "defects.xlsx", "--sheet", "Calc").json.items[0].expected_formula, "=E3*Inputs!$B$5");120  assert.equal(cli("consistency", "defects.xlsx", "--min-run", "7").json.inconsistent_cells, 0);121  assert.deepEqual(cli("controls", "defects.xlsx").json.failing_controls.map((c) => c.ref), ["Controls!E5"]);122  assert.equal(cli("controls", "clean.xlsx", "--ranges", "Outputs!B5").json.controls, 5);123  assert.equal(cli("recalc", "stale.xlsx").json.mismatches.items[0].ref, "Outputs!B5");124  assert.equal(cli("recalc", "nocache.xlsx").json.verdict, "no_cached_values");125  assert.equal(cli("range", "clean.xlsx", "Calc!B3:F3", "--limit", "2").json.next_offset, 2);126  assert.equal(cli("range", "clean.xlsx", "Calc!B3:F3", "--limit", "2", "--offset", "4").json.next_offset, null);127  assert.equal(cli("formulas", "clean.xlsx", "--sheet", "Calc", "--group").json.patterns, 7);128  assert.equal(cli("precedents", "clean.xlsx", "Outputs!B1", "--depth", "1").json.nodes[0].ref, "Calc!G3");129  assert.equal(cli("dependents", "clean.xlsx", "Inputs!B2", "--depth", "1").json.count, 6);130});131132test("command line: report as Markdown, JSON and file, never overwriting", () => {133  const md = cli("report", "defects.xlsx");134  assert.equal(md.status, 0);135  assert.match(md.out, /^# Workbook audit: defects\.xlsx/);136  assert.match(md.out, /\| Calc!D3 \| 110000 \| row \|/);137  assert.equal(cli("report", "defects.xlsx", "--json").json.findings.errors, 3);138  const w = cli("report", "clean.xlsx", "--out", "clean-audit.md");139  assert.equal(w.json.findings.errors, 0);140  assert.match(readFileSync(join(work, "clean-audit.md"), "utf8"), /Controls failing \/ unknown \/ total \| 0 \/ 0 \/ 4/);141  const again = cli("report", "clean.xlsx", "--out", "clean-audit.md");142  assert.deepEqual([again.status, again.json.error.code], [2, "exists"]);143  assert.equal(cli("report", "clean.xlsx", "--out", "a.txt").json.error.code, "bad_output");144});145146test("command line: compare and grade with the shipped rubric template", () => {147  const c = cli("compare", "reference.xlsx", "submission.xlsx").json;148  assert.deepEqual(c.counts, { right_by_formula: 21, right_but_typed_as_constant: 4, wrong: 31, missing: 1, unknown: 0 });149  const rubric = join(skill, "assets/rubric.example.json");150  const g = cli("grade", "reference.xlsx", "submission.xlsx", "--rubric", rubric, "--out", "evidence.md").json;151  assert.deepEqual([g.suggested_total, g.max_points], [20, 29]);152  assert.deepEqual(g.items.map((i) => i.awarded), [2, 5, 0, 4, 4, 3, 2]);153  assert.match(g.disclaimer, /human grader/);154  const ev = readFileSync(join(work, "evidence.md"), "utf8");155  assert.match(ev, /> This is a grading aid, not a grade/);156  assert.match(ev, /\| Calc!C2 \| correct_value_typed_as_constant \| 0\.5 \|/);157  assert.equal(cli("grade", "reference.xlsx", "clean.xlsx", "--rubric", rubric).json.suggested_total, 29);158  assert.equal(cli("grade", "reference.xlsx", "submission.xlsx").json.error.code, "rubric_invalid");159  writeFileSync(join(work, "bad.json"), "{ not json");160  assert.equal(cli("grade", "reference.xlsx", "submission.xlsx", "--rubric", "bad.json").json.error.code, "rubric_invalid");161});162163test("command line: errors are coded, exit status 2, and a root confines paths", () => {164  const cases = [[["summary", "missing.xlsx"], "not_found"], [["summary", "/etc/hosts"], "unsupported_type"], [["nonsense", "clean.xlsx"], "usage"], [["range", "clean.xlsx"], "usage"], [["summary"], "usage"], [["range", "clean.xlsx", "Nope!A1"], "sheet_not_found"]];165  for (const [args, code] of cases) { const r = cli(...args); assert.deepEqual([r.status, r.json.error.code], [2, code], args.join(" ")); }166  writeFileSync(join(work, "garbage.xlsx"), "definitely not a workbook, just text");167  assert.equal(cli("summary", "garbage.xlsx").json.error.code, "not_a_zip");168  const sub = mkdtempSync(join(work, "root-"));169  copyFileSync(join(fx, "clean.xlsx"), join(sub, "in.xlsx"));170  assert.equal(cli("summary", "in.xlsx", "--root", sub).json.totals.formulas, 57);171  assert.equal(cli("summary", join(work, "clean.xlsx"), "--root", sub).json.error.code, "outside_root");172  assert.equal(cli({ XLSX_AUDIT_ROOT: sub }, "summary", "../clean.xlsx").json.error.code, "outside_root");173  assert.equal(cli({ XLSX_AUDIT_ROOT: sub }, "report", "in.xlsx", "--out", "../escape.md").json.error.code, "outside_root");174  assert.ok(!existsSync(join(work, "escape.md")));175  assert.equal(cli({ XLSX_AUDIT_MAX_CELLS: "10" }, "summary", "clean.xlsx").json.error.code, "workbook_too_large");176  assert.equal(cli("--help").status, 0);177  assert.equal(cli().status, 2);178});179
Files main
Menu

Theme