SPB Git

Repository Public Research preview

skill.test.mjs 11.9 KB 178 lines javascript

1/**2 * =============================================================================3 *                  SP BOUCHER | SOFTWARE & RESEARCH4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : LLM API toolkits8 * File             : scholarly/skill-tests/skill.test.mjs9 * Module           : scholarly10 * Repository       : https://git.spboucher.ai/llm-api-toolkits11 * Created          : 2026-11-1012 * Last Modified    : 2026-11-1013 *14 * Author           : Simon-Pierre Boucher15 * Position         : Professor of Finance16 * Institution      : Université du Québec en Outaouais (UQO)17 * Organization     : SP Boucher18 * Website          : https://spboucher.ai19 * Git              : https://git.spboucher.ai20 *21 * Preferred Contact: dev@spboucher.ai22 * Development      : dev@spboucher.ai23 * Support          : support@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142// Tests of the skill: specification compliance, engine identity with the MCP package, and the43// command-line tool against the real services (Crossref, arXiv, doi.org; no OpenAlex call).44// Run: node --test skill-tests/skill.test.mjs45import assert from "node:assert/strict";46import { execFile } from "node:child_process";47import { existsSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";48import { tmpdir } from "node:os";49import { join } from "node:path";50import { test } from "node:test";51import { fileURLToPath } from "node:url";52import { ENGINE_FILES } from "../build-skill.mjs";5354const skill = fileURLToPath(new URL("../skill/citation-verification/", import.meta.url));55const work = mkdtempSync(join(tmpdir(), "citation-skill-"));56const B = "10.1016/0304-4076(86)90063-1", W = "10.1016/s0140-6736(97)11096-0", FAKE = "10.1234/this-doi-does-not-exist-zz9";57const TITLE = "Generalized autoregressive conditional heteroskedasticity";58const cli = (...args) => new Promise((res) => execFile(process.execPath, [join(skill, "scripts/scholarly.mjs"), ...args], { cwd: work, encoding: "utf8", maxBuffer: 32 * 1024 * 1024, timeout: 180_000, env: { PATH: process.env.PATH } }, (err, stdout, stderr) => {59  let json = null;60  try { json = JSON.parse(stdout); } catch { /* usage text or Markdown */ }61  res({ status: err ? (typeof err.code === "number" ? err.code : 1) : 0, stdout, stderr, json });62}));6364test("SKILL.md follows the Agent Skills specification and every referenced file exists", () => {65  const md = readFileSync(join(skill, "SKILL.md"), "utf8");66  const fm = /^---\n([\s\S]*?)\n---\n/.exec(md)[1];67  assert.equal(/^name: (.+)$/m.exec(fm)[1], "citation-verification");68  const d = /^description: (.+)$/m.exec(fm)[1];69  assert.ok(d.length > 50 && d.length <= 1024, `description is ${d.length} characters`);70  assert.match(fm, /^license: MIT$/m);71  assert.match(fm, /^compatibility: .{10,500}$/m);72  assert.ok(md.split("\n").length < 500);73  const refs = [...md.matchAll(/`((?:references|scripts)\/[A-Za-z0-9_./-]+)`/g)].map((m) => m[1]);74  assert.ok(refs.length >= 4);75  for (const r of refs) assert.ok(existsSync(join(skill, r)), `${r} is referenced but missing`);76  for (const f of readdirSync(join(skill, "references"))) assert.ok(md.includes(`references/${f}`), `${f} is not mentioned in SKILL.md`);77  const manifest = JSON.parse(readFileSync(join(skill, "agent-registry.json"), "utf8"));78  assert.deepEqual([manifest.name, manifest.type, manifest.entrypoint], ["@spboucher/citation-verification", "skill", "SKILL.md"]);79  assert.equal(/version: "(.+)"/.exec(fm)[1], manifest.version);80  assert.doesNotMatch(md, /[\u{1F300}-\u{1FAFF}☀-➿]/u); // no emoji81});8283test("the skill engine is byte-identical to the MCP engine and has no dependencies", () => {84  assert.deepEqual(readdirSync(join(skill, "scripts/lib")).sort(), [...ENGINE_FILES].sort());85  assert.deepEqual(readdirSync(new URL("../mcp/lib/", import.meta.url)).sort(), [...ENGINE_FILES].sort()); // nothing in the MCP engine is left behind86  for (const f of ENGINE_FILES) assert.equal(readFileSync(join(skill, "scripts/lib", f), "utf8"), readFileSync(new URL(`../mcp/lib/${f}`, import.meta.url), "utf8"), f);87  for (const f of ["scripts/scholarly.mjs", ...ENGINE_FILES.map((x) => `scripts/lib/${x}`)]) for (const m of readFileSync(join(skill, f), "utf8").matchAll(/^import .* from "([^"]+)";$/gm)) assert.ok(m[1].startsWith("node:") || m[1].startsWith("./"), `${f} imports ${m[1]}`);88  assert.ok(!existsSync(join(skill, "node_modules")));89});9091test("every command of the usage text is implemented, and every documented command exists", async () => {92  const help = await cli("help");93  assert.equal(help.status, 0);94  const src = readFileSync(join(skill, "scripts/scholarly.mjs"), "utf8");95  const cases = [...src.matchAll(/case "([a-z-]+)":/g)].map((m) => m[1]).sort();96  const usage = [...help.stdout.matchAll(/^ {2}([a-z][a-z-]+)(?= |$)/gm)].map((m) => m[1]).sort();97  assert.deepEqual(usage, cases);98  assert.equal(cases.length, 16);99  const docs = ["SKILL.md", "references/statuses.md", "references/sources.md", "references/workflows.md", "README.md"].map((f) => readFileSync(join(skill, f), "utf8")).join("\n");100  for (const m of docs.matchAll(/scholarly\.mjs ([a-z][a-z-]+)/g)) assert.ok(cases.includes(m[1]) || m[1] === "help", `documented command "${m[1]}" does not exist`);101  for (const m of docs.matchAll(/--([a-z][a-z-]+)/g)) assert.ok(help.stdout.includes(`--${m[1]}`), `documented option --${m[1]} is not in the usage text`);102  const st = await cli("status");103  assert.deepEqual([st.json.config.mailto_sent, st.json.config.openalex_api_key_set, st.json.http.requests], [false, false, 0]);104});105106test("usage errors exit 2, lookup errors exit 1 with a JSON error", async () => {107  assert.equal((await cli()).status, 0); // no command prints the usage108  const unknown = await cli("frobnicate");109  assert.deepEqual([unknown.status, /unknown command/.test(unknown.stderr)], [2, true]);110  assert.equal((await cli("verify-doi")).status, 2);111  assert.equal((await cli("verify-doi", B, "--year")).status, 2);112  assert.equal((await cli("verify-doi", B, "--year", "abc")).status, 2);113  assert.equal((await cli("retractions")).status, 2);114  const bad = await cli("get", "not an identifier");115  assert.deepEqual([bad.status, bad.json.error.code], [1, "invalid_input"]);116  const missing = await cli("parse-bib", "nothing-here.bib");117  assert.deepEqual([missing.status, missing.json.error.code], [1, "not_found"]);118  writeFileSync(join(work, "notes.env"), "x");119  assert.equal((await cli("parse-bib", "notes.env")).json.error.code, "path_not_allowed");120});121122test("verify-doi against the live registries: verified, mismatch, fabricated, retracted", async () => {123  const ok = await cli("verify-doi", B, "--title", TITLE, "--author", "Bollerslev, T.", "--year", "1986", "--venue", "J. Econometrics");124  assert.deepEqual([ok.status, ok.json.status, ok.json.verdict], [0, "verified", "match"]);125  const year = await cli("verify-doi", B, "--title", TITLE, "--author", "Bollerslev", "--year", "1996");126  assert.deepEqual([year.json.status, year.json.mismatches], ["metadata_mismatch", ["year"]]);127  const fake = await cli("verify-doi", FAKE, "--title", "A study that does not exist");128  assert.deepEqual([fake.status, fake.json.status, fake.json.doi_resolves], [0, "not_found", false]);129  const ret = await cli("retractions", W, B, "--no-openalex");130  assert.deepEqual(ret.json.results.map((r) => r.status), ["retracted", "no_notice"]);131});132133test("verify-bib on a mixed file: JSON, Markdown report, --strict, --keys, retractions --bib", async () => {134  writeFileSync(join(work, "refs.bib"), `135@article{bollerslev1986, author = {Bollerslev, Tim}, title = {Generalized Autoregressive Conditional Heteroskedasticity}, journal = {Journal of Econometrics}, year = {1986}, doi = {${B}}}136@article{bollerslev_wrong, author = {Bollerslev, Tim}, title = {Generalized Autoregressive Conditional Heteroskedasticity}, year = {1996}, doi = {${B}}}137@article{engle1982, author = {Engle, Robert F.}, title = {Autoregressive Conditional Heteroscedasticity with Estimates of the Variance of {U}nited {K}ingdom Inflation}, journal = {Econometrica}, year = {1982}}138@article{ghost2021, author = {Smith, John}, title = {Quantum hedonic regression of lunar real estate under stochastic cheese constraints}, year = {2021}, doi = {${FAKE}}}139@article{wakefield1998, author = {Wakefield, A. J.}, title = {Ileal-lymphoid-nodular hyperplasia, non-specific colitis, and pervasive developmental disorder in children}, year = {1998}, doi = {${W}}}140@article{unclosed, title = {This entry never ends141`);142  const p = await cli("parse-bib", "refs.bib");143  assert.deepEqual([p.json.entries_total, p.json.without_doi, p.json.parse_errors.length], [5, ["engle1982"], 1]);144  const r = await cli("verify-bib", "refs.bib");145  assert.equal(r.status, 0);146  assert.deepEqual(Object.fromEntries(r.json.results.map((x) => [x.key, x.status])), { bollerslev1986: "verified", bollerslev_wrong: "metadata_mismatch", engle1982: "doi_missing_found", ghost2021: "doi_not_found", wakefield1998: "retracted" });147  assert.equal(r.json.results.find((x) => x.key === "engle1982").suggested_doi, "10.2307/1912773");148149  const md = await cli("verify-bib", "refs.bib", "--format", "md", "--out", join(work, "audit.md"), "--strict");150  assert.equal(md.status, 3); // --strict: entries need attention151  assert.deepEqual(md.json.needs_attention, ["bollerslev_wrong", "ghost2021", "wakefield1998"]);152  const report = readFileSync(join(work, "audit.md"), "utf8");153  assert.match(report, /\| bollerslev_wrong \| metadata_mismatch \|.*year: cited "1996", registered "1986"/);154  assert.match(report, /\| wakefield1998 \| retracted \|.*10\.1016\/s0140-6736\(10\)60175-4/);155  assert.match(report, /\| engle1982 \| 10\.2307\/1912773 \|/);156  assert.match(report, /BibTeX syntax errors[\s\S]*line 7/);157  const again = await cli("verify-bib", "refs.bib", "--format", "md", "--out", join(work, "audit.md"));158  assert.equal(again.status, 1); // an existing report is never overwritten159160  const one = await cli("verify-bib", "refs.bib", "--keys", "bollerslev1986", "--strict");161  assert.deepEqual([one.status, one.json.entries_checked, one.json.summary], [0, 1, { verified: 1 }]);162  const ret = await cli("retractions", "--bib", "refs.bib", "--no-openalex");163  assert.deepEqual([ret.json.checked, ret.json.flagged], [3, [W]]); // three distinct DOIs in the file164});165166test("resolve, cite, bibtex and arXiv from the live registries", async () => {167  const none = await cli("resolve", "Smith, J. (2021). Quantum hedonic regression of lunar real estate under stochastic cheese constraints. Journal of Imaginary Results, 12(3), 45-67.");168  assert.deepEqual([none.json.status, none.json.confident], ["not_found", false]);169  const found = await cli("resolve", "--title", TITLE, "--author", "Bollerslev", "--year", "1986");170  assert.deepEqual([found.json.status, found.json.match.doi], ["found", B]);171  const cite = await cli("cite", B, "--style", "apa");172  assert.equal(cite.json.citation, `Bollerslev, T. (1986). ${TITLE}. Journal of Econometrics, 31(3), 307–327. https://doi.org/${B}`);173  const bib = await cli("bibtex", B, "--generated", "--key", "garch86");174  assert.match(bib.json.bibtex, /^@article\{garch86,\n {2}author = \{Bollerslev, Tim\}/);175  const ax = await cli("get", "arXiv:1706.03762");176  assert.deepEqual([ax.json.work.title, ax.json.work.year, ax.json.work.type], ["Attention Is All You Need", 2017, "preprint"]);177});178
Files main
Menu

Theme