Repository Public Research preview
1/**2 * =============================================================================3 * SP BOUCHER | SOFTWARE & RESEARCH4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : LLM API toolkits8 * File : scholarly/skill-tests/skill.test.mjs9 * Module : scholarly10 * Repository : https://git.spboucher.ai/llm-api-toolkits11 * Created : 2026-11-1012 * Last Modified : 2026-11-1013 *14 * Author : Simon-Pierre Boucher15 * Position : Professor of Finance16 * Institution : Université du Québec en Outaouais (UQO)17 * Organization : SP Boucher18 * Website : https://spboucher.ai19 * Git : https://git.spboucher.ai20 *21 * Preferred Contact: dev@spboucher.ai22 * Development : dev@spboucher.ai23 * Support : support@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142// Tests of the skill: specification compliance, engine identity with the MCP package, and the43// command-line tool against the real services (Crossref, arXiv, doi.org; no OpenAlex call).44// Run: node --test skill-tests/skill.test.mjs45import assert from "node:assert/strict";46import { execFile } from "node:child_process";47import { existsSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";48import { tmpdir } from "node:os";49import { join } from "node:path";50import { test } from "node:test";51import { fileURLToPath } from "node:url";52import { ENGINE_FILES } from "../build-skill.mjs";5354const skill = fileURLToPath(new URL("../skill/citation-verification/", import.meta.url));55const work = mkdtempSync(join(tmpdir(), "citation-skill-"));56const B = "10.1016/0304-4076(86)90063-1", W = "10.1016/s0140-6736(97)11096-0", FAKE = "10.1234/this-doi-does-not-exist-zz9";57const TITLE = "Generalized autoregressive conditional heteroskedasticity";58const cli = (...args) => new Promise((res) => execFile(process.execPath, [join(skill, "scripts/scholarly.mjs"), ...args], { cwd: work, encoding: "utf8", maxBuffer: 32 * 1024 * 1024, timeout: 180_000, env: { PATH: process.env.PATH } }, (err, stdout, stderr) => {59 let json = null;60 try { json = JSON.parse(stdout); } catch { /* usage text or Markdown */ }61 res({ status: err ? (typeof err.code === "number" ? err.code : 1) : 0, stdout, stderr, json });62}));6364test("SKILL.md follows the Agent Skills specification and every referenced file exists", () => {65 const md = readFileSync(join(skill, "SKILL.md"), "utf8");66 const fm = /^---\n([\s\S]*?)\n---\n/.exec(md)[1];67 assert.equal(/^name: (.+)$/m.exec(fm)[1], "citation-verification");68 const d = /^description: (.+)$/m.exec(fm)[1];69 assert.ok(d.length > 50 && d.length <= 1024, `description is ${d.length} characters`);70 assert.match(fm, /^license: MIT$/m);71 assert.match(fm, /^compatibility: .{10,500}$/m);72 assert.ok(md.split("\n").length < 500);73 const refs = [...md.matchAll(/`((?:references|scripts)\/[A-Za-z0-9_./-]+)`/g)].map((m) => m[1]);74 assert.ok(refs.length >= 4);75 for (const r of refs) assert.ok(existsSync(join(skill, r)), `${r} is referenced but missing`);76 for (const f of readdirSync(join(skill, "references"))) assert.ok(md.includes(`references/${f}`), `${f} is not mentioned in SKILL.md`);77 const manifest = JSON.parse(readFileSync(join(skill, "agent-registry.json"), "utf8"));78 assert.deepEqual([manifest.name, manifest.type, manifest.entrypoint], ["@spboucher/citation-verification", "skill", "SKILL.md"]);79 assert.equal(/version: "(.+)"/.exec(fm)[1], manifest.version);80 assert.doesNotMatch(md, /[\u{1F300}-\u{1FAFF}☀-➿]/u); // no emoji81});8283test("the skill engine is byte-identical to the MCP engine and has no dependencies", () => {84 assert.deepEqual(readdirSync(join(skill, "scripts/lib")).sort(), [...ENGINE_FILES].sort());85 assert.deepEqual(readdirSync(new URL("../mcp/lib/", import.meta.url)).sort(), [...ENGINE_FILES].sort()); // nothing in the MCP engine is left behind86 for (const f of ENGINE_FILES) assert.equal(readFileSync(join(skill, "scripts/lib", f), "utf8"), readFileSync(new URL(`../mcp/lib/${f}`, import.meta.url), "utf8"), f);87 for (const f of ["scripts/scholarly.mjs", ...ENGINE_FILES.map((x) => `scripts/lib/${x}`)]) for (const m of readFileSync(join(skill, f), "utf8").matchAll(/^import .* from "([^"]+)";$/gm)) assert.ok(m[1].startsWith("node:") || m[1].startsWith("./"), `${f} imports ${m[1]}`);88 assert.ok(!existsSync(join(skill, "node_modules")));89});9091test("every command of the usage text is implemented, and every documented command exists", async () => {92 const help = await cli("help");93 assert.equal(help.status, 0);94 const src = readFileSync(join(skill, "scripts/scholarly.mjs"), "utf8");95 const cases = [...src.matchAll(/case "([a-z-]+)":/g)].map((m) => m[1]).sort();96 const usage = [...help.stdout.matchAll(/^ {2}([a-z][a-z-]+)(?= |$)/gm)].map((m) => m[1]).sort();97 assert.deepEqual(usage, cases);98 assert.equal(cases.length, 16);99 const docs = ["SKILL.md", "references/statuses.md", "references/sources.md", "references/workflows.md", "README.md"].map((f) => readFileSync(join(skill, f), "utf8")).join("\n");100 for (const m of docs.matchAll(/scholarly\.mjs ([a-z][a-z-]+)/g)) assert.ok(cases.includes(m[1]) || m[1] === "help", `documented command "${m[1]}" does not exist`);101 for (const m of docs.matchAll(/--([a-z][a-z-]+)/g)) assert.ok(help.stdout.includes(`--${m[1]}`), `documented option --${m[1]} is not in the usage text`);102 const st = await cli("status");103 assert.deepEqual([st.json.config.mailto_sent, st.json.config.openalex_api_key_set, st.json.http.requests], [false, false, 0]);104});105106test("usage errors exit 2, lookup errors exit 1 with a JSON error", async () => {107 assert.equal((await cli()).status, 0); // no command prints the usage108 const unknown = await cli("frobnicate");109 assert.deepEqual([unknown.status, /unknown command/.test(unknown.stderr)], [2, true]);110 assert.equal((await cli("verify-doi")).status, 2);111 assert.equal((await cli("verify-doi", B, "--year")).status, 2);112 assert.equal((await cli("verify-doi", B, "--year", "abc")).status, 2);113 assert.equal((await cli("retractions")).status, 2);114 const bad = await cli("get", "not an identifier");115 assert.deepEqual([bad.status, bad.json.error.code], [1, "invalid_input"]);116 const missing = await cli("parse-bib", "nothing-here.bib");117 assert.deepEqual([missing.status, missing.json.error.code], [1, "not_found"]);118 writeFileSync(join(work, "notes.env"), "x");119 assert.equal((await cli("parse-bib", "notes.env")).json.error.code, "path_not_allowed");120});121122test("verify-doi against the live registries: verified, mismatch, fabricated, retracted", async () => {123 const ok = await cli("verify-doi", B, "--title", TITLE, "--author", "Bollerslev, T.", "--year", "1986", "--venue", "J. Econometrics");124 assert.deepEqual([ok.status, ok.json.status, ok.json.verdict], [0, "verified", "match"]);125 const year = await cli("verify-doi", B, "--title", TITLE, "--author", "Bollerslev", "--year", "1996");126 assert.deepEqual([year.json.status, year.json.mismatches], ["metadata_mismatch", ["year"]]);127 const fake = await cli("verify-doi", FAKE, "--title", "A study that does not exist");128 assert.deepEqual([fake.status, fake.json.status, fake.json.doi_resolves], [0, "not_found", false]);129 const ret = await cli("retractions", W, B, "--no-openalex");130 assert.deepEqual(ret.json.results.map((r) => r.status), ["retracted", "no_notice"]);131});132133test("verify-bib on a mixed file: JSON, Markdown report, --strict, --keys, retractions --bib", async () => {134 writeFileSync(join(work, "refs.bib"), `135@article{bollerslev1986, author = {Bollerslev, Tim}, title = {Generalized Autoregressive Conditional Heteroskedasticity}, journal = {Journal of Econometrics}, year = {1986}, doi = {${B}}}136@article{bollerslev_wrong, author = {Bollerslev, Tim}, title = {Generalized Autoregressive Conditional Heteroskedasticity}, year = {1996}, doi = {${B}}}137@article{engle1982, author = {Engle, Robert F.}, title = {Autoregressive Conditional Heteroscedasticity with Estimates of the Variance of {U}nited {K}ingdom Inflation}, journal = {Econometrica}, year = {1982}}138@article{ghost2021, author = {Smith, John}, title = {Quantum hedonic regression of lunar real estate under stochastic cheese constraints}, year = {2021}, doi = {${FAKE}}}139@article{wakefield1998, author = {Wakefield, A. J.}, title = {Ileal-lymphoid-nodular hyperplasia, non-specific colitis, and pervasive developmental disorder in children}, year = {1998}, doi = {${W}}}140@article{unclosed, title = {This entry never ends141`);142 const p = await cli("parse-bib", "refs.bib");143 assert.deepEqual([p.json.entries_total, p.json.without_doi, p.json.parse_errors.length], [5, ["engle1982"], 1]);144 const r = await cli("verify-bib", "refs.bib");145 assert.equal(r.status, 0);146 assert.deepEqual(Object.fromEntries(r.json.results.map((x) => [x.key, x.status])), { bollerslev1986: "verified", bollerslev_wrong: "metadata_mismatch", engle1982: "doi_missing_found", ghost2021: "doi_not_found", wakefield1998: "retracted" });147 assert.equal(r.json.results.find((x) => x.key === "engle1982").suggested_doi, "10.2307/1912773");148149 const md = await cli("verify-bib", "refs.bib", "--format", "md", "--out", join(work, "audit.md"), "--strict");150 assert.equal(md.status, 3); // --strict: entries need attention151 assert.deepEqual(md.json.needs_attention, ["bollerslev_wrong", "ghost2021", "wakefield1998"]);152 const report = readFileSync(join(work, "audit.md"), "utf8");153 assert.match(report, /\| bollerslev_wrong \| metadata_mismatch \|.*year: cited "1996", registered "1986"/);154 assert.match(report, /\| wakefield1998 \| retracted \|.*10\.1016\/s0140-6736\(10\)60175-4/);155 assert.match(report, /\| engle1982 \| 10\.2307\/1912773 \|/);156 assert.match(report, /BibTeX syntax errors[\s\S]*line 7/);157 const again = await cli("verify-bib", "refs.bib", "--format", "md", "--out", join(work, "audit.md"));158 assert.equal(again.status, 1); // an existing report is never overwritten159160 const one = await cli("verify-bib", "refs.bib", "--keys", "bollerslev1986", "--strict");161 assert.deepEqual([one.status, one.json.entries_checked, one.json.summary], [0, 1, { verified: 1 }]);162 const ret = await cli("retractions", "--bib", "refs.bib", "--no-openalex");163 assert.deepEqual([ret.json.checked, ret.json.flagged], [3, [W]]); // three distinct DOIs in the file164});165166test("resolve, cite, bibtex and arXiv from the live registries", async () => {167 const none = await cli("resolve", "Smith, J. (2021). Quantum hedonic regression of lunar real estate under stochastic cheese constraints. Journal of Imaginary Results, 12(3), 45-67.");168 assert.deepEqual([none.json.status, none.json.confident], ["not_found", false]);169 const found = await cli("resolve", "--title", TITLE, "--author", "Bollerslev", "--year", "1986");170 assert.deepEqual([found.json.status, found.json.match.doi], ["found", B]);171 const cite = await cli("cite", B, "--style", "apa");172 assert.equal(cite.json.citation, `Bollerslev, T. (1986). ${TITLE}. Journal of Econometrics, 31(3), 307–327. https://doi.org/${B}`);173 const bib = await cli("bibtex", B, "--generated", "--key", "garch86");174 assert.match(bib.json.bibtex, /^@article\{garch86,\n {2}author = \{Bollerslev, Tim\}/);175 const ax = await cli("get", "arXiv:1706.03762");176 assert.deepEqual([ax.json.work.title, ax.json.work.year, ax.json.work.type], ["Attention Is All You Need", 2017, "preprint"]);177});178