Repository Public Research preview
1/**2 * =============================================================================3 * SP BOUCHER | SOFTWARE & RESEARCH4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : LLM API toolkits8 * File : protocol/mcp/test/protocol.test.mjs9 * Module : protocol10 * Repository : https://git.spboucher.ai/llm-api-toolkits11 * Created : 2026-11-1012 * Last Modified : 2026-11-1013 *14 * Author : Simon-Pierre Boucher15 * Position : Professor of Finance16 * Institution : Université du Québec en Outaouais (UQO)17 * Organization : SP Boucher18 * Website : https://spboucher.ai19 * Git : https://git.spboucher.ai20 *21 * Preferred Contact: dev@spboucher.ai22 * Development : dev@spboucher.ai23 * Support : support@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142// The server over real stdio, with the official MCP client: every tool is called at least once.43// Run: node --test test/protocol.test.mjs44import assert from "node:assert/strict";45import fs from "node:fs";46import { tmpdir } from "node:os";47import { join } from "node:path";48import { after, before, test } from "node:test";49import { fileURLToPath } from "node:url";50import { Client } from "@modelcontextprotocol/sdk/client/index.js";51import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";5253const serverPath = fileURLToPath(new URL("../server.mjs", import.meta.url));54const root = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), "ledger-mcp-")));55const linkDir = fs.mkdtempSync(join(tmpdir(), "ledger-link-"));56let client;57const connect = async (path, env) => { const c = new Client({ name: "test", version: "1.0.0" }); await c.connect(new StdioClientTransport({ command: process.execPath, args: [path], env: { PATH: process.env.PATH, ...env }, stderr: "pipe" })); return c; };58const call = async (name, args = {}) => { const r = await client.callTool({ name, arguments: args }); return { error: r.isError ? r.structuredContent.error : null, data: r.structuredContent, text: r.content[0].text }; };5960before(async () => {61 fs.writeFileSync(join(root, "protocol.md"), "# Protocol\n");62 fs.writeFileSync(join(root, "analysis.py"), "print(1)\n");63 fs.writeFileSync(join(root, "out.json"), "{}\n");64 client = await connect(serverPath, { RESEARCH_LEDGER_ROOT: root });65});66after(async () => { await client?.close(); fs.rmSync(root, { recursive: true, force: true }); fs.rmSync(linkDir, { recursive: true, force: true }); });6768const EXPECTED = ["init_project", "register_hypothesis", "freeze_protocol", "verify_protocol", "amend_protocol", "mark_data_access", "log_decision", "log_deviation", "open_gate", "close_gate", "check_rerun", "register_run", "record_spend", "spend_report", "set_budget", "record_result", "log_question", "resolve_question", "status", "export_report", "verify_ledger"];6970test("the server lists its tools with descriptions, schemas and annotations", async () => {71 const { tools } = await client.listTools();72 assert.deepEqual(tools.map((t) => t.name).sort(), [...EXPECTED].sort());73 for (const t of tools) {74 assert.ok(t.description.length > 40, `${t.name}: description too short`);75 assert.ok(t.title, `${t.name}: no title`);76 assert.equal(t.inputSchema.type, "object");77 assert.equal(typeof t.annotations.readOnlyHint, "boolean", `${t.name}: no annotations`);78 assert.equal(t.annotations.openWorldHint, false, `${t.name}: the server never leaves the machine`);79 assert.equal(t.annotations.destructiveHint, false, `${t.name}: nothing is ever deleted`);80 }81 const readOnly = tools.filter((t) => t.annotations.readOnlyHint).map((t) => t.name).sort();82 assert.deepEqual(readOnly, ["check_rerun", "spend_report", "status", "verify_ledger", "verify_protocol"]);83 assert.match(client.getInstructions(), /Start every session with status/);84 assert.equal(client.getServerVersion().name, "research-ledger");85});8687test("errors come back as structured tool errors, not protocol failures", async () => {88 const r = await call("status");89 assert.equal(r.error.code, "not_initialised");90 assert.match(r.text, /init_project/);91 const bad = await client.callTool({ name: "log_decision", arguments: { question: "q" } });92 assert.equal(bad.isError, true, "schema violations are reported to the caller");93});9495test("read-only tools leave the project untouched", async () => {96 await call("verify_ledger");97 assert.equal(fs.existsSync(join(root, ".research")), false);98});99100test("a whole study through the protocol", async () => {101 const called = new Set();102 const go = async (name, args) => { called.add(name); const r = await call(name, args); assert.equal(r.error, null, `${name}: ${r.text}`); return r.data; };103104 assert.equal((await go("init_project", { title: "Protocol study", budget_usd: 10, outcome_data: "The y column." })).head.seq, 1);105 assert.equal((await go("register_hypothesis", { statement: "x raises y in the full sample", test: "OLS of y on x", decision_rule: "coefficient > 0, p < 0.05" })).registered, "H1");106 assert.equal((await go("register_hypothesis", { statement: "x raises y more in small units", test: "Interaction term", decision_rule: "interaction > 0, p < 0.05", role: "secondary" })).registered, "H2");107 assert.equal((await go("log_question", { question: "Is the 2019 file complete?" })).id, "Q-01");108 assert.equal((await go("resolve_question", { id: "Q-01", answer: "Yes, all months present." })).resolved, true);109 assert.equal((await go("open_gate", { name: "A", criteria: ["Design reviewed before data"] })).round, 1);110 assert.equal((await go("close_gate", { name: "A", outcome: "go", evidence: ["Two reviewers signed off on the design"] })).outcome, "go");111 const fz = await go("freeze_protocol", { files: ["protocol.md", "analysis.py"] });112 assert.equal(fz.version, 1);113 assert.equal((await call("freeze_protocol", { files: ["protocol.md"] })).error.code, "already_frozen");114 assert.equal((await go("verify_protocol")).intact, true);115 assert.equal((await go("log_decision", { question: "Cluster by unit or by period?", options: ["unit", "period"], choice: "unit", rationale: "Treatment varies at the unit level.", timing: "before_outcome_data" })).id, "D-01");116 assert.equal((await go("mark_data_access", { description: "Opened the outcome column for the first time." })).outcome_data, true);117118 const spec = { command: "python analysis.py", code_files: ["analysis.py"], seed: 1 };119 assert.equal((await go("check_rerun", spec)).rerun_advised, true);120 assert.equal((await go("register_run", { name: "Main", ...spec, output_files: ["out.json"], expensive: true })).id, "R-01");121 assert.equal((await go("check_rerun", spec)).rerun_advised, false);122 const again = await call("register_run", { name: "Main again", ...spec, expensive: true });123 assert.equal(again.error.code, "expensive_rerun");124 assert.deepEqual(again.error.identical_runs, ["R-01"]);125126 assert.equal((await go("record_spend", { provider: "api-a", model: "model-x", purpose: "annotation", cost_usd: 0.1, run_id: "R-01" })).total_usd, 0.1);127 assert.equal((await go("record_spend", { provider: "api-a", model: "model-x", purpose: "annotation", cost_usd: 0.2 })).total_usd, 0.3);128 assert.equal((await go("set_budget", { budget_usd: 12, reason: "Second annotation pass approved." })).previous_usd, 10);129 assert.equal((await go("spend_report")).remaining_usd, 11.7);130131 assert.equal((await go("record_result", { hypothesis: "H1", estimate: 0.11, uncertainty: "SE 0.04", verdict: "confirmed", run_id: "R-01" })).status, "confirmatory");132 assert.equal((await go("log_deviation", { protocol_said: "Interaction with a size dummy.", done: "Interaction with log size.", reason: "The dummy had almost no variation.", impact: "exploratory", hypotheses: ["H2"] })).id, "V-01");133 fs.appendFileSync(join(root, "analysis.py"), "# log size\n");134 assert.deepEqual((await go("verify_protocol")).changed, ["analysis.py"]);135 const am = await go("amend_protocol", { files: ["protocol.md", "analysis.py"], reason: "H2 uses log size instead of a dummy.", data_seen: true, affects: ["H2"] });136 assert.equal(am.version, 2);137 assert.equal((await go("record_result", { hypothesis: "H2", estimate: "0.02", verdict: "inconclusive" })).status, "exploratory");138139 const st = await go("status");140 assert.equal(st.ledger_ok, true);141 assert.equal(st.protocol.version, 2);142 assert.deepEqual(st.hypotheses.map((h) => h.result), ["confirmed (confirmatory)", "inconclusive (exploratory)"]);143 const rep = await go("export_report", { write: true });144 assert.match(rep.markdown, /# Transparency report — Protocol study/);145 assert.equal(rep.written.length, 2);146 assert.ok(fs.existsSync(join(root, rep.written[0])));147 const v = await go("verify_ledger");148 assert.equal(v.ok, true);149 assert.equal(v.records_verified, st.records);150 assert.deepEqual([...called].sort(), [...EXPECTED].sort(), "every tool was exercised");151});152153test("tampering is reported through the protocol, and appends are refused", async () => {154 const file = join(root, ".research", "ledger.jsonl");155 const original = fs.readFileSync(file, "utf8");156 fs.writeFileSync(file, original.replace('"choice":"unit"', '"choice":"period"'));157 const v = await call("verify_ledger");158 assert.equal(v.data.ok, false);159 assert.equal(v.data.first_problem.kind, "content_changed");160 const refused = await call("log_question", { question: "Can anything be appended now?" });161 assert.equal(refused.error.code, "ledger_corrupt");162 assert.equal(refused.error.problem.seq, v.data.first_problem.seq);163 fs.writeFileSync(file, original);164 assert.equal((await call("verify_ledger")).data.ok, true);165});166167test("paths outside the root are refused through the protocol", async () => {168 const r = await call("check_rerun", { command: "x", code_files: ["../../etc/hosts"] });169 assert.ok(["path_forbidden", "file_missing"].includes(r.error.code));170 const abs = await call("check_rerun", { command: "x", code_files: ["/etc/hosts"] });171 assert.equal(abs.error.code, "path_forbidden");172});173174test("the server starts when launched through a symlinked path", async () => {175 const link = join(linkDir, "pkg");176 fs.symlinkSync(fileURLToPath(new URL("..", import.meta.url)), link);177 const c = await connect(join(link, "server.mjs"), { RESEARCH_LEDGER_ROOT: root });178 try { assert.equal((await c.listTools()).tools.length, EXPECTED.length); } finally { await c.close(); }179});180181test("without RESEARCH_LEDGER_ROOT the working directory is the project", async () => {182 const dir = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), "ledger-cwd-")));183 const c = new Client({ name: "test", version: "1.0.0" });184 await c.connect(new StdioClientTransport({ command: process.execPath, args: [serverPath], cwd: dir, env: { PATH: process.env.PATH }, stderr: "pipe" }));185 try {186 const r = await c.callTool({ name: "init_project", arguments: { title: "Cwd study" } });187 assert.equal(r.isError, undefined);188 assert.ok(fs.existsSync(join(dir, ".research", "ledger.jsonl")));189 } finally { await c.close(); fs.rmSync(dir, { recursive: true, force: true }); }190});191