Repository Public Research preview
1/**2 * =============================================================================3 * SP BOUCHER | SOFTWARE & RESEARCH4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : LLM API toolkits8 * File : protocol/mcp/test/offline.test.mjs9 * Module : protocol10 * Repository : https://git.spboucher.ai/llm-api-toolkits11 * Created : 2026-11-1012 * Last Modified : 2026-11-1013 *14 * Author : Simon-Pierre Boucher15 * Position : Professor of Finance16 * Institution : Université du Québec en Outaouais (UQO)17 * Organization : SP Boucher18 * Website : https://spboucher.ai19 * Git : https://git.spboucher.ai20 *21 * Preferred Contact: dev@spboucher.ai22 * Development : dev@spboucher.ai23 * Support : support@spboucher.ai24 * Security/Admin : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142// Offline tests of the ledger engine. Everything happens in temporary directories.43// Run: node --test test/offline.test.mjs44import assert from "node:assert/strict";45import { execFile } from "node:child_process";46import fs from "node:fs";47import { tmpdir } from "node:os";48import { join } from "node:path";49import { after, test } from "node:test";50import { fileURLToPath } from "node:url";51import { Ledger, LedgerError, buildState, canonical, classify, sha256 } from "../lib/ledger.mjs";5253const made = [];54after(() => { for (const d of made) fs.rmSync(d, { recursive: true, force: true }); });55function tmp(prefix = "ledger-") { const d = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), prefix))); made.push(d); return d; }56/** A project directory with a protocol, code and data, and a ledger on it. */57function project({ init = true, budget } = {}) {58 const root = tmp();59 fs.writeFileSync(join(root, "protocol.md"), "# Protocol\nH1: x raises y.\n");60 fs.writeFileSync(join(root, "analysis.py"), "print('estimate')\n");61 fs.writeFileSync(join(root, "data.csv"), "x,y\n1,2\n");62 const l = new Ledger({ root });63 if (init) l.init({ title: "Test study", description: "Does x raise y?", budget_usd: budget });64 return { root, l, file: join(root, ".research", "ledger.jsonl"), head: join(root, ".research", "HEAD") };65}66const H = (l, extra = {}) => l.registerHypothesis({ statement: "x raises y in the full sample", test: "OLS of y on x, clustered errors", decision_rule: "coefficient > 0 with one-sided p < 0.05", ...extra });67const D = (l, extra = {}) => l.logDecision({ question: "Winsorise at 1% or 5%?", options: ["1%", "5%"], choice: "1%", rationale: "Convention in the literature we extend.", timing: "before_outcome_data", ...extra });68const code = (fn) => { try { fn(); } catch (e) { assert.ok(e instanceof LedgerError, `expected a LedgerError, got ${e}`); return e.code; } return null; };69const lines = (file) => fs.readFileSync(file, "utf8").trimEnd().split("\n");70const tree = (dir, base = dir) => fs.readdirSync(dir, { withFileTypes: true }).flatMap((e) => (e.isDirectory() ? tree(join(dir, e.name), base) : [join(dir, e.name).slice(base.length + 1)])).sort();7172/* ───────── basics ───────── */7374test("canonical JSON is independent of key order", () => {75 assert.equal(canonical({ b: 1, a: [2, { d: 1, c: 2 }] }), canonical({ a: [2, { c: 2, d: 1 }], b: 1 }));76 assert.equal(canonical({ a: undefined, b: null }), '{"b":null}');77});7879test("init creates the ledger once and refuses a second time", () => {80 const { root, l, file } = project({ init: false });81 assert.equal(code(() => l.status()), "not_initialised");82 assert.equal(code(() => D(l)), "not_initialised");83 const r = l.init({ title: "Study", budget_usd: 50 });84 assert.equal(r.head.seq, 1);85 assert.equal(r.project.budget_usd, 50);86 assert.equal(code(() => l.init({ title: "Again" })), "already_initialised");87 assert.equal(lines(file).length, 1);88 assert.deepEqual(tree(join(root, ".research")), ["HEAD", "ledger.jsonl"]);89});9091test("a missing root is reported", () => {92 assert.equal(code(() => new Ledger({ root: join(tmpdir(), "does-not-exist-" + Date.now()) })), "root_missing");93});9495test("hypotheses are numbered, validated and never overwritten", () => {96 const { l } = project();97 assert.equal(H(l).registered, "H1");98 assert.equal(H(l, { role: "secondary", direction: "positive" }).registered, "H2");99 assert.equal(H(l, { id: "H3a" }).registered, "H3a");100 assert.equal(code(() => H(l, { id: "H1" })), "duplicate_id");101 assert.equal(code(() => H(l, { id: "first" })), "invalid_input");102 assert.equal(code(() => H(l, { statement: "short" })), "invalid_input");103 assert.equal(code(() => H(l, { direction: "up" })), "invalid_input");104});105106/* ───────── freeze, verify, amend ───────── */107108test("freeze hashes the files; a one-byte change is pinpointed", () => {109 const { root, l } = project();110 assert.equal(code(() => l.freeze({ files: ["protocol.md"] })), "no_hypotheses");111 H(l);112 assert.equal(code(() => l.verifyProtocol()), "not_frozen");113 const f = l.freeze({ files: ["protocol.md", "analysis.py"] });114 assert.equal(f.version, 1);115 assert.match(f.root_hash, /^[0-9a-f]{64}$/);116 assert.deepEqual(f.files.map((x) => x.path), ["analysis.py", "protocol.md"]);117 assert.equal(f.files[1].sha256, sha256(fs.readFileSync(join(root, "protocol.md"))));118 assert.equal(l.verifyProtocol().intact, true);119120 const p = join(root, "analysis.py"), original = fs.readFileSync(p);121 const flipped = Buffer.from(original); flipped[0] ^= 1; // same length, one bit122 fs.writeFileSync(p, flipped);123 const v = l.verifyProtocol();124 assert.equal(v.intact, false);125 assert.deepEqual(v.changed, ["analysis.py"]);126 const entry = v.files.find((x) => x.path === "analysis.py");127 assert.equal(entry.status, "changed");128 assert.equal(entry.frozen_bytes, entry.current_bytes);129 assert.notEqual(entry.frozen_sha256, entry.current_sha256);130 assert.equal(v.files.find((x) => x.path === "protocol.md").status, "unchanged");131 fs.writeFileSync(p, original);132 assert.equal(l.verifyProtocol().intact, true);133});134135test("a renamed protocol file is reported as missing, with where its content went", () => {136 const { root, l } = project();137 H(l); l.freeze({ files: ["protocol.md", "analysis.py"] });138 fs.renameSync(join(root, "protocol.md"), join(root, "protocol-v2.md"));139 const v = l.verifyProtocol();140 assert.deepEqual(v.changed, ["protocol.md"]);141 const e = v.files.find((x) => x.path === "protocol.md");142 assert.equal(e.status, "missing");143 assert.equal(e.same_content_found_at, "protocol-v2.md");144 fs.rmSync(join(root, "analysis.py"));145 const gone = l.verifyProtocol().files.find((x) => x.path === "analysis.py");146 assert.equal(gone.status, "missing");147 assert.equal(gone.same_content_found_at, undefined);148});149150test("the root hash depends on file names, file contents and hypotheses", () => {151 const roots = [];152 for (const variant of ["base", "content", "hypothesis", "name"]) {153 const { root, l } = project();154 if (variant === "content") fs.writeFileSync(join(root, "protocol.md"), "# Protocol\nH1: x lowers y.\n");155 if (variant === "name") fs.renameSync(join(root, "protocol.md"), join(root, "plan.md"));156 H(l, variant === "hypothesis" ? { direction: "negative" } : {});157 roots.push(l.freeze({ files: [variant === "name" ? "plan.md" : "protocol.md"] }).root_hash);158 }159 assert.equal(new Set(roots).size, 4);160 const again = project(); H(again.l);161 assert.equal(again.l.freeze({ files: ["protocol.md"] }).root_hash, roots[0], "same inputs give the same root hash");162});163164test("refreezing is refused; an amendment is a linked new version with a mandatory reason", () => {165 const { root, l } = project();166 H(l);167 const v1 = l.freeze({ files: ["protocol.md"] });168 assert.equal(code(() => l.freeze({ files: ["protocol.md"] })), "already_frozen");169 assert.equal(code(() => l.freeze({ files: ["protocol.md", "analysis.py"] })), "already_frozen");170 assert.equal(code(() => l.amend({ files: ["protocol.md"], data_seen: false })), "invalid_input", "reason is mandatory");171 assert.equal(code(() => l.amend({ files: ["protocol.md"], reason: "Clarify the sample definition." })), "invalid_input", "data_seen is mandatory");172 assert.equal(code(() => l.amend({ files: ["protocol.md"], reason: "Nothing really changed here.", data_seen: false })), "nothing_changed");173 assert.equal(code(() => l.amend({ files: ["protocol.md"], reason: "Clarify the sample definition.", data_seen: false, affects: ["H9"] })), "unknown_hypothesis");174 fs.appendFileSync(join(root, "protocol.md"), "Sample: firms with at least 12 months of data.\n");175 const v2 = l.amend({ files: ["protocol.md", "analysis.py"], reason: "Clarify the sample definition.", data_seen: false });176 assert.equal(v2.version, 2);177 assert.equal(v2.kind, "amendment");178 assert.notEqual(v2.root_hash, v1.root_hash);179 assert.deepEqual(v2.changed_files, { added: ["analysis.py"], removed: [], modified: ["protocol.md"] });180 const st = l.state();181 assert.equal(st.freezes[1].previous_root_hash, v1.root_hash);182 assert.equal(l.verifyProtocol().version, 2);183 assert.equal(l.verifyProtocol().intact, true);184});185186test("amend before any freeze is refused", () => {187 const { l } = project(); H(l);188 assert.equal(code(() => l.amend({ files: ["protocol.md"], reason: "There is nothing to amend yet.", data_seen: false })), "not_frozen");189});190191/* ───────── confirmatory vs exploratory ───────── */192193test("a result under a protocol frozen before data is confirmatory", () => {194 const { l } = project();195 H(l); l.freeze({ files: ["protocol.md", "analysis.py"] });196 l.markDataAccess({ description: "Opened the outcome file and ran the main regression." });197 const r = l.recordResult({ hypothesis: "H1", estimate: 0.11, uncertainty: "95% CI [0.02, 0.19]", verdict: "confirmed" });198 assert.equal(r.id, "X-01");199 assert.equal(r.status, "confirmatory");200});201202test("no freeze, or a freeze after data, gives exploratory results", () => {203 const a = project(); H(a.l);204 assert.equal(a.l.recordResult({ hypothesis: "H1", verdict: "confirmed" }).status, "exploratory");205 const b = project(); H(b.l);206 assert.match(b.l.markDataAccess({ description: "Looked at the outcome before freezing." }).warning, /before any protocol was frozen/);207 assert.match(b.l.freeze({ files: ["protocol.md"] }).warning, /after outcome data/);208 const r = b.l.recordResult({ hypothesis: "H1", verdict: "confirmed" });209 assert.equal(r.status, "exploratory");210 assert.match(r.status_reasons[0], /after outcome data/);211});212213test("data marked as not outcome data do not end the confirmatory window", () => {214 const { l } = project(); H(l);215 l.markDataAccess({ description: "Loaded covariates only, to check coverage.", outcome_data: false });216 assert.equal(l.freeze({ files: ["protocol.md"] }).warning, undefined);217 assert.equal(l.recordResult({ hypothesis: "H1", verdict: "not_confirmed" }).status, "confirmatory");218});219220test("an amendment after data makes only the affected hypotheses exploratory, from then on", () => {221 const { root, l } = project();222 H(l); H(l); H(l);223 l.freeze({ files: ["protocol.md", "analysis.py"] });224 const first = l.recordResult({ hypothesis: "H1", estimate: 0.11, verdict: "confirmed" });225 assert.equal(first.status, "confirmatory");226 fs.appendFileSync(join(root, "analysis.py"), "# H2 now uses a log transform\n");227 // Declared as "no data seen", but a result is already on record: the ledger does not accept that.228 const am = l.amend({ files: ["protocol.md", "analysis.py"], reason: "H2 is now tested on log outcomes.", data_seen: false, affects: ["H2"] });229 assert.match(am.warning, /already records access to outcome data/);230 assert.match(am.consequence, /H2/);231 const h2 = l.recordResult({ hypothesis: "H2", verdict: "confirmed" });232 assert.equal(h2.status, "exploratory");233 assert.match(h2.status_reasons.join(" "), /amended after outcome data were seen \(version 2\)/);234 assert.equal(l.recordResult({ hypothesis: "H3", verdict: "inconclusive" }).status, "confirmatory", "H3 is not affected");235 // The report keeps H1's earlier result confirmatory: it was recorded before the amendment.236 const rep = l.exportReport().json;237 assert.deepEqual(rep.results.map((x) => [x.hypothesis, x.status]), [["H1", "confirmatory"], ["H2", "exploratory"], ["H3", "confirmatory"]]);238 assert.equal(rep.protocol.versions[1].data_seen, true);239});240241test("an amendment after data with no hypothesis list affects all of them", () => {242 const { root, l } = project(); H(l); H(l);243 l.freeze({ files: ["protocol.md"] });244 l.markDataAccess({ description: "Ran the main specification." });245 fs.appendFileSync(join(root, "protocol.md"), "Changed estimator.\n");246 l.amend({ files: ["protocol.md"], reason: "Switched estimator after seeing the fit.", data_seen: true });247 assert.equal(l.recordResult({ hypothesis: "H1", verdict: "confirmed" }).status, "exploratory");248 assert.equal(l.recordResult({ hypothesis: "H2", verdict: "confirmed" }).status, "exploratory");249});250251test("a hypothesis added after the freeze is exploratory unless amended in before data", () => {252 const a = project(); H(a.l); a.l.freeze({ files: ["protocol.md"] });253 assert.match(H(a.l).warning, /after the protocol was frozen/);254 assert.deepEqual(a.l.verifyProtocol().hypotheses_registered_since, ["H2"]);255 const r = a.l.recordResult({ hypothesis: "H2", verdict: "confirmed" });256 assert.equal(r.status, "exploratory");257 assert.match(r.status_reasons[0], /registered after/);258259 const b = project(); H(b.l); b.l.freeze({ files: ["protocol.md"] }); H(b.l);260 b.l.amend({ files: ["protocol.md"], reason: "Add H2 before any data are opened.", data_seen: false });261 assert.equal(b.l.recordResult({ hypothesis: "H2", verdict: "confirmed" }).status, "confirmatory");262});263264test("deviations: minor keeps confirmatory with a flag, exploratory removes it, scoped by hypothesis", () => {265 const { l } = project(); H(l); H(l); H(l);266 l.freeze({ files: ["protocol.md"] });267 assert.equal(code(() => l.logDeviation({ protocol_said: "Use all firms.", done: "Dropped two firms.", reason: "Their identifiers were duplicated.", impact: "large" })), "invalid_input");268 assert.equal(code(() => l.logDeviation({ protocol_said: "Use all firms.", done: "Dropped two firms.", reason: "Their identifiers were duplicated.", impact: "minor", hypotheses: ["H7"] })), "unknown_hypothesis");269 assert.equal(l.logDeviation({ protocol_said: "Use all firms.", done: "Dropped two firms.", reason: "Their identifiers were duplicated.", impact: "minor", hypotheses: ["H1"] }).id, "V-01");270 assert.equal(l.logDeviation({ protocol_said: "Cluster by firm.", done: "Clustered by industry.", reason: "Too few firms per cluster to estimate.", impact: "exploratory", hypotheses: ["H2"] }).id, "V-02");271 assert.equal(l.recordResult({ hypothesis: "H1", verdict: "confirmed" }).status, "confirmatory_with_deviations");272 assert.equal(l.recordResult({ hypothesis: "H2", verdict: "confirmed" }).status, "exploratory");273 assert.equal(l.recordResult({ hypothesis: "H3", verdict: "not_confirmed" }).status, "confirmatory");274 // A deviation written down late still applies when the report is built.275 l.logDeviation({ protocol_said: "Two-sided test.", done: "One-sided test.", reason: "Noticed after the fact that the code used one side.", impact: "exploratory", hypotheses: ["H3"] });276 assert.equal(l.exportReport().json.results.find((x) => x.hypothesis === "H3").status, "exploratory");277});278279test("a silent change to a frozen file makes the result exploratory", () => {280 const { root, l } = project(); H(l);281 l.freeze({ files: ["protocol.md", "analysis.py"] });282 fs.appendFileSync(join(root, "analysis.py"), "# tweak\n");283 const r = l.recordResult({ hypothesis: "H1", verdict: "confirmed" });284 assert.equal(r.status, "exploratory");285 assert.match(r.status_reasons.join(" "), /no longer matched the frozen manifest/);286});287288test("results need a known hypothesis, a valid verdict and a known run", () => {289 const { l } = project(); H(l);290 assert.equal(code(() => l.recordResult({ hypothesis: "H4", verdict: "confirmed" })), "unknown_hypothesis");291 assert.equal(code(() => l.recordResult({ hypothesis: "H1", verdict: "significant" })), "invalid_input");292 assert.equal(code(() => l.recordResult({ hypothesis: "H1", verdict: "confirmed", run_id: "R-09" })), "unknown_run");293});294295test("classify is a pure function of the state", () => {296 const { l } = project(); H(l); l.freeze({ files: ["protocol.md"] });297 const st = l.state();298 assert.equal(classify(st, "H1").status, "confirmatory");299 assert.equal(classify(st, "H1", Infinity, false).status, "exploratory");300 assert.equal(classify(st, "H1", 2).status, "exploratory", "before the freeze record there is no protocol");301});302303/* ───────── decisions, gates, questions ───────── */304305test("decisions are numbered in order and a false 'before data' is flagged", () => {306 const { l } = project(); H(l); l.freeze({ files: ["protocol.md"] });307 assert.equal(D(l).id, "D-01");308 assert.equal(D(l).id, "D-02");309 assert.equal(code(() => D(l, { options: ["only one"] })), "invalid_input");310 assert.equal(code(() => D(l, { timing: "sometime" })), "invalid_input");311 assert.equal(code(() => D(l, { rationale: "" })), "invalid_input");312 l.markDataAccess({ description: "Opened the outcome file." });313 const late = D(l);314 assert.equal(late.id, "D-03");315 assert.match(late.warning, /already records access to outcome data/);316 assert.equal(D(l, { timing: "after_outcome_data" }).warning, undefined);317 assert.equal(l.state().decisions[2].timing_conflict, true);318});319320test("gates: criteria first, then evidence; no-go and conditional are recorded as such", () => {321 const { l } = project();322 assert.equal(code(() => l.closeGate({ name: "A", outcome: "go", evidence: ["n = 5 000"] })), "gate_not_open");323 assert.equal(code(() => l.openGate({ name: "A", criteria: [] })), "invalid_input");324 assert.equal(l.openGate({ name: "A", criteria: ["At least 2 000 usable pairs", "Pilot cost under 40 USD"] }).round, 1);325 assert.equal(code(() => l.openGate({ name: "A", criteria: ["again"] })), "gate_open");326 assert.equal(code(() => l.closeGate({ name: "A", outcome: "conditional", evidence: ["1 800 pairs"] })), "invalid_input");327 assert.equal(code(() => l.closeGate({ name: "A", outcome: "maybe", evidence: ["1 800 pairs"] })), "invalid_input");328 l.closeGate({ name: "A", outcome: "conditional", evidence: ["1 800 pairs", "Cost 22 USD"], conditions: ["Collect 200 more pairs"] });329 assert.match(l.status().next_steps.join(" "), /conditional: Collect 200 more pairs/);330 assert.equal(l.openGate({ name: "A", criteria: ["At least 2 000 usable pairs"] }).round, 2);331 assert.equal(l.closeGate({ name: "A", outcome: "go", evidence: ["2 050 pairs"] }).round, 2);332 l.openGate({ name: "B", criteria: ["Effect detectable in the pilot"] });333 assert.match(l.closeGate({ name: "B", outcome: "no_go", evidence: ["Pilot effect indistinguishable from zero"] }).consequence, /No-go is a result/);334 const s = l.status();335 assert.deepEqual(s.gates.map((g) => [g.name, g.round, g.state]), [["A", 2, "go"], ["B", 1, "no_go"]]);336 assert.match(s.next_steps.join(" "), /no-go: do not proceed/);337 assert.equal(l.exportReport().json.gates.length, 3);338});339340test("open questions appear in status until resolved", () => {341 const { l } = project();342 assert.equal(l.logQuestion({ question: "Is the 2019 file complete?" }).id, "Q-01");343 assert.equal(l.status().open_questions.length, 1);344 assert.equal(code(() => l.resolveQuestion({ id: "Q-07", answer: "Yes." })), "unknown_question");345 l.resolveQuestion({ id: "Q-01", answer: "Yes, 12 months present." });346 assert.equal(code(() => l.resolveQuestion({ id: "Q-01", answer: "Again." })), "already_resolved");347 assert.equal(l.status().open_questions.length, 0);348});349350/* ───────── runs ───────── */351352test("runs are fingerprinted; an identical expensive run is refused without a reason", () => {353 const { root, l } = project();354 fs.writeFileSync(join(root, "out.json"), '{"beta":0.11}\n');355 const spec = { command: "python analysis.py --seed 7", code_files: ["analysis.py"], input_files: ["data.csv"], seed: 7 };356 const pre = l.checkRerun(spec);357 assert.equal(pre.rerun_advised, true);358 assert.deepEqual(pre.identical_runs, []);359 const r1 = l.registerRun({ name: "Main estimation", ...spec, output_files: ["out.json"], expensive: true, deterministic: false, started_at: "2026-01-05T10:00:00Z", ended_at: "2026-01-05T10:20:00Z", environment: "python 3.12, numpy 2.1" });360 assert.equal(r1.id, "R-01");361 assert.equal(r1.fingerprint, pre.fingerprint);362 assert.equal(r1.outputs[0].sha256, sha256(fs.readFileSync(join(root, "out.json"))));363 const chk = l.checkRerun(spec);364 assert.equal(chk.rerun_advised, false);365 assert.match(chk.warning, /R-01/);366 assert.equal(chk.identical_runs[0].outputs[0].path, "out.json");367368 let err; try { l.registerRun({ name: "Again", ...spec, expensive: true }); } catch (e) { err = e; }369 assert.equal(err.code, "expensive_rerun");370 assert.deepEqual(err.identical_runs, ["R-01"]);371 const r2 = l.registerRun({ name: "Again", ...spec, rerun_reason: "The first output file was lost with the disk." });372 assert.equal(r2.id, "R-02");373 assert.deepEqual(r2.repeats, ["R-01"]);374375 // Any change to command, code, inputs or seed is a different run.376 assert.equal(l.checkRerun({ ...spec, seed: 8 }).rerun_advised, true);377 assert.equal(l.checkRerun({ ...spec, command: "python analysis.py --seed 7 --fast" }).rerun_advised, true);378 fs.appendFileSync(join(root, "data.csv"), "2,3\n");379 assert.equal(l.checkRerun(spec).rerun_advised, true);380 assert.notEqual(l.checkRerun(spec).fingerprint, r1.fingerprint);381});382383test("cheap or failed runs can be repeated freely; inputs are validated", () => {384 const { l } = project();385 const spec = { command: "python analysis.py", code_files: ["analysis.py"] };386 l.registerRun({ name: "Cheap", ...spec });387 assert.equal(l.registerRun({ name: "Cheap again", ...spec }).id, "R-02");388 l.registerRun({ name: "Costly, crashed", command: "python fit.py", expensive: true, status: "failed" });389 assert.equal(l.checkRerun({ command: "python fit.py" }).rerun_advised, true, "a failed expensive run does not block a retry");390 assert.equal(l.registerRun({ name: "Costly, retried", command: "python fit.py", expensive: true }).id, "R-04");391 assert.equal(code(() => l.registerRun({ name: "Bad", command: "x", code_files: ["missing.py"] })), "file_missing");392 assert.equal(code(() => l.registerRun({ name: "Bad", command: "x", started_at: "yesterday" })), "invalid_input");393 assert.equal(code(() => l.registerRun({ name: "Bad", command: "x", code_files: ["analysis.py", "./analysis.py"] })), "invalid_input");394 assert.equal(code(() => l.registerRun({ name: "Bad", command: "x", status: "running" })), "invalid_input");395});396397/* ───────── spending ───────── */398399test("spend arithmetic is exact and the budget is tracked", () => {400 const { l } = project({ budget: 1 });401 l.registerRun({ name: "Extraction", command: "node extract.mjs", expensive: true });402 // 0.1 + 0.2 is 0.30000000000000004 in floating point: totals must not show that.403 assert.equal(l.recordSpend({ provider: "api-a", model: "model-x", purpose: "extraction", cost_usd: 0.1, input_tokens: 1000, output_tokens: 200, run_id: "R-01" }).total_usd, 0.1);404 const two = l.recordSpend({ provider: "api-a", model: "model-x", purpose: "extraction", cost_usd: 0.2, input_tokens: 2000, output_tokens: 300 });405 assert.equal(two.total_usd, 0.3);406 assert.equal(two.remaining_usd, 0.7);407 for (let i = 0; i < 10; i++) l.recordSpend({ provider: "api-b", model: "model-y", purpose: "judging", cost_usd: 0.01 });408 let rep = l.spendReport();409 assert.equal(rep.entries, 12);410 assert.equal(rep.total_usd, 0.4);411 assert.equal(rep.remaining_usd, 0.6);412 assert.equal(rep.budget_used_pct, 40);413 assert.equal(rep.over_budget, false);414 assert.deepEqual(rep.by_provider, [{ provider: "api-a", cost_usd: 0.3, entries: 2, input_tokens: 3000, output_tokens: 500 }, { provider: "api-b", cost_usd: 0.1, entries: 10, input_tokens: 0, output_tokens: 0 }]);415 assert.deepEqual(rep.by_purpose.map((x) => [x.purpose, x.cost_usd]), [["extraction", 0.3], ["judging", 0.1]]);416 assert.equal(rep.by_provider.reduce((s, x) => s + Math.round(x.cost_usd * 1e6), 0), 400000);417418 const over = l.recordSpend({ provider: "api-b", purpose: "judging", cost_usd: 0.600001 });419 assert.equal(over.total_usd, 1.000001);420 assert.equal(over.remaining_usd, -0.000001);421 assert.match(over.warning, /exceeds the declared budget/);422 assert.match(l.status().next_steps.join(" "), /over budget/);423424 l.setBudget({ budget_usd: 5, reason: "Second wave approved after gate B." });425 rep = l.spendReport();426 assert.equal(rep.budget_usd, 5);427 assert.equal(rep.remaining_usd, 3.999999);428 assert.equal(rep.over_budget, false);429430 assert.equal(code(() => l.recordSpend({ provider: "api-a", purpose: "x", cost_usd: -1 })), "invalid_input");431 assert.equal(code(() => l.recordSpend({ provider: "api-a", purpose: "x", cost_usd: Number.NaN })), "invalid_input");432 assert.equal(code(() => l.recordSpend({ provider: "api-a", purpose: "x", cost_usd: 1, input_tokens: 1.5 })), "invalid_input");433 assert.equal(code(() => l.recordSpend({ provider: "api-a", purpose: "x", cost_usd: 1, run_id: "R-99" })), "unknown_run");434 assert.equal(code(() => l.setBudget({ budget_usd: 9 })), "invalid_input", "a budget change needs a reason");435});436437test("without a budget the report says so instead of inventing one", () => {438 const { l } = project();439 l.recordSpend({ provider: "api-a", purpose: "pilot", cost_usd: 2.5 });440 const rep = l.spendReport();441 assert.equal(rep.budget_usd, null);442 assert.equal(rep.remaining_usd, null);443 assert.equal(rep.over_budget, false);444 assert.match(l.exportReport().markdown, /No budget was declared/);445});446447/* ───────── tamper evidence ───────── */448449function filled() {450 const p = project(); H(p.l); p.l.freeze({ files: ["protocol.md"] });451 for (let i = 0; i < 5; i++) D(p.l);452 return p; // 8 records: init, hypothesis, freeze, D-01…D-05453}454455test("an intact ledger verifies", () => {456 const { l } = filled();457 const v = l.verifyLedger();458 assert.equal(v.ok, true);459 assert.equal(v.records_verified, 8);460 assert.equal(v.first_problem, null);461 assert.deepEqual(v.warnings, []);462 assert.match(v.guarantee, /rewrites the whole chain/);463});464465test("an edited record is pinpointed", () => {466 const { l, file } = filled();467 const ls = lines(file);468 ls[4] = ls[4].replace('"choice":"1%"', '"choice":"5%"');469 fs.writeFileSync(file, ls.join("\n") + "\n");470 const v = l.verifyLedger();471 assert.equal(v.ok, false);472 assert.equal(v.records_verified, 4);473 assert.deepEqual([v.first_problem.seq, v.first_problem.line, v.first_problem.kind], [5, 5, "content_changed"]);474});475476test("an edit that also recomputes the record's own hash breaks the next link", () => {477 const { l, file } = filled();478 const ls = lines(file);479 const r = JSON.parse(ls[4]); r.data.choice = "5%";480 r.hash = sha256(canonical({ seq: r.seq, ts: r.ts, type: r.type, data: r.data, prev: r.prev }));481 ls[4] = JSON.stringify(r);482 fs.writeFileSync(file, ls.join("\n") + "\n");483 const v = l.verifyLedger();484 assert.deepEqual([v.first_problem.seq, v.first_problem.kind], [6, "chain_break"]);485});486487test("a deleted record is pinpointed", () => {488 const { l, file } = filled();489 const ls = lines(file); ls.splice(3, 1);490 fs.writeFileSync(file, ls.join("\n") + "\n");491 const v = l.verifyLedger();492 assert.equal(v.records_verified, 3);493 assert.deepEqual([v.first_problem.line, v.first_problem.kind], [4, "sequence_break"]);494});495496test("two swapped records are pinpointed at the first of them", () => {497 const { l, file } = filled();498 const ls = lines(file); [ls[4], ls[5]] = [ls[5], ls[4]];499 fs.writeFileSync(file, ls.join("\n") + "\n");500 const v = l.verifyLedger();501 assert.equal(v.records_verified, 4);502 assert.deepEqual([v.first_problem.line, v.first_problem.kind], [5, "sequence_break"]);503});504505test("truncation is detected: whole records removed, and a record cut in the middle", () => {506 const a = filled();507 fs.writeFileSync(a.file, lines(a.file).slice(0, 6).join("\n") + "\n");508 const v = a.l.verifyLedger();509 assert.equal(v.ok, false);510 assert.deepEqual([v.first_problem.seq, v.first_problem.kind], [7, "tail_missing"]);511 assert.match(v.first_problem.detail, /had 8 records but only 6 remain/);512513 const b = filled();514 const raw = fs.readFileSync(b.file, "utf8");515 fs.writeFileSync(b.file, raw.slice(0, raw.length - 40));516 const w = b.l.verifyLedger();517 assert.deepEqual([w.first_problem.seq, w.first_problem.kind], [8, "invalid_json"]);518 assert.match(w.first_problem.detail, /truncated/);519 assert.equal(w.records_verified, 7);520});521522test("a rewritten last record and an emptied ledger are detected through HEAD", () => {523 const a = filled();524 const ls = lines(a.file);525 const r = JSON.parse(ls[7]); r.data.choice = "5%";526 r.hash = sha256(canonical({ seq: r.seq, ts: r.ts, type: r.type, data: r.data, prev: r.prev }));527 ls[7] = JSON.stringify(r);528 fs.writeFileSync(a.file, ls.join("\n") + "\n");529 assert.deepEqual([a.l.verifyLedger().first_problem.seq, a.l.verifyLedger().first_problem.kind], [8, "head_mismatch"]);530531 const b = filled();532 fs.writeFileSync(b.file, "");533 assert.deepEqual([b.l.verifyLedger().first_problem.seq, b.l.verifyLedger().first_problem.kind], [1, "tail_missing"]);534});535536test("an inserted record is detected", () => {537 const { l, file } = filled();538 const ls = lines(file); ls.splice(2, 0, ls[1]);539 fs.writeFileSync(file, ls.join("\n") + "\n");540 assert.deepEqual([l.verifyLedger().first_problem.line, l.verifyLedger().first_problem.kind], [3, "sequence_break"]);541});542543test("a corrupt ledger refuses appends and says where it broke", () => {544 const { l, file } = filled();545 const before = lines(file); before.splice(3, 1);546 fs.writeFileSync(file, before.join("\n") + "\n");547 const frozen = fs.readFileSync(file, "utf8");548 let err; try { D(l); } catch (e) { err = e; }549 assert.equal(err.code, "ledger_corrupt");550 assert.equal(err.problem.line, 4);551 assert.equal(code(() => l.spendReport()), "ledger_corrupt");552 assert.equal(code(() => l.exportReport()), "ledger_corrupt");553 assert.equal(fs.readFileSync(file, "utf8"), frozen, "nothing was appended");554 const s = l.status();555 assert.equal(s.ledger_ok, false);556 assert.equal(s.problem.line, 4);557});558559test("a missing HEAD is a warning, repaired by the next append", () => {560 const { l, head } = filled();561 fs.rmSync(head);562 const v = l.verifyLedger();563 assert.equal(v.ok, true);564 assert.equal(v.warnings[0].kind, "head_missing");565 D(l);566 assert.deepEqual(l.verifyLedger().warnings, []);567 assert.equal(JSON.parse(fs.readFileSync(head, "utf8")).seq, 9);568});569570/* ───────── atomic writes, crashes, locking ───────── */571572test("a writer that dies before the rename leaves the ledger untouched", () => {573 const { root, l, file } = filled();574 const before = fs.readFileSync(file);575 const real = fs.renameSync;576 fs.renameSync = () => { throw new Error("simulated crash before rename"); };577 try { assert.throws(() => D(l), /simulated crash/); } finally { fs.renameSync = real; }578 assert.deepEqual(fs.readFileSync(file), before);579 assert.equal(fs.existsSync(join(root, ".research", "lock")), false, "the lock is released");580 assert.equal(fs.readdirSync(join(root, ".research")).filter((f) => f.includes(".tmp-")).length, 1, "the half-written temp file is all that is left");581 assert.equal(l.verifyLedger().ok, true);582 assert.equal(D(l).id, "D-06", "the lost append did not consume a number");583 assert.equal(fs.readdirSync(join(root, ".research")).filter((f) => f.includes(".tmp-")).length, 0, "the next writer removes the leftover");584 assert.equal(l.verifyLedger().records_verified, 9);585});586587test("a crash between the ledger write and the HEAD write is recovered", () => {588 const { l, head } = filled();589 const real = fs.renameSync; let n = 0;590 fs.renameSync = (a, b) => { if (++n === 2) throw new Error("simulated crash before HEAD"); return real(a, b); };591 try { assert.throws(() => D(l), /simulated crash/); } finally { fs.renameSync = real; }592 const v = l.verifyLedger();593 assert.equal(v.ok, true);594 assert.equal(v.records_verified, 9, "the record itself was committed");595 assert.equal(v.warnings[0].kind, "head_stale");596 assert.equal(D(l).id, "D-07");597 assert.deepEqual(l.verifyLedger().warnings, []);598 assert.equal(JSON.parse(fs.readFileSync(head, "utf8")).seq, 10);599});600601test("a stray partial temp file never counts as ledger content", () => {602 const { root, l } = filled();603 fs.writeFileSync(join(root, ".research", "ledger.jsonl.tmp-999-deadbeef"), '{"seq":9,"ts":"2026-01-01T00:00:00.000Z","type":"decision","da');604 assert.equal(l.verifyLedger().ok, true);605 assert.equal(l.status().records, 8);606});607608test("a stale lock from a dead writer is taken over", () => {609 const { root, l } = filled();610 const lock = join(root, ".research", "lock");611 fs.writeFileSync(lock, "99999\n");612 const old = new Date(Date.now() - 120000);613 fs.utimesSync(lock, old, old);614 assert.equal(D(l).id, "D-06");615 assert.equal(fs.existsSync(lock), false);616});617618test("decision numbers stay unique and ordered under concurrent writers", async () => {619 const { root, l, file } = project();620 const worker = fileURLToPath(new URL("./fixtures/append-worker.mjs", import.meta.url));621 const WORKERS = 6, EACH = 8;622 const runs = await Promise.all(Array.from({ length: WORKERS }, (_, w) => new Promise((res, rej) => execFile(process.execPath, [worker, root, String(w), String(EACH)], { encoding: "utf8" }, (err, stdout, stderr) => (err ? rej(new Error(stderr || err.message)) : res(JSON.parse(stdout)))))));623 const returned = runs.flat();624 assert.equal(returned.length, WORKERS * EACH);625 assert.equal(new Set(returned).size, WORKERS * EACH, "no two writers received the same number");626 for (const ids of runs) assert.deepEqual(ids, [...ids].sort(), "each writer saw its own numbers increase");627 const v = l.verifyLedger();628 assert.equal(v.ok, true);629 assert.equal(v.records_verified, 1 + WORKERS * EACH);630 assert.deepEqual(v.warnings, []);631 const inFile = lines(file).map((x) => JSON.parse(x)).filter((r) => r.type === "decision").map((r) => r.data.id);632 assert.deepEqual(inFile, Array.from({ length: WORKERS * EACH }, (_, i) => `D-${String(i + 1).padStart(2, "0")}`));633 assert.deepEqual(fs.readdirSync(join(root, ".research")).sort(), ["HEAD", "ledger.jsonl"]);634});635636/* ───────── path confinement ───────── */637638test("files outside the project root are refused, including through links", () => {639 const { root, l } = project(); H(l);640 const outside = tmp("outside-");641 fs.writeFileSync(join(outside, "secret.txt"), "not part of the project\n");642 fs.symlinkSync(join(outside, "secret.txt"), join(root, "link.txt"));643 fs.symlinkSync(outside, join(root, "linkdir"));644 fs.symlinkSync(join(root, "protocol.md"), join(root, "alias.md"));645 fs.mkdirSync(join(root, "sub"));646 for (const p of ["../" + outside.split("/").pop() + "/secret.txt", join(outside, "secret.txt"), "link.txt", "linkdir/secret.txt", "sub/../../" + outside.split("/").pop() + "/secret.txt"]) {647 assert.equal(code(() => l.freeze({ files: [p] })), "path_forbidden", p);648 assert.equal(code(() => l.registerRun({ name: "x", command: "x", input_files: [p] })), "path_forbidden", p);649 assert.equal(code(() => l.checkRerun({ command: "x", code_files: [p] })), "path_forbidden", p);650 }651 assert.equal(code(() => l.freeze({ files: [".research/ledger.jsonl"] })), "path_forbidden");652 assert.equal(code(() => l.freeze({ files: ["sub"] })), "not_a_file");653 assert.equal(code(() => l.freeze({ files: ["nope.md"] })), "file_missing");654 assert.equal(code(() => l.freeze({ files: ["a\0b"] })), "invalid_input");655 // A link that stays inside the root is fine and is recorded under its real path.656 assert.deepEqual(l.freeze({ files: ["alias.md"] }).files.map((f) => f.path), ["protocol.md"]);657});658659test("a .research that is a link to somewhere else is refused", () => {660 const root = tmp(), elsewhere = tmp("elsewhere-");661 fs.symlinkSync(elsewhere, join(root, ".research"));662 const l = new Ledger({ root });663 assert.equal(code(() => l.init({ title: "Study" })), "path_forbidden");664 assert.deepEqual(fs.readdirSync(elsewhere), []);665});666667test("nothing outside .research is created or modified", () => {668 const { root, l } = project({ budget: 10 });669 const snapshot = () => Object.fromEntries(tree(root).filter((f) => !f.startsWith(".research/")).map((f) => [f, sha256(fs.readFileSync(join(root, f)))]));670 const before = snapshot();671 H(l); l.freeze({ files: ["protocol.md", "analysis.py"] }); l.verifyProtocol();672 l.markDataAccess({ description: "Opened the outcome file." }); D(l, { timing: "after_outcome_data" });673 l.openGate({ name: "A", criteria: ["c"] }); l.closeGate({ name: "A", outcome: "go", evidence: ["e"] });674 l.registerRun({ name: "Run", command: "python analysis.py", code_files: ["analysis.py"], input_files: ["data.csv"] });675 l.recordSpend({ provider: "api-a", purpose: "pilot", cost_usd: 1 }); l.recordResult({ hypothesis: "H1", verdict: "inconclusive" });676 l.status(); l.verifyLedger(); l.exportReport({ write: true });677 assert.deepEqual(snapshot(), before);678 assert.deepEqual(tree(join(root, ".research")), ["HEAD", "ledger.jsonl", "reports/report-10.json", "reports/report-10.md"]);679});680681/* ───────── status and report ───────── */682683test("status carries what a new session needs, from the files alone", () => {684 const { root, l } = project({ budget: 20 });685 assert.match(l.status().next_steps[0], /Register the hypotheses/);686 H(l); H(l, { role: "secondary" });687 assert.match(l.status().next_steps[0], /Freeze the protocol/);688 l.freeze({ files: ["protocol.md", "analysis.py"] });689 l.openGate({ name: "A", criteria: ["Enough data"] });690 l.recordSpend({ provider: "api-a", purpose: "pilot", cost_usd: 3.25 });691 l.recordResult({ hypothesis: "H1", estimate: 0.11, verdict: "confirmed" });692 fs.appendFileSync(join(root, "analysis.py"), "# edit\n");693694 const s = new Ledger({ root }).status(); // a fresh object: nothing is kept in memory695 assert.equal(s.ledger_ok, true);696 assert.equal(s.project.title, "Test study");697 assert.equal(s.records, 7);698 assert.equal(s.protocol.frozen, true);699 assert.equal(s.protocol.intact, false);700 assert.deepEqual(s.protocol.changed_files, ["analysis.py"]);701 assert.equal(s.outcome_data_accessed, true);702 assert.deepEqual(s.hypotheses.map((h) => [h.id, h.role, h.result]), [["H1", "primary", "confirmed (confirmatory)"], ["H2", "secondary", null]]);703 assert.deepEqual(s.gates, [{ name: "A", round: 1, state: "open", criteria: ["Enough data"] }]);704 assert.deepEqual(s.spend, { total_usd: 3.25, budget_usd: 20, remaining_usd: 16.75 });705 assert.equal(s.recent.at(-1).what, "X-01: H1 confirmed (confirmatory)");706 const next = s.next_steps.join(" | ");707 assert.match(next, /Protocol files changed since the freeze \(analysis\.py\)/);708 assert.match(next, /Gate "A" is open/);709});710711test("the report shows every hypothesis, including null and unreported ones", () => {712 const { l } = project({ budget: 10 });713 H(l); H(l, { statement: "x raises z | in small firms" }); H(l);714 l.freeze({ files: ["protocol.md"] });715 D(l);716 l.logDeviation({ protocol_said: "Use all firms.", done: "Dropped two firms.", reason: "Their identifiers were duplicated.", impact: "minor", hypotheses: ["H1"] });717 l.openGate({ name: "A", criteria: ["Enough data"] }); l.closeGate({ name: "A", outcome: "go", evidence: ["n = 5 000"] });718 l.registerRun({ name: "Main", command: "python analysis.py", code_files: ["analysis.py"], expensive: true });719 l.recordSpend({ provider: "api-a", purpose: "pilot", cost_usd: 1.5 });720 l.recordResult({ hypothesis: "H1", estimate: 0.11, uncertainty: "SE 0.04", verdict: "confirmed", run_id: "R-01" });721 l.recordResult({ hypothesis: "H2", estimate: -0.01, uncertainty: "SE 0.05", verdict: "not_confirmed" });722 l.logQuestion({ question: "Is the 2019 file complete?" });723 const { markdown: md, json, written } = l.exportReport({ write: true });724 assert.deepEqual(written, [".research/reports/report-14.md", ".research/reports/report-14.json"]);725 for (const part of ["# Transparency report — Test study", "## Protocol", "## Hypotheses and results", "## Decisions", "## Deviations from the protocol", "## Gates", "## Runs", "## Spending", "## Open questions", "## How to check this report"]) assert.ok(md.includes(part), part);726 assert.match(md, /\| H1 \| primary \|.*\| confirmed \| confirmatory with deviations \| 0\.11 \| SE 0\.04 \|/);727 assert.match(md, /\| H2 \|.*x raises z \\\| in small firms.*\| not confirmed \| confirmatory \| -0\.01 \|/);728 assert.match(md, /\| H3 \|.*\| not reported \|/);729 assert.match(md, /Registered but not reported: H3/);730 assert.match(md, /\*\*D-01\*\*/);731 assert.match(md, /\*\*V-01\*\* \(impact: minor; H1\)/);732 assert.match(md, /Total: \*\*1\.5 USD\*\* over 1 entries\. Budget: 10 USD; remaining: 8\.5 USD\./);733 assert.ok(md.includes(json.ledger.head.hash));734 assert.equal(json.protocol.current_files_intact, true);735 assert.equal(json.hypotheses.length, 3);736 assert.equal(json.results.find((x) => x.hypothesis === "H1").status_now, undefined);737 const saved = JSON.parse(fs.readFileSync(join(l.dir, "reports", "report-14.json"), "utf8"));738 assert.deepEqual(saved.results, json.results);739 assert.equal(fs.readFileSync(join(l.dir, "reports", "report-14.md"), "utf8"), md);740 assert.equal(l.verifyLedger().records_verified, 14, "writing a report appends nothing");741});742743test("a report without a freeze says that every result is exploratory", () => {744 const { l } = project(); H(l);745 l.recordResult({ hypothesis: "H1", verdict: "confirmed" });746 assert.match(l.exportReport().markdown, /\*\*No protocol was frozen\.\*\* Every result below is exploratory\./);747});748749test("records from a newer engine are kept in the chain and ignored", () => {750 const st = buildState([{ seq: 1, ts: "t", type: "init", data: { title: "T", budget_micro_usd: null } }, { seq: 2, ts: "t", type: "something_new", data: {} }]);751 assert.equal(st.project.title, "T");752});753754test("timestamps come from the injected clock and are part of the hash", () => {755 const root = tmp(); let t = Date.parse("2026-02-01T00:00:00Z");756 const l = new Ledger({ root, now: () => new Date((t += 60000)) });757 l.init({ title: "Clock" });758 const recs = l.read().records;759 assert.equal(recs[0].ts, "2026-02-01T00:01:00.000Z");760 assert.equal(recs[0].prev, "0".repeat(64));761});762