SPB Git

Repository Public Research preview

offline.test.mjs 42.6 KB 762 lines javascript

1/**2 * =============================================================================3 *                  SP BOUCHER | SOFTWARE & RESEARCH4 *             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project          : LLM API toolkits8 * File             : protocol/mcp/test/offline.test.mjs9 * Module           : protocol10 * Repository       : https://git.spboucher.ai/llm-api-toolkits11 * Created          : 2026-11-1012 * Last Modified    : 2026-11-1013 *14 * Author           : Simon-Pierre Boucher15 * Position         : Professor of Finance16 * Institution      : Université du Québec en Outaouais (UQO)17 * Organization     : SP Boucher18 * Website          : https://spboucher.ai19 * Git              : https://git.spboucher.ai20 *21 * Preferred Contact: dev@spboucher.ai22 * Development      : dev@spboucher.ai23 * Support          : support@spboucher.ai24 * Security/Admin   : admin@spboucher.ai25 *26 * Copyright (c) 202627 * Simon-Pierre Boucher28 * All rights reserved where applicable. This file is governed by the29 * applicable project license, copyright notices and relevant law.30 * Existing third-party rights and notices remain unaffected.31 *32 * License          : None specified (no license file in this repository)33 *34 * Do not embed credentials, private keys, access tokens,35 * certificates or confidential personal information.36 *37 * =============================================================================38 * END OF SP BOUCHER SOURCE HEADER39 * =============================================================================40 */4142// Offline tests of the ledger engine. Everything happens in temporary directories.43// Run: node --test test/offline.test.mjs44import assert from "node:assert/strict";45import { execFile } from "node:child_process";46import fs from "node:fs";47import { tmpdir } from "node:os";48import { join } from "node:path";49import { after, test } from "node:test";50import { fileURLToPath } from "node:url";51import { Ledger, LedgerError, buildState, canonical, classify, sha256 } from "../lib/ledger.mjs";5253const made = [];54after(() => { for (const d of made) fs.rmSync(d, { recursive: true, force: true }); });55function tmp(prefix = "ledger-") { const d = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), prefix))); made.push(d); return d; }56/** A project directory with a protocol, code and data, and a ledger on it. */57function project({ init = true, budget } = {}) {58  const root = tmp();59  fs.writeFileSync(join(root, "protocol.md"), "# Protocol\nH1: x raises y.\n");60  fs.writeFileSync(join(root, "analysis.py"), "print('estimate')\n");61  fs.writeFileSync(join(root, "data.csv"), "x,y\n1,2\n");62  const l = new Ledger({ root });63  if (init) l.init({ title: "Test study", description: "Does x raise y?", budget_usd: budget });64  return { root, l, file: join(root, ".research", "ledger.jsonl"), head: join(root, ".research", "HEAD") };65}66const H = (l, extra = {}) => l.registerHypothesis({ statement: "x raises y in the full sample", test: "OLS of y on x, clustered errors", decision_rule: "coefficient > 0 with one-sided p < 0.05", ...extra });67const D = (l, extra = {}) => l.logDecision({ question: "Winsorise at 1% or 5%?", options: ["1%", "5%"], choice: "1%", rationale: "Convention in the literature we extend.", timing: "before_outcome_data", ...extra });68const code = (fn) => { try { fn(); } catch (e) { assert.ok(e instanceof LedgerError, `expected a LedgerError, got ${e}`); return e.code; } return null; };69const lines = (file) => fs.readFileSync(file, "utf8").trimEnd().split("\n");70const tree = (dir, base = dir) => fs.readdirSync(dir, { withFileTypes: true }).flatMap((e) => (e.isDirectory() ? tree(join(dir, e.name), base) : [join(dir, e.name).slice(base.length + 1)])).sort();7172/* ───────── basics ───────── */7374test("canonical JSON is independent of key order", () => {75  assert.equal(canonical({ b: 1, a: [2, { d: 1, c: 2 }] }), canonical({ a: [2, { c: 2, d: 1 }], b: 1 }));76  assert.equal(canonical({ a: undefined, b: null }), '{"b":null}');77});7879test("init creates the ledger once and refuses a second time", () => {80  const { root, l, file } = project({ init: false });81  assert.equal(code(() => l.status()), "not_initialised");82  assert.equal(code(() => D(l)), "not_initialised");83  const r = l.init({ title: "Study", budget_usd: 50 });84  assert.equal(r.head.seq, 1);85  assert.equal(r.project.budget_usd, 50);86  assert.equal(code(() => l.init({ title: "Again" })), "already_initialised");87  assert.equal(lines(file).length, 1);88  assert.deepEqual(tree(join(root, ".research")), ["HEAD", "ledger.jsonl"]);89});9091test("a missing root is reported", () => {92  assert.equal(code(() => new Ledger({ root: join(tmpdir(), "does-not-exist-" + Date.now()) })), "root_missing");93});9495test("hypotheses are numbered, validated and never overwritten", () => {96  const { l } = project();97  assert.equal(H(l).registered, "H1");98  assert.equal(H(l, { role: "secondary", direction: "positive" }).registered, "H2");99  assert.equal(H(l, { id: "H3a" }).registered, "H3a");100  assert.equal(code(() => H(l, { id: "H1" })), "duplicate_id");101  assert.equal(code(() => H(l, { id: "first" })), "invalid_input");102  assert.equal(code(() => H(l, { statement: "short" })), "invalid_input");103  assert.equal(code(() => H(l, { direction: "up" })), "invalid_input");104});105106/* ───────── freeze, verify, amend ───────── */107108test("freeze hashes the files; a one-byte change is pinpointed", () => {109  const { root, l } = project();110  assert.equal(code(() => l.freeze({ files: ["protocol.md"] })), "no_hypotheses");111  H(l);112  assert.equal(code(() => l.verifyProtocol()), "not_frozen");113  const f = l.freeze({ files: ["protocol.md", "analysis.py"] });114  assert.equal(f.version, 1);115  assert.match(f.root_hash, /^[0-9a-f]{64}$/);116  assert.deepEqual(f.files.map((x) => x.path), ["analysis.py", "protocol.md"]);117  assert.equal(f.files[1].sha256, sha256(fs.readFileSync(join(root, "protocol.md"))));118  assert.equal(l.verifyProtocol().intact, true);119120  const p = join(root, "analysis.py"), original = fs.readFileSync(p);121  const flipped = Buffer.from(original); flipped[0] ^= 1; // same length, one bit122  fs.writeFileSync(p, flipped);123  const v = l.verifyProtocol();124  assert.equal(v.intact, false);125  assert.deepEqual(v.changed, ["analysis.py"]);126  const entry = v.files.find((x) => x.path === "analysis.py");127  assert.equal(entry.status, "changed");128  assert.equal(entry.frozen_bytes, entry.current_bytes);129  assert.notEqual(entry.frozen_sha256, entry.current_sha256);130  assert.equal(v.files.find((x) => x.path === "protocol.md").status, "unchanged");131  fs.writeFileSync(p, original);132  assert.equal(l.verifyProtocol().intact, true);133});134135test("a renamed protocol file is reported as missing, with where its content went", () => {136  const { root, l } = project();137  H(l); l.freeze({ files: ["protocol.md", "analysis.py"] });138  fs.renameSync(join(root, "protocol.md"), join(root, "protocol-v2.md"));139  const v = l.verifyProtocol();140  assert.deepEqual(v.changed, ["protocol.md"]);141  const e = v.files.find((x) => x.path === "protocol.md");142  assert.equal(e.status, "missing");143  assert.equal(e.same_content_found_at, "protocol-v2.md");144  fs.rmSync(join(root, "analysis.py"));145  const gone = l.verifyProtocol().files.find((x) => x.path === "analysis.py");146  assert.equal(gone.status, "missing");147  assert.equal(gone.same_content_found_at, undefined);148});149150test("the root hash depends on file names, file contents and hypotheses", () => {151  const roots = [];152  for (const variant of ["base", "content", "hypothesis", "name"]) {153    const { root, l } = project();154    if (variant === "content") fs.writeFileSync(join(root, "protocol.md"), "# Protocol\nH1: x lowers y.\n");155    if (variant === "name") fs.renameSync(join(root, "protocol.md"), join(root, "plan.md"));156    H(l, variant === "hypothesis" ? { direction: "negative" } : {});157    roots.push(l.freeze({ files: [variant === "name" ? "plan.md" : "protocol.md"] }).root_hash);158  }159  assert.equal(new Set(roots).size, 4);160  const again = project(); H(again.l);161  assert.equal(again.l.freeze({ files: ["protocol.md"] }).root_hash, roots[0], "same inputs give the same root hash");162});163164test("refreezing is refused; an amendment is a linked new version with a mandatory reason", () => {165  const { root, l } = project();166  H(l);167  const v1 = l.freeze({ files: ["protocol.md"] });168  assert.equal(code(() => l.freeze({ files: ["protocol.md"] })), "already_frozen");169  assert.equal(code(() => l.freeze({ files: ["protocol.md", "analysis.py"] })), "already_frozen");170  assert.equal(code(() => l.amend({ files: ["protocol.md"], data_seen: false })), "invalid_input", "reason is mandatory");171  assert.equal(code(() => l.amend({ files: ["protocol.md"], reason: "Clarify the sample definition." })), "invalid_input", "data_seen is mandatory");172  assert.equal(code(() => l.amend({ files: ["protocol.md"], reason: "Nothing really changed here.", data_seen: false })), "nothing_changed");173  assert.equal(code(() => l.amend({ files: ["protocol.md"], reason: "Clarify the sample definition.", data_seen: false, affects: ["H9"] })), "unknown_hypothesis");174  fs.appendFileSync(join(root, "protocol.md"), "Sample: firms with at least 12 months of data.\n");175  const v2 = l.amend({ files: ["protocol.md", "analysis.py"], reason: "Clarify the sample definition.", data_seen: false });176  assert.equal(v2.version, 2);177  assert.equal(v2.kind, "amendment");178  assert.notEqual(v2.root_hash, v1.root_hash);179  assert.deepEqual(v2.changed_files, { added: ["analysis.py"], removed: [], modified: ["protocol.md"] });180  const st = l.state();181  assert.equal(st.freezes[1].previous_root_hash, v1.root_hash);182  assert.equal(l.verifyProtocol().version, 2);183  assert.equal(l.verifyProtocol().intact, true);184});185186test("amend before any freeze is refused", () => {187  const { l } = project(); H(l);188  assert.equal(code(() => l.amend({ files: ["protocol.md"], reason: "There is nothing to amend yet.", data_seen: false })), "not_frozen");189});190191/* ───────── confirmatory vs exploratory ───────── */192193test("a result under a protocol frozen before data is confirmatory", () => {194  const { l } = project();195  H(l); l.freeze({ files: ["protocol.md", "analysis.py"] });196  l.markDataAccess({ description: "Opened the outcome file and ran the main regression." });197  const r = l.recordResult({ hypothesis: "H1", estimate: 0.11, uncertainty: "95% CI [0.02, 0.19]", verdict: "confirmed" });198  assert.equal(r.id, "X-01");199  assert.equal(r.status, "confirmatory");200});201202test("no freeze, or a freeze after data, gives exploratory results", () => {203  const a = project(); H(a.l);204  assert.equal(a.l.recordResult({ hypothesis: "H1", verdict: "confirmed" }).status, "exploratory");205  const b = project(); H(b.l);206  assert.match(b.l.markDataAccess({ description: "Looked at the outcome before freezing." }).warning, /before any protocol was frozen/);207  assert.match(b.l.freeze({ files: ["protocol.md"] }).warning, /after outcome data/);208  const r = b.l.recordResult({ hypothesis: "H1", verdict: "confirmed" });209  assert.equal(r.status, "exploratory");210  assert.match(r.status_reasons[0], /after outcome data/);211});212213test("data marked as not outcome data do not end the confirmatory window", () => {214  const { l } = project(); H(l);215  l.markDataAccess({ description: "Loaded covariates only, to check coverage.", outcome_data: false });216  assert.equal(l.freeze({ files: ["protocol.md"] }).warning, undefined);217  assert.equal(l.recordResult({ hypothesis: "H1", verdict: "not_confirmed" }).status, "confirmatory");218});219220test("an amendment after data makes only the affected hypotheses exploratory, from then on", () => {221  const { root, l } = project();222  H(l); H(l); H(l);223  l.freeze({ files: ["protocol.md", "analysis.py"] });224  const first = l.recordResult({ hypothesis: "H1", estimate: 0.11, verdict: "confirmed" });225  assert.equal(first.status, "confirmatory");226  fs.appendFileSync(join(root, "analysis.py"), "# H2 now uses a log transform\n");227  // Declared as "no data seen", but a result is already on record: the ledger does not accept that.228  const am = l.amend({ files: ["protocol.md", "analysis.py"], reason: "H2 is now tested on log outcomes.", data_seen: false, affects: ["H2"] });229  assert.match(am.warning, /already records access to outcome data/);230  assert.match(am.consequence, /H2/);231  const h2 = l.recordResult({ hypothesis: "H2", verdict: "confirmed" });232  assert.equal(h2.status, "exploratory");233  assert.match(h2.status_reasons.join(" "), /amended after outcome data were seen \(version 2\)/);234  assert.equal(l.recordResult({ hypothesis: "H3", verdict: "inconclusive" }).status, "confirmatory", "H3 is not affected");235  // The report keeps H1's earlier result confirmatory: it was recorded before the amendment.236  const rep = l.exportReport().json;237  assert.deepEqual(rep.results.map((x) => [x.hypothesis, x.status]), [["H1", "confirmatory"], ["H2", "exploratory"], ["H3", "confirmatory"]]);238  assert.equal(rep.protocol.versions[1].data_seen, true);239});240241test("an amendment after data with no hypothesis list affects all of them", () => {242  const { root, l } = project(); H(l); H(l);243  l.freeze({ files: ["protocol.md"] });244  l.markDataAccess({ description: "Ran the main specification." });245  fs.appendFileSync(join(root, "protocol.md"), "Changed estimator.\n");246  l.amend({ files: ["protocol.md"], reason: "Switched estimator after seeing the fit.", data_seen: true });247  assert.equal(l.recordResult({ hypothesis: "H1", verdict: "confirmed" }).status, "exploratory");248  assert.equal(l.recordResult({ hypothesis: "H2", verdict: "confirmed" }).status, "exploratory");249});250251test("a hypothesis added after the freeze is exploratory unless amended in before data", () => {252  const a = project(); H(a.l); a.l.freeze({ files: ["protocol.md"] });253  assert.match(H(a.l).warning, /after the protocol was frozen/);254  assert.deepEqual(a.l.verifyProtocol().hypotheses_registered_since, ["H2"]);255  const r = a.l.recordResult({ hypothesis: "H2", verdict: "confirmed" });256  assert.equal(r.status, "exploratory");257  assert.match(r.status_reasons[0], /registered after/);258259  const b = project(); H(b.l); b.l.freeze({ files: ["protocol.md"] }); H(b.l);260  b.l.amend({ files: ["protocol.md"], reason: "Add H2 before any data are opened.", data_seen: false });261  assert.equal(b.l.recordResult({ hypothesis: "H2", verdict: "confirmed" }).status, "confirmatory");262});263264test("deviations: minor keeps confirmatory with a flag, exploratory removes it, scoped by hypothesis", () => {265  const { l } = project(); H(l); H(l); H(l);266  l.freeze({ files: ["protocol.md"] });267  assert.equal(code(() => l.logDeviation({ protocol_said: "Use all firms.", done: "Dropped two firms.", reason: "Their identifiers were duplicated.", impact: "large" })), "invalid_input");268  assert.equal(code(() => l.logDeviation({ protocol_said: "Use all firms.", done: "Dropped two firms.", reason: "Their identifiers were duplicated.", impact: "minor", hypotheses: ["H7"] })), "unknown_hypothesis");269  assert.equal(l.logDeviation({ protocol_said: "Use all firms.", done: "Dropped two firms.", reason: "Their identifiers were duplicated.", impact: "minor", hypotheses: ["H1"] }).id, "V-01");270  assert.equal(l.logDeviation({ protocol_said: "Cluster by firm.", done: "Clustered by industry.", reason: "Too few firms per cluster to estimate.", impact: "exploratory", hypotheses: ["H2"] }).id, "V-02");271  assert.equal(l.recordResult({ hypothesis: "H1", verdict: "confirmed" }).status, "confirmatory_with_deviations");272  assert.equal(l.recordResult({ hypothesis: "H2", verdict: "confirmed" }).status, "exploratory");273  assert.equal(l.recordResult({ hypothesis: "H3", verdict: "not_confirmed" }).status, "confirmatory");274  // A deviation written down late still applies when the report is built.275  l.logDeviation({ protocol_said: "Two-sided test.", done: "One-sided test.", reason: "Noticed after the fact that the code used one side.", impact: "exploratory", hypotheses: ["H3"] });276  assert.equal(l.exportReport().json.results.find((x) => x.hypothesis === "H3").status, "exploratory");277});278279test("a silent change to a frozen file makes the result exploratory", () => {280  const { root, l } = project(); H(l);281  l.freeze({ files: ["protocol.md", "analysis.py"] });282  fs.appendFileSync(join(root, "analysis.py"), "# tweak\n");283  const r = l.recordResult({ hypothesis: "H1", verdict: "confirmed" });284  assert.equal(r.status, "exploratory");285  assert.match(r.status_reasons.join(" "), /no longer matched the frozen manifest/);286});287288test("results need a known hypothesis, a valid verdict and a known run", () => {289  const { l } = project(); H(l);290  assert.equal(code(() => l.recordResult({ hypothesis: "H4", verdict: "confirmed" })), "unknown_hypothesis");291  assert.equal(code(() => l.recordResult({ hypothesis: "H1", verdict: "significant" })), "invalid_input");292  assert.equal(code(() => l.recordResult({ hypothesis: "H1", verdict: "confirmed", run_id: "R-09" })), "unknown_run");293});294295test("classify is a pure function of the state", () => {296  const { l } = project(); H(l); l.freeze({ files: ["protocol.md"] });297  const st = l.state();298  assert.equal(classify(st, "H1").status, "confirmatory");299  assert.equal(classify(st, "H1", Infinity, false).status, "exploratory");300  assert.equal(classify(st, "H1", 2).status, "exploratory", "before the freeze record there is no protocol");301});302303/* ───────── decisions, gates, questions ───────── */304305test("decisions are numbered in order and a false 'before data' is flagged", () => {306  const { l } = project(); H(l); l.freeze({ files: ["protocol.md"] });307  assert.equal(D(l).id, "D-01");308  assert.equal(D(l).id, "D-02");309  assert.equal(code(() => D(l, { options: ["only one"] })), "invalid_input");310  assert.equal(code(() => D(l, { timing: "sometime" })), "invalid_input");311  assert.equal(code(() => D(l, { rationale: "" })), "invalid_input");312  l.markDataAccess({ description: "Opened the outcome file." });313  const late = D(l);314  assert.equal(late.id, "D-03");315  assert.match(late.warning, /already records access to outcome data/);316  assert.equal(D(l, { timing: "after_outcome_data" }).warning, undefined);317  assert.equal(l.state().decisions[2].timing_conflict, true);318});319320test("gates: criteria first, then evidence; no-go and conditional are recorded as such", () => {321  const { l } = project();322  assert.equal(code(() => l.closeGate({ name: "A", outcome: "go", evidence: ["n = 5 000"] })), "gate_not_open");323  assert.equal(code(() => l.openGate({ name: "A", criteria: [] })), "invalid_input");324  assert.equal(l.openGate({ name: "A", criteria: ["At least 2 000 usable pairs", "Pilot cost under 40 USD"] }).round, 1);325  assert.equal(code(() => l.openGate({ name: "A", criteria: ["again"] })), "gate_open");326  assert.equal(code(() => l.closeGate({ name: "A", outcome: "conditional", evidence: ["1 800 pairs"] })), "invalid_input");327  assert.equal(code(() => l.closeGate({ name: "A", outcome: "maybe", evidence: ["1 800 pairs"] })), "invalid_input");328  l.closeGate({ name: "A", outcome: "conditional", evidence: ["1 800 pairs", "Cost 22 USD"], conditions: ["Collect 200 more pairs"] });329  assert.match(l.status().next_steps.join(" "), /conditional: Collect 200 more pairs/);330  assert.equal(l.openGate({ name: "A", criteria: ["At least 2 000 usable pairs"] }).round, 2);331  assert.equal(l.closeGate({ name: "A", outcome: "go", evidence: ["2 050 pairs"] }).round, 2);332  l.openGate({ name: "B", criteria: ["Effect detectable in the pilot"] });333  assert.match(l.closeGate({ name: "B", outcome: "no_go", evidence: ["Pilot effect indistinguishable from zero"] }).consequence, /No-go is a result/);334  const s = l.status();335  assert.deepEqual(s.gates.map((g) => [g.name, g.round, g.state]), [["A", 2, "go"], ["B", 1, "no_go"]]);336  assert.match(s.next_steps.join(" "), /no-go: do not proceed/);337  assert.equal(l.exportReport().json.gates.length, 3);338});339340test("open questions appear in status until resolved", () => {341  const { l } = project();342  assert.equal(l.logQuestion({ question: "Is the 2019 file complete?" }).id, "Q-01");343  assert.equal(l.status().open_questions.length, 1);344  assert.equal(code(() => l.resolveQuestion({ id: "Q-07", answer: "Yes." })), "unknown_question");345  l.resolveQuestion({ id: "Q-01", answer: "Yes, 12 months present." });346  assert.equal(code(() => l.resolveQuestion({ id: "Q-01", answer: "Again." })), "already_resolved");347  assert.equal(l.status().open_questions.length, 0);348});349350/* ───────── runs ───────── */351352test("runs are fingerprinted; an identical expensive run is refused without a reason", () => {353  const { root, l } = project();354  fs.writeFileSync(join(root, "out.json"), '{"beta":0.11}\n');355  const spec = { command: "python analysis.py --seed 7", code_files: ["analysis.py"], input_files: ["data.csv"], seed: 7 };356  const pre = l.checkRerun(spec);357  assert.equal(pre.rerun_advised, true);358  assert.deepEqual(pre.identical_runs, []);359  const r1 = l.registerRun({ name: "Main estimation", ...spec, output_files: ["out.json"], expensive: true, deterministic: false, started_at: "2026-01-05T10:00:00Z", ended_at: "2026-01-05T10:20:00Z", environment: "python 3.12, numpy 2.1" });360  assert.equal(r1.id, "R-01");361  assert.equal(r1.fingerprint, pre.fingerprint);362  assert.equal(r1.outputs[0].sha256, sha256(fs.readFileSync(join(root, "out.json"))));363  const chk = l.checkRerun(spec);364  assert.equal(chk.rerun_advised, false);365  assert.match(chk.warning, /R-01/);366  assert.equal(chk.identical_runs[0].outputs[0].path, "out.json");367368  let err; try { l.registerRun({ name: "Again", ...spec, expensive: true }); } catch (e) { err = e; }369  assert.equal(err.code, "expensive_rerun");370  assert.deepEqual(err.identical_runs, ["R-01"]);371  const r2 = l.registerRun({ name: "Again", ...spec, rerun_reason: "The first output file was lost with the disk." });372  assert.equal(r2.id, "R-02");373  assert.deepEqual(r2.repeats, ["R-01"]);374375  // Any change to command, code, inputs or seed is a different run.376  assert.equal(l.checkRerun({ ...spec, seed: 8 }).rerun_advised, true);377  assert.equal(l.checkRerun({ ...spec, command: "python analysis.py --seed 7 --fast" }).rerun_advised, true);378  fs.appendFileSync(join(root, "data.csv"), "2,3\n");379  assert.equal(l.checkRerun(spec).rerun_advised, true);380  assert.notEqual(l.checkRerun(spec).fingerprint, r1.fingerprint);381});382383test("cheap or failed runs can be repeated freely; inputs are validated", () => {384  const { l } = project();385  const spec = { command: "python analysis.py", code_files: ["analysis.py"] };386  l.registerRun({ name: "Cheap", ...spec });387  assert.equal(l.registerRun({ name: "Cheap again", ...spec }).id, "R-02");388  l.registerRun({ name: "Costly, crashed", command: "python fit.py", expensive: true, status: "failed" });389  assert.equal(l.checkRerun({ command: "python fit.py" }).rerun_advised, true, "a failed expensive run does not block a retry");390  assert.equal(l.registerRun({ name: "Costly, retried", command: "python fit.py", expensive: true }).id, "R-04");391  assert.equal(code(() => l.registerRun({ name: "Bad", command: "x", code_files: ["missing.py"] })), "file_missing");392  assert.equal(code(() => l.registerRun({ name: "Bad", command: "x", started_at: "yesterday" })), "invalid_input");393  assert.equal(code(() => l.registerRun({ name: "Bad", command: "x", code_files: ["analysis.py", "./analysis.py"] })), "invalid_input");394  assert.equal(code(() => l.registerRun({ name: "Bad", command: "x", status: "running" })), "invalid_input");395});396397/* ───────── spending ───────── */398399test("spend arithmetic is exact and the budget is tracked", () => {400  const { l } = project({ budget: 1 });401  l.registerRun({ name: "Extraction", command: "node extract.mjs", expensive: true });402  // 0.1 + 0.2 is 0.30000000000000004 in floating point: totals must not show that.403  assert.equal(l.recordSpend({ provider: "api-a", model: "model-x", purpose: "extraction", cost_usd: 0.1, input_tokens: 1000, output_tokens: 200, run_id: "R-01" }).total_usd, 0.1);404  const two = l.recordSpend({ provider: "api-a", model: "model-x", purpose: "extraction", cost_usd: 0.2, input_tokens: 2000, output_tokens: 300 });405  assert.equal(two.total_usd, 0.3);406  assert.equal(two.remaining_usd, 0.7);407  for (let i = 0; i < 10; i++) l.recordSpend({ provider: "api-b", model: "model-y", purpose: "judging", cost_usd: 0.01 });408  let rep = l.spendReport();409  assert.equal(rep.entries, 12);410  assert.equal(rep.total_usd, 0.4);411  assert.equal(rep.remaining_usd, 0.6);412  assert.equal(rep.budget_used_pct, 40);413  assert.equal(rep.over_budget, false);414  assert.deepEqual(rep.by_provider, [{ provider: "api-a", cost_usd: 0.3, entries: 2, input_tokens: 3000, output_tokens: 500 }, { provider: "api-b", cost_usd: 0.1, entries: 10, input_tokens: 0, output_tokens: 0 }]);415  assert.deepEqual(rep.by_purpose.map((x) => [x.purpose, x.cost_usd]), [["extraction", 0.3], ["judging", 0.1]]);416  assert.equal(rep.by_provider.reduce((s, x) => s + Math.round(x.cost_usd * 1e6), 0), 400000);417418  const over = l.recordSpend({ provider: "api-b", purpose: "judging", cost_usd: 0.600001 });419  assert.equal(over.total_usd, 1.000001);420  assert.equal(over.remaining_usd, -0.000001);421  assert.match(over.warning, /exceeds the declared budget/);422  assert.match(l.status().next_steps.join(" "), /over budget/);423424  l.setBudget({ budget_usd: 5, reason: "Second wave approved after gate B." });425  rep = l.spendReport();426  assert.equal(rep.budget_usd, 5);427  assert.equal(rep.remaining_usd, 3.999999);428  assert.equal(rep.over_budget, false);429430  assert.equal(code(() => l.recordSpend({ provider: "api-a", purpose: "x", cost_usd: -1 })), "invalid_input");431  assert.equal(code(() => l.recordSpend({ provider: "api-a", purpose: "x", cost_usd: Number.NaN })), "invalid_input");432  assert.equal(code(() => l.recordSpend({ provider: "api-a", purpose: "x", cost_usd: 1, input_tokens: 1.5 })), "invalid_input");433  assert.equal(code(() => l.recordSpend({ provider: "api-a", purpose: "x", cost_usd: 1, run_id: "R-99" })), "unknown_run");434  assert.equal(code(() => l.setBudget({ budget_usd: 9 })), "invalid_input", "a budget change needs a reason");435});436437test("without a budget the report says so instead of inventing one", () => {438  const { l } = project();439  l.recordSpend({ provider: "api-a", purpose: "pilot", cost_usd: 2.5 });440  const rep = l.spendReport();441  assert.equal(rep.budget_usd, null);442  assert.equal(rep.remaining_usd, null);443  assert.equal(rep.over_budget, false);444  assert.match(l.exportReport().markdown, /No budget was declared/);445});446447/* ───────── tamper evidence ───────── */448449function filled() {450  const p = project(); H(p.l); p.l.freeze({ files: ["protocol.md"] });451  for (let i = 0; i < 5; i++) D(p.l);452  return p; // 8 records: init, hypothesis, freeze, D-01…D-05453}454455test("an intact ledger verifies", () => {456  const { l } = filled();457  const v = l.verifyLedger();458  assert.equal(v.ok, true);459  assert.equal(v.records_verified, 8);460  assert.equal(v.first_problem, null);461  assert.deepEqual(v.warnings, []);462  assert.match(v.guarantee, /rewrites the whole chain/);463});464465test("an edited record is pinpointed", () => {466  const { l, file } = filled();467  const ls = lines(file);468  ls[4] = ls[4].replace('"choice":"1%"', '"choice":"5%"');469  fs.writeFileSync(file, ls.join("\n") + "\n");470  const v = l.verifyLedger();471  assert.equal(v.ok, false);472  assert.equal(v.records_verified, 4);473  assert.deepEqual([v.first_problem.seq, v.first_problem.line, v.first_problem.kind], [5, 5, "content_changed"]);474});475476test("an edit that also recomputes the record's own hash breaks the next link", () => {477  const { l, file } = filled();478  const ls = lines(file);479  const r = JSON.parse(ls[4]); r.data.choice = "5%";480  r.hash = sha256(canonical({ seq: r.seq, ts: r.ts, type: r.type, data: r.data, prev: r.prev }));481  ls[4] = JSON.stringify(r);482  fs.writeFileSync(file, ls.join("\n") + "\n");483  const v = l.verifyLedger();484  assert.deepEqual([v.first_problem.seq, v.first_problem.kind], [6, "chain_break"]);485});486487test("a deleted record is pinpointed", () => {488  const { l, file } = filled();489  const ls = lines(file); ls.splice(3, 1);490  fs.writeFileSync(file, ls.join("\n") + "\n");491  const v = l.verifyLedger();492  assert.equal(v.records_verified, 3);493  assert.deepEqual([v.first_problem.line, v.first_problem.kind], [4, "sequence_break"]);494});495496test("two swapped records are pinpointed at the first of them", () => {497  const { l, file } = filled();498  const ls = lines(file); [ls[4], ls[5]] = [ls[5], ls[4]];499  fs.writeFileSync(file, ls.join("\n") + "\n");500  const v = l.verifyLedger();501  assert.equal(v.records_verified, 4);502  assert.deepEqual([v.first_problem.line, v.first_problem.kind], [5, "sequence_break"]);503});504505test("truncation is detected: whole records removed, and a record cut in the middle", () => {506  const a = filled();507  fs.writeFileSync(a.file, lines(a.file).slice(0, 6).join("\n") + "\n");508  const v = a.l.verifyLedger();509  assert.equal(v.ok, false);510  assert.deepEqual([v.first_problem.seq, v.first_problem.kind], [7, "tail_missing"]);511  assert.match(v.first_problem.detail, /had 8 records but only 6 remain/);512513  const b = filled();514  const raw = fs.readFileSync(b.file, "utf8");515  fs.writeFileSync(b.file, raw.slice(0, raw.length - 40));516  const w = b.l.verifyLedger();517  assert.deepEqual([w.first_problem.seq, w.first_problem.kind], [8, "invalid_json"]);518  assert.match(w.first_problem.detail, /truncated/);519  assert.equal(w.records_verified, 7);520});521522test("a rewritten last record and an emptied ledger are detected through HEAD", () => {523  const a = filled();524  const ls = lines(a.file);525  const r = JSON.parse(ls[7]); r.data.choice = "5%";526  r.hash = sha256(canonical({ seq: r.seq, ts: r.ts, type: r.type, data: r.data, prev: r.prev }));527  ls[7] = JSON.stringify(r);528  fs.writeFileSync(a.file, ls.join("\n") + "\n");529  assert.deepEqual([a.l.verifyLedger().first_problem.seq, a.l.verifyLedger().first_problem.kind], [8, "head_mismatch"]);530531  const b = filled();532  fs.writeFileSync(b.file, "");533  assert.deepEqual([b.l.verifyLedger().first_problem.seq, b.l.verifyLedger().first_problem.kind], [1, "tail_missing"]);534});535536test("an inserted record is detected", () => {537  const { l, file } = filled();538  const ls = lines(file); ls.splice(2, 0, ls[1]);539  fs.writeFileSync(file, ls.join("\n") + "\n");540  assert.deepEqual([l.verifyLedger().first_problem.line, l.verifyLedger().first_problem.kind], [3, "sequence_break"]);541});542543test("a corrupt ledger refuses appends and says where it broke", () => {544  const { l, file } = filled();545  const before = lines(file); before.splice(3, 1);546  fs.writeFileSync(file, before.join("\n") + "\n");547  const frozen = fs.readFileSync(file, "utf8");548  let err; try { D(l); } catch (e) { err = e; }549  assert.equal(err.code, "ledger_corrupt");550  assert.equal(err.problem.line, 4);551  assert.equal(code(() => l.spendReport()), "ledger_corrupt");552  assert.equal(code(() => l.exportReport()), "ledger_corrupt");553  assert.equal(fs.readFileSync(file, "utf8"), frozen, "nothing was appended");554  const s = l.status();555  assert.equal(s.ledger_ok, false);556  assert.equal(s.problem.line, 4);557});558559test("a missing HEAD is a warning, repaired by the next append", () => {560  const { l, head } = filled();561  fs.rmSync(head);562  const v = l.verifyLedger();563  assert.equal(v.ok, true);564  assert.equal(v.warnings[0].kind, "head_missing");565  D(l);566  assert.deepEqual(l.verifyLedger().warnings, []);567  assert.equal(JSON.parse(fs.readFileSync(head, "utf8")).seq, 9);568});569570/* ───────── atomic writes, crashes, locking ───────── */571572test("a writer that dies before the rename leaves the ledger untouched", () => {573  const { root, l, file } = filled();574  const before = fs.readFileSync(file);575  const real = fs.renameSync;576  fs.renameSync = () => { throw new Error("simulated crash before rename"); };577  try { assert.throws(() => D(l), /simulated crash/); } finally { fs.renameSync = real; }578  assert.deepEqual(fs.readFileSync(file), before);579  assert.equal(fs.existsSync(join(root, ".research", "lock")), false, "the lock is released");580  assert.equal(fs.readdirSync(join(root, ".research")).filter((f) => f.includes(".tmp-")).length, 1, "the half-written temp file is all that is left");581  assert.equal(l.verifyLedger().ok, true);582  assert.equal(D(l).id, "D-06", "the lost append did not consume a number");583  assert.equal(fs.readdirSync(join(root, ".research")).filter((f) => f.includes(".tmp-")).length, 0, "the next writer removes the leftover");584  assert.equal(l.verifyLedger().records_verified, 9);585});586587test("a crash between the ledger write and the HEAD write is recovered", () => {588  const { l, head } = filled();589  const real = fs.renameSync; let n = 0;590  fs.renameSync = (a, b) => { if (++n === 2) throw new Error("simulated crash before HEAD"); return real(a, b); };591  try { assert.throws(() => D(l), /simulated crash/); } finally { fs.renameSync = real; }592  const v = l.verifyLedger();593  assert.equal(v.ok, true);594  assert.equal(v.records_verified, 9, "the record itself was committed");595  assert.equal(v.warnings[0].kind, "head_stale");596  assert.equal(D(l).id, "D-07");597  assert.deepEqual(l.verifyLedger().warnings, []);598  assert.equal(JSON.parse(fs.readFileSync(head, "utf8")).seq, 10);599});600601test("a stray partial temp file never counts as ledger content", () => {602  const { root, l } = filled();603  fs.writeFileSync(join(root, ".research", "ledger.jsonl.tmp-999-deadbeef"), '{"seq":9,"ts":"2026-01-01T00:00:00.000Z","type":"decision","da');604  assert.equal(l.verifyLedger().ok, true);605  assert.equal(l.status().records, 8);606});607608test("a stale lock from a dead writer is taken over", () => {609  const { root, l } = filled();610  const lock = join(root, ".research", "lock");611  fs.writeFileSync(lock, "99999\n");612  const old = new Date(Date.now() - 120000);613  fs.utimesSync(lock, old, old);614  assert.equal(D(l).id, "D-06");615  assert.equal(fs.existsSync(lock), false);616});617618test("decision numbers stay unique and ordered under concurrent writers", async () => {619  const { root, l, file } = project();620  const worker = fileURLToPath(new URL("./fixtures/append-worker.mjs", import.meta.url));621  const WORKERS = 6, EACH = 8;622  const runs = await Promise.all(Array.from({ length: WORKERS }, (_, w) => new Promise((res, rej) => execFile(process.execPath, [worker, root, String(w), String(EACH)], { encoding: "utf8" }, (err, stdout, stderr) => (err ? rej(new Error(stderr || err.message)) : res(JSON.parse(stdout)))))));623  const returned = runs.flat();624  assert.equal(returned.length, WORKERS * EACH);625  assert.equal(new Set(returned).size, WORKERS * EACH, "no two writers received the same number");626  for (const ids of runs) assert.deepEqual(ids, [...ids].sort(), "each writer saw its own numbers increase");627  const v = l.verifyLedger();628  assert.equal(v.ok, true);629  assert.equal(v.records_verified, 1 + WORKERS * EACH);630  assert.deepEqual(v.warnings, []);631  const inFile = lines(file).map((x) => JSON.parse(x)).filter((r) => r.type === "decision").map((r) => r.data.id);632  assert.deepEqual(inFile, Array.from({ length: WORKERS * EACH }, (_, i) => `D-${String(i + 1).padStart(2, "0")}`));633  assert.deepEqual(fs.readdirSync(join(root, ".research")).sort(), ["HEAD", "ledger.jsonl"]);634});635636/* ───────── path confinement ───────── */637638test("files outside the project root are refused, including through links", () => {639  const { root, l } = project(); H(l);640  const outside = tmp("outside-");641  fs.writeFileSync(join(outside, "secret.txt"), "not part of the project\n");642  fs.symlinkSync(join(outside, "secret.txt"), join(root, "link.txt"));643  fs.symlinkSync(outside, join(root, "linkdir"));644  fs.symlinkSync(join(root, "protocol.md"), join(root, "alias.md"));645  fs.mkdirSync(join(root, "sub"));646  for (const p of ["../" + outside.split("/").pop() + "/secret.txt", join(outside, "secret.txt"), "link.txt", "linkdir/secret.txt", "sub/../../" + outside.split("/").pop() + "/secret.txt"]) {647    assert.equal(code(() => l.freeze({ files: [p] })), "path_forbidden", p);648    assert.equal(code(() => l.registerRun({ name: "x", command: "x", input_files: [p] })), "path_forbidden", p);649    assert.equal(code(() => l.checkRerun({ command: "x", code_files: [p] })), "path_forbidden", p);650  }651  assert.equal(code(() => l.freeze({ files: [".research/ledger.jsonl"] })), "path_forbidden");652  assert.equal(code(() => l.freeze({ files: ["sub"] })), "not_a_file");653  assert.equal(code(() => l.freeze({ files: ["nope.md"] })), "file_missing");654  assert.equal(code(() => l.freeze({ files: ["a\0b"] })), "invalid_input");655  // A link that stays inside the root is fine and is recorded under its real path.656  assert.deepEqual(l.freeze({ files: ["alias.md"] }).files.map((f) => f.path), ["protocol.md"]);657});658659test("a .research that is a link to somewhere else is refused", () => {660  const root = tmp(), elsewhere = tmp("elsewhere-");661  fs.symlinkSync(elsewhere, join(root, ".research"));662  const l = new Ledger({ root });663  assert.equal(code(() => l.init({ title: "Study" })), "path_forbidden");664  assert.deepEqual(fs.readdirSync(elsewhere), []);665});666667test("nothing outside .research is created or modified", () => {668  const { root, l } = project({ budget: 10 });669  const snapshot = () => Object.fromEntries(tree(root).filter((f) => !f.startsWith(".research/")).map((f) => [f, sha256(fs.readFileSync(join(root, f)))]));670  const before = snapshot();671  H(l); l.freeze({ files: ["protocol.md", "analysis.py"] }); l.verifyProtocol();672  l.markDataAccess({ description: "Opened the outcome file." }); D(l, { timing: "after_outcome_data" });673  l.openGate({ name: "A", criteria: ["c"] }); l.closeGate({ name: "A", outcome: "go", evidence: ["e"] });674  l.registerRun({ name: "Run", command: "python analysis.py", code_files: ["analysis.py"], input_files: ["data.csv"] });675  l.recordSpend({ provider: "api-a", purpose: "pilot", cost_usd: 1 }); l.recordResult({ hypothesis: "H1", verdict: "inconclusive" });676  l.status(); l.verifyLedger(); l.exportReport({ write: true });677  assert.deepEqual(snapshot(), before);678  assert.deepEqual(tree(join(root, ".research")), ["HEAD", "ledger.jsonl", "reports/report-10.json", "reports/report-10.md"]);679});680681/* ───────── status and report ───────── */682683test("status carries what a new session needs, from the files alone", () => {684  const { root, l } = project({ budget: 20 });685  assert.match(l.status().next_steps[0], /Register the hypotheses/);686  H(l); H(l, { role: "secondary" });687  assert.match(l.status().next_steps[0], /Freeze the protocol/);688  l.freeze({ files: ["protocol.md", "analysis.py"] });689  l.openGate({ name: "A", criteria: ["Enough data"] });690  l.recordSpend({ provider: "api-a", purpose: "pilot", cost_usd: 3.25 });691  l.recordResult({ hypothesis: "H1", estimate: 0.11, verdict: "confirmed" });692  fs.appendFileSync(join(root, "analysis.py"), "# edit\n");693694  const s = new Ledger({ root }).status(); // a fresh object: nothing is kept in memory695  assert.equal(s.ledger_ok, true);696  assert.equal(s.project.title, "Test study");697  assert.equal(s.records, 7);698  assert.equal(s.protocol.frozen, true);699  assert.equal(s.protocol.intact, false);700  assert.deepEqual(s.protocol.changed_files, ["analysis.py"]);701  assert.equal(s.outcome_data_accessed, true);702  assert.deepEqual(s.hypotheses.map((h) => [h.id, h.role, h.result]), [["H1", "primary", "confirmed (confirmatory)"], ["H2", "secondary", null]]);703  assert.deepEqual(s.gates, [{ name: "A", round: 1, state: "open", criteria: ["Enough data"] }]);704  assert.deepEqual(s.spend, { total_usd: 3.25, budget_usd: 20, remaining_usd: 16.75 });705  assert.equal(s.recent.at(-1).what, "X-01: H1 confirmed (confirmatory)");706  const next = s.next_steps.join(" | ");707  assert.match(next, /Protocol files changed since the freeze \(analysis\.py\)/);708  assert.match(next, /Gate "A" is open/);709});710711test("the report shows every hypothesis, including null and unreported ones", () => {712  const { l } = project({ budget: 10 });713  H(l); H(l, { statement: "x raises z | in small firms" }); H(l);714  l.freeze({ files: ["protocol.md"] });715  D(l);716  l.logDeviation({ protocol_said: "Use all firms.", done: "Dropped two firms.", reason: "Their identifiers were duplicated.", impact: "minor", hypotheses: ["H1"] });717  l.openGate({ name: "A", criteria: ["Enough data"] }); l.closeGate({ name: "A", outcome: "go", evidence: ["n = 5 000"] });718  l.registerRun({ name: "Main", command: "python analysis.py", code_files: ["analysis.py"], expensive: true });719  l.recordSpend({ provider: "api-a", purpose: "pilot", cost_usd: 1.5 });720  l.recordResult({ hypothesis: "H1", estimate: 0.11, uncertainty: "SE 0.04", verdict: "confirmed", run_id: "R-01" });721  l.recordResult({ hypothesis: "H2", estimate: -0.01, uncertainty: "SE 0.05", verdict: "not_confirmed" });722  l.logQuestion({ question: "Is the 2019 file complete?" });723  const { markdown: md, json, written } = l.exportReport({ write: true });724  assert.deepEqual(written, [".research/reports/report-14.md", ".research/reports/report-14.json"]);725  for (const part of ["# Transparency report — Test study", "## Protocol", "## Hypotheses and results", "## Decisions", "## Deviations from the protocol", "## Gates", "## Runs", "## Spending", "## Open questions", "## How to check this report"]) assert.ok(md.includes(part), part);726  assert.match(md, /\| H1 \| primary \|.*\| confirmed \| confirmatory with deviations \| 0\.11 \| SE 0\.04 \|/);727  assert.match(md, /\| H2 \|.*x raises z \\\| in small firms.*\| not confirmed \| confirmatory \| -0\.01 \|/);728  assert.match(md, /\| H3 \|.*\| not reported \|/);729  assert.match(md, /Registered but not reported: H3/);730  assert.match(md, /\*\*D-01\*\*/);731  assert.match(md, /\*\*V-01\*\* \(impact: minor; H1\)/);732  assert.match(md, /Total: \*\*1\.5 USD\*\* over 1 entries\. Budget: 10 USD; remaining: 8\.5 USD\./);733  assert.ok(md.includes(json.ledger.head.hash));734  assert.equal(json.protocol.current_files_intact, true);735  assert.equal(json.hypotheses.length, 3);736  assert.equal(json.results.find((x) => x.hypothesis === "H1").status_now, undefined);737  const saved = JSON.parse(fs.readFileSync(join(l.dir, "reports", "report-14.json"), "utf8"));738  assert.deepEqual(saved.results, json.results);739  assert.equal(fs.readFileSync(join(l.dir, "reports", "report-14.md"), "utf8"), md);740  assert.equal(l.verifyLedger().records_verified, 14, "writing a report appends nothing");741});742743test("a report without a freeze says that every result is exploratory", () => {744  const { l } = project(); H(l);745  l.recordResult({ hypothesis: "H1", verdict: "confirmed" });746  assert.match(l.exportReport().markdown, /\*\*No protocol was frozen\.\*\* Every result below is exploratory\./);747});748749test("records from a newer engine are kept in the chain and ignored", () => {750  const st = buildState([{ seq: 1, ts: "t", type: "init", data: { title: "T", budget_micro_usd: null } }, { seq: 2, ts: "t", type: "something_new", data: {} }]);751  assert.equal(st.project.title, "T");752});753754test("timestamps come from the injected clock and are part of the hash", () => {755  const root = tmp(); let t = Date.parse("2026-02-01T00:00:00Z");756  const l = new Ledger({ root, now: () => new Date((t += 60000)) });757  l.init({ title: "Clock" });758  const recs = l.read().records;759  assert.equal(recs[0].ts, "2026-02-01T00:01:00.000Z");760  assert.equal(recs[0].prev, "0".repeat(64));761});762
Files main
Menu

Theme