/** * KHAELOR * File: tests/agent/verification.test.ts * Description: Verification gate tests — nudge on unverified code changes, 2-attempt budget, evidence from real events only. * * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai */ import { rmSync, writeFileSync } from "node:fs"; import * as path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { VerificationGate } from "../../src/agent/index.js"; import type { ChangeAttributor } from "../../src/agent/index.js"; import { LocalWorkspace } from "../../src/workspace/index.js"; import { assertPairingSafe, durableTypes, makeHarness, textTurn, toolTurn, } from "./fixtures.js"; import type { AgentHarness } from "./fixtures.js"; let harness: AgentHarness | null = null; afterEach(async () => { if (harness !== null) { await harness.log.close(); rmSync(harness.dir, { recursive: true, force: true }); harness = null; } }); const WRITE_TURN = (requestId: string): ReturnType => toolTurn(requestId, "Writing the module.", [ { toolUseId: `toolu_${requestId}`, toolName: "write", input: { file_path: "src/answer.ts", content: "export const answer = 42;\n" }, }, ]); describe("verification gate", () => { it("nudges when files changed without checks, and stops after 2 attempts", async () => { harness = await makeHarness({ turns: [ WRITE_TURN("req_1"), textTurn("req_2", "Done."), // stops without verifying → nudge 1 textTurn("req_3", "It looks correct to me."), // still no checks → nudge 2 textTurn("req_4", "Done (final)."), // budget exhausted → accepted, withheld honestly ], }); harness.user("Create the answer module"); const outcome = await harness.kernel.runTurn(); expect(outcome.kind).toBe("done"); if (outcome.kind !== "done") return; const events = harness.events(); const nudges = events.filter((event) => event.type === "task.verification-requested"); expect(nudges).toHaveLength(2); expect(nudges.map((n) => (n.type === "task.verification-requested" ? n.payload.attempt : 0))).toEqual([1, 2]); // 4 model calls: main, tool follow-up, nudge 1, nudge 2. expect(harness.client.requests).toHaveLength(4); // Nudge requests carry the synthetic verification message as volatile context. const third = harness.client.requests[2]; const serialized = JSON.stringify(third?.messages); expect(serialized).toContain("Verification required before completion"); // …and it was NOT in the pre-nudge requests. expect(JSON.stringify(harness.client.requests[1]?.messages)).not.toContain( "Verification required before completion", ); // Evidence is honest: changed files listed, zero checks, unresolved issue recorded. expect(outcome.evidence.changedFiles.some((p) => p.includes("answer.ts"))).toBe(true); expect(outcome.evidence.checks).toEqual([]); expect( outcome.evidence.unresolvedIssues.some((issue) => issue.includes("without fresh verification evidence"), ), ).toBe(true); assertPairingSafe(events); }); it("accepts completion without a nudge when a check ran after the last change", async () => { harness = await makeHarness({ turns: [ WRITE_TURN("req_1"), toolTurn("req_2", "Running the tests.", [ { toolUseId: "toolu_check", toolName: "bash", input: { command: "echo running tests" } }, ]), textTurn("req_3", "Done and verified."), ], }); harness.user("Create the answer module and verify it"); const outcome = await harness.kernel.runTurn(); expect(outcome.kind).toBe("done"); if (outcome.kind !== "done") return; expect(durableTypes(harness.events())).not.toContain("task.verification-requested"); expect(outcome.evidence.checks).toHaveLength(1); expect(outcome.evidence.checks[0]).toMatchObject({ command: "echo running tests", exitCode: 0, }); expect(outcome.evidence.unresolvedIssues).toEqual([]); assertPairingSafe(harness.events()); }); it("does not gate documentation-only changes", async () => { harness = await makeHarness({ turns: [ toolTurn("req_1", "Updating the docs.", [ { toolUseId: "toolu_doc", toolName: "write", input: { file_path: "NOTES.md", content: "# Notes\n" }, }, ]), textTurn("req_2", "Docs updated."), ], }); harness.user("Update the notes"); const outcome = await harness.kernel.runTurn(); expect(outcome.kind).toBe("done"); expect(durableTypes(harness.events())).not.toContain("task.verification-requested"); }); it("reports failing checks as unresolved issues — real exit codes only", async () => { harness = await makeHarness({ turns: [ WRITE_TURN("req_1"), toolTurn("req_2", "Running the tests.", [ { toolUseId: "toolu_check", toolName: "bash", input: { command: "echo tests failed; exit 3" }, }, ]), textTurn("req_3", "Tests fail but I am done."), ], }); harness.user("Create the module"); const outcome = await harness.kernel.runTurn(); expect(outcome.kind).toBe("done"); if (outcome.kind !== "done") return; expect(outcome.evidence.checks[0]).toMatchObject({ exitCode: 3 }); expect( outcome.evidence.unresolvedIssues.some((issue) => issue.includes("exited with code 3")), ).toBe(true); }); }); describe("VerificationGate services", () => { it("detects check commands from real package scripts", async () => { harness = await makeHarness({ turns: [] }); writeFileSync( path.join(harness.dir, "package.json"), JSON.stringify({ scripts: { test: "vitest run", lint: "eslint .", dev: "tsx watch" } }), "utf8", ); const gate = new VerificationGate({ workspace: new LocalWorkspace(harness.dir) }); expect(await gate.detectChecks()).toEqual(["npm test", "npm run lint"]); }); it("returns no detected checks when package.json is absent", async () => { harness = await makeHarness({ turns: [] }); expect(await harness.verifier.detectChecks()).toEqual([]); }); it("merges GitService-shaped change attribution into the evidence", async () => { harness = await makeHarness({ turns: [textTurn("req_1", "ok")] }); const attributor: ChangeAttributor = { attributeChanges: () => Promise.resolve({ kind: "ok" as const, value: { khaelor: ["src/extra.ts"], preExisting: ["user-wip.ts"] }, }), }; const gate = new VerificationGate({ workspace: harness.workspace, attributor }); harness.user("Question only"); await harness.kernel.runTurn(); const evidence = await gate.collectEvidence(harness.events()); expect(evidence.changedFiles).toContain("src/extra.ts"); expect(evidence.changedFiles).not.toContain("user-wip.ts"); // never claim user work expect(evidence.objective).toBe("Question only"); }); });