spb/khaelor Public

1/**2 * KHAELOR3 * File: tests/agent/verification.test.ts4 * Description: Verification gate tests — nudge on unverified code changes, 2-attempt budget, evidence from real events only.5 *6 * Author: Simon-Pierre Boucher7 * Contact: contact@spboucher.ai8 */910import { rmSync, writeFileSync } from "node:fs";11import * as path from "node:path";12import { afterEach, describe, expect, it } from "vitest";13import { VerificationGate } from "../../src/agent/index.js";14import type { ChangeAttributor } from "../../src/agent/index.js";15import { LocalWorkspace } from "../../src/workspace/index.js";16import {17  assertPairingSafe,18  durableTypes,19  makeHarness,20  textTurn,21  toolTurn,22} from "./fixtures.js";23import type { AgentHarness } from "./fixtures.js";2425let harness: AgentHarness | null = null;2627afterEach(async () => {28  if (harness !== null) {29    await harness.log.close();30    rmSync(harness.dir, { recursive: true, force: true });31    harness = null;32  }33});3435const WRITE_TURN = (requestId: string): ReturnType<typeof toolTurn> =>36  toolTurn(requestId, "Writing the module.", [37    {38      toolUseId: `toolu_${requestId}`,39      toolName: "write",40      input: { file_path: "src/answer.ts", content: "export const answer = 42;\n" },41    },42  ]);4344describe("verification gate", () => {45  it("nudges when files changed without checks, and stops after 2 attempts", async () => {46    harness = await makeHarness({47      turns: [48        WRITE_TURN("req_1"),49        textTurn("req_2", "Done."), // stops without verifying → nudge 150        textTurn("req_3", "It looks correct to me."), // still no checks → nudge 251        textTurn("req_4", "Done (final)."), // budget exhausted → accepted, withheld honestly52      ],53    });5455    harness.user("Create the answer module");56    const outcome = await harness.kernel.runTurn();57    expect(outcome.kind).toBe("done");58    if (outcome.kind !== "done") return;5960    const events = harness.events();61    const nudges = events.filter((event) => event.type === "task.verification-requested");62    expect(nudges).toHaveLength(2);63    expect(nudges.map((n) => (n.type === "task.verification-requested" ? n.payload.attempt : 0))).toEqual([1, 2]);6465    // 4 model calls: main, tool follow-up, nudge 1, nudge 2.66    expect(harness.client.requests).toHaveLength(4);6768    // Nudge requests carry the synthetic verification message as volatile context.69    const third = harness.client.requests[2];70    const serialized = JSON.stringify(third?.messages);71    expect(serialized).toContain("Verification required before completion");72    // …and it was NOT in the pre-nudge requests.73    expect(JSON.stringify(harness.client.requests[1]?.messages)).not.toContain(74      "Verification required before completion",75    );7677    // Evidence is honest: changed files listed, zero checks, unresolved issue recorded.78    expect(outcome.evidence.changedFiles.some((p) => p.includes("answer.ts"))).toBe(true);79    expect(outcome.evidence.checks).toEqual([]);80    expect(81      outcome.evidence.unresolvedIssues.some((issue) =>82        issue.includes("without fresh verification evidence"),83      ),84    ).toBe(true);85    assertPairingSafe(events);86  });8788  it("accepts completion without a nudge when a check ran after the last change", async () => {89    harness = await makeHarness({90      turns: [91        WRITE_TURN("req_1"),92        toolTurn("req_2", "Running the tests.", [93          { toolUseId: "toolu_check", toolName: "bash", input: { command: "echo running tests" } },94        ]),95        textTurn("req_3", "Done and verified."),96      ],97    });9899    harness.user("Create the answer module and verify it");100    const outcome = await harness.kernel.runTurn();101    expect(outcome.kind).toBe("done");102    if (outcome.kind !== "done") return;103104    expect(durableTypes(harness.events())).not.toContain("task.verification-requested");105    expect(outcome.evidence.checks).toHaveLength(1);106    expect(outcome.evidence.checks[0]).toMatchObject({107      command: "echo running tests",108      exitCode: 0,109    });110    expect(outcome.evidence.unresolvedIssues).toEqual([]);111    assertPairingSafe(harness.events());112  });113114  it("does not gate documentation-only changes", async () => {115    harness = await makeHarness({116      turns: [117        toolTurn("req_1", "Updating the docs.", [118          {119            toolUseId: "toolu_doc",120            toolName: "write",121            input: { file_path: "NOTES.md", content: "# Notes\n" },122          },123        ]),124        textTurn("req_2", "Docs updated."),125      ],126    });127    harness.user("Update the notes");128    const outcome = await harness.kernel.runTurn();129    expect(outcome.kind).toBe("done");130    expect(durableTypes(harness.events())).not.toContain("task.verification-requested");131  });132133  it("reports failing checks as unresolved issues — real exit codes only", async () => {134    harness = await makeHarness({135      turns: [136        WRITE_TURN("req_1"),137        toolTurn("req_2", "Running the tests.", [138          {139            toolUseId: "toolu_check",140            toolName: "bash",141            input: { command: "echo tests failed; exit 3" },142          },143        ]),144        textTurn("req_3", "Tests fail but I am done."),145      ],146    });147    harness.user("Create the module");148    const outcome = await harness.kernel.runTurn();149    expect(outcome.kind).toBe("done");150    if (outcome.kind !== "done") return;151    expect(outcome.evidence.checks[0]).toMatchObject({ exitCode: 3 });152    expect(153      outcome.evidence.unresolvedIssues.some((issue) => issue.includes("exited with code 3")),154    ).toBe(true);155  });156});157158describe("VerificationGate services", () => {159  it("detects check commands from real package scripts", async () => {160    harness = await makeHarness({ turns: [] });161    writeFileSync(162      path.join(harness.dir, "package.json"),163      JSON.stringify({ scripts: { test: "vitest run", lint: "eslint .", dev: "tsx watch" } }),164      "utf8",165    );166    const gate = new VerificationGate({ workspace: new LocalWorkspace(harness.dir) });167    expect(await gate.detectChecks()).toEqual(["npm test", "npm run lint"]);168  });169170  it("returns no detected checks when package.json is absent", async () => {171    harness = await makeHarness({ turns: [] });172    expect(await harness.verifier.detectChecks()).toEqual([]);173  });174175  it("merges GitService-shaped change attribution into the evidence", async () => {176    harness = await makeHarness({ turns: [textTurn("req_1", "ok")] });177    const attributor: ChangeAttributor = {178      attributeChanges: () =>179        Promise.resolve({180          kind: "ok" as const,181          value: { khaelor: ["src/extra.ts"], preExisting: ["user-wip.ts"] },182        }),183    };184    const gate = new VerificationGate({ workspace: harness.workspace, attributor });185    harness.user("Question only");186    await harness.kernel.runTurn();187188    const evidence = await gate.collectEvidence(harness.events());189    expect(evidence.changedFiles).toContain("src/extra.ts");190    expect(evidence.changedFiles).not.toContain("user-wip.ts"); // never claim user work191    expect(evidence.objective).toBe("Question only");192  });193});194