/** * KHAELOR * File: tests/verify/verify.test.ts * Description: Native-verification tests — config parsing, errors-first truncation, repair-loop accounting, runner events (v2 §4). * * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai */ import { describe, expect, it } from "vitest"; import { countRepairFailures, parseVerifyConfig, truncateErrorsFirst, VerifyRunner, } from "../../src/verify/index.js"; import type { DurableEvent, DurableEventInput } from "../../src/session/index.js"; import type { Command, ProcessResult, Workspace } from "../../src/workspace/index.js"; function fakeSession(): { events(): readonly DurableEvent[]; publishDurable(input: DurableEventInput): DurableEvent; } { const events: DurableEvent[] = []; let seq = 0; return { events: () => events, publishDurable(input: DurableEventInput): DurableEvent { seq += 1; const event = { v: 1, id: `e${seq}`, sessionId: "s1", seq, ts: seq, type: input.type, payload: input.payload, } as DurableEvent; events.push(event); return event; }, }; } function fakeWorkspace(results: Record>): Workspace { return { cwd: () => "/repo", readFile: () => Promise.reject(new Error("not used")), writeFile: () => Promise.resolve(), exec: (command: Command) => Promise.resolve({ exitCode: 0, stdout: "", stderr: "", durationMs: 1, truncated: false, ...(results[command.cmd] ?? {}), }), }; } describe("parseVerifyConfig", () => { it("parses checks, policy, and maxRepairLoops", () => { const config = parseVerifyConfig({ typecheck: { cmd: "npx tsc --noEmit", timeout: 60 }, lint: { cmd: "npx eslint --fix", timeout: 30, autofix: true }, policy: "before-final-answer", maxRepairLoops: 2, }); expect(config?.checks).toHaveLength(2); expect(config?.checks[0]?.timeoutMs).toBe(60_000); expect(config?.checks[1]?.autofix).toBe(true); expect(config?.policy).toBe("before-final-answer"); expect(config?.maxRepairLoops).toBe(2); }); it("skips invalid entries and defaults the policy", () => { const config = parseVerifyConfig({ bad: { nope: true }, test: { cmd: "npm test" } }); expect(config?.checks.map((check) => check.name)).toEqual(["test"]); expect(config?.policy).toBe("after-each-edit-batch"); expect(config?.maxRepairLoops).toBe(3); }); }); describe("truncateErrorsFirst", () => { it("keeps error-looking lines ahead of noise", () => { const noise = Array.from({ length: 300 }, (_, i) => `line ${i} of ordinary output padding`).join("\n"); const output = `${noise}\nsrc/a.ts(42): error TS2345: nope`; const truncated = truncateErrorsFirst(output, 500); expect(truncated).toContain("error TS2345"); expect(truncated).toContain("[... verify output truncated"); expect(truncated.indexOf("error TS2345")).toBeLessThan(truncated.indexOf("line 0")); }); it("returns small output untouched", () => { expect(truncateErrorsFirst("all good", 100)).toBe("all good"); }); }); describe("countRepairFailures", () => { it("counts failing rounds since the last user message", () => { const session = fakeSession(); session.publishDurable({ type: "user.message-created", payload: { text: "go", mentions: [] } }); session.publishDurable({ type: "verify.result", payload: { check: "t", command: "c", ok: false, exitCode: 1, output: "x", durationMs: 1 }, }); session.publishDurable({ type: "verify.result", payload: { check: "t", command: "c", ok: true, exitCode: 0, output: "", durationMs: 1 }, }); expect(countRepairFailures(session.events())).toBe(1); session.publishDurable({ type: "user.message-created", payload: { text: "next", mentions: [] } }); expect(countRepairFailures(session.events())).toBe(0); }); }); describe("VerifyRunner", () => { it("publishes one verify.result per check with real exit codes", async () => { const session = fakeSession(); const workspace = fakeWorkspace({ "npm test": { exitCode: 1, stderr: "1 test failed\nFAIL src/a.test.ts" }, }); const runner = new VerifyRunner({ workspace, session, config: { checks: [ { name: "typecheck", cmd: "npx tsc --noEmit", timeoutMs: 1000, autofix: false }, { name: "test", cmd: "npm test", timeoutMs: 1000, autofix: false }, ], policy: "after-each-edit-batch", maxRepairLoops: 3, }, }); const outcome = await runner.runAll(); expect(outcome.ok).toBe(false); const results = session.events().filter((event) => event.type === "verify.result"); expect(results).toHaveLength(2); const failed = results.find((event) => event.type === "verify.result" && !event.payload.ok); expect(failed !== undefined && failed.type === "verify.result" ? failed.payload.output : "").toContain( "test failed", ); }); it("withinRepairBudget flips false after maxRepairLoops failures", async () => { const session = fakeSession(); session.publishDurable({ type: "user.message-created", payload: { text: "go", mentions: [] } }); const workspace = fakeWorkspace({ bad: { exitCode: 1, stderr: "boom" } }); const runner = new VerifyRunner({ workspace, session, config: { checks: [{ name: "bad", cmd: "bad", timeoutMs: 1000, autofix: false }], policy: "after-each-edit-batch", maxRepairLoops: 2, }, }); expect(runner.withinRepairBudget()).toBe(true); await runner.runAll(); expect(runner.withinRepairBudget()).toBe(true); await runner.runAll(); expect(runner.withinRepairBudget()).toBe(false); }); });