/** * KHAELOR * File: src/agent/completion.ts * Description: Verification gate and CompletionEvidence — evidence-backed completion, never confident sentences (ADR-12, CLAUDE.md §17). * * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai */ import * as path from "node:path"; import type { CheckResult, CompletionEvidence, DurableEvent } from "../session/index.js"; import type { Workspace } from "../workspace/index.js"; // ───────────────────────── change attribution seam ───────────────────────── /** Files attributable to this session vs pre-existing user work (CLAUDE.md §16). */ export interface ChangeAttribution { khaelor: string[]; preExisting: string[]; } /** Structurally satisfied by `GitService.attributeChanges` (src/repository) — wired by the composition root. */ export type AttributionResult = | { kind: "ok"; value: ChangeAttribution } | { kind: "not-a-repo" } | { kind: "error"; code: string; message: string }; export interface ChangeAttributor { attributeChanges(): Promise; } // ───────────────────────── turn facts (pure fold) ───────────────────────── /** Commands that count as verification evidence when run through `bash`. */ export const CHECK_COMMAND_PATTERN = /\b(tests?|vitest|jest|pytest|tsc|typecheck|lint|eslint|ruff|build|check)\b/i; const DOCUMENTATION_EXTENSIONS = [".md", ".markdown", ".rst", ".adoc", ".txt"]; /** Documentation-only changes are filtered from the gate (ADR-12). */ export function isDocumentationPath(filePath: string): boolean { const lower = filePath.toLowerCase(); return DOCUMENTATION_EXTENSIONS.some((ext) => lower.endsWith(ext)); } interface TurnCheck extends CheckResult { seq: number; } /** Everything the gate and evidence builder need, derived from the recorded log in one pass. */ export interface TurnFacts { /** Seq of the turn's user message; 0 when the log has none. */ turnStartSeq: number; /** The user's request text — the evidence `objective`. */ objective: string; /** All files modified this turn (write/edit events). */ changedFiles: string[]; /** Seq of the last NON-documentation file change this turn; 0 when none. */ lastCodeChangeSeq: number; /** Check-shaped bash commands completed this turn, with real exit codes only. */ checks: TurnCheck[]; /** Verification nudges already issued this turn (budget ≤ 2). */ verificationAttempts: number; /** Seq of the last settled assistant text block — the withheld candidate (ADR-12). */ candidateSeq: number; } /** Fold the durable stream into the current turn's completion-relevant facts. */ export function collectTurnFacts(events: readonly DurableEvent[]): TurnFacts { let turnStartSeq = 0; let objective = ""; for (const event of events) { if (event.type === "user.message-created") { turnStartSeq = event.seq; objective = event.payload.text; } } const changedFiles = new Set(); let lastCodeChangeSeq = 0; const bashCommands = new Map(); // toolUseId → command const checks: TurnCheck[] = []; let verificationAttempts = 0; let candidateSeq = 0; for (const event of events) { if (event.seq <= turnStartSeq) continue; switch (event.type) { case "file.modified": { changedFiles.add(event.payload.path); if (!isDocumentationPath(event.payload.path)) { lastCodeChangeSeq = Math.max(lastCodeChangeSeq, event.seq); } break; } case "tool.requested": { if (event.payload.toolName === "bash") { const input = event.payload.input; const command = input !== null && typeof input === "object" ? (input as Record)["command"] : undefined; if (typeof command === "string") bashCommands.set(event.payload.toolUseId, command); } break; } case "tool.completed": { const command = bashCommands.get(event.payload.toolUseId); const exitCode = event.payload.ui.exitCode; if ( command !== undefined && CHECK_COMMAND_PATTERN.test(command) && typeof exitCode === "number" ) { checks.push({ command, exitCode, summary: event.payload.ui.summary, durationMs: event.payload.durationMs, seq: event.seq, }); } break; } case "task.verification-requested": { verificationAttempts += 1; break; } case "model.text-block-completed": { candidateSeq = event.seq; break; } default: break; } } return { turnStartSeq, objective, changedFiles: [...changedFiles].sort(), lastCodeChangeSeq, checks, verificationAttempts, candidateSeq, }; } // ───────────────────────── the gate ───────────────────────── export interface VerificationGateResult { /** Code changed this turn with no check run since the last change. */ required: boolean; /** Nudges already issued this turn. */ attempts: number; /** Seq of the withheld candidate answer (last settled text block). */ candidateSeq: number; } export interface VerificationGateOptions { workspace: Workspace; /** GitService (or compatible) — merged into evidence.changedFiles when available. */ attributor?: ChangeAttributor; } const CHECK_SCRIPT_NAMES = ["test", "typecheck", "lint", "build", "check"] as const; /** * The verification gate (ARCHITECTURE.md §9). When the model stops with code * changed this turn and no fresh verification evidence, the kernel records * `VerificationRequested` (max 2 attempts) and re-prompts with a nudge before * accepting completion. All evidence is real: recorded events and actual * repository state — nothing is fabricated (Absolute Rule #4). */ export class VerificationGate { readonly #workspace: Workspace; readonly #attributor: ChangeAttributor | undefined; constructor(options: VerificationGateOptions) { this.#workspace = options.workspace; this.#attributor = options.attributor; } /** Pure decision over recorded state — consumed by the kernel's deriveNext. */ needsVerification(events: readonly DurableEvent[]): VerificationGateResult { const facts = collectTurnFacts(events); const freshEvidence = facts.checks.some((check) => check.seq > facts.lastCodeChangeSeq); return { required: facts.lastCodeChangeSeq > 0 && !freshEvidence, attempts: facts.verificationAttempts, candidateSeq: facts.candidateSeq, }; } /** * Detect relevant check commands from the repository — package scripts * only in V1, never blind full suites (ADR-12). Empty when undetectable. */ async detectChecks(): Promise { try { const raw = await this.#workspace.readFile( path.join(this.#workspace.cwd(), "package.json"), ); const parsed = JSON.parse(raw) as { scripts?: Record }; const scripts = parsed.scripts ?? {}; const detected: string[] = []; for (const name of CHECK_SCRIPT_NAMES) { if (typeof scripts[name] === "string") { detected.push(name === "test" ? "npm test" : `npm run ${name}`); } } return detected; } catch { return []; } } /** The synthetic evidence-bearing nudge injected as volatile per-turn context. */ buildNudge(detectedChecks: readonly string[]): string { const suggestion = detectedChecks.length > 0 ? ` Detected project checks: ${detectedChecks.join(" · ")}.` : ""; return ( "Verification required before completion: files were modified this turn but no check " + "has been run since the last change. Run the relevant project checks now with the bash " + `tool and report their real results.${suggestion} If a check fails, fix the code and ` + "re-run it. Only then give your final answer." ); } /** * Build the CompletionEvidence record (CLAUDE.md §17) from recorded events * plus live change attribution. Every check result carries a real exit code. */ async collectEvidence(events: readonly DurableEvent[]): Promise { const facts = collectTurnFacts(events); const changed = new Set(facts.changedFiles); if (this.#attributor !== undefined) { try { const attribution = await this.#attributor.attributeChanges(); if (attribution.kind === "ok") { for (const file of attribution.value.khaelor) changed.add(file); } } catch { // Attribution is best-effort evidence enrichment — never fabricated, never fatal. } } const unresolvedIssues: string[] = []; for (const check of facts.checks) { if (check.exitCode !== 0) { unresolvedIssues.push(`"${check.command}" exited with code ${check.exitCode}.`); } } const gate = this.needsVerification(events); if (gate.required) { unresolvedIssues.push( gate.attempts >= 2 ? "Code files changed without fresh verification evidence (verification nudge budget exhausted)." : "Code files changed without fresh verification evidence.", ); } return { objective: facts.objective, changedFiles: [...changed].sort(), checks: facts.checks.map(({ command, exitCode, summary, durationMs }) => ({ command, exitCode, summary, durationMs, })), unresolvedIssues, }; } }