Repository Public
1/** The basic loop: prefix, tool round-trips, parallel reads, invalid input, nudges, refusal, max_tokens, finish gating. */2import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest';3import { operatorNotesOf, scriptedProvider, toolResultsOf, userTextsOf, type ScriptItem } from '../src/testkit/index.js';4import {5 cancelQuietly, createTestDb, criteriaOf, events, getRun, makeWorkspace, newMission, read, startEngine, steps, toolSteps, waitStatus,6 type MissionRef, type TestDb, type TestWorker,7} from './helpers.js';89let db: TestDb;10let worker: TestWorker | null = null;11let mission: MissionRef | null = null;12beforeAll(async () => {13 db = await createTestDb();14});15afterEach(async () => {16 await worker?.stop();17 worker = null;18 if (mission) await cancelQuietly(db, mission.missionId);19 mission = null;20});21afterAll(async () => {22 await db.cleanup();23});2425async function play(script: ScriptItem[], opts: { files?: Record<string, string>; git?: boolean; mission?: Record<string, unknown>; deps?: Parameters<typeof startEngine>[2] } = {}) {26 const ws = makeWorkspace(db, { files: opts.files, git: opts.git });27 const provider = scriptedProvider(script);28 mission = await newMission(db, ws, opts.mission);29 worker = startEngine(db, provider, opts.deps);30 const run = await waitStatus(db, mission.runId);31 return { run, provider, ws, m: mission };32}3334const FINISH_DONE = { name: 'finish', input: { status: 'done', summary: 'All done.', reason: null, evidence: [], openQuestions: [] } };35const PASS_C1 = { name: 'update_criteria', input: { updates: [{ id: 'c1', passes: true, evidence: 'checked: it works' }] } };3637describe('agent loop', () => {38 it('runs a tool round-trip to done, with a frozen prefix and a pinned prompt hash', async () => {39 const { run, provider, ws, m } = await play(40 [41 { text: 'Reading the file.', calls: [{ name: 'read_file', input: { path: 'a.txt', offset: null, limit: null } }] },42 { calls: [{ name: 'write_file', input: { path: 'out.txt', content: 'hello\n' } }] },43 { calls: [PASS_C1] },44 { calls: [FINISH_DONE] },45 ],46 { files: { 'a.txt': 'alpha\n', 'AGENTS.md': '# Project rules\nAlways be careful.\n' } },47 );48 expect(run.status).toBe('done');49 expect(run.result?.summary).toBe('All done.');50 expect(run.result?.evidence).toEqual([{ criterion: 'c1', kind: 'note', detail: 'checked: it works' }]);51 expect(read(ws, 'out.txt')).toBe('hello\n');52 expect(run.prompt_hash).toMatch(/^[0-9a-f]{64}$/);53 expect(run.usage.turns).toBe(4);5455 // frozen prefix: identical system + tools on every call, no id / date / path in it56 const systems = new Set(provider.requests.map((r) => r.system));57 expect(systems.size).toBe(1);58 const system = [...systems][0] as string;59 expect(system).toContain('Always be careful.');60 expect(system).not.toContain(m.missionId);61 expect(system).not.toContain(m.runId);62 expect(system).not.toContain(ws);63 expect(system).not.toMatch(/20\d\d-\d\d-\d\d/);64 expect(new Set(provider.requests.map((r) => r.toolNames.join(','))).size).toBe(1);65 expect(provider.requests[0]?.toolNames).toEqual(['read_file', 'list_dir', 'search', 'edit_file', 'write_file', 'shell', 'git_status', 'git_diff', 'git_commit', 'git_push', 'ask_owner', 'update_criteria', 'read_artifact', 'finish']);6667 // first user message: objective + criteria68 const first = userTextsOf(provider.requests[0]?.transcript ?? [])[0] ?? '';69 expect(first).toContain('Test objective');70 expect(first).toContain('c1. It works');71 // the model saw the file content, then the write result72 const seen = toolResultsOf(provider.lastTranscript());73 expect(seen[0]).toMatchObject({ name: 'read_file', ok: true });74 expect(seen[0]?.content).toContain('alpha');75 expect(seen[1]).toMatchObject({ name: 'write_file', ok: true });76 // journal: model_call requests are digests, never the transcript77 const st = await steps(db, m.runId);78 const models = st.filter((s) => s.kind === 'model_call');79 expect(models).toHaveLength(4);80 for (const s of models) {81 expect(Object.keys(s.request).sort()).toEqual(['effort', 'model', 'toolNames', 'transcriptSha256', 'turn']);82 expect(JSON.stringify(s.request).length).toBeLessThan(600);83 }84 const write = st.find((s) => s.name === 'write_file');85 expect(write?.result).toMatchObject({ ok: true });86 expect(String(write?.result?.diff)).toContain('+hello');87 expect((await criteriaOf(db, m.missionId))[0]).toMatchObject({ id: 'c1', passes: true, evidence: 'checked: it works' });88 expect((await events(db, m.missionId, 'policy.decision')).map((e) => `${String(e.tool)}:${String(e.decision)}`)).toEqual(['read_file:allow', 'write_file:allow']);89 });9091 it('runs parallel-safe reads of one turn concurrently, with seqs in call order', async () => {92 const { run, provider, m } = await play(93 [94 {95 calls: [96 { name: 'read_file', input: { path: 'a.txt', offset: null, limit: null } },97 { name: 'read_file', input: { path: 'b.txt', offset: null, limit: null } },98 { name: 'list_dir', input: { path: '.', depth: null } },99 { name: 'write_file', input: { path: 'c.txt', content: 'c' } },100 { name: 'read_file', input: { path: 'c.txt', offset: null, limit: null } },101 ],102 },103 { calls: [PASS_C1, FINISH_DONE] },104 ],105 { files: { 'a.txt': 'A', 'b.txt': 'B' } },106 );107 expect(run.status).toBe('done');108 const tools = await toolSteps(db, m.runId);109 expect(tools.map((s) => s.name)).toEqual(['read_file', 'read_file', 'list_dir', 'write_file', 'read_file', 'update_criteria', 'finish']);110 expect(tools.map((s) => s.seq)).toEqual([...tools.map((s) => s.seq)].sort((a, b) => a - b));111 // the three leading reads overlapped in time; the write started after all of them had completed112 const t = await (async () => (await db.k.pool.query('SELECT seq, started_at, completed_at FROM steps WHERE run_id = $1 AND kind = $2 ORDER BY seq', [m.runId, 'tool_call'])).rows as { started_at: Date; completed_at: Date }[])();113 const firstDone = Math.min(...t.slice(0, 3).map((r) => r.completed_at.getTime()));114 const lastStart = Math.max(...t.slice(0, 3).map((r) => r.started_at.getTime()));115 expect(lastStart).toBeLessThanOrEqual(firstDone);116 expect(t[3]!.started_at.getTime()).toBeGreaterThanOrEqual(Math.max(...t.slice(0, 3).map((r) => r.completed_at.getTime())));117 // results come back in call order, in ONE batch118 const batch = provider.requests[1]?.transcript.filter((e) => e.role === 'tool') ?? [];119 expect(batch).toHaveLength(1);120 expect(toolResultsOf(provider.requests[1]?.transcript ?? []).map((r) => r.callId)).toEqual(['call_0_0', 'call_0_1', 'call_0_2', 'call_0_3', 'call_0_4']);121 expect(toolResultsOf(provider.lastTranscript())[4]?.content).toContain('c');122 });123124 it('answers invalid tool input with an error result and executes nothing', async () => {125 const { run, provider, ws, m } = await play([126 {127 calls: [128 { name: 'write_file', input: { path: 'x.txt' } }, // zod: content missing129 { name: 'write_file', input: '__AOS_INVALID_TOOL_INPUT__ {"reason":"truncated","detail":"cut off","raw":"{\\"path\\":\\"y.txt\\""}' },130 { name: 'no_such_tool', input: {} },131 ],132 },133 { calls: [{ name: 'finish', input: { status: 'failed', summary: 'Could not.', reason: 'bad inputs', evidence: [], openQuestions: [] } }] },134 ]);135 expect(run.status).toBe('failed');136 const seen = toolResultsOf(provider.lastTranscript());137 expect(seen.map((r) => r.ok)).toEqual([false, false, false]);138 expect(seen[0]?.content).toMatch(/Invalid input for write_file: content/);139 expect(seen[1]?.content).toMatch(/truncat|cut off/i);140 expect(seen[2]?.content).toMatch(/Unknown tool "no_such_tool"/);141 expect((await toolSteps(db, m.runId)).map((s) => s.name)).toEqual(['finish']);142 expect(await import('node:fs').then((f) => f.readdirSync(ws))).toEqual([]);143 });144145 it('nudges a turn without tool calls twice, then ends partial', async () => {146 const { run, provider } = await play([{ text: 'I think I am done.' }, { text: 'Really done.' }, { text: 'Nothing more to say.' }]);147 expect(run.status).toBe('partial');148 expect(run.result?.summary).toMatch(/never called finish/);149 expect(run.result?.summary).toContain('Nothing more to say.');150 expect(provider.requests).toHaveLength(3);151 const notes = operatorNotesOf(provider.lastTranscript());152 expect(notes).toHaveLength(2);153 expect(notes[0]).toMatch(/reminder 1 of 2/);154 expect(notes[1]).toMatch(/reminder 2 of 2/);155 });156157 it('gates finish: "done" is refused until every criterion passes with evidence', async () => {158 const { run, provider, m } = await play(159 [160 { calls: [FINISH_DONE] },161 { calls: [{ name: 'update_criteria', input: { updates: [{ id: 'c1', passes: true, evidence: 'ran the test: ok' }, { id: 'c9', passes: true, evidence: 'x' }] } }] },162 { calls: [{ name: 'update_criteria', input: { updates: [{ id: 'c1', passes: true, evidence: 'ran the test: ok' }] } }, FINISH_DONE] },163 { calls: [{ name: 'update_criteria', input: { updates: [{ id: 'c2', passes: true, evidence: 'file exists' }] } }, FINISH_DONE] },164 ],165 { mission: { doneCriteria: ['Tests pass', 'File exists'] } },166 );167 expect(run.status).toBe('done');168 const seen = toolResultsOf(provider.lastTranscript());169 expect(seen[0]).toMatchObject({ name: 'finish', ok: false });170 expect(seen[0]?.content).toMatch(/2 of 2 done criteria are not passing/);171 expect(seen[1]).toMatchObject({ name: 'update_criteria', ok: false });172 expect(seen[1]?.content).toMatch(/Unknown criterion id: c9/);173 expect(seen[2]).toMatchObject({ name: 'update_criteria', ok: true });174 expect(seen[3]).toMatchObject({ name: 'finish', ok: false });175 expect(seen[3]?.content).toMatch(/c2: File exists/);176 expect(await criteriaOf(db, m.missionId)).toEqual([177 { id: 'c1', text: 'Tests pass', passes: true, evidence: 'ran the test: ok' },178 { id: 'c2', text: 'File exists', passes: true, evidence: 'file exists' },179 ]);180 const notices = (await events(db, m.missionId, 'system.notice')).filter((e) => e.code === 'criteria');181 expect(notices).toHaveLength(2);182 expect(run.result?.evidence.map((e) => e.criterion)).toEqual(['c1', 'c2']);183 });184185 it('partial / blocked are always accepted with a reason', async () => {186 const { run } = await play([{ calls: [{ name: 'finish', input: { status: 'blocked', summary: 'Need access.', reason: 'no credentials', evidence: [], openQuestions: ['Which account?'] } }] }]);187 expect(run.status).toBe('blocked');188 expect(run.result?.summary).toContain('Reason: no credentials');189 expect(run.result?.openQuestions).toEqual(['Which account?']);190 });191192 it('a refusal ends the run blocked with the category', async () => {193 const { run } = await play([{ text: 'I cannot help with that.', refusal: { category: 'cyber', explanation: 'policy' } }]);194 expect(run.status).toBe('blocked');195 expect(run.result?.summary).toMatch(/declined to continue.*category: cyber/);196 });197198 it('continues after max_tokens with an operator note', async () => {199 const { run, provider } = await play([{ text: 'A very long answer that was cut', stop: 'max_tokens' }, { calls: [PASS_C1, FINISH_DONE] }]);200 expect(run.status).toBe('done');201 expect(operatorNotesOf(provider.lastTranscript())[0]).toMatch(/cut off by the output-token limit/);202 });203204 it('a missing provider key ends the mission blocked without failing the worker', async () => {205 const { run, m } = await play([{ error: { message: 'ANTHROPIC_API_KEY is not set', code: 'missing_api_key' } }]);206 expect(run.status).toBe('blocked');207 expect(run.result?.summary).toMatch(/ANTHROPIC_API_KEY\) is not configured on the worker/);208 // the same worker still serves the next mission209 const provider2 = scriptedProvider([{ calls: [PASS_C1, FINISH_DONE] }]);210 await worker?.stop();211 const m2 = await newMission(db, m.ws);212 worker = startEngine(db, provider2);213 expect((await waitStatus(db, m2.runId)).status).toBe('done');214 });215216 it('retries a retryable model failure after a durable back-off, and fails on a permanent one', async () => {217 const a = await play(218 [(_req, info) => (info.callNo === 0 ? { error: { message: 'overloaded', code: 'overloaded_error', status: 529, retryable: true } } : { calls: [PASS_C1, FINISH_DONE] })],219 { deps: { limits: { modelRetryDelaysMs: [50] } } },220 );221 expect(a.run.status).toBe('done');222 const st = await steps(db, a.m.runId);223 expect(st.filter((s) => s.kind === 'model_call').map((s) => s.status)).toEqual(['failed', 'completed']);224 expect(st.some((s) => s.kind === 'timer')).toBe(true);225 await worker?.stop();226 const b = await play([{ error: { message: 'invalid request', code: 'invalid_request_error', status: 400, retryable: false } }]);227 expect(b.run.status).toBe('failed');228 expect(b.run.result?.summary).toMatch(/model call failed \(code invalid_request_error, HTTP 400\): invalid request/);229 expect((await getRun(db, b.m.runId)).usage.turns ?? 0).toBe(0);230 });231});232