Repository Public
1/** Real tools through the loop: edit staleness, the Seatbelt sandbox, git branch + commit, output offloading, run-end cleanup. */2import fs from 'node:fs';3import os from 'node:os';4import path from 'node:path';5import { requestCancel } from '@aos/kernel';6import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest';7import { sandboxDenyRead, sweepRunTmp } from '../src/index.js';8import { scriptedProvider, toolResultsOf, userTextsOf, type ScriptItem } from '../src/testkit/index.js';9import {10 cancelQuietly, createTestDb, exists, getRun, git, makeWorkspace, newMission, pendingWait, read, rows, startEngine, steps, waitFor, waitStatus,11 type MissionRef, type TestDb, type TestWorker,12} from './helpers.js';1314let db: TestDb;15let worker: TestWorker | null = null;16let mission: MissionRef | null = null;17beforeAll(async () => {18 db = await createTestDb();19});20afterEach(async () => {21 await worker?.stop();22 worker = null;23 if (mission) await cancelQuietly(db, mission.missionId);24 mission = null;25});26afterAll(async () => {27 await db.cleanup();28});29async function play(script: ScriptItem[], ws: string, over: Record<string, unknown> = {}, deps: Parameters<typeof startEngine>[2] = {}) {30 const provider = scriptedProvider(script);31 mission = await newMission(db, ws, over);32 worker = startEngine(db, provider, deps);33 const run = await waitStatus(db, mission.runId, undefined, 60_000);34 return { run, provider, m: mission };35}36const READ = (p: string) => ({ name: 'read_file', input: { path: p, offset: null, limit: null } });37const EDIT = (p: string, oldString: string, newString: string) => ({ name: 'edit_file', input: { path: p, oldString, newString, replaceAll: null } });38const SH = (command: string) => ({ name: 'shell', input: { command, timeoutSec: null } });39const PARTIAL = { name: 'finish', input: { status: 'partial', summary: 'Stop.', reason: 'test', evidence: [], openQuestions: [] } };40const runTmp = (runId: string): string => path.join(fs.realpathSync(os.tmpdir()), `aos-run-${runId}`);4142describe('file tools through the engine', () => {43 it('edit staleness: an edit fails when the file was never read, or changed since it was read', async () => {44 const ws = makeWorkspace(db, { files: { 'a.txt': 'one\n', 'b.txt': 'two\n' } });45 const { run, provider } = await play(46 [47 { calls: [EDIT('a.txt', 'one', 'ONE')] }, // never read48 { calls: [READ('a.txt'), READ('b.txt')] },49 { calls: [EDIT('a.txt', 'one', 'ONE'), SH('echo changed > b.txt')] },50 { calls: [EDIT('b.txt', 'two', 'TWO')] }, // changed on disk by the shell since the read51 { calls: [READ('b.txt')] },52 { calls: [EDIT('b.txt', 'changed', 'CHANGED'), EDIT('a.txt', 'ONE', 'one again')] }, // a.txt: own write is the last known state53 { calls: [PARTIAL] },54 ],55 ws,56 );57 expect(run.status).toBe('partial');58 const seen = toolResultsOf(provider.lastTranscript());59 expect(seen.map((r) => `${r.name}:${r.ok}`)).toEqual([60 'edit_file:false', 'read_file:true', 'read_file:true', 'edit_file:true', 'shell:true', 'edit_file:false', 'read_file:true', 'edit_file:true', 'edit_file:true',61 ]);62 expect(seen[0]?.content).toMatch(/read/i);63 expect(seen[5]?.content).toMatch(/changed|read it again/i);64 expect(read(ws, 'a.txt')).toBe('one again\n');65 expect(read(ws, 'b.txt')).toBe('CHANGED\n');66 });6768 it('offloads long output to an artifact that read_artifact can page through', async () => {69 const ws = makeWorkspace(db, { files: { 'big.txt': Array.from({ length: 4000 }, (_, i) => `line ${i + 1} ${'x'.repeat(20)}`).join('\n') } });70 const { run, provider, m } = await play(71 [72 { calls: [SH('cat big.txt')] },73 (req) => {74 const out = toolResultsOf(req.transcript as never)[0]?.content ?? '';75 const name = /artifact "([^"]+)"/.exec(out)?.[1] ?? 'missing';76 return { calls: [{ name: 'read_artifact', input: { name, offset: 2000, limit: 2 } }] };77 },78 { calls: [PARTIAL] },79 ],80 ws,81 );82 expect(run.status).toBe('partial');83 const seen = toolResultsOf(provider.lastTranscript());84 expect(seen[0]?.content.length).toBeLessThan(10_500);85 expect(seen[0]?.content).toMatch(/Output truncated: \d+ characters/);86 expect(seen[1]).toMatchObject({ name: 'read_artifact', ok: true });87 expect(seen[1]?.content).toContain('line 2000 ');88 const shell = (await steps(db, m.runId)).find((s) => s.name === 'shell');89 const art = shell?.result?.artifact as { name: string; sha256: string; size: number };90 expect(art.size).toBeGreaterThan(100_000);91 expect(await rows(db, 'SELECT name FROM run_artifacts WHERE run_id = $1', [m.runId])).toEqual([{ name: art.name }]);92 expect(run.result?.artifacts).toEqual([{ name: art.name, sha256: art.sha256 }]);93 });94});9596describe('sandboxed shell through the engine', () => {97 it('confines writes, hides sibling projects and secrets, has no network and no API keys', async () => {98 const ws = makeWorkspace(db, { files: { 'a.txt': 'a\n' }, name: 'proj-a' });99 makeWorkspace(db, { files: { 'secret.txt': 'sibling secret\n' }, name: 'proj-b' });100 const outside = path.join(db.tmp, 'outside.txt');101 process.env.ANTHROPIC_API_KEY_TEST_CANARY = 'canary-value';102 const { run, provider, m } = await play(103 [104 {105 calls: [106 SH('echo inside > made.txt && cat made.txt'),107 SH(`echo x > ${outside}`),108 SH('cat ../proj-b/secret.txt'),109 SH('ls ../proj-b'),110 SH('env | grep -c -E "API_KEY|CANARY" ; echo "tmp=$TMPDIR"; echo t > "$TMPDIR/t.txt" && echo tmp-ok'),111 SH('curl -s -m 3 https://example.com >/dev/null && echo NET-OK || echo NET-BLOCKED'),112 ],113 },114 { calls: [PARTIAL] },115 ],116 ws,117 );118 delete process.env.ANTHROPIC_API_KEY_TEST_CANARY;119 expect(run.status).toBe('partial');120 const seen = toolResultsOf(provider.lastTranscript());121 expect(seen[0]).toMatchObject({ ok: true });122 expect(seen[0]?.content).toContain('inside');123 expect(seen[1]?.ok).toBe(false);124 expect(fs.existsSync(outside)).toBe(false);125 expect(seen[2]?.ok).toBe(false);126 expect(seen[2]?.content).not.toContain('sibling secret');127 expect(seen[3]?.ok).toBe(false);128 expect(seen[4]?.content).toMatch(/^\[exit code 0\]\n0\ntmp=.*aos-run-/);129 expect(seen[4]?.content).toContain('tmp-ok');130 expect(seen[5]?.content).toContain('NET-BLOCKED');131 expect(read(ws, 'made.txt')).toBe('inside\n');132 // run-end cleanup: the per-run temp dir is gone once the run is terminal133 expect(fs.existsSync(runTmp(m.runId))).toBe(false);134 });135136 it('runs a project test suite inside the sandbox (node --test)', async () => {137 const ws = makeWorkspace(db, {138 files: {139 'package.json': '{ "type": "module" }\n',140 'sum.js': 'export const sum = (a, b) => a + b;\n',141 'sum.test.js': "import test from 'node:test';\nimport assert from 'node:assert/strict';\nimport { sum } from './sum.js';\ntest('sum', () => assert.equal(sum(2, 2), 4));\n",142 },143 });144 const { provider } = await play([{ calls: [SH('node --test 2>&1 | tail -12')] }, { calls: [PARTIAL] }], ws, {}, { denyOwnerFolders: true });145 const r = toolResultsOf(provider.lastTranscript())[0];146 expect(r?.ok).toBe(true);147 expect(r?.content).toMatch(/pass 1/);148 expect(r?.content).toMatch(/fail 0/);149 });150151 it('fails closed when the sandbox launcher is unavailable', async () => {152 const ws = makeWorkspace(db, { files: { 'a.txt': 'a\n' } });153 const { provider } = await play([{ calls: [SH('echo hi > x.txt')] }, { calls: [PARTIAL] }], ws, {}, { sandboxExecPath: '/nonexistent/sandbox-exec' });154 const r = toolResultsOf(provider.lastTranscript())[0];155 expect(r?.ok).toBe(false);156 expect(r?.content).toMatch(/NOT run/);157 expect(exists(ws, 'x.txt')).toBe(false);158 });159160 it('deny list: siblings of the workspace, owner folders, extras — never an ancestor of the workspace', async () => {161 const home = path.join(db.tmp, 'home');162 const root = path.join(home, 'Desktop', 'work');163 for (const d of ['Desktop/work/p1/src', 'Desktop/work/p2', 'Desktop/other', 'Documents', 'extra']) fs.mkdirSync(path.join(home, d), { recursive: true });164 fs.writeFileSync(path.join(home, 'Desktop', 'note.txt'), 'x');165 const list = await sandboxDenyRead({ workspaceAbs: path.join(root, 'p1'), workspaceRoot: root, extra: [path.join(home, 'extra'), 'relative/ignored'], ownerFolders: true, home });166 expect(list).toEqual([path.join(home, 'Desktop/other'), path.join(home, 'Desktop/work/p2'), path.join(home, 'Documents'), path.join(home, 'extra')].sort());167 });168});169170describe('git checkpointing and cleanup', () => {171 it('works on aos/<mission> created at start; commits carry the mission trailer', async () => {172 const ws = makeWorkspace(db, { git: true, files: { 'a.txt': 'a\n' } });173 const { run, provider, m } = await play(174 [175 { calls: [{ name: 'write_file', input: { path: 'new.txt', content: 'n\n' } }] },176 { calls: [{ name: 'git_commit', input: { message: 'Add new.txt', paths: null } }, { name: 'git_status', input: {} }] },177 { calls: [PARTIAL] },178 ],179 ws,180 );181 expect(run.status).toBe('partial');182 const branch = `aos/${m.missionId.slice(0, 8)}`;183 expect(git(ws, 'symbolic-ref', '--short', 'HEAD').trim()).toBe(branch);184 expect(git(ws, 'log', '--format=%s', branch).trim().split('\n')).toEqual(['Add new.txt', 'init']);185 expect(git(ws, 'log', '--format=%s', 'main').trim()).toBe('init');186 expect(git(ws, 'log', '-1', '--format=%B')).toContain(`AOS-Mission: ${m.missionId}`);187 expect(userTextsOf(provider.requests[0]?.transcript ?? [])[0]).toContain(`mission branch \`${branch}\` (branched from \`main\`)`);188 expect(toolResultsOf(provider.lastTranscript())[2]?.content).toContain(branch);189 const st = await steps(db, m.runId);190 expect(st.slice(0, 3).map((s) => `${s.kind}/${s.name}`)).toEqual(['setup/run_meta', 'setup/prefix', 'setup/git_branch']);191 expect(st[2]?.result).toMatchObject({ repo: true, branch, base: 'main', error: null });192 });193194 it('a workspace that is not a repository gets no branch and says so', async () => {195 const ws = makeWorkspace(db, { files: { 'a.txt': 'a\n' } });196 const { provider } = await play([{ calls: [PARTIAL] }], ws);197 expect(userTextsOf(provider.requests[0]?.transcript ?? [])[0]).toContain('Not a git repository');198 expect(exists(ws, '.git')).toBe(false);199 });200201 it('project instructions: AGENTS.md wins over CLAUDE.md, capped at 32 KiB, plus the owner notes', async () => {202 const ws = makeWorkspace(db, { files: { 'AGENTS.md': `HEAD-MARK\n${'é'.repeat(40_000)}\nTAIL-MARK\n`, 'CLAUDE.md': 'CLAUDE-MARK\n' } });203 await db.k.pool.query(`UPDATE projects SET instructions = 'OWNER-NOTE: use pnpm' WHERE id = $1`, [db.projectId]);204 const { provider } = await play([{ calls: [PARTIAL] }], ws);205 await db.k.pool.query(`UPDATE projects SET instructions = '' WHERE id = $1`, [db.projectId]);206 const system = provider.requests[0]?.system ?? '';207 expect(system).toContain('HEAD-MARK');208 expect(system).not.toContain('TAIL-MARK');209 expect(system).not.toContain('CLAUDE-MARK');210 expect(system).not.toContain('�');211 expect(system).toContain('<project_instructions file="AGENTS.md" truncated="true">');212 expect(system).toContain('OWNER-NOTE: use pnpm');213 expect(Buffer.byteLength(system)).toBeLessThan(32 * 1024 + 9000);214 });215216 it('a worker shut down in the middle of a tool call releases the run; another worker resumes and the call runs to its end once', async () => {217 const ws = makeWorkspace(db, { files: { 'a.txt': 'a\n' } });218 const provider = scriptedProvider([{ calls: [SH('echo started >> starts.log; sleep 2; echo tick >> counter.log; cat counter.log')] }, { calls: [PARTIAL] }]);219 mission = await newMission(db, ws);220 worker = startEngine(db, provider);221 await waitFor(async () => exists(ws, 'starts.log'), 20_000, 'the shell command to start');222 await worker.stop(); // graceful shutdown: aborts the command, releases the lease, journals nothing223 const mid = await steps(db, mission.runId);224 expect(mid.find((s) => s.name === 'shell')?.status).toBe('started');225 expect((await getRun(db, mission.runId)).status).toBe('running');226 expect(exists(ws, 'counter.log')).toBe(false);227 worker = startEngine(db, provider);228 expect((await waitStatus(db, mission.runId, undefined, 60_000)).status).toBe('partial');229 expect(read(ws, 'counter.log')).toBe('tick\n');230 expect(read(ws, 'starts.log')).toBe('started\nstarted\n'); // the interrupted attempt and the full one231 const r = toolResultsOf(provider.lastTranscript())[0];232 expect(r).toMatchObject({ name: 'shell', ok: true }); // the model never sees the aborted attempt233 expect(r?.content).toBe('[exit code 0]\ntick');234 });235236 it('sweepRunTmp removes the temp dir of a run cancelled while it was waiting', async () => {237 const ws = makeWorkspace(db, { files: { 'a.txt': 'a\n' } });238 const provider = scriptedProvider([{ calls: [SH('echo t > "$TMPDIR/keep.txt"'), { name: 'ask_owner', input: { question: 'Go on?', options: null } }] }]);239 mission = await newMission(db, ws);240 worker = startEngine(db, provider);241 await pendingWait(db, mission.runId);242 await waitStatus(db, mission.runId, ['waiting']);243 expect(fs.existsSync(path.join(runTmp(mission.runId), 'keep.txt'))).toBe(true); // a suspended run keeps its temp dir244 expect(await sweepRunTmp(db.k)).not.toContain(mission.runId);245 await requestCancel(mission.missionId, db.k);246 expect(await sweepRunTmp(db.k)).toContain(mission.runId);247 expect(fs.existsSync(runTmp(mission.runId))).toBe(false);248 });249});250