Repository Public
1/**2 * Opt-in live tests (AOS_LIVE_TESTS=1; keys from the environment): one mission per provider on the volume3 * model. A temp git repository holds a small module with a bug and a failing unit test (Node's built-in4 * runner, nothing to install). The mission: make the test pass without changing the test.5 * The real worker loop, kernel, tools and Seatbelt sandbox run it to completion.6 *7 * AOS_LIVE_ONLY=anthropic|openai restricts the run; AOS_LIVE_MODEL_ANTHROPIC / AOS_LIVE_MODEL_OPENAI8 * override the model (worker-tier fallback). Budget: USD 0.30 per mission.9 */10import { execFileSync } from 'node:child_process';11import fs from 'node:fs';12import path from 'node:path';13import { runWorkerLoop } from '@aos/kernel';14import type { Provider } from '@aos/shared';15import { afterAll, beforeAll, describe, expect, it } from 'vitest';16import { agentLogic } from '../src/index.js';17import { createTestDb, criteriaOf, getRun, git, makeWorkspace, newMission, rows, steps, waitStatus, type TestDb } from './helpers.js';1819const LIVE = process.env.AOS_LIVE_TESTS === '1';20const ONLY = process.env.AOS_LIVE_ONLY;21const BUDGET_USD = 0.3;2223const TEST_FILE = `import test from 'node:test';24import assert from 'node:assert/strict';25import { median, mean } from '../src/stats.js';2627test('mean of a list', () => {28 assert.equal(mean([1, 2, 3, 4]), 2.5);29});3031test('median of an odd-length list', () => {32 assert.equal(median([5, 1, 3]), 3);33});3435test('median of an even-length list is the mean of the two middle values', () => {36 assert.equal(median([4, 1, 3, 2]), 2.5);37});3839test('median does not reorder its input', () => {40 const xs = [3, 1, 2];41 median(xs);42 assert.deepEqual(xs, [3, 1, 2]);43});44`;45const FILES: Record<string, string> = {46 'package.json': '{\n "name": "stats",\n "version": "1.0.0",\n "type": "module",\n "scripts": { "test": "node --test" }\n}\n',47 'README.md': '# stats\n\nTiny statistics helpers. Run the tests with `node --test` (no dependencies).\n',48 'src/stats.js': `export function mean(xs) {49 return xs.reduce((a, b) => a + b, 0) / xs.length;50}5152export function median(xs) {53 const sorted = xs.sort();54 const mid = Math.floor(sorted.length / 2);55 return sorted[mid];56}57`,58 'test/stats.test.js': TEST_FILE,59};6061const CASES: { provider: Provider; model: string; key: string }[] = [62 { provider: 'anthropic', model: process.env.AOS_LIVE_MODEL_ANTHROPIC ?? 'claude-haiku-5-5', key: 'ANTHROPIC_API_KEY' },63 { provider: 'openai', model: process.env.AOS_LIVE_MODEL_OPENAI ?? 'gpt-6-luna', key: 'OPENAI_API_KEY' },64];6566describe.skipIf(!LIVE)('live missions (real providers)', () => {67 let db: TestDb;68 beforeAll(async () => {69 db = await createTestDb();70 });71 afterAll(async () => {72 await db.cleanup();73 });7475 for (const c of CASES.filter((x) => !ONLY || ONLY === x.provider)) {76 it(`${c.provider} / ${c.model}: fixes the bug, runs the test in the sandbox, commits, flips the criterion, finishes done`, async () => {77 expect(process.env[c.key], `${c.key} must be set for the live test`).toBeTruthy();78 const ws = makeWorkspace(db, { git: true, files: FILES });79 const failing = (() => {80 try {81 execFileSync('node', ['--test'], { cwd: ws, stdio: 'pipe' });82 return false;83 } catch {84 return true;85 }86 })();87 expect(failing, 'the fixture test must fail before the mission').toBe(true);8889 const m = await newMission(db, ws, {90 provider: c.provider, model: c.model, budgetUsd: BUDGET_USD, limits: { turns: 40, wallSec: 900 },91 objective: 'The unit tests of this project fail (`node --test`). Find the bug in src/ and fix it so that every test passes. Do not change anything under test/.',92 doneCriteria: ['`node --test` exits with code 0 and every test passes, with test/stats.test.js unchanged'],93 });94 const ac = new AbortController();95 const loop = runWorkerLoop({96 kernel: db.k, signal: ac.signal, concurrency: 1, pollMs: 100, janitorMs: false, workerId: `live-${c.provider}`,97 logic: agentLogic({ kernel: db.k, workspaceRoot: db.root }), // real providers, production defaults98 });99 let run;100 try {101 run = await waitStatus(db, m.runId, undefined, 14 * 60_000);102 } finally {103 ac.abort();104 await loop;105 }106 const st = await steps(db, m.runId);107 const spent = Number((await rows<{ spent_usd: string }>(db, 'SELECT spent_usd FROM missions WHERE id = $1', [m.missionId]))[0]?.spent_usd);108 const report = {109 provider: c.provider, model: c.model, status: run.status, turns: run.usage.turns, toolCalls: run.usage.toolCalls, usd: spent,110 inputTokens: run.usage.inputTokens, outputTokens: run.usage.outputTokens, cacheReadTokens: run.usage.cacheReadTokens,111 tools: st.filter((s) => s.kind === 'tool_call').map((s) => `${s.name}${s.result?.ok === false ? '!' : ''}`).join(' '),112 summary: run.result?.summary?.slice(0, 600),113 };114 // eslint-disable-next-line no-console115 console.log(`LIVE RESULT ${JSON.stringify(report)}`);116117 expect(run.status).toBe('done');118 // the test passes when WE run it, and the test file is untouched119 execFileSync('node', ['--test'], { cwd: ws, stdio: 'pipe' });120 expect(fs.readFileSync(path.join(ws, 'test/stats.test.js'), 'utf8')).toBe(TEST_FILE);121 // a commit exists on the mission branch122 const branch = `aos/${m.missionId.slice(0, 8)}`;123 const commits = git(ws, 'log', '--format=%s', `main..${branch}`).trim().split('\n').filter(Boolean);124 expect(commits.length).toBeGreaterThanOrEqual(1);125 expect(git(ws, 'log', '-1', '--format=%B', branch)).toContain(`AOS-Mission: ${m.missionId}`);126 expect(git(ws, 'status', '--porcelain')).toBe('');127 // the criterion carries evidence; spend is recorded and within budget128 const crit = await criteriaOf(db, m.missionId);129 expect(crit).toHaveLength(1);130 expect(crit[0]?.passes).toBe(true);131 expect((crit[0]?.evidence ?? '').length).toBeGreaterThan(5);132 expect(spent).toBeGreaterThan(0);133 expect(spent).toBeLessThanOrEqual(BUDGET_USD);134 // missions.spent_usd is numeric(12,6): equal to the run's own usage up to that rounding135 expect(Math.abs(((await getRun(db, m.runId)).usage.usd ?? 0) - spent)).toBeLessThan(1e-5);136 }, 15 * 60_000);137 }138});139