Repository Public
1import { execFileSync } from 'node:child_process';2import { createHmac } from 'node:crypto';3import fs from 'node:fs';4import http from 'node:http';5import path from 'node:path';6import { expect, type BrowserContext, type Locator, type Page } from '@playwright/test';7import { DATA_DIR, REPO_ROOT, SCRATCH_DIR, WORKSPACE_ROOT, providerKeys } from './env';89export function totp(secret: string, t = Date.now()): string {10 const B32 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567';11 let bits = '';12 for (const c of secret.replace(/=+$/, '').toUpperCase()) bits += B32.indexOf(c).toString(2).padStart(5, '0');13 const key = Buffer.from((bits.match(/.{8}/g) ?? []).map((b) => parseInt(b, 2)));14 const counter = Buffer.alloc(8);15 counter.writeBigUInt64BE(BigInt(Math.floor(t / 30000)));16 const h = createHmac('sha1', key).update(counter).digest();17 const o = (h[h.length - 1] ?? 0) & 0xf;18 return String((h.readUInt32BE(o) & 0x7fffffff) % 1e6).padStart(6, '0');19}20/** A code is single-use: wait for the next 30 s window before generating another one. */21export async function nextTotpWindow(): Promise<void> {22 await new Promise((r) => setTimeout(r, 30_000 - (Date.now() % 30_000) + 600));23}2425/**26 * Everything the browser must never do or receive, checked over a whole context:27 * console errors, page errors, CSP violations (under the server's real CSP), and the provider28 * key values in any HTTP response body or WebSocket frame.29 */30export class Guard {31 readonly errors: string[] = [];32 readonly leaks: string[] = [];33 frames = 0;34 responses = 0;35 private readonly keys = providerKeys();36 private readonly pending = new Set<Promise<void>>();3738 constructor(context: BrowserContext) {39 void context.addInitScript(() => {40 document.addEventListener('securitypolicyviolation', (e) => console.error(`CSP violation: ${e.effectiveDirective} ${e.blockedURI}`));41 // records that streamed assistant text was on screen at some point (it is replaced by the committed step)42 const w = window as unknown as { __sawStream?: boolean };43 new MutationObserver(() => {44 if (!w.__sawStream && document.querySelector('[data-testid="assistant-text"][data-streaming]')) w.__sawStream = true;45 }).observe(document, { subtree: true, childList: true, attributes: true, attributeFilter: ['data-streaming'] });46 });47 context.on('console', (m) => {48 if (m.type() !== 'error') return;49 const t = m.text();50 // expected: 401 on the first /auth calls before sign-in; the events socket dropped when the page navigates51 if (/Failed to load resource: the server responded with a status of 401/.test(t)) return;52 this.errors.push(t);53 });54 context.on('weberror', (e) => this.errors.push(`pageerror: ${e.error().message}`));55 context.on('response', (r) => {56 const type = r.headers()['content-type'] ?? '';57 if (!/json|text|javascript|xml|octet-stream/.test(type)) return;58 const p = r.text().then(59 (body) => {60 this.responses++;61 if (this.keys.some((k) => body.includes(k))) this.leaks.push(`response ${r.url()}`);62 },63 () => undefined, // redirects / aborted navigations have no body64 );65 this.pending.add(p);66 void p.finally(() => this.pending.delete(p));67 });68 context.on('page', (page) => {69 page.on('websocket', (ws) => {70 ws.on('framereceived', (f) => {71 this.frames++;72 const s = typeof f.payload === 'string' ? f.payload : f.payload.toString('utf8');73 if (this.keys.some((k) => s.includes(k))) this.leaks.push(`websocket frame ${ws.url()}`);74 });75 });76 });77 }78 async expectClean(): Promise<void> {79 await Promise.all([...this.pending]);80 expect(this.errors, 'console errors / CSP violations').toEqual([]);81 expect(this.leaks, 'provider key value sent to the browser').toEqual([]);82 }83}8485const git = (cwd: string, ...args: string[]): string =>86 execFileSync('git', ['-c', 'user.name=E2E Seed', '-c', 'user.email=seed@example.invalid', '-c', 'commit.gpgsign=false', ...args], { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();87export const gitIn = git;8889export const BUGGY_SUM = '// Adds every number of the list.\nexport function sum(list) {\n let total = 0;\n for (let i = 1; i < list.length; i++) total += list[i];\n return total;\n}\n';9091/** Seeds a project directory (created by the server through the UI) with a tiny module, a failing test and an initial commit. */92export function seedBugProject(relPath: string, opts: { remote?: boolean } = {}): { dir: string; remoteDir: string | null } {93 const dir = path.join(WORKSPACE_ROOT, relPath);94 expect(fs.existsSync(path.join(dir, '.git')), `project directory ${dir} created by the server`).toBe(true);95 fs.mkdirSync(path.join(dir, 'test'), { recursive: true });96 fs.writeFileSync(path.join(dir, 'package.json'), `${JSON.stringify({ name: 'calc', version: '1.0.0', type: 'module', scripts: { test: 'node --test' } }, null, 2)}\n`);97 fs.writeFileSync(path.join(dir, 'sum.js'), BUGGY_SUM);98 fs.writeFileSync(99 path.join(dir, 'test', 'sum.test.js'),100 "import test from 'node:test';\nimport assert from 'node:assert/strict';\nimport { sum } from '../sum.js';\n\ntest('sum adds every element', () => {\n assert.equal(sum([1, 2, 3]), 6);\n assert.equal(sum([5]), 5);\n});\n",101 );102 git(dir, 'add', '-A');103 git(dir, 'commit', '-q', '-m', 'initial: sum with a failing test');104 let remoteDir: string | null = null;105 if (opts.remote) {106 remoteDir = path.join(SCRATCH_DIR, `${relPath.replace(/[^a-z0-9-]/gi, '_')}.git`);107 fs.mkdirSync(SCRATCH_DIR, { recursive: true });108 execFileSync('git', ['init', '-q', '--bare', remoteDir], { stdio: 'pipe' });109 git(dir, 'remote', 'add', 'origin', remoteDir);110 }111 return { dir, remoteDir };112}113/** `node --test` in a workspace: true when it exits 0. */114export function testsPass(dir: string): boolean {115 try {116 execFileSync('node', ['--test'], { cwd: dir, stdio: 'pipe' });117 return true;118 } catch {119 return false;120 }121}122export const missionBranch = (missionId: string): string => `aos/${missionId.slice(0, 8)}`;123/** Subjects of the commits the mission added on its branch (not on main). */124export function missionCommits(dir: string, missionId: string): string[] {125 const out = git(dir, 'log', '--format=%s', `main..${missionBranch(missionId)}`);126 return out ? out.split('\n') : [];127}128129export const workerPid = (): number => Number(fs.readFileSync(path.join(DATA_DIR, 'run', 'worker.pid'), 'utf8').trim());130export function startWorker(): void {131 execFileSync('bash', ['scripts/dev-up.sh', '--worker'], { cwd: REPO_ROOT, stdio: ['ignore', 'inherit', 'inherit'] });132}133134export async function apiGet<T>(page: Page, p: string): Promise<T> {135 const r = await page.request.get(`/api/v1${p}`);136 expect(r.ok(), `${p} → ${r.status()}`).toBeTruthy();137 return (await r.json()) as T;138}139export interface StepLite { seq: number; kind: string; name: string; status: string; request: { input?: Record<string, unknown> } | null; result: { ok?: boolean; content?: string; diff?: string } | null }140export interface MissionLite { id: string; status: string; spentUsd: number; rootRunId: string; model: string; doneCriteria: { id: string; text: string; passes: boolean; evidence?: string }[] }141export interface RunLite { id: string; status: string; model: string; usage: { usd: number; turns: number; toolCalls: number } }142export const steps = (page: Page, runId: string): Promise<StepLite[]> => apiGet<StepLite[]>(page, `/runs/${runId}/steps?limit=500`);143144/** Creates a project through the Projects page; returns its relative path (= slug the server chose). */145export async function createProject(page: Page, name: string): Promise<{ id: string; path: string }> {146 await page.goto('/projects/');147 await page.getByTestId('project-new').click();148 await page.getByTestId('project-name').fill(name);149 const created = page.waitForResponse((r) => r.url().endsWith('/api/v1/projects') && r.request().method() === 'POST');150 await page.getByTestId('project-create').click();151 const res = await created;152 expect(res.status(), 'POST /projects').toBe(201);153 const p = (await res.json()) as { id: string; path: string };154 await expect(page.getByTestId('project-card').filter({ hasText: name })).toBeVisible();155 return p;156}157158/** Creates a mission through the UI (Projects → "new mission" of that project); lands on the mission page. */159export async function createMission(page: Page, o: { project: string; objective: string; criteria: string[]; model: string; budget?: string }): Promise<string> {160 await page.goto('/projects/');161 await page.getByTestId('project-card').filter({ hasText: o.project }).getByTestId('project-mission').click();162 const form = page.getByTestId('new-mission-form');163 await expect(form).toBeVisible();164 await page.getByTestId('mission-objective').fill(o.objective);165 for (let i = 0; i < o.criteria.length; i++) {166 await page.getByTestId('criterion-add').click();167 await page.getByTestId('criterion-input').nth(i).fill(o.criteria[i] as string);168 }169 const model = page.getByTestId(`model-${o.model}`);170 await model.scrollIntoViewIfNeeded();171 await expect(model, `model ${o.model} is available on this key`).toBeEnabled();172 await model.click();173 await page.getByTestId('mission-budget').fill(o.budget ?? '0.5');174 await page.getByTestId('mission-submit').click();175 await expect(page).toHaveURL(/\/mission\/\?id=/);176 const id = new URL(page.url()).searchParams.get('id');177 expect(id).toBeTruthy();178 await expect(page.getByTestId('mission-page')).toHaveAttribute('data-mission-id', id as string);179 return id as string;180}181182export const missionStatus = (page: Page) => page.getByTestId('mission-page');183export const toolRow = (page: Page, tool: string): Locator => page.locator(`[data-testid="tool-row"][data-tool="${tool}"]`);184export const rowBySeq = (page: Page, seq: number): Locator => page.locator(`[data-testid="tool-row"][data-seq="${seq}"]`);185186/** The timeline is virtualised: scroll from the top, a screen at a time, until the row exists; then bring it into view. */187export async function reveal(page: Page, row: Locator): Promise<void> {188 const scroller = page.getByTestId('timeline-scroll');189 await scroller.evaluate((el) => { el.scrollTop = 0; });190 for (let i = 0; i < 120; i++) {191 if ((await row.count()) > 0) break;192 const moved = await scroller.evaluate((el) => { const before = el.scrollTop; el.scrollTop += el.clientHeight * 0.6; return el.scrollTop > before; });193 await page.waitForTimeout(80);194 if (!moved) break;195 }196 await expect(row.first()).toBeVisible();197 await row.first().scrollIntoViewIfNeeded();198}199/**200 * Every row key of the timeline, collected while scrolling through it (virtualised list). Fails if201 * the DOM ever holds the same key twice (a duplicated row).202 */203export async function allRowKeys(page: Page): Promise<string[]> {204 const scroller = page.getByTestId('timeline-scroll');205 const seen = new Set<string>();206 await scroller.evaluate((el) => { el.scrollTop = 0; });207 for (let i = 0; i < 200; i++) {208 await page.waitForTimeout(60);209 const keys = await page.locator('[data-testid="timeline"] [data-key]').evaluateAll((els) => els.map((e) => e.getAttribute('data-key') ?? ''));210 expect(new Set(keys).size, 'a timeline row is rendered twice').toBe(keys.length);211 for (const k of keys) seen.add(k);212 const moved = await scroller.evaluate((el) => { const before = el.scrollTop; el.scrollTop += el.clientHeight * 0.5; return el.scrollTop > before; });213 if (!moved) break;214 }215 return [...seen];216}217218export interface Outcome { scenario: string; model: string; status: string; turns: number; toolCalls: number; usd: number }219const REPORT = path.join(__dirname, 'last-run.json');220/** Appends the outcome of a mission to the run report (model, turns, cost) and returns the running total. */221export async function record(page: Page, scenario: string, missionId: string): Promise<Outcome> {222 const m = await apiGet<MissionLite>(page, `/missions/${missionId}`);223 const run = (await apiGet<RunLite[]>(page, `/missions/${missionId}/runs`))[0] as RunLite;224 const out: Outcome = { scenario, model: run.model, status: m.status, turns: run.usage.turns, toolCalls: run.usage.toolCalls, usd: Number(m.spentUsd.toFixed(6)) };225 let all: Outcome[] = [];226 try {227 all = JSON.parse(fs.readFileSync(REPORT, 'utf8')) as Outcome[];228 } catch {229 /* first record */230 }231 all = [...all.filter((x) => x.scenario !== scenario), out];232 fs.mkdirSync(path.dirname(REPORT), { recursive: true });233 fs.writeFileSync(REPORT, `${JSON.stringify(all, null, 2)}\n`);234 console.log(`[real-e2e] ${scenario}: ${out.model} → ${out.status}, ${out.turns} turns, ${out.toolCalls} tool calls, $${out.usd}`);235 return out;236}237export function totalSpend(): number {238 try {239 return (JSON.parse(fs.readFileSync(REPORT, 'utf8')) as Outcome[]).reduce((n, x) => n + x.usd, 0);240 } catch {241 return 0;242 }243}244245/** HTTP status of a WebSocket upgrade on /api/v1/events (101 = accepted), with the given headers. */246export function wsUpgradeStatus(headers: Record<string, string>): Promise<number> {247 return new Promise((resolve, reject) => {248 const req = http.request({249 host: 'localhost', port: 8640, path: '/api/v1/events',250 headers: { Connection: 'Upgrade', Upgrade: 'websocket', 'Sec-WebSocket-Version': '13', 'Sec-WebSocket-Key': Buffer.from('aos-real-e2e-key').toString('base64'), ...headers },251 });252 req.on('upgrade', (_res, socket) => { socket.destroy(); resolve(101); });253 req.on('response', (res) => { res.resume(); resolve(res.statusCode ?? 0); });254 req.on('error', reject);255 req.end();256 });257}258