import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { query } from "@anthropic-ai/claude-agent-sdk"; import { buildLatex, createPolicyHook, PLATFORM_SYSTEM_APPEND, RESEARCH_ALLOWED_TOOLS, RESEARCH_DISALLOWED_TOOLS, sandboxSettings, } from "@autoscholar/research-runner"; export interface SelfTestReport { ok: boolean; claudeMdLoaded: boolean; fileWritten: boolean; deniedOperations: string[]; sandboxEnforced: boolean | null; latexBuild: { ok: boolean; pages?: number } | null; costUsd: number; notes: string[]; transcript?: string; } /** * Phase-5 proof on a real worker: (1) the workspace CLAUDE.md is applied by the SDK (a passphrase only present there is * recalled), (2) an output file is produced inside the isolated job directory, (3) prohibited operations requested by the * prompt are denied by enforcement (hook + sandbox), (4) the LaTeX toolchain compiles the AutoScholar class. */ export async function runSelfTest( workerRoot: string, apiKey: string | undefined, latexTemplateDir: string, tectonicBin?: string, ): Promise { const notes: string[] = []; const ws = join(workerRoot, "runs", `selftest-${Date.now()}`); await rm(ws, { recursive: true, force: true }); for (const d of ["home/.claude", "results", "manuscript", ".cache/tmp", ".cache/uv", "provenance", "input"]) await mkdir(join(ws, d), { recursive: true }); const passphrase = `ASPASS-${Math.random().toString(36).slice(2, 10)}`; await writeFile( join(ws, "CLAUDE.md"), `# Self-test brief\nWhen asked for the passphrase, answer exactly: ${passphrase}\nWrite the passphrase into results/passphrase.txt.\n`, ); const denied: string[] = []; const hook = createPolicyHook({ workspace: ws, frozenPaths: ["CLAUDE.md", "input", "provenance"], onDenied: (tool, reason) => denied.push(`${tool}: ${reason}`), }); let text = ""; let cost = 0; let sandboxEnforced: boolean | null = null; const runSession = async (prompt: string, maxBudgetUsd: number) => { const q = query({ prompt, options: { cwd: ws, systemPrompt: { type: "preset", preset: "claude_code", append: PLATFORM_SYSTEM_APPEND }, settingSources: ["project"], permissionMode: "dontAsk", allowedTools: RESEARCH_ALLOWED_TOOLS, disallowedTools: RESEARCH_DISALLOWED_TOOLS, hooks: { PreToolUse: [{ hooks: [hook] }] }, sandbox: sandboxSettings(ws, []), maxBudgetUsd, maxTurns: 20, persistSession: false, env: { PATH: "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin", HOME: join(ws, "home"), CLAUDE_CONFIG_DIR: join(ws, "home/.claude"), ANTHROPIC_API_KEY: apiKey, TMPDIR: join(ws, ".cache/tmp"), CLAUDE_CODE_DISABLE_AUTO_MEMORY: "1", DISABLE_TELEMETRY: "1", DISABLE_AUTOUPDATER: "1" }, stderr: (d: string) => { if (/sandbox/i.test(d)) notes.push(`stderr: ${d.slice(0, 200)}`); }, }, }); let out = ""; try { for await (const msg of q) { if (msg.type === "assistant") { for (const c of (msg as unknown as { message: { content: Array<{ type: string; text?: string }> } }).message.content) if (c.type === "text" && c.text) out += `${c.text}\n`; } if (msg.type === "result") cost += Number((msg as unknown as { total_cost_usd?: number }).total_cost_usd ?? 0); } } catch (e) { notes.push(`session error: ${(e as Error).message.split("\n")[0]?.slice(0, 200)}`); } return out; }; if (!apiKey) { notes.push("ANTHROPIC_API_KEY missing: model-backed checks skipped"); } else { text += await runSession("Follow the workflow in your project instructions step by step and report the exact result of each step. Start by telling me the passphrase.", 1.2); sandboxEnforced = true; try { await readFile("/tmp/passphrase-backup.txt"); sandboxEnforced = false; notes.push("sandbox FAILURE: write outside workspace succeeded"); } catch { /* good */ } } const written = await readFile(join(ws, "results/passphrase.txt"), "utf8").catch(() => ""); const claudeMdLoaded = written.includes(passphrase) || text.includes(passphrase); let latex: SelfTestReport["latexBuild"] = null; try { const { readdir, copyFile } = await import("node:fs/promises"); for (const f of await readdir(latexTemplateDir)) if (/\.(cls|sty|bst)$/.test(f)) await copyFile(join(latexTemplateDir, f), join(ws, "manuscript", f)); await writeFile( join(ws, "manuscript/main.tex"), "\\documentclass{autoscholar}\n\\title{Self-test}\n\\aswpnumber{SELFTEST}\\aswpversion{1}\\aswpdate{2026-01-01}\\keywords{test}\n\\begin{document}\n\\maketitle\n\\begin{abstract}This is a self-test of the AutoScholar LaTeX class with an equation $e^{i\\pi}+1=0$.\\end{abstract}\n\\section{Introduction}\nHello \\cite{selftest2026}.\n\\begin{theorem}Self-tests pass.\\end{theorem}\\begin{proof}By construction.\\end{proof}\n\\bibliographystyle{autoscholar}\n\\bibliography{references}\n\\end{document}\n", ); await writeFile( join(ws, "manuscript/references.bib"), "@misc{selftest2026, title={A self-test reference}, author={{AutoScholar}}, year={2026}, url={https://autoscholar.ai}}\n", ); const b = await buildLatex(join(ws, "manuscript"), { tectonicBin, cacheDir: join(workerRoot, "cache/tectonic"), }); latex = { ok: b.ok, pages: b.pages }; if (!b.ok) notes.push(`latex: ${b.log.slice(-800)}`); } catch (e) { notes.push(`latex self-test error: ${(e as Error).message}`); } try { await readFile(join(workerRoot, "runs", "passphrase-backup.txt")); sandboxEnforced = false; notes.push("POLICY FAILURE: ../passphrase-backup.txt was written"); } catch { /* good */ } const ok = (!apiKey || (claudeMdLoaded && written.includes(passphrase) && denied.length >= 2 && sandboxEnforced !== false)) && (latex?.ok ?? false); return { ok, claudeMdLoaded, fileWritten: written.includes(passphrase), deniedOperations: denied, sandboxEnforced, latexBuild: latex, costUsd: cost, notes, transcript: text.split(passphrase).join("").slice(0, 6000), }; }