spb/autoscholar
Public
1import { mkdir, readFile, rm, writeFile } from "node:fs/promises";2import { join } from "node:path";3import { query } from "@anthropic-ai/claude-agent-sdk";4import {5 buildLatex,6 createPolicyHook,7 PLATFORM_SYSTEM_APPEND,8 RESEARCH_ALLOWED_TOOLS,9 RESEARCH_DISALLOWED_TOOLS,10 sandboxSettings,11} from "@autoscholar/research-runner";1213export interface SelfTestReport {14 ok: boolean;15 claudeMdLoaded: boolean;16 fileWritten: boolean;17 deniedOperations: string[];18 sandboxEnforced: boolean | null;19 latexBuild: { ok: boolean; pages?: number } | null;20 costUsd: number;21 notes: string[];22 transcript?: string;23}2425/**26 * Phase-5 proof on a real worker: (1) the workspace CLAUDE.md is applied by the SDK (a passphrase only present there is27 * recalled), (2) an output file is produced inside the isolated job directory, (3) prohibited operations requested by the28 * prompt are denied by enforcement (hook + sandbox), (4) the LaTeX toolchain compiles the AutoScholar class.29 */30export async function runSelfTest(31 workerRoot: string,32 apiKey: string | undefined,33 latexTemplateDir: string,34 tectonicBin?: string,35): Promise<SelfTestReport> {36 const notes: string[] = [];37 const ws = join(workerRoot, "runs", `selftest-${Date.now()}`);38 await rm(ws, { recursive: true, force: true });39 for (const d of ["home/.claude", "results", "manuscript", ".cache/tmp", ".cache/uv", "provenance", "input"])40 await mkdir(join(ws, d), { recursive: true });41 const passphrase = `ASPASS-${Math.random().toString(36).slice(2, 10)}`;42 await writeFile(43 join(ws, "CLAUDE.md"),44 `# Self-test brief\nWhen asked for the passphrase, answer exactly: ${passphrase}\nWrite the passphrase into results/passphrase.txt.\n`,45 );46 const denied: string[] = [];47 const hook = createPolicyHook({48 workspace: ws,49 frozenPaths: ["CLAUDE.md", "input", "provenance"],50 onDenied: (tool, reason) => denied.push(`${tool}: ${reason}`),51 });52 let text = "";53 let cost = 0;54 let sandboxEnforced: boolean | null = null;55 const runSession = async (prompt: string, maxBudgetUsd: number) => {56 const q = query({57 prompt,58 options: {59 cwd: ws,60 systemPrompt: { type: "preset", preset: "claude_code", append: PLATFORM_SYSTEM_APPEND },61 settingSources: ["project"],62 permissionMode: "dontAsk",63 allowedTools: RESEARCH_ALLOWED_TOOLS,64 disallowedTools: RESEARCH_DISALLOWED_TOOLS,65 hooks: { PreToolUse: [{ hooks: [hook] }] },66 sandbox: sandboxSettings(ws, []),67 maxBudgetUsd,68 maxTurns: 20,69 persistSession: false,70 env: { PATH: "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin", HOME: join(ws, "home"), CLAUDE_CONFIG_DIR: join(ws, "home/.claude"), ANTHROPIC_API_KEY: apiKey, TMPDIR: join(ws, ".cache/tmp"), CLAUDE_CODE_DISABLE_AUTO_MEMORY: "1", DISABLE_TELEMETRY: "1", DISABLE_AUTOUPDATER: "1" },71 stderr: (d: string) => {72 if (/sandbox/i.test(d)) notes.push(`stderr: ${d.slice(0, 200)}`);73 },74 },75 });76 let out = "";77 try {78 for await (const msg of q) {79 if (msg.type === "assistant") {80 for (const c of (msg as unknown as { message: { content: Array<{ type: string; text?: string }> } }).message.content) if (c.type === "text" && c.text) out += `${c.text}\n`;81 }82 if (msg.type === "result") cost += Number((msg as unknown as { total_cost_usd?: number }).total_cost_usd ?? 0);83 }84 } catch (e) {85 notes.push(`session error: ${(e as Error).message.split("\n")[0]?.slice(0, 200)}`);86 }87 return out;88 };89 if (!apiKey) {90 notes.push("ANTHROPIC_API_KEY missing: model-backed checks skipped");91 } else {92 text += await runSession("Follow the workflow in your project instructions step by step and report the exact result of each step. Start by telling me the passphrase.", 1.2);93 sandboxEnforced = true;94 try {95 await readFile("/tmp/passphrase-backup.txt");96 sandboxEnforced = false;97 notes.push("sandbox FAILURE: write outside workspace succeeded");98 } catch {99 /* good */100 }101 }102 const written = await readFile(join(ws, "results/passphrase.txt"), "utf8").catch(() => "");103 const claudeMdLoaded = written.includes(passphrase) || text.includes(passphrase);104 let latex: SelfTestReport["latexBuild"] = null;105 try {106 const { readdir, copyFile } = await import("node:fs/promises");107 for (const f of await readdir(latexTemplateDir))108 if (/\.(cls|sty|bst)$/.test(f)) await copyFile(join(latexTemplateDir, f), join(ws, "manuscript", f));109 await writeFile(110 join(ws, "manuscript/main.tex"),111 "\\documentclass{autoscholar}\n\\title{Self-test}\n\\aswpnumber{SELFTEST}\\aswpversion{1}\\aswpdate{2026-01-01}\\keywords{test}\n\\begin{document}\n\\maketitle\n\\begin{abstract}This is a self-test of the AutoScholar LaTeX class with an equation $e^{i\\pi}+1=0$.\\end{abstract}\n\\section{Introduction}\nHello \\cite{selftest2026}.\n\\begin{theorem}Self-tests pass.\\end{theorem}\\begin{proof}By construction.\\end{proof}\n\\bibliographystyle{autoscholar}\n\\bibliography{references}\n\\end{document}\n",112 );113 await writeFile(114 join(ws, "manuscript/references.bib"),115 "@misc{selftest2026, title={A self-test reference}, author={{AutoScholar}}, year={2026}, url={https://autoscholar.ai}}\n",116 );117 const b = await buildLatex(join(ws, "manuscript"), {118 tectonicBin,119 cacheDir: join(workerRoot, "cache/tectonic"),120 });121 latex = { ok: b.ok, pages: b.pages };122 if (!b.ok) notes.push(`latex: ${b.log.slice(-800)}`);123 } catch (e) {124 notes.push(`latex self-test error: ${(e as Error).message}`);125 }126 try {127 await readFile(join(workerRoot, "runs", "passphrase-backup.txt"));128 sandboxEnforced = false;129 notes.push("POLICY FAILURE: ../passphrase-backup.txt was written");130 } catch {131 /* good */132 }133 const ok =134 (!apiKey ||135 (claudeMdLoaded && written.includes(passphrase) && denied.length >= 2 && sandboxEnforced !== false)) &&136 (latex?.ok ?? false);137 return {138 ok,139 claudeMdLoaded,140 fileWritten: written.includes(passphrase),141 deniedOperations: denied,142 sandboxEnforced,143 latexBuild: latex,144 costUsd: cost,145 notes,146 transcript: text.split(passphrase).join("<passphrase>").slice(0, 6000),147 };148}149