spb/autoscholar
Public
1import { spawn } from "node:child_process";2import { createHash } from "node:crypto";3import { statSync } from "node:fs";4import {5 appendFile,6 chmod,7 copyFile,8 cp,9 mkdir,10 readdir,11 readFile,12 realpath,13 rm,14 stat,15 writeFile,16} from "node:fs/promises";17import { join } from "node:path";18import { query, type SDKMessage } from "@anthropic-ai/claude-agent-sdk";19import type { ArtifactKind, RunEventInput, RunManifest, StageKey } from "@autoscholar/schemas";20import { MetricsFile } from "@autoscholar/schemas";21import { MAX_ARTIFACT_BYTES, syncArtifacts, writeChecksums } from "./artifacts.js";22import { buildLatex, listFiles, renderPages } from "./latex.js";23import {24 createPolicyHook,25 PLATFORM_SYSTEM_APPEND,26 RESEARCH_ALLOWED_TOOLS,27 RESEARCH_DISALLOWED_TOOLS,28 sandboxSettings,29} from "./policy.js";30import { STAGE_SPECS, type StageContext, type StageSpec } from "./stages.js";31import { createResearchTools } from "./tools.js";3233export interface RunnerJob {34 runId: string;35 attemptNo: number;36 manifest: RunManifest;37 instructionText: string;38 resumeFromStage: StageKey | null;39 checkpoint: Record<string, unknown> | null;40}4142export interface RunnerHost {43 workspaceRoot: string;44 apiKey: string;45 extraDomains: string[];46 tectonicBin?: string;47 tectonicCache?: string;48 latexTemplateDir: string;49 workerName: string;50 sdkVersion: string;51 toolchain: Record<string, string>;52 emit: (ev: RunEventInput) => void;53 stage: (report: {54 stage: StageKey;55 status: "running" | "completed" | "failed" | "skipped";56 costUsd: number;57 turns: number;58 usage?: Record<string, unknown>;59 error?: string;60 checkpoint?: Record<string, unknown>;61 }) => Promise<void>;62 upload: (path: string, kind: ArtifactKind, contentType: string, buf: Buffer) => Promise<void>;63 isCancelled: () => boolean;64 /** Optional: fetch the latest artifacts of this run from the control plane (resume on a different node). */65 fetchArtifacts?: () => Promise<Array<{ path: string; read: () => Promise<Buffer> }>>;66}6768export interface RunOutcome {69 outcome: "completed" | "failed" | "cancelled" | "budget_exceeded" | "timeout";70 error?: string;71 costUsd: number;72 turns: number;73 stagesCompleted: StageKey[];74 resourceUsage: { wallSeconds: number; diskBytes: number; peakRssBytes: number };75}7677interface SessionResult {78 costUsd: number;79 turns: number;80 subtype: string;81 resultText: string;82 usage: Record<string, unknown>;83 denied: number;84}8586async function dirSize(dir: string): Promise<number> {87 let total = 0;88 for (const f of await listFiles(dir)) {89 try {90 total += (await stat(join(dir, f))).size;91 } catch {92 /* ignore */93 }94 }95 return total;96}9798export class ResearchRunner {99 private readonly ws: string;100 private spent = 0;101 private turns = 0;102 private readonly started = Date.now();103 private completed: StageKey[] = [];104 private notes: string[] = [];105 private peakRss = 0;106107 constructor(108 private readonly job: RunnerJob,109 private readonly host: RunnerHost,110 ) {111 this.ws = join(host.workspaceRoot, "runs", job.runId);112 }113114 private emit(115 kind: string,116 message: string,117 data?: Record<string, unknown>,118 level: RunEventInput["level"] = "info",119 stage?: StageKey,120 ) {121 const ev: RunEventInput = {122 kind,123 message: message.slice(0, 4000),124 data,125 level,126 stage,127 at: new Date().toISOString(),128 };129 this.host.emit(ev);130 appendFile(131 join(this.ws, "provenance/events.jsonl"),132 `${JSON.stringify({ ...ev, runId: this.job.runId, attemptNo: this.job.attemptNo })}\n`,133 ).catch(() => {});134 const rss = process.memoryUsage().rss;135 if (rss > this.peakRss) this.peakRss = rss;136 }137138 private remainingBudget(): number {139 return this.job.manifest.maxBudgetUsd - this.spent;140 }141 private remainingMinutes(): number {142 return this.job.manifest.maxRuntimeMinutes - (Date.now() - this.started) / 60_000;143 }144145 /** Creates the isolated workspace: frozen CLAUDE.md as project instructions, contract directories, LaTeX class. */146 async prepareWorkspace(): Promise<{ resumed: boolean; startIndex: number }> {147 const ws = this.ws;148 let resumed = false;149 let startIndex = 0;150 const existed = await stat(ws)151 .then((s) => s.isDirectory())152 .catch(() => false);153 const cpStage = (this.job.checkpoint?.stage as StageKey | undefined) ?? this.job.resumeFromStage;154 if (existed && cpStage && STAGE_SPECS.some((s) => s.key === cpStage)) {155 const idx = STAGE_SPECS.findIndex((s) => s.key === cpStage);156 // verify the frozen instruction is intact; otherwise restart from scratch157 const frozen = await readFile(join(ws, "CLAUDE.md"), "utf8").catch(() => "");158 if (createHash("sha256").update(frozen).digest("hex") === this.job.manifest.instructionSha256) {159 resumed = true;160 startIndex = idx + 1;161 this.completed =162 (this.job.checkpoint?.completedStages as StageKey[] | undefined) ??163 STAGE_SPECS.slice(0, idx + 1).map((s) => s.key);164 this.spent = Number(this.job.checkpoint?.spentUsd ?? 0);165 }166 }167 if (!resumed) {168 if (existed) {169 await chmod(join(ws, "input"), 0o755).catch(() => {});170 for (const f of ["CLAUDE.md", "input/CLAUDE.md", "input/run.json"])171 await chmod(join(ws, f), 0o644).catch(() => {});172 await rm(ws, { recursive: true, force: true });173 }174 for (const d of [175 "input",176 "plan",177 "sources",178 "data",179 "code",180 "results/tables",181 "results/figures",182 "manuscript",183 "review",184 "provenance",185 "home/.claude",186 ".autoscholar",187 ".cache/uv",188 ".cache/tmp",189 ])190 await mkdir(join(ws, d), { recursive: true });191 await writeFile(join(ws, "CLAUDE.md"), this.job.instructionText);192 await writeFile(join(ws, "input/CLAUDE.md"), this.job.instructionText);193 await chmod(join(ws, "CLAUDE.md"), 0o444);194 await chmod(join(ws, "input/CLAUDE.md"), 0o444);195 await writeFile(196 join(ws, "input/run.json"),197 `${JSON.stringify({ ...this.job.manifest, workerName: this.host.workerName, attemptNo: this.job.attemptNo, frozenAt: new Date().toISOString() }, null, 2)}\n`,198 );199 await chmod(join(ws, "input/run.json"), 0o444);200 await chmod(join(ws, "input"), 0o555);201 await writeFile(202 join(ws, "home/.claude/settings.json"),203 JSON.stringify({ env: {}, includeCoAuthoredBy: false }, null, 2),204 );205 for (const f of await readdir(this.host.latexTemplateDir).catch(() => [] as string[])) {206 if (/\.(cls|sty|bst|mplstyle)$/.test(f))207 await copyFile(join(this.host.latexTemplateDir, f), join(ws, "manuscript", f));208 }209 await writeFile(join(ws, ".gitignore"), "home/\n.cache/\n.autoscholar/\n");210 }211 this.emit(212 "workspace.ready",213 resumed ? `resumed workspace at stage index ${startIndex}` : "fresh isolated workspace prepared",214 { workspace: ws, resumed, instructionSha256: this.job.manifest.instructionSha256 },215 );216 return { resumed, startIndex };217 }218219 private buildOptions(220 spec: StageSpec,221 maxBudgetUsd: number,222 maxTurns: number,223 abort: AbortController,224 denied: { n: number },225 ) {226 const ws = this.ws;227 const reviewer = !!spec.reviewer;228 const hook = createPolicyHook({229 workspace: ws,230 frozenPaths: ["CLAUDE.md", "input", "provenance", ".gitignore"],231 writableOnly: reviewer ? ["review/"] : undefined,232 onDenied: (tool, reason, input) => {233 denied.n++;234 this.emit("policy.denied", `${tool} denied: ${reason}`, { tool, reason, input }, "warn", spec.key);235 },236 });237 const tools = createResearchTools({238 workspace: ws,239 tectonicBin: this.host.tectonicBin,240 tectonicCache: this.host.tectonicCache,241 emit: (k, m, d) => this.emit(k, m, d, "info", spec.key),242 });243 const reviewerAppend = reviewer244 ? "\n\n# Reviewer session\nYou are the independent referee for this manuscript. You did not author it. You may only write files under review/. Be rigorous, specific and fair; blockers only for fabricated/unverifiable/invalid science."245 : "";246 return {247 cwd: ws,248 model: reviewer ? this.job.manifest.reviewerModel : this.job.manifest.model,249 systemPrompt: {250 type: "preset" as const,251 preset: "claude_code" as const,252 append: PLATFORM_SYSTEM_APPEND + reviewerAppend,253 },254 settingSources: ["project" as const],255 permissionMode: "dontAsk" as const,256 allowedTools: RESEARCH_ALLOWED_TOOLS,257 disallowedTools: RESEARCH_DISALLOWED_TOOLS,258 hooks: { PreToolUse: [{ hooks: [hook] }] },259 sandbox: sandboxSettings(ws, this.host.extraDomains),260 mcpServers: { autoscholar: tools },261 maxBudgetUsd,262 maxTurns,263 abortController: abort,264 persistSession: false,265 includePartialMessages: false,266 env: {267 PATH: "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin",268 HOME: join(ws, "home"),269 CLAUDE_CONFIG_DIR: join(ws, "home/.claude"),270 ANTHROPIC_API_KEY: this.host.apiKey,271 TMPDIR: join(ws, ".cache/tmp"),272 UV_CACHE_DIR: join(ws, ".cache/uv"),273 TECTONIC_CACHE_DIR: this.host.tectonicCache ?? join(ws, ".cache/tectonic"),274 LANG: "en_US.UTF-8",275 LC_ALL: "en_US.UTF-8",276 SOURCE_DATE_EPOCH: "1700000000",277 MPLBACKEND: "Agg",278 PYTHONDONTWRITEBYTECODE: "1",279 CLAUDE_CODE_DISABLE_AUTO_MEMORY: "1",280 DISABLE_TELEMETRY: "1",281 DISABLE_AUTOUPDATER: "1",282 CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",283 AUTOSCHOLAR_RUN_ID: this.job.runId,284 },285 stderr: (d: string) => {286 if (/error|fatal|sandbox/i.test(d))287 this.emit("sdk.stderr", d.slice(0, 500), undefined, "debug", spec.key);288 },289 };290 }291292 private async session(293 spec: StageSpec,294 prompt: string,295 limits: { maxBudgetUsd: number; maxTurns: number; timeoutMs: number },296 ): Promise<SessionResult> {297 const abort = new AbortController();298 const denied = { n: 0 };299 const timer = setTimeout(() => abort.abort(new Error("stage wall-clock limit")), limits.timeoutMs);300 const cancelPoll = setInterval(() => {301 if (this.host.isCancelled()) abort.abort(new Error("cancelled"));302 }, 5000);303 const result: SessionResult = {304 costUsd: 0,305 turns: 0,306 subtype: "unknown",307 resultText: "",308 usage: {},309 denied: 0,310 };311 const t0 = Date.now();312 try {313 const q = query({314 prompt,315 options: this.buildOptions(spec, limits.maxBudgetUsd, limits.maxTurns, abort, denied),316 });317 let toolCalls = 0;318 for await (const msg of q as AsyncIterable<SDKMessage>) {319 if (msg.type === "system" && (msg as { subtype?: string }).subtype === "init") {320 const m = msg as unknown as {321 model?: string;322 claude_code_version?: string;323 tools?: string[];324 mcp_servers?: Array<{ name: string; status: string }>;325 };326 this.emit(327 "session.init",328 `session started (${m.model ?? "model"}; claude code ${m.claude_code_version ?? "?"})`,329 { tools: m.tools?.length, mcp: m.mcp_servers },330 "debug",331 spec.key,332 );333 } else if (msg.type === "assistant") {334 const content =335 (336 msg as unknown as {337 message: {338 content: Array<{339 type: string;340 text?: string;341 name?: string;342 input?: Record<string, unknown>;343 }>;344 };345 }346 ).message.content ?? [];347 for (const c of content) {348 if (c.type === "text" && c.text?.trim())349 this.emit("assistant.text", c.text.slice(0, 600), undefined, "debug", spec.key);350 if (c.type === "tool_use") {351 toolCalls++;352 const summary =353 c.name === "Bash"354 ? String(c.input?.command ?? "").slice(0, 200)355 : c.name === "Write" || c.name === "Edit" || c.name === "Read"356 ? String(c.input?.file_path ?? "")357 : c.name?.startsWith("mcp__")358 ? JSON.stringify(c.input).slice(0, 200)359 : "";360 this.emit("tool.use", `${c.name}: ${summary}`, { tool: c.name }, "debug", spec.key);361 }362 }363 } else if (msg.type === "result") {364 const r = msg as unknown as {365 subtype: string;366 total_cost_usd?: number;367 num_turns?: number;368 result?: string;369 usage?: Record<string, unknown>;370 modelUsage?: Record<string, unknown>;371 errors?: string[];372 };373 result.subtype = r.subtype;374 result.costUsd = Number(r.total_cost_usd ?? 0);375 result.turns = Number(r.num_turns ?? 0);376 result.resultText = (r.result ?? (r.errors ?? []).join("; ")).slice(0, 4000);377 const u = (r.usage ?? {}) as Record<string, number>;378 result.usage = {379 model: spec.reviewer ? this.job.manifest.reviewerModel : this.job.manifest.model,380 inputTokens: u.input_tokens ?? 0,381 outputTokens: u.output_tokens ?? 0,382 cacheReadTokens: u.cache_read_input_tokens ?? 0,383 cacheCreateTokens: u.cache_creation_input_tokens ?? 0,384 modelUsage: r.modelUsage,385 toolCalls,386 };387 }388 }389 } catch (e) {390 result.subtype = abort.signal.aborted391 ? `aborted:${(abort.signal.reason as Error)?.message ?? "abort"}`392 : `error:${(e as Error).message.slice(0, 300)}`;393 } finally {394 clearTimeout(timer);395 clearInterval(cancelPoll);396 }397 result.denied = denied.n;398 await this.killLeftoverProcesses(spec.key);399 this.spent += result.costUsd;400 this.turns += result.turns;401 this.emit(402 "session.end",403 `${spec.key} session ${result.subtype}: ${result.turns} turns, $${result.costUsd.toFixed(3)}, ${Math.round((Date.now() - t0) / 1000)}s, ${denied.n} policy denials`,404 { subtype: result.subtype, costUsd: result.costUsd, turns: result.turns, denied: denied.n },405 "info",406 spec.key,407 );408 return result;409 }410411 /** Kills processes (owned by this user) whose command line references this workspace — leftover solvers, servers, watchers. */412 private async killLeftoverProcesses(stage: StageKey): Promise<void> {413 try {414 const out = await new Promise<string>((resolve) => {415 const ps = spawn("/bin/ps", ["-axo", "pid=,command="], { stdio: ["ignore", "pipe", "ignore"] });416 let buf = "";417 ps.stdout.on("data", (d) => {418 buf += d.toString();419 });420 ps.on("close", () => resolve(buf));421 ps.on("error", () => resolve(""));422 });423 const mine = out424 .split("\n")425 .map((l) => l.trim())426 .filter((l) => l.includes(this.ws) && !/claude-agent-sdk|dist\/main\.js/.test(l))427 .map((l) => Number(l.split(/\s+/)[0]))428 .filter((pid) => Number.isInteger(pid) && pid !== process.pid);429 for (const pid of mine) {430 try {431 process.kill(pid, "SIGTERM");432 } catch {433 /* gone */434 }435 }436 if (mine.length) {437 await new Promise((r) => setTimeout(r, 3000));438 for (const pid of mine) {439 try {440 process.kill(pid, "SIGKILL");441 } catch {442 /* gone */443 }444 }445 this.emit(446 "process.cleanup",447 `terminated ${mine.length} leftover process(es) after the ${stage} session`,448 { pids: mine.slice(0, 20) },449 "warn",450 stage,451 );452 }453 } catch {454 /* best effort */455 }456 }457458 private async stageContext(extra: Partial<StageContext> = {}): Promise<StageContext> {459 return {460 filesPresent: await listFiles(this.ws),461 previousStageNotes: this.notes.slice(-6).join("\n"),462 ...extra,463 };464 }465466 private async syncAndCheckpoint(467 stage: StageKey,468 status: "completed" | "failed",469 cost: number,470 turns: number,471 usage?: Record<string, unknown>,472 error?: string,473 ) {474 await syncArtifacts(this.ws, stage, this.host.upload, (k, m, d) => this.emit(k, m, d, "info", stage));475 if (status === "completed") this.completed.push(stage);476 await this.host.stage({477 stage,478 status,479 costUsd: cost,480 turns,481 usage,482 error,483 checkpoint:484 status === "completed"485 ? { stage, completedStages: this.completed, spentUsd: this.spent, at: new Date().toISOString() }486 : undefined,487 });488 }489490 /** Platform-side reproduction (stage 8, before the agent's revision session). */491 private async reproduce(): Promise<Record<string, unknown>> {492 const out: Record<string, unknown> = { at: new Date().toISOString() };493 const manuscript = join(this.ws, "manuscript");494 const cleanDir = join(this.ws, "_repro/manuscript");495 await rm(join(this.ws, "_repro"), { recursive: true, force: true });496 await mkdir(cleanDir, { recursive: true });497 for (const f of await readdir(manuscript))498 if (/\.(tex|bib|cls|sty|bst|pdf|png|jpg|csv)$/.test(f) && f !== "main.pdf")499 await cp(join(manuscript, f), join(cleanDir, f), { recursive: true });500 for (const d of ["results"])501 if (await stat(join(this.ws, d)).catch(() => null))502 await cp(join(this.ws, d), join(this.ws, "_repro", d), { recursive: true });503 const build = await buildLatex(cleanDir, {504 tectonicBin: this.host.tectonicBin,505 cacheDir: this.host.tectonicCache,506 });507 out.cleanPdfBuild = {508 ok: build.ok,509 pages: build.pages,510 durationMs: build.durationMs,511 logTail: build.log.slice(-1500),512 };513 if (build.ok && build.pdfPath) {514 const a = createHash("sha256")515 .update(await readFile(build.pdfPath))516 .digest("hex");517 out.cleanPdfSha256 = a;518 }519 const script = join(this.ws, "code/reproduce.sh");520 if (await stat(script).catch(() => null)) {521 const reproDir = join(this.ws, "_repro/work");522 await mkdir(reproDir, { recursive: true });523 for (const d of ["code", "data"])524 if (await stat(join(this.ws, d)).catch(() => null))525 await cp(join(this.ws, d), join(reproDir, d), {526 recursive: true,527 filter: (s) => !/\/(\.venv|node_modules|__pycache__)(\/|$)/.test(s),528 });529 await mkdir(join(reproDir, "results/tables"), { recursive: true });530 await mkdir(join(reproDir, "results/figures"), { recursive: true });531 await mkdir(join(reproDir, "home"), { recursive: true });532 // Seatbelt matches canonical paths (/private/var/...): resolve every path and allow both spellings533 const real = async (p: string) => realpath(p).catch(() => p);534 const reproReal = await real(reproDir);535 const wsReal = await real(this.ws);536 const cacheDir = this.host.tectonicCache ?? join(this.ws, ".cache/tectonic");537 const cacheReal = await real(cacheDir);538 const both = (p: string, q: string) => [...new Set([p, q])].map((x) => `(subpath "${x}")`).join(" ");539 const profile = [540 "(version 1)",541 "(allow default)",542 "(deny file-write*)",543 `(allow file-write* ${both(reproDir, reproReal)} (subpath "/private/tmp") (subpath "/tmp") (subpath "/private/var/folders") (literal "/dev/null") (regex #"^/dev/tty") ${both(cacheDir, cacheReal)} ${both(join(this.ws, ".cache"), join(wsReal, ".cache"))})`,544 `(deny file-read* (subpath "/Users") (subpath "/etc/autoscholar") (subpath "/private/etc/autoscholar") (subpath "/var/autoscholar/runs") (subpath "/private/var/autoscholar/runs"))`,545 `(allow file-read* ${both(reproDir, reproReal)} ${both(join(this.ws, ".cache"), join(wsReal, ".cache"))} ${both(cacheDir, cacheReal)})`,546 "",547 ].join("\n");548 const profilePath = join(this.ws, "_repro/repro.sb");549 await writeFile(profilePath, profile);550 const t0 = Date.now();551 const res = await new Promise<{ code: number | null; log: string }>((resolve) => {552 const child = spawn("/usr/bin/sandbox-exec", ["-f", profilePath, "/bin/bash", "code/reproduce.sh"], {553 cwd: reproDir,554 env: {555 PATH: "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin",556 HOME: join(reproDir, "home"),557 TMPDIR: "/tmp",558 UV_CACHE_DIR: join(this.ws, ".cache/uv"),559 MPLBACKEND: "Agg",560 LANG: "en_US.UTF-8",561 PYTHONDONTWRITEBYTECODE: "1",562 },563 stdio: ["ignore", "pipe", "pipe"],564 });565 let log = "";566 const cap = (d: Buffer) => {567 log += d.toString();568 if (log.length > 200_000) log = log.slice(-150_000);569 };570 child.stdout.on("data", cap);571 child.stderr.on("data", cap);572 const timer = setTimeout(() => child.kill("SIGKILL"), 90 * 60_000);573 child.on("close", (code) => {574 clearTimeout(timer);575 resolve({ code, log });576 });577 child.on("error", (e) => resolve({ code: -1, log: `spawn error: ${e.message}` }));578 });579 out.reproduceScript = {580 ran: res.code !== -1,581 exitCode: res.code,582 durationMs: Date.now() - t0,583 logTail: res.log.slice(-3000),584 };585 try {586 const orig = MetricsFile.parse(587 JSON.parse(await readFile(join(this.ws, "results/metrics.json"), "utf8")),588 );589 const regen = MetricsFile.parse(590 JSON.parse(await readFile(join(reproDir, "results/metrics.json"), "utf8")),591 );592 const compared: Array<{ key: string; original: unknown; reproduced: unknown; match: boolean }> = [];593 for (const [k, v] of Object.entries(orig.metrics)) {594 const r = regen.metrics[k];595 const match =596 typeof v === "number" && typeof r === "number"597 ? Math.abs(v - r) <= Math.max(1e-9, Math.abs(v) * 1e-6)598 : JSON.stringify(v) === JSON.stringify(r);599 compared.push({ key: k, original: v, reproduced: r, match });600 }601 out.metricsCompared = compared.length;602 out.metricsMismatched = compared.filter((c) => !c.match).slice(0, 25);603 out.metricsMissingInRerun = compared.filter((c) => c.reproduced === undefined).length;604 } catch (e) {605 out.metricsComparison = `not compared: ${(e as Error).message.slice(0, 200)}`;606 }607 } else out.reproduceScript = { ran: false, reason: "code/reproduce.sh not present" };608 await mkdir(join(this.ws, "results"), { recursive: true });609 await writeFile(join(this.ws, "results/reproduction.json"), `${JSON.stringify(out, null, 2)}\n`);610 return out;611 }612613 async run(): Promise<RunOutcome> {614 const { startIndex } = await this.prepareWorkspace();615 const finish = async (outcome: RunOutcome["outcome"], error?: string): Promise<RunOutcome> => {616 await this.finalize(outcome, error);617 return {618 outcome,619 error,620 costUsd: this.spent,621 turns: this.turns,622 stagesCompleted: this.completed,623 resourceUsage: {624 wallSeconds: (Date.now() - this.started) / 1000,625 diskBytes: await dirSize(this.ws),626 peakRssBytes: this.peakRss,627 },628 };629 };630 const total = this.job.manifest.maxBudgetUsd;631 if (startIndex >= STAGE_SPECS.length) {632 const r = await this.qaRevision();633 if (r) return finish(r.outcome, r.error);634 }635 for (let i = startIndex; i < STAGE_SPECS.length; i++) {636 const spec = STAGE_SPECS[i]!;637 if (this.host.isCancelled()) return finish("cancelled", "cancellation requested");638 if (this.remainingBudget() < Math.max(0.25, total * 0.02))639 return finish(640 "budget_exceeded",641 `budget exhausted before ${spec.key} ($${this.spent.toFixed(2)} of $${total})`,642 );643 if (this.remainingMinutes() < 2) return finish("timeout", `runtime cap reached before ${spec.key}`);644 await this.host.stage({ stage: spec.key, status: "running", costUsd: 0, turns: 0 });645 this.emit("stage.start", `stage ${i}: ${spec.key}`, undefined, "info", spec.key);646 const stageBudget = Math.min(this.remainingBudget() * 0.9, Math.max(1, total * spec.budgetShare * 1.6));647 const stageMinutes = Math.min(648 this.remainingMinutes() * 0.9,649 Math.max(5, this.job.manifest.maxRuntimeMinutes * spec.wallMinutesShare * 1.8),650 );651 let ctx: StageContext;652 if (spec.key === "reproduction") {653 this.emit(654 "reproduction.start",655 "platform re-executing analysis and clean LaTeX build",656 undefined,657 "info",658 spec.key,659 );660 const repro = await this.reproduce();661 const pdf = join(this.ws, "manuscript/main.pdf");662 let previews: string[] = [];663 if (await stat(pdf).catch(() => null)) {664 const pages = Number((repro.cleanPdfBuild as { pages?: number })?.pages ?? 0) || 4;665 const picks = [...new Set([1, 2, Math.max(1, Math.floor(pages / 2)), pages])].filter(666 (p) => p >= 1 && p <= pages,667 );668 previews = await renderPages(pdf, join(this.ws, "review/previews"), picks);669 this.emit(670 "reproduction.previews",671 previews.length672 ? `rendered ${previews.length} page previews`673 : "page previews unavailable (pdftoppm missing)",674 { previews: previews.map((p) => p.replace(`${this.ws}/`, "")) },675 "info",676 spec.key,677 );678 }679 ctx = await this.stageContext({680 reproduction: repro,681 previewFiles: previews.map((p) => p.replace(`${this.ws}/`, "")),682 });683 } else ctx = await this.stageContext();684 let res = await this.session(spec, spec.prompt(this.job.manifest, ctx), {685 maxBudgetUsd: stageBudget,686 maxTurns: spec.maxTurns,687 timeoutMs: stageMinutes * 60_000,688 });689 if (res.subtype.startsWith("aborted:cancelled"))690 return finish("cancelled", `cancelled during ${spec.key}`);691 if (res.subtype === "error_max_budget_usd" && this.remainingBudget() < 1) {692 await this.syncAndCheckpoint(693 spec.key,694 "failed",695 res.costUsd,696 res.turns,697 res.usage,698 "budget exhausted",699 );700 return finish("budget_exceeded", `budget exhausted during ${spec.key}`);701 }702 // manuscript stage: the platform guarantees a PDF build attempt703 if (spec.key === "manuscript" || spec.key === "reproduction") {704 const b = await buildLatex(join(this.ws, "manuscript"), {705 tectonicBin: this.host.tectonicBin,706 cacheDir: this.host.tectonicCache,707 });708 this.emit(709 "latex.build",710 b.ok ? `platform build ok: ${b.pages} pages` : "platform build failed",711 { ok: b.ok, pages: b.pages, log: b.log.slice(-1200) },712 b.ok ? "info" : "warn",713 spec.key,714 );715 if (!b.ok && this.remainingBudget() > 1 && !this.host.isCancelled()) {716 const fix = await this.session(717 spec,718 `${spec.prompt(this.job.manifest, await this.stageContext())}\n\n# BUILD FAILED — fix it\nThe platform's clean tectonic build of manuscript/main.tex failed. Log:\n\`\`\`\n${b.log.slice(-6000)}\n\`\`\`\nFix the LaTeX (or bibliography) and call latex_build until it succeeds. Do not remove scientific content to make it compile.`,719 { maxBudgetUsd: Math.min(this.remainingBudget() * 0.5, 4), maxTurns: 60, timeoutMs: 20 * 60_000 },720 );721 res = {722 ...res,723 costUsd: res.costUsd + fix.costUsd,724 turns: res.turns + fix.turns,725 denied: res.denied + fix.denied,726 };727 const b2 = await buildLatex(join(this.ws, "manuscript"), {728 tectonicBin: this.host.tectonicBin,729 cacheDir: this.host.tectonicCache,730 });731 this.emit(732 "latex.build",733 b2.ok ? `rebuild ok: ${b2.pages} pages` : "rebuild failed",734 { ok: b2.ok, pages: b2.pages },735 b2.ok ? "info" : "error",736 spec.key,737 );738 if (!b2.ok) {739 await this.syncAndCheckpoint(740 spec.key,741 "failed",742 res.costUsd,743 res.turns,744 res.usage,745 "LaTeX build failed after fix attempt",746 );747 return finish("failed", `LaTeX build failed in ${spec.key}`);748 }749 } else if (!b.ok) {750 await this.syncAndCheckpoint(751 spec.key,752 "failed",753 res.costUsd,754 res.turns,755 res.usage,756 "LaTeX build failed",757 );758 return finish("failed", `LaTeX build failed in ${spec.key}`);759 }760 }761 let problems = await spec.check(this.ws, this.job.manifest);762 if (problems.some((p) => p.startsWith("INFEASIBLE"))) {763 await this.syncAndCheckpoint(764 spec.key,765 "failed",766 res.costUsd,767 res.turns,768 res.usage,769 problems.join("; "),770 );771 return finish(772 "failed",773 problems.find((p) => p.startsWith("INFEASIBLE")),774 );775 }776 if (problems.length && this.remainingBudget() > 1 && !this.host.isCancelled()) {777 this.emit(778 "stage.check",779 `post-conditions not met, running a fix session: ${problems.join("; ")}`,780 { problems },781 "warn",782 spec.key,783 );784 const fix = await this.session(785 spec,786 `${spec.prompt(this.job.manifest, await this.stageContext())}\n\n# INCOMPLETE — the deterministic post-conditions of this stage failed:\n- ${problems.join("\n- ")}\nComplete exactly what is missing (do not redo finished work).`,787 {788 maxBudgetUsd: Math.min(this.remainingBudget() * 0.5, Math.max(1, stageBudget * 0.5)),789 maxTurns: Math.ceil(spec.maxTurns / 2),790 timeoutMs: Math.max(5, stageMinutes * 0.5) * 60_000,791 },792 );793 res = {794 ...res,795 costUsd: res.costUsd + fix.costUsd,796 turns: res.turns + fix.turns,797 denied: res.denied + fix.denied,798 };799 if (fix.subtype.startsWith("aborted:cancelled"))800 return finish("cancelled", `cancelled during ${spec.key}`);801 problems = await spec.check(this.ws, this.job.manifest);802 }803 if (problems.length) {804 await this.syncAndCheckpoint(805 spec.key,806 "failed",807 res.costUsd,808 res.turns,809 res.usage,810 problems.join("; "),811 );812 return finish("failed", `${spec.key}: ${problems.join("; ")}`);813 }814 this.notes.push(`[${spec.key}] ${res.resultText.replace(/\s+/g, " ").slice(0, 400)}`);815 await this.syncAndCheckpoint(spec.key, "completed", res.costUsd, res.turns, res.usage);816 }817 return finish("completed");818 }819820 /**821 * Retry after `quality_failed`: fetch the server's QA report, run one revision session that addresses the hard822 * failures (same rules as the reproduction stage), rebuild, and let finalize/QA re-run. Returns an outcome only on failure.823 */824 private async qaRevision(): Promise<{ outcome: RunOutcome["outcome"]; error?: string } | null> {825 const spec = STAGE_SPECS.find((s) => s.key === "reproduction")!;826 let report: { hardFailures?: string[]; warnings?: string[] } | null = null;827 try {828 const list = (await this.host.fetchArtifacts?.()) ?? [];829 const qa = list.find((a) => a.path === "review/qa-report.json");830 if (qa) {831 const buf = await qa.read();832 await mkdir(join(this.ws, "review"), { recursive: true });833 await writeFile(join(this.ws, "review/qa-report.json"), buf);834 report = JSON.parse(buf.toString("utf8"));835 }836 } catch (e) {837 this.emit(838 "qa.revision",839 `could not fetch the QA report: ${(e as Error).message}`,840 undefined,841 "warn",842 "reproduction",843 );844 }845 const hard = report?.hardFailures ?? [];846 if (hard.length === 0) {847 this.emit(848 "qa.revision",849 "no QA report with hard failures; re-finalizing only",850 undefined,851 "info",852 "reproduction",853 );854 return null;855 }856 if (this.remainingBudget() < 2)857 return { outcome: "budget_exceeded", error: "no budget left for the QA revision" };858 await this.host.stage({ stage: "reproduction", status: "running", costUsd: 0, turns: 0 });859 this.emit(860 "qa.revision",861 `revision pass for ${hard.length} failed hard gate(s)`,862 { hardFailures: hard },863 "info",864 "reproduction",865 );866 const prompt = `${spec.prompt(this.job.manifest, await this.stageContext())}867868# QUALITY GATES FAILED — revision pass869The platform's deterministic publication gates rejected the previous attempt. Fix the causes in the artifacts (never by weakening claims you cannot support; downgrade or remove unsupported claims instead), then rebuild with latex_build. Failed hard gates:870- ${hard.join("\n- ")}871Soft warnings: ${(report?.warnings ?? []).join(" | ") || "none"}.872Contract reminders: results/metrics.json must be {"metrics": {...}, "reportedValues": [...]} (register every reported number with record_reported_value); results/experiment-log.jsonl entries need "name", "command", "startedAt", "endedAt", "exitCode"; code/reproduce.sh must finish within 60 minutes (put long certificate re-checks behind an optional flag and say so in the paper); review/review-1.json must keep every finding with a resolution.`;873 const res = await this.session(spec, prompt, {874 maxBudgetUsd: Math.min(this.remainingBudget() * 0.6, 20),875 maxTurns: 120,876 timeoutMs: 60 * 60_000,877 });878 if (res.subtype.startsWith("aborted:cancelled"))879 return { outcome: "cancelled", error: "cancelled during QA revision" };880 const b = await buildLatex(join(this.ws, "manuscript"), {881 tectonicBin: this.host.tectonicBin,882 cacheDir: this.host.tectonicCache,883 });884 this.emit(885 "latex.build",886 b.ok ? `revision build ok: ${b.pages} pages` : "revision build failed",887 { ok: b.ok, pages: b.pages },888 b.ok ? "info" : "error",889 "reproduction",890 );891 const problems = await spec.check(this.ws, this.job.manifest);892 await this.syncAndCheckpoint(893 "reproduction",894 problems.length || !b.ok ? "failed" : "completed",895 res.costUsd,896 res.turns,897 res.usage,898 problems.join("; ") || (b.ok ? undefined : "build failed"),899 );900 if (!b.ok) return { outcome: "failed", error: "LaTeX build failed after QA revision" };901 if (problems.length) return { outcome: "failed", error: `QA revision: ${problems.join("; ")}` };902 return null;903 }904905 private async finalize(outcome: RunOutcome["outcome"], error?: string) {906 const summary = {907 runId: this.job.runId,908 attemptNo: this.job.attemptNo,909 researchMode: this.job.manifest.researchMode,910 stagesCompleted: this.completed,911 costUsd: Number(this.spent.toFixed(4)),912 turns: this.turns,913 model: this.job.manifest.model,914 reviewerModel: this.job.manifest.reviewerModel,915 workerName: this.host.workerName,916 startedAt: new Date(this.started).toISOString(),917 endedAt: new Date().toISOString(),918 sdkVersion: this.host.sdkVersion,919 toolchain: this.host.toolchain,920 synthetic: false,921 outcome,922 error: error ?? null,923 };924 const step = async (name: string, fn: () => Promise<unknown>) => {925 try {926 await fn();927 } catch (e) {928 this.emit(929 "run.finalize_error",930 `${name}: ${(e as Error).message}`.slice(0, 1000),931 undefined,932 "error",933 );934 }935 };936 await step("cleanup-processes", () => this.killLeftoverProcesses("reproduction"));937 await step("summary", async () => {938 await mkdir(join(this.ws, "provenance"), { recursive: true });939 await writeFile(join(this.ws, "provenance/run-summary.json"), `${JSON.stringify(summary, null, 2)}\n`);940 });941 await step("cleanup", () => rm(join(this.ws, "_repro"), { recursive: true, force: true }));942 // sync first, then derive checksums.sha256 from the digests that were actually uploaded, then upload it943 await step("sync", () =>944 syncArtifacts(this.ws, "reproduction", this.host.upload, (k, m, d) => this.emit(k, m, d)),945 );946 await step("checksums", async () => {947 const text = await writeChecksums(this.ws, join(this.ws, ".autoscholar/uploaded.json"));948 await this.host.upload(949 "provenance/checksums.sha256",950 "checksum",951 "text/plain; charset=utf-8",952 Buffer.from(text),953 );954 });955 this.emit(956 "run.finalized",957 `run ${outcome}${error ? `: ${error}` : ""} — $${this.spent.toFixed(3)}, ${this.turns} turns, ${this.completed.length} stages`,958 { outcome, costUsd: this.spent, turns: this.turns },959 outcome === "completed" ? "info" : "warn",960 );961 }962}963964export async function runResearch(job: RunnerJob, host: RunnerHost): Promise<RunOutcome> {965 return new ResearchRunner(job, host).run();966}967