spb/autoscholar
Public
1import { spawn } from "node:child_process";2import { createHash } from "node:crypto";3import {4 appendFile,5 chmod,6 copyFile,7 cp,8 mkdir,9 readdir,10 readFile,11 realpath,12 rm,13 stat,14 writeFile,15} from "node:fs/promises";16import { join } from "node:path";17import { query, type SDKMessage } from "@anthropic-ai/claude-agent-sdk";18import type { ArtifactKind, RunEventInput, RunManifest, StageKey } from "@autoscholar/schemas";19import { MetricsFile } from "@autoscholar/schemas";20import { syncArtifacts, writeChecksums } from "./artifacts.js";21import { buildLatex, listFiles, renderPages } from "./latex.js";22import {23 createPolicyHook,24 PLATFORM_SYSTEM_APPEND,25 RESEARCH_ALLOWED_TOOLS,26 RESEARCH_DISALLOWED_TOOLS,27 sandboxSettings,28} from "./policy.js";29import { STAGE_SPECS, type StageContext, type StageSpec } from "./stages.js";30import { createResearchTools } from "./tools.js";3132export interface RunnerJob {33 runId: string;34 attemptNo: number;35 manifest: RunManifest;36 instructionText: string;37 resumeFromStage: StageKey | null;38 checkpoint: Record<string, unknown> | null;39}4041export interface RunnerHost {42 workspaceRoot: string;43 apiKey: string;44 extraDomains: string[];45 tectonicBin?: string;46 tectonicCache?: string;47 latexTemplateDir: string;48 workerName: string;49 sdkVersion: string;50 toolchain: Record<string, string>;51 emit: (ev: RunEventInput) => void;52 stage: (report: {53 stage: StageKey;54 status: "running" | "completed" | "failed" | "skipped";55 costUsd: number;56 turns: number;57 usage?: Record<string, unknown>;58 error?: string;59 checkpoint?: Record<string, unknown>;60 }) => Promise<void>;61 upload: (path: string, kind: ArtifactKind, contentType: string, buf: Buffer) => Promise<void>;62 isCancelled: () => boolean;63 /** Optional: fetch the latest artifacts of this run from the control plane (resume on a different node). */64 fetchArtifacts?: () => Promise<Array<{ path: string; read: () => Promise<Buffer> }>>;65}6667export interface RunOutcome {68 outcome: "completed" | "failed" | "cancelled" | "budget_exceeded" | "timeout";69 error?: string;70 costUsd: number;71 turns: number;72 stagesCompleted: StageKey[];73 resourceUsage: { wallSeconds: number; diskBytes: number; peakRssBytes: number };74}7576interface SessionResult {77 costUsd: number;78 turns: number;79 subtype: string;80 resultText: string;81 usage: Record<string, unknown>;82 denied: number;83}8485async function dirSize(dir: string): Promise<number> {86 let total = 0;87 for (const f of await listFiles(dir)) {88 try {89 total += (await stat(join(dir, f))).size;90 } catch {91 /* ignore */92 }93 }94 return total;95}9697export class ResearchRunner {98 private readonly ws: string;99 private spent = 0;100 private turns = 0;101 private readonly started = Date.now();102 private completed: StageKey[] = [];103 private notes: string[] = [];104 private peakRss = 0;105106 constructor(107 private readonly job: RunnerJob,108 private readonly host: RunnerHost,109 ) {110 this.ws = join(host.workspaceRoot, "runs", job.runId);111 }112113 private emit(114 kind: string,115 message: string,116 data?: Record<string, unknown>,117 level: RunEventInput["level"] = "info",118 stage?: StageKey,119 ) {120 const ev: RunEventInput = {121 kind,122 message: message.slice(0, 4000),123 data,124 level,125 stage,126 at: new Date().toISOString(),127 };128 this.host.emit(ev);129 appendFile(130 join(this.ws, "provenance/events.jsonl"),131 `${JSON.stringify({ ...ev, runId: this.job.runId, attemptNo: this.job.attemptNo })}\n`,132 ).catch(() => {});133 const rss = process.memoryUsage().rss;134 if (rss > this.peakRss) this.peakRss = rss;135 }136137 private remainingBudget(): number {138 return this.job.manifest.maxBudgetUsd - this.spent;139 }140 private remainingMinutes(): number {141 return this.job.manifest.maxRuntimeMinutes - (Date.now() - this.started) / 60_000;142 }143144 /** Creates the isolated workspace: frozen CLAUDE.md as project instructions, contract directories, LaTeX class. */145 async prepareWorkspace(): Promise<{ resumed: boolean; startIndex: number }> {146 const ws = this.ws;147 let resumed = false;148 let startIndex = 0;149 const existed = await stat(ws)150 .then((s) => s.isDirectory())151 .catch(() => false);152 const cpStage = (this.job.checkpoint?.stage as StageKey | undefined) ?? this.job.resumeFromStage;153 if (existed && cpStage && STAGE_SPECS.some((s) => s.key === cpStage)) {154 const idx = STAGE_SPECS.findIndex((s) => s.key === cpStage);155 // verify the frozen instruction is intact; otherwise restart from scratch156 const frozen = await readFile(join(ws, "CLAUDE.md"), "utf8").catch(() => "");157 if (createHash("sha256").update(frozen).digest("hex") === this.job.manifest.instructionSha256) {158 resumed = true;159 startIndex = idx + 1;160 this.completed =161 (this.job.checkpoint?.completedStages as StageKey[] | undefined) ??162 STAGE_SPECS.slice(0, idx + 1).map((s) => s.key);163 this.spent = Number(this.job.checkpoint?.spentUsd ?? 0);164 }165 }166 if (!resumed) {167 if (existed) {168 await chmod(join(ws, "input"), 0o755).catch(() => {});169 for (const f of ["CLAUDE.md", "input/CLAUDE.md", "input/run.json"])170 await chmod(join(ws, f), 0o644).catch(() => {});171 await rm(ws, { recursive: true, force: true });172 }173 for (const d of [174 "input",175 "plan",176 "sources",177 "data",178 "code",179 "results/tables",180 "results/figures",181 "manuscript",182 "review",183 "provenance",184 "home/.claude",185 ".autoscholar",186 ".cache/uv",187 ".cache/tmp",188 ])189 await mkdir(join(ws, d), { recursive: true });190 await writeFile(join(ws, "CLAUDE.md"), this.job.instructionText);191 await writeFile(join(ws, "input/CLAUDE.md"), this.job.instructionText);192 await chmod(join(ws, "CLAUDE.md"), 0o444);193 await chmod(join(ws, "input/CLAUDE.md"), 0o444);194 await writeFile(195 join(ws, "input/run.json"),196 `${JSON.stringify({ ...this.job.manifest, workerName: this.host.workerName, attemptNo: this.job.attemptNo, frozenAt: new Date().toISOString() }, null, 2)}\n`,197 );198 await chmod(join(ws, "input/run.json"), 0o444);199 await chmod(join(ws, "input"), 0o555);200 await writeFile(201 join(ws, "home/.claude/settings.json"),202 JSON.stringify({ env: {}, includeCoAuthoredBy: false }, null, 2),203 );204 for (const f of await readdir(this.host.latexTemplateDir).catch(() => [] as string[])) {205 if (/\.(cls|sty|bst|mplstyle)$/.test(f))206 await copyFile(join(this.host.latexTemplateDir, f), join(ws, "manuscript", f));207 }208 await writeFile(join(ws, ".gitignore"), "home/\n.cache/\n.autoscholar/\n");209 }210 this.emit(211 "workspace.ready",212 resumed ? `resumed workspace at stage index ${startIndex}` : "fresh isolated workspace prepared",213 { workspace: ws, resumed, instructionSha256: this.job.manifest.instructionSha256 },214 );215 return { resumed, startIndex };216 }217218 private buildOptions(219 spec: StageSpec,220 maxBudgetUsd: number,221 maxTurns: number,222 abort: AbortController,223 denied: { n: number },224 ) {225 const ws = this.ws;226 const reviewer = !!spec.reviewer;227 const hook = createPolicyHook({228 workspace: ws,229 frozenPaths: ["CLAUDE.md", "input", "provenance", ".gitignore"],230 writableOnly: reviewer ? ["review/"] : undefined,231 onDenied: (tool, reason, input) => {232 denied.n++;233 this.emit("policy.denied", `${tool} denied: ${reason}`, { tool, reason, input }, "warn", spec.key);234 },235 });236 const tools = createResearchTools({237 workspace: ws,238 tectonicBin: this.host.tectonicBin,239 tectonicCache: this.host.tectonicCache,240 emit: (k, m, d) => this.emit(k, m, d, "info", spec.key),241 });242 const reviewerAppend = reviewer243 ? "\n\n# Reviewer session\nYou are the independent referee for this manuscript. You did not author it. You may only write files under review/. Be rigorous, specific and fair; blockers only for fabricated/unverifiable/invalid science."244 : "";245 return {246 cwd: ws,247 model: reviewer ? this.job.manifest.reviewerModel : this.job.manifest.model,248 systemPrompt: {249 type: "preset" as const,250 preset: "claude_code" as const,251 append: PLATFORM_SYSTEM_APPEND + reviewerAppend,252 },253 settingSources: ["project" as const],254 permissionMode: "dontAsk" as const,255 allowedTools: RESEARCH_ALLOWED_TOOLS,256 disallowedTools: RESEARCH_DISALLOWED_TOOLS,257 hooks: { PreToolUse: [{ hooks: [hook] }] },258 sandbox: sandboxSettings(ws, this.host.extraDomains),259 mcpServers: { autoscholar: tools },260 maxBudgetUsd,261 maxTurns,262 abortController: abort,263 persistSession: false,264 includePartialMessages: false,265 env: {266 PATH: "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin",267 HOME: join(ws, "home"),268 CLAUDE_CONFIG_DIR: join(ws, "home/.claude"),269 ANTHROPIC_API_KEY: this.host.apiKey,270 TMPDIR: join(ws, ".cache/tmp"),271 UV_CACHE_DIR: join(ws, ".cache/uv"),272 TECTONIC_CACHE_DIR: this.host.tectonicCache ?? join(ws, ".cache/tectonic"),273 LANG: "en_US.UTF-8",274 LC_ALL: "en_US.UTF-8",275 SOURCE_DATE_EPOCH: "1700000000",276 MPLBACKEND: "Agg",277 PYTHONDONTWRITEBYTECODE: "1",278 CLAUDE_CODE_DISABLE_AUTO_MEMORY: "1",279 DISABLE_TELEMETRY: "1",280 DISABLE_AUTOUPDATER: "1",281 CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",282 AUTOSCHOLAR_RUN_ID: this.job.runId,283 },284 stderr: (d: string) => {285 if (/error|fatal|sandbox/i.test(d))286 this.emit("sdk.stderr", d.slice(0, 500), undefined, "debug", spec.key);287 },288 };289 }290291 private async session(292 spec: StageSpec,293 prompt: string,294 limits: { maxBudgetUsd: number; maxTurns: number; timeoutMs: number },295 ): Promise<SessionResult> {296 const abort = new AbortController();297 const denied = { n: 0 };298 const timer = setTimeout(() => abort.abort(new Error("stage wall-clock limit")), limits.timeoutMs);299 const cancelPoll = setInterval(() => {300 if (this.host.isCancelled()) abort.abort(new Error("cancelled"));301 }, 5000);302 const result: SessionResult = {303 costUsd: 0,304 turns: 0,305 subtype: "unknown",306 resultText: "",307 usage: {},308 denied: 0,309 };310 const t0 = Date.now();311 try {312 const q = query({313 prompt,314 options: this.buildOptions(spec, limits.maxBudgetUsd, limits.maxTurns, abort, denied),315 });316 let toolCalls = 0;317 for await (const msg of q as AsyncIterable<SDKMessage>) {318 if (msg.type === "system" && (msg as { subtype?: string }).subtype === "init") {319 const m = msg as unknown as {320 model?: string;321 claude_code_version?: string;322 tools?: string[];323 mcp_servers?: Array<{ name: string; status: string }>;324 };325 this.emit(326 "session.init",327 `session started (${m.model ?? "model"}; claude code ${m.claude_code_version ?? "?"})`,328 { tools: m.tools?.length, mcp: m.mcp_servers },329 "debug",330 spec.key,331 );332 } else if (msg.type === "assistant") {333 const content =334 (335 msg as unknown as {336 message: {337 content: Array<{338 type: string;339 text?: string;340 name?: string;341 input?: Record<string, unknown>;342 }>;343 };344 }345 ).message.content ?? [];346 for (const c of content) {347 if (c.type === "text" && c.text?.trim())348 this.emit("assistant.text", c.text.slice(0, 600), undefined, "debug", spec.key);349 if (c.type === "tool_use") {350 toolCalls++;351 const summary =352 c.name === "Bash"353 ? String(c.input?.command ?? "").slice(0, 200)354 : c.name === "Write" || c.name === "Edit" || c.name === "Read"355 ? String(c.input?.file_path ?? "")356 : c.name?.startsWith("mcp__")357 ? JSON.stringify(c.input).slice(0, 200)358 : "";359 this.emit("tool.use", `${c.name}: ${summary}`, { tool: c.name }, "debug", spec.key);360 }361 }362 } else if (msg.type === "result") {363 const r = msg as unknown as {364 subtype: string;365 total_cost_usd?: number;366 num_turns?: number;367 result?: string;368 usage?: Record<string, unknown>;369 modelUsage?: Record<string, unknown>;370 errors?: string[];371 };372 result.subtype = r.subtype;373 result.costUsd = Number(r.total_cost_usd ?? 0);374 result.turns = Number(r.num_turns ?? 0);375 result.resultText = (r.result ?? (r.errors ?? []).join("; ")).slice(0, 4000);376 const u = (r.usage ?? {}) as Record<string, number>;377 result.usage = {378 model: spec.reviewer ? this.job.manifest.reviewerModel : this.job.manifest.model,379 inputTokens: u.input_tokens ?? 0,380 outputTokens: u.output_tokens ?? 0,381 cacheReadTokens: u.cache_read_input_tokens ?? 0,382 cacheCreateTokens: u.cache_creation_input_tokens ?? 0,383 modelUsage: r.modelUsage,384 toolCalls,385 };386 }387 }388 } catch (e) {389 result.subtype = abort.signal.aborted390 ? `aborted:${(abort.signal.reason as Error)?.message ?? "abort"}`391 : `error:${(e as Error).message.slice(0, 300)}`;392 } finally {393 clearTimeout(timer);394 clearInterval(cancelPoll);395 }396 result.denied = denied.n;397 await this.killLeftoverProcesses(spec.key);398 this.spent += result.costUsd;399 this.turns += result.turns;400 this.emit(401 "session.end",402 `${spec.key} session ${result.subtype}: ${result.turns} turns, $${result.costUsd.toFixed(3)}, ${Math.round((Date.now() - t0) / 1000)}s, ${denied.n} policy denials`,403 { subtype: result.subtype, costUsd: result.costUsd, turns: result.turns, denied: denied.n },404 "info",405 spec.key,406 );407 return result;408 }409410 /** Kills processes (owned by this user) whose command line references this workspace — leftover solvers, servers, watchers. */411 private async killLeftoverProcesses(stage: StageKey): Promise<void> {412 try {413 const out = await new Promise<string>((resolve) => {414 const ps = spawn("/bin/ps", ["-axo", "pid=,command="], { stdio: ["ignore", "pipe", "ignore"] });415 let buf = "";416 ps.stdout.on("data", (d) => {417 buf += d.toString();418 });419 ps.on("close", () => resolve(buf));420 ps.on("error", () => resolve(""));421 });422 const mine = out423 .split("\n")424 .map((l) => l.trim())425 .filter((l) => l.includes(this.ws) && !/claude-agent-sdk|dist\/main\.js/.test(l))426 .map((l) => Number(l.split(/\s+/)[0]))427 .filter((pid) => Number.isInteger(pid) && pid !== process.pid);428 for (const pid of mine) {429 try {430 process.kill(pid, "SIGTERM");431 } catch {432 /* gone */433 }434 }435 if (mine.length) {436 await new Promise((r) => setTimeout(r, 3000));437 for (const pid of mine) {438 try {439 process.kill(pid, "SIGKILL");440 } catch {441 /* gone */442 }443 }444 this.emit(445 "process.cleanup",446 `terminated ${mine.length} leftover process(es) after the ${stage} session`,447 { pids: mine.slice(0, 20) },448 "warn",449 stage,450 );451 }452 } catch {453 /* best effort */454 }455 }456457 private async stageContext(extra: Partial<StageContext> = {}): Promise<StageContext> {458 return {459 filesPresent: await listFiles(this.ws),460 previousStageNotes: this.notes.slice(-6).join("\n"),461 ...extra,462 };463 }464465 private async syncAndCheckpoint(466 stage: StageKey,467 status: "completed" | "failed",468 cost: number,469 turns: number,470 usage?: Record<string, unknown>,471 error?: string,472 ) {473 await syncArtifacts(this.ws, stage, this.host.upload, (k, m, d) => this.emit(k, m, d, "info", stage));474 if (status === "completed") this.completed.push(stage);475 await this.host.stage({476 stage,477 status,478 costUsd: cost,479 turns,480 usage,481 error,482 checkpoint:483 status === "completed"484 ? { stage, completedStages: this.completed, spentUsd: this.spent, at: new Date().toISOString() }485 : undefined,486 });487 }488489 /** Platform-side reproduction (stage 8, before the agent's revision session). */490 private async reproduce(): Promise<Record<string, unknown>> {491 const out: Record<string, unknown> = { at: new Date().toISOString() };492 const manuscript = join(this.ws, "manuscript");493 const cleanDir = join(this.ws, "_repro/manuscript");494 await rm(join(this.ws, "_repro"), { recursive: true, force: true });495 await mkdir(cleanDir, { recursive: true });496 for (const f of await readdir(manuscript))497 if (/\.(tex|bib|cls|sty|bst|pdf|png|jpg|csv)$/.test(f) && f !== "main.pdf")498 await cp(join(manuscript, f), join(cleanDir, f), { recursive: true });499 for (const d of ["results"])500 if (await stat(join(this.ws, d)).catch(() => null))501 await cp(join(this.ws, d), join(this.ws, "_repro", d), { recursive: true });502 const build = await buildLatex(cleanDir, {503 tectonicBin: this.host.tectonicBin,504 cacheDir: this.host.tectonicCache,505 });506 out.cleanPdfBuild = {507 ok: build.ok,508 pages: build.pages,509 durationMs: build.durationMs,510 logTail: build.log.slice(-1500),511 };512 if (build.ok && build.pdfPath) {513 const a = createHash("sha256")514 .update(await readFile(build.pdfPath))515 .digest("hex");516 out.cleanPdfSha256 = a;517 }518 const script = join(this.ws, "code/reproduce.sh");519 if (await stat(script).catch(() => null)) {520 const reproDir = join(this.ws, "_repro/work");521 await mkdir(reproDir, { recursive: true });522 for (const d of ["code", "data"])523 if (await stat(join(this.ws, d)).catch(() => null))524 await cp(join(this.ws, d), join(reproDir, d), {525 recursive: true,526 filter: (s) => !/\/(\.venv|node_modules|__pycache__)(\/|$)/.test(s),527 });528 await mkdir(join(reproDir, "results/tables"), { recursive: true });529 await mkdir(join(reproDir, "results/figures"), { recursive: true });530 await mkdir(join(reproDir, "home"), { recursive: true });531 // Seatbelt matches canonical paths (/private/var/...): resolve every path and allow both spellings532 const real = async (p: string) => realpath(p).catch(() => p);533 const reproReal = await real(reproDir);534 const wsReal = await real(this.ws);535 const cacheDir = this.host.tectonicCache ?? join(this.ws, ".cache/tectonic");536 const cacheReal = await real(cacheDir);537 const both = (p: string, q: string) => [...new Set([p, q])].map((x) => `(subpath "${x}")`).join(" ");538 const profile = [539 "(version 1)",540 "(allow default)",541 "(deny file-write*)",542 `(allow file-write* ${both(reproDir, reproReal)} (subpath "/private/tmp") (subpath "/tmp") (subpath "/private/var/folders") (literal "/dev/null") (regex #"^/dev/tty") ${both(cacheDir, cacheReal)} ${both(join(this.ws, ".cache"), join(wsReal, ".cache"))})`,543 `(deny file-read* (subpath "/Users") (subpath "/etc/autoscholar") (subpath "/private/etc/autoscholar") (subpath "/var/autoscholar/runs") (subpath "/private/var/autoscholar/runs"))`,544 `(allow file-read* ${both(reproDir, reproReal)} ${both(join(this.ws, ".cache"), join(wsReal, ".cache"))} ${both(cacheDir, cacheReal)})`,545 "",546 ].join("\n");547 const profilePath = join(this.ws, "_repro/repro.sb");548 await writeFile(profilePath, profile);549 const t0 = Date.now();550 const res = await new Promise<{ code: number | null; log: string }>((resolve) => {551 const child = spawn("/usr/bin/sandbox-exec", ["-f", profilePath, "/bin/bash", "code/reproduce.sh"], {552 cwd: reproDir,553 env: {554 PATH: "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin",555 HOME: join(reproDir, "home"),556 TMPDIR: "/tmp",557 UV_CACHE_DIR: join(this.ws, ".cache/uv"),558 MPLBACKEND: "Agg",559 LANG: "en_US.UTF-8",560 PYTHONDONTWRITEBYTECODE: "1",561 },562 stdio: ["ignore", "pipe", "pipe"],563 });564 let log = "";565 const cap = (d: Buffer) => {566 log += d.toString();567 if (log.length > 200_000) log = log.slice(-150_000);568 };569 child.stdout.on("data", cap);570 child.stderr.on("data", cap);571 const timer = setTimeout(() => child.kill("SIGKILL"), 40 * 60_000);572 child.on("close", (code) => {573 clearTimeout(timer);574 resolve({ code, log });575 });576 child.on("error", (e) => resolve({ code: -1, log: `spawn error: ${e.message}` }));577 });578 out.reproduceScript = {579 ran: res.code !== -1,580 exitCode: res.code,581 durationMs: Date.now() - t0,582 logTail: res.log.slice(-3000),583 };584 try {585 const orig = MetricsFile.parse(586 JSON.parse(await readFile(join(this.ws, "results/metrics.json"), "utf8")),587 );588 const regen = MetricsFile.parse(589 JSON.parse(await readFile(join(reproDir, "results/metrics.json"), "utf8")),590 );591 const compared: Array<{ key: string; original: unknown; reproduced: unknown; match: boolean }> = [];592 for (const [k, v] of Object.entries(orig.metrics)) {593 const r = regen.metrics[k];594 const match =595 typeof v === "number" && typeof r === "number"596 ? Math.abs(v - r) <= Math.max(1e-9, Math.abs(v) * 1e-6)597 : JSON.stringify(v) === JSON.stringify(r);598 compared.push({ key: k, original: v, reproduced: r, match });599 }600 out.metricsCompared = compared.length;601 out.metricsMismatched = compared.filter((c) => !c.match).slice(0, 25);602 out.metricsMissingInRerun = compared.filter((c) => c.reproduced === undefined).length;603 } catch (e) {604 out.metricsComparison = `not compared: ${(e as Error).message.slice(0, 200)}`;605 }606 } else out.reproduceScript = { ran: false, reason: "code/reproduce.sh not present" };607 await mkdir(join(this.ws, "results"), { recursive: true });608 await writeFile(join(this.ws, "results/reproduction.json"), `${JSON.stringify(out, null, 2)}\n`);609 return out;610 }611612 async run(): Promise<RunOutcome> {613 const { startIndex } = await this.prepareWorkspace();614 const finish = async (outcome: RunOutcome["outcome"], error?: string): Promise<RunOutcome> => {615 await this.finalize(outcome, error);616 return {617 outcome,618 error,619 costUsd: this.spent,620 turns: this.turns,621 stagesCompleted: this.completed,622 resourceUsage: {623 wallSeconds: (Date.now() - this.started) / 1000,624 diskBytes: await dirSize(this.ws),625 peakRssBytes: this.peakRss,626 },627 };628 };629 const total = this.job.manifest.maxBudgetUsd;630 for (let i = startIndex; i < STAGE_SPECS.length; i++) {631 const spec = STAGE_SPECS[i]!;632 if (this.host.isCancelled()) return finish("cancelled", "cancellation requested");633 if (this.remainingBudget() < Math.max(0.25, total * 0.02))634 return finish(635 "budget_exceeded",636 `budget exhausted before ${spec.key} ($${this.spent.toFixed(2)} of $${total})`,637 );638 if (this.remainingMinutes() < 2) return finish("timeout", `runtime cap reached before ${spec.key}`);639 await this.host.stage({ stage: spec.key, status: "running", costUsd: 0, turns: 0 });640 this.emit("stage.start", `stage ${i}: ${spec.key}`, undefined, "info", spec.key);641 const stageBudget = Math.min(this.remainingBudget() * 0.9, Math.max(1, total * spec.budgetShare * 1.6));642 const stageMinutes = Math.min(643 this.remainingMinutes() * 0.9,644 Math.max(5, this.job.manifest.maxRuntimeMinutes * spec.wallMinutesShare * 1.8),645 );646 let ctx: StageContext;647 if (spec.key === "reproduction") {648 this.emit(649 "reproduction.start",650 "platform re-executing analysis and clean LaTeX build",651 undefined,652 "info",653 spec.key,654 );655 const repro = await this.reproduce();656 const pdf = join(this.ws, "manuscript/main.pdf");657 let previews: string[] = [];658 if (await stat(pdf).catch(() => null)) {659 const pages = Number((repro.cleanPdfBuild as { pages?: number })?.pages ?? 0) || 4;660 const picks = [...new Set([1, 2, Math.max(1, Math.floor(pages / 2)), pages])].filter(661 (p) => p >= 1 && p <= pages,662 );663 previews = await renderPages(pdf, join(this.ws, "review/previews"), picks);664 this.emit(665 "reproduction.previews",666 previews.length667 ? `rendered ${previews.length} page previews`668 : "page previews unavailable (pdftoppm missing)",669 { previews: previews.map((p) => p.replace(`${this.ws}/`, "")) },670 "info",671 spec.key,672 );673 }674 ctx = await this.stageContext({675 reproduction: repro,676 previewFiles: previews.map((p) => p.replace(`${this.ws}/`, "")),677 });678 } else ctx = await this.stageContext();679 let res = await this.session(spec, spec.prompt(this.job.manifest, ctx), {680 maxBudgetUsd: stageBudget,681 maxTurns: spec.maxTurns,682 timeoutMs: stageMinutes * 60_000,683 });684 if (res.subtype.startsWith("aborted:cancelled"))685 return finish("cancelled", `cancelled during ${spec.key}`);686 if (res.subtype === "error_max_budget_usd" && this.remainingBudget() < 1) {687 await this.syncAndCheckpoint(688 spec.key,689 "failed",690 res.costUsd,691 res.turns,692 res.usage,693 "budget exhausted",694 );695 return finish("budget_exceeded", `budget exhausted during ${spec.key}`);696 }697 // manuscript stage: the platform guarantees a PDF build attempt698 if (spec.key === "manuscript" || spec.key === "reproduction") {699 const b = await buildLatex(join(this.ws, "manuscript"), {700 tectonicBin: this.host.tectonicBin,701 cacheDir: this.host.tectonicCache,702 });703 this.emit(704 "latex.build",705 b.ok ? `platform build ok: ${b.pages} pages` : "platform build failed",706 { ok: b.ok, pages: b.pages, log: b.log.slice(-1200) },707 b.ok ? "info" : "warn",708 spec.key,709 );710 if (!b.ok && this.remainingBudget() > 1 && !this.host.isCancelled()) {711 const fix = await this.session(712 spec,713 `${spec.prompt(this.job.manifest, await this.stageContext())}\n\n# BUILD FAILED — fix it\nThe platform's clean tectonic build of manuscript/main.tex failed. Log:\n\`\`\`\n${b.log.slice(-6000)}\n\`\`\`\nFix the LaTeX (or bibliography) and call latex_build until it succeeds. Do not remove scientific content to make it compile.`,714 { maxBudgetUsd: Math.min(this.remainingBudget() * 0.5, 4), maxTurns: 60, timeoutMs: 20 * 60_000 },715 );716 res = {717 ...res,718 costUsd: res.costUsd + fix.costUsd,719 turns: res.turns + fix.turns,720 denied: res.denied + fix.denied,721 };722 const b2 = await buildLatex(join(this.ws, "manuscript"), {723 tectonicBin: this.host.tectonicBin,724 cacheDir: this.host.tectonicCache,725 });726 this.emit(727 "latex.build",728 b2.ok ? `rebuild ok: ${b2.pages} pages` : "rebuild failed",729 { ok: b2.ok, pages: b2.pages },730 b2.ok ? "info" : "error",731 spec.key,732 );733 if (!b2.ok) {734 await this.syncAndCheckpoint(735 spec.key,736 "failed",737 res.costUsd,738 res.turns,739 res.usage,740 "LaTeX build failed after fix attempt",741 );742 return finish("failed", `LaTeX build failed in ${spec.key}`);743 }744 } else if (!b.ok) {745 await this.syncAndCheckpoint(746 spec.key,747 "failed",748 res.costUsd,749 res.turns,750 res.usage,751 "LaTeX build failed",752 );753 return finish("failed", `LaTeX build failed in ${spec.key}`);754 }755 }756 let problems = await spec.check(this.ws, this.job.manifest);757 if (problems.some((p) => p.startsWith("INFEASIBLE"))) {758 await this.syncAndCheckpoint(759 spec.key,760 "failed",761 res.costUsd,762 res.turns,763 res.usage,764 problems.join("; "),765 );766 return finish(767 "failed",768 problems.find((p) => p.startsWith("INFEASIBLE")),769 );770 }771 if (problems.length && this.remainingBudget() > 1 && !this.host.isCancelled()) {772 this.emit(773 "stage.check",774 `post-conditions not met, running a fix session: ${problems.join("; ")}`,775 { problems },776 "warn",777 spec.key,778 );779 const fix = await this.session(780 spec,781 `${spec.prompt(this.job.manifest, await this.stageContext())}\n\n# INCOMPLETE — the deterministic post-conditions of this stage failed:\n- ${problems.join("\n- ")}\nComplete exactly what is missing (do not redo finished work).`,782 {783 maxBudgetUsd: Math.min(this.remainingBudget() * 0.5, Math.max(1, stageBudget * 0.5)),784 maxTurns: Math.ceil(spec.maxTurns / 2),785 timeoutMs: Math.max(5, stageMinutes * 0.5) * 60_000,786 },787 );788 res = {789 ...res,790 costUsd: res.costUsd + fix.costUsd,791 turns: res.turns + fix.turns,792 denied: res.denied + fix.denied,793 };794 if (fix.subtype.startsWith("aborted:cancelled"))795 return finish("cancelled", `cancelled during ${spec.key}`);796 problems = await spec.check(this.ws, this.job.manifest);797 }798 if (problems.length) {799 await this.syncAndCheckpoint(800 spec.key,801 "failed",802 res.costUsd,803 res.turns,804 res.usage,805 problems.join("; "),806 );807 return finish("failed", `${spec.key}: ${problems.join("; ")}`);808 }809 this.notes.push(`[${spec.key}] ${res.resultText.replace(/\s+/g, " ").slice(0, 400)}`);810 await this.syncAndCheckpoint(spec.key, "completed", res.costUsd, res.turns, res.usage);811 }812 return finish("completed");813 }814815 private async finalize(outcome: RunOutcome["outcome"], error?: string) {816 try {817 const summary = {818 runId: this.job.runId,819 attemptNo: this.job.attemptNo,820 researchMode: this.job.manifest.researchMode,821 stagesCompleted: this.completed,822 costUsd: Number(this.spent.toFixed(4)),823 turns: this.turns,824 model: this.job.manifest.model,825 reviewerModel: this.job.manifest.reviewerModel,826 workerName: this.host.workerName,827 startedAt: new Date(this.started).toISOString(),828 endedAt: new Date().toISOString(),829 sdkVersion: this.host.sdkVersion,830 toolchain: this.host.toolchain,831 synthetic: false,832 outcome,833 error: error ?? null,834 };835 await mkdir(join(this.ws, "provenance"), { recursive: true });836 await writeFile(join(this.ws, "provenance/run-summary.json"), `${JSON.stringify(summary, null, 2)}\n`);837 await rm(join(this.ws, "_repro"), { recursive: true, force: true });838 await writeChecksums(this.ws);839 await syncArtifacts(this.ws, "reproduction", this.host.upload, (k, m, d) => this.emit(k, m, d));840 this.emit(841 "run.finalized",842 `run ${outcome}${error ? `: ${error}` : ""} — $${this.spent.toFixed(3)}, ${this.turns} turns, ${this.completed.length} stages`,843 { outcome, costUsd: this.spent, turns: this.turns },844 outcome === "completed" ? "info" : "warn",845 );846 } catch (e) {847 this.emit("run.finalize_error", (e as Error).message, undefined, "error");848 }849 }850}851852export async function runResearch(job: RunnerJob, host: RunnerHost): Promise<RunOutcome> {853 return new ResearchRunner(job, host).run();854}855