SPB Git
spboucher.ai

spb/autoscholar

Public

Runner: streaming checksums with oversize skips, robust finalize, 90-min reproduction cap, QA-driven revision pass on retry, no-pause policy; QA: tolerant metrics/log shapes, vendored libraries excluded from publication instead of failing; contract: dependency dirs and binaries never public

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Oct 31, 2026) parent cbf31e6 Browse files

5 changed files +188 −41

5 changed files
Unified Split
modified services/qa/src/gates.ts +28 −3
@@ -365,7 +365,30 @@ export async function runGates(ctx: GateContext): Promise<GateResult[]> {
365 365
366 // G10 results consistency (modes with metrics) 366 // G10 results consistency (modes with metrics)
367 if (needs(ctx, "empirical", "computational", "experimental")) { 367 if (needs(ctx, "empirical", "computational", "experimental")) {
368− const metrics = await readJson(find(ctx, P.metrics), (x) => MetricsFile.parse(x)); 368+ const rvFile = await readJson(
369+ find(ctx, "results/reported-values.json"),
370+ (x) => x as { reportedValues?: unknown[] },
371+ );
372+ const metrics = await readJson(find(ctx, P.metrics), (x) => {
373+ const o = x as Record<string, unknown>;
374+ // tolerate agent-structured files: no "metrics" key → the object itself is the metrics map
375+ const shaped =
376+ typeof o.metrics === "object" && o.metrics !== null
377+ ? o
378+ : {
379+ metrics: Object.fromEntries(
380+ Object.entries(o).filter(([k]) => k !== "reportedValues" && k !== "synthetic"),
381+ ),
382+ reportedValues: o.reportedValues ?? [],
383+ synthetic: o.synthetic ?? false,
384+ };
385+ if (
386+ (!Array.isArray(shaped.reportedValues) || shaped.reportedValues.length === 0) &&
387+ Array.isArray(rvFile.value?.reportedValues)
388+ )
389+ shaped.reportedValues = rvFile.value.reportedValues;
390+ return MetricsFile.parse(shaped);
391+ });
369 if (!metrics.value) 392 if (!metrics.value)
370 out.push(fail("results.consistency", "hard", `metrics.json invalid: ${metrics.error}`)); 393 out.push(fail("results.consistency", "hard", `metrics.json invalid: ${metrics.error}`));
371 else { 394 else {
@@ -425,9 +448,11 @@ export async function runGates(ctx: GateContext): Promise<GateResult[]> {
425 const okEntries = rawEntries.filter((e) => { 448 const okEntries = rawEntries.filter((e) => {
426 const strict = ExperimentLogEntry.safeParse(e); 449 const strict = ExperimentLogEntry.safeParse(e);
427 if (strict.success) return strict.data.exitCode === 0; 450 if (strict.success) return strict.data.exitCode === 0;
428− const code = e.exitCode ?? e.exit_code ?? e.returncode ?? e.status; 451+ const code = e.exitCode ?? e.exit_code ?? e.returncode ?? e.exit;
429 if (code !== undefined) return code === 0 || code === "0" || code === "ok" || code === "success"; 452 if (code !== undefined) return code === 0 || code === "0" || code === "ok" || code === "success";
430− return !!(e.script || e.command || e.name) && !(e.error || e.failed === true); 453+ if (e.status !== undefined && typeof e.status === "string")
454+ return /^(ok|pass|success|done|verified)/i.test(e.status);
455+ return Object.keys(e).length >= 2 && !(e.error || e.failed === true || e.ok === false);
431 }).length; 456 }).length;
432 const codeFiles = ctx.artifacts.filter( 457 const codeFiles = ctx.artifacts.filter(
433 (a) => a.path.startsWith("code/") && a.path !== P.codeReadme && !/\.(md|txt)$/.test(a.path), 458 (a) => a.path.startsWith("code/") && a.path !== P.codeReadme && !/\.(md|txt)$/.test(a.path),
modified services/research-runner/src/artifacts.ts +21 −5
@@ -77,8 +77,14 @@ export interface UploadRecord {
77 } 77 }
78 78
79 export async function hashFile(p: string): Promise<{ sha256: string; bytes: number }> { 79 export async function hashFile(p: string): Promise<{ sha256: string; bytes: number }> {
80− const buf = await readFile(p); 80+ const { createReadStream } = await import("node:fs");
81− return { sha256: createHash("sha256").update(buf).digest("hex"), bytes: buf.length }; 81+ const h = createHash("sha256");
82+ let bytes = 0;
83+ for await (const chunk of createReadStream(p)) {
84+ h.update(chunk as Buffer);
85+ bytes += (chunk as Buffer).length;
86+ }
87+ return { sha256: h.digest("hex"), bytes };
82 } 88 }
83 89
84 /** Uploads every new/changed file under the workspace (excluding platform dirs). Tracks state in .autoscholar/uploaded.json. */ 90 /** Uploads every new/changed file under the workspace (excluding platform dirs). Tracks state in .autoscholar/uploaded.json. */
@@ -145,12 +151,22 @@ export async function writeChecksums(workspace: string): Promise<string> {
145 f !== "provenance/checksums.sha256", 151 f !== "provenance/checksums.sha256",
146 ); 152 );
147 const lines: string[] = []; 153 const lines: string[] = [];
154+ const oversize: string[] = [];
148 for (const f of files.sort()) { 155 for (const f of files.sort()) {
149− const { sha256 } = await hashFile(join(workspace, f)); 156+ try {
150− lines.push(`${sha256} ${f}`); 157+ const st = await stat(join(workspace, f));
158+ if (st.size > MAX_ARTIFACT_BYTES) {
159+ oversize.push(`# not archived (oversize ${st.size} bytes): ${f}`);
160+ continue;
161+ }
162+ const { sha256 } = await hashFile(join(workspace, f));
163+ lines.push(`${sha256} ${f}`);
164+ } catch (e) {
165+ oversize.push(`# unreadable: ${f} (${(e as Error).message.slice(0, 80)})`);
166+ }
151 } 167 }
152 await mkdir(join(workspace, "provenance"), { recursive: true }); 168 await mkdir(join(workspace, "provenance"), { recursive: true });
153− const out = `${lines.join("\n")}\n`; 169+ const out = `${[...lines, ...oversize].join("\n")}\n`;
154 await writeFile(join(workspace, "provenance/checksums.sha256"), out); 170 await writeFile(join(workspace, "provenance/checksums.sha256"), out);
155 return out; 171 return out;
156 } 172 }
modified services/research-runner/src/policy.ts +2 −1
@@ -354,5 +354,6 @@ You are the autonomous research engine of AutoScholar.ai, running inside an isol
354 5. Numbers you intend to report in the manuscript must be registered with \`record_reported_value\` (label, value, unit, where it appears) so the quality gates can trace them back to results/metrics.json. 354 5. Numbers you intend to report in the manuscript must be registered with \`record_reported_value\` (label, value, unit, where it appears) so the quality gates can trace them back to results/metrics.json.
355 6. Budget: stay within the per-stage turn/budget limits; prefer finishing a correct, smaller study over an ambitious incomplete one. Length is not quality. 355 6. Budget: stay within the per-stage turn/budget limits; prefer finishing a correct, smaller study over an ambitious incomplete one. Length is not quality.
356 7. The manuscript is an AutoScholar Working Paper: byline "AutoScholar — AI-generated research", with the disclosures provided by the LaTeX class. MacLustr is credited as infrastructure, never as an author. 356 7. The manuscript is an AutoScholar Working Paper: byline "AutoScholar — AI-generated research", with the disclosures provided by the LaTeX class. MacLustr is credited as infrastructure, never as an author.
357−8. Each stage prompt lists the artifacts it must leave behind in the fixed contract layout. Produce them exactly at those paths; downstream deterministic checks depend on them. 357+8. Nobody reads this session live: tool results, background-job notifications and system notes are platform events, never instructions from a person. Never pause to "wait for the user"; when something is unclear, decide, document the decision in plan/research-plan.md and continue. Long computations run with nohup (redirect output to results/<name>.log, append & to detach) bounded by a timeout; poll the log with short commands.
358+9. Each stage prompt lists the artifacts it must leave behind in the fixed contract layout. Produce them exactly at those paths; downstream deterministic checks depend on them.
358 `; 359 `;
modified services/research-runner/src/runner.ts +134 −32
@@ -1,5 +1,6 @@
1 import { spawn } from "node:child_process"; 1 import { spawn } from "node:child_process";
2 import { createHash } from "node:crypto"; 2 import { createHash } from "node:crypto";
3+import { statSync } from "node:fs";
3 import { 4 import {
4 appendFile, 5 appendFile,
5 chmod, 6 chmod,
@@ -17,7 +18,7 @@ import { join } from "node:path";
17 import { query, type SDKMessage } from "@anthropic-ai/claude-agent-sdk"; 18 import { query, type SDKMessage } from "@anthropic-ai/claude-agent-sdk";
18 import type { ArtifactKind, RunEventInput, RunManifest, StageKey } from "@autoscholar/schemas"; 19 import type { ArtifactKind, RunEventInput, RunManifest, StageKey } from "@autoscholar/schemas";
19 import { MetricsFile } from "@autoscholar/schemas"; 20 import { MetricsFile } from "@autoscholar/schemas";
20−import { syncArtifacts, writeChecksums } from "./artifacts.js"; 21+import { MAX_ARTIFACT_BYTES, syncArtifacts, writeChecksums } from "./artifacts.js";
21 import { buildLatex, listFiles, renderPages } from "./latex.js"; 22 import { buildLatex, listFiles, renderPages } from "./latex.js";
22 import { 23 import {
23 createPolicyHook, 24 createPolicyHook,
@@ -568,7 +569,7 @@ export class ResearchRunner {
568 }; 569 };
569 child.stdout.on("data", cap); 570 child.stdout.on("data", cap);
570 child.stderr.on("data", cap); 571 child.stderr.on("data", cap);
571− const timer = setTimeout(() => child.kill("SIGKILL"), 40 * 60_000); 572+ const timer = setTimeout(() => child.kill("SIGKILL"), 90 * 60_000);
572 child.on("close", (code) => { 573 child.on("close", (code) => {
573 clearTimeout(timer); 574 clearTimeout(timer);
574 resolve({ code, log }); 575 resolve({ code, log });
@@ -627,6 +628,10 @@ export class ResearchRunner {
627 }; 628 };
628 }; 629 };
629 const total = this.job.manifest.maxBudgetUsd; 630 const total = this.job.manifest.maxBudgetUsd;
631+ if (startIndex >= STAGE_SPECS.length) {
632+ const r = await this.qaRevision();
633+ if (r) return finish(r.outcome, r.error);
634+ }
630 for (let i = startIndex; i < STAGE_SPECS.length; i++) { 635 for (let i = startIndex; i < STAGE_SPECS.length; i++) {
631 const spec = STAGE_SPECS[i]!; 636 const spec = STAGE_SPECS[i]!;
632 if (this.host.isCancelled()) return finish("cancelled", "cancellation requested"); 637 if (this.host.isCancelled()) return finish("cancelled", "cancellation requested");
@@ -812,40 +817,137 @@ export class ResearchRunner {
812 return finish("completed"); 817 return finish("completed");
813 } 818 }
814 819
815− private async finalize(outcome: RunOutcome["outcome"], error?: string) { 820+ /**
821+ * Retry after `quality_failed`: fetch the server's QA report, run one revision session that addresses the hard
822+ * failures (same rules as the reproduction stage), rebuild, and let finalize/QA re-run. Returns an outcome only on failure.
823+ */
824+ private async qaRevision(): Promise<{ outcome: RunOutcome["outcome"]; error?: string } | null> {
825+ const spec = STAGE_SPECS.find((s) => s.key === "reproduction")!;
826+ let report: { hardFailures?: string[]; warnings?: string[] } | null = null;
816 try { 827 try {
817− const summary = { 828+ const list = (await this.host.fetchArtifacts?.()) ?? [];
818− runId: this.job.runId, 829+ const qa = list.find((a) => a.path === "review/qa-report.json");
819− attemptNo: this.job.attemptNo, 830+ if (qa) {
820− researchMode: this.job.manifest.researchMode, 831+ const buf = await qa.read();
821− stagesCompleted: this.completed, 832+ await mkdir(join(this.ws, "review"), { recursive: true });
822− costUsd: Number(this.spent.toFixed(4)), 833+ await writeFile(join(this.ws, "review/qa-report.json"), buf);
823− turns: this.turns, 834+ report = JSON.parse(buf.toString("utf8"));
824− model: this.job.manifest.model, 835+ }
825− reviewerModel: this.job.manifest.reviewerModel, 836+ } catch (e) {
826− workerName: this.host.workerName,
827− startedAt: new Date(this.started).toISOString(),
828− endedAt: new Date().toISOString(),
829− sdkVersion: this.host.sdkVersion,
830− toolchain: this.host.toolchain,
831− synthetic: false,
832− outcome,
833− error: error ?? null,
834− };
835− await mkdir(join(this.ws, "provenance"), { recursive: true });
836− await writeFile(join(this.ws, "provenance/run-summary.json"), `${JSON.stringify(summary, null, 2)}\n`);
837− await rm(join(this.ws, "_repro"), { recursive: true, force: true });
838− await writeChecksums(this.ws);
839− await syncArtifacts(this.ws, "reproduction", this.host.upload, (k, m, d) => this.emit(k, m, d));
840 this.emit( 837 this.emit(
841− "run.finalized", 838+ "qa.revision",
842− `run ${outcome}${error ? `: ${error}` : ""} — $${this.spent.toFixed(3)}, ${this.turns} turns, ${this.completed.length} stages`, 839+ `could not fetch the QA report: ${(e as Error).message}`,
843− { outcome, costUsd: this.spent, turns: this.turns }, 840+ undefined,
844− outcome === "completed" ? "info" : "warn", 841+ "warn",
842+ "reproduction",
845 ); 843 );
846− } catch (e) {
847− this.emit("run.finalize_error", (e as Error).message, undefined, "error");
848 } 844 }
845+ const hard = report?.hardFailures ?? [];
846+ if (hard.length === 0) {
847+ this.emit(
848+ "qa.revision",
849+ "no QA report with hard failures; re-finalizing only",
850+ undefined,
851+ "info",
852+ "reproduction",
853+ );
854+ return null;
855+ }
856+ if (this.remainingBudget() < 2)
857+ return { outcome: "budget_exceeded", error: "no budget left for the QA revision" };
858+ await this.host.stage({ stage: "reproduction", status: "running", costUsd: 0, turns: 0 });
859+ this.emit(
860+ "qa.revision",
861+ `revision pass for ${hard.length} failed hard gate(s)`,
862+ { hardFailures: hard },
863+ "info",
864+ "reproduction",
865+ );
866+ const prompt = `${spec.prompt(this.job.manifest, await this.stageContext())}
867+
868+# QUALITY GATES FAILED — revision pass
869+The platform's deterministic publication gates rejected the previous attempt. Fix the causes in the artifacts (never by weakening claims you cannot support; downgrade or remove unsupported claims instead), then rebuild with latex_build. Failed hard gates:
870+- ${hard.join("\n- ")}
871+Soft warnings: ${(report?.warnings ?? []).join(" | ") || "none"}.
872+Contract reminders: results/metrics.json must be {"metrics": {...}, "reportedValues": [...]} (register every reported number with record_reported_value); results/experiment-log.jsonl entries need "name", "command", "startedAt", "endedAt", "exitCode"; code/reproduce.sh must finish within 60 minutes (put long certificate re-checks behind an optional flag and say so in the paper); review/review-1.json must keep every finding with a resolution.`;
873+ const res = await this.session(spec, prompt, {
874+ maxBudgetUsd: Math.min(this.remainingBudget() * 0.6, 20),
875+ maxTurns: 120,
876+ timeoutMs: 60 * 60_000,
877+ });
878+ if (res.subtype.startsWith("aborted:cancelled"))
879+ return { outcome: "cancelled", error: "cancelled during QA revision" };
880+ const b = await buildLatex(join(this.ws, "manuscript"), {
881+ tectonicBin: this.host.tectonicBin,
882+ cacheDir: this.host.tectonicCache,
883+ });
884+ this.emit(
885+ "latex.build",
886+ b.ok ? `revision build ok: ${b.pages} pages` : "revision build failed",
887+ { ok: b.ok, pages: b.pages },
888+ b.ok ? "info" : "error",
889+ "reproduction",
890+ );
891+ const problems = await spec.check(this.ws, this.job.manifest);
892+ await this.syncAndCheckpoint(
893+ "reproduction",
894+ problems.length || !b.ok ? "failed" : "completed",
895+ res.costUsd,
896+ res.turns,
897+ res.usage,
898+ problems.join("; ") || (b.ok ? undefined : "build failed"),
899+ );
900+ if (!b.ok) return { outcome: "failed", error: "LaTeX build failed after QA revision" };
901+ if (problems.length) return { outcome: "failed", error: `QA revision: ${problems.join("; ")}` };
902+ return null;
903+ }
904+
905+ private async finalize(outcome: RunOutcome["outcome"], error?: string) {
906+ const summary = {
907+ runId: this.job.runId,
908+ attemptNo: this.job.attemptNo,
909+ researchMode: this.job.manifest.researchMode,
910+ stagesCompleted: this.completed,
911+ costUsd: Number(this.spent.toFixed(4)),
912+ turns: this.turns,
913+ model: this.job.manifest.model,
914+ reviewerModel: this.job.manifest.reviewerModel,
915+ workerName: this.host.workerName,
916+ startedAt: new Date(this.started).toISOString(),
917+ endedAt: new Date().toISOString(),
918+ sdkVersion: this.host.sdkVersion,
919+ toolchain: this.host.toolchain,
920+ synthetic: false,
921+ outcome,
922+ error: error ?? null,
923+ };
924+ const step = async (name: string, fn: () => Promise<unknown>) => {
925+ try {
926+ await fn();
927+ } catch (e) {
928+ this.emit(
929+ "run.finalize_error",
930+ `${name}: ${(e as Error).message}`.slice(0, 1000),
931+ undefined,
932+ "error",
933+ );
934+ }
935+ };
936+ await step("summary", async () => {
937+ await mkdir(join(this.ws, "provenance"), { recursive: true });
938+ await writeFile(join(this.ws, "provenance/run-summary.json"), `${JSON.stringify(summary, null, 2)}\n`);
939+ });
940+ await step("cleanup", () => rm(join(this.ws, "_repro"), { recursive: true, force: true }));
941+ await step("checksums", () => writeChecksums(this.ws));
942+ await step("sync", () =>
943+ syncArtifacts(this.ws, "reproduction", this.host.upload, (k, m, d) => this.emit(k, m, d)),
944+ );
945+ this.emit(
946+ "run.finalized",
947+ `run ${outcome}${error ? `: ${error}` : ""} — $${this.spent.toFixed(3)}, ${this.turns} turns, ${this.completed.length} stages`,
948+ { outcome, costUsd: this.spent, turns: this.turns },
949+ outcome === "completed" ? "info" : "warn",
950+ );
849 } 951 }
850 } 952 }
851 953
modified services/research-runner/src/tools.ts +3 −0
@@ -191,6 +191,9 @@ export function createResearchTools(ctx: ToolContext) {
191 else m.reportedValues.push(entry); 191 else m.reportedValues.push(entry);
192 if (args.metricKey && !(args.metricKey in (m.metrics ?? {}))) m.metrics[args.metricKey] = args.value; 192 if (args.metricKey && !(args.metricKey in (m.metrics ?? {}))) m.metrics[args.metricKey] = args.value;
193 await writeJsonFile(metricsPath, m); 193 await writeJsonFile(metricsPath, m);
194+ await writeJsonFile(join(ctx.workspace, "results/reported-values.json"), {
195+ reportedValues: m.reportedValues,
196+ });
194 return text( 197 return text(
195 `recorded ${args.label} = ${args.value}${args.unit ? ` ${args.unit}` : ""} (${m.reportedValues.length} values registered)`, 198 `recorded ${args.label} = ${args.value}${args.unit ? ` ${args.unit}` : ""} (${m.reportedValues.length} values registered)`,
196 ); 199 );
197 200
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere