spb/autoscholar

1import { readFile } from "node:fs/promises";2import { join } from "node:path";3import { MODE_PROFILES } from "@autoscholar/research-contract";4import {5  DataManifestFile,6  FeasibilityFile,7  ManuscriptMetadata,8  MetricsFile,9  CONTRACT_PATHS as P,10  type ResearchMode,11  ReviewReport,12  type RunManifest,13  SourceMapFile,14  SourcesFile,15  type StageKey,16} from "@autoscholar/schemas";1718export interface StageSpec {19  key: StageKey;20  maxTurns: number;21  budgetShare: number; // fraction of the total budget allowed for this stage22  wallMinutesShare: number;23  prompt: (m: RunManifest, ctx: StageContext) => string;24  /** Deterministic post-conditions; returns problems (empty = ok). */25  check: (workspace: string, m: RunManifest) => Promise<string[]>;26  reviewer?: boolean;27}2829export interface StageContext {30  filesPresent: string[];31  previousStageNotes: string;32  reviewText?: string;33  buildLog?: string;34  previewFiles?: string[];35  reproduction?: Record<string, unknown>;36}3738async function exists(ws: string, rel: string): Promise<boolean> {39  try {40    await readFile(join(ws, rel));41    return true;42  } catch {43    return false;44  }45}46async function json<T>(ws: string, rel: string, parse: (x: unknown) => T): Promise<{ v?: T; err?: string }> {47  try {48    return { v: parse(JSON.parse(await readFile(join(ws, rel), "utf8"))) };49  } catch (e) {50    return { err: `${rel}: ${(e as Error).message.split("\n")[0]?.slice(0, 200)}` };51  }52}53const needsData = (m: ResearchMode) => ["empirical", "experimental", "review", "humanities"].includes(m);54const needsMetrics = (m: ResearchMode) => ["empirical", "computational", "experimental"].includes(m);5556const COMMON = (m: RunManifest, ctx: StageContext) => `57Run ${m.runId} · mode: ${m.researchMode}${m.secondaryModes.length ? ` (+${m.secondaryModes.join(", ")})` : ""} · discipline: ${m.discipline} · working title: "${m.title}".58The research brief is CLAUDE.md in this directory (already loaded as your project instructions). Files currently present (${ctx.filesPresent.length}): ${ctx.filesPresent.slice(0, 120).join(", ")}${ctx.filesPresent.length > 120 ? ", …" : ""}.59${ctx.previousStageNotes ? `Notes from previous stages:\n${ctx.previousStageNotes}\n` : ""}60Work autonomously; do not ask questions (nobody answers). Use relative paths. Keep a concise running log of decisions in plan/research-plan.md ("## Log" section, append-only).`;6162export const STAGE_SPECS: StageSpec[] = [63  {64    key: "preflight",65    maxTurns: 40,66    budgetShare: 0.04,67    wallMinutesShare: 0.05,68    prompt: (m, ctx) => `${COMMON(m, ctx)}69# Stage 0 — Preflight70Read CLAUDE.md carefully. Decide whether this study is scientifically feasible within an autonomous, sandboxed run (no human participants, no clinical/biosafety work, only public/licensed data, only allowlisted internet hosts, budget ≈ USD ${m.maxBudgetUsd}, ≈ ${m.maxRuntimeMinutes} min).71Write:721. plan/feasibility.json — JSON object: {"feasible": boolean, "researchMode": "${m.researchMode}", "secondaryModes": [...], "summary": "...", "restrictedActivities": [...], "requiredApprovals": [...], "missingInputs": [...], "risks": [...], "estimatedBudgetUsd": number, "rejectReason": "..." (only if infeasible)}. If the brief demands a regulated/restricted activity, set requiredApprovals and constrain the plan to permitted work instead of rejecting outright, unless nothing permitted remains.732. plan/research-plan.md — sections: "# Research plan", "## Question", "## Scope and constraints", "## Preliminary approach", "## Data / proof / corpus foundations", "## Evaluation plan", "## Failure modes and null-result pathways", "## Log".74Do not start the research itself.`,75    check: async (ws) => {76      const f = await json(ws, P.feasibility, (x) => FeasibilityFile.parse(x));77      const problems: string[] = [];78      if (f.err) problems.push(f.err);79      else if (!f.v!.feasible) problems.push(`INFEASIBLE: ${f.v!.rejectReason ?? f.v!.summary}`);80      if (!(await exists(ws, P.plan))) problems.push("plan/research-plan.md missing");81      return problems;82    },83  },84  {85    key: "framing",86    maxTurns: 40,87    budgetShare: 0.05,88    wallMinutesShare: 0.05,89    prompt: (m, ctx) => `${COMMON(m, ctx)}90# Stage 1 — Research framing91Refine plan/research-plan.md so that it states precisely: the research question; hypotheses or propositions (falsifiable where applicable); the novelty target and intended contribution, in qualified language; the methodology chosen for this discipline (${m.researchMode}); evaluation metrics / proof obligations / inclusion criteria defined BEFORE any analysis; competing explanations; likely failure modes and the honest null-result pathway; and the deliverables mapped to the artifact contract (plan/, sources/, data/, code/, results/, manuscript/, review/). Keep it under ~1500 words. Add "## Pre-registration" with the exact specifications/tests you commit to run first.`,92    check: async (ws) => {93      const plan = await readFile(join(ws, P.plan), "utf8").catch(() => "");94      const problems: string[] = [];95      for (const h of ["Question", "Evaluation", "Failure", "Pre-registration"])96        if (!new RegExp(`^##? .*${h}`, "mi").test(plan))97          problems.push(`research-plan.md lacks a "${h}" section`);98      return problems;99    },100  },101  {102    key: "literature",103    maxTurns: 120,104    budgetShare: 0.14,105    wallMinutesShare: 0.15,106    prompt: (m, ctx) => `${COMMON(m, ctx)}107# Stage 2 — Literature and prior-art investigation108Use scholar_search (OpenAlex, Crossref, arXiv, Semantic Scholar, PubMed as fits the discipline) and WebFetch for official archival pages. For every source you intend to cite, call scholar_add_source with the exact identifier and the title you believe it has; the tool verifies it and writes the BibTeX. Never write bib entries by hand for scholarly works. Read abstracts; read full texts (arXiv PDFs, open-access pages) when possible and record accessLevel honestly.109Target ≥ ${m.researchMode === "review" ? 25 : 10} verified sources. Then write:110- sources/source-map.json: {"claims": [{"claim": "...", "location": "intro|lit|method|results", "sourceKeys": ["key", ...], "kind": "literature"}]} — at least 6 key background claims mapped to verified keys.111- plan/research-plan.md: add "## Prior art and novelty" summarising what exists, what gap this study addresses, with qualified novelty language.112Keep sources/sources.json as the single registry (the tool maintains it; do not edit verified flags by hand).`,113    check: async (ws, m) => {114      const problems: string[] = [];115      const s = await json(ws, P.sources, (x) => SourcesFile.parse(x));116      if (s.err) problems.push(s.err);117      else {118        const verified = s.v!.sources.filter((x) => x.verified).length;119        const min = m.researchMode === "review" ? 12 : 5;120        if (verified < min) problems.push(`only ${verified} verified sources (minimum ${min})`);121      }122      const sm = await json(ws, P.sourceMap, (x) => SourceMapFile.parse(x));123      if (sm.err) problems.push(sm.err);124      else if (sm.v!.claims.length < 3) problems.push("source-map.json has fewer than 3 claims");125      if (!(await exists(ws, P.referencesBib))) problems.push("sources/references.bib missing");126      return problems;127    },128  },129  {130    key: "design",131    maxTurns: 120,132    budgetShare: 0.12,133    wallMinutesShare: 0.12,134    prompt: (m, ctx) => `${COMMON(m, ctx)}135# Stage 3 — Research design and data / proof foundations136${needsData(m.researchMode) ? `Acquire the genuine data or corpus the plan requires from legally usable sources (official statistical APIs, open repositories, public archives; use Bash with curl/python inside the sandbox; only allowlisted hosts are reachable). Store files under data/ (keep total < 200 MB; subsample if needed and say so). Write data/README.md (what, where from, how to re-download, license) and data/manifest.json: {"datasets":[{"name","source","url","license","acquiredAt","sha256","bytes","synthetic": false,"transformations":[...],"notes"}]}. Compute sha256 with shasum -a 256. If the only option is simulation, set synthetic: true and say so in the plan. ${m.researchMode === "review" ? "For a review: the 'datasets' are your screening records — export the search strings, dates, hit counts, inclusion/exclusion decisions as data/screening.csv and describe it in the manifest." : ""}` : ""}137${needsMetrics(m.researchMode) ? 'Set up code/ with an environment lock (code/pyproject.toml or code/requirements.txt pinned, created with `uv venv .venv && uv pip install ...` inside code/), a code/README.md, and a `code/reproduce.sh` that re-runs the whole analysis from raw inputs and regenerates results/metrics.json, tables and figures. Establish baselines, data splits, seeds and leakage controls explicitly in plan/research-plan.md ("## Design").' : ""}138${m.researchMode === "theoretical" || m.secondaryModes.includes("theoretical") ? "Write plan/formal-setup.md with precise definitions, assumptions, the propositions/theorems to be proved, proof strategy, and the counterexample search plan. If a proof assistant (Lean/Coq) or a CAS (sympy) check is practical, plan it in code/." : ""}139Also add to plan/research-plan.md a "## Design" section with the final protocol. Label any synthetic component as synthetic.`,140    check: async (ws, m) => {141      const problems: string[] = [];142      if (needsData(m.researchMode)) {143        const d = await json(ws, P.dataManifest, (x) => DataManifestFile.parse(x));144        if (d.err) problems.push(d.err);145        else if (d.v!.datasets.length === 0) problems.push("data/manifest.json lists no datasets");146        if (!(await exists(ws, P.dataReadme))) problems.push("data/README.md missing");147      }148      if (needsMetrics(m.researchMode) && !(await exists(ws, P.codeReadme)))149        problems.push("code/README.md missing");150      if (151        (m.researchMode === "theoretical" || m.secondaryModes.includes("theoretical")) &&152        !(await exists(ws, "plan/formal-setup.md"))153      )154        problems.push("plan/formal-setup.md missing");155      return problems;156    },157  },158  {159    key: "execute",160    maxTurns: 250,161    budgetShare: 0.2,162    wallMinutesShare: 0.25,163    prompt: (m, ctx) => `${COMMON(m, ctx)}164# Stage 4 — Execute the research165Carry out the pre-registered plan for real.166${needsMetrics(m.researchMode) ? `Run the analysis/experiments with code under code/ (python via the uv venv, R/Julia if installed). Record every execution in results/experiment-log.jsonl, one JSON object per line: {"name","command","startedAt","endedAt","exitCode","seed","environment":{"python":"3.x","packages":"..."},"outputs":[paths],"synthetic":false}. Write results/metrics.json {"metrics": {...all computed numbers...}, "reportedValues": [], "synthetic": false}. Save tables as results/tables/*.tex (booktabs) and figures as results/figures/*.pdf (vector, matplotlib with the style in manuscript/autoscholar-plots.mplstyle if present) plus a .png preview. Keep failures and negative findings in the log; never rerun specifications until something "works".` : ""}167${m.researchMode === "theoretical" || m.secondaryModes.includes("theoretical") ? "Write results/derivations.md with complete, step-by-step derivations/proofs for every proposition (LaTeX math in Markdown), and where practical verify algebra numerically or symbolically with sympy in code/ (log it in results/experiment-log.jsonl)." : ""}168${m.researchMode === "review" ? "Perform the screening: for every candidate from the searches, record the inclusion/exclusion decision with a reason in data/screening.csv; extract the data items defined in the plan into results/extraction.csv; synthesise in results/synthesis.md (tables as results/tables/*.tex)." : ""}169${m.researchMode === "humanities" ? "Carry out the textual/interpretive analysis on the primary sources in data/; write results/analysis.md with quoted evidence (with locators) for every interpretive claim, and results/metrics.json if you count anything." : ""}170Append progress to the "## Log" in plan/research-plan.md. Register every headline number you will report with record_reported_value when it is final.`,171    check: async (ws, m) => {172      const problems: string[] = [];173      if (needsMetrics(m.researchMode)) {174        const mm = await json(ws, P.metrics, (x) => MetricsFile.parse(x));175        if (mm.err) problems.push(mm.err);176        const log = await readFile(join(ws, P.experimentLog), "utf8").catch(() => "");177        if (!log.split("\n").some((l) => l.includes('"exitCode": 0') || l.includes('"exitCode":0')))178          problems.push("experiment-log.jsonl has no successful execution");179      }180      if (181        (m.researchMode === "theoretical" || m.secondaryModes.includes("theoretical")) &&182        !(await exists(ws, "results/derivations.md"))183      )184        problems.push("results/derivations.md missing");185      if (m.researchMode === "review" && !(await exists(ws, "results/synthesis.md")))186        problems.push("results/synthesis.md missing");187      if (m.researchMode === "humanities" && !(await exists(ws, "results/analysis.md")))188        problems.push("results/analysis.md missing");189      return problems;190    },191  },192  {193    key: "robustness",194    maxTurns: 120,195    budgetShare: 0.1,196    wallMinutesShare: 0.1,197    prompt: (m, ctx) => `${COMMON(m, ctx)}198# Stage 5 — Robustness and falsification199Try to break your own result. ${needsMetrics(m.researchMode) ? "Run alternative specifications, sensitivity to sample/period/hyper-parameters, out-of-sample or held-out checks, stronger baselines, placebo tests or multiple-testing corrections as applicable; add them to results/experiment-log.jsonl and results/metrics.json." : ""}${m.researchMode === "theoretical" || m.secondaryModes.includes("theoretical") ? "Search for counterexamples to each proposition (boundary cases, relaxed assumptions, numeric random tests with sympy/numpy), check every derivation step again." : ""}${m.researchMode === "review" ? "Re-run key searches with alternative strings/databases, check inter-rater style consistency of your screening by re-screening a random sample, test sensitivity of conclusions to excluding low-quality studies." : ""}200Write results/robustness.md: what was tested, what held, what weakened or overturned the contribution (be explicit), and the implications for the claims. Update reported values with record_reported_value where they change.`,201    check: async (ws) => ((await exists(ws, P.robustness)) ? [] : ["results/robustness.md missing"]),202  },203  {204    key: "manuscript",205    maxTurns: 200,206    budgetShare: 0.15,207    wallMinutesShare: 0.12,208    prompt: (m, ctx) => `${COMMON(m, ctx)}209# Stage 6 — Manuscript construction210Write the working paper in LaTeX under manuscript/ using the provided class: \\documentclass{autoscholar} (manuscript/autoscholar.cls is already there; read its header comments for the metadata commands \\aswpnumber, \\aswpversion, \\aswpdate, \\keywords, \\jelcodes (optional), abstract environment, theorem environments, \\asdisclosure is automatic). Use natbib with \\bibliographystyle{autoscholar} (provided) and \\bibliography{references} — manuscript/references.bib is the platform-generated file; cite ONLY keys present in it.211Structure: title, abstract (150–300 words), introduction/motivation, related work, theory/methodology, data (if any), results, robustness, discussion incl. competing explanations, limitations, conclusion, references, appendices (proofs, protocols, reproducibility, extra tables). Include figures from results/figures (\\includegraphics) and tables from results/tables (\\input). Every number must come from results/ and be registered with record_reported_value. Genuine intellectual depth, no filler, no exaggerated novelty, honest about weaknesses.212Also write: manuscript/abstract.md (plain-text abstract) and manuscript/metadata.json: {"title","abstract","keywords":[4-8],"discipline":"${m.discipline}","researchMode":"${m.researchMode}","secondaryModes":[...],"contributions":[2-5 strings],"methods":"...","mainResults":"...","limitations":"...","dataAvailability":"...","codeAvailability":"...","ethics":"...","synthetic": boolean (true if any synthetic data/simulation underlies results),"language":"en"}.213Then call latex_build until it succeeds with zero errors and inspect the log for overfull boxes and undefined references. Set \\aswpnumber{pending} and \\aswpversion{1}.`,214    check: async (ws) => {215      const problems: string[] = [];216      for (const f of [P.mainTex, P.manuscriptBib, P.abstract])217        if (!(await exists(ws, f))) problems.push(`${f} missing`);218      const md = await json(ws, P.metadata, (x) => ManuscriptMetadata.parse(x));219      if (md.err) problems.push(md.err);220      return problems;221    },222  },223  {224    key: "review",225    reviewer: true,226    maxTurns: 80,227    budgetShare: 0.08,228    wallMinutesShare: 0.08,229    prompt: (m, ctx) => `${COMMON(m, ctx)}230# Stage 7 — Independent critical review (you did NOT write this manuscript)231You are an independent, demanding referee. Read manuscript/main.tex, manuscript/main.pdf (Read the PDF), the plan, sources/sources.json, sources/source-map.json, data/manifest.json, results/ (metrics, logs, robustness) and code/. Check: novelty vs the verified prior art; scientific validity (assumptions, identification, statistics, proofs); evidence and traceability (do the numbers in the text match results/metrics.json? are all citations verified keys? is anything claimed without artifact support?); reproducibility (does code/reproduce.sh plausibly regenerate results? seeds? environment lock?); manuscript craftsmanship; integrity and disclosure (synthetic labelling, limitations, no overclaiming). Re-run small checks yourself with Bash if useful (read-only on manuscript/).232Write review/review-1.md (narrative referee report with major/minor issues and proposed tests) and review/review-1.json strictly matching: {"round":1,"reviewerModel":"${m.reviewerModel}","summary":"...","rubric":[{"dimension":"novelty_relevance|scientific_validity|evidence_traceability|reproducibility_robustness|manuscript_craftsmanship|integrity_disclosure","score":0-5,"evidence":"..."} ×6],"findings":[{"id":"F1","severity":"blocker|major|minor","category":"...","title":"...","detail":"...","proposedTest":"...","resolution":"open"}],"recommendation":"accept|minor_revision|major_revision|reject","limitationsOfThisReview":"..."}. Blockers are reserved for fabricated/unverifiable claims, results that contradict artifacts, invalid methods, or missing mandatory evidence. You may write only under review/.`,233    check: async (ws) => {234      const problems: string[] = [];235      if (!(await exists(ws, P.review1))) problems.push("review/review-1.md missing");236      const r = await json(ws, "review/review-1.json", (x) => ReviewReport.parse(x));237      if (r.err) problems.push(r.err);238      return problems;239    },240  },241  {242    key: "reproduction",243    maxTurns: 150,244    budgetShare: 0.1,245    wallMinutesShare: 0.08,246    prompt: (m, ctx) => `${COMMON(m, ctx)}247# Stage 8 — Revision, reproduction and build248An independent reviewer produced review/review-1.md and review/review-1.json. Address every finding: fix the manuscript, analysis or proofs where the reviewer is right; where you disagree, explain why with evidence. Then write review/revisions.md (point-by-point response) and update review/review-1.json by setting each finding's "resolution" to "resolved" or "wontfix" with a "resolutionNote" (you may not delete findings or change severities; the platform re-checks blockers).249${ctx.reproduction ? `The platform re-executed code/reproduce.sh in a clean environment and rebuilt the PDF; outcome: ${JSON.stringify(ctx.reproduction).slice(0, 1500)}. Reconcile any discrepancy (fix the script, the numbers, or document what could not be reproduced in the manuscript's reproducibility appendix).` : ""}250${ctx.previewFiles?.length ? `Page previews of the current PDF are at: ${ctx.previewFiles.join(", ")} — Read them (they are images) and fix real layout defects (overfull lines, broken floats, unreadable figures, empty pages, bad tables).` : ""}251Finally: ensure manuscript/metadata.json, abstract.md and the LaTeX title/abstract agree; make sure every reported number is registered; rebuild with latex_build until clean; verify the bibliography compiled (no [?] citations) by searching the PDF text. Also write manuscript/reproducibility.md: environment, versions, commands, seeds, data hashes, what reproduced and what did not.`,252    check: async (ws) => {253      const problems: string[] = [];254      if (!(await exists(ws, P.revisions))) problems.push("review/revisions.md missing");255      if (!(await exists(ws, P.mainPdf))) problems.push("manuscript/main.pdf missing");256      const r = await json(ws, "review/review-1.json", (x) => ReviewReport.parse(x));257      if (r.err) problems.push(r.err);258      return problems;259    },260  },261];262263export const STAGE_ORDER: StageKey[] = STAGE_SPECS.map((s) => s.key);264265export function requiredArtifactsFor(mode: ResearchMode): string[] {266  return MODE_PROFILES[mode].requiredArtifacts;267}268