spb/autoscholar
Public
1import { CONTRACT_PATHS, checkArtifactContract, MODE_PROFILES } from "@autoscholar/research-contract";2import {3 DataManifestFile,4 ExperimentLogEntry,5 FeasibilityFile,6 type GateResult,7 ManuscriptMetadata,8 MetricsFile,9 type ResearchMode,10 ReviewReport,11 SourceMapFile,12 SourcesFile,13} from "@autoscholar/schemas";14import { citeKeysInTex, parseBib } from "./bib.js";15import { inspectPdf } from "./pdf.js";1617export interface ArtifactView {18 path: string;19 kind: string;20 sha256: string;21 bytes: number;22 synthetic: boolean;23 read: () => Promise<Buffer>;24 storedSha256: () => Promise<string | null>;25}2627export interface GateContext {28 mode: ResearchMode;29 secondaryModes: string[];30 artifacts: ArtifactView[];31 attemptOutcome: string | null;32 cancelRequested: boolean;33 workerSimulated: boolean;34 reviewFindingsOpenBlockers: number;35}3637const P = CONTRACT_PATHS;38const SECRET_PATTERNS = [39 /sk-ant-[A-Za-z0-9_-]{20,}/,40 /asw_[A-Za-z0-9_-]{20,}/,41 /-----BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY-----/,42 /AKIA[0-9A-Z]{16}/,43 /ghp_[A-Za-z0-9]{30,}/,44 /xox[bp]-[A-Za-z0-9-]{20,}/,45];46const TEXT_EXT = /\.(md|tex|bib|json|jsonl|txt|csv|py|r|jl|sh|yaml|yml|toml|cfg|ini|sha256|log)$/i;4748const pass = (49 gate: string,50 severity: "hard" | "soft",51 message: string,52 evidence: Record<string, unknown> = {},53): GateResult => ({ gate, severity, status: "pass", message, evidence });54const fail = (55 gate: string,56 severity: "hard" | "soft",57 message: string,58 evidence: Record<string, unknown> = {},59): GateResult => ({ gate, severity, status: severity === "hard" ? "fail" : "warn", message, evidence });60const skip = (gate: string, severity: "hard" | "soft", message: string): GateResult => ({61 gate,62 severity,63 status: "skip",64 message,65 evidence: {},66});6768function find(ctx: GateContext, path: string): ArtifactView | undefined {69 return ctx.artifacts.find((a) => a.path === path);70}71async function readJson<T>(72 a: ArtifactView | undefined,73 parse: (x: unknown) => T,74): Promise<{ value?: T; error?: string }> {75 if (!a) return { error: "missing" };76 try {77 return { value: parse(JSON.parse((await a.read()).toString("utf8"))) };78 } catch (e) {79 return { error: (e as Error).message.slice(0, 300) };80 }81}82function needs(ctx: GateContext, ...modes: ResearchMode[]): boolean {83 return modes.includes(ctx.mode) || ctx.secondaryModes.some((m) => modes.includes(m as ResearchMode));84}85/** Does the tex contain the reported number (tolerant to formatting: thousands separators, trailing zeros, percent)? */86export function texContainsNumber(tex: string, value: number | string): boolean {87 const s = String(value).trim();88 if (!/^-?\d/.test(s)) return tex.includes(s);89 const n = Number(s.replace(/,/g, ""));90 if (!Number.isFinite(n)) return tex.includes(s);91 const candidates = new Set<string>([s, s.replace(/,/g, ""), n.toString()]);92 for (const d of [0, 1, 2, 3, 4]) candidates.add(n.toFixed(d));93 const compact = tex.replace(/,(?=\d{3})/g, "").replace(/\\,|;|\\ |~|\{|\}|\$/g, "");94 for (const c of candidates) {95 const re = new RegExp(`(?<![\\d.])${c.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?![\\d])`);96 if (re.test(compact)) return true;97 }98 return false;99}100101export async function runGates(ctx: GateContext): Promise<GateResult[]> {102 const out: GateResult[] = [];103 const profile = MODE_PROFILES[ctx.mode];104105 // G1 final attempt106 out.push(107 ctx.attemptOutcome === "completed" && !ctx.cancelRequested108 ? pass("attempt.final", "hard", "worker attempt completed and run not cancelled")109 : fail(110 "attempt.final",111 "hard",112 `attempt outcome is ${ctx.attemptOutcome ?? "none"}${ctx.cancelRequested ? " (cancellation requested)" : ""}`,113 ),114 );115116 // G2 not synthetic / not simulated worker117 const synth = ctx.artifacts.filter((a) => a.synthetic).map((a) => a.path);118 out.push(119 ctx.workerSimulated || synth.length > 0120 ? fail(121 "worker.real",122 "hard",123 "artifacts were produced by a simulated worker or are flagged synthetic; never publishable",124 { synthetic: synth.slice(0, 20), workerSimulated: ctx.workerSimulated },125 )126 : pass("worker.real", "hard", "artifacts come from a real worker"),127 );128129 // G3 contract130 const contract = checkArtifactContract(ctx.mode, ctx.artifacts);131 out.push(132 contract.ok133 ? pass("contract.artifacts", "hard", `all ${contract.present.length} mode-required artifacts present`, {134 mode: ctx.mode,135 })136 : fail("contract.artifacts", "hard", `missing ${contract.missing.length} required artifact(s)`, {137 missing: contract.missing,138 }),139 );140141 // G4 checksums142 const mismatches: string[] = [];143 for (const a of ctx.artifacts) {144 const stored = await a.storedSha256();145 if (stored && stored !== a.sha256) mismatches.push(a.path);146 }147 let checksumFileOk = false;148 const cf = find(ctx, P.checksums);149 if (cf) {150 const lines = (await cf.read()).toString("utf8").split(/\r?\n/).filter(Boolean);151 const map = new Map(lines.map((l) => [l.slice(66).trim().replace(/^\*/, ""), l.slice(0, 64)]));152 const bad = ctx.artifacts153 .filter(154 (a) =>155 a.path !== P.checksums &&156 a.path !== P.events &&157 a.path !== P.runSummary &&158 map.has(a.path) &&159 map.get(a.path) !== a.sha256,160 )161 .map((a) => a.path);162 const covered = ctx.artifacts.filter((a) => map.has(a.path)).length;163 checksumFileOk = bad.length === 0 && covered >= Math.max(1, Math.floor(ctx.artifacts.length * 0.5));164 if (bad.length) mismatches.push(...bad.map((b) => `${b} (checksums.sha256)`));165 }166 out.push(167 mismatches.length === 0 && checksumFileOk168 ? pass("checksums.valid", "hard", "all stored digests match and checksums.sha256 agrees", {169 artifacts: ctx.artifacts.length,170 })171 : fail(172 "checksums.valid",173 "hard",174 mismatches.length175 ? `digest mismatches: ${mismatches.length}`176 : "checksums.sha256 missing or covers too few artifacts",177 { mismatches: mismatches.slice(0, 20) },178 ),179 );180181 // G5 feasibility182 const feas = await readJson(find(ctx, P.feasibility), (x) => FeasibilityFile.parse(x));183 out.push(184 feas.value?.feasible && feas.value.requiredApprovals.length === 0185 ? pass("feasibility.ok", "hard", "preflight judged the study feasible with no outstanding approvals")186 : fail(187 "feasibility.ok",188 "hard",189 feas.error190 ? `feasibility.json invalid: ${feas.error}`191 : `not feasible or approvals outstanding: ${feas.value?.requiredApprovals.join(", ") || feas.value?.rejectReason}`,192 ),193 );194195 // G6 metadata + manuscript196 const meta = await readJson(find(ctx, P.metadata), (x) => ManuscriptMetadata.parse(x));197 const texA = find(ctx, P.mainTex);198 const tex = texA ? (await texA.read()).toString("utf8") : "";199 if (!meta.value) out.push(fail("metadata.valid", "hard", `metadata.json invalid: ${meta.error}`));200 else {201 const problems: string[] = [];202 if (!tex.includes("\\begin{document}")) problems.push("main.tex has no document body");203 if (!/\\title\{/.test(tex) && !/\\Title\{/.test(tex) && !/title=\{/.test(tex))204 problems.push("main.tex has no title");205 if (meta.value.synthetic && !/synthetic/i.test(meta.value.limitations + meta.value.abstract))206 problems.push("synthetic flag set but not disclosed in abstract/limitations");207 out.push(208 problems.length209 ? fail("metadata.valid", "hard", problems.join("; "))210 : pass("metadata.valid", "hard", "manuscript metadata and LaTeX source are well-formed", {211 title: meta.value.title,212 keywords: meta.value.keywords.length,213 }),214 );215 }216217 // G7 PDF218 const pdfA = find(ctx, P.mainPdf);219 if (!pdfA) out.push(fail("pdf.valid", "hard", "manuscript/main.pdf missing"));220 else {221 const insp = await inspectPdf(await pdfA.read());222 const minPages = ctx.mode === "theoretical" ? 4 : 6;223 const problems = [...insp.errors];224 if (insp.pages < minPages)225 problems.push(`only ${insp.pages} pages (minimum ${minPages} for ${ctx.mode})`);226 if (insp.fonts.unembedded.length > 0)227 problems.push(`unembedded fonts: ${insp.fonts.unembedded.join(", ")}`);228 out.push(229 problems.length230 ? fail("pdf.valid", "hard", problems.join("; "), { pages: insp.pages, fonts: insp.fonts })231 : pass(232 "pdf.valid",233 "hard",234 `${insp.pages} pages, ${insp.textChars} text chars, ${insp.fonts.embedded}/${insp.fonts.total} fonts embedded`,235 { pages: insp.pages, fonts: insp.fonts.names },236 ),237 );238 const t = insp.text.replace(/\s+/g, " ");239 const hasWP = /AutoScholar Working Paper/i.test(t);240 const hasAI = /AI-generated|generated autonomously|autonomous(ly)? (AI|research)/i.test(t);241 const hasNoPeer = /not (formally |been )?peer[- ]reviewed/i.test(t);242 const hasMac = /MacLustr/.test(t);243 out.push(244 hasWP && hasAI && hasNoPeer && hasMac245 ? pass(246 "disclosure.present",247 "hard",248 "PDF carries working-paper, AI-generation, non-peer-review and MacLustr statements",249 )250 : fail("disclosure.present", "hard", "PDF lacks required disclosures", {251 workingPaper: hasWP,252 aiGenerated: hasAI,253 notPeerReviewed: hasNoPeer,254 maclustr: hasMac,255 }),256 );257 if (meta.value) {258 const titleWords = meta.value.title259 .toLowerCase()260 .replace(/[^a-z0-9 ]/g, " ")261 .split(/\s+/)262 .filter((w) => w.length > 3);263 const hit = titleWords.filter((w) => t.toLowerCase().includes(w)).length;264 out.push(265 titleWords.length === 0 || hit / titleWords.length >= 0.6266 ? pass("pdf.title_matches", "soft", "PDF text contains the declared title")267 : fail("pdf.title_matches", "soft", "declared title not found in PDF text", {268 hit,269 titleWords: titleWords.length,270 }),271 );272 }273 if (ctx.mode !== "theoretical" && !/Figure \d/.test(t) && !/\\begin\{figure\}/.test(tex))274 out.push(fail("pdf.figures", "soft", "no figures detected in a non-theoretical paper"));275 }276277 // G8 references278 const sources = await readJson(find(ctx, P.sources), (x) => SourcesFile.parse(x));279 const bibA = find(ctx, P.manuscriptBib);280 const bib = bibA ? parseBib((await bibA.read()).toString("utf8")) : [];281 const bibKeys = new Set(bib.map((b) => b.key));282 const cited = citeKeysInTex(tex);283 if (!sources.value) out.push(fail("references.verified", "hard", `sources.json invalid: ${sources.error}`));284 else {285 const srcKeys = new Map(sources.value.sources.map((s) => [s.key, s]));286 const unverified = sources.value.sources.filter((s) => !s.verified).map((s) => s.key);287 const citedUnknown = cited.filter((k) => !bibKeys.has(k));288 const citedUnverified = cited.filter(289 (k) => bibKeys.has(k) && (!srcKeys.has(k) || !srcKeys.get(k)!.verified),290 );291 const bibWithoutSource = bib.filter((b) => !srcKeys.has(b.key)).map((b) => b.key);292 const problems: string[] = [];293 if (citedUnknown.length) problems.push(`${citedUnknown.length} \\cite keys not in references.bib`);294 if (citedUnverified.length)295 problems.push(`${citedUnverified.length} cited references not verified in sources.json`);296 if (bibWithoutSource.length)297 problems.push(`${bibWithoutSource.length} bib entries without a verified source record`);298 if (cited.length === 0) problems.push("manuscript cites nothing");299 const bibMissingId = bib300 .filter((b) => !(b.fields.doi || b.fields.url || b.fields.eprint || b.fields.arxivid || b.fields.isbn))301 .map((b) => b.key);302 const minSources = ctx.mode === "review" ? 10 : 5;303 out.push(304 problems.length305 ? fail("references.verified", "hard", problems.join("; "), {306 citedUnknown: citedUnknown.slice(0, 20),307 citedUnverified: citedUnverified.slice(0, 20),308 bibWithoutSource: bibWithoutSource.slice(0, 20),309 unverified: unverified.slice(0, 20),310 })311 : pass(312 "references.verified",313 "hard",314 `${cited.length} cited keys all verified (${sources.value.sources.length} sources)`,315 { cited: cited.length, sources: sources.value.sources.length },316 ),317 );318 out.push(319 bibMissingId.length320 ? fail(321 "references.identifiers",322 "soft",323 `${bibMissingId.length} bib entries lack DOI/URL/arXiv identifiers`,324 { keys: bibMissingId.slice(0, 20) },325 )326 : pass("references.identifiers", "soft", "every bib entry carries an identifier"),327 );328 out.push(329 sources.value.sources.length >= minSources330 ? pass("references.count", "soft", `${sources.value.sources.length} sources`)331 : fail(332 "references.count",333 "soft",334 `only ${sources.value.sources.length} sources (expected ≥ ${minSources})`,335 ),336 );337 }338339 // G9 claims traced340 const smap = await readJson(find(ctx, P.sourceMap), (x) => SourceMapFile.parse(x));341 if (!smap.value) out.push(fail("claims.traced", "hard", `source-map.json invalid: ${smap.error}`));342 else {343 const srcKeys = new Set(sources.value?.sources.map((s) => s.key) ?? []);344 const paths = new Set(ctx.artifacts.map((a) => a.path));345 const untraced = smap.value.claims.filter(346 (c) => c.sourceKeys.every((k) => !srcKeys.has(k)) && c.artifactPaths.every((p) => !paths.has(p)),347 );348 out.push(349 smap.value.claims.length >= 3 && untraced.length === 0350 ? pass(351 "claims.traced",352 "hard",353 `${smap.value.claims.length} key claims mapped to verified sources or artifacts`,354 )355 : fail(356 "claims.traced",357 "hard",358 smap.value.claims.length < 3359 ? `only ${smap.value.claims.length} claims mapped (minimum 3)`360 : `${untraced.length} claims have no existing supporting source/artifact`,361 { untraced: untraced.slice(0, 10).map((c) => c.claim) },362 ),363 );364 }365366 // G10 results consistency (modes with metrics)367 if (needs(ctx, "empirical", "computational", "experimental")) {368 const metrics = await readJson(find(ctx, P.metrics), (x) => MetricsFile.parse(x));369 if (!metrics.value)370 out.push(fail("results.consistency", "hard", `metrics.json invalid: ${metrics.error}`));371 else {372 const rv = metrics.value.reportedValues;373 const missing = rv.filter((v) => !texContainsNumber(tex, v.value));374 out.push(375 rv.length >= 3 && missing.length === 0376 ? pass(377 "results.consistency",378 "hard",379 `${rv.length} reported values found verbatim in the manuscript`,380 )381 : fail(382 "results.consistency",383 "hard",384 rv.length < 3385 ? `only ${rv.length} reported values recorded (minimum 3)`386 : `${missing.length} reported values absent from main.tex`,387 { missing: missing.slice(0, 10) },388 ),389 );390 if (metrics.value.synthetic && ctx.mode === "empirical")391 out.push(fail("results.real", "hard", "empirical results flagged synthetic"));392 }393 const logA = find(ctx, P.experimentLog);394 const entries = logA395 ? (await logA.read())396 .toString("utf8")397 .split(/\r?\n/)398 .filter(Boolean)399 .map((l) => {400 try {401 return ExperimentLogEntry.parse(JSON.parse(l));402 } catch {403 return null;404 }405 })406 : [];407 const ok = entries.filter((e) => e && e.exitCode === 0).length;408 const codeFiles = ctx.artifacts.filter(409 (a) => a.path.startsWith("code/") && a.path !== P.codeReadme,410 ).length;411 out.push(412 ok >= 1 && codeFiles >= 1413 ? pass("execution.evidence", "hard", `${ok} successful executions logged, ${codeFiles} code files`)414 : fail(415 "execution.evidence",416 "hard",417 `insufficient execution evidence (successful runs: ${ok}, code files: ${codeFiles})`,418 ),419 );420 } else out.push(skip("results.consistency", "hard", "not required for this mode"));421422 // G11 data provenance423 if (needs(ctx, "empirical", "experimental", "review", "humanities")) {424 const dm = await readJson(find(ctx, P.dataManifest), (x) => DataManifestFile.parse(x));425 if (!dm.value) out.push(fail("data.provenance", "hard", `data/manifest.json invalid: ${dm.error}`));426 else {427 const ds = dm.value.datasets;428 const problems: string[] = [];429 if (ds.length === 0) problems.push("no datasets/records declared");430 for (const d of ds) {431 if (!d.license) problems.push(`${d.name}: no license`);432 if (!d.source) problems.push(`${d.name}: no source`);433 }434 if (ctx.mode === "empirical" && ds.length && ds.every((d) => d.synthetic))435 problems.push("all datasets are synthetic in empirical mode");436 if (ds.some((d) => d.synthetic) && !meta.value?.synthetic)437 problems.push("synthetic dataset present but manuscript metadata does not disclose synthetic data");438 out.push(439 problems.length440 ? fail("data.provenance", "hard", problems.slice(0, 5).join("; "), { problems })441 : pass("data.provenance", "hard", `${ds.length} dataset/record set(s) with source and license`),442 );443 }444 } else out.push(skip("data.provenance", "hard", "not required for this mode"));445446 // G12 theoretical rigor447 if (needs(ctx, "theoretical")) {448 const hasThm = /\\begin\{(theorem|proposition|lemma|corollary)\}/.test(tex);449 const hasProof = /\\begin\{proof\}/.test(tex);450 const rob = find(ctx, P.robustness);451 out.push(452 hasThm && hasProof && rob453 ? pass(454 "theory.formal",455 "hard",456 "formal statements, proofs and a counterexample/robustness note are present",457 )458 : fail(459 "theory.formal",460 "hard",461 `missing: ${[!hasThm && "theorem/proposition environments", !hasProof && "proof environments", !rob && "results/robustness.md"].filter(Boolean).join(", ")}`,462 ),463 );464 }465466 // G13 review467 const rr = await readJson(find(ctx, "review/review-1.json"), (x) => ReviewReport.parse(x));468 const revisions = find(ctx, P.revisions);469 if (!rr.value)470 out.push(fail("review.completed", "hard", `review/review-1.json invalid or missing: ${rr.error}`));471 else {472 const openBlockers = rr.value.findings.filter((f) => f.severity === "blocker" && f.resolution === "open");473 const openMajors = rr.value.findings.filter((f) => f.severity === "major" && f.resolution === "open");474 out.push(475 openBlockers.length === 0 && revisions && rr.value.recommendation !== "reject"476 ? pass(477 "review.completed",478 "hard",479 `independent review round ${rr.value.round}: ${rr.value.findings.length} findings, 0 open blockers, recommendation ${rr.value.recommendation}`,480 { rubric: rr.value.rubric, recommendation: rr.value.recommendation },481 )482 : fail(483 "review.completed",484 "hard",485 openBlockers.length486 ? `${openBlockers.length} blocker finding(s) unresolved`487 : !revisions488 ? "no response-to-review document"489 : "reviewer recommended reject",490 { openBlockers: openBlockers.map((f) => f.title) },491 ),492 );493 out.push(494 openMajors.length === 0495 ? pass("review.majors", "soft", "no open major findings")496 : fail("review.majors", "soft", `${openMajors.length} open major finding(s)`, {497 titles: openMajors.map((f) => f.title),498 }),499 );500 const avg = rr.value.rubric.length501 ? rr.value.rubric.reduce((s, r) => s + r.score, 0) / rr.value.rubric.length502 : 0;503 const low = rr.value.rubric.filter((r) => r.score <= 1);504 out.push(505 low.length === 0506 ? pass("review.rubric", "soft", `rubric average ${avg.toFixed(2)}/5`)507 : fail(508 "review.rubric",509 "soft",510 `rubric dimensions scored ≤1: ${low.map((l) => l.dimension).join(", ")}`,511 ),512 );513 }514515 // G14 secrets / dangerous content in promotable artifacts516 const leaks: string[] = [];517 for (const a of ctx.artifacts) {518 if (!TEXT_EXT.test(a.path) || a.bytes > 8 * 1024 * 1024 || a.path.startsWith("input/")) continue;519 const t = (await a.read()).toString("utf8");520 if (SECRET_PATTERNS.some((re) => re.test(t))) leaks.push(a.path);521 }522 const dangerous = ctx.artifacts523 .filter((a) => /\.(exe|dll|dylib|so|app|pkg|dmg|bat|cmd|scr)$/i.test(a.path))524 .map((a) => a.path);525 out.push(526 leaks.length === 0 && dangerous.length === 0527 ? pass("secrets.none", "hard", "no credential patterns or executable binaries in artifacts")528 : fail(529 "secrets.none",530 "hard",531 `${leaks.length} artifact(s) contain secret-like strings; ${dangerous.length} dangerous binaries`,532 { leaks, dangerous },533 ),534 );535536 // G15 abstract sanity537 if (meta.value) {538 const words = meta.value.abstract.split(/\s+/).length;539 out.push(540 words >= 100 && words <= 400541 ? pass("abstract.length", "soft", `${words} words`)542 : fail("abstract.length", "soft", `abstract has ${words} words (expected 100–400)`),543 );544 }545546 // G16 profile evidence summary547 out.push(548 pass("profile.evidence", "soft", `mode ${ctx.mode} requires: ${profile.requiredEvidence.join(", ")}`),549 );550 return out;551}552