import { CONTRACT_PATHS, checkArtifactContract, MODE_PROFILES } from "@autoscholar/research-contract"; import { DataManifestFile, ExperimentLogEntry, FeasibilityFile, type GateResult, ManuscriptMetadata, MetricsFile, type ResearchMode, ReviewReport, SourceMapFile, SourcesFile, } from "@autoscholar/schemas"; import { citeKeysInTex, parseBib } from "./bib.js"; import { inspectPdf } from "./pdf.js"; export interface ArtifactView { path: string; kind: string; sha256: string; bytes: number; synthetic: boolean; read: () => Promise; storedSha256: () => Promise; } export interface GateContext { mode: ResearchMode; secondaryModes: string[]; artifacts: ArtifactView[]; attemptOutcome: string | null; cancelRequested: boolean; workerSimulated: boolean; reviewFindingsOpenBlockers: number; } const P = CONTRACT_PATHS; const SECRET_PATTERNS = [ /sk-ant-[A-Za-z0-9_-]{20,}/, /asw_[A-Za-z0-9_-]{20,}/, /-----BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY-----/, /AKIA[0-9A-Z]{16}/, /ghp_[A-Za-z0-9]{30,}/, /xox[bp]-[A-Za-z0-9-]{20,}/, ]; const TEXT_EXT = /\.(md|tex|bib|json|jsonl|txt|csv|py|r|jl|sh|yaml|yml|toml|cfg|ini|sha256|log)$/i; const pass = ( gate: string, severity: "hard" | "soft", message: string, evidence: Record = {}, ): GateResult => ({ gate, severity, status: "pass", message, evidence }); const fail = ( gate: string, severity: "hard" | "soft", message: string, evidence: Record = {}, ): GateResult => ({ gate, severity, status: severity === "hard" ? "fail" : "warn", message, evidence }); const skip = (gate: string, severity: "hard" | "soft", message: string): GateResult => ({ gate, severity, status: "skip", message, evidence: {}, }); function find(ctx: GateContext, path: string): ArtifactView | undefined { return ctx.artifacts.find((a) => a.path === path); } async function readJson( a: ArtifactView | undefined, parse: (x: unknown) => T, ): Promise<{ value?: T; error?: string }> { if (!a) return { error: "missing" }; try { return { value: parse(JSON.parse((await a.read()).toString("utf8"))) }; } catch (e) { return { error: (e as Error).message.slice(0, 300) }; } } function needs(ctx: GateContext, ...modes: ResearchMode[]): boolean { return modes.includes(ctx.mode) || ctx.secondaryModes.some((m) => modes.includes(m as ResearchMode)); } /** Does the tex contain the reported number (tolerant to formatting: thousands separators, trailing zeros, percent)? */ export function texContainsNumber(tex: string, value: number | string): boolean { const s = String(value).trim(); if (!/^-?\d/.test(s)) return tex.includes(s); const n = Number(s.replace(/,/g, "")); if (!Number.isFinite(n)) return tex.includes(s); const candidates = new Set([s, s.replace(/,/g, ""), n.toString()]); for (const d of [0, 1, 2, 3, 4]) candidates.add(n.toFixed(d)); const compact = tex.replace(/,(?=\d{3})/g, "").replace(/\\,|;|\\ |~|\{|\}|\$/g, ""); for (const c of candidates) { const re = new RegExp(`(? { const out: GateResult[] = []; const profile = MODE_PROFILES[ctx.mode]; // G1 final attempt out.push( ctx.attemptOutcome === "completed" && !ctx.cancelRequested ? pass("attempt.final", "hard", "worker attempt completed and run not cancelled") : fail( "attempt.final", "hard", `attempt outcome is ${ctx.attemptOutcome ?? "none"}${ctx.cancelRequested ? " (cancellation requested)" : ""}`, ), ); // G2 not synthetic / not simulated worker const synth = ctx.artifacts.filter((a) => a.synthetic).map((a) => a.path); out.push( ctx.workerSimulated || synth.length > 0 ? fail( "worker.real", "hard", "artifacts were produced by a simulated worker or are flagged synthetic; never publishable", { synthetic: synth.slice(0, 20), workerSimulated: ctx.workerSimulated }, ) : pass("worker.real", "hard", "artifacts come from a real worker"), ); // G3 contract const contract = checkArtifactContract(ctx.mode, ctx.artifacts); out.push( contract.ok ? pass("contract.artifacts", "hard", `all ${contract.present.length} mode-required artifacts present`, { mode: ctx.mode, }) : fail("contract.artifacts", "hard", `missing ${contract.missing.length} required artifact(s)`, { missing: contract.missing, }), ); // G4 checksums const mismatches: string[] = []; for (const a of ctx.artifacts) { const stored = await a.storedSha256(); if (stored && stored !== a.sha256) mismatches.push(a.path); } let checksumFileOk = false; const cf = find(ctx, P.checksums); if (cf) { const lines = (await cf.read()).toString("utf8").split(/\r?\n/).filter(Boolean); const map = new Map(lines.map((l) => [l.slice(66).trim().replace(/^\*/, ""), l.slice(0, 64)])); const bad = ctx.artifacts .filter( (a) => a.path !== P.checksums && a.path !== P.events && a.path !== P.runSummary && map.has(a.path) && map.get(a.path) !== a.sha256, ) .map((a) => a.path); const covered = ctx.artifacts.filter((a) => map.has(a.path)).length; checksumFileOk = bad.length === 0 && covered >= Math.max(1, Math.floor(ctx.artifacts.length * 0.5)); if (bad.length) mismatches.push(...bad.map((b) => `${b} (checksums.sha256)`)); } out.push( mismatches.length === 0 && checksumFileOk ? pass("checksums.valid", "hard", "all stored digests match and checksums.sha256 agrees", { artifacts: ctx.artifacts.length, }) : fail( "checksums.valid", "hard", mismatches.length ? `digest mismatches: ${mismatches.length}` : "checksums.sha256 missing or covers too few artifacts", { mismatches: mismatches.slice(0, 20) }, ), ); // G5 feasibility const feas = await readJson(find(ctx, P.feasibility), (x) => FeasibilityFile.parse(x)); out.push( feas.value?.feasible && feas.value.requiredApprovals.length === 0 ? pass("feasibility.ok", "hard", "preflight judged the study feasible with no outstanding approvals") : fail( "feasibility.ok", "hard", feas.error ? `feasibility.json invalid: ${feas.error}` : `not feasible or approvals outstanding: ${feas.value?.requiredApprovals.join(", ") || feas.value?.rejectReason}`, ), ); // G6 metadata + manuscript const meta = await readJson(find(ctx, P.metadata), (x) => ManuscriptMetadata.parse(x)); const texA = find(ctx, P.mainTex); const tex = texA ? (await texA.read()).toString("utf8") : ""; if (!meta.value) out.push(fail("metadata.valid", "hard", `metadata.json invalid: ${meta.error}`)); else { const problems: string[] = []; if (!tex.includes("\\begin{document}")) problems.push("main.tex has no document body"); if (!/\\title\{/.test(tex) && !/\\Title\{/.test(tex) && !/title=\{/.test(tex)) problems.push("main.tex has no title"); if (meta.value.synthetic && !/synthetic/i.test(meta.value.limitations + meta.value.abstract)) problems.push("synthetic flag set but not disclosed in abstract/limitations"); out.push( problems.length ? fail("metadata.valid", "hard", problems.join("; ")) : pass("metadata.valid", "hard", "manuscript metadata and LaTeX source are well-formed", { title: meta.value.title, keywords: meta.value.keywords.length, }), ); } // G7 PDF const pdfA = find(ctx, P.mainPdf); if (!pdfA) out.push(fail("pdf.valid", "hard", "manuscript/main.pdf missing")); else { const insp = await inspectPdf(await pdfA.read()); const minPages = ctx.mode === "theoretical" ? 4 : 6; const problems = [...insp.errors]; if (insp.pages < minPages) problems.push(`only ${insp.pages} pages (minimum ${minPages} for ${ctx.mode})`); if (insp.fonts.unembedded.length > 0) problems.push(`unembedded fonts: ${insp.fonts.unembedded.join(", ")}`); out.push( problems.length ? fail("pdf.valid", "hard", problems.join("; "), { pages: insp.pages, fonts: insp.fonts }) : pass( "pdf.valid", "hard", `${insp.pages} pages, ${insp.textChars} text chars, ${insp.fonts.embedded}/${insp.fonts.total} fonts embedded`, { pages: insp.pages, fonts: insp.fonts.names }, ), ); const t = insp.text.replace(/\s+/g, " "); const hasWP = /AutoScholar Working Paper/i.test(t); const hasAI = /AI-generated|generated autonomously|autonomous(ly)? (AI|research)/i.test(t); const hasNoPeer = /not (formally |been )?peer[- ]reviewed/i.test(t); const hasMac = /MacLustr/.test(t); out.push( hasWP && hasAI && hasNoPeer && hasMac ? pass( "disclosure.present", "hard", "PDF carries working-paper, AI-generation, non-peer-review and MacLustr statements", ) : fail("disclosure.present", "hard", "PDF lacks required disclosures", { workingPaper: hasWP, aiGenerated: hasAI, notPeerReviewed: hasNoPeer, maclustr: hasMac, }), ); if (meta.value) { const titleWords = meta.value.title .toLowerCase() .replace(/[^a-z0-9 ]/g, " ") .split(/\s+/) .filter((w) => w.length > 3); const hit = titleWords.filter((w) => t.toLowerCase().includes(w)).length; out.push( titleWords.length === 0 || hit / titleWords.length >= 0.6 ? pass("pdf.title_matches", "soft", "PDF text contains the declared title") : fail("pdf.title_matches", "soft", "declared title not found in PDF text", { hit, titleWords: titleWords.length, }), ); } if (ctx.mode !== "theoretical" && !/Figure \d/.test(t) && !/\\begin\{figure\}/.test(tex)) out.push(fail("pdf.figures", "soft", "no figures detected in a non-theoretical paper")); } // G8 references const sources = await readJson(find(ctx, P.sources), (x) => SourcesFile.parse(x)); const bibA = find(ctx, P.manuscriptBib); const bib = bibA ? parseBib((await bibA.read()).toString("utf8")) : []; const bibKeys = new Set(bib.map((b) => b.key)); const cited = citeKeysInTex(tex); if (!sources.value) out.push(fail("references.verified", "hard", `sources.json invalid: ${sources.error}`)); else { const srcKeys = new Map(sources.value.sources.map((s) => [s.key, s])); const unverified = sources.value.sources.filter((s) => !s.verified).map((s) => s.key); const citedUnknown = cited.filter((k) => !bibKeys.has(k)); const citedUnverified = cited.filter( (k) => bibKeys.has(k) && (!srcKeys.has(k) || !srcKeys.get(k)!.verified), ); const bibWithoutSource = bib.filter((b) => !srcKeys.has(b.key)).map((b) => b.key); const problems: string[] = []; if (citedUnknown.length) problems.push(`${citedUnknown.length} \\cite keys not in references.bib`); if (citedUnverified.length) problems.push(`${citedUnverified.length} cited references not verified in sources.json`); if (bibWithoutSource.length) problems.push(`${bibWithoutSource.length} bib entries without a verified source record`); if (cited.length === 0) problems.push("manuscript cites nothing"); const bibMissingId = bib .filter((b) => !(b.fields.doi || b.fields.url || b.fields.eprint || b.fields.arxivid || b.fields.isbn)) .map((b) => b.key); const minSources = ctx.mode === "review" ? 10 : 5; out.push( problems.length ? fail("references.verified", "hard", problems.join("; "), { citedUnknown: citedUnknown.slice(0, 20), citedUnverified: citedUnverified.slice(0, 20), bibWithoutSource: bibWithoutSource.slice(0, 20), unverified: unverified.slice(0, 20), }) : pass( "references.verified", "hard", `${cited.length} cited keys all verified (${sources.value.sources.length} sources)`, { cited: cited.length, sources: sources.value.sources.length }, ), ); out.push( bibMissingId.length ? fail( "references.identifiers", "soft", `${bibMissingId.length} bib entries lack DOI/URL/arXiv identifiers`, { keys: bibMissingId.slice(0, 20) }, ) : pass("references.identifiers", "soft", "every bib entry carries an identifier"), ); out.push( sources.value.sources.length >= minSources ? pass("references.count", "soft", `${sources.value.sources.length} sources`) : fail( "references.count", "soft", `only ${sources.value.sources.length} sources (expected ≥ ${minSources})`, ), ); } // G9 claims traced const smap = await readJson(find(ctx, P.sourceMap), (x) => SourceMapFile.parse(x)); if (!smap.value) out.push(fail("claims.traced", "hard", `source-map.json invalid: ${smap.error}`)); else { const srcKeys = new Set(sources.value?.sources.map((s) => s.key) ?? []); const paths = new Set(ctx.artifacts.map((a) => a.path)); const untraced = smap.value.claims.filter( (c) => c.sourceKeys.every((k) => !srcKeys.has(k)) && c.artifactPaths.every((p) => !paths.has(p)), ); out.push( smap.value.claims.length >= 3 && untraced.length === 0 ? pass( "claims.traced", "hard", `${smap.value.claims.length} key claims mapped to verified sources or artifacts`, ) : fail( "claims.traced", "hard", smap.value.claims.length < 3 ? `only ${smap.value.claims.length} claims mapped (minimum 3)` : `${untraced.length} claims have no existing supporting source/artifact`, { untraced: untraced.slice(0, 10).map((c) => c.claim) }, ), ); } // G10 results consistency (modes with metrics) if (needs(ctx, "empirical", "computational", "experimental")) { const metrics = await readJson(find(ctx, P.metrics), (x) => MetricsFile.parse(x)); if (!metrics.value) out.push(fail("results.consistency", "hard", `metrics.json invalid: ${metrics.error}`)); else { const rv = metrics.value.reportedValues; const missing = rv.filter((v) => !texContainsNumber(tex, v.value)); out.push( rv.length >= 3 && missing.length === 0 ? pass( "results.consistency", "hard", `${rv.length} reported values found verbatim in the manuscript`, ) : fail( "results.consistency", "hard", rv.length < 3 ? `only ${rv.length} reported values recorded (minimum 3)` : `${missing.length} reported values absent from main.tex`, { missing: missing.slice(0, 10) }, ), ); if (metrics.value.synthetic && ctx.mode === "empirical") out.push(fail("results.real", "hard", "empirical results flagged synthetic")); } const logA = find(ctx, P.experimentLog); const entries = logA ? (await logA.read()) .toString("utf8") .split(/\r?\n/) .filter(Boolean) .map((l) => { try { return ExperimentLogEntry.parse(JSON.parse(l)); } catch { return null; } }) : []; const ok = entries.filter((e) => e && e.exitCode === 0).length; const codeFiles = ctx.artifacts.filter( (a) => a.path.startsWith("code/") && a.path !== P.codeReadme, ).length; out.push( ok >= 1 && codeFiles >= 1 ? pass("execution.evidence", "hard", `${ok} successful executions logged, ${codeFiles} code files`) : fail( "execution.evidence", "hard", `insufficient execution evidence (successful runs: ${ok}, code files: ${codeFiles})`, ), ); } else out.push(skip("results.consistency", "hard", "not required for this mode")); // G11 data provenance if (needs(ctx, "empirical", "experimental", "review", "humanities")) { const dm = await readJson(find(ctx, P.dataManifest), (x) => DataManifestFile.parse(x)); if (!dm.value) out.push(fail("data.provenance", "hard", `data/manifest.json invalid: ${dm.error}`)); else { const ds = dm.value.datasets; const problems: string[] = []; if (ds.length === 0) problems.push("no datasets/records declared"); for (const d of ds) { if (!d.license) problems.push(`${d.name}: no license`); if (!d.source) problems.push(`${d.name}: no source`); } if (ctx.mode === "empirical" && ds.length && ds.every((d) => d.synthetic)) problems.push("all datasets are synthetic in empirical mode"); if (ds.some((d) => d.synthetic) && !meta.value?.synthetic) problems.push("synthetic dataset present but manuscript metadata does not disclose synthetic data"); out.push( problems.length ? fail("data.provenance", "hard", problems.slice(0, 5).join("; "), { problems }) : pass("data.provenance", "hard", `${ds.length} dataset/record set(s) with source and license`), ); } } else out.push(skip("data.provenance", "hard", "not required for this mode")); // G12 theoretical rigor if (needs(ctx, "theoretical")) { const hasThm = /\\begin\{(theorem|proposition|lemma|corollary)\}/.test(tex); const hasProof = /\\begin\{proof\}/.test(tex); const rob = find(ctx, P.robustness); out.push( hasThm && hasProof && rob ? pass( "theory.formal", "hard", "formal statements, proofs and a counterexample/robustness note are present", ) : fail( "theory.formal", "hard", `missing: ${[!hasThm && "theorem/proposition environments", !hasProof && "proof environments", !rob && "results/robustness.md"].filter(Boolean).join(", ")}`, ), ); } // G13 review const rr = await readJson(find(ctx, "review/review-1.json"), (x) => ReviewReport.parse(x)); const revisions = find(ctx, P.revisions); if (!rr.value) out.push(fail("review.completed", "hard", `review/review-1.json invalid or missing: ${rr.error}`)); else { const openBlockers = rr.value.findings.filter((f) => f.severity === "blocker" && f.resolution === "open"); const openMajors = rr.value.findings.filter((f) => f.severity === "major" && f.resolution === "open"); out.push( openBlockers.length === 0 && revisions && rr.value.recommendation !== "reject" ? pass( "review.completed", "hard", `independent review round ${rr.value.round}: ${rr.value.findings.length} findings, 0 open blockers, recommendation ${rr.value.recommendation}`, { rubric: rr.value.rubric, recommendation: rr.value.recommendation }, ) : fail( "review.completed", "hard", openBlockers.length ? `${openBlockers.length} blocker finding(s) unresolved` : !revisions ? "no response-to-review document" : "reviewer recommended reject", { openBlockers: openBlockers.map((f) => f.title) }, ), ); out.push( openMajors.length === 0 ? pass("review.majors", "soft", "no open major findings") : fail("review.majors", "soft", `${openMajors.length} open major finding(s)`, { titles: openMajors.map((f) => f.title), }), ); const avg = rr.value.rubric.length ? rr.value.rubric.reduce((s, r) => s + r.score, 0) / rr.value.rubric.length : 0; const low = rr.value.rubric.filter((r) => r.score <= 1); out.push( low.length === 0 ? pass("review.rubric", "soft", `rubric average ${avg.toFixed(2)}/5`) : fail( "review.rubric", "soft", `rubric dimensions scored ≤1: ${low.map((l) => l.dimension).join(", ")}`, ), ); } // G14 secrets / dangerous content in promotable artifacts const leaks: string[] = []; for (const a of ctx.artifacts) { if (!TEXT_EXT.test(a.path) || a.bytes > 8 * 1024 * 1024 || a.path.startsWith("input/")) continue; const t = (await a.read()).toString("utf8"); if (SECRET_PATTERNS.some((re) => re.test(t))) leaks.push(a.path); } const dangerous = ctx.artifacts .filter((a) => /\.(exe|dll|dylib|so|app|pkg|dmg|bat|cmd|scr)$/i.test(a.path)) .map((a) => a.path); out.push( leaks.length === 0 && dangerous.length === 0 ? pass("secrets.none", "hard", "no credential patterns or executable binaries in artifacts") : fail( "secrets.none", "hard", `${leaks.length} artifact(s) contain secret-like strings; ${dangerous.length} dangerous binaries`, { leaks, dangerous }, ), ); // G15 abstract sanity if (meta.value) { const words = meta.value.abstract.split(/\s+/).length; out.push( words >= 100 && words <= 400 ? pass("abstract.length", "soft", `${words} words`) : fail("abstract.length", "soft", `abstract has ${words} words (expected 100–400)`), ); } // G16 profile evidence summary out.push( pass("profile.evidence", "soft", `mode ${ctx.mode} requires: ${profile.requiredEvidence.join(", ")}`), ); return out; }