Finalize: sync then checksums from uploaded digests; QA revision skips platform-side gates; stale registered values become a soft warning
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
3 changed files +47 −6
modified
services/qa/src/gates.ts
+16 −4
@@ -399,22 +399,34 @@ export async function runGates(ctx: GateContext): Promise<GateResult[]> { | |||||
| 399 | reportTexts.push((await a.read()).toString("utf8")); | 399 | reportTexts.push((await a.read()).toString("utf8")); | ||
| 400 | const corpus = `${tex}\n${reportTexts.join("\n")}`; | 400 | const corpus = `${tex}\n${reportTexts.join("\n")}`; | ||
| 401 | const missing = rv.filter((v) => !texContainsNumber(corpus, v.value)); | 401 | const missing = rv.filter((v) => !texContainsNumber(corpus, v.value)); | ||
| 402 | + const found = rv.length - missing.length; | ||||
| 403 | + const ratio = rv.length ? found / rv.length : 0; | ||||
| 402 | out.push( | 404 | out.push( | ||
| 403 | − rv.length >= 3 && missing.length === 0 | 405 | + found >= 3 && ratio >= 0.7 | ||
| 404 | ? pass( | 406 | ? pass( | ||
| 405 | "results.consistency", | 407 | "results.consistency", | ||
| 406 | "hard", | 408 | "hard", | ||
| 407 | − `${rv.length} reported values found verbatim in the manuscript`, | 409 | + `${found}/${rv.length} registered values found verbatim in the manuscript or results reports`, | ||
| 410 | + { found, total: rv.length }, | ||||
| 408 | ) | 411 | ) | ||
| 409 | : fail( | 412 | : fail( | ||
| 410 | "results.consistency", | 413 | "results.consistency", | ||
| 411 | "hard", | 414 | "hard", | ||
| 412 | rv.length < 3 | 415 | rv.length < 3 | ||
| 413 | ? `only ${rv.length} reported values recorded (minimum 3)` | 416 | ? `only ${rv.length} reported values recorded (minimum 3)` | ||
| 414 | − : `${missing.length} reported values absent from main.tex`, | 417 | + : `only ${found}/${rv.length} registered values found in the manuscript or results reports`, | ||
| 415 | − { missing: missing.slice(0, 10) }, | 418 | + { missing: missing.slice(0, 10), found, total: rv.length }, | ||
| 416 | ), | 419 | ), | ||
| 417 | ); | 420 | ); | ||
| 421 | + if (missing.length > 0 && found >= 3 && ratio >= 0.7) | ||||
| 422 | + out.push( | ||||
| 423 | + fail( | ||||
| 424 | + "results.stale_values", | ||||
| 425 | + "soft", | ||||
| 426 | + `${missing.length} registered value(s) no longer appear in the manuscript (stale registrations)`, | ||||
| 427 | + { missing: missing.slice(0, 10) }, | ||||
| 428 | + ), | ||||
| 429 | + ); | ||||
| 418 | if (metrics.value.synthetic && ctx.mode === "empirical") | 430 | if (metrics.value.synthetic && ctx.mode === "empirical") | ||
| 419 | out.push(fail("results.real", "hard", "empirical results flagged synthetic")); | 431 | out.push(fail("results.real", "hard", "empirical results flagged synthetic")); | ||
| 420 | } | 432 | } | ||
modified
services/research-runner/src/artifacts.ts
+20 −1
@@ -140,7 +140,26 @@ export async function syncArtifacts( | |||||
| 140 | } | 140 | } | ||
| 141 | 141 | ||||
| 142 | /** Writes provenance/checksums.sha256 for every artifact file (excluding itself and platform dirs). */ | 142 | /** Writes provenance/checksums.sha256 for every artifact file (excluding itself and platform dirs). */ | ||
| 143 | −export async function writeChecksums(workspace: string): Promise<string> { | 143 | +/** | ||
| 144 | + * Writes provenance/checksums.sha256. When an upload-state file is given, digests come from what was actually uploaded | ||||
| 145 | + * (consistent with the artifact store even if files change afterwards); otherwise files are hashed in place. | ||||
| 146 | + */ | ||||
| 147 | +export async function writeChecksums(workspace: string, uploadStatePath?: string): Promise<string> { | ||||
| 148 | + if (uploadStatePath) { | ||||
| 149 | + try { | ||||
| 150 | + const state = JSON.parse(await readFile(uploadStatePath, "utf8")) as Record<string, string>; | ||||
| 151 | + const lines = Object.entries(state) | ||||
| 152 | + .filter(([p]) => p !== "provenance/checksums.sha256") | ||||
| 153 | + .sort(([a], [b]) => (a < b ? -1 : 1)) | ||||
| 154 | + .map(([p, sha]) => `${sha} ${p}`); | ||||
| 155 | + await mkdir(join(workspace, "provenance"), { recursive: true }); | ||||
| 156 | + const out = `${lines.join("\n")}\n`; | ||||
| 157 | + await writeFile(join(workspace, "provenance/checksums.sha256"), out); | ||||
| 158 | + return out; | ||||
| 159 | + } catch { | ||||
| 160 | + /* fall back to hashing */ | ||||
| 161 | + } | ||||
| 162 | + } | ||||
| 144 | const files = (await listFiles(workspace)).filter( | 163 | const files = (await listFiles(workspace)).filter( | ||
| 145 | (f) => | 164 | (f) => | ||
| 146 | !SKIP.some((r) => r.test(f)) && | 165 | !SKIP.some((r) => r.test(f)) && | ||
modified
services/research-runner/src/runner.ts
+11 −1
@@ -933,15 +933,25 @@ Contract reminders: results/metrics.json must be {"metrics": {...}, "reportedVal | |||||
| 933 | ); | 933 | ); | ||
| 934 | } | 934 | } | ||
| 935 | }; | 935 | }; | ||
| 936 | + await step("cleanup-processes", () => this.killLeftoverProcesses("reproduction")); | ||||
| 936 | await step("summary", async () => { | 937 | await step("summary", async () => { | ||
| 937 | await mkdir(join(this.ws, "provenance"), { recursive: true }); | 938 | await mkdir(join(this.ws, "provenance"), { recursive: true }); | ||
| 938 | await writeFile(join(this.ws, "provenance/run-summary.json"), `${JSON.stringify(summary, null, 2)}\n`); | 939 | await writeFile(join(this.ws, "provenance/run-summary.json"), `${JSON.stringify(summary, null, 2)}\n`); | ||
| 939 | }); | 940 | }); | ||
| 940 | await step("cleanup", () => rm(join(this.ws, "_repro"), { recursive: true, force: true })); | 941 | await step("cleanup", () => rm(join(this.ws, "_repro"), { recursive: true, force: true })); | ||
| 941 | − await step("checksums", () => writeChecksums(this.ws)); | 942 | + // sync first, then derive checksums.sha256 from the digests that were actually uploaded, then upload it | ||
| 942 | await step("sync", () => | 943 | await step("sync", () => | ||
| 943 | syncArtifacts(this.ws, "reproduction", this.host.upload, (k, m, d) => this.emit(k, m, d)), | 944 | syncArtifacts(this.ws, "reproduction", this.host.upload, (k, m, d) => this.emit(k, m, d)), | ||
| 944 | ); | 945 | ); | ||
| 946 | + await step("checksums", async () => { | ||||
| 947 | + const text = await writeChecksums(this.ws, join(this.ws, ".autoscholar/uploaded.json")); | ||||
| 948 | + await this.host.upload( | ||||
| 949 | + "provenance/checksums.sha256", | ||||
| 950 | + "checksum", | ||||
| 951 | + "text/plain; charset=utf-8", | ||||
| 952 | + Buffer.from(text), | ||||
| 953 | + ); | ||||
| 954 | + }); | ||||
| 945 | this.emit( | 955 | this.emit( | ||
| 946 | "run.finalized", | 956 | "run.finalized", | ||
| 947 | `run ${outcome}${error ? `: ${error}` : ""} — $${this.spent.toFixed(3)}, ${this.turns} turns, ${this.completed.length} stages`, | 957 | `run ${outcome}${error ? `: ${error}` : ""} — $${this.spent.toFixed(3)}, ${this.turns} turns, ${this.completed.length} stages`, | ||
| 948 | 958 | ||||