Finalize: sync then checksums from uploaded digests; QA revision skips platform-side gates; stale registered values become a soft warning
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
3 changed files +47 −6
modified
services/qa/src/gates.ts
+16 −4
@@ -399,22 +399,34 @@ export async function runGates(ctx: GateContext): Promise<GateResult[]> { | ||
| 399 | 399 | reportTexts.push((await a.read()).toString("utf8")); |
| 400 | 400 | const corpus = `${tex}\n${reportTexts.join("\n")}`; |
| 401 | 401 | const missing = rv.filter((v) => !texContainsNumber(corpus, v.value)); |
| 402 | + const found = rv.length - missing.length; | |
| 403 | + const ratio = rv.length ? found / rv.length : 0; | |
| 402 | 404 | out.push( |
| 403 | − rv.length >= 3 && missing.length === 0 | |
| 405 | + found >= 3 && ratio >= 0.7 | |
| 404 | 406 | ? pass( |
| 405 | 407 | "results.consistency", |
| 406 | 408 | "hard", |
| 407 | − `${rv.length} reported values found verbatim in the manuscript`, | |
| 409 | + `${found}/${rv.length} registered values found verbatim in the manuscript or results reports`, | |
| 410 | + { found, total: rv.length }, | |
| 408 | 411 | ) |
| 409 | 412 | : fail( |
| 410 | 413 | "results.consistency", |
| 411 | 414 | "hard", |
| 412 | 415 | rv.length < 3 |
| 413 | 416 | ? `only ${rv.length} reported values recorded (minimum 3)` |
| 414 | − : `${missing.length} reported values absent from main.tex`, | |
| 415 | − { missing: missing.slice(0, 10) }, | |
| 417 | + : `only ${found}/${rv.length} registered values found in the manuscript or results reports`, | |
| 418 | + { missing: missing.slice(0, 10), found, total: rv.length }, | |
| 416 | 419 | ), |
| 417 | 420 | ); |
| 421 | + if (missing.length > 0 && found >= 3 && ratio >= 0.7) | |
| 422 | + out.push( | |
| 423 | + fail( | |
| 424 | + "results.stale_values", | |
| 425 | + "soft", | |
| 426 | + `${missing.length} registered value(s) no longer appear in the manuscript (stale registrations)`, | |
| 427 | + { missing: missing.slice(0, 10) }, | |
| 428 | + ), | |
| 429 | + ); | |
| 418 | 430 | if (metrics.value.synthetic && ctx.mode === "empirical") |
| 419 | 431 | out.push(fail("results.real", "hard", "empirical results flagged synthetic")); |
| 420 | 432 | } |
modified
services/research-runner/src/artifacts.ts
+20 −1
@@ -140,7 +140,26 @@ export async function syncArtifacts( | ||
| 140 | 140 | } |
| 141 | 141 | |
| 142 | 142 | /** Writes provenance/checksums.sha256 for every artifact file (excluding itself and platform dirs). */ |
| 143 | −export async function writeChecksums(workspace: string): Promise<string> { | |
| 143 | +/** | |
| 144 | + * Writes provenance/checksums.sha256. When an upload-state file is given, digests come from what was actually uploaded | |
| 145 | + * (consistent with the artifact store even if files change afterwards); otherwise files are hashed in place. | |
| 146 | + */ | |
| 147 | +export async function writeChecksums(workspace: string, uploadStatePath?: string): Promise<string> { | |
| 148 | + if (uploadStatePath) { | |
| 149 | + try { | |
| 150 | + const state = JSON.parse(await readFile(uploadStatePath, "utf8")) as Record<string, string>; | |
| 151 | + const lines = Object.entries(state) | |
| 152 | + .filter(([p]) => p !== "provenance/checksums.sha256") | |
| 153 | + .sort(([a], [b]) => (a < b ? -1 : 1)) | |
| 154 | + .map(([p, sha]) => `${sha} ${p}`); | |
| 155 | + await mkdir(join(workspace, "provenance"), { recursive: true }); | |
| 156 | + const out = `${lines.join("\n")}\n`; | |
| 157 | + await writeFile(join(workspace, "provenance/checksums.sha256"), out); | |
| 158 | + return out; | |
| 159 | + } catch { | |
| 160 | + /* fall back to hashing */ | |
| 161 | + } | |
| 162 | + } | |
| 144 | 163 | const files = (await listFiles(workspace)).filter( |
| 145 | 164 | (f) => |
| 146 | 165 | !SKIP.some((r) => r.test(f)) && |
modified
services/research-runner/src/runner.ts
+11 −1
@@ -933,15 +933,25 @@ Contract reminders: results/metrics.json must be {"metrics": {...}, "reportedVal | ||
| 933 | 933 | ); |
| 934 | 934 | } |
| 935 | 935 | }; |
| 936 | + await step("cleanup-processes", () => this.killLeftoverProcesses("reproduction")); | |
| 936 | 937 | await step("summary", async () => { |
| 937 | 938 | await mkdir(join(this.ws, "provenance"), { recursive: true }); |
| 938 | 939 | await writeFile(join(this.ws, "provenance/run-summary.json"), `${JSON.stringify(summary, null, 2)}\n`); |
| 939 | 940 | }); |
| 940 | 941 | await step("cleanup", () => rm(join(this.ws, "_repro"), { recursive: true, force: true })); |
| 941 | − await step("checksums", () => writeChecksums(this.ws)); | |
| 942 | + // sync first, then derive checksums.sha256 from the digests that were actually uploaded, then upload it | |
| 942 | 943 | await step("sync", () => |
| 943 | 944 | syncArtifacts(this.ws, "reproduction", this.host.upload, (k, m, d) => this.emit(k, m, d)), |
| 944 | 945 | ); |
| 946 | + await step("checksums", async () => { | |
| 947 | + const text = await writeChecksums(this.ws, join(this.ws, ".autoscholar/uploaded.json")); | |
| 948 | + await this.host.upload( | |
| 949 | + "provenance/checksums.sha256", | |
| 950 | + "checksum", | |
| 951 | + "text/plain; charset=utf-8", | |
| 952 | + Buffer.from(text), | |
| 953 | + ); | |
| 954 | + }); | |
| 945 | 955 | this.emit( |
| 946 | 956 | "run.finalized", |
| 947 | 957 | `run ${outcome}${error ? `: ${error}` : ""} — $${this.spent.toFixed(3)}, ${this.turns} turns, ${this.completed.length} stages`, |
| 948 | 958 | |