1#!/usr/bin/env python2"""Results-to-claim audit. Reads results/claim_registry.csv (claim, metric, value, benchmark, run_ids, artifact,3status) and verifies that (i) every artifact path exists, (ii) every run id has a manifest with state completed4(or the registry explicitly says 'partial'), (iii) every numeric macro used in the manuscript exists in5paper/tables/numbers.tex, and (iv) the manuscript does not use forbidden promotional words.6Exit code 1 on any failure. Usage: python scripts/verify_manuscript_claims.py7"""8from __future__ import annotations910import csv, glob, json, re, sys11from pathlib import Path1213FORBIDDEN = ["revolutionary", "state-of-the-art", "breakthrough", "groundbreaking", "unprecedented", "novel architecture that outperforms"]141516def main() -> int:17 ok = True18 reg = Path("results/claim_registry.csv")19 if not reg.exists():20 print("FAIL: results/claim_registry.csv missing"); return 121 rows = list(csv.DictReader(open(reg)))22 print(f"{len(rows)} registered claims")23 for r in rows:24 for art in filter(None, [a.strip() for a in r["artifact"].split(";")]):25 if not Path(art).exists():26 print(f"FAIL artifact missing for claim '{r['claim'][:60]}': {art}"); ok = False27 for rid in filter(None, [x.strip() for x in r["run_ids"].split(";")]):28 mans = glob.glob(f"experiments/runs/{rid}*/manifest.json") + glob.glob(f"experiments/runs/{rid}/manifest.json")29 if not mans:30 print(f"FAIL run manifest missing for '{r['claim'][:60]}': {rid}"); ok = False; continue31 st = json.load(open(mans[0])).get("state", "")32 if st != "completed" and r.get("status", "") not in ("partial", "stopped-run-used"):33 print(f"FAIL run {rid} state={st} but claim status={r.get('status')}"); ok = False34 nums = Path("paper/tables/numbers.tex").read_text() if Path("paper/tables/numbers.tex").exists() else ""35 defined = set(re.findall(r"\\newcommand\{\\([A-Za-z]+)\}", nums))36 tex = "\n".join(Path(p).read_text() for p in glob.glob("paper/sections/*.tex") + glob.glob("paper/appendix/*.tex"))37 used = set(re.findall(r"\\(params[A-Za-z]+|labels[A-Za-z]+|ctrl[A-Za-z]+|oracle[A-Za-z]+|vtwo[A-Za-z]+|latency[A-Za-z]+|cost[A-Za-z]+|ntrain[A-Za-z]+|pilot[A-Za-z]+|em[A-Z][A-Za-z]+)", tex))38 for u in sorted(used - defined):39 print(f"FAIL macro \\{u} used in manuscript but not generated"); ok = False40 low = tex.lower()41 for w in FORBIDDEN:42 if w in low:43 print(f"FAIL forbidden phrase in manuscript: '{w}'"); ok = False44 # unresolved placeholders45 for p in glob.glob("paper/sections/*.tex") + glob.glob("paper/appendix/*.tex"):46 if "% to be written" in Path(p).read_text() or "TODO" in Path(p).read_text():47 print(f"FAIL placeholder left in {p}"); ok = False48 print("AUDIT", "PASS" if ok else "FAIL")49 return 0 if ok else 1505152if __name__ == "__main__":53 raise SystemExit(main())54