spb/wpai0002_uqo

1#!/usr/bin/env python32"""Freeze the pre-registration: compute the corpus/splits/pairs hashes, write §6 of docs/PREREGISTRATION.md, commit,3then record the freeze commit hash in the document and in docs/DECISION_LOG.md with a second commit.45Usage: .venv/bin/python scripts/freeze_preregistration.py --corpus data/parsed/pilot_pmc --splits data/splits/pilot_pmc/splits.jsonl \6          --pairs data/pairs/pilot_opus/validated/pairs.jsonl --version 1.0 [--no-commit]7"""8from __future__ import annotations910import argparse11import datetime as dt12import hashlib13import re14import subprocess15import sys16from pathlib import Path1718ROOT = Path(__file__).resolve().parents[1]192021def sha(p: Path) -> str:22    return hashlib.sha256(p.read_bytes()).hexdigest()232425def git(*args: str) -> str:26    return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()272829def main() -> None:30    ap = argparse.ArgumentParser()31    ap.add_argument("--corpus", required=True)32    ap.add_argument("--splits", required=True)33    ap.add_argument("--pairs", required=True)34    ap.add_argument("--heldout-pairs", nargs="*", default=[])35    ap.add_argument("--version", default="1.0")36    ap.add_argument("--mstar-rule", default="the candidate run (same size, data and step budget) with the lowest validation token cross-entropy, recorded with its checkpoint sha256 before any confirmatory scoring")37    ap.add_argument("--decoding", default="greedy, KV-cached, placeholder protocol of the pair's protect_version, max_new_tokens = 1.6 x input + 16")38    ap.add_argument("--no-commit", action="store_true")39    a = ap.parse_args()40    corpus, splits, pairs = Path(a.corpus), Path(a.splits), Path(a.pairs)41    now = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d %H:%M UTC")42    hashes = {43        "corpus build_summary.json": sha(corpus / "build_summary.json"),44        "documents.jsonl": sha(corpus / "documents.jsonl"),45        "splits.jsonl": sha(splits),46        "validated pairs.jsonl": sha(pairs),47        **{f"held-out pairs {Path(h).parent.parent.name}": sha(Path(h)) for h in a.heldout_pairs},48    }49    doc = ROOT / "docs/PREREGISTRATION.md"50    text = doc.read_text()51    text = text.replace("# Pre-registration — WP-AI-0002 (v0.1, DRAFT, not frozen)", f"# Pre-registration — WP-AI-0002 (v{a.version}, FROZEN {now})")52    text = re.sub(r"Status: \*\*draft v0\.1, 2026-11-09\*\*\. The SESOI values below are proposals and need the author's confirmation\.",53                  f"Status: **v{a.version}, frozen {now}, pre-specified in-session by the operating assistant** (the SESOI values are the v0.1 proposals; the author had not yet confirmed them at freeze time — this is stated in the paper as *pre-specified in-session*, not as author-reviewed pre-registration).", text)54    record = (55        f"## 6. Freeze record (v{a.version})\n\n"56        f"- Date: {now}\n- M\\* selection rule: {a.mstar_rule}\n- Decoding: {a.decoding}\n"57        f"- Confirmatory bootstrap: 10 000 resamples, seed 0 (`scripts/evaluate.py --n-boot 10000 --confirmatory`)\n"58        f"- Corpus: `{corpus}` · splits: `{splits}` · pairs: `{pairs}`\n"59        + "".join(f"- sha256 {k}: `{v}`\n" for k, v in hashes.items())60        + "- Held-out sets: `test`, `heldout_domain` (chemistry as labelled at split time), `heldout_period` (pub_year < 1950; 4 passages only), held-out prompts H1/H2, held-out models Sonnet 5.5 / Haiku 5.5\n"61        "- Commit hash: pending (filled by the second commit of scripts/freeze_preregistration.py)\n"62    )63    text = re.sub(r"## 6\. Freeze record.*\Z", record, text, flags=re.S)64    doc.write_text(text)65    if a.no_commit:66        print(text[-1500:])67        return68    git("add", "docs/PREREGISTRATION.md")69    git("commit", "-q", "-m", f"PREREGISTRATION v{a.version} freeze (pre-specified in-session)\n\nCo-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>")70    h = git("rev-parse", "HEAD")71    text = doc.read_text().replace("- Commit hash: pending (filled by the second commit of scripts/freeze_preregistration.py)", f"- Commit hash: `{h}` (the freeze commit; this line was added by the following commit)")72    doc.write_text(text)73    log = ROOT / "docs/DECISION_LOG.md"74    log.write_text(log.read_text().rstrip() + f"\n\n## D-FREEZE — {now}\n\nPREREGISTRATION v{a.version} frozen at commit `{h}` (pre-specified in-session; SESOIs = v0.1 proposals, author confirmation pending). No confirmatory split had been scored with a model before this commit; only B0/B1 sanity checks on the smoke corpus.\n")75    git("add", "docs/PREREGISTRATION.md", "docs/DECISION_LOG.md")76    git("commit", "-q", "-m", f"Record PREREGISTRATION v{a.version} freeze commit {h[:12]}\n\nCo-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>")77    print(f"frozen at {h}")787980if __name__ == "__main__":81    main()82