1#!/usr/bin/env python2"""Pre-registered uncertainty analysis: paired item-level bootstrap (10,000 resamples) of exact-match differences3within each seed, and t-intervals over seeds. Writes results/analyzed/statistics_v2.json and paper/tables/stats.tex."""4import glob, json, sys5from pathlib import Path6import numpy as np7sys.path.insert(0, ".")8from evaluation.statistical_analysis import paired_bootstrap, seed_interval910def load(pre):11 out = []12 for run in sorted(glob.glob(f"experiments/runs/{pre}_seed*")):13 if json.load(open(Path(run) / "manifest.json")).get("state") != "completed":14 continue15 recs = {}16 for b in (0, 1, 2, 8):17 p = Path(run) / f"em_dev_iid_b{b}.json"18 if p.exists():19 recs[b] = {r["uid"]: r["correct"] for r in json.load(open(p))}20 out.append((Path(run).name, recs))21 return out2223res = {}24systems = {"prisme": load("p4v2_prisme"), "gr": load("p4v2_gr"), "dense": load("p4v2_dense")}25for name, runs in systems.items():26 for b in (1, 2, 8):27 diffs, boots = [], []28 for rid, recs in runs:29 if b in recs and 0 in recs:30 common = sorted(set(recs[0]) & set(recs[b]))31 a = np.array([recs[b][u] for u in common]); c = np.array([recs[0][u] for u in common])32 bs = paired_bootstrap(a, c); bs["run_id"] = rid; boots.append(bs); diffs.append(bs["diff"])33 if diffs:34 res[f"{name}_b{b}_vs_b0"] = {"per_seed": boots, "seeds": seed_interval(diffs)}35# PRISME vs GR at the same budget (paired by item, per seed index)36for b in (1, 2, 8):37 diffs, boots = [], []38 for (rp, rec_p), (rg, rec_g) in zip(systems["prisme"], systems["gr"]):39 if b in rec_p and b in rec_g:40 common = sorted(set(rec_p[b]) & set(rec_g[b]))41 a = np.array([rec_p[b][u] for u in common]); c = np.array([rec_g[b][u] for u in common])42 bs = paired_bootstrap(a, c); bs["run_ids"] = [rp, rg]; boots.append(bs); diffs.append(bs["diff"])43 if diffs:44 res[f"prisme_minus_gr_b{b}"] = {"per_seed": boots, "seeds": seed_interval(diffs)}45Path("results/analyzed").mkdir(exist_ok=True, parents=True)46Path("results/analyzed/statistics_v2.json").write_text(json.dumps(res, indent=1))47rows = []48for k, v in res.items():49 s = v["seeds"]; lo = min(b["ci_low"] for b in v["per_seed"]); hi = max(b["ci_high"] for b in v["per_seed"])50 rows.append(f"{k.replace('_', ' ')} & {s['n']} & {100*s['mean']:+.2f} & [{100*s['ci_low']:+.2f}, {100*s['ci_high']:+.2f}] & [{100*lo:+.2f}, {100*hi:+.2f}]\\\\")51 print(k, "seeds", s["n"], f"mean {100*s['mean']:+.2f} pp, t-CI [{100*s['ci_low']:+.2f}, {100*s['ci_high']:+.2f}]")52Path("paper/tables/stats.tex").write_text(r"""\begin{table}[t]\centering\small53\caption{Pre-registered uncertainty analysis of dev-iid exact match (percentage points). ``t-CI'' is the 95\,\%54$t$-interval over independent adapter seeds of the per-seed paired difference; ``item CI range'' spans the55smallest lower and largest upper bound of the per-seed paired item bootstraps (10,000 resamples, 1,000 items).}56\label{tab:stats}57\begin{tabular}{lrrll}\toprule58Comparison & seeds & mean & t-CI over seeds & item CI range\\\midrule59""" + "\n".join(rows) + "\n" + r"\bottomrule\end{tabular}\end{table}" + "\n")60