#!/usr/bin/env python
"""Pre-registered uncertainty analysis: paired item-level bootstrap (10,000 resamples) of exact-match differences
within each seed, and t-intervals over seeds. Writes results/analyzed/statistics_v2.json and paper/tables/stats.tex."""
import glob, json, sys
from pathlib import Path
import numpy as np
sys.path.insert(0, ".")
from evaluation.statistical_analysis import paired_bootstrap, seed_interval

def load(pre):
    out = []
    for run in sorted(glob.glob(f"experiments/runs/{pre}_seed*")):
        if json.load(open(Path(run) / "manifest.json")).get("state") != "completed":
            continue
        recs = {}
        for b in (0, 1, 2, 8):
            p = Path(run) / f"em_dev_iid_b{b}.json"
            if p.exists():
                recs[b] = {r["uid"]: r["correct"] for r in json.load(open(p))}
        out.append((Path(run).name, recs))
    return out

res = {}
systems = {"prisme": load("p4v2_prisme"), "gr": load("p4v2_gr"), "dense": load("p4v2_dense")}
for name, runs in systems.items():
    for b in (1, 2, 8):
        diffs, boots = [], []
        for rid, recs in runs:
            if b in recs and 0 in recs:
                common = sorted(set(recs[0]) & set(recs[b]))
                a = np.array([recs[b][u] for u in common]); c = np.array([recs[0][u] for u in common])
                bs = paired_bootstrap(a, c); bs["run_id"] = rid; boots.append(bs); diffs.append(bs["diff"])
        if diffs:
            res[f"{name}_b{b}_vs_b0"] = {"per_seed": boots, "seeds": seed_interval(diffs)}
# PRISME vs GR at the same budget (paired by item, per seed index)
for b in (1, 2, 8):
    diffs, boots = [], []
    for (rp, rec_p), (rg, rec_g) in zip(systems["prisme"], systems["gr"]):
        if b in rec_p and b in rec_g:
            common = sorted(set(rec_p[b]) & set(rec_g[b]))
            a = np.array([rec_p[b][u] for u in common]); c = np.array([rec_g[b][u] for u in common])
            bs = paired_bootstrap(a, c); bs["run_ids"] = [rp, rg]; boots.append(bs); diffs.append(bs["diff"])
    if diffs:
        res[f"prisme_minus_gr_b{b}"] = {"per_seed": boots, "seeds": seed_interval(diffs)}
Path("results/analyzed").mkdir(exist_ok=True, parents=True)
Path("results/analyzed/statistics_v2.json").write_text(json.dumps(res, indent=1))
rows = []
for k, v in res.items():
    s = v["seeds"]; lo = min(b["ci_low"] for b in v["per_seed"]); hi = max(b["ci_high"] for b in v["per_seed"])
    rows.append(f"{k.replace('_', ' ')} & {s['n']} & {100*s['mean']:+.2f} & [{100*s['ci_low']:+.2f}, {100*s['ci_high']:+.2f}] & [{100*lo:+.2f}, {100*hi:+.2f}]\\\\")
    print(k, "seeds", s["n"], f"mean {100*s['mean']:+.2f} pp, t-CI [{100*s['ci_low']:+.2f}, {100*s['ci_high']:+.2f}]")
Path("paper/tables/stats.tex").write_text(r"""\begin{table}[t]\centering\small
\caption{Pre-registered uncertainty analysis of dev-iid exact match (percentage points). ``t-CI'' is the 95\,\%
$t$-interval over independent adapter seeds of the per-seed paired difference; ``item CI range'' spans the
smallest lower and largest upper bound of the per-seed paired item bootstraps (10,000 resamples, 1,000 items).}
\label{tab:stats}
\begin{tabular}{lrrll}\toprule
Comparison & seeds & mean & t-CI over seeds & item CI range\\\midrule
""" + "\n".join(rows) + "\n" + r"\bottomrule\end{tabular}\end{table}" + "\n")