1"""Paired bootstrap confidence intervals at the item level and seed-level t-intervals."""2from __future__ import annotations34import numpy as np567def paired_bootstrap(a: np.ndarray, b: np.ndarray, n_boot: int = 10000, seed: int = 0, alpha: float = 0.05):8 """a, b: per-item outcomes (e.g. 0/1 correctness) for two systems on the SAME items. Returns mean diff + CI."""9 assert a.shape == b.shape10 rng = np.random.default_rng(seed)11 d = a.astype(float) - b.astype(float)12 n = len(d)13 idx = rng.integers(0, n, size=(n_boot, n))14 boots = d[idx].mean(1)15 lo, hi = np.percentile(boots, [100 * alpha / 2, 100 * (1 - alpha / 2)])16 return {"diff": float(d.mean()), "ci_low": float(lo), "ci_high": float(hi), "n": int(n), "p_two_sided": float(2 * min((boots <= 0).mean(), (boots >= 0).mean()))}171819def seed_interval(values: list[float], alpha: float = 0.05):20 """t-interval over independent seeds (values = per-seed statistic)."""21 from scipy import stats22 v = np.asarray(values, dtype=float)23 n = len(v)24 if n < 2:25 return {"mean": float(v.mean()) if n else float("nan"), "sd": float("nan"), "ci_low": float("nan"), "ci_high": float("nan"), "n": n}26 m, sd = v.mean(), v.std(ddof=1)27 h = stats.t.ppf(1 - alpha / 2, n - 1) * sd / np.sqrt(n)28 return {"mean": float(m), "sd": float(sd), "ci_low": float(m - h), "ci_high": float(m + h), "n": n}29