"""Paired bootstrap confidence intervals at the item level and seed-level t-intervals.""" from __future__ import annotations import numpy as np def paired_bootstrap(a: np.ndarray, b: np.ndarray, n_boot: int = 10000, seed: int = 0, alpha: float = 0.05): """a, b: per-item outcomes (e.g. 0/1 correctness) for two systems on the SAME items. Returns mean diff + CI.""" assert a.shape == b.shape rng = np.random.default_rng(seed) d = a.astype(float) - b.astype(float) n = len(d) idx = rng.integers(0, n, size=(n_boot, n)) boots = d[idx].mean(1) lo, hi = np.percentile(boots, [100 * alpha / 2, 100 * (1 - alpha / 2)]) return {"diff": float(d.mean()), "ci_low": float(lo), "ci_high": float(hi), "n": int(n), "p_two_sided": float(2 * min((boots <= 0).mean(), (boots >= 0).mean()))} def seed_interval(values: list[float], alpha: float = 0.05): """t-interval over independent seeds (values = per-seed statistic).""" from scipy import stats v = np.asarray(values, dtype=float) n = len(v) if n < 2: return {"mean": float(v.mean()) if n else float("nan"), "sd": float("nan"), "ci_low": float("nan"), "ci_high": float("nan"), "n": n} m, sd = v.mean(), v.std(ddof=1) h = stats.t.ppf(1 - alpha / 2, n - 1) * sd / np.sqrt(n) return {"mean": float(m), "sd": float(sd), "ci_low": float(m - h), "ci_high": float(m + h), "n": n}