1"""Shared helpers for the WP29 analysis pipeline (frozen interfaces)."""2import os, json, numpy as np, pandas as pd3REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))4DATA = os.path.join(REPO, "results", "analysis_data"); EST = os.path.join(REPO, "results", "estimates")5ARMS = ["A", "B", "C", "D"]67def load_runs(split=None, path=None, include_postfix=False):8 df = pd.read_csv(path or os.path.join(DATA, "runs.csv"))9 if split: df = df[df["split"] == split]10 df = df.copy()11 if "notes" in df.columns:12 df["postfix_verification"] = df["notes"].fillna("").str.contains("seed 102|seed 402")13 if not include_postfix: df = df[~df["postfix_verification"]]14 df["Y"] = ((df["scientific_success"] == 1) & (df["status"].isin(["evaluated"])) & (df["terminal_reason"].fillna("").str.startswith("executor_finished") | df["terminal_reason"].fillna("").str.startswith("result_") | df["terminal_reason"].fillna("").eq("max_turns_reached"))).astype(int)15 # Y requires completion within the envelope: timeout/budget_exhausted → 0 (status is 'evaluated' for all evaluated runs; terminal_reason keeps the cause)16 df.loc[df["terminal_reason"].fillna("").isin(["wall_clock_limit"]) | df["terminal_reason"].fillna("").str.contains("budget", na=False), "Y"] = 017 df["Y_any"] = (df["scientific_success"] == 1).astype(int) # correct regardless of envelope18 df["viol"] = (df["n_violations"].fillna(0) > 0).astype(int)19 df["family"] = df["task_id"].str.split("-").str[0]20 return df2122def task_weighted_rate(df, arm, col="Y"):23 d = df[df["treatment"] == arm]24 if d.empty: return np.nan25 return d.groupby("task_id")[col].mean().mean()2627def paired_task_diff(df, a, b, col="Y"):28 """Task-level paired differences (mean over repeats within task) a − b, as a Series indexed by task."""29 ma = df[df["treatment"] == a].groupby("task_id")[col].mean(); mb = df[df["treatment"] == b].groupby("task_id")[col].mean()30 common = ma.index.intersection(mb.index)31 return (ma.loc[common] - mb.loc[common])3233def cluster_bootstrap(diffs, B=5000, seed=20261030):34 rng = np.random.default_rng(seed); x = diffs.values; n = len(x)35 if n == 0: return (np.nan, np.nan, np.nan)36 boots = np.array([x[rng.integers(0, n, n)].mean() for _ in range(B)])37 return float(x.mean()), float(np.percentile(boots, 2.5)), float(np.percentile(boots, 97.5))3839def permutation_p(diffs, n_perm=20000, seed=20261030):40 rng = np.random.default_rng(seed); x = diffs.values; n = len(x)41 if n == 0: return np.nan42 obs = abs(x.mean()); signs = rng.choice([-1, 1], size=(n_perm, n))43 perm = np.abs((signs * x).mean(axis=1))44 return float((np.sum(perm >= obs - 1e-12) + 1) / (n_perm + 1))4546def holm(pvals: dict):47 items = sorted(pvals.items(), key=lambda kv: kv[1]); m = len(items); out = {}; prev = 0.048 for i, (k, p) in enumerate(items):49 adj = min(1.0, max(prev, (m - i) * p)); out[k] = adj; prev = adj50 return out51