1"""Shared helpers for the WP29 analysis pipeline (frozen interfaces)."""2import os, json, numpy as np, pandas as pd3REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))4DATA = os.path.join(REPO, "results", "analysis_data"); EST = os.path.join(REPO, "results", "estimates")5ARMS = ["A", "B", "C", "D"]67def load_runs(split=None, path=None, include_postfix=False):8 df = pd.read_csv(path or os.path.join(DATA, "runs.csv"))9 if split: df = df[df["split"] == split]10 df = df.copy()11 if "notes" in df.columns:12 df["postfix_verification"] = df["notes"].fillna("").str.contains("seed 102|seed 402|seed 403|seed 802")13 if not include_postfix: df = df[~df["postfix_verification"]]14 no_intervention = df["human_interventions"].fillna("").eq("") if "human_interventions" in df.columns else True15 df["Y"] = ((df["scientific_success"] == 1) & (df["status"].isin(["evaluated"])) & (df["terminal_reason"].fillna("").str.startswith("executor_finished")) & no_intervention).astype(int) # max_turns_reached and result_* are outside the envelope16 # Y requires completion within the envelope: timeout/budget_exhausted → 0 (status is 'evaluated' for all evaluated runs; terminal_reason keeps the cause)17 df.loc[df["terminal_reason"].fillna("").isin(["wall_clock_limit"]) | df["terminal_reason"].fillna("").str.contains("budget", na=False), "Y"] = 018 df["Y_any"] = (df["scientific_success"] == 1).astype(int) # correct regardless of envelope19 df["viol"] = (df["n_violations"].fillna(0) > 0).astype(int)20 df["family"] = df["task_id"].str.split("-").str[0]21 return df2223def task_weighted_rate(df, arm, col="Y"):24 d = df[df["treatment"] == arm]25 if d.empty: return np.nan26 return d.groupby("task_id")[col].mean().mean()2728def paired_task_diff(df, a, b, col="Y"):29 """Task-level paired differences (mean over repeats within task) a − b, as a Series indexed by task."""30 ma = df[df["treatment"] == a].groupby("task_id")[col].mean(); mb = df[df["treatment"] == b].groupby("task_id")[col].mean()31 common = ma.index.intersection(mb.index)32 return (ma.loc[common] - mb.loc[common])3334def cluster_bootstrap(diffs, B=5000, seed=20261030):35 rng = np.random.default_rng(seed); x = diffs.values; n = len(x)36 if n == 0: return (np.nan, np.nan, np.nan)37 boots = np.array([x[rng.integers(0, n, n)].mean() for _ in range(B)])38 return float(x.mean()), float(np.percentile(boots, 2.5)), float(np.percentile(boots, 97.5))3940def permutation_p(diffs, n_perm=20000, seed=20261030):41 rng = np.random.default_rng(seed); x = diffs.values; n = len(x)42 if n == 0: return np.nan43 obs = abs(x.mean()); signs = rng.choice([-1, 1], size=(n_perm, n))44 perm = np.abs((signs * x).mean(axis=1))45 return float((np.sum(perm >= obs - 1e-12) + 1) / (n_perm + 1))4647def holm(pvals: dict):48 items = sorted(pvals.items(), key=lambda kv: kv[1]); m = len(items); out = {}; prev = 0.049 for i, (k, p) in enumerate(items):50 adj = min(1.0, max(prev, (m - i) * p)); out[k] = adj; prev = adj51 return out52