1"""Shared helpers for the WP29 analysis pipeline (frozen interfaces)."""2import os, json, numpy as np, pandas as pd3REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))4DATA = os.path.join(REPO, "results", "analysis_data"); EST = os.path.join(REPO, "results", "estimates")5ARMS = ["A", "B", "C", "D"]67def load_runs(split=None, path=None):8 df = pd.read_csv(path or os.path.join(DATA, "runs.csv"))9 if split: df = df[df["split"] == split]10 df = df.copy()11 df["Y"] = ((df["scientific_success"] == 1) & (df["status"].isin(["evaluated"])) & (df["terminal_reason"].fillna("").str.startswith("executor_finished") | df["terminal_reason"].fillna("").str.startswith("result_") | df["terminal_reason"].fillna("").eq("max_turns_reached"))).astype(int)12 # Y requires completion within the envelope: timeout/budget_exhausted → 0 (status is 'evaluated' for all evaluated runs; terminal_reason keeps the cause)13 df.loc[df["terminal_reason"].fillna("").isin(["wall_clock_limit"]) | df["terminal_reason"].fillna("").str.contains("budget", na=False), "Y"] = 014 df["Y_any"] = (df["scientific_success"] == 1).astype(int) # correct regardless of envelope15 df["viol"] = (df["n_violations"].fillna(0) > 0).astype(int)16 df["family"] = df["task_id"].str.split("-").str[0]17 return df1819def task_weighted_rate(df, arm, col="Y"):20 d = df[df["treatment"] == arm]21 if d.empty: return np.nan22 return d.groupby("task_id")[col].mean().mean()2324def paired_task_diff(df, a, b, col="Y"):25 """Task-level paired differences (mean over repeats within task) a − b, as a Series indexed by task."""26 ma = df[df["treatment"] == a].groupby("task_id")[col].mean(); mb = df[df["treatment"] == b].groupby("task_id")[col].mean()27 common = ma.index.intersection(mb.index)28 return (ma.loc[common] - mb.loc[common])2930def cluster_bootstrap(diffs, B=5000, seed=20261030):31 rng = np.random.default_rng(seed); x = diffs.values; n = len(x)32 if n == 0: return (np.nan, np.nan, np.nan)33 boots = np.array([x[rng.integers(0, n, n)].mean() for _ in range(B)])34 return float(x.mean()), float(np.percentile(boots, 2.5)), float(np.percentile(boots, 97.5))3536def permutation_p(diffs, n_perm=20000, seed=20261030):37 rng = np.random.default_rng(seed); x = diffs.values; n = len(x)38 if n == 0: return np.nan39 obs = abs(x.mean()); signs = rng.choice([-1, 1], size=(n_perm, n))40 perm = np.abs((signs * x).mean(axis=1))41 return float((np.sum(perm >= obs - 1e-12) + 1) / (n_perm + 1))4243def holm(pvals: dict):44 items = sorted(pvals.items(), key=lambda kv: kv[1]); m = len(items); out = {}; prev = 0.045 for i, (k, p) in enumerate(items):46 adj = min(1.0, max(prev, (m - i) * p)); out[k] = adj; prev = adj47 return out48