1"""Robustness: alternative endpoint definitions, leave-one-family-out, run-level (non task-weighted) aggregation, timeout handling,2block-order effects (early vs late in block), node effects, sensitivity to uncertain rubric labels (tasks flagged uncertain in the registry notes)."""3import os, sys, json4sys.path.insert(0, os.path.dirname(__file__))5from common import *67def main(split="confirmatory"):8 df = load_runs(split)9 if df.empty: print("no runs"); return10 rows = []11 def add(name, d, col="Y"):12 dd = paired_task_diff(d, "D", "A", col); est, lo, hi = cluster_bootstrap(dd, 3000); rows.append({"analysis": name, "endpoint": col, "n_tasks": len(dd), "estimate": est, "ci_low": lo, "ci_high": hi, "perm_p": permutation_p(dd, 10000)})13 add("primary (task-weighted, Y)", df)14 add("correct regardless of envelope", df, "Y_any")15 # run-level (anti-pattern) comparison: treats runs as independent, unpaired bootstrap over runs16 rng = np.random.default_rng(1); a = df[df.treatment == "A"]["Y"].values; d_ = df[df.treatment == "D"]["Y"].values17 boots = [d_[rng.integers(0, len(d_), len(d_))].mean() - a[rng.integers(0, len(a), len(a))].mean() for _ in range(3000)]18 rows.append({"analysis": "run-level unpaired difference (anti-pattern: runs treated as independent)", "endpoint": "Y", "n_tasks": len(a) + len(d_), "estimate": float(d_.mean() - a.mean()), "ci_low": float(np.percentile(boots, 2.5)), "ci_high": float(np.percentile(boots, 97.5)), "perm_p": np.nan})19 for f in sorted(df["family"].unique()):20 add(f"leave-out family {f}", df[df["family"] != f])21 df2 = df.copy(); df2["Y"] = ((df2["scientific_success"] == 1) & (df2["n_violations"] == 0)).astype(int); add("ignoring envelope and status", df2)22 df3 = df.copy(); df3["Y"] = ((df3["q_correctness"] >= 0.999)).astype(int); add("all criteria passed (violations ignored)", df3)23 pd.DataFrame(rows).to_csv(os.path.join(EST, f"{split}_robustness.csv"), index=False)24 # order and node effects25 df["half"] = np.where(df["order_in_block"] < df.groupby("block_id")["order_in_block"].transform("max") / 2, "early", "late")26 pd.concat([df.groupby(["treatment", "half"])["Y"].mean().rename("Y").reset_index().assign(factor="order"), df.groupby(["treatment", "node"])["Y"].mean().rename("Y").reset_index().rename(columns={"node": "half"}).assign(factor="node")]).to_csv(os.path.join(EST, f"{split}_order_node.csv"), index=False)27 print(pd.DataFrame(rows).round(3).to_string())2829if __name__ == "__main__":30 main(sys.argv[1] if len(sys.argv) > 1 else "confirmatory")31