1"""Robustness: alternative endpoint definitions, leave-one-family-out, run-level (non task-weighted) aggregation, timeout handling,2block-order effects (early vs late in block), node effects, sensitivity to uncertain rubric labels (tasks flagged uncertain in the registry notes)."""3import os, sys, json4sys.path.insert(0, os.path.dirname(__file__))5from common import *67def main(split="confirmatory"):8 df = load_runs(split)9 if df.empty: print("no runs"); return10 rows = []11 def add(name, d, col="Y"):12 dd = paired_task_diff(d, "D", "A", col); est, lo, hi = cluster_bootstrap(dd, 3000); rows.append({"analysis": name, "endpoint": col, "n_tasks": len(dd), "estimate": est, "ci_low": lo, "ci_high": hi, "perm_p": permutation_p(dd, 10000)})13 add("primary (task-weighted, Y)", df)14 add("correct regardless of envelope", df, "Y_any")15 add("run-level unweighted mean difference", df.assign(task_id=df["run_id"])) # treats runs as independent — reported as an anti-pattern comparison16 for f in sorted(df["family"].unique()):17 add(f"leave-out family {f}", df[df["family"] != f])18 df2 = df.copy(); df2["Y"] = ((df2["scientific_success"] == 1) & (df2["n_violations"] == 0)).astype(int); add("ignoring envelope and status", df2)19 df3 = df.copy(); df3["Y"] = ((df3["q_correctness"] >= 0.999)).astype(int); add("all criteria passed (violations ignored)", df3)20 pd.DataFrame(rows).to_csv(os.path.join(EST, f"{split}_robustness.csv"), index=False)21 # order and node effects22 df["half"] = np.where(df["order_in_block"] < df.groupby("block_id")["order_in_block"].transform("max") / 2, "early", "late")23 pd.concat([df.groupby(["treatment", "half"])["Y"].mean().rename("Y").reset_index().assign(factor="order"), df.groupby(["treatment", "node"])["Y"].mean().rename("Y").reset_index().rename(columns={"node": "half"}).assign(factor="node")]).to_csv(os.path.join(EST, f"{split}_order_node.csv"), index=False)24 print(pd.DataFrame(rows).round(3).to_string())2526if __name__ == "__main__":27 main(sys.argv[1] if len(sys.argv) > 1 else "confirmatory")28