"""Robustness: alternative endpoint definitions, leave-one-family-out, run-level (non task-weighted) aggregation, timeout handling,
block-order effects (early vs late in block), node effects, sensitivity to uncertain rubric labels (tasks flagged uncertain in the registry notes)."""
import os, sys, json
sys.path.insert(0, os.path.dirname(__file__))
from common import *

def main(split="confirmatory"):
    df = load_runs(split)
    if df.empty: print("no runs"); return
    rows = []
    def add(name, d, col="Y"):
        dd = paired_task_diff(d, "D", "A", col); est, lo, hi = cluster_bootstrap(dd, 3000); rows.append({"analysis": name, "endpoint": col, "n_tasks": len(dd), "estimate": est, "ci_low": lo, "ci_high": hi, "perm_p": permutation_p(dd, 10000)})
    add("primary (task-weighted, Y)", df)
    add("correct regardless of envelope", df, "Y_any")
    add("run-level unweighted mean difference", df.assign(task_id=df["run_id"]))   # treats runs as independent — reported as an anti-pattern comparison
    for f in sorted(df["family"].unique()):
        add(f"leave-out family {f}", df[df["family"] != f])
    df2 = df.copy(); df2["Y"] = ((df2["scientific_success"] == 1) & (df2["n_violations"] == 0)).astype(int); add("ignoring envelope and status", df2)
    df3 = df.copy(); df3["Y"] = ((df3["q_correctness"] >= 0.999)).astype(int); add("all criteria passed (violations ignored)", df3)
    pd.DataFrame(rows).to_csv(os.path.join(EST, f"{split}_robustness.csv"), index=False)
    # order and node effects
    df["half"] = np.where(df["order_in_block"] < df.groupby("block_id")["order_in_block"].transform("max") / 2, "early", "late")
    pd.concat([df.groupby(["treatment", "half"])["Y"].mean().rename("Y").reset_index().assign(factor="order"), df.groupby(["treatment", "node"])["Y"].mean().rename("Y").reset_index().rename(columns={"node": "half"}).assign(factor="node")]).to_csv(os.path.join(EST, f"{split}_order_node.csv"), index=False)
    print(pd.DataFrame(rows).round(3).to_string())

if __name__ == "__main__":
    main(sys.argv[1] if len(sys.argv) > 1 else "confirmatory")