"""Model sensitivity: arm A3 under claude-opus-5-5 (pilot, R=3) vs claude-sonnet-5-5 and claude-fable-5-1 (stage sensitivity, R=2).
Outputs analysis/tables/table9_model_sensitivity.csv and per-scenario table."""
import os, sys, json
import numpy as np, pandas as pd
REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")); sys.path.insert(0, os.path.join(REPO, "analysis", "econometrics"))
from estimate import load, cluster_bootstrap, permutation_p
def main():
    pilot = load("pilot", 70); sens = load("sensitivity", 70)
    a3 = pilot[pilot.arm == "A3"].copy(); a3["model_tag"] = "opus"
    sens["model_tag"] = sens["model"].map({"claude-sonnet-5-5": "sonnet", "claude-fable-5-1": "fable"})
    df = pd.concat([a3, sens]); rows = []
    for m, g in df.groupby("model_tag"):
        acc = g["S"].sum()
        rows.append({"model": m, "n_runs": len(g), "acceptance_taskw": g.groupby("task_id")["S"].mean().mean(), "det_completion": g["D"].mean(), "critical_error_rate": g["E"].mean(),
                     "quality_mean": g["quality"].mean(), "cost_mean": g["C"].mean(), "cost_per_accepted": g["C"].sum() / acc if acc else np.nan, "wall_median_s": g["T"].median(),
                     "tool_calls_mean": g["tool_calls"].mean(), "timeout_rate": g["censored"].mean()})
    t = pd.DataFrame(rows); t.to_csv(os.path.join(REPO, "analysis", "tables", "table9_model_sensitivity.csv"), index=False)
    # contrasts vs opus (scenario-cluster bootstrap on S and quality); reuse helpers by relabelling arm := model_tag
    d2 = df.copy(); d2["arm"] = d2["model_tag"]; rng = np.random.default_rng(42); crows = []
    for m in ("sonnet", "fable"):
        for col in ("S", "D", "quality", "C"):
            est, lo, hi, J = cluster_bootstrap(d2, col, m, "opus", 5000, rng); p = permutation_p(d2, col, m, "opus", 20000, rng)
            crows.append({"contrast": f"{m}-opus", "outcome": col, "estimate": est, "ci_lo": lo, "ci_hi": hi, "perm_p": p, "J": J})
    c = pd.DataFrame(crows); c.to_csv(os.path.join(REPO, "analysis", "tables", "table9b_model_contrasts.csv"), index=False)
    per = df.groupby(["task_id", "model_tag"]).agg(acceptance=("S", "mean"), quality=("quality", "mean"), cost=("C", "mean")).reset_index()
    per.to_csv(os.path.join(REPO, "analysis", "tables", "table9c_model_by_scenario.csv"), index=False)
    print(t.round(3).to_string()); print(c.round(3).to_string())
if __name__ == "__main__": main()