1"""Model sensitivity: arm A3 under claude-opus-5-5 (pilot, R=3) vs claude-sonnet-5-5 and claude-fable-5-1 (stage sensitivity, R=2).2Outputs analysis/tables/table9_model_sensitivity.csv and per-scenario table."""3import os, sys, json4import numpy as np, pandas as pd5REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")); sys.path.insert(0, os.path.join(REPO, "analysis", "econometrics"))6from estimate import load, cluster_bootstrap, permutation_p7def main():8 pilot = load("pilot", 70); sens = load("sensitivity", 70)9 a3 = pilot[pilot.arm == "A3"].copy(); a3["model_tag"] = "opus"10 sens["model_tag"] = sens["model"].map({"claude-sonnet-5-5": "sonnet", "claude-fable-5-1": "fable"})11 df = pd.concat([a3, sens]); rows = []12 for m, g in df.groupby("model_tag"):13 acc = g["S"].sum()14 rows.append({"model": m, "n_runs": len(g), "acceptance_taskw": g.groupby("task_id")["S"].mean().mean(), "det_completion": g["D"].mean(), "critical_error_rate": g["E"].mean(),15 "quality_mean": g["quality"].mean(), "cost_mean": g["C"].mean(), "cost_per_accepted": g["C"].sum() / acc if acc else np.nan, "wall_median_s": g["T"].median(),16 "tool_calls_mean": g["tool_calls"].mean(), "timeout_rate": g["censored"].mean()})17 t = pd.DataFrame(rows); t.to_csv(os.path.join(REPO, "analysis", "tables", "table9_model_sensitivity.csv"), index=False)18 # contrasts vs opus (scenario-cluster bootstrap on S and quality); reuse helpers by relabelling arm := model_tag19 d2 = df.copy(); d2["arm"] = d2["model_tag"]; rng = np.random.default_rng(42); crows = []20 for m in ("sonnet", "fable"):21 for col in ("S", "D", "quality", "C"):22 est, lo, hi, J = cluster_bootstrap(d2, col, m, "opus", 5000, rng); p = permutation_p(d2, col, m, "opus", 20000, rng)23 crows.append({"contrast": f"{m}-opus", "outcome": col, "estimate": est, "ci_lo": lo, "ci_hi": hi, "perm_p": p, "J": J})24 c = pd.DataFrame(crows); c.to_csv(os.path.join(REPO, "analysis", "tables", "table9b_model_contrasts.csv"), index=False)25 per = df.groupby(["task_id", "model_tag"]).agg(acceptance=("S", "mean"), quality=("quality", "mean"), cost=("C", "mean")).reset_index()26 per.to_csv(os.path.join(REPO, "analysis", "tables", "table9c_model_by_scenario.csv"), index=False)27 print(t.round(3).to_string()); print(c.round(3).to_string())28if __name__ == "__main__": main()29