1"""Economic sensitivity (mandate §12/§14 fig. 15) and supervision proxy (fig. 10).2- Cost per accepted deliverable by arm under API price multipliers (0.5x, 1x, 2x, 4x) and with a fully-loaded overhead (controller + judges + node time at an assumed node cost).3- Autonomy-Adjusted Productivity: accepted deliverables per 100 USD of (API cost + wage × human minutes), human minutes = 5 min per unanswered question (assumption) + interventions, wage ∈ {40, 80, 120} CAD/h.4- Human-cost benchmark (ASSUMPTION, not measured): the scenario authors' professional_minutes_estimate × wage, for comparison only.5Outputs table11_economic_sensitivity.csv, table12_supervision.csv, fig15, fig10."""6import os, sys, json7import numpy as np, pandas as pd8import matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt9from matplotlib.ticker import NullFormatter10REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")); sys.path.insert(0, os.path.join(REPO, "analysis", "econometrics")); sys.path.insert(0, os.path.join(REPO, "analysis", "figures"))11from estimate import load12from make_figures import COL, LABEL, ARMS, save13plt.rcParams.update({"font.family": "serif", "font.size": 9, "axes.spines.top": False, "axes.spines.right": False, "axes.grid": True, "grid.color": "#e6e6e6", "grid.linewidth": 0.5, "legend.frameon": False})1415def main():16 df = load("pilot", 70); jd = pd.read_parquet(os.path.join(REPO, "data", "validated", "judgments_pilot.parquet"))17 judge_cost = jd.groupby("run_id")["cost_usd"].sum(); df["judge_cost"] = df["run_id"].map(judge_cost).fillna(0)18 tasks = {t: json.load(open(os.path.join(REPO, "benchmark", "scenarios", t, "task.json"))) for t in df.task_id.unique()}19 df["prof_min"] = df.task_id.map({t: v["professional_minutes_estimate"] for t, v in tasks.items()})20 df["questions"] = df["n_questions"].fillna(0).astype(float); df["H_min"] = df["human_intervention_min"].astype(float) + 5.0 * df["questions"]21 rows = []22 for arm in ARMS:23 d = df[df.arm == arm]; acc = d["S"].sum()24 for mult in (0.5, 1.0, 2.0, 4.0):25 api = d["C"].sum() * mult26 rows.append({"arm": arm, "price_multiplier": mult, "cost_per_accepted_api": api / acc if acc else np.nan,27 "cost_per_accepted_loaded": (api + d["judge_cost"].sum() + 0.05 * d["wall_total_s"].sum() / 3600 * 1.0) / acc if acc else np.nan}) # +1 USD/node-hour assumption28 for wage in (40, 80, 120):29 spend = d["C"].sum() + wage / 60 * d["H_min"].sum()30 rows.append({"arm": arm, "wage_cad_h": wage, "accepted_per_100usd_aap": 100 * acc / spend if spend else np.nan, "mean_H_min": d["H_min"].mean()})31 t11 = pd.DataFrame(rows); t11.to_csv(os.path.join(REPO, "analysis", "tables", "table11_economic_sensitivity_pilot.csv"), index=False)32 sup = df.groupby("arm").agg(n=("S", "size"), abstention_rate=("abstention_status", lambda s: s.isin(["needs_clarification", "abstained"]).mean()), mean_questions=("questions", "mean"),33 justified_abstention_rate=("abstention_justified", lambda s: pd.Series(s).fillna(False).astype(bool).mean()), mean_H_min=("H_min", "mean")).reset_index()34 sup.to_csv(os.path.join(REPO, "analysis", "tables", "table12_supervision_pilot.csv"), index=False)35 # human-cost benchmark (assumption)36 hb = pd.DataFrame([{"task_id": t, "prof_minutes": v["professional_minutes_estimate"], **{f"human_cost_{w}": v["professional_minutes_estimate"] / 60 * w for w in (40, 80, 120)}} for t, v in tasks.items()])37 hb.to_csv(os.path.join(REPO, "analysis", "tables", "table13_human_cost_assumption.csv"), index=False)38 # fig15: cost per accepted deliverable vs price multiplier39 fig, ax = plt.subplots(figsize=(5.0, 3.0))40 for arm in ARMS:41 d = t11[(t11.arm == arm) & t11["price_multiplier"].notna()]42 if d["cost_per_accepted_api"].notna().any(): ax.plot(d["price_multiplier"], d["cost_per_accepted_api"], marker="o", color=COL[arm], label=LABEL[arm], linewidth=1.6)43 ax.set_xscale("log"); ax.set_xticks([0.5, 1, 2, 4]); ax.set_xticklabels(["0.5×", "1×", "2×", "4×"]); ax.xaxis.set_minor_formatter(NullFormatter()); ax.set_xlabel("API price multiplier (relative to November 2026 list prices)"); ax.set_ylabel("marginal API cost per accepted deliverable (USD)")44 ax.axhspan(hb["human_cost_40"].min(), hb["human_cost_120"].max(), color="#c3c2b7", alpha=0.25, lw=0); ax.text(0.52, hb["human_cost_40"].min() * 1.05, "authors' professional-time estimate × 40–120 CAD/h (assumption)", fontsize=6, color="#52514e")45 ax.set_yscale("log"); ax.yaxis.set_minor_formatter(NullFormatter()); ax.legend(fontsize=7); save(fig, "fig15_economic_sensitivity_pilot")46 # fig10: supervision proxy47 fig, ax = plt.subplots(figsize=(4.6, 2.8)); x = np.arange(len(sup))48 ax.bar(x - 0.2, sup["abstention_rate"], width=0.4, color=[COL[a] for a in sup.arm], label="runs flagging questions / abstaining")49 ax.bar(x + 0.2, sup["justified_abstention_rate"], width=0.4, facecolor="none", edgecolor="#444", linewidth=0.9, label="of which justified by the scenario design")50 ax.set_xticks(x); ax.set_xticklabels(sup.arm); ax.set_ylabel("share of runs"); ax.set_ylim(0, 1.05); ax.legend(fontsize=7, loc="upper right"); save(fig, "fig10_supervision_pilot")51 # fig13 (replaces the perturbation figure, not run): model sensitivity on A352 t9 = pd.read_csv(os.path.join(REPO, "analysis", "tables", "table9_model_sensitivity.csv"))53 fig, ax = plt.subplots(figsize=(4.6, 2.8)); cols = {"sonnet": "#2a78d6", "opus": "#eb6834", "fable": "#1baf7a"}54 for _, r in t9.iterrows():55 ax.scatter(r["cost_mean"], r["acceptance_taskw"], s=90, color=cols.get(r["model"], "#444"), edgecolor="#222"); ax.annotate(f"{r['model']}\nQ={r['quality_mean']:.0f}", (r["cost_mean"], r["acceptance_taskw"]), textcoords="offset points", xytext=(6, -14), fontsize=7)56 ax.set_xscale("log"); ax.xaxis.set_minor_formatter(NullFormatter()); ax.set_xlabel("mean marginal API cost per run (USD, log)"); ax.set_ylabel("acceptance rate (arm A3)"); ax.set_ylim(0.5, 1.0); save(fig, "fig13_model_sensitivity")57 print(t11.round(3).to_string()); print(sup.round(3).to_string())58if __name__ == "__main__": main()59