1"""Extra descriptives for the manuscript's empirical sections (Agent H). Everything is recomputed from data/validated/*.csv|parquet,2analysis/tables/*.csv and the archived first attempts of amendment A-010; nothing is typed by hand.3Output: analysis/tables/table14_extra_descriptives.csv (long format: block, key, arm/group, metric, value) and a printed summary.4Usage: .venv/bin/python analysis/robustness/extra_descriptives.py"""5from __future__ import annotations6import json, os, sys7import numpy as np, pandas as pd8REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))9VAL = os.path.join(REPO, "data", "validated"); TAB = os.path.join(REPO, "analysis", "tables"); RAW = os.path.join(REPO, "data", "raw", "runs")10sys.path.insert(0, os.path.join(REPO, "analysis", "econometrics"))11from estimate import load, cluster_bootstrap, permutation_p, weighted_rate, ARMS12ROWS = []1314def add(block, key, group, metric, value):15 ROWS.append({"block": block, "key": key, "group": group, "metric": metric, "value": value})1617def tokens(df):18 # same supervision proxy as estimate.py main(): 5 min per unanswered question (assumption) + logged interventions19 df["H_min"] = df["human_intervention_min"].astype(float) + 5.0 * df["n_questions"].fillna(0).astype(float)20 tok = df["tokens"].apply(lambda x: json.loads(x) if isinstance(x, str) else (x or {}))21 for k in ("input", "output", "cache_read", "cache_create"):22 df["tok_" + k] = tok.apply(lambda t: t.get(k, 0)).astype(float)23 df["tok_total"] = df[["tok_input", "tok_output", "tok_cache_read", "tok_cache_create"]].sum(axis=1)24 return df2526def main():27 pilot = tokens(load("pilot", 70)); sens = tokens(load("sensitivity", 70)); rec = tokens(load("recovery", 70))28 # ---- A. process descriptives by arm (pilot)29 for arm, d in pilot.groupby("arm"):30 for m in ("tok_input", "tok_output", "tok_cache_read", "tok_cache_create", "tok_total", "num_turns", "tool_time_s", "first_tool_s", "retries", "n_artifacts"):31 add("A_process", "mean", arm, m, float(d[m].mean()))32 for qn, qv in (("p10", 0.10), ("p50", 0.50), ("p90", 0.90), ("max", 1.0), ("min", 0.0)):33 add("A_process", qn, arm, "wall_s", float(d["T"].quantile(qv))); add("A_process", qn, arm, "cost_usd", float(d["C"].quantile(qv)))34 add("A_process", qn, arm, "tool_calls", float(d["tool_calls"].quantile(qv))); add("A_process", qn, arm, "quality", float(d["quality"].quantile(qv)))35 add("A_process", "sd", arm, "wall_s", float(d["T"].std())); add("A_process", "n", arm, "runs", int(len(d)))36 # ---- B. node × arm (Assumption: no node effect) — runs and acceptance37 for (node, arm), d in pilot.groupby(["node", "arm"]):38 add("B_node", node, arm, "n", int(len(d))); add("B_node", node, arm, "acceptance", float(d["S"].mean())); add("B_node", node, arm, "quality", float(d["quality"].mean()))39 for node, d in pilot.groupby("node"):40 add("B_node", node, "all", "n", int(len(d))); add("B_node", node, "all", "acceptance", float(d["S"].mean()))41 dd = d[d.arm != "A0"]; add("B_node", node, "A1-A4", "acceptance", float(dd["S"].mean())); add("B_node", node, "A1-A4", "n", int(len(dd)))42 # ---- C. questions / supervision proxy43 for arm, d in pilot.groupby("arm"):44 nq = d["n_questions"].fillna(0).astype(float)45 add("C_questions", "share_zero_questions", arm, "share", float((nq == 0).mean())); add("C_questions", "share_le_3", arm, "share", float((nq <= 3).mean()))46 add("C_questions", "mean", arm, "n_questions", float(nq.mean())); add("C_questions", "median", arm, "n_questions", float(nq.median())); add("C_questions", "max", arm, "n_questions", float(nq.max()))47 add("C_questions", "mean", arm, "H_min", float(d["H_min"].mean()))48 # ---- D. ALF with the human-time allowance unbounded (budget-only frontier) and pooled over occupations49 budgets = [0.25, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 4.0]50 for arm in ARMS:51 g = pilot[pilot.arm == arm]52 for B in budgets:53 ok = ((g["S"] == 1) & (g["C"] <= B)).groupby(g["task_id"]).mean()54 add("D_alf_Hinf", f"B={B}", arm, "ALF_pooled_equal", float(ok.mean()))55 for H in (0.0, 15.0, 30.0):56 okh = ((g["S"] == 1) & (g["C"] <= B) & (g["H_min"] <= H)).groupby(g["task_id"]).mean()57 add("D_alf_H", f"B={B};H={H:g}", arm, "ALF_pooled_equal", float(okh.mean()))58 for occ, go in g.groupby("occupation"):59 for B in (1.0, 2.0, 4.0):60 ok = ((go["S"] == 1) & (go["C"] <= B)).groupby(go["task_id"]).mean()61 add("D_alf_Hinf_occ", f"B={B}", f"{occ}|{arm}", "ALF", float(ok.mean()))62 # ---- E. A-008 sensitivity: the 8 re-judged runs (RES-01, arms A1–A4, repeats 1–2, judged after the sanitisation fix)63 rejudged = [f"RES-01_{a}_r{r}" for a in ("A1", "A2", "A3", "A4") for r in (1, 2)]64 jd = pd.read_parquet(os.path.join(VAL, "judgments_pilot.parquet")); first = jd.groupby("run_id")["judged_utc"].min()65 add("E_rejudged", "n_rejudged_runs_present", "pilot", "n", int(sum(r in first.index for r in rejudged)))66 for r in rejudged: add("E_rejudged", r, "pilot", "first_judged_utc", str(first.get(r)))67 rng = np.random.default_rng(42)68 for label, dsub in (("full", pilot), ("excl_8_rejudged", pilot[~pilot.run_id.isin(rejudged)]), ("excl_RES-01", pilot[pilot.task_id != "RES-01"])):69 for arm in ARMS: add("E_rejudged", label, arm, "acceptance_taskw", weighted_rate(dsub, "S", arm))70 for x, y in (("A3", "A1"), ("A1", "A0"), ("A4", "A3"), ("A2", "A1")):71 est, lo, hi, J = cluster_bootstrap(dsub, "S", x, y, 5000, rng); p = permutation_p(dsub, "S", x, y, 20000, rng)72 add("E_rejudged", label, f"{x}-{y}", "estimate", est); add("E_rejudged", label, f"{x}-{y}", "ci_lo", lo); add("E_rejudged", label, f"{x}-{y}", "ci_hi", hi); add("E_rejudged", label, f"{x}-{y}", "perm_p", p); add("E_rejudged", label, f"{x}-{y}", "J", J)73 res = pilot[pilot.task_id == "RES-01"]74 for _, r in res.iterrows(): add("E_rejudged", r["run_id"], r["arm"], "quality", float(r["quality"])); add("E_rejudged", r["run_id"], r["arm"], "S", float(r["S"]))75 # ---- F. A-010: the three lost collections (first attempts archived as *.attempt1) and their reruns76 for rid in ("RE-01_A2_r1", "RE-01_A2_r2", "LEGAL-01_A2_r1"):77 p1 = os.path.join(RAW, rid, "raw_stream.jsonl.attempt1")78 ev = [json.loads(l) for l in open(p1) if l.strip()]79 ntool = sum(1 for e in ev if e.get("type") == "assistant" for c in (e.get("message", {}).get("content") or []) if isinstance(c, dict) and c.get("type") == "tool_use")80 resv = [e for e in ev if e.get("type") == "result"]81 add("F_a010", rid, "attempt1", "events", len(ev)); add("F_a010", rid, "attempt1", "tool_use", ntool)82 add("F_a010", rid, "attempt1", "result_subtype", resv[0].get("subtype") if resv else None); add("F_a010", rid, "attempt1", "cost_usd", float(resv[0].get("total_cost_usd")) if resv else None)83 add("F_a010", rid, "attempt1", "num_turns", resv[0].get("num_turns") if resv else None); add("F_a010", rid, "attempt1", "duration_s", float(resv[0].get("duration_ms")) / 1000 if resv else None)84 rc1 = json.load(open(os.path.join(RAW, rid, "receipt.json.attempt1"))); add("F_a010", rid, "attempt1", "det_completion", bool(rc1.get("det_completion"))); add("F_a010", rid, "attempt1", "det_score", float(rc1.get("det_score")))85 j1 = json.load(open(os.path.join(RAW, rid, "judgments.json.attempt1")))86 add("F_a010", rid, "attempt1", "quality_mean", float(np.mean([x.get("score") for x in j1])) if isinstance(j1, list) and j1 else None)87 r2 = pilot[pilot.run_id == rid].iloc[0]88 for m in ("D", "quality", "S", "C", "tool_calls", "T", "node"): add("F_a010", rid, "rerun", m, r2[m] if m == "node" else float(r2[m]))89 # ---- G. recovery stage: cost imputation for the interrupted first sessions (no result event → tokens × fitted prices)90 ref = pd.read_parquet(os.path.join(VAL, "runs_pilot.parquet")); ref = ref[ref["cost_usd"].fillna(0) > 0]91 rtok = ref["tokens"].apply(lambda x: json.loads(x) if isinstance(x, str) else (x or {}))92 RX = np.column_stack([rtok.apply(lambda t: t.get(k, 0)).astype(float).values for k in ("input", "output", "cache_read", "cache_create")])93 beta, *_ = np.linalg.lstsq(RX, ref["cost_usd"].values, rcond=None)94 for k, b in zip(("input", "output", "cache_read", "cache_create"), beta): add("G_recovery_cost", "fitted_price_per_Mtoken_usd", "claude-opus-5-5", k, float(b * 1e6))95 pred = RX @ beta; add("G_recovery_cost", "fit", "claude-opus-5-5", "n_runs", int(len(ref))); add("G_recovery_cost", "fit", "claude-opus-5-5", "R2", float(1 - ((ref["cost_usd"].values - pred) ** 2).sum() / ((ref["cost_usd"].values - ref["cost_usd"].mean()) ** 2).sum()))96 add("G_recovery_cost", "fit", "claude-opus-5-5", "mean_abs_err_usd", float(np.abs(ref["cost_usd"].values - pred).mean()))97 for arm, d in rec.groupby("arm"):98 first_cost = d["C"] - d["recovery_cost_usd"].fillna(0).astype(float)99 add("G_recovery_cost", "imputed_first_session", arm, "mean_usd", float(first_cost.mean())); add("G_recovery_cost", "imputed_first_session", arm, "n_imputed", int(d["cost_imputed_flag"].sum()))100 add("G_recovery_cost", "recovery_session", arm, "mean_usd", float(d["recovery_cost_usd"].fillna(0).mean())); add("G_recovery_cost", "recovery_session", arm, "min_usd", float(d["recovery_cost_usd"].fillna(0).min())); add("G_recovery_cost", "recovery_session", arm, "max_usd", float(d["recovery_cost_usd"].fillna(0).max()))101 add("G_recovery_cost", "total", arm, "mean_usd", float(d["C"].mean())); add("G_recovery_cost", "wall_total_s", arm, "mean", float(d["wall_total_s"].mean())); add("G_recovery_cost", "wall_total_s", arm, "median", float(d["wall_total_s"].median()))102 add("G_recovery_cost", "wall_total_s", arm, "max", float(d["wall_total_s"].max())); add("G_recovery_cost", "wall_first_s", arm, "mean", float(d["wall_s"].mean())); add("G_recovery_cost", "wall_first_s", arm, "max", float(d["wall_s"].max()))103 add("G_recovery_cost", "n_artifacts", arm, "mean", float(d["n_artifacts"].mean())); add("G_recovery_cost", "n_artifacts", arm, "share_zero", float((d["n_artifacts"] == 0).mean()))104 add("G_recovery_cost", "tool_calls", arm, "mean", float(d["tool_calls"].mean())); add("G_recovery_cost", "n_questions", arm, "mean", float(d["n_questions"].fillna(0).mean()))105 add("G_recovery_cost", "quality_missing", arm, "n", int(d["quality"].isna().sum())); add("G_recovery_cost", "S", arm, "sum", float(d["S"].sum())); add("G_recovery_cost", "S", arm, "n_nonmissing", int(d["S"].notna().sum()))106 add("G_recovery_cost", "critical", arm, "n", int(d["E"].sum())); add("G_recovery_cost", "det", arm, "n", int(d["D"].sum()))107 rr = pd.read_csv(os.path.join(VAL, "receipts_recovery.csv"))108 for _, x in rr.iterrows():109 v = json.loads(x["violations"]) if isinstance(x["violations"], str) else []110 for i in v: add("G_recovery_cost", "violation", x["run_id"], i["id"], i.get("description", "")[:120])111 rs = pd.read_csv(os.path.join(VAL, "receipts_sensitivity.csv"))112 for _, x in rs.iterrows():113 v = json.loads(x["violations"]) if isinstance(x["violations"], str) else []114 for i in v: add("J_sensitivity", "violation", x["run_id"], i["id"], i.get("description", "")[:120])115 # ---- H. spend by stage (validated data): worker cost (observed or imputed) and judge cost116 for tag, d in (("pilot", pilot), ("sensitivity", sens), ("recovery", rec)):117 j = pd.read_parquet(os.path.join(VAL, f"judgments_{tag}.parquet"))118 add("H_spend", tag, "worker", "sum_usd", float(d["C"].sum())); add("H_spend", tag, "worker", "n_runs", int(len(d))); add("H_spend", tag, "worker", "n_cost_imputed", int(d["cost_imputed_flag"].sum()))119 add("H_spend", tag, "judges", "sum_usd", float(j["cost_usd"].sum())); add("H_spend", tag, "judges", "n_judgments", int(len(j))); add("H_spend", tag, "judges", "mean_per_run_usd", float(j.groupby("run_id")["cost_usd"].sum().mean()))120 for jm, jj in j.groupby("judge_model"): add("H_spend", tag, f"judge:{jm}", "mean_cost_usd", float(jj["cost_usd"].mean()))121 add("H_spend", tag, "runs", "n_judges_0", int((d["n_judges"] == 0).sum())); add("H_spend", tag, "runs", "n_judges_3", int((d["n_judges"] == 3).sum()))122 add("H_spend", tag, "runs", "timed_out", int(d["censored"].sum())); add("H_spend", tag, "runs", "result_subtypes", ";".join(sorted(d["result_subtype"].astype(str).unique())))123 add("H_spend", tag, "runs", "nodes", ";".join(f"{k}:{v}" for k, v in d["node"].value_counts().sort_index().items()))124 # ---- I. judge-vote agreement (acceptable votes) vs D and vs Q>=70, and judge means by arm × model125 pilot["acc_votes"] = pilot["judge_acceptable_votes"].apply(lambda s: sum(1 for v in json.loads(s) if v) if isinstance(s, str) else np.nan)126 for (dv, av), n in pilot.groupby(["D", "acc_votes"]).size().items(): add("I_judges", f"D={int(dv)}", f"votes={int(av)}", "n", int(n))127 for (qv, av), n in pilot.groupby([(pilot["quality"] >= 70), "acc_votes"]).size().items(): add("I_judges", f"Q>=70={bool(qv)}", f"votes={int(av)}", "n", int(n))128 add("I_judges", "unanimous", "all", "share", float(pilot["acc_votes"].isin([0, 3]).mean()))129 jd2 = jd.merge(pilot[["run_id", "arm"]], on="run_id")130 for (arm, jm), g in jd2.groupby(["arm", "judge_model"]): add("I_judges", arm, jm, "mean_score", float(g["score"].mean()))131 for (arm, ji), g in jd2.groupby(["arm", "judge_idx"]): add("I_judges", arm, f"judge{ji}", "mean_score", float(g["score"].mean()))132 piv = jd.pivot_table(index="run_id", columns="judge_idx", values="score")133 add("I_judges", "opus_minus_sonnet_mean", "all", "points", float((piv[0] - piv[[1, 2]].mean(axis=1)).mean()))134 add("I_judges", "abs_diff_opus_sonnet", "all", "median_points", float((piv[0] - piv[[1, 2]].mean(axis=1)).abs().median()))135 add("I_judges", "abs_diff_opus_sonnet", "all", "p90_points", float((piv[0] - piv[[1, 2]].mean(axis=1)).abs().quantile(0.9)))136 add("I_judges", "abs_diff_sonnet_sonnet", "all", "median_points", float((piv[1] - piv[2]).abs().median()))137 add("I_judges", "runs_with_judge_disagreement_on_q", "all", "n", int(((piv >= 70).sum(axis=1).isin([1, 2])).sum()))138 # ---- J. model sensitivity: process by model (A3)139 a3 = pilot[pilot.arm == "A3"].copy(); a3["model_tag"] = "opus"; sens["model_tag"] = sens["model"].map({"claude-sonnet-5-5": "sonnet", "claude-fable-5-1": "fable"})140 for tag, d in pd.concat([a3, sens]).groupby("model_tag"):141 for m in ("num_turns", "tool_errors", "app_transitions", "tok_output", "tok_cache_read", "n_questions"): add("J_sensitivity", tag, "mean", m, float(d[m].fillna(0).mean()))142 for qn, qv in (("p10", 0.1), ("p50", 0.5), ("p90", 0.9), ("max", 1.0)): add("J_sensitivity", tag, qn, "wall_s", float(d["T"].quantile(qv))); add("J_sensitivity", tag, qn, "cost_usd", float(d["C"].quantile(qv)))143 add("J_sensitivity", tag, "abstention_rate", "share", float(d["abstention_status"].isin(["needs_clarification", "abstained"]).mean()))144 add("J_sensitivity", tag, "n", "runs", int(len(d)))145 # ---- K. q-gate anatomy: deterministic passes failing the gate, by scenario146 gate_fail = pilot[(pilot["D"] == 1) & (pilot["E"] == 0) & (pilot["S"] == 0)]147 for t, g in gate_fail.groupby("task_id"): add("K_gate", t, "det_pass_quality_below_q", "n", int(len(g))); add("K_gate", t, "det_pass_quality_below_q", "quality_min", float(g["quality"].min())); add("K_gate", t, "det_pass_quality_below_q", "quality_max", float(g["quality"].max()))148 det_fail_hi = pilot[(pilot["D"] == 0) & (pilot["quality"] >= 70)]149 for _, r in det_fail_hi.iterrows(): add("K_gate", r["run_id"], "det_fail_quality_above_q", "quality", float(r["quality"]))150 adm = pilot[pilot.task_id == "ADM-01"]151 for arm, g in adm.groupby("arm"): add("K_gate", "ADM-01", arm, "quality_mean", float(g["quality"].mean())); add("K_gate", "ADM-01", arm, "n_questions_mean", float(g["n_questions"].fillna(0).mean()))152 # judges' per-criterion means for ADM-01 (which rubric criteria pull the score below q)153 jadm = jd[jd.run_id.str.startswith("ADM-01") & ~jd.run_id.str.contains("_A0_")]154 crit = {}155 for _, x in jadm.iterrows():156 sc = json.loads(x["scores"]).get("scores", {}) if isinstance(x["scores"], str) else {}157 for k, v in sc.items(): crit.setdefault(k, []).append(v)158 for k, v in crit.items(): add("K_gate", "ADM-01_A1-A4", k, "mean_criterion_score_0to4", float(np.mean(v)))159 # ---- L. dependency: within-arm correlation of acceptance with depth and with tool calls (A1–A4 pooled and per arm)160 tasks = {t: json.load(open(os.path.join(REPO, "benchmark", "scenarios", t, "task.json"))) for t in pilot.task_id.unique()}161 pilot["depth"] = pilot.task_id.map({t: v.get("designed_dependency_depth") for t, v in tasks.items()})162 for arm in ("A1", "A2", "A3", "A4"):163 d = pilot[pilot.arm == arm]; tm = d.groupby("task_id").agg(S=("S", "mean"), depth=("depth", "first"), tc=("tool_calls", "mean"))164 add("L_dependency", arm, "scenario_level", "spearman_S_depth", float(tm[["S", "depth"]].corr(method="spearman").iloc[0, 1]))165 add("L_dependency", arm, "scenario_level", "spearman_S_toolcalls", float(tm[["S", "tc"]].corr(method="spearman").iloc[0, 1]))166 add("L_dependency", arm, "scenario_level", "spearman_toolcalls_depth", float(tm[["tc", "depth"]].corr(method="spearman").iloc[0, 1]))167 add("L_dependency", arm, "run_level", "mean_toolcalls_accepted", float(d[d.S == 1]["tool_calls"].mean())); add("L_dependency", arm, "run_level", "mean_toolcalls_rejected", float(d[d.S == 0]["tool_calls"].mean()) if (d.S == 0).any() else np.nan)168 shell = pilot[pilot.arm != "A0"]; tm = shell.groupby("task_id").agg(S=("S", "mean"), depth=("depth", "first"), tc=("tool_calls", "mean"))169 add("L_dependency", "A1-A4", "scenario_level", "spearman_S_depth", float(tm[["S", "depth"]].corr(method="spearman").iloc[0, 1])); add("L_dependency", "A1-A4", "scenario_level", "spearman_S_toolcalls", float(tm[["S", "tc"]].corr(method="spearman").iloc[0, 1]))170 for dcls, g in shell.groupby(shell["depth"].map(lambda x: "standard(21-24)" if x <= 24 else ("complex(31-34)" if x <= 34 else "workflow(44)"))):171 add("L_dependency", "A1-A4", dcls, "acceptance", float(g["S"].mean())); add("L_dependency", "A1-A4", dcls, "n", int(len(g))); add("L_dependency", "A1-A4", dcls, "tool_calls_mean", float(g["tool_calls"].mean()))172 # ---- M. tool families per run (pilot) and denied calls context173 fam = pd.read_csv(os.path.join(TAB, "audit_families_pilot.csv"))174 for arm, g in fam.groupby("arm"):175 tot = g["calls"].sum(); add("M_families", arm, "total", "calls_per_run", float(tot / 30))176 for _, r in g.iterrows(): add("M_families", arm, r["family"], "share", float(r["calls"] / tot))177178 # ---- N. cost accounting of the recovery stage, re-derived from the raw streams (per-API-call usage deduplicated by message id)179 # Prices are not taken from a list: they are fitted on the 150 pilot runs' result usage (input, output, cache read, cache creation).180 def dedup_usage(path):181 ev = [json.loads(l) for l in open(path) if l.strip()]; by = {}182 for e in ev:183 if e.get("type") != "assistant": continue184 m = e.get("message", {}); u = m.get("usage"); mid = m.get("id")185 if u and mid: by[mid] = u186 res = [e for e in ev if e.get("type") == "result"]187 return {"calls": len(by), "inp": sum(u.get("input_tokens", 0) for u in by.values()), "cr": sum(u.get("cache_read_input_tokens", 0) for u in by.values()),188 "cc": sum(u.get("cache_creation_input_tokens", 0) for u in by.values()), "res_usage": res[0].get("usage") if res else None, "res_cost": res[0].get("total_cost_usd") if res else None}189 pil = pd.read_parquet(os.path.join(VAL, "runs_pilot.parquet")); U = []; calls = []190 for rid in pil.run_id:191 d = dedup_usage(os.path.join(RAW, rid, "raw_stream.jsonl")); u = d["res_usage"]192 U.append([u["input_tokens"], u["output_tokens"], u["cache_read_input_tokens"], u["cache_creation_input_tokens"], d["res_cost"], d["calls"]])193 U = np.array(U, dtype=float); price, *_ = np.linalg.lstsq(U[:, :4], U[:, 4], rcond=None); predc = U[:, :4] @ price194 for k, b in zip(("input", "output", "cache_read", "cache_create"), price): add("N_recovery_cost_v2", "fitted_price_usd_per_Mtoken", "claude-opus-5-5", k, float(b * 1e6))195 add("N_recovery_cost_v2", "fit_on_result_usage", "claude-opus-5-5", "R2", float(1 - ((U[:, 4] - predc) ** 2).sum() / ((U[:, 4] - U[:, 4].mean()) ** 2).sum()))196 add("N_recovery_cost_v2", "fit_on_result_usage", "claude-opus-5-5", "max_abs_err_usd", float(np.abs(U[:, 4] - predc).max()))197 pil_arm = pil["arm"].values198 for arm in ("A3", "A4"):199 sel = pil_arm == arm; add("N_recovery_cost_v2", "pilot_output_tokens_per_call", arm, "mean", float((U[sel, 1] / U[sel, 5]).mean()))200 add("N_recovery_cost_v2", "pilot_cost_per_wall_second", arm, "usd_per_s", float(pil.loc[sel, "cost_usd"].sum() / pil.loc[sel, "wall_s"].sum()))201 add("N_recovery_cost_v2", "pilot_output_share_of_cost", arm, "share", float((U[sel, 1] * price[1]).sum() / U[sel, 4].sum()))202 opc = {arm: (U[pil_arm == arm, 1] / U[pil_arm == arm, 5]).mean() for arm in ("A3", "A4")}203 cps = {arm: pil.loc[pil_arm == arm, "cost_usd"].sum() / pil.loc[pil_arm == arm, "wall_s"].sum() for arm in ("A3", "A4")}204 recv = pd.read_csv(os.path.join(VAL, "runs_recovery.csv")); nrows = []205 for _, r in recv.iterrows():206 d1 = dedup_usage(os.path.join(RAW, r["run_id"], "raw_stream.jsonl"))207 lb = (d1["inp"] * price[0] + d1["cr"] * price[2] + d1["cc"] * price[3]); est = lb + d1["calls"] * opc[r["arm"]] * price[1]208 row = {"run_id": r["run_id"], "arm": r["arm"], "first_calls": d1["calls"], "first_cost_lower_bound": lb, "first_cost_est_tokens": est, "first_cost_est_wall": cps[r["arm"]] * r["wall_s"], "reported_recovery_cost": r["recovery_cost_usd"]}209 p2 = os.path.join(RAW, r["run_id"], "raw_stream_recovery.jsonl")210 if os.path.exists(p2):211 d2 = dedup_usage(p2); u2 = d2["res_usage"] or {}212 row["recovery_only_cost_from_usage"] = u2.get("input_tokens", 0) * price[0] + u2.get("output_tokens", 0) * price[1] + u2.get("cache_read_input_tokens", 0) * price[2] + u2.get("cache_creation_input_tokens", 0) * price[3]213 row["recovery_result_cost"] = d2["res_cost"]; row["recovery_calls"] = d2["calls"]214 nrows.append(row)215 N = pd.DataFrame(nrows)216 for arm, g in N.groupby("arm"):217 for m in ("first_calls", "first_cost_lower_bound", "first_cost_est_tokens", "first_cost_est_wall", "reported_recovery_cost", "recovery_only_cost_from_usage", "recovery_result_cost", "recovery_calls"):218 if m in g and g[m].notna().any(): add("N_recovery_cost_v2", arm, m, "mean", float(g[m].mean()))219 if arm == "A4":220 tot = g["first_cost_est_tokens"] + g["recovery_only_cost_from_usage"]; add("N_recovery_cost_v2", arm, "total_est_tokens(first+recovery_only)", "mean", float(tot.mean()))221 add("N_recovery_cost_v2", arm, "reported_minus_recovery_only", "mean", float((g["reported_recovery_cost"] - g["recovery_only_cost_from_usage"]).mean()))222 add("N_recovery_cost_v2", arm, "reported_minus_recovery_only", "min", float((g["reported_recovery_cost"] - g["recovery_only_cost_from_usage"]).min()))223 add("N_recovery_cost_v2", arm, "corr_reported_vs_total_est", "pearson", float(np.corrcoef(g["reported_recovery_cost"], tot)[0, 1]))224 add("N_recovery_cost_v2", arm, "corr_reported_vs_recovery_only", "pearson", float(np.corrcoef(g["reported_recovery_cost"], g["recovery_only_cost_from_usage"])[0, 1]))225 S4 = rec[rec.arm == "A4"].set_index("run_id").loc[g["run_id"], "S"]226 add("N_recovery_cost_v2", arm, "cost_per_accepted_if_reported_is_cumulative", "usd", float(g["reported_recovery_cost"].sum() / S4.sum()))227 add("N_recovery_cost_v2", arm, "cost_per_accepted_tokens_est", "usd", float(tot.sum() / S4.sum()))228 if arm == "A3":229 S3 = rec[rec.arm == "A3"].set_index("run_id").loc[g["run_id"], "S"]230 add("N_recovery_cost_v2", arm, "cost_per_accepted_tokens_est", "usd", float(g["first_cost_est_tokens"].sum() / S3.sum()) if S3.sum() else np.nan)231 for _, r in N.iterrows(): add("N_recovery_cost_v2", r["run_id"], r["arm"], "first_cost_est_tokens", float(r["first_cost_est_tokens"])); 232 for _, r in N[N.arm == "A4"].iterrows(): add("N_recovery_cost_v2", r["run_id"], r["arm"], "recovery_only_cost_from_usage", float(r["recovery_only_cost_from_usage"])); add("N_recovery_cost_v2", r["run_id"], r["arm"], "reported_recovery_cost", float(r["reported_recovery_cost"]))233234 # ---- O. session token totals from the client's per-model usage (result event), A0 tool-error anatomy, criteria counts per scenario235 def mu(x):236 try: d = json.loads(x)237 except Exception: return {} # noqa: BLE001238 out = {}239 for m, v in d.items():240 for k in ("inputTokens", "outputTokens", "cacheReadInputTokens", "cacheCreationInputTokens", "thinkingTokens", "costUSD"): out[k] = out.get(k, 0) + (v.get(k, 0) or 0)241 return out242 for tag, d in (("pilot", pilot), ("sensitivity", sens)):243 m = pd.DataFrame(list(d["model_usage"].apply(mu))); m["grp"] = (d["arm"] if tag == "pilot" else d["model"]).values244 for g, gg in m.groupby("grp"):245 for k in ("inputTokens", "outputTokens", "cacheReadInputTokens", "cacheCreationInputTokens", "thinkingTokens"): add("O_tokens", tag, g, k + "_mean", float(gg[k].mean()))246 add("O_tokens", tag, g, "n_with_usage", int(gg["outputTokens"].notna().sum()))247 evp = pd.read_parquet(os.path.join(VAL, "events_pilot.parquet"))248 ev0 = evp[evp.run_id.isin(pilot[pilot.arm == "A0"].run_id)]249 add("O_tokens", "events_columns", "pilot", "cols", ";".join(evp.columns))250 if "error_class" in evp.columns:251 err = ev0[ev0["error_class"].notna() & (ev0["error_class"].astype(str) != "")]252 for (tool, ec), n in err.groupby(["tool_name", "error_class"]).size().sort_values(ascending=False).head(8).items(): add("O_a0_errors", str(tool), str(ec), "n", int(n))253 add("O_a0_errors", "total", "A0", "n", int(len(err)))254 pref = err["summary"].astype(str).str.replace("tool_result:error:", "", regex=False).str[:40]255 for k, n in pref.value_counts().head(6).items(): add("O_a0_errors", "summary_prefix", k, "n", int(n))256 for arm in ("A1", "A2", "A3", "A4"):257 e1 = evp[evp.run_id.isin(pilot[pilot.arm == arm].run_id)]; e1 = e1[e1["error_class"].notna() & (e1["error_class"].astype(str) != "")]258 for (tool, ec), n in e1.groupby(["tool_name", "error_class"]).size().sort_values(ascending=False).head(4).items(): add("O_a0_errors", f"{arm}:{tool}", str(ec), "n", int(n))259 rcp = pd.read_parquet(os.path.join(VAL, "receipts_pilot.parquet"))260 for t, g in rcp.assign(task_id=rcp.run_id.str.split("_").str[0]).groupby("task_id"):261 c = json.loads(g.iloc[0]["criteria"]) if isinstance(g.iloc[0]["criteria"], str) else g.iloc[0]["criteria"]262 req = [x for x in c if x.get("required", True)] if isinstance(c, list) else c263 add("O_criteria", t, "required_criteria", "n", int(len(req)) if isinstance(req, list) else None); add("O_criteria", t, "evaluator_version", "v", str(g.iloc[0]["evaluator_version"]))264 add("O_criteria", t, "oracle_sha256_prefix", "v", str(g.iloc[0]["oracle_sha256"])[:12]); add("O_criteria", t, "n_distinct_oracle_hashes", "n", int(g["oracle_sha256"].nunique()))265 out = pd.DataFrame(ROWS); out.to_csv(os.path.join(TAB, "table14_extra_descriptives.csv"), index=False)266 pd.set_option("display.width", 220); pd.set_option("display.max_rows", 2000); pd.set_option("display.max_colwidth", 60)267 print(out.to_string())268269if __name__ == "__main__":270 main()271