1"""Emit paper/tables/numbers.tex: every number quoted in the prose comes from this file (registry, configs, state store)."""2import os, sys, csv, json, yaml3sys.path.insert(0, os.path.dirname(__file__)); from common import *4def m(name, val): return f"\\newcommand{{\\{name}}}{{{val}}}"5def _n_infra_errors(split):6 import sqlite37 c = sqlite3.connect(os.path.join(REPO, "runs", "state.sqlite"))8 return c.execute("SELECT COUNT(DISTINCT t.run_id) FROM run_transitions t JOIN runs r ON r.run_id=t.run_id WHERE r.split=? AND t.to_status='error'", (split,)).fetchone()[0]9def main():10 reg = list(csv.DictReader(open(os.path.join(REPO, "tasks", "registry.csv"))))11 conf = [r for r in reg if r["split"] == "conf"]; dev = [r for r in reg if r["split"] == "dev"]; tr = [r for r in reg if r["split"] == "transfer"]12 b = yaml.safe_load(open(os.path.join(REPO, "configs", "budgets.yaml")))["budget"]13 out = [m("nConfTasks", len(conf)), m("nDevTasks", len(dev)), m("nTransferTasks", len(tr)), m("nConfRealData", sum(r["real_data"] == "yes" for r in conf)),14 m("nConfSourceDocs", sum(r["source_documents"] == "yes" for r in conf)), m("nConfNullTruth", sum(r["infeasible_or_null_truth"] == "yes" for r in conf)),15 m("nOracles", len([r for r in reg if r["split"] in ("conf", "dev")])), m("envUSD", f"{b['max_budget_usd']:.2f}"), m("envWall", f"{b['wall_seconds']:,}"), m("envTurns", b["max_turns_per_session"])]16 try:17 s = pd.read_csv(os.path.join(DATA, "sessions.csv")); r = pd.read_csv(os.path.join(DATA, "runs.csv"))18 sm = s[s.run_id.str.startswith("RD") & (s.kind == "initial")].merge(r[["run_id", "treatment"]], on="run_id")19 for a in ARMS:20 v = sm[sm.treatment == a].first_turn_input_tokens21 out.append(m(f"ctx{a}", f"{int(v.iloc[0]):,}" if len(v) else "--"))22 for split, tag in [("confirmatory", "Main"), ("pilot", "Pilot"), ("transfer", "Transfer"), ("recovery", "Recovery"), ("ablation", "Ablation")]:23 d = r[r.split == split]24 out += [m(f"n{tag}Runs", len(d)), m(f"n{tag}Evaluated", int((d.status == "evaluated").sum())), m(f"cost{tag}Total", f"{d.spent_usd.sum():.2f}")]25 rep = r[r.split == "confirmatory"].groupby(["task_id", "treatment"]).size()26 out.append(m("nRepeats", int(rep.max()) if len(rep) else 3))27 out.append(m("costAllTotal", f"{r.spent_usd.sum():.2f}")); out.append(m("nAllRuns", len(r)))28 except Exception as e:29 out.append(f"% sessions/runs not available: {e}")30 # headline results (confirmatory) + sensitivity + accounting31 try:32 at = pd.read_csv(os.path.join(EST, "confirmatory_arm_table.csv")).set_index("arm"); ct = pd.read_csv(os.path.join(EST, "confirmatory_contrasts.csv"))33 for a in ARMS:34 out += [m(f"p{a}", f"{at.loc[a, 'Y_task_weighted']:.3f}"), m(f"viol{a}", f"{at.loc[a, 'violation_rate']:.3f}"), m(f"nviol{a}", int(at.loc[a, 'n_runs_with_violation'])), m(f"cost{a}", f"{at.loc[a, 'cost_mean_usd']:.2f}"),35 m(f"costMed{a}", f"{at.loc[a, 'cost_median_usd']:.2f}"), m(f"wall{a}", f"{at.loc[a, 'wall_median_s']:.0f}"), m(f"qual{a}", f"{at.loc[a, 'quality_mean']:.3f}"), m(f"nRuns{a}", int(at.loc[a, 'n_runs'])), m(f"costPerSuccess{a}", f"{at.loc[a, 'cost_per_success_usd']:.2f}" if pd.notna(at.loc[a, 'cost_per_success_usd']) else "undefined")]36 for _, r in ct[ct.endpoint == "Y"].iterrows():37 k = r["contrast"].replace("-", "")38 out += [m(f"d{k}", f"{r['estimate']:+.3f}"), m(f"d{k}lo", f"{r['ci_low']:+.3f}"), m(f"d{k}hi", f"{r['ci_high']:+.3f}"), m(f"d{k}p", f"{r['perm_p']:.2f}"), m(f"d{k}fav", f"{int(r['n_tasks_favoring_first'])}/{int(r['n_tasks_tied'])}/{int(r['n_tasks_favoring_second'])}")]39 rv = pd.read_csv(os.path.join(EST, "confirmatory_contrasts.csv")); q = rv[(rv.endpoint == "quality_score") & (rv.contrast == "D-A")].iloc[0]40 out += [m("dDAqual", f"{q['estimate']:+.4f}"), m("dDAquallo", f"{q['ci_low']:+.4f}"), m("dDAqualhi", f"{q['ci_high']:+.4f}")]41 sa = pd.read_csv(os.path.join(EST, "sensitivity_rubric_arm_table.csv")).set_index("arm"); sc = pd.read_csv(os.path.join(EST, "sensitivity_rubric_contrasts.csv")); sd = sc[sc.contrast == "D-A"].iloc[0]42 for a in ARMS: out.append(m(f"sens{a}", f"{sa.loc[a, 'Y_task_weighted']:.3f}"))43 out += [m("sensDA", f"{sd['estimate']:+.3f}"), m("sensDAlo", f"{sd['ci_low']:+.3f}"), m("sensDAhi", f"{sd['ci_high']:+.3f}")]44 r = pd.read_csv(os.path.join(DATA, "runs.csv")); c = r[r.split == "confirmatory"]45 out += [m("nMainTimeouts", int((c.terminal_reason == "wall_clock_limit").sum())), m("nMainBudget", int(c.terminal_reason.fillna("").str.contains("budget").sum())), m("nMainInfraErrors", _n_infra_errors("confirmatory")),46 m("costMainMean", f"{c.spent_usd.mean():.2f}"), m("nMainViolRuns", int((c.n_violations > 0).sum())), m("nMainSuccess", int((c.scientific_success == 1).sum()))]47 ses = pd.read_csv(os.path.join(DATA, "sessions.csv")); out.append(m("costAllSessions", f"{ses.session_cost_usd.sum():.2f}"))48 for a in ("B", "C", "D"): out.append(m(f"costRatio{a}", f"{at.loc[a, 'cost_mean_usd'] / at.loc['A', 'cost_mean_usd']:.1f}"))49 gr = pd.read_csv(os.path.join(DATA, "gate_receipts.csv")); g = gr[gr.run_id.str.startswith("RC")]50 out += [m("nGateControllerFail", int(((g.origin == "controller") & (g.passed == 0)).sum())), m("nContractFailA", int((g[(g.origin == "final_common_contract_check") & (g.passed == 0)].merge(c[["run_id", "treatment"]], on="run_id").treatment == "A").sum())), m("nContractFailB", int((g[(g.origin == "final_common_contract_check") & (g.passed == 0)].merge(c[["run_id", "treatment"]], on="run_id").treatment == "B").sum()))]51 rvw = pd.read_csv(os.path.join(DATA, "reviews.csv")); rvw = rvw[rvw.run_id.str.startswith("RC")]52 out += [m("nReviews", len(rvw)), m("nReviewsRevise", int((rvw.verdict == "revise").sum())), m("reviewIssuesMean", f"{rvw.n_issues.mean():.1f}")]53 except Exception as e:54 out.append(f"% results macros not available: {e}")55 # recovery and ablation macros56 try:57 r1 = pd.read_csv(os.path.join(EST, "recovery_r1_summary.csv")).set_index("treatment")58 for a in ("A", "D"):59 out += [m(f"rOneN{a}", int(r1.loc[a, "n"])), m(f"rOneSucc{a}", f"{r1.loc[a, 'success']:.2f}"), m(f"rOneDup{a}", f"{r1.loc[a, 'dup_runs']:.1f}"), m(f"rOneLost{a}", int(r1.loc[a, "lost"])), m(f"rOneAck{a}", f"{r1.loc[a, 'ack']:.2f}"), m(f"rOneCost{a}", f"{r1.loc[a, 'cost']:.2f}")]60 r3 = json.load(open(os.path.join(EST, "recovery_r3_duplicate.json"))); out += [m("rThreeSubmissions", r3["submissions"]), m("rThreeSessions", r3["n_sessions_launched"])]61 rr = pd.read_csv(os.path.join(EST, "recovery_r1_r4.csv")); r4 = rr[rr.fault_type == "missing_fixture"]62 out += [m("rFourN", len(r4)), m("rFourInfeasible", int((r4.feasibility_reported == "infeasible").sum()))]63 r2 = json.load(open(os.path.join(EST, "recovery_r2_stale_input.json"))); out += [m("rTwoN", len(r2)), m("rTwoInvalidated", sum(1 for x in r2 if x.get("reproducibility_check_after_mutation") is False)), m("rTwoUnaffected", sum(1 for x in r2 if x.get("reproducibility_check_after_mutation") is True))]64 ab = pd.read_csv(os.path.join(EST, "ablation_length_control.csv")).set_index("contrast")65 out += [m("pL", f"{ab.loc['L rate', 'estimate']:.3f}"), m("dLA", f"{ab.loc['L-A', 'estimate']:+.3f}"), m("dLAlo", f"{ab.loc['L-A', 'ci_low']:+.3f}"), m("dLAhi", f"{ab.loc['L-A', 'ci_high']:+.3f}"), m("dLAp", f"{ab.loc['L-A', 'perm_p']:.2f}"), m("dBL", f"{ab.loc['B-L', 'estimate']:+.3f}"), m("dBLlo", f"{ab.loc['B-L', 'ci_low']:+.3f}"), m("dBLhi", f"{ab.loc['B-L', 'ci_high']:+.3f}"), m("nAblTasks", int(ab.loc['L rate', 'n_tasks']))]66 rl = pd.read_csv(os.path.join(DATA, "runs.csv")); abl = rl[(rl.split == "ablation") & (rl.status == "evaluated")]; out += [m("costL", f"{abl.spent_usd.mean():.2f}"), m("nRunsL", len(abl)), m("nViolL", int((abl.n_violations > 0).sum()))]67 except Exception as e:68 out.append(f"% recovery/ablation macros not available: {e}")69 # transfer macros70 try:71 rl = pd.read_csv(os.path.join(DATA, "runs.csv")); t = rl[(rl.split == "transfer") & (rl.status == "evaluated")]72 out += [m("nTransferEval", len(t)), m("nTransferSuccess", int((t.scientific_success == 1).sum())), m("costTransferMean", f"{t.spent_usd.mean():.2f}"), m("wallTransferMed", f"{t.spent_wall_s.median()/60:.0f}")]73 for a in ARMS:74 d = t[t.treatment == a]; out += [m(f"transCost{a}", f"{d.spent_usd.mean():.2f}"), m(f"transSucc{a}", f"{int((d.scientific_success == 1).sum())}/{len(d)}")]75 g = pd.read_csv(os.path.join(DATA, "gate_receipts.csv")); gt = g[g.run_id.str.startswith("RT")]76 out += [m("nTransGateCtrlFail", int(((gt.origin == "controller") & (gt.passed == 0)).sum())), m("nTransGateCtrlPass", int(((gt.origin == "controller") & (gt.passed == 1)).sum())), m("nTransRunsGateFail", int(gt[(gt.origin.str.startswith("controller")) & (gt.passed == 0)].run_id.nunique()))]77 t1cd = t[(t.task_id == "T1-forecast") & (t.treatment.isin(["C", "D"]))]; t1ab = t[(t.task_id == "T1-forecast") & (t.treatment.isin(["A", "B"]))]78 out += [m("transTOneCostCD", f"{t1cd.spent_usd.mean():.2f}"), m("transTOneCostAB", f"{t1ab.spent_usd.mean():.2f}"), m("transTOneSessCD", f"{t1cd.n_sessions.mean():.0f}")]79 except Exception as e:80 out.append(f"% transfer macros not available: {e}")81 os.makedirs(os.path.join(REPO, "paper", "tables"), exist_ok=True)82 open(os.path.join(REPO, "paper", "tables", "numbers.tex"), "w").write("\n".join(out) + "\n"); print("\n".join(out))83if __name__ == "__main__": main()84