"""Emit paper/tables/numbers.tex: every number quoted in the prose comes from this file (registry, configs, state store).""" import os, sys, csv, json, yaml sys.path.insert(0, os.path.dirname(__file__)); from common import * def m(name, val): return f"\\newcommand{{\\{name}}}{{{val}}}" def main(): reg = list(csv.DictReader(open(os.path.join(REPO, "tasks", "registry.csv")))) conf = [r for r in reg if r["split"] == "conf"]; dev = [r for r in reg if r["split"] == "dev"]; tr = [r for r in reg if r["split"] == "transfer"] b = yaml.safe_load(open(os.path.join(REPO, "configs", "budgets.yaml")))["budget"] out = [m("nConfTasks", len(conf)), m("nDevTasks", len(dev)), m("nTransferTasks", len(tr)), m("nConfRealData", sum(r["real_data"] == "yes" for r in conf)), m("nConfSourceDocs", sum(r["source_documents"] == "yes" for r in conf)), m("nConfNullTruth", sum(r["infeasible_or_null_truth"] == "yes" for r in conf)), m("nOracles", len([r for r in reg if r["split"] in ("conf", "dev")])), m("envUSD", f"{b['max_budget_usd']:.2f}"), m("envWall", f"{b['wall_seconds']:,}"), m("envTurns", b["max_turns_per_session"])] try: s = pd.read_csv(os.path.join(DATA, "sessions.csv")); r = pd.read_csv(os.path.join(DATA, "runs.csv")) sm = s[s.run_id.str.startswith("RD") & (s.kind == "initial")].merge(r[["run_id", "treatment"]], on="run_id") for a in ARMS: v = sm[sm.treatment == a].first_turn_input_tokens out.append(m(f"ctx{a}", f"{int(v.iloc[0]):,}" if len(v) else "--")) for split, tag in [("confirmatory", "Main"), ("pilot", "Pilot"), ("transfer", "Transfer"), ("recovery", "Recovery"), ("ablation", "Ablation")]: d = r[r.split == split] out += [m(f"n{tag}Runs", len(d)), m(f"n{tag}Evaluated", int((d.status == "evaluated").sum())), m(f"cost{tag}Total", f"{d.spent_usd.sum():.2f}")] rep = r[r.split == "confirmatory"].groupby(["task_id", "treatment"]).size() out.append(m("nRepeats", int(rep.max()) if len(rep) else 3)) out.append(m("costAllTotal", f"{r.spent_usd.sum():.2f}")); out.append(m("nAllRuns", len(r))) except Exception as e: out.append(f"% sessions/runs not available: {e}") # headline results (confirmatory) + sensitivity + accounting try: at = pd.read_csv(os.path.join(EST, "confirmatory_arm_table.csv")).set_index("arm"); ct = pd.read_csv(os.path.join(EST, "confirmatory_contrasts.csv")) for a in ARMS: out += [m(f"p{a}", f"{at.loc[a, 'Y_task_weighted']:.3f}"), m(f"viol{a}", f"{at.loc[a, 'violation_rate']:.3f}"), m(f"nviol{a}", int(at.loc[a, 'n_runs_with_violation'])), m(f"cost{a}", f"{at.loc[a, 'cost_mean_usd']:.2f}"), m(f"costMed{a}", f"{at.loc[a, 'cost_median_usd']:.2f}"), m(f"wall{a}", f"{at.loc[a, 'wall_median_s']:.0f}"), m(f"qual{a}", f"{at.loc[a, 'quality_mean']:.3f}"), m(f"nRuns{a}", int(at.loc[a, 'n_runs'])), m(f"costPerSuccess{a}", f"{at.loc[a, 'cost_per_success_usd']:.2f}" if pd.notna(at.loc[a, 'cost_per_success_usd']) else "undefined")] for _, r in ct[ct.endpoint == "Y"].iterrows(): k = r["contrast"].replace("-", "") out += [m(f"d{k}", f"{r['estimate']:+.3f}"), m(f"d{k}lo", f"{r['ci_low']:+.3f}"), m(f"d{k}hi", f"{r['ci_high']:+.3f}"), m(f"d{k}p", f"{r['perm_p']:.2f}"), m(f"d{k}fav", f"{int(r['n_tasks_favoring_first'])}/{int(r['n_tasks_tied'])}/{int(r['n_tasks_favoring_second'])}")] rv = pd.read_csv(os.path.join(EST, "confirmatory_contrasts.csv")); q = rv[(rv.endpoint == "quality_score") & (rv.contrast == "D-A")].iloc[0] out += [m("dDAqual", f"{q['estimate']:+.4f}"), m("dDAquallo", f"{q['ci_low']:+.4f}"), m("dDAqualhi", f"{q['ci_high']:+.4f}")] sa = pd.read_csv(os.path.join(EST, "sensitivity_rubric_arm_table.csv")).set_index("arm"); sc = pd.read_csv(os.path.join(EST, "sensitivity_rubric_contrasts.csv")); sd = sc[sc.contrast == "D-A"].iloc[0] for a in ARMS: out.append(m(f"sens{a}", f"{sa.loc[a, 'Y_task_weighted']:.3f}")) out += [m("sensDA", f"{sd['estimate']:+.3f}"), m("sensDAlo", f"{sd['ci_low']:+.3f}"), m("sensDAhi", f"{sd['ci_high']:+.3f}")] r = pd.read_csv(os.path.join(DATA, "runs.csv")); c = r[r.split == "confirmatory"] out += [m("nMainTimeouts", int((c.terminal_reason == "wall_clock_limit").sum())), m("nMainBudget", int(c.terminal_reason.fillna("").str.contains("budget").sum())), m("nMainInfraErrors", int((c.status == "error").sum()) + int(c.run_id.isin(["RC0094"]).sum() * 0)), m("costMainMean", f"{c.spent_usd.mean():.2f}"), m("nMainViolRuns", int((c.n_violations > 0).sum())), m("nMainSuccess", int((c.scientific_success == 1).sum()))] ses = pd.read_csv(os.path.join(DATA, "sessions.csv")); out.append(m("costAllSessions", f"{ses.session_cost_usd.sum():.2f}")) for a in ("B", "C", "D"): out.append(m(f"costRatio{a}", f"{at.loc[a, 'cost_mean_usd'] / at.loc['A', 'cost_mean_usd']:.1f}")) gr = pd.read_csv(os.path.join(DATA, "gate_receipts.csv")); g = gr[gr.run_id.str.startswith("RC")] out += [m("nGateControllerFail", int(((g.origin == "controller") & (g.passed == 0)).sum())), m("nContractFailA", int((g[(g.origin == "final_common_contract_check") & (g.passed == 0)].merge(c[["run_id", "treatment"]], on="run_id").treatment == "A").sum())), m("nContractFailB", int((g[(g.origin == "final_common_contract_check") & (g.passed == 0)].merge(c[["run_id", "treatment"]], on="run_id").treatment == "B").sum()))] rvw = pd.read_csv(os.path.join(DATA, "reviews.csv")); rvw = rvw[rvw.run_id.str.startswith("RC")] out += [m("nReviews", len(rvw)), m("nReviewsRevise", int((rvw.verdict == "revise").sum())), m("reviewIssuesMean", f"{rvw.n_issues.mean():.1f}")] except Exception as e: out.append(f"% results macros not available: {e}") os.makedirs(os.path.join(REPO, "paper", "tables"), exist_ok=True) open(os.path.join(REPO, "paper", "tables", "numbers.tex"), "w").write("\n".join(out) + "\n"); print("\n".join(out)) if __name__ == "__main__": main()