SPB Git

spb/wp29_uqo

Public
2.4 KB · 29 lines python
1"""Emit paper/tables/numbers.tex: every number quoted in the prose comes from this file (registry, configs, state store)."""2import os, sys, csv, json, yaml3sys.path.insert(0, os.path.dirname(__file__)); from common import *4def m(name, val): return f"\\newcommand{{\\{name}}}{{{val}}}"5def main():6    reg = list(csv.DictReader(open(os.path.join(REPO, "tasks", "registry.csv"))))7    conf = [r for r in reg if r["split"] == "conf"]; dev = [r for r in reg if r["split"] == "dev"]; tr = [r for r in reg if r["split"] == "transfer"]8    b = yaml.safe_load(open(os.path.join(REPO, "configs", "budgets.yaml")))["budget"]9    out = [m("nConfTasks", len(conf)), m("nDevTasks", len(dev)), m("nTransferTasks", len(tr)), m("nConfRealData", sum(r["real_data"] == "yes" for r in conf)),10           m("nConfSourceDocs", sum(r["source_documents"] == "yes" for r in conf)), m("nConfNullTruth", sum(r["infeasible_or_null_truth"] == "yes" for r in conf)),11           m("nOracles", len([r for r in reg if r["split"] in ("conf", "dev")])), m("envUSD", f"{b['max_budget_usd']:.2f}"), m("envWall", f"{b['wall_seconds']:,}"), m("envTurns", b["max_turns_per_session"])]12    try:13        s = pd.read_csv(os.path.join(DATA, "sessions.csv")); r = pd.read_csv(os.path.join(DATA, "runs.csv"))14        sm = s[s.run_id.str.startswith("RD") & (s.kind == "initial")].merge(r[["run_id", "treatment"]], on="run_id")15        for a in ARMS:16            v = sm[sm.treatment == a].first_turn_input_tokens17            out.append(m(f"ctx{a}", f"{int(v.iloc[0]):,}" if len(v) else "--"))18        for split, tag in [("confirmatory", "Main"), ("pilot", "Pilot"), ("transfer", "Transfer"), ("recovery", "Recovery"), ("ablation", "Ablation")]:19            d = r[r.split == split]20            out += [m(f"n{tag}Runs", len(d)), m(f"n{tag}Evaluated", int((d.status == "evaluated").sum())), m(f"cost{tag}Total", f"{d.spent_usd.sum():.2f}")]21        rep = r[r.split == "confirmatory"].groupby(["task_id", "treatment"]).size()22        out.append(m("nRepeats", int(rep.max()) if len(rep) else 3))23        out.append(m("costAllTotal", f"{r.spent_usd.sum():.2f}")); out.append(m("nAllRuns", len(r)))24    except Exception as e:25        out.append(f"% sessions/runs not available: {e}")26    os.makedirs(os.path.join(REPO, "paper", "tables"), exist_ok=True)27    open(os.path.join(REPO, "paper", "tables", "numbers.tex"), "w").write("\n".join(out) + "\n"); print("\n".join(out))28if __name__ == "__main__": main()29
Files · 2c82656f6a48d002fcaa9c62ccf1d2b88a2e285b
Menu

Theme