SPB Git

spb/wp29_uqo

Public
3.0 KB · 20 lines python
1"""reports/experiment_accounting.md: planned/assigned/launched/completed/failed/blocked/evaluated/successes, inference cost per split incl. infrastructure attempts, and human/builder effort pointers."""2import os, sys, json, sqlite3, datetime3sys.path.insert(0, os.path.dirname(__file__)); from common import *4def main():5    r = pd.read_csv(os.path.join(DATA, "runs.csv")); s = pd.read_csv(os.path.join(DATA, "sessions.csv"))6    con = sqlite3.connect(os.path.join(REPO, "runs", "state.sqlite"))7    err = pd.read_sql("SELECT r.split, COUNT(DISTINCT t.run_id) AS n_runs_with_error FROM run_transitions t JOIN runs r ON r.run_id=t.run_id WHERE t.to_status='error' GROUP BY r.split", con).set_index("split")8    L = ["# Experiment accounting", "", f"Generated {datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')} from the state store. Costs are the client's list-price estimates (USD). 'Infra attempts' counts runs that had at least one infrastructure-error attempt before completion; their aborted sessions' cost is included in 'Sessions cost (all attempts)'.", "",9         "| Split | Assigned | Launched | Completed (executor finished) | Timeout | Budget exhausted | Infra attempts | Evaluated | Successes | Envelope cost (final attempts) | Sessions cost (all attempts) | Mean cost/run | Median wall s |", "|---|---|---|---|---|---|---|---|---|---|---|---|---|"]10    for sp in ["development", "pilot", "confirmatory", "transfer", "recovery", "ablation"]:11        d = r[r.split == sp]12        if d.empty: continue13        tr = d.terminal_reason.fillna(""); ss = s[s.run_id.isin(d.run_id)]14        L.append(f"| {sp} | {len(d)} | {int((d.status != 'not_started').sum())} | {int(tr.str.startswith('executor_finished').sum() + tr.eq('max_turns_reached').sum())} | {int(tr.eq('wall_clock_limit').sum())} | {int(tr.str.contains('budget').sum())} | {int(err.loc[sp, 'n_runs_with_error']) if sp in err.index else 0} | {int((d.status == 'evaluated').sum())} | {int((d.scientific_success == 1).sum())} | {d.spent_usd.sum():.2f} | {ss.session_cost_usd.sum():.2f} | {d.spent_usd.mean():.3f} | {d.spent_wall_s.median():.0f} |")15    L += ["", f"**Total inference (all sessions, all attempts): {s.session_cost_usd.sum():.2f} USD; total sessions: {len(s)}; executor sessions: {int((s.role == 'executor').sum())}; reviewer sessions: {int((s.role == 'reviewer').sum())}.** Probes (sentinel/capability) ≈ 0.002 USD, not in the store.", "",16          "Human effort: see `reports/interventions.csv` (mandate preparation: unknown; no intervention in any executor session; final manuscript review: pending at build time). Builder effort: `reports/builder_effort.json` (sub-session tokens and durations from client notifications; builder wall time from the phase log).",17          "", "Reserved budget (amendment A-001): 800 USD. Spent: see total above."]18    open(os.path.join(REPO, "reports", "experiment_accounting.md"), "w").write("\n".join(L) + "\n"); print("\n".join(L[4:12]))19if __name__ == "__main__": main()20
Files · 46a1f53512bec3cd5d9c16e8fc9d9da4129c09cc
Menu

Theme