1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca3"""Run one budget tier from configs/budgets.yaml (sequence of `av run-arm` runs, resumable, capped).45Usage: python3 scripts/33_run_tier.py <tier> [--live] [--dry-run]6Each run gets its own ledger (results/ledger/<run_id>.jsonl); the tier cap is enforced on the sum of the7ledgers of the tier's runs (estimated list-price spend).8"""9from __future__ import annotations1011import argparse12import json13import sys14from pathlib import Path1516sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))17from agentic_valuation.config import RESULTS, load_config # noqa: E40218from agentic_valuation.experiments.run_arm import RunSpec, run_arm # noqa: E402192021def tier_spent(run_ids: list[str]) -> float:22 tot = 0.023 for rid in run_ids:24 f = RESULTS / "ledger" / f"{rid}.jsonl"25 if f.exists():26 for line in f.read_text().splitlines():27 try:28 tot += float(json.loads(line).get("cost_usd", 0.0))29 except Exception:30 pass31 return tot323334def main(argv: list[str] | None = None) -> None:35 ap = argparse.ArgumentParser()36 ap.add_argument("tier")37 ap.add_argument("--live", action="store_true")38 ap.add_argument("--dry-run", action="store_true")39 ap.add_argument("--n", type=int, default=None, help="override n for every run")40 ap.add_argument("--only", default=None, help="comma-separated substrings; keep runs whose run_id contains any of them")41 ap.add_argument("--concurrency", type=int, default=6, help="properties processed concurrently per run")42 a = ap.parse_args(argv)43 cfg = load_config("budgets")44 t = cfg["tiers"][a.tier]45 model_tier = t.get("tier", cfg.get("tier_default", "mid"))46 specs: list[RunSpec] = []47 for r in t["runs"]:48 for c in r["committees"]:49 n = a.n or r.get("n", t["n"])50 rounds = r.get("rounds")51 rid = "_".join([a.tier, r["arm"] + (f"_r{rounds}" if rounds is not None else ""), c, model_tier]) + ("" if a.live else "_mock")52 if not a.live:53 rid = "_" + rid54 specs.append(RunSpec(arm=r["arm"], rounds=rounds, committee=c, split=t["split"], n=n, live=a.live, tier=model_tier, run_id=rid, property_concurrency=a.concurrency))55 if a.only:56 keys = [k.strip() for k in a.only.split(",") if k.strip()]57 specs = [s for s in specs if any(k in s.run_id for k in keys)]58 all_ids = []59 for r in t["runs"]:60 for c in r["committees"]:61 rr = r.get("rounds")62 all_ids.append("_".join([a.tier, r["arm"] + (f"_r{rr}" if rr is not None else ""), c, model_tier]) + ("" if a.live else "_mock"))63 ids = [s.run_id for s in specs]64 print(f"[tier {a.tier}] {len(specs)} runs, split={t['split']}, model tier={model_tier}, cap={t['cap_usd']} USD, spent so far={tier_spent(ids):.2f}")65 for s in specs:66 print(f" - {s.run_id}: n={s.n}")67 if a.dry_run:68 return69 for s in specs:70 spent = tier_spent(all_ids)71 if spent >= t["cap_usd"]:72 print(f"[tier {a.tier}] cap reached ({spent:.2f} >= {t['cap_usd']}); stopping before {s.run_id}")73 break74 import os7576 os.environ["AV_MAX_SPEND_USD"] = str(max(t["cap_usd"] - spent, 1.0))77 run_arm(s)78 print(f"[tier {a.tier}] done; estimated spend of the whole tier {tier_spent(all_ids):.2f} USD")79 (RESULTS / f"tier_{a.tier}.json").write_text(json.dumps({"tier": a.tier, "runs": ids, "spent_usd": tier_spent(ids), "live": a.live}, indent=1))808182if __name__ == "__main__":83 main()84