1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca3"""Rounds curve: accuracy and dispersion of the committee by deliberation round, from the debate transcripts4(round 0 = independent estimates, round r = revisions). Writes results/evaluation/rounds.csv5(columns: round, value, ci_lo, ci_hi, arm, metric, n) consumed by scripts/21_figures.py.67Usage: python3 scripts/35_rounds_curve.py --split validation [--runs validation_G_debate_r4_mixed_random_mid ...]8"""9from __future__ import annotations1011import argparse12import json13import math14import re15import sys16from pathlib import Path1718import numpy as np19import pandas as pd20import polars as pl2122sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))23from agentic_valuation.config import PROCESSED, RESULTS # noqa: E402242526def per_round_points(tr: dict) -> dict[int, dict[str, float]]:27 out: dict[int, dict[str, float]] = {}28 for turn in tr["turns"]:29 if turn["kind"] in ("estimate", "revision") and turn.get("estimate"):30 out.setdefault(int(turn["round"]), {})[turn["role"]] = float(turn["estimate"]["point_estimate"])31 return out323334def block_ci(vals: np.ndarray, blocks: np.ndarray, B: int = 2000, seed: int = 20261027) -> tuple[float, float]:35 rng = np.random.default_rng(seed)36 ub = np.unique(blocks)37 idx = {b: np.where(blocks == b)[0] for b in ub}38 stats = []39 for _ in range(B):40 draw = rng.choice(ub, size=len(ub), replace=True)41 sel = np.concatenate([idx[b] for b in draw])42 stats.append(vals[sel].mean())43 return float(np.percentile(stats, 2.5)), float(np.percentile(stats, 97.5))444546def main(argv: list[str] | None = None) -> None:47 ap = argparse.ArgumentParser()48 ap.add_argument("--split", default="validation")49 ap.add_argument("--runs", nargs="*", default=None)50 ap.add_argument("--out", default=None)51 a = ap.parse_args(argv)52 truth = pl.read_parquet(PROCESSED / "analytic.parquet", columns=["tx_id", "price", "mun_code", "date"]).with_columns(pl.col("date").dt.strftime("%Y-%m").alias("ym"))53 tmap = {r["tx_id"]: (math.log(r["price"]), f"{r['mun_code']}_{r['ym']}") for r in truth.iter_rows(named=True)}54 runs = a.runs or [d.name for d in sorted((RESULTS / "runs").glob(f"{a.split}_G_*")) + sorted((RESULTS / "runs").glob(f"final_G_*")) if a.split in d.name]55 rows = []56 for run in runs:57 d = RESULTS / "runs" / run58 recs = []59 for f in sorted((d / "transcripts").glob("*.json")):60 tr = json.loads(f.read_text())61 if tr["tx_id"] not in tmap:62 continue63 lp, blk = tmap[tr["tx_id"]]64 prp = per_round_points(tr)65 m_r = re.search(r"_r(\d+)_", run)66 max_r = int(m_r.group(1)) if m_r else int(tr.get("rounds", max(prp))) # nominal rounds of the arm67 # carry forward: a committee that stopped early (IQR rule) keeps its last estimates in later rounds,68 # so every round averages the SAME set of properties69 last: dict[str, float] = {}70 for r in range(0, max(max_r, max(prp)) + 1):71 if r in prp:72 last = {**last, **prp[r]}73 if not last:74 continue75 v = np.log(np.array(list(last.values())))76 recs.append({"tx_id": tr["tx_id"], "round": r, "ae_median": abs(float(np.median(v)) - lp), "ae_mean": abs(float(v.mean()) - lp), "iqr": float(np.subtract(*np.percentile(v, [75, 25]))), "block": blk, "n_roles": len(v), "carried": r not in prp})77 if not recs:78 continue79 df = pd.DataFrame(recs)80 man = json.loads((d / "manifest.json").read_text()) if (d / "manifest.json").exists() else {}81 arm = man.get("arm_label") or "_".join(run.split("_")[1:4])82 committee = man.get("committee") or ("mixed_random" if "mixed_random" in run else "all_anthropic" if "all_anthropic" in run else "all_openai")83 for r, g in df.groupby("round"):84 for metric in ("ae_median", "ae_mean", "iqr"):85 lo, hi = block_ci(g[metric].to_numpy(), g["block"].to_numpy())86 rows.append({"round": int(r), "value": float(g[metric].mean()), "ci_lo": lo, "ci_hi": hi, "arm": arm, "committee": committee, "metric": metric, "n": int(len(g)), "run_id": run})87 out = Path(a.out) if a.out else RESULTS / "evaluation" / f"rounds_{a.split}.csv"88 out.parent.mkdir(parents=True, exist_ok=True)89 res = pd.DataFrame(rows)90 res.to_csv(out, index=False)91 if a.split == "test":92 res[res.metric == "ae_median"].to_csv(RESULTS / "evaluation" / "rounds.csv", index=False)93 print(res[res.metric.isin(["ae_median", "iqr"])].pivot_table(index=["arm", "round"], columns="metric", values="value").round(4))949596if __name__ == "__main__":97 main()98