1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""Evaluate every run under results/runs/ (one predictions.parquet per run) against the truth table.34Outputs (results/evaluation/): summary.csv (all arm × committee × aggregation × run_id), summary_main.csv (one row per5arm, main configuration), contrasts.csv, coverage.csv, strata.csv, faithfulness.csv (+ faithfulness_turns.csv,6faithfulness_committee.csv), pit.csv, records.parquet (per-record errors of the main configuration), manifest.json.7LaTeX fragments + CSV twins in results/tables/: main_results, contrasts, coverage, strata, cost, faithfulness.89Run ids starting with `_` (e.g. `_fake_demo`) are SKIPPED unless --include-fake is given.1011 python3 scripts/20_evaluate.py # real data: data/processed/analytic.parquet + splits.parquet12 python3 scripts/20_evaluate.py --synthetic --include-fake --B 500 # smoke run on the fake demo13"""14from __future__ import annotations1516import argparse17import json18import sys19from datetime import datetime, timezone20from pathlib import Path2122import numpy as np23import pandas as pd24import polars as pl2526from agentic_valuation.config import PROCESSED, RESULTS, SEED, SYNTHETIC27from agentic_valuation.evaluation import (audit_run, compare_arms, coverage_by, pit_from_quantiles, pit_histogram, reliability,28 stratified_metrics, summarise_by, write_contrasts, write_cost, write_coverage,29 write_faithfulness, write_main_results, write_strata)30from agentic_valuation.evaluation._common import join_truth, prepare_truth, to_pandas31from agentic_valuation.evaluation.faithfulness import summarise_faithfulness32from agentic_valuation.evaluation.metrics import PRIMARY_METRIC, per_record33from agentic_valuation.evaluation.stratify import add_strata3435# Pre-registered primary family (docs/preregistration.md H1–H4): (A, B) with Δ = metric(B) − metric(A)36PRIMARY_PAIRS = [("D_avm", "G_debate_r2"), ("F_independent", "G_debate_r2"), ("J_cost_matched", "G_debate_r2"), ("I_placebo", "G_debate_r2")]37PRIMARY_METRICS = [PRIMARY_METRIC]38TAIL_PAIRS = [("I_placebo", "G_debate_r2")] # H4b: P95 |e|39SECONDARY_PAIRS = [("E_single_llm", "G_debate_r2"), ("H_debate_skeptic", "G_debate_r2"), ("A_hedonic", "G_debate_r2"),40 ("C_gbm", "G_debate_r2"), ("D_avm", "F_independent"), ("E_single_llm", "F_independent"),41 ("G_debate_r1", "G_debate_r2"), ("G_debate_r2", "G_debate_r4")]42SECONDARY_METRICS = [PRIMARY_METRIC, "mdape", "p95_abs_log_error", "severe_rate", "coverage_90", "pinball_log"]434445def collect_runs(runs_dir: Path, include_fake: bool) -> tuple[pd.DataFrame, list[str]]:46 frames, used = [], []47 for d in sorted(p for p in runs_dir.iterdir() if p.is_dir()):48 if d.name.startswith("_") and not include_fake:49 continue50 f = d / "predictions.parquet"51 if not f.exists():52 continue53 df = pl.read_parquet(f).to_pandas()54 if "run_id" not in df.columns:55 df["run_id"] = d.name56 if not include_fake:57 df = df[~df["run_id"].astype(str).str.startswith("_")]58 if df.empty:59 continue60 frames.append(df)61 used.append(d.name)62 if not frames:63 return pd.DataFrame(), used64 return pd.concat(frames, ignore_index=True), used656667def load_truth(args: argparse.Namespace) -> tuple[pd.DataFrame, str]:68 if args.synthetic:69 path = SYNTHETIC / "synthetic_transactions.parquet"70 t = prepare_truth(pl.read_parquet(path))71 return t, f"SYNTHETIC {path}"72 path = Path(args.truth)73 t = prepare_truth(pl.read_parquet(path))74 sp = Path(args.splits)75 if sp.exists():76 s = pl.read_parquet(sp).to_pandas()77 t = t.merge(s[["tx_id", "split"]], on="tx_id", how="left")78 return t, str(path)798081def select_main(summary: pd.DataFrame, committee: str, aggregation: str) -> pd.DataFrame:82 """One row per arm, in order of preference: (committee, aggregation) → (any committee, aggregation) → baseline83 config (none/model) → first available. The choice is recorded in `main_selection`; ties broken by the last run_id."""84 rows = []85 for arm, g in summary.groupby("arm", sort=True):86 candidates = [87 ("preferred", g[(g["committee"] == committee) & (g["aggregation"] == aggregation)]),88 ("aggregation_any_committee", g[g["aggregation"] == aggregation]),89 ("baseline", g[(g["committee"] == "none") & (g["aggregation"] == "model")]),90 ("fallback_first", g),91 ]92 label, pick = next((lab, c) for lab, c in candidates if not c.empty)93 row = pick.sort_values(["committee", "run_id"]).iloc[-1].to_dict()94 row["main_selection"] = label95 rows.append(row)96 return pd.DataFrame(rows)979899def main_predictions(preds: pd.DataFrame, main: pd.DataFrame) -> pd.DataFrame:100 keys = main[["arm", "committee", "aggregation", "run_id"]]101 return preds.merge(keys, on=["arm", "committee", "aggregation", "run_id"], how="inner")102103104def run_contrasts(preds_main: pd.DataFrame, truth: pd.DataFrame, pairs, metrics, B, seed, family: str, holm_family: bool) -> pd.DataFrame:105 arms = set(preds_main["arm"].unique())106 ok_pairs = [(a, b) for a, b in pairs if a in arms and b in arms]107 if not ok_pairs:108 return pd.DataFrame()109 c = compare_arms(preds_main, truth, ok_pairs, metric=metrics, B=B, seed=seed, holm_family=holm_family)110 c.insert(0, "family", family)111 return c112113114def main(argv: list[str] | None = None) -> int:115 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)116 ap.add_argument("--runs-dir", default=str(RESULTS / "runs"))117 ap.add_argument("--truth", default=str(PROCESSED / "analytic.parquet"))118 ap.add_argument("--splits", default=str(PROCESSED / "splits.parquet"))119 ap.add_argument("--synthetic", action="store_true", help="use data/synthetic as truth (tests / smoke only)")120 ap.add_argument("--split", default="test", help="partition to evaluate (rows of predictions with this split)")121 ap.add_argument("--include-fake", action="store_true", help="also collect run ids starting with `_`")122 ap.add_argument("--B", type=int, default=2000)123 ap.add_argument("--seed", type=int, default=SEED)124 ap.add_argument("--main-committee", default="mixed_random")125 ap.add_argument("--main-aggregation", default="median")126 ap.add_argument("--out", default=str(RESULTS / "evaluation"))127 ap.add_argument("--tables", default=str(RESULTS / "tables"))128 ap.add_argument("--min-cell", type=int, default=30)129 args = ap.parse_args(argv)130131 out, tables = Path(args.out), Path(args.tables)132 out.mkdir(parents=True, exist_ok=True)133 tables.mkdir(parents=True, exist_ok=True)134 runs_dir = Path(args.runs_dir)135 preds, used = collect_runs(runs_dir, args.include_fake)136 if preds.empty:137 print(f"no predictions found under {runs_dir} (fake runs skipped={not args.include_fake}); nothing to evaluate", file=sys.stderr)138 return 1139 preds = preds[preds["split"].astype(str) == args.split].copy()140 if preds.empty:141 print(f"no predictions for split={args.split!r}", file=sys.stderr)142 return 1143 truth, truth_src = load_truth(args)144 if "split" in truth.columns:145 truth = truth[truth["split"].astype(str) == args.split]146 truth = add_strata(truth)147 n_before = len(preds)148 preds = preds[preds["tx_id"].astype(str).isin(set(truth["tx_id"]))]149 print(f"runs: {used} | prediction rows: {n_before} (matched to truth: {len(preds)}) | truth: {truth_src}")150151 # 1. summaries152 summary = summarise_by(preds, truth, by=["arm", "committee", "aggregation", "run_id"])153 summary.to_csv(out / "summary.csv", index=False)154 main_sel = select_main(summary, args.main_committee, args.main_aggregation)155 main_sel.to_csv(out / "summary_main.csv", index=False)156 pm = main_predictions(preds, main_sel)157 records = per_record(pm, truth)158 records = add_strata(records)159 records.to_parquet(out / "records.parquet", index=False)160161 # 2. contrasts162 parts = [163 run_contrasts(pm, truth, PRIMARY_PAIRS, PRIMARY_METRICS, args.B, args.seed, "primary", True),164 run_contrasts(pm, truth, TAIL_PAIRS, ["p95_abs_log_error"], args.B, args.seed, "primary_tail", False),165 run_contrasts(pm, truth, SECONDARY_PAIRS, SECONDARY_METRICS, args.B, args.seed, "secondary", False),166 ]167 contrasts = pd.concat([p for p in parts if not p.empty], ignore_index=True) if any(not p.empty for p in parts) else pd.DataFrame()168 contrasts.to_csv(out / "contrasts.csv", index=False)169170 # 3. coverage / calibration171 cov_rows, pit_rows = [], []172 for arm, g in records.groupby("arm", sort=True):173 if not g["has_interval"].any():174 continue175 r = reliability(g)176 r.insert(0, "arm", arm)177 r.insert(1, "group", "all")178 r.insert(2, "level_value", "all")179 cov_rows.append(r)180 for strat in ("region_group", "property_class", "price_tercile"):181 if strat in g.columns:182 cb = coverage_by(g, [strat], min_n=args.min_cell)183 cb = cb.rename(columns={strat: "level_value", "coverage": "empirical"})184 cb.insert(0, "arm", arm)185 cb.insert(1, "group", strat)186 cb["kind"] = "interval"187 cov_rows.append(cb[["arm", "group", "level_value", "kind", "nominal", "empirical", "n", "suppressed"]])188 ph = pit_histogram(pit_from_quantiles(g))189 ph.insert(0, "arm", arm)190 pit_rows.append(ph)191 coverage = pd.concat(cov_rows, ignore_index=True) if cov_rows else pd.DataFrame()192 coverage.to_csv(out / "coverage.csv", index=False)193 (pd.concat(pit_rows, ignore_index=True) if pit_rows else pd.DataFrame()).to_csv(out / "pit.csv", index=False)194195 # 4. strata196 strata = stratified_metrics(pm, truth, min_n=args.min_cell)197 strata.to_csv(out / "strata.csv", index=False)198199 # 5. faithfulness200 turns, comms = [], []201 for name in used:202 t, c = audit_run(runs_dir / name)203 if not t.empty:204 turns.append(t)205 if not c.empty:206 comms.append(c)207 turns_df = pd.concat(turns, ignore_index=True) if turns else pd.DataFrame()208 comms_df = pd.concat(comms, ignore_index=True) if comms else pd.DataFrame()209 faith = summarise_faithfulness(turns_df, comms_df) if not turns_df.empty else pd.DataFrame()210 turns_df.to_csv(out / "faithfulness_turns.csv", index=False)211 comms_df.to_csv(out / "faithfulness_committee.csv", index=False)212 faith.to_csv(out / "faithfulness.csv", index=False)213 if not comms_df.empty:214 rounds = comms_df.groupby(["arm", "round"], as_index=False).agg(iqr_log=("iqr_log", "mean"), convergence=("convergence", "mean"),215 anchoring_index=("anchoring_index", "mean"), share_toward_median=("share_toward_median", "mean"), n=("tx_id", "nunique"))216 rounds.to_csv(out / "committee_rounds.csv", index=False)217218 # 6. LaTeX fragments219 write_main_results(main_sel, tables / "main_results.tex")220 write_coverage(main_sel, tables / "coverage.tex")221 write_cost(main_sel, tables / "cost.tex")222 write_contrasts(contrasts if not contrasts.empty else pd.DataFrame(columns=["arm_a", "arm_b", "metric", "n", "value_a", "value_b", "diff", "ci_lo", "ci_hi", "p_boot", "p_holm", "dm_p"]), tables / "contrasts.tex")223 write_strata(strata, tables / "strata.tex")224 write_faithfulness(faith, tables / "faithfulness.tex")225226 manifest = {227 "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "script": "scripts/20_evaluate.py",228 "runs": used, "truth": truth_src, "split": args.split, "n_prediction_rows": int(len(preds)), "n_truth": int(len(truth)),229 "B": args.B, "seed": args.seed, "main_committee": args.main_committee, "main_aggregation": args.main_aggregation,230 "min_cell": args.min_cell, "arms": sorted(preds["arm"].unique().tolist()), "fake_included": args.include_fake,231 "note": "SYNTHETIC / FAKE — not an empirical result" if (args.synthetic or args.include_fake) else "",232 }233 (out / "manifest.json").write_text(json.dumps(manifest, indent=1, ensure_ascii=False), encoding="utf-8")234 print(main_sel[["arm", "committee", "aggregation", "n", PRIMARY_METRIC, "coverage_90", "main_selection"]].to_string(index=False))235 if not contrasts.empty:236 print(contrasts[["family", "arm_a", "arm_b", "metric", "diff", "ci_lo", "ci_hi", "p_boot", "p_holm", "dm_p"]].to_string(index=False))237 print(f"wrote {out} and {tables}")238 return 0239240241if __name__ == "__main__":242 raise SystemExit(main())243