1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""Figures (PDF, figures/) from results/evaluation/. No titles inside the figures (captions live in LaTeX);3one idea per figure; colour-blind-safe palette validated with the dataviz checker (6 slots + markers/hatches).45 fig_error_distribution.pdf log error e by arm (box + whiskers at P5/P95)6 fig_pit.pdf PIT histograms per arm (small multiples, uniform reference)7 fig_coverage_vs_nominal.pdf empirical vs nominal interval coverage per arm8 fig_rounds.pdf rounds curve (only if results/evaluation/rounds.csv exists)9 fig_cost_accuracy.pdf cost per property vs mean |e| with the Pareto frontier10 fig_strata.pdf mean |e| by stratum level × arm (grouped bars, suppressed cells omitted)1112 python3 scripts/21_figures.py [--stratum region_group] [--arms D_avm,E_single_llm,F_independent,G_debate_r2,J_cost_matched]13"""14from __future__ import annotations1516import argparse17from pathlib import Path1819import matplotlib20matplotlib.use("Agg")21import matplotlib.pyplot as plt # noqa: E40222import numpy as np # noqa: E40223import pandas as pd # noqa: E4022425from agentic_valuation.config import FIGURES, RESULTS # noqa: E40226from agentic_valuation.evaluation.metrics import PRIMARY_METRIC # noqa: E40227from agentic_valuation.evaluation.stratify import STRATUM_LABELS # noqa: E4022829PALETTE = ["#1f5fa8", "#c8541c", "#0b7a5a", "#8b4e8d", "#b58b00", "#2a9dcf"] # validated: dataviz validate_palette.js30MARKERS = ["o", "s", "^", "D", "v", "P"]31HATCHES = ["", "//", "..", "xx", "\\\\", "++"]32GREY = "#7a7a7a"33INK = "#222222"34DEFAULT_ARMS = ["D_avm", "E_single_llm", "F_independent", "G_debate_r2", "I_placebo", "J_cost_matched"]3536plt.rcParams.update({37 "font.size": 8.5, "axes.labelsize": 8.5, "xtick.labelsize": 8, "ytick.labelsize": 8, "legend.fontsize": 7.5,38 "axes.spines.top": False, "axes.spines.right": False, "axes.edgecolor": GREY, "axes.labelcolor": INK,39 "xtick.color": GREY, "ytick.color": GREY, "axes.grid": True, "grid.color": "#e3e3e3", "grid.linewidth": 0.5,40 "axes.axisbelow": True, "pdf.fonttype": 42, "ps.fonttype": 42, "figure.dpi": 150, "legend.frameon": False,41 "text.color": INK, "font.family": "sans-serif",42})434445def pretty(arm: str) -> str:46 parts = arm.split("_")47 return f"{parts[0]} ({' '.join(parts[1:])})" if len(parts) > 1 else arm484950def arm_style(arms: list[str]) -> dict[str, tuple[str, str, str]]:51 """Fixed, order-stable assignment arm → (colour, marker, hatch); beyond 6 arms → grey, folded."""52 style = {}53 for i, a in enumerate(sorted(arms)):54 style[a] = (PALETTE[i], MARKERS[i], HATCHES[i]) if i < len(PALETTE) else (GREY, "x", "")55 return style565758def save(fig: plt.Figure, path: Path) -> None:59 fig.savefig(path, bbox_inches="tight", pad_inches=0.02)60 plt.close(fig)61 print("wrote", path)626364def fig_error_distribution(records: pd.DataFrame, out: Path) -> None:65 arms = sorted(records["arm"].unique())66 data = [records.loc[records["arm"] == a, "e"].dropna().to_numpy() for a in arms]67 fig, ax = plt.subplots(figsize=(3.4, 0.32 * len(arms) + 0.8))68 bp = ax.boxplot(data, vert=False, whis=(5, 95), showfliers=False, widths=0.55, patch_artist=True,69 medianprops={"color": INK, "linewidth": 1.2}, boxprops={"facecolor": "#d6e2f1", "edgecolor": PALETTE[0], "linewidth": 0.9},70 whiskerprops={"color": PALETTE[0], "linewidth": 0.9}, capprops={"color": PALETTE[0], "linewidth": 0.9})71 ax.axvline(0, color=GREY, linewidth=0.8, linestyle="--")72 ax.set_yticks(range(1, len(arms) + 1), [pretty(a) for a in arms])73 ax.set_xlabel(r"log error $e=\log\hat P-\log P$ (box IQR, whiskers P5–P95)")74 ax.invert_yaxis()75 save(fig, out / "fig_error_distribution.pdf")767778def fig_pit(pit: pd.DataFrame, out: Path) -> None:79 arms = sorted(pit["arm"].unique())80 if not arms:81 return82 ncol = min(3, len(arms))83 nrow = int(np.ceil(len(arms) / ncol))84 fig, axes = plt.subplots(nrow, ncol, figsize=(2.1 * ncol, 1.6 * nrow + 0.3), sharex=True, sharey=True, squeeze=False)85 for ax, a in zip(axes.flat, arms):86 g = pit[pit["arm"] == a]87 ax.bar((g["bin_lo"] + g["bin_hi"]) / 2, g["share"], width=(g["bin_hi"] - g["bin_lo"]) * 0.92, color=PALETTE[0], edgecolor="white", linewidth=0.5)88 ax.axhline(g["expected"].iloc[0], color=PALETTE[1], linewidth=1.0, linestyle="--")89 ax.text(0.0, 1.02, pretty(a), transform=ax.transAxes, ha="left", va="bottom", fontsize=7.5)90 ax.set_xlim(0, 1)91 ax.set_xticks([0, 0.5, 1])92 for ax in axes.flat[len(arms):]:93 ax.axis("off")94 for j in range(ncol):95 col_axes = [axes[i, j] for i in range(nrow) if i * ncol + j < len(arms)]96 if col_axes:97 col_axes[-1].set_xlabel("PIT")98 col_axes[-1].tick_params(labelbottom=True)99 for ax in axes[:, 0]:100 ax.set_ylabel("share")101 save(fig, out / "fig_pit.pdf")102103104def fig_coverage(coverage: pd.DataFrame, out: Path) -> None:105 c = coverage[(coverage["group"] == "all") & (coverage["kind"] == "interval")]106 arms = sorted(c["arm"].unique())107 if not arms:108 return109 style = arm_style(arms)110 fig, ax = plt.subplots(figsize=(3.3, 3.0))111 ax.plot([0.4, 1.0], [0.4, 1.0], color=GREY, linewidth=0.8, linestyle="--", zorder=1)112 for a in arms:113 g = c[c["arm"] == a].sort_values("nominal")114 col, mk, _ = style[a]115 ax.plot(g["nominal"], g["empirical"], color=col, marker=mk, markersize=4.5, linewidth=1.4, label=pretty(a), markeredgecolor="white", markeredgewidth=0.6)116 ax.set_xlabel("nominal coverage")117 ax.set_ylabel("empirical coverage")118 ax.set_xticks([0.5, 0.8, 0.9, 0.95], ["0.50", "0.80", "0.90", "0.95"], rotation=45, ha="right")119 ax.set_xlim(0.45, 1.0)120 ax.set_ylim(min(0.3, float(c["empirical"].min()) - 0.05), 1.0)121 ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.22), ncol=2, handlelength=1.6, columnspacing=1.0)122 save(fig, out / "fig_coverage_vs_nominal.pdf")123124125def fig_rounds(rounds: pd.DataFrame, out: Path) -> None:126 """Expects columns: round, value (+ optional ci_lo, ci_hi, arm/series, metric)."""127 if "arm" not in rounds.columns:128 rounds = rounds.assign(arm="G_debate")129 if "metric" in rounds.columns:130 rounds = rounds[rounds["metric"] == PRIMARY_METRIC] if (rounds["metric"] == PRIMARY_METRIC).any() else rounds131 arms = sorted(rounds["arm"].unique())132 style = arm_style(arms)133 fig, ax = plt.subplots(figsize=(3.3, 2.5))134 for a in arms:135 g = rounds[rounds["arm"] == a].sort_values("round")136 col, mk, _ = style[a]137 if {"ci_lo", "ci_hi"} <= set(g.columns):138 ax.fill_between(g["round"], g["ci_lo"], g["ci_hi"], color=col, alpha=0.15, linewidth=0)139 ax.plot(g["round"], g["value"], color=col, marker=mk, markersize=4.5, linewidth=1.4, label=pretty(a), markeredgecolor="white", markeredgewidth=0.6)140 ax.set_xlabel("debate rounds")141 ax.set_ylabel(r"mean $|e|$")142 ax.set_xticks(sorted(rounds["round"].unique()))143 if len(arms) > 1:144 ax.legend()145 save(fig, out / "fig_rounds.pdf")146147148def fig_cost_accuracy(summary: pd.DataFrame, out: Path) -> None:149 s = summary.dropna(subset=[PRIMARY_METRIC]).copy()150 s["cost"] = s["cost_usd_mean"].fillna(0.0)151 fig, ax = plt.subplots(figsize=(3.4, 2.8))152 floor = max(s.loc[s["cost"] > 0, "cost"].min() / 10 if (s["cost"] > 0).any() else 1e-3, 1e-4)153 x = np.where(s["cost"] > 0, s["cost"], floor)154 fam = s["arm"].str[0].le("D")155 ax.scatter(x[fam], s.loc[fam, PRIMARY_METRIC], color=PALETTE[0], marker="s", s=26, label="statistical baselines (A–D, cost ≈ 0)", zorder=3, edgecolor="white", linewidth=0.6)156 ax.scatter(x[~fam], s.loc[~fam, PRIMARY_METRIC], color=PALETTE[1], marker="o", s=26, label="LLM arms (E–J)", zorder=3, edgecolor="white", linewidth=0.6)157 # Pareto frontier: lowest error for each cost level (non-dominated points), sorted by cost158 pts = sorted(zip(x, s[PRIMARY_METRIC], s["arm"]), key=lambda t: (t[0], t[1]))159 frontier, best = [], np.inf160 for cx, cy, a in pts:161 if cy < best:162 frontier.append((cx, cy))163 best = cy164 if len(frontier) > 1:165 ax.plot([f[0] for f in frontier], [f[1] for f in frontier], color=GREY, linewidth=0.9, linestyle="-", zorder=2)166 for cx, cy, a in zip(x, s[PRIMARY_METRIC], s["arm"]):167 ax.annotate(a.split("_")[0], (cx, cy), textcoords="offset points", xytext=(4, 3), fontsize=7, color=INK)168 ax.set_xscale("log")169 ax.set_xlabel("cost per property (USD, log; baselines at floor)")170 ax.set_ylabel(r"mean $|e|$")171 ax.legend(loc="upper right")172 save(fig, out / "fig_cost_accuracy.pdf")173174175def fig_strata(strata: pd.DataFrame, out: Path, stratum: str, arms: list[str]) -> None:176 s = strata[(strata["metric"] == PRIMARY_METRIC) & (strata["stratum"] == stratum) & (strata["arm"].isin(arms))]177 if s.empty:178 print(f"no strata rows for {stratum}; skipping fig_strata")179 return180 levels = list(dict.fromkeys(s["level"]))181 arms = [a for a in sorted(arms) if a in set(s["arm"])]182 style = arm_style(arms)183 w = 0.8 / max(len(arms), 1)184 fig, ax = plt.subplots(figsize=(max(3.3, 0.9 * len(levels) + 1.2), 2.6))185 for j, a in enumerate(arms):186 g = s[s["arm"] == a].set_index("level").reindex(levels)187 vals = g["value"].to_numpy(dtype=float)188 col, _, hatch = style[a]189 xs = np.arange(len(levels)) + (j - (len(arms) - 1) / 2) * w190 ax.bar(xs, np.nan_to_num(vals, nan=0.0), width=w * 0.92, color=col, hatch=hatch, edgecolor="white", linewidth=0.5, label=pretty(a))191 for xi, v, n in zip(xs, vals, g["n"]):192 if not np.isfinite(v):193 ax.text(xi, 0.002, "n<30", rotation=90, ha="center", va="bottom", fontsize=6, color=GREY)194 ax.set_xticks(range(len(levels)), [f"{lv}\n(n={int(s[s['level'] == lv]['n'].max())})" for lv in levels])195 ax.set_ylabel(r"mean $|e|$")196 ax.set_xlabel(STRATUM_LABELS.get(stratum, stratum))197 ax.legend(ncol=min(3, len(arms)), loc="upper center", bbox_to_anchor=(0.5, -0.32))198 save(fig, out / "fig_strata.pdf")199200201def main(argv: list[str] | None = None) -> int:202 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)203 ap.add_argument("--eval-dir", default=str(RESULTS / "evaluation"))204 ap.add_argument("--out", default=str(FIGURES))205 ap.add_argument("--stratum", default="region_group")206 ap.add_argument("--arms", default=",".join(DEFAULT_ARMS), help="arms shown in the stratified figure (≤ 6)")207 args = ap.parse_args(argv)208 ev, out = Path(args.eval_dir), Path(args.out)209 out.mkdir(parents=True, exist_ok=True)210 if not (ev / "records.parquet").exists():211 print(f"{ev}/records.parquet not found — run scripts/20_evaluate.py first")212 return 1213 records = pd.read_parquet(ev / "records.parquet")214 summary = pd.read_csv(ev / "summary_main.csv")215 fig_error_distribution(records, out)216 if (ev / "pit.csv").exists() and (ev / "pit.csv").stat().st_size > 1:217 pit = pd.read_csv(ev / "pit.csv")218 if not pit.empty:219 fig_pit(pit, out)220 if (ev / "coverage.csv").exists():221 cov = pd.read_csv(ev / "coverage.csv")222 if not cov.empty:223 fig_coverage(cov, out)224 if (ev / "rounds.csv").exists():225 fig_rounds(pd.read_csv(ev / "rounds.csv"), out)226 else:227 print("results/evaluation/rounds.csv absent — fig_rounds.pdf not produced (placeholder)")228 fig_cost_accuracy(summary, out)229 if (ev / "strata.csv").exists():230 strata = pd.read_csv(ev / "strata.csv")231 if not strata.empty:232 fig_strata(strata, out, args.stratum, [a for a in args.arms.split(",") if a][:6])233 return 0234235236if __name__ == "__main__":237 raise SystemExit(main())238