1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca3"""Additional LaTeX table fragments for the results, ablations and calibration sections (paper/tables/tab_*.tex,4booktabs, siunitx-safe, CSV twin next to each fragment). Every number is read from the versioned outputs of5scripts/20_evaluate.py (results/evaluation*/), scripts/35_rounds_curve.py (rounds_validation.csv) and6results/evaluation/contrasts.csv; the only new computations are paired block bootstraps on the stored per-record7frames (records.parquet) for contrasts that scripts/20_evaluate.py does not pre-register, using the same8block_bootstrap_contrast routine, the same blocks (municipality × month), B = 2000 and seed 20261027.910Fragments11 tab_primary_effects primary family on test: Δ, CI, SE, relative CI, minimum detectable effect (80 % power)12 tab_aggregation test: mean |e| of the committee arms under each aggregation rule13 tab_rounds validation rounds grid: per-round mean |e| of the committee median, IQR of log estimates14 tab_deliberation per run: IQR round 0 / final, convergence, anchoring index, share moving toward the median15 tab_committees validation: E, F, G2 by provider committee + paired contrasts vs the mixed committee16 tab_contrasts_extra validation exploratory paired contrasts (G2 vs G-fullview, committees)17 tab_spatial spatial holdout: in-region vs withheld-region models paired on the 8,988 withheld sales18 tab_strata_compact test: mean |e| by stratum for D, E, F, G2, I, J (mid) and G2 (small); suppressed cells = --19 tab_coverage_subgroups test: conditional 50 / 90 % coverage by subgroup (non-suppressed cells)20 tab_drift test / holdout / prospective: accuracy and calibration of D (full and paired), E, F, G221 tab_faithfulness faithfulness audit per run with the split made explicit2223Usage: python3 scripts/22_extra_tables.py24"""25from __future__ import annotations2627import json28import sys29from pathlib import Path3031import numpy as np32import pandas as pd3334ROOT = Path(__file__).resolve().parents[1]35sys.path.insert(0, str(ROOT / "src"))36from agentic_valuation.evaluation.inference import block_bootstrap_contrast # noqa: E40237from agentic_valuation.evaluation.metrics import per_record # noqa: E40238from agentic_valuation.evaluation.tables import arm_pretty, escape, num, tabular # noqa: E4023940RES = ROOT / "results"41OUT = ROOT / "paper" / "tables"42B = 200043SEED = 2026102744Z80 = 1.959964 + 0.841621 # two-sided 5 % test, 80 % power454647def write(tex: str, stem: str, csv_df: pd.DataFrame | None = None) -> None:48 body = tex.rstrip()49 assert not body.endswith(r"\\"), "fragment must not end with \\\\"50 path = OUT / f"{stem}.tex"51 path.write_text("% Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca\n% generated by scripts/22_extra_tables.py — do not edit\n" + body + "\n", encoding="utf-8")52 if csv_df is not None:53 csv_df.to_csv(path.with_suffix(".csv"), index=False)54 print("wrote", path.relative_to(ROOT))555657def ci(lo: float, hi: float, d: int = 3) -> str:58 if not (np.isfinite(lo) and np.isfinite(hi)):59 return "{--}"60 return f"{{[{lo:.{d}f}, {hi:.{d}f}]}}"616263def short(arm: str) -> str:64 """Compact arm label: G_debate_r2 → G2, G_fullview → G-fullview, E_single_llm_small → E (small)."""65 base = {"A_hedonic": "A", "B_hierarchical": "B", "C_gbm": "C (gbm)", "C_catboost": "C (catboost)", "C_xgboost": "C (xgboost)", "D_avm": "D",66 "E_single_llm": "E", "F_independent": "F", "G_debate_r1": "G1", "G_debate_r2": "G2", "G_debate_r4": "G4", "G_fullview": "G-fullview",67 "H_debate_skeptic": "H", "I_placebo": "I", "J_cost_matched": "J"}68 if arm.endswith("_small"):69 return short(arm[:-6]) + " (small)"70 if arm.endswith("_spatial"):71 return short(arm[:-8]) + " (spatial)"72 return base.get(arm, arm_pretty(arm))737475COMM = {"mixed_random": "mixed", "all_anthropic": "all-Anthropic", "all_openai": "all-OpenAI", "none": "--"}767778def records(split_dir: str) -> pd.DataFrame:79 return pd.read_parquet(RES / split_dir / "records.parquet")808182def run_records(run_id: str, aggregation: str = "median") -> pd.DataFrame:83 """Per-record frame of one stored run (results/runs/<run_id>/predictions.parquet joined to the analytic table);84 needed for the provider committees, which records.parquet (main configuration only) does not carry."""85 preds = pd.read_parquet(RES / "runs" / run_id / "predictions.parquet")86 preds = preds[preds["aggregation"] == aggregation]87 truth = pd.read_parquet(ROOT / "data" / "processed" / "analytic.parquet")88 return per_record(preds, truth)899091def pnum(p: float) -> str:92 if not np.isfinite(p):93 return "{--}"94 return r"{$<$0.001}" if p < 0.001 else num(p)959697def sub(df: pd.DataFrame, arm: str, committee: str | None = None, aggregation: str | None = None) -> pd.DataFrame:98 m = df["arm"] == arm99 if committee is not None:100 m &= df["committee"] == committee101 if aggregation is not None:102 m &= df["aggregation"] == aggregation103 s = df.loc[m]104 if s["tx_id"].duplicated().any():105 raise ValueError(f"ambiguous selection {arm}/{committee}/{aggregation}")106 return s107108109def paired(a: pd.DataFrame, b: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]:110 a = a.set_index("tx_id")111 b = b.set_index("tx_id")112 common = a.index.intersection(b.index)113 return a.loc[common].reset_index(), b.loc[common].reset_index()114115116def contrast(a: pd.DataFrame, b: pd.DataFrame, metric: str = "mean_abs_log_error") -> dict[str, float]:117 a, b = paired(a, b)118 return block_bootstrap_contrast(a, b, metric, B=B, seed=SEED)119120121# ------------------------------------------------------------------------------------------ 1. primary effects122123def tab_primary_effects() -> None:124 c = pd.read_csv(RES / "evaluation" / "contrasts.csv")125 c = c[c["family"].isin(["primary", "primary_tail"])].copy()126 rows, out = [], []127 for _, r in c.iterrows():128 se = r["se_boot"]129 half = 1.959964 * se130 mde = Z80 * se131 rel_lo, rel_hi = r["ci_lo"] / r["value_a"], r["ci_hi"] / r["value_a"]132 lab = f"{short(r['arm_b'])} vs {short(r['arm_a'])}"133 met = r"Mean $|e|$" if r["metric"] == "mean_abs_log_error" else r"P95 $|e|$"134 rows.append([lab, f"{{{met}}}", num(r["diff"]), ci(r["ci_lo"], r["ci_hi"]), num(se, 4), num(100 * r["rel_diff"], 1),135 ci(100 * rel_lo, 100 * rel_hi, 1), num(mde, 3), num(100 * mde / r["value_a"], 1), num(r["p_holm"]) if np.isfinite(r["p_holm"]) else "{--}"])136 out.append({"contrast": lab, "metric": r["metric"], "diff": r["diff"], "ci_lo": r["ci_lo"], "ci_hi": r["ci_hi"], "se_boot": se,137 "rel_diff_pct": 100 * r["rel_diff"], "rel_ci_lo_pct": 100 * rel_lo, "rel_ci_hi_pct": 100 * rel_hi, "half_width": half,138 "mde80_abs": mde, "mde80_rel_pct": 100 * mde / r["value_a"], "p_holm": r["p_holm"], "value_a": r["value_a"], "value_b": r["value_b"]})139 header = ["Contrast", "{Metric}", r"{$\Delta$}", r"{95\% CI}", "{SE}", r"{$\Delta$ (\%)}", r"{95\% CI (\%)}", "{MDE}", r"{MDE (\%)}", r"{$p$ (Holm)}"]140 write(tabular(header, rows, "llrcrrcrrr"), "tab_primary_effects", pd.DataFrame(out))141142143# ------------------------------------------------------------------------------------------ 2. aggregation rules144145def tab_aggregation() -> None:146 s = pd.read_csv(RES / "evaluation" / "summary.csv")147 arms = ["F_independent", "G_debate_r2", "H_debate_skeptic", "I_placebo", "J_cost_matched", "F_independent_small", "G_debate_r2_small"]148 rules = [("median", "Median (primary)"), ("mean", "Arithmetic mean"), ("trimmed_mean_20", r"Trimmed mean (20\%)"),149 ("inverse_variance", "Inverse variance (validation weights)"), ("stacking", "Stacking (validation weights)"),150 ("robust_bayes", "Robust Student-$t$ pool"), ("mixture", "Mixture of role distributions"),151 ("samples_mean", "J: mean of 12 samples"), ("samples_median", "J: median of 12 samples"), ("self_refine", "J: self-refined estimate")]152 rows, out = [], []153 for key, lab in rules:154 row = [lab]155 rec = {"aggregation": key}156 for a in arms:157 x = s[(s["arm"] == a) & (s["aggregation"] == key)]158 v = float(x["mean_abs_log_error"].iloc[0]) if len(x) else np.nan159 row.append(num(v))160 rec[a] = v161 rows.append(row)162 out.append(rec)163 header = ["Aggregation rule"] + [f"{{{short(a)}}}" for a in arms]164 write(tabular(header, rows, "l" + "r" * len(arms), midrules_after=[6]), "tab_aggregation", pd.DataFrame(out))165166167# ------------------------------------------------------------------------------------------ 3. rounds grid168169def tab_rounds() -> None:170 r = pd.read_csv(RES / "evaluation" / "rounds_validation.csv")171 order = [("G_debate_r1", "mixed_random"), ("G_debate_r2", "mixed_random"), ("G_debate_r2", "all_anthropic"), ("G_debate_r2", "all_openai"),172 ("G_debate_r4", "mixed_random"), ("G_fullview", "mixed_random")]173 rows, out, i = [], [], 0174 mid = []175 for arm, comm in order:176 g = r[(r["arm"] == arm) & (r["committee"] == comm)]177 rounds = sorted(g["round"].unique())178 for k, rd in enumerate(rounds):179 med = g[(g["round"] == rd) & (g["metric"] == "ae_median")].iloc[0]180 mean = g[(g["round"] == rd) & (g["metric"] == "ae_mean")].iloc[0]181 iqr = g[(g["round"] == rd) & (g["metric"] == "iqr")].iloc[0]182 rows.append([short(arm) if k == 0 else "", COMM[comm] if k == 0 else "", str(int(rd)), str(int(med["n"])), num(med["value"]), ci(med["ci_lo"], med["ci_hi"]),183 num(mean["value"]), num(iqr["value"]), ci(iqr["ci_lo"], iqr["ci_hi"])])184 out.append({"arm": arm, "committee": comm, "round": int(rd), "n": int(med["n"]), "ae_median": med["value"], "ae_median_lo": med["ci_lo"], "ae_median_hi": med["ci_hi"],185 "ae_mean": mean["value"], "iqr": iqr["value"], "iqr_lo": iqr["ci_lo"], "iqr_hi": iqr["ci_hi"]})186 i += 1187 mid.append(i - 1)188 header = ["Arm", "Committee", "{Round}", "{$n$}", r"{Mean $|e|$ (median)}", r"{95\% CI}", r"{Mean $|e|$ (mean)}", "{IQR}", r"{95\% CI}"]189 write(tabular(header, rows, "llrrrcrrc", midrules_after=mid[:-1]), "tab_rounds", pd.DataFrame(out))190191192# ------------------------------------------------------------------------------------------ 4. deliberation statistics193194SPLIT_OF = {"final_": "test (mid)", "final_small_": "test (small)", "holdout_": "holdout 2026", "prospective_": "prospective", "validation_": "validation"}195196197def split_label(run_id: str) -> str:198 for k in ["final_small_", "final_", "holdout_", "prospective_", "validation_"]:199 if run_id.startswith(k):200 return SPLIT_OF[k]201 return run_id202203204def tab_deliberation() -> None:205 f = pd.read_csv(RES / "evaluation" / "faithfulness.csv")206 f = f[f["arm"].str.startswith(("F_", "G_", "H_", "I_"))].copy()207 f["split"] = f["run_id"].map(split_label)208 order = ["validation", "test (mid)", "test (small)", "holdout 2026", "prospective"]209 f["k"] = f["split"].map({s: i for i, s in enumerate(order)})210 f = f.sort_values(["k", "arm", "committee"])211 rows, mid, i, last = [], [], 0, None212 for _, r in f.iterrows():213 if last is not None and r["split"] != last:214 mid.append(i - 1)215 last = r["split"]216 rows.append([escape(r["split"]), short(r["arm"]), COMM[r["committee"]], str(int(r["n_properties"])), num(r["iqr_log_round0"]), num(r["iqr_log_final"]),217 num(r["convergence"]), num(r["anchoring_index"]), num(r["share_toward_median"])])218 i += 1219 header = ["Split", "Arm", "Committee", "{$n$}", r"{IQR$_0$}", r"{IQR$_R$}", "{Convergence}", "{Anchoring}", "{To median}"]220 cols = ["split", "run_id", "arm", "committee", "n_properties", "iqr_log_round0", "iqr_log_final", "convergence", "anchoring_index", "share_toward_median"]221 write(tabular(header, rows, "lllrrrrrr", midrules_after=mid), "tab_deliberation", f[cols])222223224# ------------------------------------------------------------------------------------------ 5. committees + extra contrasts225226def tab_committees_and_extra() -> None:227 s = pd.read_csv(RES / "evaluation_validation" / "summary.csv")228 f = pd.read_csv(RES / "evaluation" / "faithfulness.csv").set_index("run_id")229 rec = records("evaluation_validation")230 rows, out, i, mid = [], [], 0, []231 for arm in ["E_single_llm", "F_independent", "G_debate_r2"]:232 for comm in ["all_anthropic", "all_openai", "mixed_random"]:233 x = s[(s["arm"] == arm) & (s["committee"] == comm) & (s["aggregation"] == "median")].iloc[0]234 run = x["run_id"]235 fa = f.loc[run] if run in f.index else None236 if comm != "mixed_random":237 a = run_records(f"validation_{arm}_mixed_random_mid")238 b = run_records(run)239 c = contrast(a, b)240 d, lo, hi, p = c["diff"], c["ci_lo"], c["ci_hi"], c["p_boot"]241 else:242 d = lo = hi = p = np.nan243 rows.append([short(arm), COMM[comm], str(int(x["n"])), num(x["mean_abs_log_error"]), num(x["p95_abs_log_error"]), num(x["bias_log"]),244 num(x["coverage_90"]), num(x["mean_log_width_90"]),245 num(fa["iqr_log_final"]) if fa is not None and arm == "G_debate_r2" else "{--}",246 num(fa["anchoring_index"]) if fa is not None and arm == "G_debate_r2" else "{--}",247 num(d) if np.isfinite(d) else "{--}", ci(lo, hi), pnum(p)])248 out.append({"arm": arm, "committee": comm, "run_id": run, "n": x["n"], "mean_abs_log_error": x["mean_abs_log_error"], "mdape": x["mdape"],249 "p95_abs_log_error": x["p95_abs_log_error"], "bias_log": x["bias_log"], "coverage_90": x["coverage_90"], "mean_log_width_90": x["mean_log_width_90"],250 "cost_usd_mean": x["cost_usd_mean"], "iqr_log_final": fa["iqr_log_final"] if fa is not None else np.nan,251 "anchoring_index": fa["anchoring_index"] if fa is not None else np.nan, "diff_vs_mixed": d, "ci_lo": lo, "ci_hi": hi, "p_boot": p})252 i += 1253 mid.append(i - 1)254 header = ["Arm", "Committee", "{$n$}", r"{Mean $|e|$}", r"{P95 $|e|$}", r"{Bias}", r"{Cov. 90\%}", r"{Width 90\%}",255 r"{IQR$_R$}", "{Anchoring}", r"{$\Delta$ vs mixed}", r"{95\% CI}", r"{$p$}"]256 write(tabular(header, rows, "llrrrrrrrrrcr", midrules_after=mid[:-1]), "tab_committees", pd.DataFrame(out))257258 # exploratory paired contrasts on validation that 20_evaluate.py does not pre-register259 pairs = [("G_debate_r2", "mixed_random", "G_fullview", "mixed_random", "G-fullview vs G2 (evidence partition)"),260 ("I_placebo", "mixed_random", "G_fullview", "mixed_random", "G-fullview vs I (prompts at full view)"),261 ("F_independent", "mixed_random", "G_fullview", "mixed_random", "G-fullview vs F")]262 rows, out = [], []263 for arm_a, ca, arm_b, cb, lab in pairs:264 for metric, mlab in [("mean_abs_log_error", r"Mean $|e|$"), ("mdape", "MdAPE"), ("p95_abs_log_error", r"P95 $|e|$"), ("coverage_90", r"Cov. 90\%")]:265 c = contrast(sub(rec, arm_a, ca, "median"), sub(rec, arm_b, cb, "median"), metric)266 rows.append([lab if metric == "mean_abs_log_error" else "", f"{{{mlab}}}", str(int(c["n"])), num(c["value_a"]), num(c["value_b"]), num(c["diff"]),267 ci(c["ci_lo"], c["ci_hi"]), pnum(c["p_boot"])])268 out.append({"contrast": lab, "metric": metric, **{k: c[k] for k in ["n", "n_blocks", "value_a", "value_b", "diff", "ci_lo", "ci_hi", "se_boot", "p_boot"]}})269 header = ["Contrast (B vs A)", "{Metric}", "{$n$}", "{A}", "{B}", r"{$\Delta$}", r"{95\% CI}", r"{$p$ (boot)}"]270 write(tabular(header, rows, "llrrrrcr", midrules_after=[3, 7]), "tab_contrasts_extra", pd.DataFrame(out))271272273# ------------------------------------------------------------------------------------------ 6. spatial holdout274275def tab_spatial() -> None:276 rec = records("evaluation")277 rows, out = [], []278 for base in ["A_hedonic", "C_gbm", "D_avm"]:279 sp = sub(rec, base + "_spatial")280 full = sub(rec, base)281 a, b = paired(full, sp) # A = model trained with the regions, B = model trained without them282 for metric, mlab in [("mean_abs_log_error", r"Mean $|e|$"), ("mdape", "MdAPE"), ("p95_abs_log_error", r"P95 $|e|$"), ("bias_log", r"Bias $\bar e$"), ("coverage_90", r"Cov. 90\%")]:283 c = block_bootstrap_contrast(a, b, metric, B=B, seed=SEED)284 rows.append([short(base) if metric == "mean_abs_log_error" else "", f"{{{mlab}}}", str(int(c["n"])), num(c["value_a"]), num(c["value_b"]), num(c["diff"]),285 ci(c["ci_lo"], c["ci_hi"]), pnum(c["p_boot"])])286 out.append({"arm": base, "metric": metric, **{k: c[k] for k in ["n", "n_blocks", "value_a", "value_b", "diff", "ci_lo", "ci_hi", "se_boot", "p_boot"]}})287 header = ["Arm", "{Metric}", "{$n$}", "{In-region model}", "{Withheld model}", r"{$\Delta$}", r"{95\% CI}", r"{$p$ (boot)}"]288 write(tabular(header, rows, "llrrrrcr", midrules_after=[4, 9]), "tab_spatial", pd.DataFrame(out))289290291# ------------------------------------------------------------------------------------------ 7. compact strata292293STRATA = [("region_group", "Region"), ("urban_class", "Urban class"), ("property_class", "Property class"), ("price_tercile", "Price tercile"), ("year_built_decade", "Construction decade")]294295296def tab_strata_compact() -> None:297 s = pd.read_csv(RES / "evaluation" / "strata.csv")298 s = s[s["metric"] == "mean_abs_log_error"]299 arms = ["D_avm", "E_single_llm", "F_independent", "G_debate_r2", "I_placebo", "J_cost_matched", "G_debate_r2_small"]300 rows, out, mid, i = [], [], [], 0301 for stratum, lab in STRATA:302 g = s[s["stratum"] == stratum]303 levels = list(dict.fromkeys(g["level"]))304 for lv in levels:305 row = [escape(f"{lab}: {lv}")]306 rec = {"stratum": stratum, "level": lv}307 nd = g[(g["arm"] == "D_avm") & (g["level"] == lv)]308 nm = g[(g["arm"] == "G_debate_r2") & (g["level"] == lv)]309 ns = g[(g["arm"] == "G_debate_r2_small") & (g["level"] == lv)]310 row += [str(int(nd["n"].iloc[0])) if len(nd) else "{--}", str(int(nm["n"].iloc[0])) if len(nm) else "{--}", str(int(ns["n"].iloc[0])) if len(ns) else "{--}"]311 rec.update({"n_full": nd["n"].iloc[0] if len(nd) else np.nan, "n_mid": nm["n"].iloc[0] if len(nm) else np.nan, "n_small": ns["n"].iloc[0] if len(ns) else np.nan})312 for a in arms:313 x = g[(g["arm"] == a) & (g["level"] == lv)]314 v = float(x["value"].iloc[0]) if len(x) and not bool(x["suppressed"].iloc[0]) else np.nan315 row.append(num(v))316 rec[a] = v317 rows.append(row)318 out.append(rec)319 i += 1320 mid.append(i - 1)321 header = ["Stratum", "{$n$ full}", "{$n$ mid}", "{$n$ small}"] + [f"{{{short(a)}}}" for a in arms]322 write(tabular(header, rows, "lrrr" + "r" * len(arms), midrules_after=mid[:-1]), "tab_strata_compact", pd.DataFrame(out))323324325# ------------------------------------------------------------------------------------------ 8. conditional coverage326327def tab_coverage_subgroups() -> None:328 c = pd.read_csv(RES / "evaluation" / "coverage.csv")329 c = c[(c["kind"] == "interval") & (c["group"] != "all") & (c["nominal"].isin([0.5, 0.9]))]330 arms = ["D_avm", "E_single_llm", "F_independent", "G_debate_r2", "H_debate_skeptic", "I_placebo", "J_cost_matched", "G_debate_r2_small"]331 groups = [("region_group", "Region"), ("property_class", "Property class"), ("price_tercile", "Price tercile")]332 rows, out, mid, i = [], [], [], 0333 for nominal in [0.9, 0.5]:334 for grp, lab in groups:335 g = c[(c["group"] == grp) & (c["nominal"] == nominal)]336 for lv in list(dict.fromkeys(g["level_value"])):337 row = [escape(f"{lab}: {lv}")]338 rec = {"nominal": nominal, "group": grp, "level": lv}339 nm = g[(g["arm"] == "G_debate_r2") & (g["level_value"] == lv)]340 ns = g[(g["arm"] == "G_debate_r2_small") & (g["level_value"] == lv)]341 row += [str(int(nm["n"].iloc[0])) if len(nm) else "{--}", str(int(ns["n"].iloc[0])) if len(ns) else "{--}"]342 rec.update({"n_mid": nm["n"].iloc[0] if len(nm) else np.nan, "n_small": ns["n"].iloc[0] if len(ns) else np.nan})343 for a in arms:344 x = g[(g["arm"] == a) & (g["level_value"] == lv)]345 v = float(x["empirical"].iloc[0]) if len(x) and not bool(x["suppressed"].iloc[0]) else np.nan346 row.append(num(v))347 rec[a] = v348 rows.append(row)349 out.append(rec)350 i += 1351 mid.append(i - 1)352 header = ["Subgroup", "{$n$ mid}", "{$n$ small}"] + [f"{{{short(a)}}}" for a in arms]353 write(tabular(header, rows, "lrr" + "r" * len(arms), midrules_after=mid[:-1]), "tab_coverage_subgroups", pd.DataFrame(out))354355356# ------------------------------------------------------------------------------------------ 9. drift357358def tab_drift() -> None:359 rows, out = [], []360 cols = ["mean_abs_log_error", "mdape", "p95_abs_log_error", "bias_log", "severe_rate", "over_rate", "coverage_50", "coverage_90", "mean_log_width_90"]361 for split_dir, lab in [("evaluation", "test"), ("evaluation_holdout", "holdout 2026"), ("evaluation_prospective", "prospective")]:362 s = pd.read_csv(RES / split_dir / "summary_main.csv").set_index("arm")363 rec = records(split_dir)364 g2 = sub(rec, "G_debate_r2", "mixed_random", "median")365 d_full = s.loc["D_avm"]366 d_pair, _ = paired(sub(rec, "D_avm"), g2)367 from agentic_valuation.evaluation.metrics import summarise_records # noqa: E402368 dp = summarise_records(d_pair)369 entries = [("D (full partition)", d_full.to_dict()), ("D (paired subsample)", dp)]370 for arm in ["E_single_llm", "F_independent", "G_debate_r2"]:371 entries.append((short(arm), s.loc[arm].to_dict()))372 for k, (name, v) in enumerate(entries):373 rows.append([escape(lab) if k == 0 else "", name, str(int(v["n"]))] + [num(v[c]) for c in cols])374 out.append({"split": lab, "arm": name, "n": v["n"], **{c: v[c] for c in cols}})375 header = ["Split", "Arm", "{$n$}", r"{Mean $|e|$}", "{MdAPE}", r"{P95 $|e|$}", r"{Bias $\bar e$}", r"{$|e|>\log 1.25$}", r"{Over +25\%}", r"{Cov. 50\%}", r"{Cov. 90\%}", r"{Width 90\%}"]376 write(tabular(header, rows, "llr" + "r" * len(cols), midrules_after=[4, 9]), "tab_drift", pd.DataFrame(out))377378379# ------------------------------------------------------------------------------------------ 10. faithfulness with split380381def tab_faithfulness() -> None:382 f = pd.read_csv(RES / "evaluation" / "faithfulness.csv").copy()383 f["split"] = f["run_id"].map(split_label)384 order = ["test (mid)", "test (small)", "holdout 2026", "prospective", "validation"]385 f["k"] = f["split"].map({s: i for i, s in enumerate(order)})386 f = f.sort_values(["k", "arm", "committee"])387 rows, mid, i, last = [], [], 0, None388 for _, r in f.iterrows():389 if last is not None and r["split"] != last:390 mid.append(i - 1)391 last = r["split"]392 rows.append([escape(r["split"]), short(r["arm"]), COMM[r["committee"]], str(int(r["n_properties"])), str(int(r["n_turns"])), num(r["cited_share_valid"]),393 num(r["adj_share_within_40"]), num(r["share_turns_adj_ok"]), num(r["point_in_hull"]), num(r["quantile_sane"]), num(r["summary_words"], 0),394 num(r["summary_within_limit"]), num(r["validation_retries"])])395 i += 1396 header = ["Split", "Arm", "Committee", "{$n$}", "{Turns}", "{Valid ids}", r"{Adj. $\pm$40\%}", "{Turns ok}", "{In hull}", "{Quantiles ok}", "{Words}", r"{$\le$120}", "{Retries}"]397 cols = ["split", "run_id", "arm", "committee", "n_properties", "n_turns", "cited_share_valid", "share_turns_all_ids_valid", "adj_share_within_40", "share_turns_adj_ok",398 "point_in_hull", "hull_deviation_justified", "quantile_sane", "summary_words", "summary_within_limit", "validation_retries", "packet_hash_match"]399 write(tabular(header, rows, "lllrrrrrrrrrr", midrules_after=mid), "tab_faithfulness", f[cols])400401402# ------------------------------------------------------------------------------------------ 11. conformal rescaling403404def tab_conformal() -> None:405 """Split-conformal rescaling of the LLM intervals with the factors of results/evaluation/llm_conformal_scales.json406 (estimated on the validation runs only, Section 3.6 of docs/evaluation_protocol.md), applied unchanged to the407 test, holdout and prospective runs of the same arm and committee. Point and median are untouched."""408 scales = json.loads((RES / "evaluation" / "llm_conformal_scales.json").read_text())409 by_arm = {(v["arm"], v["committee"]): v["scales"] for v in scales.values()}410 rows, out, mid, i = [], [], [], 0411 for split_dir, lab, arms in [("evaluation", "test (mid)", ["E_single_llm", "F_independent", "G_debate_r2", "J_cost_matched"]),412 ("evaluation_holdout", "holdout 2026", ["E_single_llm", "F_independent", "G_debate_r2"]),413 ("evaluation_prospective", "prospective", ["E_single_llm", "F_independent", "G_debate_r2"])]:414 rec = records(split_dir)415 for k, arm in enumerate(arms):416 r = sub(rec, arm, "mixed_random", "median")417 sc = by_arm[(arm, "mixed_random")]418 y, c = np.log(r["price"].to_numpy(float)), np.log(r["q50"].to_numpy(float))419 l90, u90 = c - np.log(r["q05"].to_numpy(float)), np.log(r["q95"].to_numpy(float)) - c420 l50, u50 = c - np.log(r["q25"].to_numpy(float)), np.log(r["q75"].to_numpy(float)) - c421 k90, k50 = sc["scale_90"], sc["scale_50"]422 cov90 = float(np.mean((y >= c - k90 * l90) & (y <= c + k90 * u90)))423 cov50 = float(np.mean((y >= c - k50 * l50) & (y <= c + k50 * u50)))424 w90 = float(np.mean(k90 * (l90 + u90)))425 rows.append([escape(lab) if k == 0 else "", short(arm), str(len(r)), num(float(r["cov_50"].mean())), num(float(r["cov_90"].mean())), num(float(r["width_90"].mean())),426 num(k50), num(k90), num(cov50), num(cov90), num(w90)])427 out.append({"split": lab, "arm": arm, "n": len(r), "coverage_50_raw": float(r["cov_50"].mean()), "coverage_90_raw": float(r["cov_90"].mean()),428 "width_90_raw": float(r["width_90"].mean()), "scale_50": k50, "scale_90": k90, "coverage_50_conformal": cov50, "coverage_90_conformal": cov90, "width_90_conformal": w90})429 i += 1430 mid.append(i - 1)431 header = ["Split", "Arm", "{$n$}", r"{Cov. 50\% raw}", r"{Cov. 90\% raw}", r"{Width 90\% raw}", r"{$k_{50}$}", r"{$k_{90}$}", r"{Cov. 50\% conf.}", r"{Cov. 90\% conf.}", r"{Width 90\% conf.}"]432 write(tabular(header, rows, "llrrrrrrrrr", midrules_after=mid[:-1]), "tab_conformal", pd.DataFrame(out))433434435def main() -> None:436 OUT.mkdir(parents=True, exist_ok=True)437 tab_conformal()438 tab_primary_effects()439 tab_aggregation()440 tab_rounds()441 tab_deliberation()442 tab_committees_and_extra()443 tab_spatial()444 tab_strata_compact()445 tab_coverage_subgroups()446 tab_drift()447 tab_faithfulness()448 (RES / "tables" / "extra_tables_manifest.json").write_text(json.dumps({"script": "scripts/22_extra_tables.py", "B": B, "seed": SEED,449 "fragments": sorted(p.name for p in OUT.glob("tab_*.tex"))}, indent=1))450451452if __name__ == "__main__":453 main()454