1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""Arm errors against the realised price and against the resale-implied value (paper §sec:fundamental, Table tab:resale).34Uses every live, real run under results/runs/ (run ids starting with `_` are mock/synthetic and skipped), restricted to5the requested partitions (default: test and holdout, the inferential partitions) and to subsample properties with a6later resale (results/fundamental/resale_targets.parquet from scripts/50_repeat_sales.py; pairs held < 12 months and7flagged later legs excluded unless --keep-short / --keep-flagged).89Outputs: results/fundamental/eval_vs_resale.csv (+ eval_vs_resale_contrasts.csv, eval_vs_resale.json) and the fragment10paper/tables/tab_resale.tex. When no eligible run exists the script still runs, says so, and writes a fragment that11reads NOT YET ESTIMATED (the resale-sample description table comes from scripts/50_repeat_sales.py).1213 python3 scripts/51_eval_vs_resale.py # test + holdout14 python3 scripts/51_eval_vs_resale.py --splits validation # pilot runs (illustration only, not inferential)15"""16from __future__ import annotations1718import argparse19import json20import sys21import time22from datetime import datetime, timezone23from pathlib import Path2425import numpy as np26import pandas as pd27import polars as pl2829from agentic_valuation.config import PAPER, PROCESSED, RESULTS, SEED30from agentic_valuation.evaluation._common import prepare_truth31from agentic_valuation.evaluation.tables import arm_pretty, escape, num, tabular32from agentic_valuation.fundamental.resale_eval import PRICE_ARM, collect_live_runs, evaluate_vs_resale, select_main_config3334OUT = RESULTS / "fundamental"35PRIMARY_PAIRS = [("D_avm", "G_debate_r2"), ("F_independent", "G_debate_r2"), ("J_cost_matched", "G_debate_r2"), ("I_placebo", "G_debate_r2")]36HEADER = "% Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca\n% generated by scripts/51_eval_vs_resale.py — do not edit\n"373839def write_not_estimated(path: Path, reason: str) -> None:40 """Fragment with the table skeleton and a NOT YET ESTIMATED row; the full reason is kept in eval_vs_resale.json."""41 path.parent.mkdir(parents=True, exist_ok=True)42 short = "no live run on the test/holdout partitions yet" if "none has predictions" in reason or "no live prediction run" in reason \43 else "no evaluated property has a later resale in the snapshot"44 tex = (r"\begin{tabular}{@{}lrrrrrr@{}}" "\n" r"\toprule" "\n"45 r"Arm & {N} & {Mean $|e^{P}|$} & {Mean $|e^{R}|$} & {$\Delta$} & {95\% CI} & {$p$ (Holm)} \\" "\n"46 r"\midrule" "\n"47 r"\multicolumn{7}{@{}l}{\textbf{NOT YET ESTIMATED} --- " + escape(short) + r"} \\" "\n"48 r"\bottomrule" "\n" r"\end{tabular}")49 path.write_text(HEADER + tex + "\n", encoding="utf-8")505152def write_table(summary: pd.DataFrame, contrasts: pd.DataFrame, path: Path) -> None:53 c = contrasts[(contrasts["arm_a"] == PRICE_ARM) & (contrasts["metric"] == "mean_abs_log_error")].set_index("arm_b")54 header = ["Arm", "{N}", "{Mean $|e^{P}|$}", "{Mean $|e^{R}|$}", "{$\\Delta$}", "{95\\% CI}", "{$p$ (Holm)}"]55 rows = []56 for _, r in summary.sort_values("arm").iterrows():57 arm = r["arm"]58 if arm != PRICE_ARM and r["n"] < 30:59 continue # arms with fewer than 30 resale-linked properties are not tabulated (LLM subsamples: deferred test)60 label = "Realised price $P_{t_1}$" if arm == PRICE_ARM else arm_pretty(arm)61 if arm == PRICE_ARM:62 rows.append([label, num(r["n"], 0), "{--}", num(r["mean_abs_log_error_resale"]), "{--}", "{--}", "{--}"])63 continue64 cc = c.loc[arm] if arm in c.index else None65 rows.append([label, num(r["n"], 0), num(r["mean_abs_log_error_price"]), num(r["mean_abs_log_error_resale"]),66 num(cc["diff"]) if cc is not None else "{--}",67 f"[{cc['ci_lo']:.3f}, {cc['ci_hi']:.3f}]" if cc is not None and np.isfinite(cc["ci_lo"]) else "{--}",68 num(cc["p_holm"], 3) if cc is not None else "{--}"])69 path.parent.mkdir(parents=True, exist_ok=True)70 path.write_text(HEADER + tabular(header, rows, "@{}lrrrrrr@{}", midrules_after=[0]) + "\n", encoding="utf-8")717273def main(argv: list[str] | None = None) -> int:74 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)75 ap.add_argument("--runs-dir", default=str(RESULTS / "runs"))76 ap.add_argument("--truth", default=str(PROCESSED / "analytic.parquet"))77 ap.add_argument("--splits-file", default=str(PROCESSED / "splits.parquet"))78 ap.add_argument("--targets", default=str(OUT / "resale_targets.parquet"))79 ap.add_argument("--splits", default="test,holdout", help="comma-separated partitions to evaluate")80 ap.add_argument("--keep-short", action="store_true", help="keep pairs held < 12 months")81 ap.add_argument("--keep-flagged", action="store_true", help="keep pairs whose later leg has a data-quality flag")82 ap.add_argument("--B", type=int, default=2000)83 ap.add_argument("--seed", type=int, default=SEED)84 ap.add_argument("--main-committee", default="mixed_random")85 ap.add_argument("--main-aggregation", default="median")86 ap.add_argument("--include-fake", action="store_true")87 ap.add_argument("--out", default=str(OUT))88 ap.add_argument("--tex", default=str(PAPER / "tables" / "tab_resale.tex"))89 args = ap.parse_args(argv)90 out, tex = Path(args.out), Path(args.tex)91 out.mkdir(parents=True, exist_ok=True)92 t0 = time.time()93 splits = [s.strip() for s in args.splits.split(",") if s.strip()]9495 tp = Path(args.targets)96 if not tp.exists():97 print(f"missing {tp}: run scripts/50_repeat_sales.py first", file=sys.stderr)98 return 199 targets = pd.read_parquet(tp)100 preds, used = collect_live_runs(Path(args.runs_dir), include_fake=args.include_fake)101 status: dict = {"splits": splits, "runs_found": used, "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds")}102 if preds.empty:103 reason = f"no live prediction run under results/runs (runs starting with `_` are mock and skipped)"104 print(reason)105 write_not_estimated(tex, reason)106 status["status"] = "not_estimated"107 status["reason"] = reason108 (out / "eval_vs_resale.json").write_text(json.dumps(status, indent=2), encoding="utf-8")109 return 0110 preds = preds[preds["split"].astype(str).isin(splits)].copy()111 if preds.empty:112 reason = f"live runs exist ({', '.join(used)}) but none has predictions on the partitions {splits}"113 print(reason)114 write_not_estimated(tex, reason)115 status["status"] = "not_estimated"116 status["reason"] = reason117 (out / "eval_vs_resale.json").write_text(json.dumps(status, indent=2), encoding="utf-8")118 return 0119120 truth = prepare_truth(pl.read_parquet(args.truth))121 sp = Path(args.splits_file)122 if sp.exists():123 truth = truth.merge(pl.read_parquet(sp).select(["tx_id", "split"]).to_pandas(), on="tx_id", how="left")124 truth = truth[truth["split"].astype(str).isin(splits)]125 main = select_main_config(preds, committee=args.main_committee, aggregation=args.main_aggregation)126 n_pred_tx = main["tx_id"].nunique()127 summary, contrasts = evaluate_vs_resale(main, truth, targets, exclude_short=not args.keep_short, exclude_flagged=not args.keep_flagged,128 B=args.B, seed=args.seed, arm_pairs=PRIMARY_PAIRS)129 if summary.empty:130 reason = f"runs {', '.join(used)} cover {n_pred_tx} properties on {splits} but none has a later resale in the snapshot"131 print(reason)132 write_not_estimated(tex, reason)133 status.update({"status": "not_estimated", "reason": reason, "n_predicted_properties": int(n_pred_tx)})134 (out / "eval_vs_resale.json").write_text(json.dumps(status, indent=2), encoding="utf-8")135 return 0136 summary.insert(0, "splits", "+".join(splits))137 summary.to_csv(out / "eval_vs_resale.csv", index=False)138 contrasts.to_csv(out / "eval_vs_resale_contrasts.csv", index=False)139 write_table(summary, contrasts, tex)140 status.update({"status": "estimated", "n_predicted_properties": int(n_pred_tx), "n_with_resale": int(summary["n"].max()),141 "arms": summary["arm"].tolist(), "B": args.B, "exclude_short": not args.keep_short, "exclude_flagged": not args.keep_flagged,142 "seconds": round(time.time() - t0, 1)})143 (out / "eval_vs_resale.json").write_text(json.dumps(status, indent=2), encoding="utf-8")144 print(f"runs: {used}; partitions {splits}; predicted properties {n_pred_tx}; with resale target {int(summary['n'].max())}")145 print(summary.round(4).to_string(index=False))146 print(contrasts[contrasts["metric"] == "mean_abs_log_error"][["arm_a", "arm_b", "n", "value_a", "value_b", "diff", "ci_lo", "ci_hi", "p_boot", "p_holm"]].round(4).to_string(index=False))147 return 0148149150if __name__ == "__main__":151 sys.exit(main())152