1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""Evaluate prediction runs against the resale-implied value (docs/fundamental_value.md §5).34Reading. For a sale (i, t1) with resale target V^R (repeat_sales.resale_targets), an arm's estimate ŷ has two errors:5 e^P = log ŷ − log P1 (price-based, the paper's primary metric)6 e^R = log ŷ − log V^R (resale-based)7and the price itself has e^R_P = log P1 − log V^R = −(ε_{t2} − ε_{t1}) under the repeat-sales assumptions. The8diagnostic contrast is Δ = mean|e^R(arm)| − mean|e^R(price)|: an arm with Δ < 0 is closer to the market-adjusted9resale than the first transaction price was, which is evidence that it captured a component of value obscured by10transaction noise. Δ is estimated with the paired block bootstrap of `evaluation.inference` (blocks = municipality ×11month), the price playing the role of a pseudo-arm `P_price`. Arm-vs-arm contrasts against V^R use the same machinery.12Selection: by default, pairs held < 12 months and pairs whose later leg carries a data-quality flag are excluded.13"""14from __future__ import annotations1516from pathlib import Path17from typing import Any1819import numpy as np20import pandas as pd2122from ..evaluation._common import to_pandas23from ..evaluation.inference import compare_arms24from ..evaluation.metrics import per_record, summarise_records2526PRICE_ARM = "P_price"27RESALE_METRICS = ["mean_abs_log_error", "rmse_log", "bias_log"]282930def collect_live_runs(runs_dir: Path, *, include_fake: bool = False) -> tuple[pd.DataFrame, list[str]]:31 """All prediction tables under `runs_dir`; run ids starting with `_` (mock / synthetic) are skipped."""32 import polars as pl3334 frames, used = [], []35 for d in sorted(p for p in Path(runs_dir).iterdir() if p.is_dir()):36 if d.name.startswith("_") and not include_fake:37 continue38 f = d / "predictions.parquet"39 if not f.exists():40 continue41 df = pl.read_parquet(f).to_pandas()42 if "run_id" not in df.columns:43 df["run_id"] = d.name44 if not include_fake:45 df = df[~df["run_id"].astype(str).str.startswith("_")]46 if df.empty:47 continue48 frames.append(df)49 used.append(d.name)50 if not frames:51 return pd.DataFrame(), used52 return pd.concat(frames, ignore_index=True), used535455def select_main_config(preds: pd.DataFrame, *, committee: str = "mixed_random", aggregation: str = "median") -> pd.DataFrame:56 """One (committee, aggregation, run_id) per arm — same preference order as scripts/20_evaluate.py — so that each57 arm has one row per tx_id."""58 out = []59 for arm, g in preds.groupby("arm", sort=True):60 cands = [g[(g["committee"] == committee) & (g["aggregation"] == aggregation)], g[g["aggregation"] == aggregation],61 g[(g["committee"] == "none") & (g["aggregation"] == "model")], g]62 pick = next(c for c in cands if not c.empty)63 key = pick.sort_values(["committee", "run_id"]).iloc[-1][["committee", "aggregation", "run_id"]]64 sel = g[(g["committee"] == key["committee"]) & (g["aggregation"] == key["aggregation"]) & (g["run_id"] == key["run_id"])]65 out.append(sel.drop_duplicates(subset=["tx_id"], keep="last"))66 return pd.concat(out, ignore_index=True) if out else preds.iloc[0:0]676869def resale_truth(truth: Any, targets: pd.DataFrame, *, exclude_short: bool = True, exclude_flagged: bool = True) -> pd.DataFrame:70 """Truth table restricted to sales with a resale target; `price` is replaced by the resale-implied value so that the71 standard metric code measures e^R, and the realised price is kept in `price_realised`."""72 t = to_pandas(truth)73 t["tx_id"] = t["tx_id"].astype(str)74 tg = targets.copy()75 tg["tx_id"] = tg["tx_id"].astype(str)76 if exclude_short:77 tg = tg[~tg["short_hold"].astype(bool)]78 if exclude_flagged:79 tg = tg[~tg["resale_flagged"].astype(bool)]80 tg = tg[np.isfinite(tg["resale_implied_log_value"])]81 cols = ["tx_id", "resale_implied_value", "resale_implied_log_value", "resale_price", "resale_date", "months_held", "index_change"]82 m = t.merge(tg[cols], on="tx_id", how="inner", validate="one_to_one")83 m["price_realised"] = m["price"].astype(float)84 m["price"] = m["resale_implied_value"].astype(float)85 return m868788def evaluate_vs_resale(preds: pd.DataFrame, truth: Any, targets: pd.DataFrame, *, exclude_short: bool = True, exclude_flagged: bool = True,89 B: int = 2000, seed: int = 20261027, arm_pairs: list[tuple[str, str]] | None = None) -> tuple[pd.DataFrame, pd.DataFrame]:90 """Per-arm summary against the realised price and against V^R on the common resale subsample, and the paired91 contrasts (arm vs price-as-estimate; optional arm-vs-arm pairs) against V^R.9293 Returns (summary, contrasts). `summary` has one row per arm (plus `P_price`): n, mean|e| / rmse / bias vs price and94 vs resale-implied value. `contrasts` is the tidy output of `compare_arms` with `arm_a = P_price`."""95 tr = resale_truth(truth, targets, exclude_short=exclude_short, exclude_flagged=exclude_flagged)96 p = preds.copy()97 p["tx_id"] = p["tx_id"].astype(str)98 p = p[p["tx_id"].isin(set(tr["tx_id"]))]99 if p.empty:100 return pd.DataFrame(), pd.DataFrame()101 # pseudo-arm: the realised price as an estimate (quantiles absent)102 price_rows = tr[["tx_id", "price_realised"]].rename(columns={"price_realised": "point"})103 price_rows = price_rows[price_rows["tx_id"].isin(set(p["tx_id"]))]104 price_rows = price_rows.assign(arm=PRICE_ARM, committee="none", aggregation="model", run_id="price", split=p["split"].iloc[0] if "split" in p.columns else "n/a")105 all_preds = pd.concat([p, price_rows], ignore_index=True, sort=False)106107 truth_price = tr.drop(columns=["price"]).rename(columns={"price_realised": "price"})108 rows = []109 for arm, g in all_preds.groupby("arm", sort=True):110 rp = summarise_records(per_record(g, truth_price))111 rr = summarise_records(per_record(g, tr))112 rows.append({"arm": arm, "n": rp["n"], "mean_abs_log_error_price": rp["mean_abs_log_error"], "rmse_log_price": rp["rmse_log"], "bias_log_price": rp["bias_log"],113 "mean_abs_log_error_resale": rr["mean_abs_log_error"], "rmse_log_resale": rr["rmse_log"], "bias_log_resale": rr["bias_log"]})114 summary = pd.DataFrame(rows)115 arms = [a for a in all_preds["arm"].unique() if a != PRICE_ARM]116 pairs = [(PRICE_ARM, a) for a in sorted(arms)]117 if arm_pairs:118 pairs += [(a, b) for a, b in arm_pairs if a in arms and b in arms]119 contrasts = compare_arms(all_preds, tr, pairs, metric=RESALE_METRICS, B=B, seed=seed, dm=False, holm_family=True)120 contrasts.insert(0, "target", "resale_implied")121 return summary, contrasts122