1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca3"""Phase 6 — non-LLM baselines (A hedonic, B hierarchical/repeat-sales, C LightGBM [+CatBoost/XGBoost], D AVM) on the4prospective evaluation set.56The fitted baseline objects are not persisted by scripts/10_train_baselines.py, so the arms are refit here with the7identical protocol and seeds: train on `train`, tune on `validation` (same random search, 10 trials), and predict the8`prospective` rows. The feature matrix is built with features.matrix.build_feature_matrix on the prospective bundle9(snapshot ∪ prospective rows) because the as-of neighbourhood / municipality features of a prospective sale need the10snapshot history; the matrix is restricted to train, validation and prospective rows. As-of invariance is verified:11train/validation feature rows rebuilt on the union must equal those of data/processed/feature_matrix.parquet.1213Usage: python3 scripts/62_prospective_baselines.py [--trials 10] [--skip-robustness]14Output: results/runs/prospective_baselines/{predictions.parquet, manifest.json, *_search.json}15"""16from __future__ import annotations1718import argparse19import json20import platform21import sys22import time23from pathlib import Path2425import numpy as np26import polars as pl2728sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))29from agentic_valuation.config import PROCESSED, RESULTS # noqa: E40230from agentic_valuation.baselines.gbm import AVM, CatBoostModel, LightGBMModel, XGBoostModel, dump_search_log # noqa: E40231from agentic_valuation.baselines.hedonic import HedonicRidge, HierarchicalHedonic # noqa: E40232from agentic_valuation.features.matrix import build_feature_matrix # noqa: E4023334BUNDLE = PROCESSED / "prospective_bundle"35RUN_ID = "prospective_baselines"363738def asof_invariance(fm_new: pl.DataFrame, fm_ref: pl.DataFrame) -> dict:39 """Max absolute difference of every numeric feature between the union-built matrix and the reference matrix,40 on train ∪ validation rows (must be 0: only later-dated sales were added)."""41 keep = fm_ref.filter(pl.col("split").is_in(["train", "validation"]))42 num = [c for c, t in keep.schema.items() if t == pl.Float64 and c not in ("price", "log_price")]43 j = keep.select(["tx_id", *num]).join(fm_new.select(["tx_id", *[pl.col(c).alias(f"{c}__new") for c in num]]), on="tx_id", how="inner")44 diffs = {}45 for c in num:46 a, b = j[c].to_numpy(), j[f"{c}__new"].to_numpy()47 both_nan = np.isnan(a) & np.isnan(b)48 d = np.where(both_nan, 0.0, np.abs(a - b))49 diffs[c] = float(np.nanmax(d)) if d.size else 0.050 return {"rows_compared": j.height, "max_abs_diff": max(diffs.values()) if diffs else 0.0, "by_feature_nonzero": {k: v for k, v in diffs.items() if v > 0}}515253def main(argv: list[str] | None = None) -> None:54 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)55 ap.add_argument("--trials", type=int, default=10, help="random-search trials (10 = the count used by results/runs/baselines)")56 ap.add_argument("--skip-robustness", action="store_true", help="skip C_catboost / C_xgboost")57 a = ap.parse_args(argv)58 out = RESULTS / "runs" / RUN_ID59 out.mkdir(parents=True, exist_ok=True)60 t0 = time.perf_counter()61 analytic = pl.read_parquet(BUNDLE / "analytic.parquet")62 splits = pl.read_parquet(BUNDLE / "splits.parquet")63 geo_path = BUNDLE / "geo_features.parquet"64 geo = pl.read_parquet(geo_path) if geo_path.exists() else None65 fm = build_feature_matrix(analytic, splits, geo=geo, comps_summary=True)66 fm = fm.filter(pl.col("split").is_in(["train", "validation", "prospective"]))67 t_fm = time.perf_counter() - t068 ref_path = PROCESSED / "feature_matrix.parquet"69 invariance = asof_invariance(fm, pl.read_parquet(ref_path)) if ref_path.exists() else {"skipped": "reference feature matrix missing"}70 train = fm.filter(pl.col("split") == "train")71 valid = fm.filter(pl.col("split") == "validation")72 pros = fm.filter(pl.col("split") == "prospective")73 print(f"feature matrix {fm.height:,} rows in {t_fm:.0f}s | train {train.height:,} | valid {valid.height:,} | prospective {pros.height:,} | as-of invariance max diff {invariance.get('max_abs_diff')}")74 if pros.height == 0:75 sys.exit("no prospective rows in the bundle: run scripts/61_build_prospective.py first")7677 LAGS = ("mun_", "nb_")78 models = {79 "A_hedonic": lambda: HedonicRidge(),80 "B_hierarchical": lambda: HierarchicalHedonic(),81 "C_gbm": lambda: LightGBMModel(n_trials=a.trials, exclude_prefixes=LAGS),82 "D_avm": lambda: AVM(n_trials=a.trials),83 }84 if not a.skip_robustness:85 models["C_catboost"] = lambda: CatBoostModel(exclude_prefixes=LAGS)86 models["C_xgboost"] = lambda: XGBoostModel(exclude_prefixes=LAGS)87 records, manifest = [], {88 "run_id": RUN_ID, "python": platform.python_version(), "machine": platform.machine(), "trials": a.trials,89 "data_dir": str(BUNDLE.relative_to(PROCESSED.parents[1])), "geo_joined": geo is not None,90 "n_train": train.height, "n_validation": valid.height, "n_prospective": pros.height, "feature_matrix_seconds": round(t_fm, 1),91 "asof_invariance_train_validation": invariance, "protocol": "fit on train, tune on validation, predict prospective (identical to scripts/10_train_baselines.py)",92 "arms": {},93 }94 for arm, ctor in models.items():95 t1 = time.perf_counter()96 m = ctor().fit(train, valid)97 fit_s = time.perf_counter() - t198 t2 = time.perf_counter()99 pred = m.predict(pros)100 records.append(pred.to_records("prospective", arm, RUN_ID, latency_s=time.perf_counter() - t2))101 info = {"fit_seconds": round(fit_s, 1)}102 if isinstance(m, LightGBMModel):103 dump_search_log(m, out / f"{arm}_search.json")104 info["params"] = m.params105 if isinstance(m, AVM):106 dump_search_log(m.point, out / f"{arm}_search.json")107 info["params"] = m.point.params108 info["conformal_scale"] = m.scale109 if isinstance(m, HierarchicalHedonic):110 info["w_repeat"] = m.w_repeat111 manifest["arms"][arm] = info112 print(f"{arm}: fit {fit_s:.0f}s {info}", flush=True)113 preds = pl.concat(records)114 preds.write_parquet(out / "predictions.parquet")115 manifest["n_prediction_rows"] = preds.height116 manifest["seconds"] = round(time.perf_counter() - t0, 1)117 (out / "manifest.json").write_text(json.dumps(manifest, indent=1, default=float))118 truth = fm.select(["tx_id", "log_price"])119 s = preds.join(truth, on="tx_id").with_columns((pl.col("point").log() - pl.col("log_price")).abs().alias("ae")).group_by(["arm", "split"]).agg(pl.col("ae").mean().round(4).alias("mean_abs_log_error"), pl.col("ae").median().round(4).alias("median_abs_log_error"), pl.len()).sort(["split", "arm"])120 print(s)121122123if __name__ == "__main__":124 main()125