1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""Comparable retrieval, market context, geo context and feature matrix (synthetic data)."""3from __future__ import annotations45from datetime import date, timedelta67import numpy as np8import polars as pl9import pytest1011from agentic_valuation.features.comparables import retrieve_comparables, retrieve_comparables_detailed12from agentic_valuation.features.geo import GeoLayers, geo_context13from agentic_valuation.features.market import market_context14from agentic_valuation.features.matrix import CATEGORICAL, ID_COLUMNS, build_feature_matrix, feature_columns, municipality_rolling, neighbour_summary15from agentic_valuation.schemas import Comparable, GeoContext, MarketContext161718def _subjects(df: pl.DataFrame, n: int = 60) -> pl.DataFrame:19 return df.filter(pl.col("date") >= date(2024, 1, 1)).sample(n=n, seed=1)202122def test_comparables_contract_and_order(syn_analytic: pl.DataFrame, cfg: dict) -> None:23 for r in _subjects(syn_analytic, 60).iter_rows(named=True):24 cutoff = r["date"] - timedelta(days=1)25 cs = retrieve_comparables_detailed(syn_analytic, r, cutoff, cfg)26 comps = cs.comparables27 assert all(isinstance(c, Comparable) for c in comps)28 assert len(comps) <= cfg["comparables"]["max_comps"]29 assert [c.evidence_id for c in comps] == [f"C{i:02d}" for i in range(1, len(comps) + 1)]30 assert all(c.sale_date < cutoff for c in comps)31 assert all((cutoff - c.sale_date).days <= cfg["comparables"]["lookback_days"] for c in comps)32 assert all(c.property_class == r["property_class"] for c in comps)33 assert all(c.distance_km <= cs.radius_km + 1e-6 for c in comps)34 assert all(c.months_before_valuation > 0 for c in comps)35 if len(comps) < cfg["comparables"]["min_comps"]:36 assert cs.expanded and cs.notes373839def test_comparables_exclude_subject_parcel(syn_analytic: pl.DataFrame, cfg: dict) -> None:40 repeat = syn_analytic.filter(pl.col("prior_sale_date").is_not_null()).head(30)41 for r in repeat.iter_rows(named=True):42 cutoff = r["date"] - timedelta(days=1)43 own = syn_analytic.filter(pl.col("parcel_id") == r["parcel_id"])44 own_keys = set(zip(own["date"].to_list(), own["price"].to_list()))45 for c in retrieve_comparables(syn_analytic, r, cutoff, cfg):46 assert not ((c.sale_date, c.price) in own_keys and c.distance_km == 0.0)474849def test_comparables_accept_frame_and_future_blind(syn_analytic: pl.DataFrame, cfg: dict) -> None:50 r = _subjects(syn_analytic, 1)51 cutoff = r["date"][0] - timedelta(days=1)52 a = retrieve_comparables(syn_analytic, r, cutoff, cfg)53 pert = syn_analytic.with_columns(pl.when(pl.col("date") >= cutoff).then(pl.col("price") * 3).otherwise(pl.col("price")).alias("price"))54 b = retrieve_comparables(pert, r.row(0, named=True), cutoff, cfg)55 assert [c.model_dump() for c in a] == [c.model_dump() for c in b]565758def test_market_context_asof(syn_analytic: pl.DataFrame, cfg: dict) -> None:59 for r in _subjects(syn_analytic, 10).iter_rows(named=True):60 cutoff = r["date"] - timedelta(days=1)61 m = market_context(syn_analytic, r, cutoff, cfg)62 assert isinstance(m, MarketContext) and m.as_of == cutoff63 assert all(rec["month"] <= cutoff.strftime("%Y-%m") for rec in m.municipality_price_index)64 if m.municipality_price_index:65 assert m.municipality_price_index[0]["index"] == 100.066 assert m.mortgage_rate_5y_fixed is None67 pert = syn_analytic.with_columns(pl.when(pl.col("date") > cutoff).then(pl.col("price") * 5).otherwise(pl.col("price")).alias("price"))68 assert market_context(pert, r, cutoff, cfg).model_dump() == m.model_dump()697071def test_feature_matrix_shape_and_columns(syn_matrix: pl.DataFrame, syn_analytic: pl.DataFrame) -> None:72 assert syn_matrix.height == syn_analytic.height73 num, cat = feature_columns(syn_matrix)74 assert set(cat) == set(CATEGORICAL)75 assert "price" not in num and "log_price" not in num and "price_tercile" not in num76 assert {"nb_n_12m", "nb_med_ppm2", "mun_med_ppm2_12m", "prior_sale_log_price", "month_index"} <= set(num)77 assert syn_matrix["tx_id"].n_unique() == syn_matrix.height78 for c in num:79 assert syn_matrix[c].dtype == pl.Float64808182def test_neighbour_summary_matches_brute_force(syn_analytic: pl.DataFrame) -> None:83 nb = neighbour_summary(syn_analytic)84 sample = syn_analytic.filter(pl.col("date") >= date(2023, 1, 1)).sample(n=40, seed=3)85 for r in sample.iter_rows(named=True):86 cand = syn_analytic.filter(87 (pl.col("property_class") == r["property_class"]) & (pl.col("parcel_id") != r["parcel_id"])88 & (pl.col("date") < r["date"]) & (pl.col("date") >= r["date"] - timedelta(days=365))89 ).with_columns(((pl.col("x_m") - r["x_m"]) ** 2 + (pl.col("y_m") - r["y_m"]) ** 2).sqrt().alias("d"))90 cand = cand.filter(pl.col("d") <= 3000.0)91 got = nb.filter(pl.col("tx_id") == r["tx_id"]).row(0, named=True)92 assert got["nb_n_12m"] == cand.height93 if cand.height:94 assert abs(got["nb_med_log_price"] - float(cand["log_price"].median())) < 1e-995 ppm2 = (cand["price"] / cand["floor_area_m2"]).drop_nulls()96 assert abs(got["nb_med_ppm2"] - float(ppm2.median())) < 1e-997 else:98 assert got["nb_med_log_price"] is None or np.isnan(got["nb_med_log_price"])99100101def test_municipality_rolling_matches_brute_force(syn_analytic: pl.DataFrame) -> None:102 mr = municipality_rolling(syn_analytic)103 sample = syn_analytic.filter(pl.col("date") >= date(2022, 6, 1)).sample(n=40, seed=4)104 for r in sample.iter_rows(named=True):105 cand = syn_analytic.filter(106 (pl.col("mun_code") == r["mun_code"]) & (pl.col("property_class") == r["property_class"])107 & (pl.col("date") < r["date"]) & (pl.col("date") >= r["date"] - timedelta(days=365))108 )109 got = mr.filter(pl.col("tx_id") == r["tx_id"]).row(0, named=True)110 assert got["mun_n_12m"] == cand.height111 if cand.height:112 assert abs(got["mun_med_log_price_12m"] - float(cand["log_price"].median())) < 1e-9113114115def test_feature_matrix_future_blind(syn_analytic: pl.DataFrame, syn_splits: pl.DataFrame) -> None:116 cutoff = date(2025, 4, 1)117 base = build_feature_matrix(syn_analytic, syn_splits).filter(pl.col("date") < cutoff).sort("tx_id")118 trunc = build_feature_matrix(syn_analytic.filter(pl.col("date") < cutoff), syn_splits).sort("tx_id")119 assert base["tx_id"].to_list() == trunc["tx_id"].to_list()120 for c in base.columns:121 a, b = base[c], trunc[c]122 if a.dtype == pl.Float64:123 a, b = a.fill_nan(None), b.fill_nan(None)124 assert (((a - b).abs() < 1e-9) | (a.is_null() & b.is_null())).all(), c125 else:126 assert a.eq_missing(b).all(), c127128129def test_geo_context_without_layers_returns_none(syn_analytic: pl.DataFrame) -> None:130 r = syn_analytic.head(1).row(0, named=True)131 assert geo_context(r, None) is None132 cached = pl.DataFrame({"tx_id": [r["tx_id"]], "flood_mapped": [True], "in_flood_zone": [False], "flood_zone_kind": [None], "transit_stops_within_1km": [12],133 "dauid": ["24660001"], "da_population": [500], "da_population_density": [4000.0], "da_median_age": [40.0], "da_median_income": [45000.0],134 "da_pct_renters": [60.0], "cbd_name": ["Montréal"], "distance_to_cbd_km": [3.2]})135 g = geo_context(r, None, cached=cached)136 assert isinstance(g, GeoContext) and g.transit_stops_within_1km == 12 and g.in_flood_zone is False and g.distance_to_cbd_km == 3.2137138139@pytest.mark.skipif(not GeoLayers.available(), reason="external layers not copied")140def test_geo_layers_point_queries(syn_analytic: pl.DataFrame) -> None:141 layers = GeoLayers.load()142 # Old Montréal: many stops, in a DA, near the Montréal CBD143 g = geo_context({"tx_id": "x", "lat": 45.5075, "lng": -73.5540}, layers)144 assert g is not None and g.transit_stops_within_1km > 10 and g.distance_to_cbd_km < 2.0 and g.da_population_density is not None145 # Gatineau point → CBD Gatineau146 g2 = geo_context({"tx_id": "y", "lat": 45.4290, "lng": -75.7140}, layers)147 assert g2 is not None and "Gatineau" in g2.notes[-1]148149150@pytest.mark.real151def test_real_feature_matrix_future_blind_sample(real_bundle: dict[str, pl.DataFrame]) -> None:152 an = real_bundle["analytic"]153 muns = an["mun_code"].unique().sort().head(15).to_list()154 sub = an.filter(pl.col("mun_code").is_in(muns))155 cutoff = date(2025, 4, 1)156 base = build_feature_matrix(sub, real_bundle["splits"]).filter(pl.col("date") < cutoff).sort("tx_id")157 trunc = build_feature_matrix(sub.filter(pl.col("date") < cutoff), real_bundle["splits"]).sort("tx_id")158 assert base.drop(["price", "log_price"]).equals(trunc.drop(["price", "log_price"]))159