1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""Analytic contract, derived columns, splits and subsamples (synthetic data)."""3from __future__ import annotations45from datetime import date67import numpy as np8import polars as pl9import pytest1011from agentic_valuation.data.analytic_schema import (12 ANALYTIC_COLUMNS, DERIVED_COLUMNS, REQUIRED_NON_NULL, SYNTHETIC_COLUMNS, add_prior_sale, coerce_analytic, haversine_km,13 project_xy, region_group_expr, validate_analytic,14)15from agentic_valuation.data.build import roll_reference_at_sale_expr, urban_class_expr16from agentic_valuation.data.splits import SPLIT_ORDER, assign_splits, proportional_allocation, split_windows, stratified_subsample17from tests.conftest import SYN_N_BY_SPLIT181920def test_contract_shape() -> None:21 assert len(SYNTHETIC_COLUMNS) == 2422 assert len(DERIVED_COLUMNS) == 923 assert list(ANALYTIC_COLUMNS)[:24] == list(SYNTHETIC_COLUMNS)24 assert set(REQUIRED_NON_NULL) <= set(ANALYTIC_COLUMNS)252627def test_synthetic_analytic_validates(syn_analytic: pl.DataFrame) -> None:28 validate_analytic(syn_analytic)29 assert syn_analytic.columns == list(ANALYTIC_COLUMNS)30 assert syn_analytic.height == 20_00031 # no NaN survives coercion (nulls only)32 for c, t in ANALYTIC_COLUMNS.items():33 if t == pl.Float64:34 assert syn_analytic[c].is_nan().fill_null(False).sum() == 0, c353637def test_validate_raises_on_violations(syn_analytic: pl.DataFrame) -> None:38 with pytest.raises(ValueError, match="missing columns"):39 validate_analytic(syn_analytic.drop("grid1k"))40 with pytest.raises(ValueError, match="dtype"):41 validate_analytic(syn_analytic.with_columns(pl.col("price").cast(pl.Int64)))42 with pytest.raises(ValueError, match="nulls"):43 validate_analytic(syn_analytic.with_columns(pl.when(pl.arange(0, pl.len()) == 0).then(None).otherwise(pl.col("parcel_id")).alias("parcel_id")))44 with pytest.raises(ValueError, match="unexpected columns"):45 validate_analytic(syn_analytic.with_columns(pl.lit(1).alias("extra")))46 validate_analytic(syn_analytic.with_columns(pl.lit(1).alias("extra")), allow_extra=True)47 with pytest.raises(ValueError, match="prior_sale_date"):48 bad = syn_analytic.with_columns(pl.col("date").alias("prior_sale_date"))49 validate_analytic(bad)505152def test_coerce_adds_missing_and_orders(syn_analytic: pl.DataFrame) -> None:53 shuffled = syn_analytic.select(list(reversed(syn_analytic.columns))).drop("borough_name")54 out = coerce_analytic(shuffled)55 assert out.columns == list(ANALYTIC_COLUMNS)56 assert out["borough_name"].null_count() == out.height57 assert out.schema["date"] == pl.Date585960def test_projection_matches_epsg32198() -> None:61 x, y = project_xy(np.array([46.973765046]), np.array([-69.799217111]))62 assert abs(x[0] - (-98672.84)) < 1.0 and abs(y[0] - 331793.78) < 1.0 # value from the WP 22 table63 assert abs(haversine_km(45.5017, -73.5673, 46.8139, -71.2080) - 233) < 3 # Montréal–Québec ≈ 233 km646566def test_region_group_rule() -> None:67 df = pl.DataFrame({68 "region_name": ["Montréal", "Montérégie", "Montérégie", "Capitale-Nationale", "Capitale-Nationale", "Outaouais", "Estrie"],69 "lat": [45.50, 45.40, 45.00, 46.80, 47.40, 45.45, 45.40],70 "lng": [-73.57, -73.30, -72.50, -71.22, -71.00, -75.72, -71.90],71 }).with_columns(region_group_expr().alias("rg"))72 assert df["rg"].to_list() == ["Montréal CMA", "Montréal CMA", "other", "Québec CMA", "other", "Outaouais", "other"]737475def test_prior_sale_strictly_earlier_and_same_day_invisible() -> None:76 df = pl.DataFrame({77 "tx_id": ["a", "b", "c", "d", "e"],78 "parcel_id": ["P1", "P1", "P1", "P2", "P1"],79 "date": [date(2021, 1, 1), date(2021, 1, 1), date(2022, 6, 1), date(2021, 3, 3), date(2023, 1, 1)],80 "price": [100.0, 120.0, 200.0, 50.0, 300.0],81 })82 out = add_prior_sale(df).sort("tx_id")83 assert out["prior_sale_price"].to_list() == [None, None, 120.0, None, 200.0]84 assert out["prior_sale_date"].to_list() == [None, None, date(2021, 1, 1), None, date(2022, 6, 1)]858687def test_price_tercile_balanced_by_class(syn_analytic: pl.DataFrame) -> None:88 t = syn_analytic.group_by("property_class", "price_tercile").len().sort(["property_class", "price_tercile"])89 for cls, g in t.group_by("property_class"):90 counts = g["len"].to_list()91 assert max(counts) - min(counts) <= 1, cls92 # monotone in price93 q = syn_analytic.group_by("property_class", "price_tercile").agg(pl.col("price").min().alias("lo"), pl.col("price").max().alias("hi")).sort(["property_class", "price_tercile"])94 for cls, g in q.group_by("property_class"):95 his = g.sort("price_tercile")["hi"].to_list(); los = g.sort("price_tercile")["lo"].to_list()96 assert his[0] <= los[1] and his[1] <= los[2]979899def test_urban_class_and_roll_reference_rules() -> None:100 df = pl.DataFrame({"mamh_population": [250000, 15000, 900, None], "roll_reference_date": ["2024-07-01", "2023-07-01", "2022-07-01", None],101 "date": [date(2026, 2, 1), date(2024, 12, 31), date(2021, 3, 1), date(2022, 1, 1)]})102 out = df.with_columns(urban_class_expr().alias("u"), roll_reference_at_sale_expr().alias("r"))103 assert out["u"].to_list() == ["metro", "suburban", "rural", "rural"]104 # 2024-07-01 roll in force from 2026-01-01 → k=0 ; 2023-07-01 roll in force 2025-01-01 > 2024-12-31 → step back 1 cycle ;105 # 2022-07-01 roll in force 2024-01-01, sale 2021-03 → back 1 cycle (2019-07-01 in force 2021-01-01) ; null stays null106 assert out["r"].to_list() == ["2024-07-01", "2020-07-01", "2019-07-01", None]107108109def test_splits_windows_and_assignment(syn_analytic: pl.DataFrame, syn_splits: pl.DataFrame, cfg: dict) -> None:110 w = split_windows(cfg)111 for a, b in zip(SPLIT_ORDER, SPLIT_ORDER[1:]):112 assert w[a][1] < w[b][0]113 joined = syn_analytic.select("tx_id", "date").join(syn_splits, on="tx_id")114 for name, (a, b) in w.items():115 part = joined.filter(pl.col("split") == name)116 assert part.height > 0117 assert part["date"].min() >= a and part["date"].max() <= b118 assert set(syn_splits["split"].unique()) <= set(SPLIT_ORDER) | {"unused"}119 assert syn_splits["spatial_holdout_region"].dtype == pl.Boolean120 assert syn_splits.filter(pl.col("spatial_holdout_region")).height == syn_analytic.filter(pl.col("region_name").is_in(cfg["spatial_holdout"]["regions"])).height121122123def test_overlapping_windows_rejected(syn_analytic: pl.DataFrame, cfg: dict) -> None:124 bad = {**cfg, "splits": {**cfg["splits"], "validation": {"start": "2024-06-01", "end": "2025-03-31"}}}125 with pytest.raises(ValueError, match="overlaps"):126 assign_splits(syn_analytic, bad)127128129def test_proportional_allocation() -> None:130 alloc = proportional_allocation({"a": 100, "b": 50, "c": 2}, 30)131 assert sum(alloc.values()) == 30 and alloc["c"] <= 2 and alloc["a"] > alloc["b"]132 alloc = proportional_allocation({"a": 3, "b": 3}, 10)133 assert alloc == {"a": 3, "b": 3}134135136def test_subsample_counts_strata_and_determinism(syn_analytic: pl.DataFrame, syn_splits: pl.DataFrame, syn_subsamples: pl.DataFrame, cfg: dict) -> None:137 counts = {k: int(v) for k, v in syn_subsamples.group_by("split").len().iter_rows()}138 assert counts == SYN_N_BY_SPLIT139 assert syn_subsamples["tx_id"].n_unique() == syn_subsamples.height140 assert syn_subsamples.columns == ["tx_id", "split", "stratum", "draw_seed"]141 again = stratified_subsample(syn_analytic, syn_splits, cfg, n_by_split=SYN_N_BY_SPLIT)142 assert again.equals(syn_subsamples)143 other = stratified_subsample(syn_analytic, syn_splits, cfg, n_by_split=SYN_N_BY_SPLIT, seed=1)144 assert not other.equals(syn_subsamples)145 # each stratum label = region_group|property_class|price_tercile of the row146 chk = syn_subsamples.join(syn_analytic.select("tx_id", "region_group", "property_class", "price_tercile"), on="tx_id")147 lab = chk.select(pl.concat_str([pl.col("region_group"), pl.col("property_class"), pl.col("price_tercile").cast(pl.String)], separator="|"))148 assert lab.to_series().to_list() == chk["stratum"].to_list()149150151@pytest.mark.real152def test_real_analytic_contract(real_bundle: dict[str, pl.DataFrame]) -> None:153 validate_analytic(real_bundle["analytic"])154 assert real_bundle["analytic"].height > 600_000155