spb/qwhpi Public

1# =============================================================================2# QWHPI — Quebec Weekly Housing Price Index3# Author  : Simon-Pierre Boucher4# Contact : contact@spboucher.ai5# File    : engine/tests/test_features_clean.py6# Purpose : Unit tests for cleaning flags and feature engineering.7# =============================================================================8"""Cleaning + feature unit tests on synthetic rows."""910from __future__ import annotations1112import datetime as dt13import sys14from pathlib import Path1516sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))1718import polars as pl1920from qwhpi.clean import add_flags21from qwhpi.features import build_features222324def synthetic(n: int = 8) -> tuple[pl.DataFrame, pl.DataFrame]:25    tx = pl.DataFrame({26        "id": [f"t{i}" for i in range(n)],27        "date": [dt.date(2023, 1, 9)] * n,28        "amount": [300000.0, 300000.0, 55000.0, 20_000_000.0,29                   400000.0, 500000.0, 350000.0, 360000.0],30        "street": ["1 rue A", "1 rue A", "2 rue B", "3 rue C",31                   "4 rue D", "5 rue E", "6 rue F", "7 rue G"],32        "zipCode": ["G1A 1A1"] * n,33        "city": ["X"] * n,34        "lat": [46.8] * n,35        "lng": [-71.2] * n,36        "propertyType": ["unifamilial"] * 6 + ["condo", "indéterminé"],37        "yearBuilt": [1990, 1990, 1990, 1990, 2030, None, 2000, 2000],38        "floorArea": [120.0, 120.0, 120.0, 120.0, 5000.0, None, 80.0, 80.0],39        "buildingType": ["single-story"] * 6 + [None, None],40        "previousValue": [250000.0] * n,41        "totalArValue": [280000.0, 280000.0, 280000.0, 280000.0,42                         380000.0, 100.0, 330000.0, 330000.0],43        "ownerType": ["physical_person"] * n,44        "week": [dt.date(2023, 1, 9)] * n,45        "log_amount": [12.6] * n,46    })47    geo = pl.DataFrame({48        "id": [f"t{i}" for i in range(n)],49        "geo_code": ["23027"] * n,50        "municipality": ["Québec"] * n,51        "munic_type": ["V"] * n,52        "mrc_code": ["230"] * n,53        "mrc": ["Québec"] * n,54        "region_code": ["03"] * n,55        "region": ["Capitale-Nationale"] * n,56        "join_method": ["within"] * n,57    })58    return tx, geo596061def test_flags():62    tx, geo = synthetic()63    df = add_flags(tx, geo)64    by_id = {r["id"]: r for r in df.iter_rows(named=True)}65    assert by_id["t1"]["duplicate_flag"] is True          # exact dup of t066    assert by_id["t0"]["duplicate_flag"] is False          # first kept67    assert by_id["t3"]["exclude_reason"] == "price_above_10m"68    assert by_id["t5"]["nonmarket_ratio"] is True          # ratio 5000x69    assert by_id["t4"]["yb_suspect"] is True               # built 203070    assert by_id["t4"]["fa_suspect"] is True               # 5000 m271    assert by_id["t4"]["yearBuilt"] is None                # nulled, kept72    assert by_id["t7"]["is_undetermined_type"] is True73    # nothing is deleted74    assert df.height == tx.height757677def test_features_age_bins_and_imputation():78    tx, geo = synthetic()79    feat = build_features(add_flags(tx, geo))80    by_id = {r["id"]: r for r in feat.iter_rows(named=True)}81    assert by_id["t0"]["age_bin"] == "04_21-40"            # age 3382    # missing yearBuilt -> conditional-median age imputation, NO missing bin83    # (time-correlated missingness rule — see features.py docstring)84    assert by_id["t5"]["yb_missing"] is True85    assert by_id["t5"]["age_filled"] is not None86    assert by_id["t5"]["age_bin"] != "99_missing"87    assert by_id["t5"]["fa_missing"] is True88    assert by_id["t5"]["floor_area_filled"] is not None    # imputed89    assert by_id["t0"]["loc_fine"] == "G1A"90