spb/wp22_uqo

1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca3"""Phase 6 — real-time vintages. For each monthly publication date v (10 days after month end, the measured publication4delay), re-estimate SHARP-HPI using only transactions dated ≤ v − 10 days (filtered mode: one-sided state-space deviations,5stage 1 re-estimated on the available sample). Records every vintage of every cell path in two modes — `filtered` (one-sided: the value of month m never6changes after its first release except through stage-1 re-estimation; the no-revision policy) and `smoothed` (two-sided given the data7available at the vintage: full re-estimation policy) — then the revision statistics8(1st/2nd/3rd release vs final), timeliness and three publication policies: (a) full re-estimation, (b) movement splice of9the newest month onto the frozen history, (c) revision-limited (history frozen after 3 releases).10Writes data/processed/index_vintages.parquet and results/vintages/*.csv."""11from __future__ import annotations1213import sys14import time15from datetime import date, timedelta16from pathlib import Path1718sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))19import numpy as np20import polars as pl2122from sharp_hpi import config as C23from sharp_hpi.estimators import month_grid24from sharp_hpi.hedonic import build_features25from sharp_hpi.sharp import cell_log_index, run_sharp2627OUT = C.RESULTS / "vintages"28OUT.mkdir(parents=True, exist_ok=True)29PUB_LAG_DAYS = 1030FIRST_VINTAGE = "2022-07"   # 18 months of history before the first simulated release31LEVELS_KEEP = {"province", "region", "municipality"}32MIN_MUN_N = 1500            # municipalities tracked in the vintage database (cost)333435def month_end(m: str) -> date:36    y, mm = int(m[:4]), int(m[5:7])37    return (date(y + (mm == 12), (mm % 12) + 1, 1) - timedelta(days=1))383940def main() -> None:41    a = pl.read_parquet(C.ANALYTIC_TX)42    f = build_features(a)43    big_mun = set(f.group_by("mun_code").agg(pl.len().alias("n")).filter(pl.col("n") >= MIN_MUN_N)["mun_code"].to_list())44    rows = []45    months_all = month_grid("2021-01", "2026-10")46    vintages = [m for m in months_all if m >= FIRST_VINTAGE]47    for cls in C.PROPERTY_TYPES:48        d = f.filter(pl.col("property_class") == cls)49        for vm in vintages:50            pub = month_end(vm) + timedelta(days=PUB_LAG_DAYS)51            cutoff = pub - timedelta(days=PUB_LAG_DAYS)  # transactions visible at publication52            if cutoff > date.fromisoformat(C.SAMPLE_END):53                cutoff = date.fromisoformat(C.SAMPLE_END)54            avail = d.filter(pl.col("date") <= cutoff)55            grid = month_grid("2021-01", vm)56            t0 = time.time()57            r = run_sharp(avail, grid, realtime_parent=True)58            n_prov = np.array([avail.filter(pl.col("month") == m).height for m in grid], float)59            for mode in ("filtered", "smoothed"):60                rows.append(pl.DataFrame({"class": cls, "vintage": vm, "mode": mode, "level": "province", "geo_id": "QC", "month": grid, "log_index": r.stage1.mu, "se": r.stage1.mu_se, "n": n_prov}))61                for (lvl, gid), c in r.cells.items():62                    if lvl not in LEVELS_KEEP or (lvl == "municipality" and gid not in big_mun):63                        continue64                    p, v = cell_log_index(r, lvl, gid, mode)65                    rows.append(pl.DataFrame({"class": cls, "vintage": vm, "mode": mode, "level": lvl, "geo_id": gid, "month": grid, "log_index": p, "se": np.sqrt(v), "n": c.n}))66            print(cls, vm, f"{avail.height} tx, {time.time() - t0:.1f}s", flush=True)67        pl.concat(rows).write_parquet(C.PROCESSED / "index_vintages.parquet")68    print("done")697071if __name__ == "__main__":72    main()73