TeX 62.3%
Python 37.6%
1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""Task B dataset: cell × origin panel of real-time predictors and final targets.34For origin o (a month), the predictor history of cell g is the SHARP-HPI vintage published for o (WP22 `index_vintages`,5smoothed mode, estimated on sales ≤ o − 10 days): log index levels for months ≤ o. Volumes and dispersion come from sales ≤ o.6Target over horizon h: Δ_h y = y_{o+h} − y_o where y is the FINAL retrospective smoothed log index (WP22 index_panel),7i.e. the best available estimate of the true quality-adjusted movement; the real-time level at o is also recorded so that8the "revision" component of the target can be separated. The last complete month is LAST_MONTH; origins run to LAST_MONTH − h.9"""10from __future__ import annotations1112import numpy as np13import polars as pl1415from wp23 import config as C16from wp23.data import month_add, months_between1718LAGS = 12192021def load_vintages() -> pl.DataFrame:22 v = pl.read_parquet(C.WP22_VINTAGES)23 if "mode" in v.columns:24 v = v.filter(pl.col("mode") == "smoothed")25 return v.select(["class", "vintage", "level", "geo_id", "month", "log_index", "se", "n"])262728def load_final() -> pl.DataFrame:29 p = pl.read_parquet(C.WP22_PANEL)30 return p.select(pl.col("property_type").alias("class"), pl.col("geography_level").alias("level"), pl.col("geography_id").alias("geo_id"), "month",31 (pl.col("index_value") / 100).log().alias("y_final"), pl.col("actual_transaction_count").alias("n_final"), "reliability_grade", "partial_month_flag")323334def cell_history_at(v: pl.DataFrame, origin: str) -> pl.DataFrame:35 """Rows: class, level, geo_id with arrays of the last LAGS+1 real-time log-index levels (ending at the origin), counts and s.e."""36 h = v.filter((pl.col("vintage") == origin) & (pl.col("month") <= origin) & (pl.col("month") > month_add(origin, -(LAGS + 1)))).sort(["class", "level", "geo_id", "month"])37 g = h.group_by(["class", "level", "geo_id"], maintain_order=True).agg(pl.col("log_index").alias("hist"), pl.col("n").alias("n_hist"), pl.col("se").alias("se_hist"), pl.len().alias("len"))38 return g.filter(pl.col("len") == LAGS + 1).drop("len")394041def build_panel(horizons=C.HORIZONS, origins: list[str] | None = None) -> pl.DataFrame:42 v = load_vintages(); fin = load_final()43 all_origins = sorted(v["vintage"].unique().to_list())44 origins = origins or [o for o in all_origins if C.FIRST_ORIGIN <= o <= month_add(C.LAST_MONTH, -1)]45 rows = []46 for o in origins:47 hist = cell_history_at(v, o)48 if hist.height == 0:49 continue50 H = np.array(hist["hist"].to_list(), float); N = np.array(hist["n_hist"].to_list(), float); SE = np.array(hist["se_hist"].to_list(), float)51 lv = H[:, -1]52 feats = {53 "rt_level": lv, "d1": H[:, -1] - H[:, -2], "d3": H[:, -1] - H[:, -4], "d6": H[:, -1] - H[:, -7], "d12": H[:, -1] - H[:, 0],54 "vol6": np.std(np.diff(H[:, -7:], axis=1), axis=1), "vol12": np.std(np.diff(H, axis=1), axis=1),55 "n_last": N[:, -1], "n_mean12": N.mean(axis=1), "n_ratio": (N[:, -3:].mean(axis=1) + 1) / (N.mean(axis=1) + 1), "se_last": SE[:, -1],56 "accel": (H[:, -1] - H[:, -4]) - (H[:, -4] - H[:, -7]),57 }58 base = hist.select(["class", "level", "geo_id"]).with_columns([pl.Series(k, val) for k, val in feats.items()]).with_columns(59 pl.lit(o).alias("origin"), pl.lit(int(o[5:7])).alias("month_of_year"), pl.lit(months_between("2021-01", o)).alias("origin_t"),60 pl.Series("hist", [list(r) for r in H]))61 fo = fin.filter(pl.col("month") == o).select(["class", "level", "geo_id", pl.col("y_final").alias("y_final_o")])62 base = base.join(fo, on=["class", "level", "geo_id"], how="left")63 for h in horizons:64 tm = month_add(o, h)65 if tm > C.LAST_MONTH:66 continue67 ft = fin.filter(pl.col("month") == tm).select(["class", "level", "geo_id", pl.col("y_final").alias("y_final_h"), pl.col("n_final").alias("n_target"), pl.col("reliability_grade").alias("grade_target")])68 d = base.join(ft, on=["class", "level", "geo_id"], how="inner").with_columns(69 pl.lit(h).alias("h"), (pl.col("y_final_h") - pl.col("y_final_o")).alias("target"), (pl.col("y_final_h") - pl.col("rt_level")).alias("target_vs_rt"))70 rows.append(d)71 out = pl.concat(rows)72 # province- and region-level context at the origin for every cell (parent momentum): join the region row of the same origin/class73 reg = out.filter(pl.col("level") == "region").select(["class", "origin", "h", pl.col("geo_id").alias("region_code"), pl.col("d3").alias("reg_d3"), pl.col("d12").alias("reg_d12"), pl.col("vol12").alias("reg_vol12")]).unique()74 prov = out.filter(pl.col("level") == "province").select(["class", "origin", "h", pl.col("d3").alias("prov_d3"), pl.col("d12").alias("prov_d12"), pl.col("vol12").alias("prov_vol12")]).unique()75 geo = pl.read_parquet(C.WP22_GEO, columns=["mun_code", "region_code", "mrc_code", "x_m", "y_m"]).group_by("mun_code").agg(pl.col("region_code").first(), pl.col("mrc_code").first(), pl.col("x_m").mean().alias("cx"), pl.col("y_m").mean().alias("cy"))76 out = out.join(geo, left_on="geo_id", right_on="mun_code", how="left").with_columns(77 pl.when(pl.col("level") == "region").then(pl.col("geo_id")).otherwise(pl.col("region_code")).alias("region_code"))78 out = out.join(reg, on=["class", "origin", "h", "region_code"], how="left").join(prov, on=["class", "origin", "h"], how="left")79 return out808182FEATURES_B = ["d1", "d3", "d6", "d12", "vol6", "vol12", "n_last", "n_mean12", "n_ratio", "se_last", "accel", "month_of_year", "origin_t", "reg_d3", "reg_d12", "reg_vol12", "prov_d3", "prov_d12", "prov_vol12"]83