Repository Public Research preview
1# Author: Simon-Pierre Boucher — contact@spboucher.ai2#3"""The single design matrix used by every model family (November 2026 review,4WP03-EDIT-02 / ADVE-06 / DATA-07 / THEO-09).56Two codings of the same 62 regressors:7 * ``legacy`` (v1.1/v1.2): ln(lot) = 0 when the raw lot is missing or zero,8 parking_spaces and stories as imputed (corrupt values kept). Reproduces9 ``model_data.pkl::X_const`` once standardised (``tests/test_design.py``).10 * ``v2`` (corrected): ln(lot) = ln(raw lot) if raw lot > 1 sq. ft., else11 ln(state median of raw lots > 1) plus a ``lot_missing`` indicator;12 parking_spaces winsorised at 20, stories at 10; ``implausible`` flag13 (parking > 20, stories > 10, lot > 640 acres, age >= 200 in the legacy14 coding) so that every model uses the trimmed sample by default; six15 original-missingness indicators as an optional block.1617Standardisation (OLS/QR): the 25 continuous columns of the v1.1 OLS are18centred and scaled with the sample mean and SD (ddof = 1) of the sample on19which the model is estimated; binaries and dummies are left as is.20"""2122from __future__ import annotations2324import json2526import numpy as np27import pandas as pd2829from . import config3031DESIGN_V2 = config.DATA_PROCESSED / "design_v2.parquet"32DESIGN_META = config.DATA_PROCESSED / "design_v2_meta.json"3334# the 25 columns standardised in model_data.pkl (keys of md["means"])35CONTINUOUS_STD = ["ln_sqft", "bedrooms", "bathrooms", "age", "age_sq", "stories", "bath_per_bed",36 "sqft_per_bed", "ln_lot", "walk_score", "bike_score", "transit_score",37 "avg_school_rating", "school_count", "nearest_school_distance",38 "property_tax_rate", "ln_hoa", "parking_spaces", "luxury_score", "sqft_x_age",39 "pool_x_south", "waterfront_x_sqft", "basement_x_north", "condo_x_walkscore",40 "age_x_luxury"]41MISS_BLOCK = ["miss_year_built", "miss_lot", "miss_school", "miss_walk", "miss_garage", "miss_transit"]42ID_COLS = ["zpid", "ln_price", "address_state", "region", "zip3", "zip5", "latitude", "longitude",43 "implausible", "dup_group_id", "lot_missing", "ln_lot_v1", "parking_spaces_v1",44 "stories_v1", "lot_size_sqft"]45PARKING_CAP, STORIES_CAP = 20, 10464748def regressor_columns(sample: pd.DataFrame):49 """62 regressor names in the X_const order (base features + drop-first dummies)."""50 dummies = pd.get_dummies(sample[config.CATEGORICAL_COLS], drop_first=True).astype(np.int8)51 return config.ALL_BASE_FEATS + list(dummies.columns), dummies525354def build_design(sample: pd.DataFrame) -> tuple[pd.DataFrame, dict]:55 """From analytical_sample_v2 (with raw flags) build the v2 design frame."""56 cols, dummies = regressor_columns(sample)57 X = pd.concat([sample[config.ALL_BASE_FEATS].astype(float), dummies], axis=1)58 # --- v2 recodings59 lot_raw = sample["lot_raw"]60 lot_missing = (lot_raw.isna() | (lot_raw <= 1)).astype(np.int8)61 med = lot_raw.where(lot_raw > 1).groupby(sample["address_state"]).transform("median")62 ln_lot_v2 = np.log(lot_raw.where(lot_raw > 1, med))63 assert ln_lot_v2.notna().all()64 out = pd.DataFrame({65 "zpid": sample["zpid"].values, "ln_price": sample["ln_price"].values,66 "address_state": sample["address_state"].values, "region": sample["region"].values,67 "zip3": sample["zip3"].values,68 "zip5": sample["address_zipcode"].fillna("00000").astype(str).values,69 "latitude": sample["latitude"].values, "longitude": sample["longitude"].values,70 "implausible": sample["implausible"].values.astype(np.int8),71 "dup_group_id": sample["dup_group_id"].values,72 "lot_missing": lot_missing.values,73 "ln_lot_v1": X["ln_lot"].values, "parking_spaces_v1": X["parking_spaces"].values,74 "stories_v1": X["stories"].values, "lot_size_sqft": sample["lot_size_sqft"].values,75 })76 X["ln_lot"] = ln_lot_v2.values77 X["parking_spaces"] = X["parking_spaces"].clip(upper=PARKING_CAP)78 X["stories"] = X["stories"].clip(upper=STORIES_CAP)79 for c in cols:80 out[c] = X[c].values81 for c in MISS_BLOCK:82 out[c] = sample[c].values.astype(np.int8)83 trimmed = out["implausible"] == 084 meta = {85 "regressors": cols,86 "continuous_standardised": CONTINUOUS_STD,87 "missingness_block": MISS_BLOCK,88 "reference_categories": {"roof_cat": "Flat", "construction_cat": "Brick",89 "foundation_cat": "Basement", "region": "Midwest"},90 "lot_rule_v2": "ln(raw lot) if raw lot > 1 sq ft else ln(state median of raw lots > 1); "91 "lot_missing = 1 if raw lot null, 0 or <= 1",92 "parking_cap": PARKING_CAP, "stories_cap": STORIES_CAP,93 "n_rows": int(len(out)), "n_trimmed": int(trimmed.sum()),94 "n_implausible": int((~trimmed).sum()),95 "lot_missing_count": int(lot_missing.sum()),96 "lot_classes": sample["lot_class"].value_counts().to_dict(),97 "n_parking_winsorised": int((sample["parking_spaces"] > PARKING_CAP).sum()),98 "n_stories_winsorised": int((sample["stories"] > STORIES_CAP).sum()),99 "means_trimmed": out.loc[trimmed, CONTINUOUS_STD].mean().to_dict(),100 "stds_trimmed_ddof1": out.loc[trimmed, CONTINUOUS_STD].std(ddof=1).to_dict(),101 "means_full": out[CONTINUOUS_STD].mean().to_dict(),102 "stds_full_ddof1": out[CONTINUOUS_STD].std(ddof=1).to_dict(),103 }104 return out, meta105106107def load_design(coding: str = "v2", trimmed: bool = True, standardize: bool = False,108 missing_block: bool = False, lot_indicator: bool | None = None):109 """Return (X: DataFrame, y: ndarray, info: DataFrame of ids/groups, meta).110111 coding 'v2' (corrected) or 'legacy' (v1.1 columns ln_lot_v1, parking_v1, stories_v1)112 trimmed drop the 4,970 implausible rows (default True; use False for the v1.1 replication)113 standardize z-score the 25 continuous columns on the returned sample (ddof = 1)114 missing_block append the six original-missingness indicators115 lot_indicator append lot_missing (default: True for v2, False for legacy)116 """117 d = pd.read_parquet(DESIGN_V2)118 meta = json.loads(DESIGN_META.read_text())119 cols = list(meta["regressors"])120 if lot_indicator is None:121 lot_indicator = coding == "v2"122 if trimmed:123 d = d[d["implausible"] == 0].reset_index(drop=True)124 X = d[cols].copy()125 if coding == "legacy":126 X["ln_lot"] = d["ln_lot_v1"].values127 X["parking_spaces"] = d["parking_spaces_v1"].values128 X["stories"] = d["stories_v1"].values129 elif coding != "v2":130 raise ValueError(coding)131 if lot_indicator:132 X["lot_missing"] = d["lot_missing"].values.astype(float)133 if missing_block:134 for c in MISS_BLOCK:135 X[c] = d[c].values.astype(float)136 scale = None137 if standardize:138 mu = X[CONTINUOUS_STD].mean()139 sd = X[CONTINUOUS_STD].std(ddof=1)140 X[CONTINUOUS_STD] = (X[CONTINUOUS_STD] - mu) / sd141 scale = {"means": mu.to_dict(), "stds": sd.to_dict()}142 info = d[ID_COLS]143 return X, d["ln_price"].values, info, {"meta": meta, "scale": scale, "coding": coding,144 "trimmed": trimmed}145