spb/wp9_uqo Public

1# Author: Simon-Pierre Boucher — contact@spboucher.ai2"""Design matrices and the M1–M5 hedonic specification ladder.34The semi-logarithmic hedonic equation regresses ln(price) on structural5attributes, dwelling-type/ownership dummies and location fixed effects.6FSA fixed effects are absorbed with ``linearmodels.AbsorbingLS`` (numerically7identical to full-dummy OLS); standard errors are clustered by FSA.8"""9import numpy as np10import pandas as pd11import statsmodels.api as sm12from linearmodels.iv.absorbing import AbsorbingLS1314from .config import STRUCT151617def design(data: pd.DataFrame, extra: pd.DataFrame | None = None) -> pd.DataFrame:18    """Structural regressors + dwelling-type, ownership and category dummies."""19    blocks = [data[STRUCT].astype(float)]20    if extra is not None:21        blocks.append(extra)22    blocks += [pd.get_dummies(data[col], prefix=pfx, drop_first=True).astype(float)23               for col, pfx in (("btype_c", "bt"), ("own_c", "ow"), ("cat", "cat"))]24    return sm.add_constant(pd.concat(blocks, axis=1))252627def fit_absorbing(data: pd.DataFrame, extra: pd.DataFrame | None = None):28    """FSA-fixed-effects hedonic model with FSA-clustered standard errors.2930    Returns the fitted AbsorbingLS results and the design column order.31    """32    X = design(data, extra=extra)33    res = AbsorbingLS(data["ln_price"], X,34                      absorb=data[["fsa_c"]].astype("category"),35                      drop_absorbed=True).fit(cov_type="clustered",36                                              clusters=data[["fsa_c"]])37    return res, X.columns.tolist()383940def coef_table(res, columns=None) -> pd.DataFrame:41    """Coefficient / SE / p-value table indexed by regressor name.4243    Works for both statsmodels (``bse``) and linearmodels (``std_errors``);44    uses the estimator's own parameter index, which may exclude regressors45    dropped as collinear with the absorbed effects.46    """47    se = res.std_errors if hasattr(res, "std_errors") else res.bse48    index = res.params.index if hasattr(res.params, "index") else columns49    return pd.DataFrame({50        "coef": pd.Series(np.asarray(res.params).ravel(), index=index),51        "se": pd.Series(np.asarray(se).ravel(), index=index),52        "p": pd.Series(np.asarray(res.pvalues).ravel(), index=index),53    })545556def fsa_fixed_effects(data: pd.DataFrame, res, columns=None) -> pd.Series:57    """Recover the absorbed FSA effects as within-FSA mean residuals of X'b."""58    beta = pd.Series(np.asarray(res.params).ravel(), index=res.params.index)59    xb = design(data).reindex(columns=beta.index, fill_value=0.0).values @ beta.values60    return (data["ln_price"] - xb).groupby(data["fsa_c"]).mean()616263def predict_with_fe(data: pd.DataFrame, res, columns, fe: pd.Series) -> np.ndarray:64    """Linear prediction X'b + absorbed FSA effect."""65    beta = pd.Series(np.asarray(res.params).ravel(), index=res.params.index)66    xb = design(data).reindex(columns=beta.index, fill_value=0.0).values @ beta.values67    return xb + data["fsa_c"].map(fe).values686970def specification_ladder(sample: pd.DataFrame) -> dict:71    """Estimate the M1–M5 ladder; returns {name: (results, columns, data)}.7273    M1–M3 use the house subsample (structural; +type/ownership; +province);74    M4 is houses with FSA fixed effects; M5 is the grand model over all75    residential dwellings with FSA fixed effects.76    """77    houses = sample[sample["cat"] == "house"]78    out = {}7980    X1 = sm.add_constant(houses[STRUCT].astype(float))81    out["M1"] = (sm.OLS(houses["ln_price"], X1)82                 .fit(cov_type="cluster", cov_kwds={"groups": houses["fsa_c"]}),83                 X1.columns.tolist(), houses)8485    X2 = design(houses)86    out["M2"] = (sm.OLS(houses["ln_price"], X2)87                 .fit(cov_type="cluster", cov_kwds={"groups": houses["fsa_c"]}),88                 X2.columns.tolist(), houses)8990    X3 = X2.join(pd.get_dummies(houses["prov"], prefix="pv", drop_first=True).astype(float))91    out["M3"] = (sm.OLS(houses["ln_price"], X3)92                 .fit(cov_type="cluster", cov_kwds={"groups": houses["fsa_c"]}),93                 X3.columns.tolist(), houses)9495    res4, cols4 = fit_absorbing(houses)96    out["M4"] = (res4, cols4, houses)9798    res5, cols5 = fit_absorbing(sample)99    out["M5"] = (res5, cols5, sample)100    return out101102103def duan_smearing(residuals: np.ndarray) -> float:104    """Duan (1983) smearing factor for retransformation from logs."""105    return float(np.mean(np.exp(residuals)))106