UQO Working Paper No. 2 — Decoding Real Estate Descriptions: text-based hedonic analysis of housing listings.
1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — contact@spboucher.ai3#4"""Step 7 — Revision extensions (editorial-board review, 2026-10-17).56Addresses the identification and inference issues raised in the review:7 (a) language of the listing (French / English / mixed) and its effect on the8 similarity scores computed against French references;9 (b) extended specifications: language controls, FSA (forward sortation area)10 and municipality fixed effects, cluster-robust standard errors, unit-11 corrected lot size, floor-area subsample, French-only / English-only;12 (c) domain-level block tests of the semantic dimensions;13 (d) out-of-sample validation: random 5-fold and spatial (FSA-grouped) CV;14 (e) sensitivity of the estimates to the phrasing of the reference texts;15 (f) a multilingual embedding model (paraphrase-multilingual-MiniLM-L12-v2).1617Inputs : data/processed/houses.parquet, hedonic_maison_results.csv,18 embeddings_maisons.npy, data/raw/louka.db19Outputs: results/revision_language.csv20 results/revision_specifications.csv, revision_spec_summary.csv21 results/revision_block_tests.csv22 results/revision_oos.csv23 results/revision_reference_sensitivity.csv (+ _summary)24 results/revision_multilingual.csv25 data/processed/embeddings_multilingual.npy (cache)26"""2728import sys29import time30from pathlib import Path3132import numpy as np33import pandas as pd34import statsmodels.api as sm35from sklearn.decomposition import PCA36from sklearn.model_selection import GroupKFold, KFold3738sys.path.insert(0, str(Path(__file__).resolve().parents[1]))3940from src import config41from src.extensions import (block_f_test, detect_language, load_revision_fields,42 ols, oos_metrics, within, zscore)43from src.references import ENGLISH_LABELS, REFERENCES, SIM_COLS4445STRUCT = config.STRUCTURAL_VARS46LEN = ["remarks_length"]47LANG = ["lang_en", "lang_mixed"]48MULTI_MODEL = "paraphrase-multilingual-MiniLM-L12-v2"49EMB_MULTI = config.DATA_PROCESSED / "embeddings_multilingual.npy"5051DOMAINS = {52 "Quality & Standing": ["sim_luxe", "sim_entree_de_gamme"],53 "Condition": ["sim_renove", "sim_a_renover"],54 "Physical Attributes": ["sim_lumineux_spacieux", "sim_garage_stationnement",55 "sim_sous_sol_amenage", "sim_piscine_amenagement"],56 "Location & Setting": ["sim_terrain_nature", "sim_vue_panoramique",57 "sim_localisation_premium", "sim_tranquillite", "sim_bord_eau"],58 "Style & Character": ["sim_moderne_contemporain", "sim_cachet_patrimonial",59 "sim_neuf_construction", "sim_ecoefficace", "sim_familial"],60 "Market Signals": ["sim_revenu_investissement", "sim_urgence_motivation"],61}6263T0 = time.time()646566def log(msg):67 print(f"[{time.time() - T0:6.1f}s] {msg}", flush=True)686970# --------------------------------------------------------------------------71# Data assembly72# --------------------------------------------------------------------------73def build_dataset():74 df = pd.read_csv(config.ANALYSIS_CSV)75 houses = pd.read_parquet(config.HOUSES_PARQUET)76 df["id"] = df["id"].astype(str)77 assert (df["id"].values == houses["id"].astype(str).values).all()78 df["remarks"] = houses["remarks"].values79 df["latitude"] = houses["latitude"].values80 df["longitude"] = houses["longitude"].values81 extra = load_revision_fields(config.DB_PATH, df["id"].tolist())82 for c in ["fsa", "municipality", "lot_sqm", "floor_sqm", "scraped_at"]:83 df[c] = extra[c].values84 df["lang"] = [detect_language(t) for t in df["remarks"]]85 df["lang_en"] = (df["lang"] == "en").astype(float)86 df["lang_mixed"] = (df["lang"] == "mixed").astype(float)87 df["log_lot_sqm"] = np.where(df["lot_sqm"] > 0, np.log(df["lot_sqm"].fillna(1.0)), 0.0)88 df["lot_missing"] = (~(df["lot_sqm"] > 0)).astype(float)89 df["log_floor_sqm"] = np.where(df["floor_sqm"] > 0, np.log(df["floor_sqm"].fillna(1.0)), np.nan)90 df["fsa_id"] = pd.factorize(df["fsa"])[0]91 df["mun_id"] = pd.factorize(df["municipality"])[0]92 return df939495# --------------------------------------------------------------------------96# Estimation helpers97# --------------------------------------------------------------------------98def design(df, columns, fe=None):99 """Standardized design (z-scores on the estimation sample), optional FE demeaning."""100 X = zscore(df, columns)101 if fe is not None:102 X = X - X.groupby(df[fe].values).transform("mean")103 return X # no constant after within transformation104 return sm.add_constant(X)105106107def fit_spec(df, columns, sims, fe=None, cluster=None):108 """Fit y on columns (+sims); return model, model-without-sims and summary dict."""109 y = df["log_price"]110 if fe is not None:111 y = y - y.groupby(df[fe].values).transform("mean")112 groups = df[cluster].values if cluster else None113 cov = "cluster" if cluster else "HC3"114 m_full = ols(y, design(df, columns + sims, fe), cov, groups)115 m_base = ols(y, design(df, columns, fe), cov, groups)116 f, p, q = block_f_test(m_full, sims)117 # R² relative to total variance of log price (not within variance) when FE118 if fe is not None:119 yfull = df["log_price"]120 sst = float(((yfull - yfull.mean()) ** 2).sum())121 r2_full = 1 - float((m_full.resid ** 2).sum()) / sst122 r2_base = 1 - float((m_base.resid ** 2).sum()) / sst123 r2_within_full = m_full.rsquared124 else:125 r2_full, r2_base, r2_within_full = m_full.rsquared, m_base.rsquared, np.nan126 summ = {"n": int(m_full.nobs), "r2": r2_full, "r2_without_sims": r2_base,127 "delta_r2_sims": r2_full - r2_base, "r2_within": r2_within_full,128 "block_F": f, "block_p": p, "block_df": q,129 "n_sig_5pct": int(sum(m_full.pvalues[s] < 0.05 for s in sims))}130 return m_full, summ131132133def coef_rows(spec, model, sims, label_map):134 rows = []135 for v in sims + [c for c in model.params.index if c in STRUCT + LEN + LANG + ["log_lot_sqm", "lot_missing", "log_floor_sqm"]]:136 if v not in model.params.index:137 continue138 rows.append({"spec": spec, "variable": v, "label": label_map.get(v, v),139 "coefficient": model.params[v], "std_error": model.bse[v],140 "p_value": model.pvalues[v],141 "impact_pct": (np.exp(model.params[v]) - 1) * 100})142 return rows143144145LABELS = {**{f"sim_{k}": v for k, v in ENGLISH_LABELS.items()},146 "bedrooms": "Bedrooms", "bathrooms": "Bathrooms", "half_baths": "Half-bathrooms",147 "parking": "Parking", "stories": "Stories", "land_size": "Lot size (raw field)",148 "remarks_length": "Description length", "lang_en": "English listing",149 "lang_mixed": "Mixed / uninformative", "log_lot_sqm": "log lot area (m2)",150 "lot_missing": "Lot area missing", "log_floor_sqm": "log floor area (m2)"}151152153# --------------------------------------------------------------------------154# (a) Language155# --------------------------------------------------------------------------156def language_analysis(df, S, tag, out_rows):157 msim = S.mean(axis=1)158 for lang in ["fr", "en", "mixed"]:159 mask = (df["lang"] == lang).values160 out_rows.append({"model": tag, "language": lang, "n": int(mask.sum()),161 "share": mask.mean(), "mean_similarity": msim[mask].mean(),162 "mean_log_price": df.loc[mask, "log_price"].mean(),163 "median_price": df.loc[mask, "price"].median(),164 "mean_length": df.loc[mask, "remarks_length"].mean()})165 # eta² of language on mean similarity166 grand = msim.mean()167 ss_b = sum(((df["lang"] == l).sum()) * (msim[(df["lang"] == l).values].mean() - grand) ** 2168 for l in ["fr", "en", "mixed"])169 ss_t = ((msim - grand) ** 2).sum()170 out_rows.append({"model": tag, "language": "eta2_language_on_mean_similarity",171 "n": len(df), "share": np.nan, "mean_similarity": ss_b / ss_t,172 "mean_log_price": np.nan, "median_price": np.nan, "mean_length": np.nan})173 # correlation of mean similarity with length and with English dummy174 out_rows.append({"model": tag, "language": "corr_mean_similarity_length",175 "n": len(df), "share": np.nan,176 "mean_similarity": np.corrcoef(msim, df["remarks_length"])[0, 1],177 "mean_log_price": np.nan, "median_price": np.nan, "mean_length": np.nan})178 out_rows.append({"model": tag, "language": "corr_mean_similarity_english",179 "n": len(df), "share": np.nan,180 "mean_similarity": np.corrcoef(msim, df["lang_en"])[0, 1],181 "mean_log_price": np.nan, "median_price": np.nan, "mean_length": np.nan})182 # first PC share of the 20 similarities183 pc = PCA(1).fit(S)184 out_rows.append({"model": tag, "language": "pc1_share_of_similarity_variance",185 "n": len(df), "share": np.nan,186 "mean_similarity": pc.explained_variance_ratio_[0],187 "mean_log_price": np.nan, "median_price": np.nan, "mean_length": np.nan})188189190# --------------------------------------------------------------------------191# (d) Out-of-sample192# --------------------------------------------------------------------------193def oos_validation(df, E, sim_sets, out_rows):194 y = df["log_price"].values195 folds = {"random_5fold": list(KFold(5, shuffle=True, random_state=config.SEED).split(df)),196 "spatial_fsa_5fold": list(GroupKFold(5).split(df, groups=df["fsa_id"]))}197 # The raw lot-size field (mixed units, max 7.2e8) makes out-of-sample198 # predictions explode when an extreme value falls in the test fold; all199 # specifications below therefore use the unit-corrected lot area, except200 # the first row, kept to document the breakdown.201 STRUCT_OOS = STRUCT[:-1] + ["log_lot_sqm", "lot_missing"]202 specs = {"A (raw lot-size field)": (STRUCT, None, False),203 "A: structural": (STRUCT_OOS, None, False),204 "B: + length": (STRUCT_OOS + LEN, None, False),205 "B-L: + length + language": (STRUCT_OOS + LEN + LANG, None, False),206 "D: + length + 20 similarities": (STRUCT_OOS + LEN, "base", False),207 "D-L: D + language": (STRUCT_OOS + LEN + LANG, "base", False),208 "PCA-20 + length": (STRUCT_OOS + LEN, None, True),209 "PCA-20 + length + language": (STRUCT_OOS + LEN + LANG, None, True),210 "ML-D: multilingual similarities": (STRUCT_OOS + LEN, "multi", False),211 "ML-D-L: multilingual similarities + language": (STRUCT_OOS + LEN + LANG, "multi", False)}212 for scheme, splits in folds.items():213 for name, (cols, simkey, use_pca) in specs.items():214 if simkey == "multi" and "multi" not in sim_sets:215 continue216 preds = np.full(len(df), np.nan)217 for tr, te in splits:218 Xtr = df.iloc[tr][cols].values.astype(float)219 Xte = df.iloc[te][cols].values.astype(float)220 if simkey:221 Xtr = np.hstack([Xtr, sim_sets[simkey][tr]])222 Xte = np.hstack([Xte, sim_sets[simkey][te]])223 if use_pca:224 p = PCA(20, random_state=config.SEED).fit(E[tr])225 Xtr = np.hstack([Xtr, p.transform(E[tr])])226 Xte = np.hstack([Xte, p.transform(E[te])])227 Xtr = np.hstack([np.ones((len(tr), 1)), Xtr])228 Xte = np.hstack([np.ones((len(te), 1)), Xte])229 beta, *_ = np.linalg.lstsq(Xtr, y[tr], rcond=None)230 preds[te] = Xte @ beta231 met = oos_metrics(pd.Series(y), pd.Series(preds))232 out_rows.append({"scheme": scheme, "specification": name, **met})233 log(f" OOS {scheme:18s} {name:48s} R2={met['r2_oos']:.4f} MdAPE={met['mdape_pct']:.1f}")234235236# --------------------------------------------------------------------------237# (e) Reference phrasing sensitivity238# --------------------------------------------------------------------------239def reference_variants():240 """Three systematic perturbations of each reference text."""241 variants = {"first_half": {}, "second_half": {}, "reversed": {}}242 for key, text in REFERENCES.items():243 phrases = [p.strip() for p in text.replace(".", ",").split(",") if p.strip()]244 h = int(np.ceil(len(phrases) / 2))245 variants["first_half"][key] = ", ".join(phrases[:h]) + "."246 variants["second_half"][key] = ", ".join(phrases[h:]) + "."247 variants["reversed"][key] = ", ".join(phrases[::-1]) + "."248 return variants249250251def reference_sensitivity(df, E, base_model, out_rows, summ_rows):252 from sentence_transformers import SentenceTransformer253 model = SentenceTransformer(config.EMBEDDING_MODEL)254 base_coef = base_model.params[SIM_COLS]255 sig = [s for s in SIM_COLS if base_model.pvalues[s] < 0.05]256 for vname, refs in reference_variants().items():257 R = model.encode([refs[k] for k in REFERENCES], normalize_embeddings=True)258 S_v = E @ R.T259 dv = df.copy()260 for i, c in enumerate(SIM_COLS):261 dv[c] = S_v[:, i]262 m, s = fit_spec(dv, STRUCT + LEN, SIM_COLS)263 coef = m.params[SIM_COLS]264 for c in SIM_COLS:265 out_rows.append({"variant": vname, "variable": c, "label": LABELS[c],266 "coefficient": coef[c], "std_error": m.bse[c],267 "p_value": m.pvalues[c],268 "corr_similarity_with_baseline": np.corrcoef(S_v[:, SIM_COLS.index(c)], df[c])[0, 1]})269 summ_rows.append({"variant": vname, "r2": s["r2"], "delta_r2_sims": s["delta_r2_sims"],270 "block_F": s["block_F"],271 "corr_coef_with_baseline": np.corrcoef(coef, base_coef)[0, 1],272 "sign_agreement_sig16": np.mean([np.sign(coef[c]) == np.sign(base_coef[c]) for c in sig]),273 "n_sig_5pct": s["n_sig_5pct"],274 "mean_abs_coef_change": float(np.mean(np.abs(coef - base_coef))),275 "mean_corr_similarity": float(np.mean([np.corrcoef(S_v[:, i], df[c])[0, 1] for i, c in enumerate(SIM_COLS)]))})276 log(f" variant {vname}: R2={s['r2']:.4f} corr(coef)={summ_rows[-1]['corr_coef_with_baseline']:.3f}")277278279# --------------------------------------------------------------------------280# (f) Multilingual embeddings281# --------------------------------------------------------------------------282def multilingual_similarities(df):283 from sentence_transformers import SentenceTransformer284 model = SentenceTransformer(MULTI_MODEL)285 R = model.encode(list(REFERENCES.values()), normalize_embeddings=True)286 if EMB_MULTI.exists():287 E = np.load(EMB_MULTI)288 if len(E) != len(df):289 E = None290 else:291 E = None292 if E is None:293 log(f" encoding {len(df):,} descriptions with {MULTI_MODEL} ...")294 E = model.encode(df["remarks"].tolist(), batch_size=128,295 show_progress_bar=False, normalize_embeddings=True)296 np.save(EMB_MULTI, E)297 return E @ R.T, E298299300def main():301 log("assembling dataset ...")302 df = build_dataset()303 E = np.load(config.EMBEDDINGS_NPY)304 S = df[SIM_COLS].values305 log(f"n={len(df):,}; FSA={df['fsa'].nunique()}, municipalities={df['municipality'].nunique()}; "306 f"lot area parsed for {(df['lot_sqm'] > 0).mean():.1%}; floor area for {(df['floor_sqm'] > 0).mean():.1%}")307 log(f"language shares: {df['lang'].value_counts(normalize=True).round(3).to_dict()}")308309 config.RESULTS_DIR.mkdir(parents=True, exist_ok=True)310311 # (a) language ----------------------------------------------------------312 lang_rows = []313 language_analysis(df, S, "all-MiniLM-L6-v2", lang_rows)314 per_dim = []315 for c in SIM_COLS:316 per_dim.append({"variable": c, "label": LABELS[c],317 "mean_fr": df.loc[df.lang == "fr", c].mean(),318 "mean_en": df.loc[df.lang == "en", c].mean(),319 "sd_all": df[c].std(ddof=0)})320 pd.DataFrame(per_dim).to_csv(config.RESULTS_DIR / "revision_language_by_dimension.csv", index=False)321322 # (b) specifications ----------------------------------------------------323 spec_rows, summ_rows = [], []324325 def run(spec, data, cols, sims=SIM_COLS, fe=None, cluster=None, note=""):326 m, s = fit_spec(data, cols, sims, fe, cluster)327 s.update({"spec": spec, "fe": fe or "none", "cov": cluster or "HC3", "note": note})328 summ_rows.append(s)329 spec_rows.extend(coef_rows(spec, m, sims, LABELS))330 log(f" {spec:34s} n={s['n']:6d} R2={s['r2']:.4f} dR2(sims)={s['delta_r2_sims']:.4f} "331 f"F={s['block_F']:6.2f} sig={s['n_sig_5pct']:2d}/20")332 return m333334 log("extended specifications ...")335 mD = run("D (baseline)", df, STRUCT + LEN)336 run("D-L: + language", df, STRUCT + LEN + LANG)337 run("D-L, FSA-clustered SE", df, STRUCT + LEN + LANG, cluster="fsa_id")338 run("D-L + FSA FE", df, STRUCT + LEN + LANG, fe="fsa_id", cluster="fsa_id")339 run("D-L + municipality FE", df, STRUCT + LEN + LANG, fe="mun_id", cluster="mun_id")340 run("D-L + FSA FE, lot area (m2)", df, STRUCT[:-1] + ["log_lot_sqm", "lot_missing"] + LEN + LANG,341 fe="fsa_id", cluster="fsa_id", note="land_size replaced by unit-corrected lot area")342 dff = df[df["floor_sqm"] > 0].copy()343 run("D-L + FSA FE, floor-area subsample", dff,344 STRUCT[:-1] + ["log_lot_sqm", "lot_missing", "log_floor_sqm"] + LEN + LANG,345 fe="fsa_id", cluster="fsa_id", note="listings reporting interior floor area")346 dfr = df[df["lang"] == "fr"].copy()347 run("French-only, D + FSA FE", dfr, STRUCT + LEN, fe="fsa_id", cluster="fsa_id")348 dfe = df[df["lang"] == "en"].copy()349 run("English-only, D + FSA FE", dfe, STRUCT + LEN, fe="fsa_id", cluster="fsa_id")350 run("D (baseline), FSA-clustered SE", df, STRUCT + LEN, cluster="fsa_id")351352 # (c) domain block tests under the preferred specification ---------------353 log("domain block tests ...")354 y_w = df["log_price"] - df["log_price"].groupby(df["fsa_id"].values).transform("mean")355 X_w = design(df, STRUCT + LEN + LANG + SIM_COLS, fe="fsa_id")356 m_fe = ols(y_w, X_w, "cluster", df["fsa_id"].values)357 m_hc = ols(df["log_price"], design(df, STRUCT + LEN + LANG + SIM_COLS), "HC3")358 block_rows = []359 for dom, vars_ in DOMAINS.items():360 for spec, m in [("D-L (HC3)", m_hc), ("D-L + FSA FE (clustered)", m_fe)]:361 f, p, q = block_f_test(m, vars_)362 block_rows.append({"spec": spec, "domain": dom, "n_dims": q, "F": f, "p_value": p})363 f, p, q = block_f_test(m_fe, SIM_COLS)364 block_rows.append({"spec": "D-L + FSA FE (clustered)", "domain": "All 20 dimensions", "n_dims": q, "F": f, "p_value": p})365 f, p, q = block_f_test(m_hc, SIM_COLS)366 block_rows.append({"spec": "D-L (HC3)", "domain": "All 20 dimensions", "n_dims": q, "F": f, "p_value": p})367 pd.DataFrame(block_rows).to_csv(config.RESULTS_DIR / "revision_block_tests.csv", index=False)368369 # (e) reference phrasing sensitivity -------------------------------------370 log("reference phrasing sensitivity ...")371 sens_rows, sens_summ = [], []372 reference_sensitivity(df, E, mD, sens_rows, sens_summ)373 pd.DataFrame(sens_rows).to_csv(config.RESULTS_DIR / "revision_reference_sensitivity.csv", index=False)374 pd.DataFrame(sens_summ).to_csv(config.RESULTS_DIR / "revision_reference_sensitivity_summary.csv", index=False)375376 # (f) multilingual model -------------------------------------------------377 log("multilingual embeddings ...")378 S_multi, E_multi = multilingual_similarities(df)379 language_analysis(df, S_multi, MULTI_MODEL, lang_rows)380 pd.DataFrame(lang_rows).to_csv(config.RESULTS_DIR / "revision_language.csv", index=False)381 ML_COLS = [c.replace("sim_", "ml_") for c in SIM_COLS]382 for i, c in enumerate(ML_COLS):383 df[c] = S_multi[:, i]384 ml_labels = {c.replace("sim_", "ml_"): v for c, v in LABELS.items() if c.startswith("sim_")}385 LABELS.update(ml_labels)386 corr_models = pd.DataFrame({"variable": SIM_COLS, "label": [LABELS[c] for c in SIM_COLS],387 "corr_minilm_multilingual": [np.corrcoef(df[c], df[m])[0, 1] for c, m in zip(SIM_COLS, ML_COLS)],388 "mean_fr_multi": [df.loc[df.lang == "fr", m].mean() for m in ML_COLS],389 "mean_en_multi": [df.loc[df.lang == "en", m].mean() for m in ML_COLS]})390 corr_models.to_csv(config.RESULTS_DIR / "revision_multilingual.csv", index=False)391 run("ML-D: multilingual similarities", df, STRUCT + LEN, sims=ML_COLS)392 run("ML-D-L: + language", df, STRUCT + LEN + LANG, sims=ML_COLS)393 run("ML-D-L + FSA FE", df, STRUCT + LEN + LANG, sims=ML_COLS, fe="fsa_id", cluster="fsa_id")394 run("ML-D-L + FSA FE, lot area (m2)", df, STRUCT[:-1] + ["log_lot_sqm", "lot_missing"] + LEN + LANG,395 sims=ML_COLS, fe="fsa_id", cluster="fsa_id")396397 pd.DataFrame(spec_rows).to_csv(config.RESULTS_DIR / "revision_specifications.csv", index=False)398 pd.DataFrame(summ_rows).to_csv(config.RESULTS_DIR / "revision_spec_summary.csv", index=False)399400 # (d) out-of-sample -------------------------------------------------------401 log("out-of-sample validation ...")402 oos_rows = []403 oos_validation(df, E, {"base": S, "multi": S_multi}, oos_rows)404 pd.DataFrame(oos_rows).to_csv(config.RESULTS_DIR / "revision_oos.csv", index=False)405406 df[["id", "lang", "fsa", "municipality", "lot_sqm", "floor_sqm"]].to_csv(407 config.DATA_PROCESSED / "revision_fields.csv", index=False)408 log("done.")409410411if __name__ == "__main__":412 main()413