Python 57%
TeX 42.9%
1#!/usr/bin/env python32# -*- coding: utf-8 -*-3"""4Nettoyage des aberrations + modèle hédonique de prix des maisons5(Realtor + DuProprio combinés). Log-linéaire, effets fixes ville, SE robustes.6"""7import json, re, warnings8import numpy as np9import pandas as pd10import statsmodels.formula.api as smf1112warnings.filterwarnings("ignore")13SRC = "data/processed/maisons_avec_prix.json"14CUR_YEAR = 20261516# ---------- Chargement ----------17with open(SRC, encoding="utf-8") as f:18 data = json.load(f)19print(f"Chargé : {len(data)} enregistrements")2021# ---------- Parsers ----------22def parse_area(s):23 """Retourne une superficie en pieds carrés (ft²)."""24 if not s or not isinstance(s, str):25 return np.nan26 txt = s.lower()27 is_m2 = ("m2" in txt) or ("m²" in txt)28 # garder chiffres, points, virgules, espaces ; retirer unités29 num = re.sub(r"[^0-9.,\s]", "", txt).strip()30 num = num.replace(" ", "").replace(",", "") # séparateurs de milliers31 try:32 v = float(num)33 except ValueError:34 return np.nan35 if v <= 0:36 return np.nan37 return v * 10.7639 if is_m2 else v3839def clean_city(city, source):40 if not city or not isinstance(city, str):41 return None42 c = city.split("(")[0].strip() # retire quartiers realtor43 c = re.sub(r"\s+", " ", c).lower().strip()44 return c or None4546def to_int(x):47 try:48 return int(x)49 except (TypeError, ValueError):50 return np.nan5152# amenity keywords -> recherche dans features + description + title (bilingue)53AMENITIES = {54 "pool": ["swimming pool", "piscine", "inground pool", "pool"],55 "waterfront": ["waterfront", "bord de l'eau", "navigable", "water access",56 "lakefront", "riverfront", "au bord de", "waterfront"],57 "garage": ["garage"],58 "fireplace": ["fireplace", "foyer", "wood stove", "poêle"],59 "central_air": ["central air", "air conditionné", "climatisation", "a/c",60 "heat-pump", "thermopompe", "air exchanger"],61 "finished_bsmt":["totally finished", "finished basement", "sous-sol fini",62 "basement"],63 "near_water": ["near water", "waterfront", "navigable"],64 "cul_de_sac": ["cul-de-sac", "cul de sac"],65 "golf": ["golf"],66 "ski": ["ski"],67 "near_park": ["park", "parc"],68 "near_school": ["school", "école", "cegep", "university", "université",69 "elementary", "high school"],70 "near_highway":["highway", "autoroute"],71 "transit": ["public transit", "public transportation", "transport"],72 "no_rear_neighbor":["no backyard neighbors", "no rear neighbor",73 "sans voisin arrière"],74}7576# ---------- Parsing des pièces (Realtor) ----------77def _side_ft(side):78 s = side.lower()79 if "ft" in s:80 ft = re.search(r"([\d.]+)\s*ft", s)81 inch = re.search(r"([\d.]+)\s*in", s)82 return (float(ft.group(1)) if ft else 0) + \83 (float(inch.group(1)) / 12 if inch else 0)84 m = re.search(r"([\d.,]+)\s*m", s)85 if m:86 try:87 return float(m.group(1).replace(",", ".")) * 3.2808488 except ValueError:89 return None90 return None9192def _room_area(dim):93 if not dim or "x" not in dim:94 return None95 a, _, b = dim.partition("x")96 la, lb = _side_ft(a), _side_ft(b)97 if la and lb and 1 < la < 200 and 1 < lb < 200:98 return la * lb99 return None100101ROOM_FLAGS = { # pièces à valeur potentielle102 "r_office": ["office"],103 "r_family": ["family room"],104 "r_workshop": ["workshop"],105 "r_winecellar": ["wine cellar"],106 "r_sunroom": ["sunroom", "solarium"],107 "r_mezzanine": ["mezzanine"],108 "r_library": ["library"],109 "r_laundry": ["laundry"],110}111112def parse_rooms(rooms):113 out = {"room_area": np.nan, "n_rooms": 0, "n_bed_r": 0, "n_bath_r": 0,114 "has_primary_bed": 0, "has_basement": 0, "n_levels_r": 0}115 for k in ROOM_FLAGS:116 out[k] = 0117 if not rooms:118 return out119 area, levels = 0.0, set()120 for r in rooms:121 nm = (r.get("name") or "").lower()122 lv = (r.get("level") or "").lower()123 out["n_rooms"] += 1124 if lv:125 levels.add(lv)126 if "basement" in lv or "sous-sol" in lv:127 out["has_basement"] = 1128 if "primary bedroom" in nm:129 out["has_primary_bed"] = 1130 out["n_bed_r"] += 1131 elif "bedroom" in nm:132 out["n_bed_r"] += 1133 if "bathroom" in nm:134 out["n_bath_r"] += 1135 for k, kws in ROOM_FLAGS.items():136 if any(w in nm for w in kws):137 out[k] = 1138 a = _room_area(r.get("dimensions"))139 if a:140 area += a141 out["room_area"] = area if area > 0 else np.nan142 out["n_levels_r"] = len(levels)143 return out144145rows = []146for d in data:147 prop = d.get("property") or {}148 addr = d.get("address") or {}149 feats = d.get("features") or []150 blob = (" ".join(feats) + " " + (d.get("description") or "") + " " +151 (d.get("title") or "")).lower()152153 yb = to_int(prop.get("yearBuilt"))154 age = (CUR_YEAR - yb) if (not pd.isna(yb) and 1700 <= yb <= CUR_YEAR) else np.nan155156 ptype = prop.get("buildingType") or prop.get("type") # realtor/duproprio157 storeys = prop.get("storeys") or prop.get("levels")158 try:159 storeys = float(str(storeys).replace(",", "."))160 except (TypeError, ValueError):161 storeys = np.nan162163 rec = {164 "price": d.get("priceNumber"),165 "source": d.get("source"),166 "sqft": parse_area(prop.get("squareFeet")),167 "lot_sqft": parse_area(prop.get("lotSize")),168 "beds": to_int(prop.get("bedrooms")),169 "baths": to_int(prop.get("bathrooms")),170 "parking": to_int(prop.get("parkingSpaces")), # realtor seulement171 "storeys": storeys,172 "age": age,173 "ptype": (ptype or "Unknown").strip(),174 "city": clean_city(addr.get("city"), d.get("source")),175 "region": (addr.get("region") or "").strip().lower() or None,176 }177 for name, kws in AMENITIES.items():178 rec[name] = int(any(k in blob for k in kws))179 # signaux "autre chose" : photos + visite virtuelle180 rec["n_images"] = len(d.get("images") or [])181 rec["virtual_tour"] = int(bool(d.get("virtualTourUrl")))182 # détail des pièces183 rec.update(parse_rooms(d.get("rooms")))184 rows.append(rec)185186df = pd.DataFrame(rows)187print(f"DataFrame brut : {df.shape}")188189# ---------- Nettoyage des aberrations ----------190n0 = len(df)191df = df[df["price"].notna()]192# prix plausibles pour une maison au Québec193df = df[(df["price"] >= 50_000) & (df["price"] <= 15_000_000)]194# superficie habitable : borner si présente, garder les NA (imputés + flag)195df.loc[(df["sqft"] < 300) | (df["sqft"] > 20_000), "sqft"] = np.nan196# terrain plausible (garder NA autorisé -> mais on impute plus bas)197df.loc[(df["lot_sqft"] < 200) | (df["lot_sqft"] > 5_000_000), "lot_sqft"] = np.nan198# chambres / sdb plausibles199df = df[(df["beds"].between(0, 15)) | (df["beds"].isna())]200df = df[(df["baths"].between(0, 12)) | (df["baths"].isna())]201df.loc[df["beds"] > 15, "beds"] = np.nan202df.loc[df["baths"] > 12, "baths"] = np.nan203# prix au pied carré aberrant (sur les lignes qui ont une superficie réelle)204df["ppsf"] = df["price"] / df["sqft"]205lo, hi = df["ppsf"].quantile([0.005, 0.995])206bad = df["ppsf"].notna() & ((df["ppsf"] < lo) | (df["ppsf"] > hi))207df = df[~bad]208print(f"Après nettoyage : {len(df)} (retirés : {n0 - len(df)}) "209 f"[filtre ppsf {lo:.0f}–{hi:.0f} $/ft² sur superficies réelles]")210211# ---------- Superficie : réelle > reconstruite via pièces > imputée ----------212# calibration : ratio médian (superficie réelle / somme des pièces) sur recouvrement213ov = df[df["sqft"].notna() & df["room_area"].notna() &214 (df["room_area"] > 200)]215ratio = (ov["sqft"] / ov["room_area"]).median()216print(f"Calibration surface pièces : ratio réel/pièces = {ratio:.3f} "217 f"(n={len(ov)})")218room_est = df["room_area"] * ratio219room_ok = df["room_area"].notna() & (room_est.between(300, 20_000))220221df["sqft_final"] = df["sqft"]222fill = df["sqft_final"].isna() & room_ok223df.loc[fill, "sqft_final"] = room_est[fill]224df["sqft_measured"] = df["sqft_final"].notna().astype(int) # réelle ou pièces225n_room_fill = int(fill.sum())226print(f"Superficies : réelles={int(df['sqft'].notna().sum())}, "227 f"complétées via pièces=+{n_room_fill}, "228 f"mesurées au total={int(df['sqft_measured'].sum())}/{len(df)}")229230# imputation du reste = médiane par type231df["has_sqft"] = df["sqft_measured"]232df["sqft_imp"] = df.groupby("ptype")["sqft_final"].transform(233 lambda s: s.fillna(s.median()))234df["sqft_imp"] = df["sqft_imp"].fillna(df["sqft_final"].median())235236df["has_lot"] = df["lot_sqft"].notna().astype(int)237df["lot_sqft"] = df["lot_sqft"].fillna(df["lot_sqft"].median())238df["has_age"] = df["age"].notna().astype(int)239df["age"] = df["age"].fillna(df["age"].median()).clip(lower=0)240df["beds"] = df["beds"].fillna(df["beds"].median())241df["baths"] = df["baths"].fillna(df["baths"].median())242df["parking"] = df["parking"].fillna(0)243df["storeys"] = df["storeys"].fillna(df["storeys"].median())244245df["ln_price"] = np.log(df["price"])246df["ln_sqft"] = np.log(df["sqft_imp"])247df["ln_lot"] = np.log(df["lot_sqft"])248df["age2"] = df["age"] ** 2249250# Effets fixes ville : regrouper les villes rares251vc = df["city"].value_counts()252keep = set(vc[vc >= 20].index)253df["city_fe"] = df["city"].where(df["city"].isin(keep), other="__other__")254df["city_fe"] = df["city_fe"].fillna("__other__")255print(f"Villes en effets fixes : {df['city_fe'].nunique()} "256 f"(seuil ≥20 listings ; {len(keep)} villes nommées)")257258# Type de propriété harmonisé (regrouper rares)259tc = df["ptype"].value_counts()260keept = set(tc[tc >= 30].index)261df["ptype_fe"] = df["ptype"].where(df["ptype"].isin(keept), other="Other")262263# ---------- Modèles hédoniques ----------264amen = list(AMENITIES.keys())265def pct(b):266 return (np.exp(b) - 1) * 100267268# variables "détail des pièces" + autres signaux (Realtor surtout)269df["n_images_c"] = df["n_images"].clip(upper=60) # écrête les extrêmes270room_vars = ["n_rooms", "n_levels_r", "has_primary_bed", "has_basement",271 "r_office", "r_family", "r_workshop", "r_winecellar",272 "r_sunroom", "r_mezzanine", "r_library", "r_laundry"]273extra = ["n_images_c", "virtual_tour"] + room_vars274275base = ("ln_lot + has_lot + beds + baths + parking + storeys "276 "+ age + age2 + has_age + C(source) + C(ptype_fe) + "277 + " + ".join(amen) + " + " + " + ".join(extra)278 + " + C(city_fe)")279fA = "ln_price ~ ln_sqft + " + base # riche (superficie réelle)280fB = "ln_price ~ " + base # large (sans superficie)281282dfA = df[df["has_sqft"] == 1].copy()283print(f"\nModèle A (avec superficie réelle) : N={len(dfA)}")284mA = smf.ols(fA, data=dfA).fit(cov_type="HC3")285print(f"Modèle B (sans superficie, couverture large) : N={len(df)}")286mB = smf.ols(fB, data=df).fit(cov_type="HC3")287288sdA, sdB = np.std(mA.resid), np.std(mB.resid)289290def report(m, sd, name):291 print("\n" + "=" * 70)292 print(f"{name} N={int(m.nobs)} R²={m.rsquared:.4f} "293 f"R²_adj={m.rsquared_adj:.4f} "294 f"erreur typique ±{(np.exp(sd)-1)*100:.1f}%")295 print("=" * 70)296 pr, pv = m.params, m.pvalues297 show = [p for p in pr.index if not p.startswith("C(city_fe)")]298 print(f"{'variable':<30}{'coef':>10}{'~%impact':>11}{'p':>9}")299 for p in show:300 st = "***" if pv[p]<.01 else "**" if pv[p]<.05 else "*" if pv[p]<.1 else ""301 pc = f"{pct(pr[p]):+.1f}%" if abs(pr[p]) < 3 else ""302 print(f"{p:<30}{pr[p]:>10.4f}{pc:>11}{pv[p]:>9.3g} {st}")303304report(mA, sdA, "MODÈLE A — hédonique riche (superficie incluse)")305report(mB, sdB, "MODÈLE B — couverture large (sans superficie)")306307# ---------- Prédictions combinées ----------308# A pour les biens à superficie réelle, B sinon309predA = np.exp(mA.predict(df)) * np.exp(sdA**2 / 2)310predB = np.exp(mB.predict(df)) * np.exp(sdB**2 / 2)311df["pred_price"] = np.where(df["has_sqft"] == 1, predA, predB)312df["model_used"] = np.where(df["has_sqft"] == 1, "A", "B")313df["resid_pct"] = (df["price"] / df["pred_price"] - 1) * 100314315out_cols = (["source", "city", "ptype", "price", "pred_price", "resid_pct",316 "model_used", "sqft", "room_area", "sqft_final", "sqft_measured",317 "lot_sqft", "beds", "baths", "parking", "storeys", "age",318 "n_images", "virtual_tour"] + amen + room_vars)319df[out_cols].to_csv("data/processed/maisons_hedonic.csv", index=False)320print(f"\nDataset final + prédictions -> maisons_hedonic.csv ({len(df)} lignes)")321322# top villes (effets fixes du modèle A)323cityfe = {k.split("T.")[1].rstrip("]"): v for k, v in mA.params.items()324 if k.startswith("C(city_fe)")}325cf = pd.Series(cityfe).sort_values(ascending=False)326print("\nTop 12 villes — prime de localisation (Modèle A, vs référence):")327for c, b in cf.head(12).items():328 print(f" {c:<30}{pct(b):+.1f}%")329print("Bottom 6:")330for c, b in cf.tail(6).items():331 print(f" {c:<30}{pct(b):+.1f}%")332333with open("results/hedonic_summary.txt", "w", encoding="utf-8") as fh:334 fh.write("=== MODELE A (avec superficie) ===\n")335 fh.write(str(mA.summary()))336 fh.write("\n\n=== MODELE B (sans superficie) ===\n")337 fh.write(str(mB.summary()))338print("Résumés complets -> hedonic_summary.txt")339