Python 57%
TeX 42.9%
1#!/usr/bin/env python32# -*- coding: utf-8 -*-3"""4Modèle hédonique v4 : ajoute les specs extraites par GPT-4o-mini (specs.jsonl)5au modèle v3. Réutilise la même logique de nettoyage/features + effets fixes.6"""7import json, re, warnings8import numpy as np9import pandas as pd10import statsmodels.formula.api as smf1112warnings.filterwarnings("ignore")13CUR_YEAR = 20261415# ---------- charge specs GPT ----------16specs = {}17for line in open("data/processed/specs.jsonl", encoding="utf-8"):18 try:19 r = json.loads(line)20 except Exception:21 continue22 if "specs" in r and r.get("specs"):23 specs[r["uid"]] = r["specs"]24print(f"Specs GPT chargées : {len(specs)}")2526# specs -> features numériques (dummies + catégorielles réduites)27COND_SCORE = {"a_renover":0,"original":1,"bon":2,"renove":3,"refait_neuf":4,28 "neuf":5,"inconnu":2}29BOOL_SPECS = ["turnkey","recently_renovated","garage","pool","waterfront",30 "finished_basement","fireplace","central_ac","new_construction",31 "income_property","renovated_kitchen","renovated_bathroom",32 "large_lot","quiet_area"]3334def spec_feats(uid):35 s = specs.get(uid)36 out = {"g_" + b: 0 for b in BOOL_SPECS}37 out["g_cond"] = 238 out["g_hardwood"] = 0; out["g_quartz_granite"] = 0; out["g_view_water"] = 039 out["has_gpt"] = 040 if not s:41 return out42 out["has_gpt"] = 143 for b in BOOL_SPECS:44 out["g_" + b] = int(bool(s.get(b)))45 out["g_cond"] = COND_SCORE.get(s.get("condition", "inconnu"), 2)46 fl = s.get("flooring") or []47 out["g_hardwood"] = int("bois_franc" in fl)48 out["g_quartz_granite"] = int(s.get("countertop") in ("quartz", "granit", "marbre"))49 out["g_view_water"] = int(s.get("view") == "eau")50 return out5152# ===================== pipeline v3 (repris) =====================53SRC = "data/processed/maisons_avec_prix.json"54data = json.load(open(SRC, encoding="utf-8"))5556def parse_area(s):57 if not s or not isinstance(s, str): return np.nan58 txt = s.lower(); is_m2 = ("m2" in txt) or ("m²" in txt)59 num = re.sub(r"[^0-9.,\s]", "", txt).strip().replace(" ", "").replace(",", "")60 try: v = float(num)61 except ValueError: return np.nan62 if v <= 0: return np.nan63 return v * 10.7639 if is_m2 else v6465def clean_city(city, source):66 if not city or not isinstance(city, str): return None67 c = city.split("(")[0].strip(); c = re.sub(r"\s+", " ", c).lower().strip()68 return c or None6970def to_int(x):71 try: return int(x)72 except (TypeError, ValueError): return np.nan7374AMENITIES = {75 "pool":["swimming pool","piscine","inground pool","pool"],76 "waterfront":["waterfront","bord de l'eau","navigable","water access","lakefront","riverfront","au bord de"],77 "garage":["garage"],"fireplace":["fireplace","foyer","wood stove","poêle"],78 "central_air":["central air","air conditionné","climatisation","a/c","heat-pump","thermopompe","air exchanger"],79 "finished_bsmt":["totally finished","finished basement","sous-sol fini","basement"],80 "near_water":["near water","waterfront","navigable"],81 "cul_de_sac":["cul-de-sac","cul de sac"],"golf":["golf"],"ski":["ski"],82 "near_park":["park","parc"],83 "near_school":["school","école","cegep","university","université","elementary","high school"],84 "near_highway":["highway","autoroute"],85 "transit":["public transit","public transportation","transport"],86 "no_rear_neighbor":["no backyard neighbors","no rear neighbor","sans voisin arrière"],87}8889def _side_ft(side):90 s = side.lower()91 if "ft" in s:92 ft = re.search(r"([\d.]+)\s*ft", s); inch = re.search(r"([\d.]+)\s*in", s)93 return (float(ft.group(1)) if ft else 0)+(float(inch.group(1))/12 if inch else 0)94 m = re.search(r"([\d.,]+)\s*m", s)95 if m:96 try: return float(m.group(1).replace(",", ".")) * 3.2808497 except ValueError: return None98 return None99100def _room_area(dim):101 if not dim or "x" not in dim: return None102 a, _, b = dim.partition("x"); la, lb = _side_ft(a), _side_ft(b)103 if la and lb and 1 < la < 200 and 1 < lb < 200: return la * lb104 return None105106ROOM_FLAGS = {"r_office":["office"],"r_family":["family room"],"r_workshop":["workshop"],107 "r_winecellar":["wine cellar"],"r_sunroom":["sunroom","solarium"],108 "r_mezzanine":["mezzanine"],"r_library":["library"],"r_laundry":["laundry"]}109110def parse_rooms(rooms):111 out = {"room_area":np.nan,"n_rooms":0,"has_primary_bed":0,"has_basement":0,"n_levels_r":0}112 for k in ROOM_FLAGS: out[k] = 0113 if not rooms: return out114 area, levels = 0.0, set()115 for r in rooms:116 nm = (r.get("name") or "").lower(); lv = (r.get("level") or "").lower()117 out["n_rooms"] += 1118 if lv: levels.add(lv)119 if "basement" in lv or "sous-sol" in lv: out["has_basement"] = 1120 if "primary bedroom" in nm: out["has_primary_bed"] = 1121 for k, kws in ROOM_FLAGS.items():122 if any(w in nm for w in kws): out[k] = 1123 a = _room_area(r.get("dimensions"))124 if a: area += a125 out["room_area"] = area if area > 0 else np.nan126 out["n_levels_r"] = len(levels)127 return out128129rows = []130for d in data:131 prop = d.get("property") or {}; addr = d.get("address") or {}132 feats = d.get("features") or []133 blob = (" ".join(feats)+" "+(d.get("description") or "")+" "+(d.get("title") or "")).lower()134 yb = to_int(prop.get("yearBuilt"))135 age = (CUR_YEAR-yb) if (not pd.isna(yb) and 1700 <= yb <= CUR_YEAR) else np.nan136 ptype = prop.get("buildingType") or prop.get("type")137 storeys = prop.get("storeys") or prop.get("levels")138 try: storeys = float(str(storeys).replace(",", "."))139 except (TypeError, ValueError): storeys = np.nan140 uid = f"{d.get('source')}:{d.get('listingId')}"141 rec = {"uid":uid,"price":d.get("priceNumber"),"source":d.get("source"),142 "sqft":parse_area(prop.get("squareFeet")),"lot_sqft":parse_area(prop.get("lotSize")),143 "beds":to_int(prop.get("bedrooms")),"baths":to_int(prop.get("bathrooms")),144 "parking":to_int(prop.get("parkingSpaces")),"storeys":storeys,"age":age,145 "ptype":(ptype or "Unknown").strip(),"city":clean_city(addr.get("city"),d.get("source"))}146 for name, kws in AMENITIES.items():147 rec[name] = int(any(k in blob for k in kws))148 rec["n_images"] = len(d.get("images") or [])149 rec["virtual_tour"] = int(bool(d.get("virtualTourUrl")))150 rec.update(parse_rooms(d.get("rooms")))151 rec.update(spec_feats(uid))152 rows.append(rec)153154df = pd.DataFrame(rows)155156# nettoyage identique v3157n0 = len(df)158df = df[df["price"].notna()]159df = df[(df["price"] >= 50_000) & (df["price"] <= 15_000_000)]160df.loc[(df["sqft"] < 300) | (df["sqft"] > 20_000), "sqft"] = np.nan161df.loc[(df["lot_sqft"] < 200) | (df["lot_sqft"] > 5_000_000), "lot_sqft"] = np.nan162df.loc[df["beds"] > 15, "beds"] = np.nan163df.loc[df["baths"] > 12, "baths"] = np.nan164df["ppsf"] = df["price"] / df["sqft"]165lo, hi = df["ppsf"].quantile([0.005, 0.995])166df = df[~(df["ppsf"].notna() & ((df["ppsf"] < lo) | (df["ppsf"] > hi)))]167168# surface finale169ov = df[df["sqft"].notna() & df["room_area"].notna() & (df["room_area"] > 200)]170ratio = (ov["sqft"] / ov["room_area"]).median()171room_est = df["room_area"] * ratio172room_ok = df["room_area"].notna() & (room_est.between(300, 20_000))173df["sqft_final"] = df["sqft"]174fill = df["sqft_final"].isna() & room_ok175df.loc[fill, "sqft_final"] = room_est[fill]176df["has_sqft"] = df["sqft_final"].notna().astype(int)177df["sqft_imp"] = df.groupby("ptype")["sqft_final"].transform(lambda s: s.fillna(s.median()))178df["sqft_imp"] = df["sqft_imp"].fillna(df["sqft_final"].median())179180df["has_lot"] = df["lot_sqft"].notna().astype(int)181df["lot_sqft"] = df["lot_sqft"].fillna(df["lot_sqft"].median())182df["has_age"] = df["age"].notna().astype(int)183df["age"] = df["age"].fillna(df["age"].median()).clip(lower=0)184for c in ["beds","baths","storeys"]:185 df[c] = df[c].fillna(df[c].median())186df["parking"] = df["parking"].fillna(0)187df["ln_price"] = np.log(df["price"]); df["ln_sqft"] = np.log(df["sqft_imp"])188df["ln_lot"] = np.log(df["lot_sqft"]); df["age2"] = df["age"]**2189df["n_images_c"] = df["n_images"].clip(upper=60)190191vc = df["city"].value_counts(); keep = set(vc[vc >= 20].index)192df["city_fe"] = df["city"].where(df["city"].isin(keep), "__other__").fillna("__other__")193tc = df["ptype"].value_counts(); keept = set(tc[tc >= 30].index)194df["ptype_fe"] = df["ptype"].where(df["ptype"].isin(keept), "Other")195print(f"N nettoyé={len(df)} villes FE={df['city_fe'].nunique()} "196 f"avec specs GPT={int(df['has_gpt'].sum())}")197198# ---------- modèles ----------199amen = list(AMENITIES.keys())200room_vars = ["n_rooms","n_levels_r","has_primary_bed","has_basement","r_office",201 "r_family","r_workshop","r_winecellar","r_sunroom","r_mezzanine",202 "r_library","r_laundry"]203gpt_vars = ["g_cond","g_hardwood","g_quartz_granite","g_view_water","has_gpt"] + \204 ["g_" + b for b in BOOL_SPECS]205extra = ["n_images_c","virtual_tour"] + room_vars + gpt_vars206base = ("ln_lot + has_lot + beds + baths + parking + storeys + age + age2 + has_age "207 "+ C(source) + C(ptype_fe) + " + " + ".join(amen) + " + " + " + ".join(extra)208 + " + C(city_fe)")209fA = "ln_price ~ ln_sqft + " + base210fB = "ln_price ~ " + base211212def pct(b): return (np.exp(b)-1)*100213dfA = df[df["has_sqft"] == 1].copy()214mA = smf.ols(fA, data=dfA).fit(cov_type="HC3")215mB = smf.ols(fB, data=df).fit(cov_type="HC3")216sdA, sdB = np.std(mA.resid), np.std(mB.resid)217218def report(m, sd, name):219 print("\n" + "=" * 70)220 print(f"{name} N={int(m.nobs)} R²={m.rsquared:.4f} R²adj={m.rsquared_adj:.4f} "221 f"±{(np.exp(sd)-1)*100:.1f}%")222 print("=" * 70)223 pr, pv = m.params, m.pvalues224 print("-- specs GPT (impact %) --")225 for p in pr.index:226 if p.startswith("g_") or p == "has_gpt":227 st = "***" if pv[p]<.01 else "**" if pv[p]<.05 else "*" if pv[p]<.1 else ""228 print(f" {p:<22}{pr[p]:>9.4f}{pct(pr[p]):>+8.1f}% p={pv[p]:.2g} {st}")229230report(mA, sdA, "MODÈLE A+GPT (avec superficie)")231report(mB, sdB, "MODÈLE B+GPT (couverture large)")232233predA = np.exp(mA.predict(df)) * np.exp(sdA**2/2)234predB = np.exp(mB.predict(df)) * np.exp(sdB**2/2)235df["pred_price"] = np.where(df["has_sqft"] == 1, predA, predB)236df["model_used"] = np.where(df["has_sqft"] == 1, "A", "B")237df["resid_pct"] = (df["price"]/df["pred_price"] - 1) * 100238print(f"\nErreur abs médiane: {df.resid_pct.abs().median():.1f}% "239 f"| ±15%: {(df.resid_pct.abs()<=15).mean()*100:.0f}% "240 f"±25%: {(df.resid_pct.abs()<=25).mean()*100:.0f}%")241242out = (["uid","source","city","ptype","price","pred_price","resid_pct","model_used",243 "sqft_final","has_sqft","lot_sqft","beds","baths","parking","age","n_images",244 "virtual_tour"] + amen + room_vars + gpt_vars)245df[out].to_csv("data/processed/maisons_hedonic_v4.csv", index=False)246print(f"-> maisons_hedonic_v4.csv ({len(df)} lignes)")247with open("results/hedonic_v4_summary.txt", "w", encoding="utf-8") as fh:248 fh.write("=== A+GPT ===\n"+str(mA.summary())+"\n\n=== B+GPT ===\n"+str(mB.summary()))249print("-> hedonic_v4_summary.txt")250