Python 57%
TeX 42.9%
1#!/usr/bin/env python32# -*- coding: utf-8 -*-3"""Blend (stacking) NN robuste + HistGradientBoosting, mêmes folds (seed=42)."""4import json, re, warnings5import numpy as np, pandas as pd6from sklearn.ensemble import HistGradientBoostingRegressor7from sklearn.model_selection import KFold, cross_val_predict8from sklearn.metrics import r2_score9warnings.filterwarnings("ignore")10CUR_YEAR=202611# --- rebuild df exactement comme torch_nn_robust (même ordre) ---12specs={}13for l in open("data/processed/specs.jsonl",encoding="utf-8"):14 r=json.loads(l)15 if r.get("specs"): specs[r["uid"]]=r["specs"]16COND={"a_renover":0,"original":1,"bon":2,"renove":3,"refait_neuf":4,"neuf":5,"inconnu":2}17BOOL=["turnkey","recently_renovated","garage","pool","waterfront","finished_basement","fireplace","central_ac","new_construction","income_property","renovated_kitchen","renovated_bathroom","large_lot","quiet_area"]18def gfeat(uid):19 s=specs.get(uid);o={"g_"+b:0 for b in BOOL};o.update({"g_cond":2,"g_hardwood":0,"g_quartz_granite":0,"g_view_water":0})20 if not s: return o21 for b in BOOL: o["g_"+b]=int(bool(s.get(b)))22 o["g_cond"]=COND.get(s.get("condition","inconnu"),2)23 fl=s.get("flooring") or [];o["g_hardwood"]=int("bois_franc" in fl)24 o["g_quartz_granite"]=int(s.get("countertop") in ("quartz","granit","marbre"));o["g_view_water"]=int(s.get("view")=="eau");return o25def parse_area(s):26 if not s or not isinstance(s,str): return np.nan27 t=s.lower();m=("m2" in t) or ("m²" in t);n=re.sub(r"[^0-9.,\s]","",t).strip().replace(" ","").replace(",","")28 try: v=float(n)29 except: return np.nan30 return (v*10.7639 if m else v) if v>0 else np.nan31def cc(c):32 if not c or not isinstance(c,str): return None33 c=c.split("(")[0].strip();return re.sub(r"\s+"," ",c).lower().strip() or None34def ti(x):35 try: return int(x)36 except: return np.nan37AMEN={"pool":["swimming pool","piscine","pool"],"waterfront":["waterfront","bord de l'eau","navigable","lakefront","riverfront","au bord de"],"garage":["garage"],"fireplace":["fireplace","foyer","wood stove","poêle"],"central_air":["central air","air conditionné","climatisation","a/c","heat-pump","thermopompe","air exchanger"],"finished_bsmt":["totally finished","finished basement","sous-sol fini","basement"],"near_water":["near water","waterfront","navigable"],"cul_de_sac":["cul-de-sac","cul de sac"],"golf":["golf"],"ski":["ski"],"near_park":["park","parc"],"near_school":["school","école","cegep","university","université","elementary","high school"],"near_highway":["highway","autoroute"],"transit":["public transit","public transportation","transport"],"no_rear_neighbor":["no backyard neighbors","sans voisin arrière"]}38def sd(s):39 s=s.lower()40 if "ft" in s:41 f=re.search(r"([\d.]+)\s*ft",s);i=re.search(r"([\d.]+)\s*in",s)42 return (float(f.group(1)) if f else 0)+(float(i.group(1))/12 if i else 0)43 m=re.search(r"([\d.,]+)\s*m",s)44 if m:45 try: return float(m.group(1).replace(",","."))*3.2808446 except: return None47 return None48def ra(d):49 if not d or "x" not in d: return None50 a,_,b=d.partition("x");la,lb=sd(a),sd(b);return la*lb if (la and lb and 1<la<200 and 1<lb<200) else None51RF={"r_office":["office"],"r_family":["family room"],"r_workshop":["workshop"],"r_winecellar":["wine cellar"],"r_sunroom":["sunroom","solarium"],"r_mezzanine":["mezzanine"],"r_library":["library"],"r_laundry":["laundry"]}52def pr(rooms):53 o={"room_area":np.nan,"n_rooms":0,"has_primary_bed":0,"has_basement":0,"n_levels_r":0}54 for k in RF: o[k]=055 if not rooms: return o56 ar=0.0;lv=set()57 for r in rooms:58 nm=(r.get("name") or "").lower();l=(r.get("level") or "").lower();o["n_rooms"]+=159 if l: lv.add(l)60 if "basement" in l or "sous-sol" in l: o["has_basement"]=161 if "primary bedroom" in nm: o["has_primary_bed"]=162 for k,kw in RF.items():63 if any(w in nm for w in kw): o[k]=164 a=ra(r.get("dimensions"))65 if a: ar+=a66 o["room_area"]=ar if ar>0 else np.nan;o["n_levels_r"]=len(lv);return o67data=json.load(open("data/processed/maisons_avec_prix.json",encoding="utf-8"));rows=[]68for d in data:69 p=d.get("property") or {};ad=d.get("address") or {};ft=d.get("features") or []70 blob=(" ".join(ft)+" "+(d.get("description") or "")+" "+(d.get("title") or "")).lower()71 yb=ti(p.get("yearBuilt"));age=(CUR_YEAR-yb) if (not pd.isna(yb) and 1700<=yb<=CUR_YEAR) else np.nan72 pt=p.get("buildingType") or p.get("type");st=p.get("storeys") or p.get("levels")73 try: st=float(str(st).replace(",","."))74 except: st=np.nan75 uid=f"{d.get('source')}:{d.get('listingId')}"76 rec={"uid":uid,"price":d.get("priceNumber"),"source":d.get("source"),"sqft":parse_area(p.get("squareFeet")),"lot_sqft":parse_area(p.get("lotSize")),"beds":ti(p.get("bedrooms")),"baths":ti(p.get("bathrooms")),"parking":ti(p.get("parkingSpaces")),"storeys":st,"age":age,"ptype":(pt or "Unknown").strip(),"city":cc(ad.get("city"))}77 for n,kw in AMEN.items(): rec[n]=int(any(k in blob for k in kw))78 rec["n_images"]=len(d.get("images") or []);rec["virtual_tour"]=int(bool(d.get("virtualTourUrl")))79 rec.update(pr(d.get("rooms")));rec.update(gfeat(uid));rows.append(rec)80df=pd.DataFrame(rows)81df=df[df["price"].notna()];df=df[(df["price"]>=50_000)&(df["price"]<=15_000_000)]82df.loc[(df["sqft"]<300)|(df["sqft"]>20_000),"sqft"]=np.nan83df.loc[(df["lot_sqft"]<200)|(df["lot_sqft"]>5_000_000),"lot_sqft"]=np.nan84df.loc[df["beds"]>15,"beds"]=np.nan;df.loc[df["baths"]>12,"baths"]=np.nan85df["ppsf"]=df["price"]/df["sqft"];lo,hi=df["ppsf"].quantile([0.005,0.995])86df=df[~(df["ppsf"].notna()&((df["ppsf"]<lo)|(df["ppsf"]>hi)))]87ov=df[df["sqft"].notna()&df["room_area"].notna()&(df["room_area"]>200)];ratio=(ov["sqft"]/ov["room_area"]).median()88re_=df["room_area"]*ratio;ok=df["room_area"].notna()&(re_.between(300,20000))89df["sqft_final"]=df["sqft"];fill=df["sqft_final"].isna()&ok;df.loc[fill,"sqft_final"]=re_[fill]90df["has_sqft"]=df["sqft_final"].notna().astype(int);df["age2"]=df["age"]**2;df["n_images_c"]=df["n_images"].clip(upper=60)91vc=df["city"].value_counts();keep=set(vc[vc>=20].index);df["city_fe"]=df["city"].where(df["city"].isin(keep),"__other__").fillna("__other__")92tc=df["ptype"].value_counts();df["ptype_fe"]=df["ptype"].where(df["ptype"].isin(set(tc[tc>=30].index)),"Other")93df=df.reset_index(drop=True)94y=np.log(df["price"].values)95amen=list(AMEN.keys());rv=["n_rooms","n_levels_r","has_primary_bed","has_basement","r_office","r_family","r_workshop","r_winecellar","r_sunroom","r_mezzanine","r_library","r_laundry"]96gv=["g_cond","g_hardwood","g_quartz_granite","g_view_water"]+["g_"+b for b in BOOL]97num=["sqft_final","has_sqft","lot_sqft","beds","baths","parking","storeys","age","age2","n_images_c","virtual_tour"]+amen+rv+gv98Xcat=pd.get_dummies(df[["city_fe","ptype_fe","source"]],dtype=float)99X=pd.concat([df[num].reset_index(drop=True),Xcat.reset_index(drop=True)],axis=1)100kf=KFold(n_splits=5,shuffle=True,random_state=42)101gbm=HistGradientBoostingRegressor(max_iter=600,learning_rate=0.05,max_leaf_nodes=63,l2_regularization=1.0,random_state=42)102pgbm=cross_val_predict(gbm,X,y,cv=kf,n_jobs=-1)103pnn=np.load("data/processed/nn_robust_oos.npy")104assert len(pnn)==len(y),f"désalignement {len(pnn)} vs {len(y)}"105def stat(p):106 e=np.abs(np.exp(p-y)-1)107 return r2_score(y,p),np.median(e)*100,np.mean(e<=.15)*100,np.mean(e<=.25)*100108print(f"{'Modèle':<30}{'R² OOS':>9}{'MAE méd':>9}{'±15%':>7}{'±25%':>7}")109for nm,p in [("HistGBM",pgbm),("NN robuste",pnn)]:110 r2,m,a,b=stat(p);print(f"{nm:<30}{r2:>9.4f}{m:>8.1f}%{a:>6.0f}%{b:>6.0f}%")111best=None112for w in [0.3,0.4,0.5,0.6,0.7]:113 p=w*pgbm+(1-w)*pnn;r2,m,a,b=stat(p)114 tag=f"Blend {w:.1f}·GBM+{1-w:.1f}·NN"115 print(f"{tag:<30}{r2:>9.4f}{m:>8.1f}%{a:>6.0f}%{b:>6.0f}%")116 if best is None or r2>best[0]: best=(r2,w)117print(f"\nMeilleur blend: w_GBM={best[1]:.1f} R²={best[0]:.4f}")118