Python 57%
TeX 42.9%
1#!/usr/bin/env python32# -*- coding: utf-8 -*-3"""Réseau de neurones tabulaire ROBUSTE : embeddings + ResNet-MLP,4ensemble multi-seed, Huber loss, AdamW + cosine, grad-clip, early stopping.55-fold CV out-of-sample, comparable aux autres modèles (seed=42)."""6import json, re, warnings7import numpy as np, pandas as pd8import torch, torch.nn as nn9from sklearn.model_selection import KFold10from sklearn.metrics import r2_score11warnings.filterwarnings("ignore")12CUR_YEAR=202613dev="mps" if torch.backends.mps.is_available() else "cpu"14print("device:",dev)15SEEDS=[42,1,7] # ensemble par fold16D_MAIN=256; N_BLOCKS=3; D_HIDDEN=5121718# ======== reconstruction du dataset (identique v4) ========19specs={}20for l in open("data/processed/specs.jsonl",encoding="utf-8"):21 r=json.loads(l)22 if r.get("specs"): specs[r["uid"]]=r["specs"]23COND={"a_renover":0,"original":1,"bon":2,"renove":3,"refait_neuf":4,"neuf":5,"inconnu":2}24BOOL=["turnkey","recently_renovated","garage","pool","waterfront","finished_basement","fireplace","central_ac","new_construction","income_property","renovated_kitchen","renovated_bathroom","large_lot","quiet_area"]25def gfeat(uid):26 s=specs.get(uid);o={"g_"+b:0 for b in BOOL};o.update({"g_cond":2,"g_hardwood":0,"g_quartz_granite":0,"g_view_water":0})27 if not s: return o28 for b in BOOL: o["g_"+b]=int(bool(s.get(b)))29 o["g_cond"]=COND.get(s.get("condition","inconnu"),2)30 fl=s.get("flooring") or [];o["g_hardwood"]=int("bois_franc" in fl)31 o["g_quartz_granite"]=int(s.get("countertop") in ("quartz","granit","marbre"));o["g_view_water"]=int(s.get("view")=="eau");return o32def parse_area(s):33 if not s or not isinstance(s,str): return np.nan34 t=s.lower();m=("m2" in t) or ("m²" in t);n=re.sub(r"[^0-9.,\s]","",t).strip().replace(" ","").replace(",","")35 try: v=float(n)36 except: return np.nan37 return (v*10.7639 if m else v) if v>0 else np.nan38def cc(c):39 if not c or not isinstance(c,str): return None40 c=c.split("(")[0].strip();return re.sub(r"\s+"," ",c).lower().strip() or None41def ti(x):42 try: return int(x)43 except: return np.nan44AMEN={"pool":["swimming pool","piscine","pool"],"waterfront":["waterfront","bord de l'eau","navigable","lakefront","riverfront","au bord de"],"garage":["garage"],"fireplace":["fireplace","foyer","wood stove","poêle"],"central_air":["central air","air conditionné","climatisation","a/c","heat-pump","thermopompe","air exchanger"],"finished_bsmt":["totally finished","finished basement","sous-sol fini","basement"],"near_water":["near water","waterfront","navigable"],"cul_de_sac":["cul-de-sac","cul de sac"],"golf":["golf"],"ski":["ski"],"near_park":["park","parc"],"near_school":["school","école","cegep","university","université","elementary","high school"],"near_highway":["highway","autoroute"],"transit":["public transit","public transportation","transport"],"no_rear_neighbor":["no backyard neighbors","sans voisin arrière"]}45def sd(s):46 s=s.lower()47 if "ft" in s:48 f=re.search(r"([\d.]+)\s*ft",s);i=re.search(r"([\d.]+)\s*in",s)49 return (float(f.group(1)) if f else 0)+(float(i.group(1))/12 if i else 0)50 m=re.search(r"([\d.,]+)\s*m",s)51 if m:52 try: return float(m.group(1).replace(",","."))*3.2808453 except: return None54 return None55def ra(d):56 if not d or "x" not in d: return None57 a,_,b=d.partition("x");la,lb=sd(a),sd(b);return la*lb if (la and lb and 1<la<200 and 1<lb<200) else None58RF={"r_office":["office"],"r_family":["family room"],"r_workshop":["workshop"],"r_winecellar":["wine cellar"],"r_sunroom":["sunroom","solarium"],"r_mezzanine":["mezzanine"],"r_library":["library"],"r_laundry":["laundry"]}59def pr(rooms):60 o={"room_area":np.nan,"n_rooms":0,"has_primary_bed":0,"has_basement":0,"n_levels_r":0}61 for k in RF: o[k]=062 if not rooms: return o63 ar=0.0;lv=set()64 for r in rooms:65 nm=(r.get("name") or "").lower();l=(r.get("level") or "").lower();o["n_rooms"]+=166 if l: lv.add(l)67 if "basement" in l or "sous-sol" in l: o["has_basement"]=168 if "primary bedroom" in nm: o["has_primary_bed"]=169 for k,kw in RF.items():70 if any(w in nm for w in kw): o[k]=171 a=ra(r.get("dimensions"))72 if a: ar+=a73 o["room_area"]=ar if ar>0 else np.nan;o["n_levels_r"]=len(lv);return o74data=json.load(open("data/processed/maisons_avec_prix.json",encoding="utf-8"));rows=[]75for d in data:76 p=d.get("property") or {};ad=d.get("address") or {};ft=d.get("features") or []77 blob=(" ".join(ft)+" "+(d.get("description") or "")+" "+(d.get("title") or "")).lower()78 yb=ti(p.get("yearBuilt"));age=(CUR_YEAR-yb) if (not pd.isna(yb) and 1700<=yb<=CUR_YEAR) else np.nan79 pt=p.get("buildingType") or p.get("type");st=p.get("storeys") or p.get("levels")80 try: st=float(str(st).replace(",","."))81 except: st=np.nan82 uid=f"{d.get('source')}:{d.get('listingId')}"83 rec={"uid":uid,"price":d.get("priceNumber"),"source":d.get("source"),"sqft":parse_area(p.get("squareFeet")),"lot_sqft":parse_area(p.get("lotSize")),"beds":ti(p.get("bedrooms")),"baths":ti(p.get("bathrooms")),"parking":ti(p.get("parkingSpaces")),"storeys":st,"age":age,"ptype":(pt or "Unknown").strip(),"city":cc(ad.get("city"))}84 for n,kw in AMEN.items(): rec[n]=int(any(k in blob for k in kw))85 rec["n_images"]=len(d.get("images") or []);rec["virtual_tour"]=int(bool(d.get("virtualTourUrl")))86 rec.update(pr(d.get("rooms")));rec.update(gfeat(uid));rows.append(rec)87df=pd.DataFrame(rows)88df=df[df["price"].notna()];df=df[(df["price"]>=50_000)&(df["price"]<=15_000_000)]89df.loc[(df["sqft"]<300)|(df["sqft"]>20_000),"sqft"]=np.nan90df.loc[(df["lot_sqft"]<200)|(df["lot_sqft"]>5_000_000),"lot_sqft"]=np.nan91df.loc[df["beds"]>15,"beds"]=np.nan;df.loc[df["baths"]>12,"baths"]=np.nan92df["ppsf"]=df["price"]/df["sqft"];lo,hi=df["ppsf"].quantile([0.005,0.995])93df=df[~(df["ppsf"].notna()&((df["ppsf"]<lo)|(df["ppsf"]>hi)))]94ov=df[df["sqft"].notna()&df["room_area"].notna()&(df["room_area"]>200)];ratio=(ov["sqft"]/ov["room_area"]).median()95re_=df["room_area"]*ratio;ok=df["room_area"].notna()&(re_.between(300,20000))96df["sqft_final"]=df["sqft"];fill=df["sqft_final"].isna()&ok;df.loc[fill,"sqft_final"]=re_[fill]97df["has_sqft"]=df["sqft_final"].notna().astype(int);df["age2"]=df["age"]**2;df["n_images_c"]=df["n_images"].clip(upper=60)98vc=df["city"].value_counts();keep=set(vc[vc>=20].index);df["city_fe"]=df["city"].where(df["city"].isin(keep),"__other__").fillna("__other__")99tc=df["ptype"].value_counts();df["ptype_fe"]=df["ptype"].where(df["ptype"].isin(set(tc[tc>=30].index)),"Other")100df=df.reset_index(drop=True);print("N=",len(df))101102amen=list(AMEN.keys());rv=["n_rooms","n_levels_r","has_primary_bed","has_basement","r_office","r_family","r_workshop","r_winecellar","r_sunroom","r_mezzanine","r_library","r_laundry"]103gv=["g_cond","g_hardwood","g_quartz_granite","g_view_water"]+["g_"+b for b in BOOL]104num=["sqft_final","has_sqft","lot_sqft","beds","baths","parking","storeys","age","age2","n_images_c","virtual_tour"]+amen+rv+gv105# indicateurs de manquant (robustesse) pour les colonnes à NaN106missnum=["sqft_final","lot_sqft","beds","baths","storeys","age","age2"]107for c in missnum: df["miss_"+c]=df[c].isna().astype(np.float32)108num=num+["miss_"+c for c in missnum]109y=np.log(df["price"].values).astype(np.float32)110cats=["city_fe","ptype_fe","source"];card={}111for c in cats:112 u=sorted(df[c].astype(str).unique());m={v:i for i,v in enumerate(u)};card[c]=len(u)113 df[c+"_ix"]=df[c].astype(str).map(m).astype(np.int64)114Xnum=df[num].astype(np.float32).values115Xcat=df[[c+"_ix" for c in cats]].values.astype(np.int64)116117class ResBlock(nn.Module):118 def __init__(self,d,dh,p):119 super().__init__()120 self.bn=nn.BatchNorm1d(d);self.l1=nn.Linear(d,dh);self.act=nn.GELU()121 self.dp1=nn.Dropout(p);self.l2=nn.Linear(dh,d);self.dp2=nn.Dropout(p)122 def forward(self,x):123 z=self.bn(x);z=self.act(self.l1(z));z=self.dp1(z);z=self.l2(z);z=self.dp2(z);return x+z124125class ResNetTab(nn.Module):126 def __init__(self,ncard,nnum):127 super().__init__()128 self.embs=nn.ModuleList();tot=0129 for c in ncard:130 d=max(2,min(64,round(1.6*c**0.56)));self.embs.append(nn.Embedding(c,d));tot+=d131 self.embdp=nn.Dropout(0.1)132 self.bn0=nn.BatchNorm1d(nnum)133 self.proj=nn.Linear(tot+nnum,D_MAIN)134 self.blocks=nn.ModuleList([ResBlock(D_MAIN,D_HIDDEN,0.2) for _ in range(N_BLOCKS)])135 self.head=nn.Sequential(nn.BatchNorm1d(D_MAIN),nn.GELU(),nn.Linear(D_MAIN,1))136 def forward(self,xc,xn):137 e=[emb(xc[:,i]) for i,emb in enumerate(self.embs)]138 x=torch.cat([self.embdp(torch.cat(e,1)),self.bn0(xn)],1)139 x=self.proj(x)140 for b in self.blocks: x=b(x)141 return self.head(x).squeeze(1)142143def train_one(tr,te,seed):144 torch.manual_seed(seed);np.random.seed(seed)145 med=np.nanmedian(Xnum[tr],axis=0);med=np.where(np.isnan(med),0,med)146 Xn=Xnum.copy()147 for j in range(Xn.shape[1]):148 col=Xn[:,j];col[np.isnan(col)]=med[j];Xn[:,j]=col149 mu=Xn[tr].mean(0);sg=Xn[tr].std(0)+1e-6;Xn=(Xn-mu)/sg150 ym=y[tr].mean();ys=y[tr].std()+1e-6;yz=(y-ym)/ys151 ntr=len(tr);perm=np.random.permutation(ntr);vc_=int(ntr*0.9)152 tri=tr[perm[:vc_]];vai=tr[perm[vc_:]]153 def T(a,dt): return torch.tensor(a,device=dev,dtype=dt)154 model=ResNetTab([card[c] for c in cats],Xn.shape[1]).to(dev)155 opt=torch.optim.AdamW(model.parameters(),lr=2e-3,weight_decay=1e-4)156 EPOCHS=200157 sched=torch.optim.lr_scheduler.CosineAnnealingLR(opt,T_max=EPOCHS)158 lossf=nn.SmoothL1Loss(beta=1.0) # Huber : robuste aux queues159 Xc_t=T(Xcat,torch.long);Xn_t=T(Xn,torch.float32);y_t=T(yz,torch.float32)160 best=1e9;bstate=None;pat=0161 for ep in range(EPOCHS):162 model.train();idx=np.random.permutation(tri)163 for i in range(0,len(idx),512):164 b=idx[i:i+512]165 opt.zero_grad();p=model(Xc_t[b],Xn_t[b]);l=lossf(p,y_t[b])166 l.backward();torch.nn.utils.clip_grad_norm_(model.parameters(),5.0);opt.step()167 sched.step()168 model.eval()169 with torch.no_grad():170 vp=model(Xc_t[vai],Xn_t[vai]);vl=((vp-y_t[vai])**2).mean().item()171 if vl<best-1e-4: best=vl;bstate={k:v.clone() for k,v in model.state_dict().items()};pat=0172 else:173 pat+=1174 if pat>=20: break175 model.load_state_dict(bstate);model.eval()176 with torch.no_grad():177 pz=model(Xc_t[te],Xn_t[te]).cpu().numpy()178 return pz*ys+ym179180kf=KFold(n_splits=5,shuffle=True,random_state=42)181pred=np.zeros_like(y)182for f,(tr,te) in enumerate(kf.split(Xnum)):183 ps=np.zeros(len(te),dtype=np.float64)184 for sd_ in SEEDS:185 ps+=train_one(tr,te,sd_)186 pred[te]=ps/len(SEEDS) # moyenne d'ensemble187 r2f=r2_score(y[te],pred[te]);print(f" fold {f+1}: R²={r2f:.4f} (ensemble de {len(SEEDS)})")188r2=r2_score(y,pred);err=np.abs(np.exp(pred-y)-1)189print("\n=== NN ROBUSTE (ResNet-tab, ensemble multi-seed) — 5-fold CV OOS ===")190print(f"R² OOS = {r2:.4f} | erreur médiane = {np.median(err)*100:.1f}% | ±15% = {np.mean(err<=.15)*100:.0f}% | ±25% = {np.mean(err<=.25)*100:.0f}%")191print("\nRappel (mêmes folds): OLS 0.628 | RF 0.678 | NN simple 0.711 | HistGBM 0.743")192np.save("data/processed/nn_robust_oos.npy",pred)193