Python 57%
TeX 42.9%
1#!/usr/bin/env python32# -*- coding: utf-8 -*-3"""v5 : ablation texte (v4) vs texte+vision (v5). GBM + NN robuste + blend, 5-fold CV."""4import json, re, warnings5import numpy as np, pandas as pd6import torch, torch.nn as nn7from sklearn.ensemble import HistGradientBoostingRegressor8from sklearn.model_selection import KFold, cross_val_predict9from sklearn.metrics import r2_score10warnings.filterwarnings("ignore")11torch.manual_seed(42); np.random.seed(42)12CUR_YEAR=202613dev="mps" if torch.backends.mps.is_available() else "cpu"; print("device:",dev)1415# ---- GPT specs ----16specs={}17for l in open("data/processed/specs.jsonl",encoding="utf-8"):18 r=json.loads(l)19 if r.get("specs"): specs[r["uid"]]=r["specs"]20# ---- vision ----21facade={}22for l in open("data/processed/facade_specs.jsonl",encoding="utf-8"):23 r=json.loads(l)24 if r.get("facade"): facade[r["uid"]]=r["facade"]25print("specs GPT:",len(specs),"| façades:",len(facade))2627COND={"a_renover":0,"original":1,"bon":2,"renove":3,"refait_neuf":4,"neuf":5,"inconnu":2}28BOOL=["turnkey","recently_renovated","garage","pool","waterfront","finished_basement","fireplace","central_ac","new_construction","income_property","renovated_kitchen","renovated_bathroom","large_lot","quiet_area"]29def gfeat(uid):30 s=specs.get(uid);o={"g_"+b:0 for b in BOOL};o.update({"g_cond":2,"g_hardwood":0,"g_quartz_granite":0,"g_view_water":0})31 if not s: return o32 for b in BOOL: o["g_"+b]=int(bool(s.get(b)))33 o["g_cond"]=COND.get(s.get("condition","inconnu"),2)34 fl=s.get("flooring") or [];o["g_hardwood"]=int("bois_franc" in fl)35 o["g_quartz_granite"]=int(s.get("countertop") in ("quartz","granit","marbre"));o["g_view_water"]=int(s.get("view")=="eau");return o36RENO={"a_renover":0,"date":1,"standard":2,"renove":3,"haut_de_gamme":4,"inconnu":2}37LAND={"aucun":0,"minimal":1,"entretenu":2,"amenage_soigne":3}38SETS=["urbain_dense","banlieue","rural","bord_eau","boise"]39MATS=["brique","pierre","vinyle","bois","aluminium","stucco","crepi","beton"]40def vfeat(uid):41 f=facade.get(uid)42 o={"v_cond":5,"v_curb":5,"v_photoq":5,"v_reno":2,"v_land":1,"v_luxury":0,"v_garage_vis":0,"v_is_ext":0}43 for s in SETS: o["v_set_"+s]=044 for m in MATS: o["v_mat_"+m]=045 if not f: return o46 o["v_cond"]=f.get("condition_score") or 5;o["v_curb"]=f.get("curb_appeal") or 5;o["v_photoq"]=f.get("photo_quality") or 547 o["v_reno"]=RENO.get(f.get("renovation_level","inconnu"),2);o["v_land"]=LAND.get(f.get("landscaping","minimal"),1)48 o["v_luxury"]=int(bool(f.get("curb_luxury")));o["v_garage_vis"]=int(bool(f.get("has_garage_visible")));o["v_is_ext"]=int(bool(f.get("is_exterior")))49 st=f.get("setting")50 if st in SETS: o["v_set_"+st]=151 for m in (f.get("exterior_materials") or []):52 if m in MATS: o["v_mat_"+m]=153 return o54# ---- parsing structurel ----55def pa(s):56 if not s or not isinstance(s,str): return np.nan57 t=s.lower();m=("m2" in t) or ("m²" in t);n=re.sub(r"[^0-9.,\s]","",t).strip().replace(" ","").replace(",","")58 try: v=float(n)59 except: return np.nan60 return (v*10.7639 if m else v) if v>0 else np.nan61def cc(c):62 if not c or not isinstance(c,str): return None63 c=c.split("(")[0].strip();return re.sub(r"\s+"," ",c).lower().strip() or None64def ti(x):65 try: return int(x)66 except: return np.nan67AMEN={"pool":["swimming pool","piscine","pool"],"waterfront":["waterfront","bord de l'eau","navigable","lakefront","riverfront","au bord de"],"garage":["garage"],"fireplace":["fireplace","foyer","wood stove","poêle"],"central_air":["central air","air conditionné","climatisation","a/c","heat-pump","thermopompe","air exchanger"],"finished_bsmt":["totally finished","finished basement","sous-sol fini","basement"],"near_water":["near water","waterfront","navigable"],"cul_de_sac":["cul-de-sac","cul de sac"],"golf":["golf"],"ski":["ski"],"near_park":["park","parc"],"near_school":["school","école","cegep","university","université","elementary","high school"],"near_highway":["highway","autoroute"],"transit":["public transit","public transportation","transport"],"no_rear_neighbor":["no backyard neighbors","sans voisin arrière"]}68def sd(s):69 s=s.lower()70 if "ft" in s:71 f=re.search(r"([\d.]+)\s*ft",s);i=re.search(r"([\d.]+)\s*in",s)72 return (float(f.group(1)) if f else 0)+(float(i.group(1))/12 if i else 0)73 m=re.search(r"([\d.,]+)\s*m",s)74 if m:75 try: return float(m.group(1).replace(",","."))*3.2808476 except: return None77 return None78def ra(d):79 if not d or "x" not in d: return None80 a,_,b=d.partition("x");la,lb=sd(a),sd(b);return la*lb if (la and lb and 1<la<200 and 1<lb<200) else None81RFR={"r_office":["office"],"r_family":["family room"],"r_workshop":["workshop"],"r_winecellar":["wine cellar"],"r_sunroom":["sunroom","solarium"],"r_mezzanine":["mezzanine"],"r_library":["library"],"r_laundry":["laundry"]}82def pr(rooms):83 o={"room_area":np.nan,"n_rooms":0,"has_primary_bed":0,"has_basement":0,"n_levels_r":0}84 for k in RFR: o[k]=085 if not rooms: return o86 ar=0.0;lv=set()87 for r in rooms:88 nm=(r.get("name") or "").lower();l=(r.get("level") or "").lower();o["n_rooms"]+=189 if l: lv.add(l)90 if "basement" in l or "sous-sol" in l: o["has_basement"]=191 if "primary bedroom" in nm: o["has_primary_bed"]=192 for k,kw in RFR.items():93 if any(w in nm for w in kw): o[k]=194 a=ra(r.get("dimensions"))95 if a: ar+=a96 o["room_area"]=ar if ar>0 else np.nan;o["n_levels_r"]=len(lv);return o9798data=json.load(open("data/processed/maisons_avec_prix.json",encoding="utf-8"));rows=[]99for d in data:100 p=d.get("property") or {};ad=d.get("address") or {};ft=d.get("features") or []101 blob=(" ".join(ft)+" "+(d.get("description") or "")+" "+(d.get("title") or "")).lower()102 yb=ti(p.get("yearBuilt"));age=(CUR_YEAR-yb) if (not pd.isna(yb) and 1700<=yb<=CUR_YEAR) else np.nan103 pt=p.get("buildingType") or p.get("type");st=p.get("storeys") or p.get("levels")104 try: st=float(str(st).replace(",","."))105 except: st=np.nan106 uid=f"{d.get('source')}:{d.get('listingId')}"107 rec={"uid":uid,"price":d.get("priceNumber"),"source":d.get("source"),"sqft":pa(p.get("squareFeet")),"lot_sqft":pa(p.get("lotSize")),"beds":ti(p.get("bedrooms")),"baths":ti(p.get("bathrooms")),"parking":ti(p.get("parkingSpaces")),"storeys":st,"age":age,"ptype":(pt or "Unknown").strip(),"city":cc(ad.get("city"))}108 for n,kw in AMEN.items(): rec[n]=int(any(k in blob for k in kw))109 rec["n_images"]=len(d.get("images") or []);rec["virtual_tour"]=int(bool(d.get("virtualTourUrl")))110 rec.update(pr(d.get("rooms")));rec.update(gfeat(uid));rec.update(vfeat(uid));rows.append(rec)111df=pd.DataFrame(rows)112df=df[df["price"].notna()];df=df[(df["price"]>=50_000)&(df["price"]<=15_000_000)]113df.loc[(df["sqft"]<300)|(df["sqft"]>20_000),"sqft"]=np.nan114df.loc[(df["lot_sqft"]<200)|(df["lot_sqft"]>5_000_000),"lot_sqft"]=np.nan115df.loc[df["beds"]>15,"beds"]=np.nan;df.loc[df["baths"]>12,"baths"]=np.nan116df["ppsf"]=df["price"]/df["sqft"];lo,hi=df["ppsf"].quantile([0.005,0.995])117df=df[~(df["ppsf"].notna()&((df["ppsf"]<lo)|(df["ppsf"]>hi)))]118ov=df[df["sqft"].notna()&df["room_area"].notna()&(df["room_area"]>200)];ratio=(ov["sqft"]/ov["room_area"]).median()119re_=df["room_area"]*ratio;ok=df["room_area"].notna()&(re_.between(300,20000))120df["sqft_final"]=df["sqft"];fill=df["sqft_final"].isna()&ok;df.loc[fill,"sqft_final"]=re_[fill]121df["has_sqft"]=df["sqft_final"].notna().astype(int);df["age2"]=df["age"]**2;df["n_images_c"]=df["n_images"].clip(upper=60)122vc=df["city"].value_counts();keep=set(vc[vc>=20].index);df["city_fe"]=df["city"].where(df["city"].isin(keep),"__other__").fillna("__other__")123tc=df["ptype"].value_counts();df["ptype_fe"]=df["ptype"].where(df["ptype"].isin(set(tc[tc>=30].index)),"Other")124df=df.reset_index(drop=True);print("N=",len(df))125y=np.log(df["price"].values).astype(np.float32)126127amen=list(AMEN.keys());rv=["n_rooms","n_levels_r","has_primary_bed","has_basement","r_office","r_family","r_workshop","r_winecellar","r_sunroom","r_mezzanine","r_library","r_laundry"]128gv=["g_cond","g_hardwood","g_quartz_granite","g_view_water"]+["g_"+b for b in BOOL]129vv=["v_cond","v_curb","v_photoq","v_reno","v_land","v_luxury","v_garage_vis","v_is_ext"]+["v_set_"+s for s in SETS]+["v_mat_"+m for m in MATS]130base=["sqft_final","has_sqft","lot_sqft","beds","baths","parking","storeys","age","age2","n_images_c","virtual_tour"]+amen+rv131NUM_v4=base+gv132NUM_v5=base+gv+vv133missnum=["sqft_final","lot_sqft","beds","baths","storeys","age","age2"]134for c in missnum: df["miss_"+c]=df[c].isna().astype(np.float32)135MISS=["miss_"+c for c in missnum]136137cats=["city_fe","ptype_fe","source"];card={}138for c in cats:139 u=sorted(df[c].astype(str).unique());m={v:i for i,v in enumerate(u)};card[c]=len(u);df[c+"_ix"]=df[c].astype(str).map(m).astype(np.int64)140Xcat_ix=df[[c+"_ix" for c in cats]].values.astype(np.int64)141kf=KFold(n_splits=5,shuffle=True,random_state=42)142143def gbm_cv(numcols):144 X=pd.concat([df[numcols].reset_index(drop=True),pd.get_dummies(df[cats],dtype=float).reset_index(drop=True)],axis=1)145 m=HistGradientBoostingRegressor(max_iter=600,learning_rate=0.05,max_leaf_nodes=63,l2_regularization=1.0,random_state=42)146 return cross_val_predict(m,X,y,cv=kf,n_jobs=-1)147148# ---- NN robuste ----149D_MAIN=256;D_HID=512;NB=3;SEEDS=[42,1,7]150class RB(nn.Module):151 def __init__(s,d,dh,p):152 super().__init__();s.bn=nn.BatchNorm1d(d);s.l1=nn.Linear(d,dh);s.act=nn.GELU();s.d1=nn.Dropout(p);s.l2=nn.Linear(dh,d);s.d2=nn.Dropout(p)153 def forward(s,x): z=s.d1(s.act(s.l1(s.bn(x))));return x+s.d2(s.l2(z))154class Net(nn.Module):155 def __init__(s,nc,nn_):156 super().__init__();s.embs=nn.ModuleList();tot=0157 for c in nc: d=max(2,min(64,round(1.6*c**0.56)));s.embs.append(nn.Embedding(c,d));tot+=d158 s.ed=nn.Dropout(0.1);s.bn0=nn.BatchNorm1d(nn_);s.proj=nn.Linear(tot+nn_,D_MAIN)159 s.bl=nn.ModuleList([RB(D_MAIN,D_HID,0.2) for _ in range(NB)])160 s.head=nn.Sequential(nn.BatchNorm1d(D_MAIN),nn.GELU(),nn.Linear(D_MAIN,1))161 def forward(s,xc,xn):162 e=[emb(xc[:,i]) for i,emb in enumerate(s.embs)]163 x=s.proj(torch.cat([s.ed(torch.cat(e,1)),s.bn0(xn)],1))164 for b in s.bl: x=b(x)165 return s.head(x).squeeze(1)166def nn_fold(Xnum,tr,te,seed):167 torch.manual_seed(seed);np.random.seed(seed)168 med=np.nanmedian(Xnum[tr],0);med=np.where(np.isnan(med),0,med);Xn=Xnum.copy()169 for j in range(Xn.shape[1]): col=Xn[:,j];col[np.isnan(col)]=med[j];Xn[:,j]=col170 mu=Xn[tr].mean(0);sg=Xn[tr].std(0)+1e-6;Xn=(Xn-mu)/sg171 ym=y[tr].mean();ys=y[tr].std()+1e-6;yz=(y-ym)/ys172 perm=np.random.permutation(len(tr));cut=int(len(tr)*0.9);tri=tr[perm[:cut]];vai=tr[perm[cut:]]173 T=lambda a,dt: torch.tensor(a,device=dev,dtype=dt)174 model=Net([card[c] for c in cats],Xn.shape[1]).to(dev)175 opt=torch.optim.AdamW(model.parameters(),lr=2e-3,weight_decay=1e-4)176 sch=torch.optim.lr_scheduler.CosineAnnealingLR(opt,T_max=200);lf=nn.SmoothL1Loss(beta=1.0)177 Xc=T(Xcat_ix,torch.long);Xnt=T(Xn,torch.float32);yt=T(yz,torch.float32)178 best=1e9;bs=None;pat=0179 for ep in range(200):180 model.train();idx=np.random.permutation(tri)181 for i in range(0,len(idx),512):182 b=idx[i:i+512];opt.zero_grad();l=lf(model(Xc[b],Xnt[b]),yt[b]);l.backward()183 torch.nn.utils.clip_grad_norm_(model.parameters(),5.0);opt.step()184 sch.step();model.eval()185 with torch.no_grad(): vl=((model(Xc[vai],Xnt[vai])-yt[vai])**2).mean().item()186 if vl<best-1e-4: best=vl;bs={k:v.clone() for k,v in model.state_dict().items()};pat=0187 else:188 pat+=1189 if pat>=20: break190 model.load_state_dict(bs);model.eval()191 with torch.no_grad(): return model(Xc[te],Xnt[te]).cpu().numpy()*ys+ym192def nn_cv(numcols):193 Xnum=df[numcols+MISS].astype(np.float32).values;pred=np.zeros_like(y)194 for tr,te in kf.split(Xnum):195 ps=np.zeros(len(te))196 for s in SEEDS: ps+=nn_fold(Xnum,tr,te,s)197 pred[te]=ps/len(SEEDS)198 return pred199def stat(p):200 e=np.abs(np.exp(p-y)-1);return r2_score(y,p),np.median(e)*100,np.mean(e<=.15)*100,np.mean(e<=.25)*100201202print("\n>>> GBM ablation")203g4=gbm_cv(NUM_v4);g5=gbm_cv(NUM_v5)204print("\n>>> NN v5 (peut prendre ~2 min)")205n5=nn_cv(NUM_v5)206b5=0.5*g5+0.5*n5207208print("\n================ RÉSULTATS v5 (5-fold CV OOS) ================")209print(f"{'Modèle':<34}{'R²OOS':>8}{'MAEméd':>9}{'±15%':>7}{'±25%':>7}")210for nm,p in [("GBM v4 (texte)",g4),("GBM v5 (texte+vision)",g5),("NN v5 (texte+vision)",n5),("BLEND v5 (NN+GBM)",b5)]:211 r2,m,a,b=stat(p);print(f"{nm:<34}{r2:>8.4f}{m:>8.1f}%{a:>6.0f}%{b:>6.0f}%")212print("\nRappel v4 blend: 0.760 | GBM 0.743 | NN 0.741")213np.save("data/processed/v5_blend_oos.npy",b5)214