#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Comparaison OLS vs modèles ML (RF, Gradient Boosting) en validation croisée 5-fold.
Même jeu de variables que la v4 (structurel + aménités + pièces + listing + GPT + FE)."""
import json, re, warnings
import numpy as np, pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor, HistGradientBoostingRegressor
from sklearn.model_selection import KFold, cross_val_predict
from sklearn.metrics import r2_score
warnings.filterwarnings("ignore")
CUR_YEAR = 2026

# ---- specs GPT ----
specs = {}
for l in open("data/processed/specs.jsonl", encoding="utf-8"):
    r = json.loads(l)
    if r.get("specs"): specs[r["uid"]] = r["specs"]
COND = {"a_renover":0,"original":1,"bon":2,"renove":3,"refait_neuf":4,"neuf":5,"inconnu":2}
BOOL = ["turnkey","recently_renovated","garage","pool","waterfront","finished_basement",
        "fireplace","central_ac","new_construction","income_property","renovated_kitchen",
        "renovated_bathroom","large_lot","quiet_area"]
def gfeat(uid):
    s = specs.get(uid); o = {"g_"+b:0 for b in BOOL}
    o.update({"g_cond":2,"g_hardwood":0,"g_quartz_granite":0,"g_view_water":0})
    if not s: return o
    for b in BOOL: o["g_"+b] = int(bool(s.get(b)))
    o["g_cond"] = COND.get(s.get("condition","inconnu"),2)
    fl = s.get("flooring") or []; o["g_hardwood"] = int("bois_franc" in fl)
    o["g_quartz_granite"] = int(s.get("countertop") in ("quartz","granit","marbre"))
    o["g_view_water"] = int(s.get("view")=="eau"); return o

# ---- parsing (repris) ----
def parse_area(s):
    if not s or not isinstance(s,str): return np.nan
    t=s.lower(); m=("m2" in t) or ("m²" in t)
    n=re.sub(r"[^0-9.,\s]","",t).strip().replace(" ","").replace(",","")
    try: v=float(n)
    except: return np.nan
    return (v*10.7639 if m else v) if v>0 else np.nan
def cc(c):
    if not c or not isinstance(c,str): return None
    c=c.split("(")[0].strip(); return re.sub(r"\s+"," ",c).lower().strip() or None
def ti(x):
    try: return int(x)
    except: return np.nan
AMEN={"pool":["swimming pool","piscine","pool"],"waterfront":["waterfront","bord de l'eau","navigable","lakefront","riverfront","au bord de"],
"garage":["garage"],"fireplace":["fireplace","foyer","wood stove","poêle"],
"central_air":["central air","air conditionné","climatisation","a/c","heat-pump","thermopompe","air exchanger"],
"finished_bsmt":["totally finished","finished basement","sous-sol fini","basement"],
"near_water":["near water","waterfront","navigable"],"cul_de_sac":["cul-de-sac","cul de sac"],
"golf":["golf"],"ski":["ski"],"near_park":["park","parc"],
"near_school":["school","école","cegep","university","université","elementary","high school"],
"near_highway":["highway","autoroute"],"transit":["public transit","public transportation","transport"],
"no_rear_neighbor":["no backyard neighbors","sans voisin arrière"]}
def sd(s):
    s=s.lower()
    if "ft" in s:
        f=re.search(r"([\d.]+)\s*ft",s);i=re.search(r"([\d.]+)\s*in",s)
        return (float(f.group(1)) if f else 0)+(float(i.group(1))/12 if i else 0)
    m=re.search(r"([\d.,]+)\s*m",s)
    if m:
        try: return float(m.group(1).replace(",","."))*3.28084
        except: return None
    return None
def ra(d):
    if not d or "x" not in d: return None
    a,_,b=d.partition("x");la,lb=sd(a),sd(b)
    return la*lb if (la and lb and 1<la<200 and 1<lb<200) else None
RF={"r_office":["office"],"r_family":["family room"],"r_workshop":["workshop"],"r_winecellar":["wine cellar"],
"r_sunroom":["sunroom","solarium"],"r_mezzanine":["mezzanine"],"r_library":["library"],"r_laundry":["laundry"]}
def pr(rooms):
    o={"room_area":np.nan,"n_rooms":0,"has_primary_bed":0,"has_basement":0,"n_levels_r":0}
    for k in RF: o[k]=0
    if not rooms: return o
    ar=0.0;lv=set()
    for r in rooms:
        nm=(r.get("name") or "").lower();l=(r.get("level") or "").lower();o["n_rooms"]+=1
        if l: lv.add(l)
        if "basement" in l or "sous-sol" in l: o["has_basement"]=1
        if "primary bedroom" in nm: o["has_primary_bed"]=1
        for k,kw in RF.items():
            if any(w in nm for w in kw): o[k]=1
        a=ra(r.get("dimensions"))
        if a: ar+=a
    o["room_area"]=ar if ar>0 else np.nan;o["n_levels_r"]=len(lv);return o

data=json.load(open("data/processed/maisons_avec_prix.json",encoding="utf-8"))
rows=[]
for d in data:
    p=d.get("property") or {};ad=d.get("address") or {};ft=d.get("features") or []
    blob=(" ".join(ft)+" "+(d.get("description") or "")+" "+(d.get("title") or "")).lower()
    yb=ti(p.get("yearBuilt"));age=(CUR_YEAR-yb) if (not pd.isna(yb) and 1700<=yb<=CUR_YEAR) else np.nan
    pt=p.get("buildingType") or p.get("type");st=p.get("storeys") or p.get("levels")
    try: st=float(str(st).replace(",","."))
    except: st=np.nan
    uid=f"{d.get('source')}:{d.get('listingId')}"
    rec={"uid":uid,"price":d.get("priceNumber"),"source":d.get("source"),"sqft":parse_area(p.get("squareFeet")),
    "lot_sqft":parse_area(p.get("lotSize")),"beds":ti(p.get("bedrooms")),"baths":ti(p.get("bathrooms")),
    "parking":ti(p.get("parkingSpaces")),"storeys":st,"age":age,"ptype":(pt or "Unknown").strip(),"city":cc(ad.get("city"))}
    for n,kw in AMEN.items(): rec[n]=int(any(k in blob for k in kw))
    rec["n_images"]=len(d.get("images") or []);rec["virtual_tour"]=int(bool(d.get("virtualTourUrl")))
    rec.update(pr(d.get("rooms")));rec.update(gfeat(uid));rows.append(rec)
df=pd.DataFrame(rows)

# nettoyage (identique)
df=df[df["price"].notna()];df=df[(df["price"]>=50_000)&(df["price"]<=15_000_000)]
df.loc[(df["sqft"]<300)|(df["sqft"]>20_000),"sqft"]=np.nan
df.loc[(df["lot_sqft"]<200)|(df["lot_sqft"]>5_000_000),"lot_sqft"]=np.nan
df.loc[df["beds"]>15,"beds"]=np.nan;df.loc[df["baths"]>12,"baths"]=np.nan
df["ppsf"]=df["price"]/df["sqft"];lo,hi=df["ppsf"].quantile([0.005,0.995])
df=df[~(df["ppsf"].notna()&((df["ppsf"]<lo)|(df["ppsf"]>hi)))]
ov=df[df["sqft"].notna()&df["room_area"].notna()&(df["room_area"]>200)]
ratio=(ov["sqft"]/ov["room_area"]).median();re_=df["room_area"]*ratio
ok=df["room_area"].notna()&(re_.between(300,20000))
df["sqft_final"]=df["sqft"];fill=df["sqft_final"].isna()&ok;df.loc[fill,"sqft_final"]=re_[fill]
df["has_sqft"]=df["sqft_final"].notna().astype(int)
# pour ML : garder NaN (les arbres gèrent) ; pour OLS on impute une copie
df["ln_price"]=np.log(df["price"]);df["age2"]=df["age"]**2;df["n_images_c"]=df["n_images"].clip(upper=60)
vc=df["city"].value_counts();keep=set(vc[vc>=20].index)
df["city_fe"]=df["city"].where(df["city"].isin(keep),"__other__").fillna("__other__")
tc=df["ptype"].value_counts();df["ptype_fe"]=df["ptype"].where(df["ptype"].isin(set(tc[tc>=30].index)),"Other")
print(f"N={len(df)}")

amen=list(AMEN.keys());rv=["n_rooms","n_levels_r","has_primary_bed","has_basement","r_office","r_family","r_workshop","r_winecellar","r_sunroom","r_mezzanine","r_library","r_laundry"]
gv=["g_cond","g_hardwood","g_quartz_granite","g_view_water"]+["g_"+b for b in BOOL]
num=["sqft_final","has_sqft","lot_sqft","beds","baths","parking","storeys","age","age2","n_images_c","virtual_tour"]+amen+rv+gv
cat=["city_fe","ptype_fe","source"]

y=df["ln_price"].values
price=df["price"].values
Xcat=pd.get_dummies(df[cat],prefix=cat,dtype=float)
# --- matrice ML (NaN autorisés pour arbres) ---
Xml=pd.concat([df[num].reset_index(drop=True),Xcat.reset_index(drop=True)],axis=1)
# --- matrice OLS (imputée médiane par type) ---
dfi=df.copy()
for c in ["sqft_final","lot_sqft","beds","baths","storeys","age","age2"]:
    dfi[c]=dfi.groupby("ptype")[c].transform(lambda s:s.fillna(s.median()))
    dfi[c]=dfi[c].fillna(dfi[c].median())
dfi["parking"]=dfi["parking"].fillna(0)
Xols=pd.concat([dfi[num].reset_index(drop=True),Xcat.reset_index(drop=True)],axis=1)

kf=KFold(n_splits=5,shuffle=True,random_state=42)
def evalcv(model,X):
    pred=cross_val_predict(model,X,y,cv=kf,n_jobs=-1)
    r2=r2_score(y,pred)
    mape=np.median(np.abs(np.exp(pred-y)-1))*100
    w15=np.mean(np.abs(np.exp(pred-y)-1)<=.15)*100
    w25=np.mean(np.abs(np.exp(pred-y)-1)<=.25)*100
    return r2,mape,w15,w25

print("\nValidation croisée 5-fold (out-of-sample) — cible = ln(prix)\n")
print(f"{'Modèle':<34}{'R² OOS':>9}{'MAE méd.':>10}{'±15%':>7}{'±25%':>7}")
res={}
res["OLS (linéaire, imputé)"]=evalcv(LinearRegression(),Xols)
res["Random Forest (400)"]=evalcv(RandomForestRegressor(n_estimators=400,max_features=0.5,min_samples_leaf=3,n_jobs=-1,random_state=42),Xml)
res["HistGradientBoosting"]=evalcv(HistGradientBoostingRegressor(max_iter=600,learning_rate=0.05,max_leaf_nodes=63,l2_regularization=1.0,random_state=42),Xml)
res["HistGB (tuned depth)"]=evalcv(HistGradientBoostingRegressor(max_iter=900,learning_rate=0.03,max_leaf_nodes=127,min_samples_leaf=20,l2_regularization=2.0,random_state=42),Xml)
for k,(r2,mape,w15,w25) in res.items():
    print(f"{k:<34}{r2:>9.4f}{mape:>9.1f}%{w15:>6.0f}%{w25:>6.0f}%")

# importance GBM (permutation rapide via feature_importances_ du HistGB n'existe pas → refit + importances via gain n/a)
gb=HistGradientBoostingRegressor(max_iter=600,learning_rate=0.05,max_leaf_nodes=63,l2_regularization=1.0,random_state=42).fit(Xml,y)
from sklearn.inspection import permutation_importance
import numpy as _np
idx=_np.random.RandomState(0).choice(len(Xml),4000,replace=False)
pi=permutation_importance(gb,Xml.iloc[idx],y[idx],n_repeats=3,random_state=0,n_jobs=-1)
imp=pd.Series(pi.importances_mean,index=Xml.columns).sort_values(ascending=False)
print("\nTop 20 variables (importance par permutation, GBM) :")
for name,v in imp.head(20).items(): print(f"  {name:<26}{v:.4f}")
print("\nImportance cumulée des variables GPT (g_*) :","%.4f"%imp[[c for c in Xml.columns if c.startswith('g_')]].sum())