#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Modèle hédonique v4 : ajoute les specs extraites par GPT-4o-mini (specs.jsonl)
au modèle v3. Réutilise la même logique de nettoyage/features + effets fixes.
"""
import json, re, warnings
import numpy as np
import pandas as pd
import statsmodels.formula.api as smf

warnings.filterwarnings("ignore")
CUR_YEAR = 2026

# ---------- charge specs GPT ----------
specs = {}
for line in open("data/processed/specs.jsonl", encoding="utf-8"):
    try:
        r = json.loads(line)
    except Exception:
        continue
    if "specs" in r and r.get("specs"):
        specs[r["uid"]] = r["specs"]
print(f"Specs GPT chargées : {len(specs)}")

# specs -> features numériques (dummies + catégorielles réduites)
COND_SCORE = {"a_renover":0,"original":1,"bon":2,"renove":3,"refait_neuf":4,
              "neuf":5,"inconnu":2}
BOOL_SPECS = ["turnkey","recently_renovated","garage","pool","waterfront",
              "finished_basement","fireplace","central_ac","new_construction",
              "income_property","renovated_kitchen","renovated_bathroom",
              "large_lot","quiet_area"]

def spec_feats(uid):
    s = specs.get(uid)
    out = {"g_" + b: 0 for b in BOOL_SPECS}
    out["g_cond"] = 2
    out["g_hardwood"] = 0; out["g_quartz_granite"] = 0; out["g_view_water"] = 0
    out["has_gpt"] = 0
    if not s:
        return out
    out["has_gpt"] = 1
    for b in BOOL_SPECS:
        out["g_" + b] = int(bool(s.get(b)))
    out["g_cond"] = COND_SCORE.get(s.get("condition", "inconnu"), 2)
    fl = s.get("flooring") or []
    out["g_hardwood"] = int("bois_franc" in fl)
    out["g_quartz_granite"] = int(s.get("countertop") in ("quartz", "granit", "marbre"))
    out["g_view_water"] = int(s.get("view") == "eau")
    return out

# ===================== pipeline v3 (repris) =====================
SRC = "data/processed/maisons_avec_prix.json"
data = json.load(open(SRC, encoding="utf-8"))

def parse_area(s):
    if not s or not isinstance(s, str): return np.nan
    txt = s.lower(); is_m2 = ("m2" in txt) or ("m²" in txt)
    num = re.sub(r"[^0-9.,\s]", "", txt).strip().replace(" ", "").replace(",", "")
    try: v = float(num)
    except ValueError: return np.nan
    if v <= 0: return np.nan
    return v * 10.7639 if is_m2 else v

def clean_city(city, source):
    if not city or not isinstance(city, str): return None
    c = city.split("(")[0].strip(); c = re.sub(r"\s+", " ", c).lower().strip()
    return c or None

def to_int(x):
    try: return int(x)
    except (TypeError, ValueError): return np.nan

AMENITIES = {
    "pool":["swimming pool","piscine","inground pool","pool"],
    "waterfront":["waterfront","bord de l'eau","navigable","water access","lakefront","riverfront","au bord de"],
    "garage":["garage"],"fireplace":["fireplace","foyer","wood stove","poêle"],
    "central_air":["central air","air conditionné","climatisation","a/c","heat-pump","thermopompe","air exchanger"],
    "finished_bsmt":["totally finished","finished basement","sous-sol fini","basement"],
    "near_water":["near water","waterfront","navigable"],
    "cul_de_sac":["cul-de-sac","cul de sac"],"golf":["golf"],"ski":["ski"],
    "near_park":["park","parc"],
    "near_school":["school","école","cegep","university","université","elementary","high school"],
    "near_highway":["highway","autoroute"],
    "transit":["public transit","public transportation","transport"],
    "no_rear_neighbor":["no backyard neighbors","no rear neighbor","sans voisin arrière"],
}

def _side_ft(side):
    s = side.lower()
    if "ft" in s:
        ft = re.search(r"([\d.]+)\s*ft", s); inch = re.search(r"([\d.]+)\s*in", s)
        return (float(ft.group(1)) if ft else 0)+(float(inch.group(1))/12 if inch else 0)
    m = re.search(r"([\d.,]+)\s*m", s)
    if m:
        try: return float(m.group(1).replace(",", ".")) * 3.28084
        except ValueError: return None
    return None

def _room_area(dim):
    if not dim or "x" not in dim: return None
    a, _, b = dim.partition("x"); la, lb = _side_ft(a), _side_ft(b)
    if la and lb and 1 < la < 200 and 1 < lb < 200: return la * lb
    return None

ROOM_FLAGS = {"r_office":["office"],"r_family":["family room"],"r_workshop":["workshop"],
              "r_winecellar":["wine cellar"],"r_sunroom":["sunroom","solarium"],
              "r_mezzanine":["mezzanine"],"r_library":["library"],"r_laundry":["laundry"]}

def parse_rooms(rooms):
    out = {"room_area":np.nan,"n_rooms":0,"has_primary_bed":0,"has_basement":0,"n_levels_r":0}
    for k in ROOM_FLAGS: out[k] = 0
    if not rooms: return out
    area, levels = 0.0, set()
    for r in rooms:
        nm = (r.get("name") or "").lower(); lv = (r.get("level") or "").lower()
        out["n_rooms"] += 1
        if lv: levels.add(lv)
        if "basement" in lv or "sous-sol" in lv: out["has_basement"] = 1
        if "primary bedroom" in nm: out["has_primary_bed"] = 1
        for k, kws in ROOM_FLAGS.items():
            if any(w in nm for w in kws): out[k] = 1
        a = _room_area(r.get("dimensions"))
        if a: area += a
    out["room_area"] = area if area > 0 else np.nan
    out["n_levels_r"] = len(levels)
    return out

rows = []
for d in data:
    prop = d.get("property") or {}; addr = d.get("address") or {}
    feats = d.get("features") or []
    blob = (" ".join(feats)+" "+(d.get("description") or "")+" "+(d.get("title") or "")).lower()
    yb = to_int(prop.get("yearBuilt"))
    age = (CUR_YEAR-yb) if (not pd.isna(yb) and 1700 <= yb <= CUR_YEAR) else np.nan
    ptype = prop.get("buildingType") or prop.get("type")
    storeys = prop.get("storeys") or prop.get("levels")
    try: storeys = float(str(storeys).replace(",", "."))
    except (TypeError, ValueError): storeys = np.nan
    uid = f"{d.get('source')}:{d.get('listingId')}"
    rec = {"uid":uid,"price":d.get("priceNumber"),"source":d.get("source"),
           "sqft":parse_area(prop.get("squareFeet")),"lot_sqft":parse_area(prop.get("lotSize")),
           "beds":to_int(prop.get("bedrooms")),"baths":to_int(prop.get("bathrooms")),
           "parking":to_int(prop.get("parkingSpaces")),"storeys":storeys,"age":age,
           "ptype":(ptype or "Unknown").strip(),"city":clean_city(addr.get("city"),d.get("source"))}
    for name, kws in AMENITIES.items():
        rec[name] = int(any(k in blob for k in kws))
    rec["n_images"] = len(d.get("images") or [])
    rec["virtual_tour"] = int(bool(d.get("virtualTourUrl")))
    rec.update(parse_rooms(d.get("rooms")))
    rec.update(spec_feats(uid))
    rows.append(rec)

df = pd.DataFrame(rows)

# nettoyage identique v3
n0 = len(df)
df = df[df["price"].notna()]
df = df[(df["price"] >= 50_000) & (df["price"] <= 15_000_000)]
df.loc[(df["sqft"] < 300) | (df["sqft"] > 20_000), "sqft"] = np.nan
df.loc[(df["lot_sqft"] < 200) | (df["lot_sqft"] > 5_000_000), "lot_sqft"] = np.nan
df.loc[df["beds"] > 15, "beds"] = np.nan
df.loc[df["baths"] > 12, "baths"] = np.nan
df["ppsf"] = df["price"] / df["sqft"]
lo, hi = df["ppsf"].quantile([0.005, 0.995])
df = df[~(df["ppsf"].notna() & ((df["ppsf"] < lo) | (df["ppsf"] > hi)))]

# surface finale
ov = df[df["sqft"].notna() & df["room_area"].notna() & (df["room_area"] > 200)]
ratio = (ov["sqft"] / ov["room_area"]).median()
room_est = df["room_area"] * ratio
room_ok = df["room_area"].notna() & (room_est.between(300, 20_000))
df["sqft_final"] = df["sqft"]
fill = df["sqft_final"].isna() & room_ok
df.loc[fill, "sqft_final"] = room_est[fill]
df["has_sqft"] = df["sqft_final"].notna().astype(int)
df["sqft_imp"] = df.groupby("ptype")["sqft_final"].transform(lambda s: s.fillna(s.median()))
df["sqft_imp"] = df["sqft_imp"].fillna(df["sqft_final"].median())

df["has_lot"] = df["lot_sqft"].notna().astype(int)
df["lot_sqft"] = df["lot_sqft"].fillna(df["lot_sqft"].median())
df["has_age"] = df["age"].notna().astype(int)
df["age"] = df["age"].fillna(df["age"].median()).clip(lower=0)
for c in ["beds","baths","storeys"]:
    df[c] = df[c].fillna(df[c].median())
df["parking"] = df["parking"].fillna(0)
df["ln_price"] = np.log(df["price"]); df["ln_sqft"] = np.log(df["sqft_imp"])
df["ln_lot"] = np.log(df["lot_sqft"]); df["age2"] = df["age"]**2
df["n_images_c"] = df["n_images"].clip(upper=60)

vc = df["city"].value_counts(); keep = set(vc[vc >= 20].index)
df["city_fe"] = df["city"].where(df["city"].isin(keep), "__other__").fillna("__other__")
tc = df["ptype"].value_counts(); keept = set(tc[tc >= 30].index)
df["ptype_fe"] = df["ptype"].where(df["ptype"].isin(keept), "Other")
print(f"N nettoyé={len(df)}  villes FE={df['city_fe'].nunique()}  "
      f"avec specs GPT={int(df['has_gpt'].sum())}")

# ---------- modèles ----------
amen = list(AMENITIES.keys())
room_vars = ["n_rooms","n_levels_r","has_primary_bed","has_basement","r_office",
             "r_family","r_workshop","r_winecellar","r_sunroom","r_mezzanine",
             "r_library","r_laundry"]
gpt_vars = ["g_cond","g_hardwood","g_quartz_granite","g_view_water","has_gpt"] + \
           ["g_" + b for b in BOOL_SPECS]
extra = ["n_images_c","virtual_tour"] + room_vars + gpt_vars
base = ("ln_lot + has_lot + beds + baths + parking + storeys + age + age2 + has_age "
        "+ C(source) + C(ptype_fe) + " + " + ".join(amen) + " + " + " + ".join(extra)
        + " + C(city_fe)")
fA = "ln_price ~ ln_sqft + " + base
fB = "ln_price ~ " + base

def pct(b): return (np.exp(b)-1)*100
dfA = df[df["has_sqft"] == 1].copy()
mA = smf.ols(fA, data=dfA).fit(cov_type="HC3")
mB = smf.ols(fB, data=df).fit(cov_type="HC3")
sdA, sdB = np.std(mA.resid), np.std(mB.resid)

def report(m, sd, name):
    print("\n" + "=" * 70)
    print(f"{name}  N={int(m.nobs)}  R²={m.rsquared:.4f}  R²adj={m.rsquared_adj:.4f}  "
          f"±{(np.exp(sd)-1)*100:.1f}%")
    print("=" * 70)
    pr, pv = m.params, m.pvalues
    print("-- specs GPT (impact %) --")
    for p in pr.index:
        if p.startswith("g_") or p == "has_gpt":
            st = "***" if pv[p]<.01 else "**" if pv[p]<.05 else "*" if pv[p]<.1 else ""
            print(f"  {p:<22}{pr[p]:>9.4f}{pct(pr[p]):>+8.1f}%  p={pv[p]:.2g} {st}")

report(mA, sdA, "MODÈLE A+GPT (avec superficie)")
report(mB, sdB, "MODÈLE B+GPT (couverture large)")

predA = np.exp(mA.predict(df)) * np.exp(sdA**2/2)
predB = np.exp(mB.predict(df)) * np.exp(sdB**2/2)
df["pred_price"] = np.where(df["has_sqft"] == 1, predA, predB)
df["model_used"] = np.where(df["has_sqft"] == 1, "A", "B")
df["resid_pct"] = (df["price"]/df["pred_price"] - 1) * 100
print(f"\nErreur abs médiane: {df.resid_pct.abs().median():.1f}%  "
      f"| ±15%: {(df.resid_pct.abs()<=15).mean()*100:.0f}%  "
      f"±25%: {(df.resid_pct.abs()<=25).mean()*100:.0f}%")

out = (["uid","source","city","ptype","price","pred_price","resid_pct","model_used",
        "sqft_final","has_sqft","lot_sqft","beds","baths","parking","age","n_images",
        "virtual_tour"] + amen + room_vars + gpt_vars)
df[out].to_csv("data/processed/maisons_hedonic_v4.csv", index=False)
print(f"-> maisons_hedonic_v4.csv ({len(df)} lignes)")
with open("results/hedonic_v4_summary.txt", "w", encoding="utf-8") as fh:
    fh.write("=== A+GPT ===\n"+str(mA.summary())+"\n\n=== B+GPT ===\n"+str(mB.summary()))
print("-> hedonic_v4_summary.txt")