#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Robustesse (par segment) + figures primes de localisation & validation IA."""
import numpy as np, pandas as pd, warnings, statsmodels.formula.api as smf
import matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt
warnings.filterwarnings("ignore")
plt.rcParams.update({"font.family":"serif","font.size":10,"axes.grid":True,
    "grid.alpha":0.25,"axes.axisbelow":True,"figure.dpi":150,"savefig.bbox":"tight"})
C1,C2,C3="#3B6EA5","#C4462E","#4A8C5B"
exec(open("v5_ml.py").read().split("kf=KFold")[0])
amen_=list(AMEN.keys())
bl=np.load("data/processed/v5_blend_oos.npy"); price=df["price"].values; y=np.log(price)
from sklearn.metrics import r2_score
def stat(mask):
    yy=y[mask];pp=bl[mask];e=np.abs(np.exp(pp-yy)-1)
    return len(yy),r2_score(yy,pp),np.median(e)*100,np.mean(e<=.15)*100

# ---- robustesse par segment ----
seg=[]
seg.append(("\\textit{Full sample}",np.ones(len(y),bool)))
q1,q2=np.quantile(price,[1/3,2/3])
seg.append(("\\quad Price tercile 1 (low)",price<=q1))
seg.append(("\\quad Price tercile 2 (mid)",(price>q1)&(price<=q2)))
seg.append(("\\quad Price tercile 3 (high)",price>q2))
seg.append(("\\quad Realtor.ca",df["source"].values=="realtor"))
seg.append(("\\quad DuProprio",df["source"].values=="duproprio"))
seg.append(("\\quad Measured area",df["has_sqft"].values==1))
seg.append(("\\quad Imputed area",df["has_sqft"].values==0))
rows=[]
for nm,m in seg:
    n,r2,md,w=stat(m)
    # R^2 non informatif sur une plage de prix restreinte (terciles) -> n/a
    r2c = "n/a$^{\\dagger}$" if "tercile" in nm else f"{r2:.3f}"
    rows.append(f"      {nm} & {n:,} & {r2c} & {md:.1f}\\% & {w:.0f}\\% \\\\".replace(",","{,}"))
tab=r"""% Auto-generated
\begin{table}[htbp]
  \centering
  \begin{threeparttable}
    \caption{Robustness: Blend Performance Across Subsamples}
    \label{tab:robustness}
    \begin{tabular}{@{}lcccc@{}}
      \toprule
      Subsample & $N$ & $R^2_{\text{OOS}}$ & Median AE & Within $\pm$15\% \\
      \midrule
"""+"\n".join(rows)+r"""
      \bottomrule
    \end{tabular}
    \begin{tablenotes}[flushleft]
      \footnotesize
      \item \textit{Notes:} Out-of-sample performance of the text\,+\,vision blend
      (Table~\ref{tab:models}, Panel~B) computed on subsamples of the same 5-fold
      cross-validated predictions. Price terciles are defined on the actual list price.
      ``Measured area'' are listings with a reported or room-reconstructed living area
      (priced by Model~A); ``Imputed area'' listings are priced by the area-free model.
      $^{\dagger}$Within a single price tercile the range of $\ln P$ is heavily
      restricted, so $R^2$---which normalizes by within-group variance---is
      uninformative; the median absolute error is the comparable accuracy metric across
      terciles.
    \end{tablenotes}
  \end{threeparttable}
\end{table}
"""
open("paper/tables/robustness.tex","w").write(tab)
print("-> tables/robustness.tex"); [print(f"  {nm[:28]:<30} n={stat(m)[0]:5d} R2={stat(m)[1]:.3f} md={stat(m)[2]:.1f}%") for nm,m in seg]

# ---- OLS v5 pour primes de localisation ----
df["ln_price"]=y
df["ln_sqft"]=np.log(df["sqft_final"].fillna(df.groupby("ptype")["sqft_final"].transform("median")).fillna(df["sqft_final"].median()))
df["ln_lot"]=np.log(df["lot_sqft"].fillna(df["lot_sqft"].median()));df["has_lot"]=df["lot_sqft"].notna().astype(int)
df["has_age"]=df["age"].notna().astype(int)
for c in ["beds","baths","storeys","age","age2"]:
    df[c]=df[c].fillna(df.groupby("ptype")[c].transform("median")).fillna(df[c].median())
df["parking"]=df["parking"].fillna(0)
form=("ln_price ~ ln_sqft+ln_lot+has_lot+beds+baths+parking+storeys+age+age2+has_age+C(source)+C(ptype_fe)+"+"+".join(amen_)+"+n_images_c+virtual_tour+"+"+".join(rv)+"+"+"+".join(gv)+"+"+"+".join(vv)+"+C(city_fe)")
mA=smf.ols(form,data=df[df["has_sqft"]==1]).fit(cov_type="HC3")
pct=lambda b:(np.exp(b)-1)*100
cf={k.split("T.")[1].rstrip("]"):pct(v) for k,v in mA.params.items() if k.startswith("C(city_fe)")}
s=pd.Series(cf).sort_values()
top=s.tail(12);bot=s.head(8)
fig,ax=plt.subplots(figsize=(6.6,4.6))
sel=pd.concat([bot,top])
cols=[C2 if v<0 else C1 for v in sel.values]
ax.barh(range(len(sel)),sel.values,color=cols,edgecolor="white")
ax.set_yticks(range(len(sel)));ax.set_yticklabels([c.title() for c in sel.index],fontsize=8)
ax.axvline(0,color="black",lw=0.8)
ax.set_xlabel("Location premium vs reference city (\\%)")
ax.set_title("Municipality fixed effects (largest and smallest)",fontsize=11)
fig.savefig("paper/figures/fig_premia.pdf");plt.close(fig);print("-> fig_premia.pdf")

# ---- validation IA : prix moyen par score visuel ----
ext=df[df["v_is_ext"]==1]
fig,axs=plt.subplots(1,2,figsize=(7.6,3.5))
for ax,col,lab,c in [(axs[0],"v_curb","Vision curb appeal (1--10)",C3),(axs[1],"v_cond","Vision condition score (1--10)",C1)]:
    g=ext.groupby(col)["price"].median()/1e3
    g=g[g.index>=3]
    ax.plot(g.index,g.values,"-o",color=c,lw=2)
    ax.set_xlabel(lab);ax.set_ylabel("Median price (\\$000)")
axs[0].set_title("Price rises with visual curb appeal",fontsize=10)
axs[1].set_title("Price rises with visual condition",fontsize=10)
fig.savefig("paper/figures/fig_validation.pdf");plt.close(fig);print("-> fig_validation.pdf")

# ---- table résumé features IA ----
def sh(col): return df[col].mean()*100
aitab=r"""% Auto-generated
\begin{table}[htbp]
  \centering
  \begin{threeparttable}
    \caption{Machine-Generated Feature Summary}
    \label{tab:ai_summary}
    \begin{tabular}{@{}lc@{\hskip 1.2cm}lc@{}}
      \toprule
      Text (LLM) & Share & Vision (façade) & Mean / share \\
      \midrule
"""+f"""      Recently renovated & {sh('g_recently_renovated'):.0f}\\% & Condition score (1--10) & {df['v_cond'].mean():.1f} \\\\
      Renovated kitchen & {sh('g_renovated_kitchen'):.0f}\\% & Curb appeal (1--10) & {df['v_curb'].mean():.1f} \\\\
      Hardwood floor & {sh('g_hardwood'):.0f}\\% & Luxury cue & {sh('v_luxury'):.0f}\\% \\\\
      Quartz/granite counter & {sh('g_quartz_granite'):.0f}\\% & Garage visible & {sh('v_garage_vis'):.0f}\\% \\\\
      New construction & {sh('g_new_construction'):.0f}\\% & Brick façade & {sh('v_mat_brique'):.0f}\\% \\\\
      Income property & {sh('g_income_property'):.0f}\\% & Waterfront setting & {sh('v_set_bord_eau'):.0f}\\% \\\\
      Water view & {sh('g_view_water'):.0f}\\% & Exterior shot detected & {sh('v_is_ext'):.0f}\\% \\\\
""" + r"""      \bottomrule
    \end{tabular}
    \begin{tablenotes}[flushleft]
      \footnotesize
      \item \textit{Notes:} Prevalence of selected machine-generated attributes across
      the """ + f"{len(df):,}".replace(",","{,}") + r""" cleaned listings. Text attributes are extracted from the
      description by \texttt{gpt-4o-mini}; vision attributes from the façade photograph by
      \texttt{claude-haiku-4-5}. Scores are 1--10 means; the remainder are shares.
    \end{tablenotes}
  \end{threeparttable}
\end{table}
"""
open("paper/tables/ai_summary.tex","w").write(aitab)
print("-> tables/ai_summary.tex")