# -*- coding: utf-8 -*-
"""Génère tables/*.tex, tables/numbers.tex (macros) et copie les figures depuis out/<version>/ pour le working paper."""
import json, os, shutil, sys
from pathlib import Path
import numpy as np, pandas as pd
ROOT = Path(__file__).resolve().parents[1]; V = os.environ.get("CV_VERSION", "2026-10-10")
OUT = ROOT / "out" / V; TAB = OUT / "tables"; P = Path(__file__).resolve().parent
(P / "tables").mkdir(exist_ok=True); (P / "figures").mkdir(exist_ok=True)
R = json.load(open(OUT / "results.json"))
comp = R["competition"]; compv = R["competition_val"]
def key(prefix): return next(k for k in comp if k.startswith(prefix + " "))
M = {k: key(k) for k in ["M0", "M1", "M2", "M3", "M4", "M5", "M6", "M7", "M8a", "M8b", "M9", "M10", "M11a", "M11b", "M12", "M13"]}
def f1(x): return f"{x:.1f}"
def f2(x): return f"{x:.2f}"
def n_(x): return f"{int(round(x)):,}"
def pc(x): return f"{x:.1f}\\%"
def esc(s):
    s = str(s)
    if "$" in s or "\\" in s: return s   # déjà en LaTeX
    return s.replace("%", "\\%").replace("&", "\\&").replace("_", "\\_").replace("≥", "$\\ge$").replace("≤", "$\\le$").replace("<", "$<$").replace(">", "$>$")
LAB = {"raw": "none", "none": "time only", "conv": "conventional grid", "hedo": "hedonic grid", "hedo_reg": "regional grid", "local": "local grid", "gbm": "GBM adjustments",
       "role": "assessment ratio", "geo": "nearest", "sim": "most similar", "mix": "compromise", "adj": "least adjusted", "equal": "equal", "dist": "1/distance",
       "rec": "recency", "all": "distance $\\times$ recency $\\times$ similarity", "median": "median", "mean": "mean"}
DENS = ["< 5", "5-19", "20-49", "50-199", "≥ 200"]; DENS_EN = {"< 5": "$<5$", "5-19": "5--19", "20-49": "20--49", "50-199": "50--199", "≥ 200": "$\\ge 200$"}
def tabular(df, cols, align=None, fmt=None, bold_min_cols=None):
    fmt = fmt or {}; align = align or ("l" + "r" * (len(cols) - 1))
    lines = ["\\begin{tabular}{" + align + "}", "\\toprule", " & ".join(esc(c) for c in cols) + " \\\\", "\\midrule"]
    for _, r in df.iterrows():
        cells = []
        rowmin = None
        if bold_min_cols: rowmin = min(float(r[c]) for c in bold_min_cols if pd.notna(r[c]))
        for c in cols:
            v = r[c]
            if c in fmt: t = fmt[c](v)
            elif isinstance(v, (float, np.floating)): t = f1(v) if pd.notna(v) else "--"
            else: t = esc(v)
            if bold_min_cols and c in bold_min_cols and pd.notna(v) and float(v) == rowmin: t = "\\textbf{" + t + "}"
            cells.append(t)
        lines.append(" & ".join(cells) + " \\\\")
    lines += ["\\bottomrule", "\\end{tabular}"]
    return "\n".join(lines) + "\n"
def write(name, txt): (P / "tables" / f"{name}.tex").write_text(txt)

# ------------------------------------------------------------------ tables
D = R["dataset"]
ex = pd.DataFrame(D["exclusions"], columns=["Rule", "Removed"]); ex["Rule"] = ex["Rule"].str.replace(r"^R\d+b? ", "", regex=True)
EN = ["Not matched to the assessment roll", "Duplicates (same unit, same date)", "Roll unit not a single dwelling or more than one dwelling", "Coordinates missing or outside Quebec",
      "Living area missing or outside [40, 600] m$^2$", "Lot area missing or outside [80, 20,000] m$^2$", "Year built missing, before 1850 or after the sale (land sale)",
      "Sale in the year of construction (lot or unfinished building)", "Price outside [\\$50,000, \\$3,000,000]", "Assessed value missing or price/assessment ratio outside [0.33, 3]",
      "Resale within 180 days with price change above 40\\% (flip)", "Physical link or construction type missing", "Municipality absent from the MAMH directory"]
ex["Rule"] = EN[:len(ex)]; ex["Removed"] = ex["Removed"].map(n_)
rows = [["Single-family transactions, raw", n_(D["brut"])]] + ex.values.tolist() + [["Final sample", n_(D["propre"])]]
write("exclusions", tabular(pd.DataFrame(rows, columns=["Step", "n"]), ["Step", "n"], "p{0.8\\textwidth}r"))
ds = R.get("descriptives") or {p: dict(n=0, prix_med=0, prix_moy=0, aire_med=0, terrain_med=0, age_med=0, detache=0, ratio_role_med=0) for p in ["train", "val", "test"]}
dd = pd.DataFrame({"Statistic": ["Sales", "Median price (\\$)", "Mean price (\\$)", "Median living area (m$^2$)", "Median lot area (m$^2$)", "Median age (years)", "Detached (\\%)", "Median price/assessment ratio", "Median sales within 2 km, 12 mo.", "Median candidate pool (25 km, 24 mo.)"],
                   **{{"train": "2021--23 (dev.)", "val": "2024 (val.)", "test": "2025--26 (test)"}[p]: [n_(ds[p]["n"]), n_(ds[p]["prix_med"]), n_(ds[p]["prix_moy"]), f1(ds[p]["aire_med"]), n_(ds[p]["terrain_med"]), f1(ds[p]["age_med"]), f1(ds[p]["detache"]), f2(ds[p]["ratio_role_med"]), (n_(ds[p]["dens_med"]) if "dens_med" in ds[p] else "--"), (n_(ds[p]["n_cand_med"]) if "n_cand_med" in ds[p] else "--")] for p in ["train", "val", "test"]}})
write("descriptives", tabular(dd, list(dd.columns), "p{0.46\\textwidth}rrr"))
dist = R["densite_distribution"]; tot = sum(dist.values())
dd = pd.DataFrame({"Density (sales within 2 km, 12 months)": [DENS_EN[d] for d in DENS], "Test targets": [n_(dist[d]) for d in DENS], "Share (\\%)": [f1(dist[d] / tot * 100) for d in DENS]})
write("density_dist", tabular(dd, list(dd.columns), "lrr"))
surf = pd.read_csv(TAB / "surface_E1.csv")
def surf_tab(sel, adj, T_, val="mdape_test"):
    g = surf[(surf.sel == sel) & (surf.adj == adj) & (surf["T"] == T_)].pivot(index="d", columns="k", values=val)
    df = g.reset_index(); df["d"] = df["d"].map(lambda d: f"$\\le {d:g}$ km"); cols = ["d"] + [c for c in g.columns]
    df.columns = ["Radius"] + [f"$k={c}$" for c in g.columns]
    return tabular(df, list(df.columns), "l" + "r" * len(g.columns), bold_min_cols=list(df.columns[1:]))
write("surface_geo_hedo", surf_tab("geo", "hedo", 12)); write("surface_geo_none", surf_tab("geo", "none", 12)); write("surface_sim_hedo", surf_tab("sim", "hedo", 12))
g = surf[(surf.sel == "geo") & (surf.adj == "hedo") & (surf.k == 5)].pivot(index="d", columns="T", values="couverture_test").reset_index()
g["d"] = g["d"].map(lambda d: f"$\\le {d:g}$ km"); g.columns = ["Radius"] + [f"{c} months" for c in g.columns[1:]]
write("surface_cov", tabular(g, list(g.columns), "lrrrr"))
eb = pd.DataFrame(R["E1_best_by_sel_adj"]).T.reset_index()
eb["Selection"] = eb["index"].map(lambda s: LAB[s.split("|")[0]]); eb["Adjustment"] = eb["index"].map(lambda s: LAB[s.split("|")[1]])
eb = eb.rename(columns={"k": "$k$", "d": "$d$ (km)", "T": "$t$ (months)", "mdape_val": "MdAPE 2024", "mdape_test": "MdAPE 2025--26", "couverture_test": "Coverage"})
write("e1_best", tabular(eb, ["Selection", "Adjustment", "$k$", "$d$ (km)", "$t$ (months)", "MdAPE 2024", "MdAPE 2025--26", "Coverage"], "llrrrrrr", fmt={"$k$": lambda v: str(int(v)), "$t$ (months)": lambda v: str(int(v)), "$d$ (km)": lambda v: f"{v:g}"}))
H = R["hypotheses"]
h2 = pd.DataFrame({DENS_EN[k.split("|")[1]]: v for k, v in H["H2"].items() if k.startswith("hedo")}).T.loc[[DENS_EN[d] for d in DENS]].reset_index()
h2.columns = ["Density"] + [f"$k={c}$" for c in h2.columns[1:]]
write("h2", tabular(h2, list(h2.columns), "l" + "r" * (len(h2.columns) - 1), bold_min_cols=list(h2.columns[1:])))
fac = pd.read_csv(TAB / "facteurs_E2.csv"); anchor = fac[(fac.k == 5) & (fac.d == 5) & (fac["T"] == 12)]
adj = anchor[(anchor.sel == "geo") & (anchor.w == "equal") & (anchor.recon == "median")].set_index("adj").loc[["raw", "none", "conv", "hedo", "hedo_reg", "local", "gbm", "role"]].reset_index()
adj["Adjustment"] = adj["adj"].map(LAB)
adj = adj.rename(columns={"mdape_val": "MdAPE 2024", "mdape_test": "MdAPE 2025--26", "mape_test": "MAPE", "biais_test": "Bias", "p10_test": "Within $\\pm10\\%$", "couverture_test": "Coverage"})
write("adjustments", tabular(adj, ["Adjustment", "MdAPE 2024", "MdAPE 2025--26", "MAPE", "Bias", "Within $\\pm10\\%$", "Coverage"], "lrrrrrr"))
def h3tab(prefix):
    h = pd.DataFrame({DENS_EN[k.split("|")[3]]: v for k, v in H["H3"].items() if k.startswith(prefix)}).T.loc[[DENS_EN[d] for d in DENS]].reset_index()
    h.columns = ["Density"] + [LAB[c] for c in h.columns[1:]]; return h
h3a, h3b = h3tab("5|5|12"), h3tab("3|2|6")
t = tabular(h3a, list(h3a.columns), "lrrrrr", bold_min_cols=list(h3a.columns[1:]))
t2 = tabular(h3b, list(h3b.columns), "lrrrrr", bold_min_cols=list(h3b.columns[1:]))
t = t.replace("\\midrule", "\\midrule\n\\multicolumn{6}{l}{\\emph{Anchor $(5,\\,5\\text{ km},\\,12\\text{ months})$}} \\\\", 1)
body2 = t2.split("\\midrule\n", 1)[1].replace("\\bottomrule\n\\end{tabular}\n", "")
t = t.replace("\\bottomrule", "\\midrule\n\\multicolumn{6}{l}{\\emph{Anchor $(3,\\,2\\text{ km},\\,6\\text{ months})$}} \\\\\n" + body2 + "\\bottomrule")
write("h3", t)
mod = json.load(open(OUT / "models_48.json")); b = mod["beta_global"]
NAMES = {"ln_floor": "log living area", "ln_land": "log lot area", "age": "age (years)", "age2": "age$^2$/100", "lien1": "semi-detached", "lien2": "row / integrated", "genre1": "construction type: full storeys", "genre2": "split-level", "genre3": "mansard", "etages": "storeys", "neuf": "built within 3 years"}
gd = pd.DataFrame({"Characteristic": [NAMES[k] for k in mod["feats"]], "Coefficient": [f"{b[k]:.3f}" for k in mod["feats"]]})
write("grid", tabular(gd, list(gd.columns), "lr") + f"\\par\\footnotesize Within-$R^2$ = {mod['r2_within']:.3f}; $n$ = {n_(mod['n_train'])} sales before {mod['start']}.\n")
sel = anchor[(anchor.adj == "hedo") & (anchor.w == "equal") & (anchor.recon == "median")].copy(); sel["Selection"] = sel["sel"].map(LAB)
sel = sel.rename(columns={"mdape_val": "MdAPE 2024", "mdape_test": "MdAPE 2025--26", "biais_test": "Bias", "p10_test": "Within $\\pm10\\%$"})
write("selection", tabular(sel, ["Selection", "MdAPE 2024", "MdAPE 2025--26", "Bias", "Within $\\pm10\\%$"], "lrrrr"))
wr = anchor[(anchor.adj == "hedo") & (anchor.sel == "geo")].copy(); wr["Weighting"] = wr["w"].map({**LAB, "all": "product of the three"}); wr["Reconciliation"] = wr["recon"].map(LAB)
wr = wr.rename(columns={"mdape_val": "MdAPE 2024", "mdape_test": "MdAPE 2025--26", "biais_test": "Bias", "p10_test": "Within $\\pm10\\%$"})
write("weighting", tabular(wr, ["Weighting", "Reconciliation", "MdAPE 2024", "MdAPE 2025--26", "Bias", "Within $\\pm10\\%$"], "llrrrr"))
h4 = pd.DataFrame(H["H4"]).T.loc[DENS].reset_index(); h4["index"] = h4["index"].map(DENS_EN)
SH = {"equal": "equal", "dist": "dist.", "rec": "rec.", "sim": "sim.", "all": "prod.", "median": "med.", "mean": "mean"}
h4.columns = ["Density"] + [f"{SH[c.split('|')[0]]}/{SH[c.split('|')[1]]}" for c in h4.columns[1:]]
write("h4", tabular(h4, list(h4.columns), "l" + "r" * (len(h4.columns) - 1), bold_min_cols=list(h4.columns[1:])))
marg = R["E2_marginal"]; rows = []
for f, lab in [("sel", "Selection"), ("adj", "Adjustment"), ("w", "Weighting"), ("recon", "Reconciliation")]:
    for k, v in marg[f].items(): rows.append([lab, LAB[k], v])
mg = pd.DataFrame(rows, columns=["Factor", "Level", "Mean MdAPE"]); write("marginal", tabular(mg, list(mg.columns), "llr"))
cm = pd.DataFrame(comp).T; cm["val"] = [compv[m]["mdape"] for m in cm.index]; cm = cm.reset_index().rename(columns={"index": "Model"})
cm["Model"] = cm["Model"].str.replace("M0 meilleure configuration empirique (choisie en 2024, couverture ≥ 97 %)", "M0 best configuration, chosen on 2024", regex=False)
TR = {"M1 médiane des 3 plus proches (temps seul)": "M1 median of 3 nearest, time only", "M2 moyenne des 5 plus proches (temps seul)": "M2 mean of 5 nearest, time only",
      "M3 sélection par similarité physique": "M3 5 most similar, time only", "M4 pondération par distance": "M4 5 nearest, distance-weighted", "M5 pondération par récence": "M5 5 nearest, recency-weighted",
      "M6 grille d'ajustement conventionnelle": "M6 conventional grid", "M7 grille d'ajustement hédonique": "M7 hedonic grid", "M8a grille locale (régression sur le voisinage)": "M8a local grid",
      "M8b ajustements non linéaires (GBM)": "M8b GBM adjustments", "M13 ratio au rôle (5 plus proches, même rôle)": "M13 assessment ratio", "M10 hédonique globale directe": "M10 global hedonic model",
      "M11a gradient boosting (sans rôle)": "M11a gradient boosting, no assessed value", "M11b gradient boosting (avec rôle)": "M11b gradient boosting, with assessed value",
      "M9 réconciliation optimisée (poids appris en 2024)": "M9 optimised reconciliation", "M12 hybride comparables + GBM (poids appris en 2024)": "M12 hybrid comparables + GBM"}
cm["Model"] = cm["Model"].map(lambda m: TR.get(m, m)); cm = cm.sort_values("mdape")
cm = cm.rename(columns={"val": "MdAPE 2024", "mdape": "MdAPE 2025--26", "mape": "MAPE", "biais": "Bias", "p5": "$\\pm5\\%$", "p10": "$\\pm10\\%$", "p20": "$\\pm20\\%$", "couverture": "Coverage"})
write("competition", tabular(cm, ["Model", "MdAPE 2024", "MdAPE 2025--26", "MAPE", "Bias", "$\\pm5\\%$", "$\\pm10\\%$", "$\\pm20\\%$", "Coverage"], "lrrrrrrrr"))
pr = pd.read_csv(TAB / "paires_bootstrap.csv"); sh = lambda s: s.split(" ")[0]
pr["Comparison"] = pr.apply(lambda r: f"{sh(r.A)} $-$ {sh(r.B)}", axis=1)
pr["$\\Delta$ MdAPE [95\\% CI]"] = pr.apply(lambda r: f"{r.diff_mdape:.2f} [{eval(r.ci_mdape)[0]:.2f}; {eval(r.ci_mdape)[1]:.2f}]", axis=1)
pr["$\\Delta$ MAPE [95\\% CI]"] = pr.apply(lambda r: f"{r.diff_mape:.2f} [{eval(r.ci_mape)[0]:.2f}; {eval(r.ci_mape)[1]:.2f}]", axis=1)
pr["$A$ more accurate (\\%)"] = pr["part_A_meilleur"].map(f1); pr["$n$"] = pr["n"].map(n_)
write("pairs", tabular(pr, ["Comparison", "$\\Delta$ MdAPE [95\\% CI]", "$\\Delta$ MAPE [95\\% CI]", "$A$ more accurate (\\%)", "$n$"], "lrrrr"))
def strata(by, label, order=None):
    st = pd.read_csv(TAB / f"strates_{by}.csv"); g = st.pivot(index=by, columns="modele", values="mdape").round(1)
    g["n"] = st[st.modele == "M7"].set_index(by)["n"]; g = g[["n", "M2", "M7", "M8a", "M13", "M0", "M10", "M11b", "M12"]]
    if order: g = g.loc[order]
    g = g.reset_index(); g[by] = g[by].map(lambda x: DENS_EN.get(x, x)); g = g.rename(columns={by: label, "n": "$n$"})
    return tabular(g, list(g.columns), "lr" + "r" * 8, fmt={"$n$": n_})
write("density", strata("dens", "Density", DENS))
st = pd.read_csv(TAB / "strates_region_nom.csv"); reg_order = st[st.modele == "M7"].sort_values("n", ascending=False)["region_nom"].tolist()
write("regions", strata("region_nom", "Region", reg_order))
parts = []
for by, lab in [("prix_q", "Price quintile"), ("age_cl", "Age"), ("lien", "Physical link")]:
    t = strata(by, lab); parts.append(t)
LIEN = {"0": "detached", "1": "semi-detached", "2": "row", "3": "integrated"}; AGE = {"≤ 3 ans": "$\\le 3$ years", "4-15": "4--15", "16-40": "16--40", "41-70": "41--70", "> 70": "$>70$"}
PQ = {"Q1 (bas)": "Q1 (low)", "Q5 (haut)": "Q5 (high)"}
txt = parts[0]
for p_ in parts[1:]:
    body = p_.split("\\midrule\n", 1)[1].replace("\\bottomrule\n\\end{tabular}\n", "")
    head = p_.split("\\toprule\n", 1)[1].split(" \\\\", 1)[0]
    txt = txt.replace("\\bottomrule", "\\midrule\n" + head + " \\\\\n\\midrule\n" + body + "\\bottomrule")
for k, v in {**LIEN, **AGE, **PQ}.items(): txt = txt.replace(f"\n{esc(k)} &", f"\n{v} &")
write("strata_other", txt)
h5 = pd.DataFrame(H["H5"]).T.reset_index().rename(columns={"index": "Submarket", "sd_median": "Median $\\hat\\sigma$"})
h5["Submarket"] = h5["Submarket"].map({"homogène": "homogeneous", "moyen": "intermediate", "hétérogène": "heterogeneous"})
write("h5", tabular(h5, ["Submarket", "Median $\\hat\\sigma$", "M2", "M7", "M8a", "M13", "M0", "M10", "M11b", "M12"], "lrrrrrrrrr", fmt={"Median $\\hat\\sigma$": lambda v: f"{v:.3f}"}))
iv = pd.DataFrame(R["intervalles"]).T.reset_index()
iv["Configuration"] = iv["index"].map(lambda i: f"{LAB[i.split('|')[0]]}, $k={i.split('|')[1]}$, $\\le {i.split('|')[2]}$ km, {i.split('|')[3]} months")
iv = iv.rename(columns={"z80_val": "$z$ (calibrated on 2024)", "couverture_nominale_80": "Coverage at $z=1.28$", "couverture_calibree": "Coverage, calibrated $z$", "largeur_mediane_pct": "Median width (\\%)", "n_test": "$n$"})
write("intervals", tabular(iv, ["Configuration", "$z$ (calibrated on 2024)", "Coverage at $z=1.28$", "Coverage, calibrated $z$", "Median width (\\%)", "$n$"], "lrrrrr", fmt={"$n$": n_, "$z$ (calibrated on 2024)": f2}))
rc = pd.DataFrame(R["regles_conditionnelles"]).T.loc[DENS].reset_index(); rc["Density"] = rc["index"].map(DENS_EN)
rc["Rule"] = rc.apply(lambda r: f"{LAB[r.sel]}, $k={int(r.k)}$, $\\le {float(r.d):g}$ km, {int(r['T'])} mo, {LAB[r.adj]}, {LAB[r.w]} w., {LAB[r.recon]}", axis=1)
rc = rc.rename(columns={"mdape": "MdAPE 2024", "mdape_test": "MdAPE 2025--26", "couverture": "Coverage 2024", "n_test": "$n$ test"})
write("cond_rules", tabular(rc, ["Density", "Rule", "MdAPE 2024", "MdAPE 2025--26", "Coverage 2024", "$n$ test"], "llrrrr", fmt={"$n$ test": n_}))
stab = pd.DataFrame(R["stabilite"]).rename(columns={"trimestre": "Quarter", "n": "$n$", "biais_M7": "Bias M7", "biais_M11b": "Bias M11b"})
write("stability", tabular(stab, ["Quarter", "$n$", "M2", "M7", "M0", "M11b", "M12", "Bias M7", "Bias M11b"], "lrrrrrrrr", fmt={"$n$": n_}))
# ------------------------------------------------------------------ figures (version anglaise, même palette)
import matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt
from matplotlib import rcParams
from matplotlib.colors import LinearSegmentedColormap
PAL = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#6250d6", "#e34948"]
CMAP = LinearSegmentedColormap.from_list("seqblue", ["#cde2fb", "#9ec5f4", "#6da7ec", "#3987e5", "#256abf", "#184f95", "#0d366b"])
rcParams.update({"font.family": "Helvetica Neue", "font.size": 9, "axes.edgecolor": "#c3c2b7", "axes.linewidth": 0.6, "axes.grid": True, "grid.color": "#e8e7e3",
                 "grid.linewidth": 0.5, "axes.spines.top": False, "axes.spines.right": False, "legend.frameon": False, "figure.dpi": 150, "savefig.bbox": "tight",
                 "axes.titleweight": "bold", "axes.titlesize": 10, "axes.labelcolor": "#52514e", "xtick.color": "#52514e", "ytick.color": "#52514e"})
def savefig(fig, name): fig.savefig(P / "figures" / f"{name}.pdf"); plt.close(fig)
DENS_LAB = {"< 5": "< 5", "5-19": "5–19", "20-49": "20–49", "50-199": "50–199", "≥ 200": "≥ 200"}
# f1
fig, axes = plt.subplots(1, 2, figsize=(9, 3.4), sharey=True)
for ax, a in zip(axes, ["none", "hedo"]):
    for j, d in enumerate([0.5, 1, 2, 5, 10, 25]):
        g = surf[(surf.sel == "geo") & (surf["T"] == 12) & (surf.adj == a) & (surf.d == d)].sort_values("k")
        ax.plot(g.k, g.mdape_test, color=PAL[j], lw=2, marker="o", ms=4, label=f"d ≤ {d} km")
    ax.set_xscale("log"); ax.set_xticks([1, 3, 5, 10, 20, 50]); ax.set_xticklabels([1, 3, 5, 10, 20, 50])
    ax.set_xlabel("number of comparables k"); ax.set_title({"none": "time adjustment only", "hedo": "hedonic adjustment grid"}[a])
axes[0].set_ylabel("MdAPE, test 2025–26 (%)"); axes[1].legend(ncol=2, fontsize=8); savefig(fig, "f1_mdape_vs_k")
# f2
fig, axes = plt.subplots(1, 2, figsize=(8, 3.2))
for ax, a in zip(axes, ["none", "hedo"]):
    g = surf[(surf.sel == "geo") & (surf.adj == a)].groupby(["d", "T"])["mdape_test"].min().unstack("T")
    ax.imshow(g.to_numpy(), cmap=CMAP, aspect="auto")
    ax.set_xticks(range(len(g.columns))); ax.set_xticklabels([f"{t} months" for t in g.columns]); ax.set_yticks(range(len(g.index))); ax.set_yticklabels([f"{d} km" for d in g.index])
    for i in range(g.shape[0]):
        for j in range(g.shape[1]):
            v = g.iloc[i, j]; ax.text(j, i, f"{v:.1f}", ha="center", va="center", fontsize=7.5, color="white" if v < np.nanpercentile(g.to_numpy(), 40) else "#0b0b0b")
    ax.grid(False); ax.set_title({"none": "time adjustment only", "hedo": "hedonic grid"}[a] + " — MdAPE at the best k")
savefig(fig, "f2_surface_d_T")
# f3
fig, ax = plt.subplots(figsize=(7, 3.2))
g = fac[(fac.k == 5) & (fac.d == 5) & (fac["T"] == 12) & (fac.sel == "geo") & (fac.w == "equal") & (fac.recon == "median")].set_index("adj").loc[["raw", "none", "conv", "hedo", "hedo_reg", "local", "gbm", "role"]]
lab = {"raw": "none", "none": "time only", "conv": "conventional", "hedo": "hedonic", "hedo_reg": "regional", "local": "local", "gbm": "GBM", "role": "assessment ratio"}
x = np.arange(len(g)); ax.bar(x - 0.2, g.mdape_val, 0.38, color=PAL[0], label="validation 2024"); ax.bar(x + 0.2, g.mdape_test, 0.38, color=PAL[1], label="test 2025–26")
for i, (v1, v2) in enumerate(zip(g.mdape_val, g.mdape_test)):
    ax.text(i - 0.2, v1 + 0.2, f"{v1:.1f}", ha="center", fontsize=7); ax.text(i + 0.2, v2 + 0.2, f"{v2:.1f}", ha="center", fontsize=7)
ax.set_xticks(x); ax.set_xticklabels([lab[a] for a in g.index]); ax.set_ylabel("MdAPE (%)"); ax.legend(); ax.set_title("Adjustment scheme — 5 nearest sales, ≤ 5 km, ≤ 12 months, median")
savefig(fig, "f3_ajustements")
# f4
cm2 = cm.sort_values("MdAPE 2025--26", ascending=False)
cols = [PAL[0] if n.startswith(("M10", "M11", "M12")) else (PAL[2] if n.startswith(("M0", "M9")) else PAL[1]) for n in cm2.Model]
fig, ax = plt.subplots(figsize=(7.5, 5)); ax.barh(cm2.Model, cm2["MdAPE 2025--26"], color=cols, height=0.65)
for i, v in enumerate(cm2["MdAPE 2025--26"]): ax.text(v + 0.15, i, f"{v:.1f} %", va="center", fontsize=8)
ax.set_xlabel("MdAPE, test 2025–26 (%)"); ax.set_title("Horse race (orange: comparables; green: optimised; blue: external benchmarks)")
savefig(fig, "f4_competition")
# f5
fig, ax = plt.subplots(figsize=(6.5, 3.4)); st = pd.read_csv(TAB / "strates_dens.csv")
for j, (name, lab_) in enumerate([("M2", "M2 5 nearest, time only"), ("M7", "M7 hedonic grid"), ("M0", "M0 best configuration"), ("M11b", "M11b GBM with assessed value"), ("M12", "M12 hybrid")]):
    g = st[st.modele == name].set_index("dens").loc[DENS]; ax.plot([DENS_LAB[d] for d in DENS], g.mdape, color=PAL[j], lw=2, marker="o", ms=4, label=lab_)
ax.set_xlabel("sales within 2 km in the preceding 12 months"); ax.set_ylabel("MdAPE, test (%)"); ax.legend(fontsize=8); ax.set_title("Accuracy by submarket liquidity")
savefig(fig, "f5_densite")
# f6
h2d = H["H2"]; fig, axes = plt.subplots(1, 2, figsize=(9, 3.3), sharey=True)
for ax, a in zip(axes, ["none", "hedo"]):
    for j, lev in enumerate(DENS):
        v = h2d[f"{a}|{lev}"]; ax.plot([int(k) for k in v], list(v.values()), color=PAL[j], lw=2, marker="o", ms=3.5, label=f"density {DENS_LAB[lev]}")
    ax.set_xscale("log"); ax.set_xticks([1, 3, 5, 10, 20, 50]); ax.set_xticklabels([1, 3, 5, 10, 20, 50]); ax.set_xlabel("k"); ax.set_title({"none": "time only", "hedo": "hedonic grid"}[a] + " — d ≤ 10 km, 12 months")
axes[0].set_ylabel("MdAPE, test (%)"); axes[1].legend(fontsize=8); savefig(fig, "f6_k_par_densite")
# f7
fig, ax = plt.subplots(figsize=(7, 3.2)); idxf = pd.read_csv(OUT / "index_full.csv")
for j, (r_, lab_) in enumerate([("QC", "Quebec (province)"), ("6", "Montréal"), ("3", "Capitale-Nationale"), ("7", "Outaouais"), ("5", "Estrie")]):
    g = idxf[idxf.region.astype(str) == r_]
    if len(g): ax.plot(pd.to_datetime(g.ym), np.exp(g.log_idx) * 100, color=PAL[j], lw=2 if r_ == "QC" else 1.4, label=lab_)
ax.set_ylabel("hedonic index (2021-01 = 100)"); ax.legend(fontsize=8); ax.set_title("Monthly price index (full sample, descriptive)")
savefig(fig, "f7_indice")

# ------------------------------------------------------------------ macros
m = {}
m["nSalesRaw"] = n_(D["brut"]); m["nSalesClean"] = n_(D["propre"]); m["nTrain"] = n_(D["par_periode"]["train"]); m["nVal"] = n_(D["par_periode"]["val"]); m["nTest"] = n_(D["par_periode"]["test"])
m["nTargetsVal"] = n_(R["n_targets"]["val"]); m["nTargetsTest"] = n_(R["n_targets"]["test"]); m["nTargetsAll"] = n_(R["n_targets"]["val"] + R["n_targets"]["test"])
m["nConfigs"] = "2,256"; m["nTransactionsDb"] = "766,596"; m["nUnits"] = "3.75 million"; m["dateMin"] = D["dates"][0]; m["dateMax"] = D["dates"][1]; m["medianPrice"] = n_(D["prix"]["50%"])
for k, name in [("M0", "mdapeMzero"), ("M2", "mdapeMtwo"), ("M3", "mdapeMthree"), ("M6", "mdapeMsix"), ("M7", "mdapeMseven"), ("M9", "mdapeMnine"), ("M10", "mdapeMten"), ("M11a", "mdapeMelevenA"), ("M11b", "mdapeMelevenB"), ("M12", "mdapeMtwelve"), ("M13", "mdapeMthirteen")]:
    m[name] = pc(comp[M[k]]["mdape"])
m["convMdape"] = pc(comp[M["M6"]]["mdape"]); m["localMdape"] = pc(comp[M["M8a"]]["mdape"]); m["covMthirteen"] = pc(comp[M["M13"]]["couverture"])
m["pTenMzero"] = pc(comp[M["M0"]]["p10"]); m["pTenMelevenB"] = pc(comp[M["M11b"]]["p10"])
r15 = [comp[M[k]]["mdape"] for k in ["M1", "M2", "M4", "M5"]]; m["mdapeMoneRange"] = f"{min(r15):.1f}--{max(r15):.1f}\\%"
rg = [comp[M[k]]["mdape"] for k in ["M3", "M6", "M7"]]; m["mdapeGridRange"] = f"{min(rg):.1f}--{max(rg):.1f}\\%"
m["transparencyCost"] = f1(comp[M["M0"]]["mdape"] - comp[M["M11b"]]["mdape"]); m["transparencyCostNoRole"] = f1(comp[M["M0"]]["mdape"] - comp[M["M11a"]]["mdape"])
m["roleGainRatio"] = f1(comp[M["M7"]]["mdape"] - comp[M["M13"]]["mdape"]); m["roleGainGbm"] = f1(comp[M["M11a"]]["mdape"] - comp[M["M11b"]]["mdape"])
ea = R["E2_adjust_anchor"]
m["gainTime"] = f1(ea["raw"]["mdape_test"] - ea["none"]["mdape_test"])
grids = [ea[a]["mdape_test"] for a in ["conv", "hedo", "hedo_reg", "local", "gbm"]]
m["gainGridLo"] = f1(ea["none"]["mdape_test"] - max(grids)); m["gainGridHi"] = f1(ea["none"]["mdape_test"] - min(grids))
m["localVsHedo"] = "slightly worse" if ea["local"]["mdape_test"] > ea["hedo"]["mdape_test"] else "slightly better"
m["gbmVsHedo"] = f1(ea["hedo"]["mdape_test"] - ea["gbm"]["mdape_test"])
es = R["E2_sel_anchor"]; m["gainSelHedo"] = f1(es["geo"]["mdape_test"] - es["sim"]["mdape_test"])
wrv = [v["mdape_test"] for v in R["E2_wr_anchor"].values()]; m["wrRange"] = f1(max(wrv) - min(wrv))
mg_ = R["E2_marginal"]
for f, nm in [("sel", "Sel"), ("adj", "Adj"), ("w", "W")]:
    m[f"marg{nm}Lo"] = f1(min(mg_[f].values())); m[f"marg{nm}Hi"] = f1(max(mg_[f].values()))
m["margReconMedian"] = f1(mg_["recon"]["median"]); m["margReconMean"] = f1(mg_["recon"]["mean"])
kb = surf[(surf.sel == "geo") & (surf.adj == "hedo") & (surf.d == 10) & (surf["T"] == 12)].set_index("k")["mdape_test"]
m["kOpt"] = str(int(kb.idxmin())); m["mdapeKone"] = pc(kb[1]); m["mdapeKthree"] = pc(kb[3]); m["mdapeKfive"] = pc(kb[5]); m["mdapeKten"] = pc(kb[10]); m["mdapeKopt"] = pc(kb.min()); m["mdapeKfifty"] = pc(kb[50])
nb = surf[(surf.sel == "geo") & (surf.adj == "hedo") & (surf.k == 3) & (surf.d == 1) & (surf["T"] == 3)].iloc[0]
m["narrowMdape"] = pc(nb.mdape_test); m["narrowCov"] = pc(nb.couverture_test)
e1b = R["E1_best_val"]; m["bestSel"] = LAB[e1b["sel"]]; m["bestK"] = str(int(e1b["k"])); m["bestD"] = f"{float(e1b['d']):g}"; m["bestT"] = str(int(e1b["T"])); m["bestAdj"] = LAB[e1b["adj"]]
m["bestValMdape"] = pc(e1b["mdape_val"]); m["bestTestMdape"] = pc(e1b["mdape_test"])
m0 = R["M0"]; m["mZeroDesc"] = f"{LAB[m0['sel']]}, $k={int(m0['k'])}$, $\\le {float(m0['d']):g}$ km, {int(m0['T'])} months, {LAB[m0['adj']]}, {LAB[m0['w']]} weights, {LAB[m0['recon']]}"
mu = R["M0_sans_contrainte"]; m["mZeroUncDesc"] = f"{LAB[mu['sel']]}, $k={int(mu['k'])}$, $\\le {float(mu['d']):g}$ km, {int(mu['T'])} months, {LAB[mu['adj']]}"; m["mZeroUncMdape"] = pc(mu["mdape_test"]); m["mZeroUncCov"] = pc(mu["couverture_test"])
dn = R["strates_dens"]; m["densHighMseven"] = pc(dn["≥ 200"]["M7"]); m["densLowMseven"] = pc(dn["< 5"]["M7"])
h1 = H["H1"]; m["hOneGeoDone"] = pc(h1["geo_d1"]["mdape_test"]); m["hOneGeoDoneCov"] = pc(h1["geo_d1"]["couverture_test"]); m["hOneSimDten"] = pc(h1["sim_d10"]["mdape_test"])
m["hOneGeoDtenHedo"] = pc(h1["geo_d10_hedo"]["mdape_test"]); m["hOneSimDtenHedo"] = pc(h1["sim_d10_hedo"]["mdape_test"]); m["hOneMixDtenHedo"] = pc(h1["mix_d10_hedo"]["mdape_test"])
h5 = H["H5"]; m["hFiveGapHomog"] = f1(h5["homogène"]["M0"] - h5["homogène"]["M11b"]); m["hFiveGapHeter"] = f1(h5["hétérogène"]["M0"] - h5["hétérogène"]["M11b"])
ivh = R["intervalles"]["hedo|10|10|24"]; m["ivWidth"] = pc(ivh["largeur_mediane_pct"])
cg = R["regles_conditionnelles_global_test"]; m["condMdape"] = pc(cg["mdape"]); m["condPten"] = pc(cg["p10"]); m["condCov"] = pc(cg["couverture"])
m["betaFloor"] = f"{b['ln_floor']:.2f}"; m["betaLand"] = f"{b['ln_land']:.2f}"; m["betaAge"] = f"{abs(b['age']) * 100:.1f}\\%"; m["betaSemi"] = f"{abs(b['lien1']) * 100:.1f}\\%"; m["betaRow"] = f"{abs(b['lien2']) * 100:.1f}\\%"
ea24 = fac[(fac.k == 10) & (fac.d == 10) & (fac["T"] == 24) & (fac.sel == "geo") & (fac.w == "equal") & (fac.recon == "median")].set_index("adj")["mdape_test"]
m["gainTimeLong"] = f1(ea24["raw"] - ea24["none"])
kn = surf[(surf.sel == "geo") & (surf.adj == "none") & (surf.d == 10) & (surf["T"] == 12)].set_index("k")["mdape_test"]
m["noneKten"] = pc(kn[10]); m["noneKfifty"] = pc(kn[50])
pq = R["strates_prix_q"]; m["prixQoneMseven"] = pc(pq["Q1 (bas)"]["M7"]); m["prixQfourMseven"] = pc(pq["Q4"]["M7"])
stb = {t["trimestre"]: t for t in R["stabilite"]}; m["biasQoneMseven"] = f"{stb['2025Q1']['biais_M7']:+.1f}\\%"; m["biasQthreeMseven"] = f"{stb['2025Q3']['biais_M7']:+.1f}\\%"
lines = [f"\\newcommand{{\\{k}}}{{{v}}}" for k, v in m.items()]
(P / "tables" / "numbers.tex").write_text("% généré par make_paper_tables.py\n" + "\n".join(lines) + "\n")
print(f"{len(m)} macros, tables et figures écrites")