#!/usr/bin/env python3
# Author: Simon-Pierre Boucher — contact@spboucher.ai
#
"""Step 8 — LaTeX tables and figure for the revision extensions (step 7).

Inputs : results/revision_*.csv
Outputs: paper/tables/tab_language.tex
         paper/tables/tab_spec_summary.tex
         paper/tables/tab_extended_specs.tex
         paper/tables/tab_block_tests.tex
         paper/tables/tab_oos.tex
         paper/tables/tab_reference_sensitivity.tex
         figures/fig12_specification_stability.{pdf,png}
"""

import sys
from pathlib import Path

import matplotlib
import numpy as np
import pandas as pd

matplotlib.use("Agg")
import matplotlib.pyplot as plt

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from src import config
from src.references import ENGLISH_LABELS, REFERENCES

TABLES_DIR = config.ROOT / "paper" / "tables"
HEADER = ("% Author: Simon-Pierre Boucher — contact@spboucher.ai\n"
          "% Generated by scripts/08_revision_tables.py — do not edit by hand.\n")
R = config.RESULTS_DIR

plt.rcParams.update({
    "font.family": "serif", "font.serif": ["Times New Roman", "Times", "DejaVu Serif"],
    "font.size": 10, "axes.labelsize": 11, "xtick.labelsize": 9, "ytick.labelsize": 9,
    "legend.fontsize": 8.5, "figure.dpi": 300, "savefig.dpi": 300, "savefig.bbox": "tight",
    "savefig.pad_inches": 0.08, "axes.spines.top": False, "axes.spines.right": False,
})


def tex(label):
    return label.replace("&", "\\&")


def num(x, dec=3):
    s = f"{x:,.{dec}f}".replace(",", "{,}")
    return s.replace("-", "$-$")


def stars(p):
    return "***" if p < 0.001 else "**" if p < 0.01 else "*" if p < 0.05 else ""


def write(name, content):
    (TABLES_DIR / name).write_text(HEADER + content)
    print(f"  -> {TABLES_DIR / name}")


# --------------------------------------------------------------------------
def tab_language():
    d = pd.read_csv(R / "revision_language.csv")
    base = d[d.model == "all-MiniLM-L6-v2"].set_index("language")
    multi = d[d.model == "paraphrase-multilingual-MiniLM-L12-v2"].set_index("language")
    rows = []
    for lang, name in [("fr", "French"), ("en", "English"), ("mixed", "Mixed / uninformative")]:
        b, m = base.loc[lang], multi.loc[lang]
        n_tex = f"{int(b['n']):,}".replace(",", "{,}")
        price_tex = f"{b['median_price']:,.0f}".replace(",", "{,}")
        rows.append(f"{name} & {n_tex} & {b['share'] * 100:.1f} & {b['mean_length']:.0f} & "
                    f"\\${price_tex} & {b['mean_log_price']:.3f} & "
                    f"{b['mean_similarity']:.3f} & {m['mean_similarity']:.3f} \\\\")
    body = "\n".join(rows)
    e_b = base.loc["eta2_language_on_mean_similarity", "mean_similarity"]
    e_m = multi.loc["eta2_language_on_mean_similarity", "mean_similarity"]
    c_b = base.loc["corr_mean_similarity_english", "mean_similarity"]
    c_m = multi.loc["corr_mean_similarity_english", "mean_similarity"]
    l_b = base.loc["corr_mean_similarity_length", "mean_similarity"]
    l_m = multi.loc["corr_mean_similarity_length", "mean_similarity"]
    p_b = base.loc["pc1_share_of_similarity_variance", "mean_similarity"]
    p_m = multi.loc["pc1_share_of_similarity_variance", "mean_similarity"]
    content = rf"""\begin{{table}}[!htbp]
\centering
\caption{{Language of the listing descriptions and mean cosine similarity to the 20 French references.}}
\label{{tab:language}}
\small
\begin{{adjustbox}}{{max width=\textwidth}}
\begin{{tabular}}{{lR{{1.2cm}}R{{1.1cm}}R{{1.3cm}}R{{1.6cm}}R{{1.3cm}}R{{1.5cm}}R{{1.7cm}}}}
\toprule
& & & & & & \multicolumn{{2}}{{c}}{{\textbf{{Mean similarity}}}} \\
\cmidrule(lr){{7-8}}
\textbf{{Language}} & $\boldsymbol{{n}}$ & \textbf{{\%}} & \textbf{{Length}} & \textbf{{Median price}} & $\boldsymbol{{\overline{{\ln P}}}}$ & \textbf{{MiniLM-L6}} & \textbf{{Multilingual}} \\
\midrule
{body}
\midrule
\multicolumn{{6}}{{l}}{{Share of the variance of the mean similarity explained by language ($\eta^2$)}} & {e_b:.3f} & {e_m:.3f} \\
\multicolumn{{6}}{{l}}{{Correlation of the mean similarity with the English indicator}} & {num(c_b)} & {num(c_m)} \\
\multicolumn{{6}}{{l}}{{Correlation of the mean similarity with description length}} & {l_b:.3f} & {l_m:.3f} \\
\multicolumn{{6}}{{l}}{{Share of the variance of the 20 similarities on their first principal component}} & {p_b:.3f} & {p_m:.3f} \\
\bottomrule
\end{{tabular}}
\end{{adjustbox}}
\begin{{minipage}}{{\textwidth}}
\vspace{{4pt}}\footnotesize
\textit{{Notes:}} Language is classified from French and English stop-word counts (ratio rule of 1.5; ``mixed'' listings are mostly short boilerplate such as ``See listing broker(s)''). Length is the mean number of characters. MiniLM-L6 is the paper's baseline encoder (all-MiniLM-L6-v2); Multilingual is paraphrase-multilingual-MiniLM-L12-v2 \citep{{reimers2020multilingual}}. The mean similarity is the average of the 20 cosine similarities of a listing.
\end{{minipage}}
\end{{table}}
"""
    write("tab_language.tex", content)


# --------------------------------------------------------------------------
SPEC_ORDER = [
    ("D (baseline)", "D: baseline (Table~\\ref{tab:model_comparison})"),
    ("D (baseline), FSA-clustered SE", "D, standard errors clustered by FSA"),
    ("D-L: + language", "D-L: D + language indicators"),
    ("D-L, FSA-clustered SE", "D-L, standard errors clustered by FSA"),
    ("D-L + FSA FE", "D-L + FSA fixed effects"),
    ("D-L + municipality FE", "D-L + municipality fixed effects"),
    ("D-L + FSA FE, lot area (m2)", "D-L + FSA FE, unit-corrected lot area"),
    ("D-L + FSA FE, floor-area subsample", "D-L + FSA FE + floor area (subsample)"),
    ("French-only, D + FSA FE", "French listings only, D + FSA FE"),
    ("English-only, D + FSA FE", "English listings only, D + FSA FE"),
    ("ML-D: multilingual similarities", "ML-D: multilingual encoder, as D"),
    ("ML-D-L: + language", "ML-D-L: multilingual encoder + language"),
    ("ML-D-L + FSA FE", "ML-D-L + FSA fixed effects"),
    ("ML-D-L + FSA FE, lot area (m2)", "ML-D-L + FSA FE, unit-corrected lot area"),
]


def tab_spec_summary():
    s = pd.read_csv(R / "revision_spec_summary.csv").set_index("spec")
    rows = []
    for key, label in SPEC_ORDER:
        r = s.loc[key]
        cov = "cluster" if r["cov"] != "HC3" else "HC3"
        r2w = "---" if pd.isna(r["r2_within"]) else f"{r['r2_within']:.3f}"
        n_tex = f"{int(r['n']):,}".replace(",", "{,}")
        rows.append(f"{label} & {n_tex} & {r['r2']:.3f} & {r2w} & {r['delta_r2_sims']:.3f} & "
                    f"{r['block_F']:.1f} & {int(r['n_sig_5pct'])}/20 & {cov} \\\\")
    body = "\n".join(rows)
    content = rf"""\begin{{table}}[!htbp]
\centering
\caption{{Incremental explanatory power of the 20 semantic similarities across extended specifications.}}
\label{{tab:spec_summary}}
\small
\begin{{adjustbox}}{{max width=\textwidth}}
\begin{{tabular}}{{lR{{1.2cm}}R{{1.0cm}}R{{1.2cm}}R{{1.4cm}}R{{1.3cm}}R{{1.1cm}}l}}
\toprule
\textbf{{Specification}} & $\boldsymbol{{n}}$ & $\boldsymbol{{R^2}}$ & \textbf{{Within}} $\boldsymbol{{R^2}}$ & $\boldsymbol{{\Delta R^2}}$ \textbf{{(sem.)}} & \textbf{{Block}} $\boldsymbol{{F}}$ & \textbf{{Sig.}} & \textbf{{Cov.}} \\
\midrule
{body}
\bottomrule
\end{{tabular}}
\end{{adjustbox}}
\begin{{minipage}}{{\textwidth}}
\vspace{{4pt}}\footnotesize
\textit{{Notes:}} All specifications include the six structural variables and description length; ``D-L'' adds indicators for English and mixed-language listings; ``ML'' replaces the all-MiniLM-L6-v2 similarities by those computed with paraphrase-multilingual-MiniLM-L12-v2. Fixed effects are absorbed by within-group demeaning (415 forward sortation areas, FSA, or 1{{,}}061 municipalities); $R^2$ is then measured against the total variance of log price and ``Within $R^2$'' against the within-group variance. $\Delta R^2$ (sem.) is the gain in $R^2$ from adding the 20 similarities to the otherwise identical specification. Block $F$ is the robust Wald test (HC3 or clustered by FSA/municipality) that the 20 semantic coefficients are jointly zero; all $p < 0.001$. ``Sig.'' counts similarities individually significant at the 5\% level. The unit-corrected lot area converts the raw field to square metres (m$^2$, sq.\ ft., acres, hectares) and enters in logs with a missing-value indicator; the floor-area subsample is restricted to the 6{{,}}228 listings reporting interior floor area.
\end{{minipage}}
\end{{table}}
"""
    write("tab_spec_summary.tex", content)


# --------------------------------------------------------------------------
EXT_COLS = [("D (baseline)", "D"), ("D-L: + language", "D-L"), ("D-L + FSA FE", "D-L + FSA FE"),
            ("French-only, D + FSA FE", "French only"), ("English-only, D + FSA FE", "English only"),
            ("ML-D-L + FSA FE", "ML-D-L + FSA FE")]
DIM_ORDER = [ENGLISH_LABELS[k] for k in REFERENCES]


def tab_extended_specs():
    sp = pd.read_csv(R / "revision_specifications.csv")
    sims = sp[sp.variable.str.startswith(("sim_", "ml_"))]
    imp = sims.pivot_table(index="label", columns="spec", values="impact_pct")
    pv = sims.pivot_table(index="label", columns="spec", values="p_value")
    summ = pd.read_csv(R / "revision_spec_summary.csv").set_index("spec")
    rows = []
    for lab in DIM_ORDER:
        cells = []
        for key, _ in EXT_COLS:
            v, p = imp.loc[lab, key], pv.loc[lab, key]
            cells.append(f"{num(v, 1)}{stars(p)}")
        rows.append(f"{tex(lab)} & " + " & ".join(cells) + " \\\\")
    body = "\n".join(rows)
    foot_n = " & ".join(f"{int(summ.loc[k, 'n']):,}".replace(",", "{,}") for k, _ in EXT_COLS)
    foot_r2 = " & ".join(f"{summ.loc[k, 'r2']:.3f}" for k, _ in EXT_COLS)
    foot_d = " & ".join(f"{summ.loc[k, 'delta_r2_sims']:.3f}" for k, _ in EXT_COLS)
    head = " & ".join(f"\\textbf{{{lbl}}}" for _, lbl in EXT_COLS)
    content = rf"""\begin{{table}}[!htbp]
\centering
\caption{{Semantic implicit price gradients (\% per standard deviation) across specifications and encoders.}}
\label{{tab:extended_specs}}
\small
\begin{{adjustbox}}{{max width=\textwidth}}
\begin{{tabular}}{{lR{{1.5cm}}R{{1.5cm}}R{{1.8cm}}R{{1.6cm}}R{{1.6cm}}R{{1.9cm}}}}
\toprule
\textbf{{Dimension}} & {head} \\
\midrule
{body}
\midrule
$n$ & {foot_n} \\
$R^2$ & {foot_r2} \\
$\Delta R^2$ (semantic block) & {foot_d} \\
\bottomrule
\end{{tabular}}
\end{{adjustbox}}
\begin{{minipage}}{{\textwidth}}
\vspace{{4pt}}\footnotesize
\textit{{Notes:}} Entries are $100\,(e^{{\hat\gamma_j}} - 1)$ for a one-standard-deviation increase in similarity $j$ (standard deviation computed on the estimation sample). D is the baseline full model (HC3 standard errors); D-L adds English and mixed-language indicators (HC3); ``+ FSA FE'' absorbs 415 forward-sortation-area fixed effects with standard errors clustered by FSA; the French-only and English-only columns re-estimate D with FSA fixed effects on each language subsample; ML-D-L replaces the encoder by paraphrase-multilingual-MiniLM-L12-v2. \signote
\end{{minipage}}
\end{{table}}
"""
    write("tab_extended_specs.tex", content)


# --------------------------------------------------------------------------
def tab_block_tests():
    b = pd.read_csv(R / "revision_block_tests.csv")
    order = ["Quality & Standing", "Condition", "Physical Attributes", "Location & Setting",
             "Style & Character", "Market Signals", "All 20 dimensions"]
    hc = b[b.spec == "D-L (HC3)"].set_index("domain")
    fe = b[b.spec == "D-L + FSA FE (clustered)"].set_index("domain")
    rows = []
    for dom in order:
        rows.append(f"{tex(dom)} & {int(hc.loc[dom, 'n_dims'])} & {hc.loc[dom, 'F']:.2f} & "
                    f"{'$< 0.001$' if hc.loc[dom, 'p_value'] < 0.001 else f'{hc.loc[dom, 'p_value']:.3f}'} & "
                    f"{fe.loc[dom, 'F']:.2f} & "
                    f"{'$< 0.001$' if fe.loc[dom, 'p_value'] < 0.001 else f'{fe.loc[dom, 'p_value']:.3f}'} \\\\")
    body = "\n".join(rows)
    content = rf"""\begin{{table}}[!htbp]
\centering
\caption{{Domain-level block tests of the semantic dimensions.}}
\label{{tab:block_tests}}
\small
\begin{{tabular}}{{lR{{1.0cm}}R{{1.4cm}}R{{1.6cm}}R{{1.4cm}}R{{1.6cm}}}}
\toprule
& & \multicolumn{{2}}{{c}}{{\textbf{{D-L (HC3)}}}} & \multicolumn{{2}}{{c}}{{\textbf{{D-L + FSA FE (clustered)}}}} \\
\cmidrule(lr){{3-4}} \cmidrule(lr){{5-6}}
\textbf{{Domain}} & $\boldsymbol{{q}}$ & $\boldsymbol{{F}}$ & $\boldsymbol{{p}}$ & $\boldsymbol{{F}}$ & $\boldsymbol{{p}}$ \\
\midrule
{body}
\bottomrule
\end{{tabular}}
\begin{{minipage}}{{\textwidth}}
\vspace{{4pt}}\footnotesize
\textit{{Notes:}} Wald tests that the $q$ coefficients of the dimensions belonging to each domain of Table~\ref{{tab:references}} are jointly zero, under the D-L specification with HC3 standard errors and under D-L with 415 FSA fixed effects and standard errors clustered by FSA.
\end{{minipage}}
\end{{table}}
"""
    write("tab_block_tests.tex", content)


# --------------------------------------------------------------------------
def tab_oos():
    o = pd.read_csv(R / "revision_oos.csv")
    rnd = o[o.scheme == "random_5fold"].set_index("specification")
    spa = o[o.scheme == "spatial_fsa_5fold"].set_index("specification")
    order = ["A (raw lot-size field)", "A: structural", "B: + length", "B-L: + length + language",
             "D: + length + 20 similarities", "D-L: D + language", "PCA-20 + length",
             "PCA-20 + length + language", "ML-D: multilingual similarities",
             "ML-D-L: multilingual similarities + language"]
    names = {"A (raw lot-size field)": "A, raw lot-size field (for reference)",
             "A: structural": "A: structural", "B: + length": "B: A + description length",
             "B-L: + length + language": "B-L: B + language indicators",
             "D: + length + 20 similarities": "D: B + 20 similarities",
             "D-L: D + language": "D-L: D + language indicators",
             "PCA-20 + length": "B + 20 principal components of the embeddings",
             "PCA-20 + length + language": "B-L + 20 principal components",
             "ML-D: multilingual similarities": "ML-D: B + 20 multilingual similarities",
             "ML-D-L: multilingual similarities + language": "ML-D-L: B-L + 20 multilingual similarities"}
    rows = []
    for k in order:
        r, s = rnd.loc[k], spa.loc[k]
        rows.append(f"{names[k]} & {num(r['r2_oos'])} & {r['rmse_log']:.3f} & {r['mdape_pct']:.1f} & "
                    f"{num(s['r2_oos'])} & {s['rmse_log']:.3f} & {s['mdape_pct']:.1f} \\\\")
    body = "\n".join(rows)
    content = rf"""\begin{{table}}[!htbp]
\centering
\caption{{Out-of-sample performance: random and spatial five-fold cross-validation.}}
\label{{tab:oos}}
\small
\begin{{adjustbox}}{{max width=\textwidth}}
\begin{{tabular}}{{lR{{1.3cm}}R{{1.3cm}}R{{1.4cm}}R{{1.3cm}}R{{1.3cm}}R{{1.4cm}}}}
\toprule
& \multicolumn{{3}}{{c}}{{\textbf{{Random folds}}}} & \multicolumn{{3}}{{c}}{{\textbf{{FSA-grouped folds}}}} \\
\cmidrule(lr){{2-4}} \cmidrule(lr){{5-7}}
\textbf{{Specification}} & $\boldsymbol{{R^2_{{\text{{oos}}}}}}$ & \textbf{{RMSE}} & \textbf{{MdAPE}} & $\boldsymbol{{R^2_{{\text{{oos}}}}}}$ & \textbf{{RMSE}} & \textbf{{MdAPE}} \\
\midrule
{body}
\bottomrule
\end{{tabular}}
\end{{adjustbox}}
\begin{{minipage}}{{\textwidth}}
\vspace{{4pt}}\footnotesize
\textit{{Notes:}} Five-fold cross-validation on the $n = 17{{,}}087$ listings; predictions for each fold are made from a model estimated on the other four folds, and the metrics are computed on the pooled out-of-fold predictions (Equations~\ref{{eq:r2oos}}--\ref{{eq:mdape}}). Random folds are drawn with seed 42; FSA-grouped folds hold out entire forward sortation areas, so that no held-out listing shares an FSA with the training data. RMSE is in log points; MdAPE is the median absolute percentage error of the price level. Except for the first row, the lot-size variable is the unit-corrected log lot area with a missing indicator; the raw field (mixed units, maximum $7.2 \times 10^8$) yields exploding out-of-fold predictions and a negative $R^2$. Principal components are estimated on the training folds only.
\end{{minipage}}
\end{{table}}
"""
    write("tab_oos.tex", content)


# --------------------------------------------------------------------------
def tab_reference_sensitivity():
    s = pd.read_csv(R / "revision_reference_sensitivity_summary.csv").set_index("variant")
    names = {"first_half": "First half of each reference's phrases",
             "second_half": "Second half of each reference's phrases",
             "reversed": "Phrases in reverse order"}
    rows = []
    for k in ["first_half", "second_half", "reversed"]:
        r = s.loc[k]
        rows.append(f"{names[k]} & {r['mean_corr_similarity']:.3f} & {r['r2']:.4f} & {r['delta_r2_sims']:.3f} & "
                    f"{r['block_F']:.1f} & {r['corr_coef_with_baseline']:.3f} & "
                    f"{r['sign_agreement_sig16'] * 100:.0f}\\% & {int(r['n_sig_5pct'])}/20 \\\\")
    body = "\n".join(rows)
    content = rf"""\begin{{table}}[!htbp]
\centering
\caption{{Sensitivity of Model~D to the phrasing of the reference descriptions.}}
\label{{tab:reference_sensitivity}}
\small
\begin{{adjustbox}}{{max width=\textwidth}}
\begin{{tabular}}{{lR{{1.6cm}}R{{1.1cm}}R{{1.4cm}}R{{1.3cm}}R{{1.7cm}}R{{1.6cm}}R{{1.1cm}}}}
\toprule
\textbf{{Reference variant}} & \textbf{{Corr.\ of similarities}} & $\boldsymbol{{R^2}}$ & $\boldsymbol{{\Delta R^2}}$ \textbf{{(sem.)}} & \textbf{{Block}} $\boldsymbol{{F}}$ & \textbf{{Corr.\ of coefficients}} & \textbf{{Sign agreement}} & \textbf{{Sig.}} \\
\midrule
Baseline (full references) & 1.000 & 0.5122 & 0.048 & 54.3 & 1.000 & 100\% & 16/20 \\
{body}
\bottomrule
\end{{tabular}}
\end{{adjustbox}}
\begin{{minipage}}{{\textwidth}}
\vspace{{4pt}}\footnotesize
\textit{{Notes:}} Each reference text in Appendix Table~\ref{{tab:reference_texts}} is split into its comma-separated phrases and re-encoded with all-MiniLM-L6-v2 after the indicated perturbation; Model~D is then re-estimated with the 20 perturbed similarities. ``Corr.\ of similarities'' is the mean, over the 20 dimensions, of the correlation between perturbed and baseline similarity scores; ``Corr.\ of coefficients'' is the correlation between the 20 perturbed and baseline semantic coefficients; ``Sign agreement'' is the share of the 16 dimensions significant in the baseline whose coefficient keeps its sign; ``Sig.'' counts dimensions significant at 5\%. Block $F$ is the HC3 Wald test of the 20 semantic coefficients.
\end{{minipage}}
\end{{table}}
"""
    write("tab_reference_sensitivity.tex", content)


# --------------------------------------------------------------------------
def fig12_specification_stability():
    sp = pd.read_csv(R / "revision_specifications.csv")
    sims = sp[sp.variable.str.startswith(("sim_", "ml_"))]
    imp = sims.pivot_table(index="label", columns="spec", values="impact_pct")
    pv = sims.pivot_table(index="label", columns="spec", values="p_value")
    order = imp["D (baseline)"].sort_values().index.tolist()
    specs = [("D (baseline)", "D: baseline (MiniLM-L6)", "#1565C0", "o"),
             ("D-L + FSA FE", "D-L + FSA FE (MiniLM-L6)", "#2E7D32", "s"),
             ("ML-D-L + FSA FE", "ML-D-L + FSA FE (multilingual)", "#C62828", "D")]
    fig, ax = plt.subplots(figsize=(6.4, 6.0))
    yy = np.arange(len(order))
    for j, (key, lbl, col, mk) in enumerate(specs):
        off = (j - 1) * 0.24
        vals = imp.loc[order, key].values
        sig = pv.loc[order, key].values < 0.05
        ax.scatter(vals[sig], yy[sig] + off, s=26, color=col, marker=mk, label=lbl, zorder=3)
        ax.scatter(vals[~sig], yy[~sig] + off, s=26, facecolors="none", edgecolors=col,
                   marker=mk, zorder=3)
    for y in yy[:-1]:
        ax.axhline(y + 0.5, color="#DDDDDD", lw=0.5, zorder=1)
    ax.axvline(0, color="black", lw=0.8)
    ax.set_yticks(yy)
    ax.set_yticklabels(order, fontsize=8.5)
    ax.set_xlabel("Price impact of a one-standard-deviation increase in similarity (%)")
    ax.legend(loc="lower right", frameon=False)
    ax.set_ylim(-0.7, len(order) - 0.3)
    plt.tight_layout()
    for ext in ("pdf", "png"):
        plt.savefig(config.FIGURES_DIR / f"fig12_specification_stability.{ext}")
    plt.close()
    print("  -> figures/fig12_specification_stability")


def main():
    tab_language()
    tab_spec_summary()
    tab_extended_specs()
    tab_block_tests()
    tab_oos()
    tab_reference_sensitivity()
    fig12_specification_stability()


if __name__ == "__main__":
    main()