#!/usr/bin/env python3
# Author: Simon-Pierre Boucher — contact@spboucher.ai
#
"""Step 6 — Generate the paper's LaTeX tables from the pipeline results.

Every number printed in these tables comes from results/*.csv (steps 3-4),
so the paper cannot drift from the analysis outputs.

Inputs : results/*.csv, data/processed/hedonic_maison_results.csv
Outputs: paper/tables/tab_descriptive.tex
         paper/tables/tab_similarity_stats.tex
         paper/tables/tab_model_comparison.tex
         paper/tables/tab_full_results.tex
         paper/tables/tab_quantile.tex
         paper/tables/tab_parsimonious.tex      (appendix)
         paper/tables/tab_reference_texts.tex   (appendix)
"""

import sys
from pathlib import Path

import numpy as np
import pandas as pd

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from src import config
from src.references import ENGLISH_LABELS, REFERENCES

TABLES_DIR = config.ROOT / "paper" / "tables"

HEADER = "% Author: Simon-Pierre Boucher — contact@spboucher.ai\n% Generated by scripts/06_tables.py — do not edit by hand.\n"


def tex_num(x, dec=0, signed=False):
    """Format a number with LaTeX minus signs and {,} thousand separators."""
    if signed:
        s = f"{x:+,.{dec}f}"
    else:
        s = f"{x:,.{dec}f}"
    s = s.replace(",", "{,}").replace("-", r"$-$").replace("+", "+")
    return s


def stars(p):
    return "***" if p < 0.001 else "**" if p < 0.01 else "*" if p < 0.05 else ""


def pval_str(p):
    return "$< 0.001$" if p < 0.001 else f"{p:.3f}"


def write(name, content):
    path = TABLES_DIR / name
    path.write_text(HEADER + content)
    print(f"  -> {path}")


def tab_descriptive():
    d = pd.read_csv(config.RESULTS_DIR / "descriptive_stats.csv", index_col=0)
    rows_spec = [
        ("price", "Price (\\$)", 0),
        ("log_price", "log(Price)", 2),
        ("bedrooms", "Bedrooms", 1),
        ("bathrooms", "Bathrooms", 1),
        ("half_baths", "Half-bathrooms", 1),
        ("parking", "Parking spaces", 1),
        ("stories", "Stories", 1),
        ("land_size", "Lot size (sq.\\ ft.)", 0),
        ("remarks_length", "Description length (char.)", 0),
    ]
    lines = []
    for var, label, dec in rows_spec:
        r = d.loc[var]
        cells = [tex_num(r["mean"], dec), tex_num(r["std"], dec),
                 tex_num(r["min"], 0 if var != "log_price" else 2),
                 tex_num(r["25%"], 0 if var != "log_price" else 2),
                 tex_num(r["50%"], 0 if var != "log_price" else 2),
                 tex_num(r["75%"], 0 if var != "log_price" else 2),
                 tex_num(r["max"], 0 if var != "log_price" else 2)]
        lines.append(f"{label} & " + " & ".join(cells) + " \\\\")
    body = "\n".join(lines)
    content = rf"""\begin{{table}}[!htbp]
\centering
\caption{{Descriptive statistics for structural and textual variables ($n = 17{{,}}087$).}}
\label{{tab:descriptive}}
\begin{{threeparttable}}
\small
\begin{{adjustbox}}{{max width=\textwidth}}
\begin{{tabular}}{{lR{{1.6cm}}R{{1.7cm}}R{{1.2cm}}R{{1.4cm}}R{{1.4cm}}R{{1.4cm}}R{{1.9cm}}}}
\toprule
\textbf{{Variable}} & \textbf{{Mean}} & \textbf{{Std.\ Dev.}} & \textbf{{Min}} & \textbf{{P25}} & \textbf{{Median}} & \textbf{{P75}} & \textbf{{Max}} \\
\midrule
{body}
\bottomrule
\end{{tabular}}
\end{{adjustbox}}
\begin{{tablenotes}}
\footnotesize
\item \textit{{Notes:}} The sample is restricted to single-family houses with a positive listing price and a description of at least 20 characters. Descriptions are truncated at approximately 700 characters in the data export, which bounds the description-length variable. The lot-size field is noisy (units are not standardized at the source), which motivates its cautious interpretation in the regressions.
\end{{tablenotes}}
\end{{threeparttable}}
\end{{table}}
"""
    write("tab_descriptive.tex", content)


def tab_similarity_stats():
    s = pd.read_csv(config.RESULTS_DIR / "similarity_stats.csv", index_col=0)
    lines = []
    for label, r in s.iterrows():
        label_tex = label.replace("&", "\\&")
        cells = [f"{r['mean']:.3f}", f"{r['std']:.3f}",
                 tex_num(r["min"], 3), f"{r['50%']:.3f}", f"{r['max']:.3f}",
                 tex_num(r["corr_log_price"], 3)]
        lines.append(f"{label_tex} & " + " & ".join(cells) + " \\\\")
    body = "\n".join(lines)
    content = rf"""\begin{{table}}[!htbp]
\centering
\caption{{Descriptive statistics for cosine similarity features and bivariate correlations with log(price).}}
\label{{tab:sim_stats}}
\begin{{threeparttable}}
\small
\begin{{adjustbox}}{{max width=\textwidth}}
\begin{{tabular}}{{lR{{1.2cm}}R{{1.2cm}}R{{1.2cm}}R{{1.2cm}}R{{1.2cm}}R{{1.4cm}}}}
\toprule
\textbf{{Reference}} & \textbf{{Mean}} & \textbf{{Std.~Dev.}} & \textbf{{Min}} & \textbf{{Median}} & \textbf{{Max}} & \textbf{{Corr.\ $\ln P$}} \\
\midrule
{body}
\bottomrule
\end{{tabular}}
\end{{adjustbox}}
\begin{{tablenotes}}
\footnotesize
\item \textit{{Notes:}} Cosine similarity is computed between each listing embedding and the corresponding reference description embedding using the all-MiniLM-L6-v2 sentence transformer. Corr.\ $\ln P$ denotes the Pearson correlation with log listing price.
\end{{tablenotes}}
\end{{threeparttable}}
\end{{table}}
"""
    write("tab_similarity_stats.tex", content)


def tab_model_comparison():
    m = pd.read_csv(config.RESULTS_DIR / "model_comparison.csv")
    spec_labels = {
        "A": "Structural only",
        "B": "+ Description length",
        "C": "+ Semantic similarities (20)",
        "D": "Full model (B + C)",
        "E": "Parsimonious (sig.\\ only)",
    }
    best_r2 = m["r2"].idxmax()
    best_adj = m["adj_r2"].idxmax()
    best_aic = m["aic"].idxmin()
    best_bic = m["bic"].idxmin()
    lines = []
    for i, r in m.iterrows():
        def fmt(val, dec, best):
            s = tex_num(val, dec)
            return rf"\textbf{{{s}}}" if i == best else s
        delta = "---" if r["model"] == "A" else tex_num(r["delta_r2_vs_A"], 3, signed=True)
        lines.append(
            f"{r['model']} & {spec_labels[r['model']]} & "
            f"{fmt(r['r2'], 4, best_r2)} & {fmt(r['adj_r2'], 4, best_adj)} & "
            f"{fmt(r['aic'], 0, best_aic)} & {fmt(r['bic'], 0, best_bic)} & "
            f"{int(r['k'])} & {delta} \\\\"
        )
    body = "\n".join(lines)
    content = rf"""\begin{{table}}[!htbp]
\centering
\caption{{Hedonic model comparison ($n = 17{{,}}087$).}}
\label{{tab:model_comparison}}
\begin{{threeparttable}}
\small
\begin{{adjustbox}}{{max width=\textwidth}}
\begin{{tabular}}{{clR{{1.3cm}}R{{1.3cm}}R{{1.6cm}}R{{1.6cm}}R{{0.8cm}}R{{1.8cm}}}}
\toprule
& \textbf{{Specification}} & $\boldsymbol{{R^2}}$ & \textbf{{Adj.}}~$\boldsymbol{{R^2}}$ & \textbf{{AIC}} & \textbf{{BIC}} & $\boldsymbol{{k}}$ & $\boldsymbol{{\Delta R^2}}$ \textbf{{vs.\ A}} \\
\midrule
{body}
\bottomrule
\end{{tabular}}
\end{{adjustbox}}
\begin{{tablenotes}}
\footnotesize
\item \textit{{Notes:}} $k$ denotes the number of regressors excluding the intercept. Bold values indicate the best fit for each criterion. Models are estimated by OLS with HC3 robust standard errors.
\end{{tablenotes}}
\end{{threeparttable}}
\end{{table}}
"""
    write("tab_model_comparison.tex", content)


def _coef_rows(d, variables):
    rows = []
    for _, r in d[d.variable.isin(variables)].iterrows():
        rows.append((r["label"] if r["variable"].startswith("sim_") else None,
                     r["variable"], r["coefficient"], r["std_error"],
                     r["t_value"], r["p_value"], r["impact_pct"]))
    return rows


STRUCT_LABELS = {
    "bedrooms": "Bedrooms", "bathrooms": "Bathrooms",
    "half_baths": "Half-bathrooms", "parking": "Parking",
    "stories": "Stories", "land_size": "Lot size",
    "remarks_length": "Description length",
}


def tab_full_results():
    d = pd.read_csv(config.RESULTS_DIR / "coefficients_model_D.csv")
    struct = d[~d.variable.str.startswith("sim_")].copy()
    struct["label"] = struct.variable.map(STRUCT_LABELS)
    sims = d[d.variable.str.startswith("sim_")].copy()
    pos = sims[sims.coefficient >= 0].sort_values("coefficient", ascending=False)
    neg = sims[sims.coefficient < 0].sort_values("coefficient")
    struct = struct.sort_values("coefficient", ascending=False)

    def rows(block):
        out = []
        for _, r in block.iterrows():
            label = str(r["label"]).replace("&", "\\&")
            out.append(
                f"\\quad {label} & {tex_num(r['coefficient'], 4)} & "
                f"{tex_num(r['std_error'], 4)} & {tex_num(r['t_value'], 2)} & "
                f"{pval_str(r['p_value'])} & {tex_num(r['impact_pct'], 1, signed=True)} & "
                f"{stars(r['p_value'])} \\\\"
            )
        return "\n".join(out)

    content = rf"""\begin{{table}}[!htbp]
\centering
\caption{{Full model (D) coefficient estimates. HC3 robust standard errors. All variables standardized.}}
\label{{tab:full_results}}
\small
\begin{{adjustbox}}{{max width=\textwidth}}
\begin{{tabular}}{{lR{{1.2cm}}R{{1.2cm}}R{{1.1cm}}R{{1.6cm}}R{{1.3cm}}c}}
\toprule
\textbf{{Variable}} & \textbf{{Coeff.}} & \textbf{{Std.\ Err.}} & \textbf{{$t$-stat}} & \textbf{{$p$-value}} & \textbf{{Impact (\%)}} & \\
\midrule
\multicolumn{{7}}{{l}}{{\textit{{Panel A: Structural variables}}}} \\
{rows(struct)}
\midrule
\multicolumn{{7}}{{l}}{{\textit{{Panel B: Semantic similarities --- positive price effects}}}} \\
{rows(pos)}
\midrule
\multicolumn{{7}}{{l}}{{\textit{{Panel C: Semantic similarities --- negative price effects}}}} \\
{rows(neg)}
\bottomrule
\multicolumn{{7}}{{l}}{{\footnotesize{{\signote\ Impact $= (e^{{\hat{{\beta}}}} - 1) \times 100$\%. $n = 17{{,}}087$; Adj.\ $R^2 = 0.511$.}}}}
\end{{tabular}}
\end{{adjustbox}}
\end{{table}}
"""
    write("tab_full_results.tex", content)


def tab_quantile():
    r = pd.read_csv(config.RESULTS_DIR / "robustness_results.csv")
    d = pd.read_csv(config.RESULTS_DIR / "coefficients_model_D.csv").set_index("variable")
    dims = ["sim_moderne_contemporain", "sim_luxe", "sim_terrain_nature",
            "sim_familial", "sim_urgence_motivation", "sim_a_renover"]
    patterns = {
        "sim_moderne_contemporain": "High at both tails",
        "sim_luxe": "Increasing",
        "sim_terrain_nature": "Stable",
        "sim_familial": "Mildly attenuating",
        "sim_urgence_motivation": "Attenuating",
        "sim_a_renover": "Attenuating",
    }
    lines = []
    for v in dims:
        label = ENGLISH_LABELS[v[4:]].replace("&", "\\&")
        cells = []
        for tau in ("0.25", "0.5", "0.75"):
            q = r[(r.test == f"QuantReg_tau{tau}") & (r.variable == v)].iloc[0]
            cells.append(f"{tex_num(q['value'], 3)}{stars(q['p_value'])}")
        ols = d.loc[v]
        cells.append(f"{tex_num(ols['coefficient'], 3)}{stars(ols['p_value'])}")
        lines.append(f"{label} & " + " & ".join(cells) + f" & {patterns[v]} \\\\")
    body = "\n".join(lines)
    content = rf"""\begin{{table}}[!htbp]
\centering
\caption{{Quantile regression coefficients for selected semantic dimensions.}}
\label{{tab:quantile}}
\small
\begin{{adjustbox}}{{max width=\textwidth}}
\begin{{tabular}}{{lR{{1.5cm}}R{{1.5cm}}R{{1.5cm}}R{{1.5cm}}l}}
\toprule
\textbf{{Dimension}} & $\boldsymbol{{\tau = 0.25}}$ & $\boldsymbol{{\tau = 0.50}}$ & $\boldsymbol{{\tau = 0.75}}$ & \textbf{{OLS}} & \textbf{{Pattern}} \\
\midrule
{body}
\bottomrule
\multicolumn{{6}}{{l}}{{\footnotesize{{\signote\ Quantile regressions of Model~D; standard errors follow the kernel-based estimator of \citet{{koenker1978regression}}.}}}}
\end{{tabular}}
\end{{adjustbox}}
\end{{table}}
"""
    write("tab_quantile.tex", content)


def tab_parsimonious():
    d = pd.read_csv(config.RESULTS_DIR / "coefficients_model_E.csv")
    struct = d[~d.variable.str.startswith("sim_")].copy()
    struct["label"] = struct.variable.map(STRUCT_LABELS)
    sims = d[d.variable.str.startswith("sim_")].sort_values("coefficient", ascending=False)

    def rows(block):
        out = []
        for _, r in block.iterrows():
            label = str(r["label"]).replace("&", "\\&")
            out.append(
                f"\\quad {label} & {tex_num(r['coefficient'], 4)} & "
                f"{tex_num(r['std_error'], 4)} & {pval_str(r['p_value'])} & "
                f"{tex_num(r['impact_pct'], 1, signed=True)} & {stars(r['p_value'])} \\\\"
            )
        return "\n".join(out)

    content = rf"""\begin{{table}}[!htbp]
\centering
\caption{{Parsimonious model (E) coefficient estimates. HC3 robust standard errors. All variables standardized.}}
\label{{tab:parsimonious}}
\small
\begin{{adjustbox}}{{max width=\textwidth}}
\begin{{tabular}}{{lR{{1.3cm}}R{{1.3cm}}R{{1.6cm}}R{{1.4cm}}c}}
\toprule
\textbf{{Variable}} & \textbf{{Coeff.}} & \textbf{{Std.\ Err.}} & \textbf{{$p$-value}} & \textbf{{Impact (\%)}} & \\
\midrule
\multicolumn{{6}}{{l}}{{\textit{{Panel A: Structural variables}}}} \\
{rows(struct.sort_values('coefficient', ascending=False))}
\midrule
\multicolumn{{6}}{{l}}{{\textit{{Panel B: Semantic similarities (significant at 5\% in Model D)}}}} \\
{rows(sims)}
\bottomrule
\multicolumn{{6}}{{l}}{{\footnotesize{{\signote\ Impact $= (e^{{\hat{{\beta}}}} - 1) \times 100$\%. $n = 17{{,}}087$; Adj.\ $R^2 = 0.511$.}}}}
\end{{tabular}}
\end{{adjustbox}}
\end{{table}}
"""
    write("tab_parsimonious.tex", content)


def tab_reference_texts():
    lines = []
    for slug, text in REFERENCES.items():
        label = ENGLISH_LABELS[slug].replace("&", "\\&")
        text_tex = text.replace("&", "\\&").replace("%", "\\%")
        lines.append(f"{label} & \\textit{{{text_tex}}} \\\\[2pt]")
    body = "\n".join(lines)
    content = rf"""\begin{{footnotesize}}
\begin{{longtable}}{{p{{3.2cm}}p{{11.5cm}}}}
\caption{{The 20 reference descriptions (verbatim French text used for embedding).}}
\label{{tab:reference_texts}} \\
\toprule
\textbf{{Dimension}} & \textbf{{Reference description}} \\
\midrule
\endfirsthead
\toprule
\textbf{{Dimension}} & \textbf{{Reference description}} \\
\midrule
\endhead
\bottomrule
\endfoot
{body}
\end{{longtable}}
\end{{footnotesize}}
"""
    write("tab_reference_texts.tex", content)


def main():
    TABLES_DIR.mkdir(parents=True, exist_ok=True)
    tab_descriptive()
    tab_similarity_stats()
    tab_model_comparison()
    tab_full_results()
    tab_quantile()
    tab_parsimonious()
    tab_reference_texts()
    print("All tables generated.")


if __name__ == "__main__":
    main()