spb/wp2_uqo

1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — contact@spboucher.ai3#4"""Step 6 — Generate the paper's LaTeX tables from the pipeline results.56Every number printed in these tables comes from results/*.csv (steps 3-4),7so the paper cannot drift from the analysis outputs.89Inputs : results/*.csv, data/processed/hedonic_maison_results.csv10Outputs: paper/tables/tab_descriptive.tex11         paper/tables/tab_similarity_stats.tex12         paper/tables/tab_model_comparison.tex13         paper/tables/tab_full_results.tex14         paper/tables/tab_quantile.tex15         paper/tables/tab_parsimonious.tex      (appendix)16         paper/tables/tab_reference_texts.tex   (appendix)17"""1819import sys20from pathlib import Path2122import numpy as np23import pandas as pd2425sys.path.insert(0, str(Path(__file__).resolve().parents[1]))2627from src import config28from src.references import ENGLISH_LABELS, REFERENCES2930TABLES_DIR = config.ROOT / "paper" / "tables"3132HEADER = "% Author: Simon-Pierre Boucher — contact@spboucher.ai\n% Generated by scripts/06_tables.py — do not edit by hand.\n"333435def tex_num(x, dec=0, signed=False):36    """Format a number with LaTeX minus signs and {,} thousand separators."""37    if signed:38        s = f"{x:+,.{dec}f}"39    else:40        s = f"{x:,.{dec}f}"41    s = s.replace(",", "{,}").replace("-", r"$-$").replace("+", "+")42    return s434445def stars(p):46    return "***" if p < 0.001 else "**" if p < 0.01 else "*" if p < 0.05 else ""474849def pval_str(p):50    return "$< 0.001$" if p < 0.001 else f"{p:.3f}"515253def write(name, content):54    path = TABLES_DIR / name55    path.write_text(HEADER + content)56    print(f"  -> {path}")575859def tab_descriptive():60    d = pd.read_csv(config.RESULTS_DIR / "descriptive_stats.csv", index_col=0)61    rows_spec = [62        ("price", "Price (\\$)", 0),63        ("log_price", "log(Price)", 2),64        ("bedrooms", "Bedrooms", 1),65        ("bathrooms", "Bathrooms", 1),66        ("half_baths", "Half-bathrooms", 1),67        ("parking", "Parking spaces", 1),68        ("stories", "Stories", 1),69        ("land_size", "Lot size (raw field, mixed units)", 0),70        ("remarks_length", "Description length (char.)", 0),71    ]72    lines = []73    for var, label, dec in rows_spec:74        r = d.loc[var]75        cells = [tex_num(r["mean"], dec), tex_num(r["std"], dec),76                 tex_num(r["min"], 0 if var != "log_price" else 2),77                 tex_num(r["25%"], 0 if var != "log_price" else 2),78                 tex_num(r["50%"], 0 if var != "log_price" else 2),79                 tex_num(r["75%"], 0 if var != "log_price" else 2),80                 tex_num(r["max"], 0 if var != "log_price" else 2)]81        lines.append(f"{label} & " + " & ".join(cells) + " \\\\")82    body = "\n".join(lines)83    content = rf"""\begin{{table}}[!htbp]84\centering85\caption{{Descriptive statistics for structural and textual variables ($n = 17{{,}}087$).}}86\label{{tab:descriptive}}87\small88\begin{{adjustbox}}{{max width=\textwidth}}89\begin{{tabular}}{{lR{{1.6cm}}R{{1.7cm}}R{{1.2cm}}R{{1.4cm}}R{{1.4cm}}R{{1.4cm}}R{{1.9cm}}}}90\toprule91\textbf{{Variable}} & \textbf{{Mean}} & \textbf{{Std.\ Dev.}} & \textbf{{Min}} & \textbf{{P25}} & \textbf{{Median}} & \textbf{{P75}} & \textbf{{Max}} \\92\midrule93{body}94\bottomrule95\end{{tabular}}96\end{{adjustbox}}97\begin{{minipage}}{{\textwidth}}98\vspace{{4pt}}\footnotesize99\textit{{Notes:}} The sample is restricted to single-family houses with a positive listing price and a description of at least 20 characters. Descriptions are truncated at approximately 700 characters in the data export, which bounds the description-length variable. The raw lot-size field mixes units at the source (61\\% of values are reported in m$^2$, 32\\% in sq.\\ ft., 2\\% in acres or hectares, 4\\% missing) and is used as reported in the baseline models for comparability with the first version of the paper; a unit-corrected lot area is used in Section~\\ref{{sec:extended}}.100\end{{minipage}}101\end{{table}}102"""103    write("tab_descriptive.tex", content)104105106def tab_similarity_stats():107    s = pd.read_csv(config.RESULTS_DIR / "similarity_stats.csv", index_col=0)108    lines = []109    for label, r in s.iterrows():110        label_tex = label.replace("&", "\\&")111        cells = [f"{r['mean']:.3f}", f"{r['std']:.3f}",112                 tex_num(r["min"], 3), f"{r['50%']:.3f}", f"{r['max']:.3f}",113                 tex_num(r["corr_log_price"], 3)]114        lines.append(f"{label_tex} & " + " & ".join(cells) + " \\\\")115    body = "\n".join(lines)116    content = rf"""\begin{{table}}[!htbp]117\centering118\caption{{Descriptive statistics for cosine similarity features and bivariate correlations with log(price).}}119\label{{tab:sim_stats}}120\begin{{threeparttable}}121\small122\begin{{adjustbox}}{{max width=\textwidth}}123\begin{{tabular}}{{lR{{1.2cm}}R{{1.2cm}}R{{1.2cm}}R{{1.2cm}}R{{1.2cm}}R{{1.4cm}}}}124\toprule125\textbf{{Reference}} & \textbf{{Mean}} & \textbf{{Std.~Dev.}} & \textbf{{Min}} & \textbf{{Median}} & \textbf{{Max}} & \textbf{{Corr.\ $\ln P$}} \\126\midrule127{body}128\bottomrule129\end{{tabular}}130\end{{adjustbox}}131\begin{{tablenotes}}132\footnotesize133\item \textit{{Notes:}} Cosine similarity is computed between each listing embedding and the corresponding reference description embedding using the all-MiniLM-L6-v2 sentence transformer. Corr.\ $\ln P$ denotes the Pearson correlation with log listing price.134\end{{tablenotes}}135\end{{threeparttable}}136\end{{table}}137"""138    write("tab_similarity_stats.tex", content)139140141def tab_model_comparison():142    m = pd.read_csv(config.RESULTS_DIR / "model_comparison.csv")143    spec_labels = {144        "A": "Structural only",145        "B": "+ Description length",146        "C": "+ Semantic similarities (20)",147        "D": "Full model (B + C)",148        "E": "Parsimonious (sig.\\ only)",149    }150    best_r2 = m["r2"].idxmax()151    best_adj = m["adj_r2"].idxmax()152    best_aic = m["aic"].idxmin()153    best_bic = m["bic"].idxmin()154    lines = []155    for i, r in m.iterrows():156        def fmt(val, dec, best):157            s = tex_num(val, dec)158            return rf"\textbf{{{s}}}" if i == best else s159        delta = "---" if r["model"] == "A" else tex_num(r["delta_r2_vs_A"], 3, signed=True)160        lines.append(161            f"{r['model']} & {spec_labels[r['model']]} & "162            f"{fmt(r['r2'], 4, best_r2)} & {fmt(r['adj_r2'], 4, best_adj)} & "163            f"{fmt(r['aic'], 0, best_aic)} & {fmt(r['bic'], 0, best_bic)} & "164            f"{int(r['k'])} & {delta} \\\\"165        )166    body = "\n".join(lines)167    content = rf"""\begin{{table}}[!htbp]168\centering169\caption{{Hedonic model comparison ($n = 17{{,}}087$).}}170\label{{tab:model_comparison}}171\begin{{threeparttable}}172\small173\begin{{adjustbox}}{{max width=\textwidth}}174\begin{{tabular}}{{clR{{1.3cm}}R{{1.3cm}}R{{1.6cm}}R{{1.6cm}}R{{0.8cm}}R{{1.8cm}}}}175\toprule176& \textbf{{Specification}} & $\boldsymbol{{R^2}}$ & \textbf{{Adj.}}~$\boldsymbol{{R^2}}$ & \textbf{{AIC}} & \textbf{{BIC}} & $\boldsymbol{{k}}$ & $\boldsymbol{{\Delta R^2}}$ \textbf{{vs.\ A}} \\177\midrule178{body}179\bottomrule180\end{{tabular}}181\end{{adjustbox}}182\begin{{tablenotes}}183\footnotesize184\item \textit{{Notes:}} $k$ denotes the number of regressors excluding the intercept. Bold values indicate the best fit for each criterion. Models are estimated by OLS with HC3 robust standard errors.185\end{{tablenotes}}186\end{{threeparttable}}187\end{{table}}188"""189    write("tab_model_comparison.tex", content)190191192def _coef_rows(d, variables):193    rows = []194    for _, r in d[d.variable.isin(variables)].iterrows():195        rows.append((r["label"] if r["variable"].startswith("sim_") else None,196                     r["variable"], r["coefficient"], r["std_error"],197                     r["t_value"], r["p_value"], r["impact_pct"]))198    return rows199200201STRUCT_LABELS = {202    "bedrooms": "Bedrooms", "bathrooms": "Bathrooms",203    "half_baths": "Half-bathrooms", "parking": "Parking",204    "stories": "Stories", "land_size": "Lot size",205    "remarks_length": "Description length",206}207208209def tab_full_results():210    d = pd.read_csv(config.RESULTS_DIR / "coefficients_model_D.csv")211    struct = d[~d.variable.str.startswith("sim_")].copy()212    struct["label"] = struct.variable.map(STRUCT_LABELS)213    sims = d[d.variable.str.startswith("sim_")].copy()214    pos = sims[sims.coefficient >= 0].sort_values("coefficient", ascending=False)215    neg = sims[sims.coefficient < 0].sort_values("coefficient")216    struct = struct.sort_values("coefficient", ascending=False)217218    def rows(block):219        out = []220        for _, r in block.iterrows():221            label = str(r["label"]).replace("&", "\\&")222            out.append(223                f"\\quad {label} & {tex_num(r['coefficient'], 4)} & "224                f"{tex_num(r['std_error'], 4)} & {tex_num(r['t_value'], 2)} & "225                f"{pval_str(r['p_value'])} & {tex_num(r['impact_pct'], 1, signed=True)} & "226                f"{stars(r['p_value'])} \\\\"227            )228        return "\n".join(out)229230    content = rf"""\begin{{table}}[!htbp]231\centering232\caption{{Full model (D) coefficient estimates. HC3 robust standard errors. All variables standardised.}}233\label{{tab:full_results}}234\small235\begin{{adjustbox}}{{max width=\textwidth}}236\begin{{tabular}}{{lR{{1.2cm}}R{{1.2cm}}R{{1.1cm}}R{{1.6cm}}R{{1.3cm}}c}}237\toprule238\textbf{{Variable}} & \textbf{{Coeff.}} & \textbf{{Std.\ Err.}} & \textbf{{$t$-stat}} & \textbf{{$p$-value}} & \textbf{{Impact (\%)}} & \\239\midrule240\multicolumn{{7}}{{l}}{{\textit{{Panel A: Structural variables}}}} \\241{rows(struct)}242\midrule243\multicolumn{{7}}{{l}}{{\textit{{Panel B: Semantic similarities --- positive price effects}}}} \\244{rows(pos)}245\midrule246\multicolumn{{7}}{{l}}{{\textit{{Panel C: Semantic similarities --- negative price effects}}}} \\247{rows(neg)}248\bottomrule249\multicolumn{{7}}{{l}}{{\footnotesize{{\signote\ Impact $= (e^{{\hat{{\beta}}}} - 1) \times 100$\%. $n = 17{{,}}087$; Adj.\ $R^2 = 0.511$.}}}}250\end{{tabular}}251\end{{adjustbox}}252\end{{table}}253"""254    write("tab_full_results.tex", content)255256257def tab_quantile():258    r = pd.read_csv(config.RESULTS_DIR / "robustness_results.csv")259    d = pd.read_csv(config.RESULTS_DIR / "coefficients_model_D.csv").set_index("variable")260    dims = ["sim_moderne_contemporain", "sim_luxe", "sim_terrain_nature",261            "sim_familial", "sim_urgence_motivation", "sim_a_renover"]262    patterns = {263        "sim_moderne_contemporain": "High at both tails",264        "sim_luxe": "Increasing",265        "sim_terrain_nature": "Stable",266        "sim_familial": "Mildly attenuating",267        "sim_urgence_motivation": "Attenuating",268        "sim_a_renover": "Attenuating",269    }270    lines = []271    for v in dims:272        label = ENGLISH_LABELS[v[4:]].replace("&", "\\&")273        cells = []274        for tau in ("0.25", "0.5", "0.75"):275            q = r[(r.test == f"QuantReg_tau{tau}") & (r.variable == v)].iloc[0]276            cells.append(f"{tex_num(q['value'], 3)}{stars(q['p_value'])}")277        ols = d.loc[v]278        cells.append(f"{tex_num(ols['coefficient'], 3)}{stars(ols['p_value'])}")279        lines.append(f"{label} & " + " & ".join(cells) + f" & {patterns[v]} \\\\")280    body = "\n".join(lines)281    content = rf"""\begin{{table}}[!htbp]282\centering283\caption{{Quantile regression coefficients for selected semantic dimensions.}}284\label{{tab:quantile}}285\small286\begin{{adjustbox}}{{max width=\textwidth}}287\begin{{tabular}}{{lR{{1.5cm}}R{{1.5cm}}R{{1.5cm}}R{{1.5cm}}l}}288\toprule289\textbf{{Dimension}} & $\boldsymbol{{\tau = 0.25}}$ & $\boldsymbol{{\tau = 0.50}}$ & $\boldsymbol{{\tau = 0.75}}$ & \textbf{{OLS}} & \textbf{{Pattern}} \\290\midrule291{body}292\bottomrule293\multicolumn{{6}}{{l}}{{\footnotesize{{\signote\ Quantile regressions of Model~D; standard errors follow the kernel-based estimator of \citet{{koenker1978regression}}.}}}}294\end{{tabular}}295\end{{adjustbox}}296\end{{table}}297"""298    write("tab_quantile.tex", content)299300301def tab_parsimonious():302    d = pd.read_csv(config.RESULTS_DIR / "coefficients_model_E.csv")303    struct = d[~d.variable.str.startswith("sim_")].copy()304    struct["label"] = struct.variable.map(STRUCT_LABELS)305    sims = d[d.variable.str.startswith("sim_")].sort_values("coefficient", ascending=False)306307    def rows(block):308        out = []309        for _, r in block.iterrows():310            label = str(r["label"]).replace("&", "\\&")311            out.append(312                f"\\quad {label} & {tex_num(r['coefficient'], 4)} & "313                f"{tex_num(r['std_error'], 4)} & {pval_str(r['p_value'])} & "314                f"{tex_num(r['impact_pct'], 1, signed=True)} & {stars(r['p_value'])} \\\\"315            )316        return "\n".join(out)317318    content = rf"""\begin{{table}}[!htbp]319\centering320\caption{{Parsimonious model (E) coefficient estimates. HC3 robust standard errors. All variables standardised.}}321\label{{tab:parsimonious}}322\small323\begin{{adjustbox}}{{max width=\textwidth}}324\begin{{tabular}}{{lR{{1.3cm}}R{{1.3cm}}R{{1.6cm}}R{{1.4cm}}c}}325\toprule326\textbf{{Variable}} & \textbf{{Coeff.}} & \textbf{{Std.\ Err.}} & \textbf{{$p$-value}} & \textbf{{Impact (\%)}} & \\327\midrule328\multicolumn{{6}}{{l}}{{\textit{{Panel A: Structural variables}}}} \\329{rows(struct.sort_values('coefficient', ascending=False))}330\midrule331\multicolumn{{6}}{{l}}{{\textit{{Panel B: Semantic similarities (significant at 5\% in Model D)}}}} \\332{rows(sims)}333\bottomrule334\multicolumn{{6}}{{l}}{{\footnotesize{{\signote\ Impact $= (e^{{\hat{{\beta}}}} - 1) \times 100$\%. $n = 17{{,}}087$; Adj.\ $R^2 = 0.511$.}}}}335\end{{tabular}}336\end{{adjustbox}}337\end{{table}}338"""339    write("tab_parsimonious.tex", content)340341342def tab_reference_texts():343    lines = []344    for slug, text in REFERENCES.items():345        label = ENGLISH_LABELS[slug].replace("&", "\\&")346        text_tex = text.replace("&", "\\&").replace("%", "\\%")347        lines.append(f"{label} & \\textit{{{text_tex}}} \\\\[2pt]")348    body = "\n".join(lines)349    content = rf"""\begin{{footnotesize}}350\begin{{longtable}}{{p{{3.2cm}}p{{11.5cm}}}}351\caption{{The 20 reference descriptions (verbatim French text used for embedding).}}352\label{{tab:reference_texts}} \\353\toprule354\textbf{{Dimension}} & \textbf{{Reference description}} \\355\midrule356\endfirsthead357\toprule358\textbf{{Dimension}} & \textbf{{Reference description}} \\359\midrule360\endhead361\bottomrule362\endfoot363{body}364\end{{longtable}}365\end{{footnotesize}}366"""367    write("tab_reference_texts.tex", content)368369370def main():371    TABLES_DIR.mkdir(parents=True, exist_ok=True)372    tab_descriptive()373    tab_similarity_stats()374    tab_model_comparison()375    tab_full_results()376    tab_quantile()377    tab_parsimonious()378    tab_reference_texts()379    print("All tables generated.")380381382if __name__ == "__main__":383    main()384