spb/wp2_uqo Public

1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — contact@spboucher.ai3#4"""Step 6 — Generate the paper's LaTeX tables from the pipeline results.56Every number printed in these tables comes from results/*.csv (steps 3-4),7so the paper cannot drift from the analysis outputs.89Inputs : results/*.csv, data/processed/hedonic_maison_results.csv10Outputs: paper/tables/tab_descriptive.tex11         paper/tables/tab_similarity_stats.tex12         paper/tables/tab_model_comparison.tex13         paper/tables/tab_full_results.tex14         paper/tables/tab_quantile.tex15         paper/tables/tab_parsimonious.tex      (appendix)16         paper/tables/tab_reference_texts.tex   (appendix)17"""1819import sys20from pathlib import Path2122import numpy as np23import pandas as pd2425sys.path.insert(0, str(Path(__file__).resolve().parents[1]))2627from src import config28from src.references import ENGLISH_LABELS, REFERENCES2930TABLES_DIR = config.ROOT / "paper" / "tables"3132HEADER = "% Author: Simon-Pierre Boucher — contact@spboucher.ai\n% Generated by scripts/06_tables.py — do not edit by hand.\n"333435def tex_num(x, dec=0, signed=False):36    """Format a number with LaTeX minus signs and {,} thousand separators."""37    if signed:38        s = f"{x:+,.{dec}f}"39    else:40        s = f"{x:,.{dec}f}"41    s = s.replace(",", "{,}").replace("-", r"$-$").replace("+", "+")42    return s434445def stars(p):46    return "***" if p < 0.001 else "**" if p < 0.01 else "*" if p < 0.05 else ""474849def pval_str(p):50    return "$< 0.001$" if p < 0.001 else f"{p:.3f}"515253def write(name, content):54    path = TABLES_DIR / name55    path.write_text(HEADER + content)56    print(f"  -> {path}")575859def tab_descriptive():60    d = pd.read_csv(config.RESULTS_DIR / "descriptive_stats.csv", index_col=0)61    rows_spec = [62        ("price", "Price (\\$)", 0),63        ("log_price", "log(Price)", 2),64        ("bedrooms", "Bedrooms", 1),65        ("bathrooms", "Bathrooms", 1),66        ("half_baths", "Half-bathrooms", 1),67        ("parking", "Parking spaces", 1),68        ("stories", "Stories", 1),69        ("land_size", "Lot size (sq.\\ ft.)", 0),70        ("remarks_length", "Description length (char.)", 0),71    ]72    lines = []73    for var, label, dec in rows_spec:74        r = d.loc[var]75        cells = [tex_num(r["mean"], dec), tex_num(r["std"], dec),76                 tex_num(r["min"], 0 if var != "log_price" else 2),77                 tex_num(r["25%"], 0 if var != "log_price" else 2),78                 tex_num(r["50%"], 0 if var != "log_price" else 2),79                 tex_num(r["75%"], 0 if var != "log_price" else 2),80                 tex_num(r["max"], 0 if var != "log_price" else 2)]81        lines.append(f"{label} & " + " & ".join(cells) + " \\\\")82    body = "\n".join(lines)83    content = rf"""\begin{{table}}[!htbp]84\centering85\caption{{Descriptive statistics for structural and textual variables ($n = 17{{,}}087$).}}86\label{{tab:descriptive}}87\begin{{threeparttable}}88\small89\begin{{adjustbox}}{{max width=\textwidth}}90\begin{{tabular}}{{lR{{1.6cm}}R{{1.7cm}}R{{1.2cm}}R{{1.4cm}}R{{1.4cm}}R{{1.4cm}}R{{1.9cm}}}}91\toprule92\textbf{{Variable}} & \textbf{{Mean}} & \textbf{{Std.\ Dev.}} & \textbf{{Min}} & \textbf{{P25}} & \textbf{{Median}} & \textbf{{P75}} & \textbf{{Max}} \\93\midrule94{body}95\bottomrule96\end{{tabular}}97\end{{adjustbox}}98\begin{{tablenotes}}99\footnotesize100\item \textit{{Notes:}} The sample is restricted to single-family houses with a positive listing price and a description of at least 20 characters. Descriptions are truncated at approximately 700 characters in the data export, which bounds the description-length variable. The lot-size field is noisy (units are not standardized at the source), which motivates its cautious interpretation in the regressions.101\end{{tablenotes}}102\end{{threeparttable}}103\end{{table}}104"""105    write("tab_descriptive.tex", content)106107108def tab_similarity_stats():109    s = pd.read_csv(config.RESULTS_DIR / "similarity_stats.csv", index_col=0)110    lines = []111    for label, r in s.iterrows():112        label_tex = label.replace("&", "\\&")113        cells = [f"{r['mean']:.3f}", f"{r['std']:.3f}",114                 tex_num(r["min"], 3), f"{r['50%']:.3f}", f"{r['max']:.3f}",115                 tex_num(r["corr_log_price"], 3)]116        lines.append(f"{label_tex} & " + " & ".join(cells) + " \\\\")117    body = "\n".join(lines)118    content = rf"""\begin{{table}}[!htbp]119\centering120\caption{{Descriptive statistics for cosine similarity features and bivariate correlations with log(price).}}121\label{{tab:sim_stats}}122\begin{{threeparttable}}123\small124\begin{{adjustbox}}{{max width=\textwidth}}125\begin{{tabular}}{{lR{{1.2cm}}R{{1.2cm}}R{{1.2cm}}R{{1.2cm}}R{{1.2cm}}R{{1.4cm}}}}126\toprule127\textbf{{Reference}} & \textbf{{Mean}} & \textbf{{Std.~Dev.}} & \textbf{{Min}} & \textbf{{Median}} & \textbf{{Max}} & \textbf{{Corr.\ $\ln P$}} \\128\midrule129{body}130\bottomrule131\end{{tabular}}132\end{{adjustbox}}133\begin{{tablenotes}}134\footnotesize135\item \textit{{Notes:}} Cosine similarity is computed between each listing embedding and the corresponding reference description embedding using the all-MiniLM-L6-v2 sentence transformer. Corr.\ $\ln P$ denotes the Pearson correlation with log listing price.136\end{{tablenotes}}137\end{{threeparttable}}138\end{{table}}139"""140    write("tab_similarity_stats.tex", content)141142143def tab_model_comparison():144    m = pd.read_csv(config.RESULTS_DIR / "model_comparison.csv")145    spec_labels = {146        "A": "Structural only",147        "B": "+ Description length",148        "C": "+ Semantic similarities (20)",149        "D": "Full model (B + C)",150        "E": "Parsimonious (sig.\\ only)",151    }152    best_r2 = m["r2"].idxmax()153    best_adj = m["adj_r2"].idxmax()154    best_aic = m["aic"].idxmin()155    best_bic = m["bic"].idxmin()156    lines = []157    for i, r in m.iterrows():158        def fmt(val, dec, best):159            s = tex_num(val, dec)160            return rf"\textbf{{{s}}}" if i == best else s161        delta = "---" if r["model"] == "A" else tex_num(r["delta_r2_vs_A"], 3, signed=True)162        lines.append(163            f"{r['model']} & {spec_labels[r['model']]} & "164            f"{fmt(r['r2'], 4, best_r2)} & {fmt(r['adj_r2'], 4, best_adj)} & "165            f"{fmt(r['aic'], 0, best_aic)} & {fmt(r['bic'], 0, best_bic)} & "166            f"{int(r['k'])} & {delta} \\\\"167        )168    body = "\n".join(lines)169    content = rf"""\begin{{table}}[!htbp]170\centering171\caption{{Hedonic model comparison ($n = 17{{,}}087$).}}172\label{{tab:model_comparison}}173\begin{{threeparttable}}174\small175\begin{{adjustbox}}{{max width=\textwidth}}176\begin{{tabular}}{{clR{{1.3cm}}R{{1.3cm}}R{{1.6cm}}R{{1.6cm}}R{{0.8cm}}R{{1.8cm}}}}177\toprule178& \textbf{{Specification}} & $\boldsymbol{{R^2}}$ & \textbf{{Adj.}}~$\boldsymbol{{R^2}}$ & \textbf{{AIC}} & \textbf{{BIC}} & $\boldsymbol{{k}}$ & $\boldsymbol{{\Delta R^2}}$ \textbf{{vs.\ A}} \\179\midrule180{body}181\bottomrule182\end{{tabular}}183\end{{adjustbox}}184\begin{{tablenotes}}185\footnotesize186\item \textit{{Notes:}} $k$ denotes the number of regressors excluding the intercept. Bold values indicate the best fit for each criterion. Models are estimated by OLS with HC3 robust standard errors.187\end{{tablenotes}}188\end{{threeparttable}}189\end{{table}}190"""191    write("tab_model_comparison.tex", content)192193194def _coef_rows(d, variables):195    rows = []196    for _, r in d[d.variable.isin(variables)].iterrows():197        rows.append((r["label"] if r["variable"].startswith("sim_") else None,198                     r["variable"], r["coefficient"], r["std_error"],199                     r["t_value"], r["p_value"], r["impact_pct"]))200    return rows201202203STRUCT_LABELS = {204    "bedrooms": "Bedrooms", "bathrooms": "Bathrooms",205    "half_baths": "Half-bathrooms", "parking": "Parking",206    "stories": "Stories", "land_size": "Lot size",207    "remarks_length": "Description length",208}209210211def tab_full_results():212    d = pd.read_csv(config.RESULTS_DIR / "coefficients_model_D.csv")213    struct = d[~d.variable.str.startswith("sim_")].copy()214    struct["label"] = struct.variable.map(STRUCT_LABELS)215    sims = d[d.variable.str.startswith("sim_")].copy()216    pos = sims[sims.coefficient >= 0].sort_values("coefficient", ascending=False)217    neg = sims[sims.coefficient < 0].sort_values("coefficient")218    struct = struct.sort_values("coefficient", ascending=False)219220    def rows(block):221        out = []222        for _, r in block.iterrows():223            label = str(r["label"]).replace("&", "\\&")224            out.append(225                f"\\quad {label} & {tex_num(r['coefficient'], 4)} & "226                f"{tex_num(r['std_error'], 4)} & {tex_num(r['t_value'], 2)} & "227                f"{pval_str(r['p_value'])} & {tex_num(r['impact_pct'], 1, signed=True)} & "228                f"{stars(r['p_value'])} \\\\"229            )230        return "\n".join(out)231232    content = rf"""\begin{{table}}[!htbp]233\centering234\caption{{Full model (D) coefficient estimates. HC3 robust standard errors. All variables standardized.}}235\label{{tab:full_results}}236\small237\begin{{adjustbox}}{{max width=\textwidth}}238\begin{{tabular}}{{lR{{1.2cm}}R{{1.2cm}}R{{1.1cm}}R{{1.6cm}}R{{1.3cm}}c}}239\toprule240\textbf{{Variable}} & \textbf{{Coeff.}} & \textbf{{Std.\ Err.}} & \textbf{{$t$-stat}} & \textbf{{$p$-value}} & \textbf{{Impact (\%)}} & \\241\midrule242\multicolumn{{7}}{{l}}{{\textit{{Panel A: Structural variables}}}} \\243{rows(struct)}244\midrule245\multicolumn{{7}}{{l}}{{\textit{{Panel B: Semantic similarities --- positive price effects}}}} \\246{rows(pos)}247\midrule248\multicolumn{{7}}{{l}}{{\textit{{Panel C: Semantic similarities --- negative price effects}}}} \\249{rows(neg)}250\bottomrule251\multicolumn{{7}}{{l}}{{\footnotesize{{\signote\ Impact $= (e^{{\hat{{\beta}}}} - 1) \times 100$\%. $n = 17{{,}}087$; Adj.\ $R^2 = 0.511$.}}}}252\end{{tabular}}253\end{{adjustbox}}254\end{{table}}255"""256    write("tab_full_results.tex", content)257258259def tab_quantile():260    r = pd.read_csv(config.RESULTS_DIR / "robustness_results.csv")261    d = pd.read_csv(config.RESULTS_DIR / "coefficients_model_D.csv").set_index("variable")262    dims = ["sim_moderne_contemporain", "sim_luxe", "sim_terrain_nature",263            "sim_familial", "sim_urgence_motivation", "sim_a_renover"]264    patterns = {265        "sim_moderne_contemporain": "High at both tails",266        "sim_luxe": "Increasing",267        "sim_terrain_nature": "Stable",268        "sim_familial": "Mildly attenuating",269        "sim_urgence_motivation": "Attenuating",270        "sim_a_renover": "Attenuating",271    }272    lines = []273    for v in dims:274        label = ENGLISH_LABELS[v[4:]].replace("&", "\\&")275        cells = []276        for tau in ("0.25", "0.5", "0.75"):277            q = r[(r.test == f"QuantReg_tau{tau}") & (r.variable == v)].iloc[0]278            cells.append(f"{tex_num(q['value'], 3)}{stars(q['p_value'])}")279        ols = d.loc[v]280        cells.append(f"{tex_num(ols['coefficient'], 3)}{stars(ols['p_value'])}")281        lines.append(f"{label} & " + " & ".join(cells) + f" & {patterns[v]} \\\\")282    body = "\n".join(lines)283    content = rf"""\begin{{table}}[!htbp]284\centering285\caption{{Quantile regression coefficients for selected semantic dimensions.}}286\label{{tab:quantile}}287\small288\begin{{adjustbox}}{{max width=\textwidth}}289\begin{{tabular}}{{lR{{1.5cm}}R{{1.5cm}}R{{1.5cm}}R{{1.5cm}}l}}290\toprule291\textbf{{Dimension}} & $\boldsymbol{{\tau = 0.25}}$ & $\boldsymbol{{\tau = 0.50}}$ & $\boldsymbol{{\tau = 0.75}}$ & \textbf{{OLS}} & \textbf{{Pattern}} \\292\midrule293{body}294\bottomrule295\multicolumn{{6}}{{l}}{{\footnotesize{{\signote\ Quantile regressions of Model~D; standard errors follow the kernel-based estimator of \citet{{koenker1978regression}}.}}}}296\end{{tabular}}297\end{{adjustbox}}298\end{{table}}299"""300    write("tab_quantile.tex", content)301302303def tab_parsimonious():304    d = pd.read_csv(config.RESULTS_DIR / "coefficients_model_E.csv")305    struct = d[~d.variable.str.startswith("sim_")].copy()306    struct["label"] = struct.variable.map(STRUCT_LABELS)307    sims = d[d.variable.str.startswith("sim_")].sort_values("coefficient", ascending=False)308309    def rows(block):310        out = []311        for _, r in block.iterrows():312            label = str(r["label"]).replace("&", "\\&")313            out.append(314                f"\\quad {label} & {tex_num(r['coefficient'], 4)} & "315                f"{tex_num(r['std_error'], 4)} & {pval_str(r['p_value'])} & "316                f"{tex_num(r['impact_pct'], 1, signed=True)} & {stars(r['p_value'])} \\\\"317            )318        return "\n".join(out)319320    content = rf"""\begin{{table}}[!htbp]321\centering322\caption{{Parsimonious model (E) coefficient estimates. HC3 robust standard errors. All variables standardized.}}323\label{{tab:parsimonious}}324\small325\begin{{adjustbox}}{{max width=\textwidth}}326\begin{{tabular}}{{lR{{1.3cm}}R{{1.3cm}}R{{1.6cm}}R{{1.4cm}}c}}327\toprule328\textbf{{Variable}} & \textbf{{Coeff.}} & \textbf{{Std.\ Err.}} & \textbf{{$p$-value}} & \textbf{{Impact (\%)}} & \\329\midrule330\multicolumn{{6}}{{l}}{{\textit{{Panel A: Structural variables}}}} \\331{rows(struct.sort_values('coefficient', ascending=False))}332\midrule333\multicolumn{{6}}{{l}}{{\textit{{Panel B: Semantic similarities (significant at 5\% in Model D)}}}} \\334{rows(sims)}335\bottomrule336\multicolumn{{6}}{{l}}{{\footnotesize{{\signote\ Impact $= (e^{{\hat{{\beta}}}} - 1) \times 100$\%. $n = 17{{,}}087$; Adj.\ $R^2 = 0.511$.}}}}337\end{{tabular}}338\end{{adjustbox}}339\end{{table}}340"""341    write("tab_parsimonious.tex", content)342343344def tab_reference_texts():345    lines = []346    for slug, text in REFERENCES.items():347        label = ENGLISH_LABELS[slug].replace("&", "\\&")348        text_tex = text.replace("&", "\\&").replace("%", "\\%")349        lines.append(f"{label} & \\textit{{{text_tex}}} \\\\[2pt]")350    body = "\n".join(lines)351    content = rf"""\begin{{footnotesize}}352\begin{{longtable}}{{p{{3.2cm}}p{{11.5cm}}}}353\caption{{The 20 reference descriptions (verbatim French text used for embedding).}}354\label{{tab:reference_texts}} \\355\toprule356\textbf{{Dimension}} & \textbf{{Reference description}} \\357\midrule358\endfirsthead359\toprule360\textbf{{Dimension}} & \textbf{{Reference description}} \\361\midrule362\endhead363\bottomrule364\endfoot365{body}366\end{{longtable}}367\end{{footnotesize}}368"""369    write("tab_reference_texts.tex", content)370371372def main():373    TABLES_DIR.mkdir(parents=True, exist_ok=True)374    tab_descriptive()375    tab_similarity_stats()376    tab_model_comparison()377    tab_full_results()378    tab_quantile()379    tab_parsimonious()380    tab_reference_texts()381    print("All tables generated.")382383384if __name__ == "__main__":385    main()386