#!/usr/bin/env python3 # Author: Simon-Pierre Boucher — contact@spboucher.ai """Step 05 — Generate every LaTeX table used in the paper (6 files). Tables are built from the results tier selected with ``--results``; the default ``reference`` tier reproduces the published numbers exactly. The summary-statistics table additionally needs the micro sample when the ``reproduced`` tier is selected. Usage: python scripts/05_make_tables.py [--results reference|reproduced] """ import argparse import json import sys from pathlib import Path import pandas as pd sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from wp9 import sample # noqa: E402 from wp9.config import REFERENCE, REPRODUCED, TABLES, ensure_dirs # noqa: E402 SUMMARY_VARS = [("price_cad", "List price (CAD)", "{:,.0f}"), ("ppm2", "Price per m$^2$ (CAD)", "{:,.0f}"), ("living_m2", "Living area (m$^2$)", "{:,.1f}"), ("bedrooms", "Bedrooms", "{:.2f}"), ("bathrooms", "Full bathrooms", "{:.2f}"), ("half_baths", "Half bathrooms", "{:.2f}"), ("parking_n", "Parking spaces", "{:.2f}"), ("stories_n", "Storeys", "{:.2f}"), ("lot_m2_f", "Lot area (m$^2$)", "{:,.0f}")] COEF_ORDER = [("ln_living", "$\\ln$ living area"), ("bathrooms", "Full bathrooms"), ("half_baths", "Half bathrooms"), ("bedrooms", "Bedrooms"), ("parking_n", "Parking spaces"), ("stories_n", "Storeys"), ("has_lot", "Has lot info"), ("ln_lot", "$\\ln(1+$lot m$^2)$")] def write(name: str, content: str) -> None: (TABLES / name).write_text(content + "\n") print(" tab", name) def stars(p: float) -> str: return "$^{***}$" if p < 0.01 else "$^{**}$" if p < 0.05 else "$^{*}$" if p < 0.1 else "" def cell(coefs: pd.DataFrame, key: str) -> str: if key not in coefs.index: return "" b, se, p = coefs.loc[key, ["coef", "se", "p"]] return f"\\makecell{{{b:.3f}{stars(p)}\\\\\\scriptsize({se:.3f})}}" # -------------------------------------------------------------------------- def table_summary_stats(res_dir: Path) -> None: stats_file = res_dir / "summary_stats.csv" if stats_file.exists(): stats = pd.read_csv(stats_file, index_col=0) n = int(stats.attrs.get("n", 0)) or int(stats["n"].iloc[0]) else: # reproduced tier: compute from the micro sample s = sample.load_sample() rows = {} for var, _, _ in SUMMARY_VARS: col = s[var] rows[var] = {"mean": col.mean(), "sd": col.std(), "p25": col.quantile(0.25), "median": col.median(), "p75": col.quantile(0.75), "n": len(s)} stats = pd.DataFrame(rows).T stats.to_csv(res_dir / "summary_stats.csv") n = len(s) lines = [] for var, label, fmt in SUMMARY_VARS: r = stats.loc[var] lines.append(f"{label} & {fmt.format(r['mean'])} & {fmt.format(r['sd'])} & " f"{fmt.format(r['p25'])} & {fmt.format(r['median'])} & " f"{fmt.format(r['p75'])} \\\\") write("summary_stats.tex", "\\begin{table}[t]\\centering\n\\caption{Descriptive statistics}\n" "\\label{tab:summary_stats}\n\\begin{threeparttable}\n" "\\begin{tabular}{lccccc}\n\\toprule\n" "Variable & Mean & SD & P25 & Median & P75 \\\\\n\\midrule\n" + "\n".join(lines) + "\n\\bottomrule\n\\end{tabular}\n\\begin{tablenotes}\\footnotesize\\item " f"Notes: Estimation sample of {n:,} residential dwellings (houses, " "condominiums, plexes, townhouses and apartments) from the Canadian MLS, " "after parsing and trimming the extreme 1\\% tails of price and living area.\n" "\\end{tablenotes}\n\\end{threeparttable}\n\\end{table}") # -------------------------------------------------------------------------- def table_regression(res_dir: Path) -> None: fit = json.load(open(res_dir / "fit.json")) coefs = {m: pd.read_csv(res_dir / f"coef_{m}.csv", index_col=0) for m in ("M1", "M2", "M3", "M5")} lines = [label + " & " + " & ".join(cell(coefs[m], key) for m in ("M1", "M2", "M3", "M5")) + " \\\\" for key, label in COEF_ORDER] footer = ["\\midrule", "Dwelling-type FE & No & Yes & Yes & Yes \\\\", "Ownership FE & No & Yes & Yes & Yes \\\\", "Province FE & No & No & Yes & --- \\\\", "FSA fixed effects & No & No & No & Yes \\\\", f"$R^2$ & {fit['M1']['r2']:.3f} & {fit['M2']['r2']:.3f} & " f"{fit['M3']['r2']:.3f} & {fit['M5']['r2']:.3f} \\\\", f"Observations & {fit['M1']['n']:,} & {fit['M2']['n']:,} & " f"{fit['M3']['n']:,} & {fit['M5']['n']:,} \\\\"] write("regression.tex", "\\begin{table}[t]\\centering\n\\caption{Hedonic regression estimates}\n" "\\label{tab:regression}\n\\begin{threeparttable}\n" "\\begin{tabular}{lcccc}\n\\toprule\n" " & (1) & (2) & (3) & (4) \\\\\n" " & Structural & +Type/Own. & +Province & Grand+FSA \\\\\n\\midrule\n" + "\n".join(lines + footer) + "\n\\bottomrule\n\\end{tabular}\n\\begin{tablenotes}\\footnotesize\\item " "Notes: Dependent variable is $\\ln(\\text{price})$. Columns (1)--(3) use the " "house subsample; column (4) is the grand model over all residential dwellings " "with 1{,}153 absorbed FSA fixed effects. Cluster-robust standard errors " "(by FSA) in parentheses. $^{*}p<0.1$, $^{**}p<0.05$, $^{***}p<0.01$.\n" "\\end{tablenotes}\n\\end{threeparttable}\n\\end{table}") # -------------------------------------------------------------------------- def table_robustness(res_dir: Path) -> None: rob = pd.read_csv(res_dir / "robustness.csv") lines = [] for _, r in rob.iterrows(): lines.append( f"{r['label']} & {r['n']:,.0f} & {r['r2']:.3f} & " f"\\makecell{{{r['ln_living']:.3f}\\\\\\scriptsize({r['se_living']:.3f})}} & " f"\\makecell{{{r['bathrooms']:.3f}\\\\\\scriptsize({r['se_bath']:.3f})}} & " f"\\makecell{{{r['ln_lot']:.3f}\\\\\\scriptsize({r['se_lot']:.3f})}} \\\\") write("robustness.tex", "\\begin{table}[t]\\centering\n" "\\caption{Robustness of key implicit prices across samples}\n" "\\label{tab:robustness}\n\\begin{threeparttable}\n" "\\begin{tabular}{lccccc}\n\\toprule\n" "Sample / specification & $N$ & $R^2$ & $\\ln$ area & Full bath & $\\ln$ lot \\\\\n" "\\midrule\n" + "\n".join(lines) + "\n\\bottomrule\n\\end{tabular}\n\\begin{tablenotes}\\footnotesize\\item " "Notes: Each row re-estimates the grand FSA-fixed-effects model on a different " "sample. Cluster-robust (FSA) standard errors in parentheses. The size elasticity " "and bathroom premium are stable across all cuts.\n" "\\end{tablenotes}\n\\end{threeparttable}\n\\end{table}") # -------------------------------------------------------------------------- def table_quantile(res_dir: Path) -> None: q = pd.read_csv(res_dir / "quantile.csv") taus = q[q["tau"].notna()] ols = q[q["tau"].isna()].iloc[0] lines = [] for _, r in taus.iterrows(): tau = f"{r['tau']:g}" lines.append( f"$\\tau={tau}$ & " f"\\makecell{{{r['ln_living']:.3f}\\\\\\scriptsize({r['se_living']:.3f})}} & " f"\\makecell{{{r['bathrooms']:.3f}\\\\\\scriptsize({r['se_bath']:.3f})}} & " f"\\makecell{{{r['ln_lot']:.3f}\\\\\\scriptsize({r['se_lot']:.3f})}} \\\\") write("quantile.tex", "\\begin{table}[t]\\centering" "\\caption{Quantile hedonic estimates across the price distribution}\n" "\\label{tab:quantile}\\begin{threeparttable}\\begin{tabular}{lccc}\n\\toprule\n" "Quantile & $\\ln$ area & Full bath & $\\ln$ lot \\\\\n\\midrule\n" + "\n".join(lines) + "\n\\midrule\nOLS & " f"{ols['ln_living']:.3f} & {ols['bathrooms']:.3f} & {ols['ln_lot']:.3f} \\\\\n" "\\bottomrule\\end{tabular}\\begin{tablenotes}\\footnotesize\\item Notes: House " "subsample, province fixed effects, structural controls. Analytical standard " "errors in parentheses. The size elasticity rises and the lot elasticity rises " "with price.\\end{tablenotes}\\end{threeparttable}\\end{table}") # -------------------------------------------------------------------------- def table_lopo(res_dir: Path) -> None: lopo = pd.read_csv(res_dir / "lopo.csv") lines = [f"{r['prov']} & {r['n']:,.0f} & {r['r2_within']:.3f} \\\\" for _, r in lopo.iterrows()] write("lopo.tex", "\\begin{table}[t]\\centering" "\\caption{Leave-one-province-out spatial cross-validation}\n" "\\label{tab:lopo}\\begin{threeparttable}\\begin{tabular}{lcc}\n\\toprule\n" "Held-out province & $N$ & Within-province $R^2$ \\\\\n\\midrule\n" + "\n".join(lines) + f"\n\\midrule\nMean & --- & {lopo['r2_within'].mean():.3f} \\\\\n" "\\bottomrule\\end{tabular}\\begin{tablenotes}\\footnotesize\\item Notes: The " "structural hedonic model is estimated on all provinces except one and used to " "predict the held-out province; a province-specific intercept is allowed (the " "price \\emph{level} is not identified out of sample), so the metric captures " "whether the \\emph{structural} implicit prices transfer across space." "\\end{tablenotes}\\end{threeparttable}\\end{table}") # -------------------------------------------------------------------------- def table_oos(res_dir: Path) -> None: oos = json.load(open(res_dir / "oos.json")) write("oos.tex", "\\begin{table}[t]\\centering\n" "\\caption{Out-of-sample valuation performance (80/20 split)}\n" "\\label{tab:oos}\n\\begin{threeparttable}\n\\begin{tabular}{lc}\n\\toprule\n" "Metric & Value \\\\\n\\midrule\n" f"Training listings & {oos['n_train']:,} \\\\\n" f"Test listings & {oos['n_test']:,} \\\\\n" f"Out-of-sample $R^2$ (log price) & {oos['oos_r2']:.3f} \\\\\n" f"RMSE (log points) & {oos['rmse_log']:.3f} \\\\\n" f"Median absolute \\% error & {oos['median_ape']:.1f}\\% \\\\\n" f"Mean absolute \\% error & {oos['mean_ape']:.1f}\\% \\\\\n" f"Share priced within $\\pm$10\\% & {oos['within10']:.1f}\\% \\\\\n" f"Share priced within $\\pm$20\\% & {oos['within20']:.1f}\\% \\\\\n" "\\bottomrule\n\\end{tabular}\n\\begin{tablenotes}\\footnotesize\\item Notes: " "Model trained on a random 80\\% of listings and scored on the held-out 20\\% " "(restricted to neighbourhoods observed in training). Prices back-transformed " "with Duan's smearing estimator.\n" "\\end{tablenotes}\n\\end{threeparttable}\n\\end{table}") def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--results", choices=["reference", "reproduced"], default="reference", help="results tier (default: reference — published numbers)") args = parser.parse_args() res_dir = REFERENCE if args.results == "reference" else REPRODUCED ensure_dirs() table_summary_stats(res_dir) table_regression(res_dir) table_robustness(res_dir) table_quantile(res_dir) table_lopo(res_dir) table_oos(res_dir) print("all tables written to", TABLES) if __name__ == "__main__": main()