spb/wp9_uqo Public

1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — contact@spboucher.ai3"""Step 05 — Generate every LaTeX table used in the paper (6 files).45Tables are built from the results tier selected with ``--results``; the6default ``reference`` tier reproduces the published numbers exactly.7The summary-statistics table additionally needs the micro sample when the8``reproduced`` tier is selected.910Usage:  python scripts/05_make_tables.py [--results reference|reproduced]11"""12import argparse13import json14import sys15from pathlib import Path1617import pandas as pd1819sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))2021from wp9 import sample  # noqa: E40222from wp9.config import REFERENCE, REPRODUCED, TABLES, ensure_dirs  # noqa: E4022324SUMMARY_VARS = [("price_cad", "List price (CAD)", "{:,.0f}"),25                ("ppm2", "Price per m$^2$ (CAD)", "{:,.0f}"),26                ("living_m2", "Living area (m$^2$)", "{:,.1f}"),27                ("bedrooms", "Bedrooms", "{:.2f}"),28                ("bathrooms", "Full bathrooms", "{:.2f}"),29                ("half_baths", "Half bathrooms", "{:.2f}"),30                ("parking_n", "Parking spaces", "{:.2f}"),31                ("stories_n", "Storeys", "{:.2f}"),32                ("lot_m2_f", "Lot area (m$^2$)", "{:,.0f}")]3334COEF_ORDER = [("ln_living", "$\\ln$ living area"), ("bathrooms", "Full bathrooms"),35              ("half_baths", "Half bathrooms"), ("bedrooms", "Bedrooms"),36              ("parking_n", "Parking spaces"), ("stories_n", "Storeys"),37              ("has_lot", "Has lot info"), ("ln_lot", "$\\ln(1+$lot m$^2)$")]383940def write(name: str, content: str) -> None:41    (TABLES / name).write_text(content + "\n")42    print(" tab", name)434445def stars(p: float) -> str:46    return "$^{***}$" if p < 0.01 else "$^{**}$" if p < 0.05 else "$^{*}$" if p < 0.1 else ""474849def cell(coefs: pd.DataFrame, key: str) -> str:50    if key not in coefs.index:51        return ""52    b, se, p = coefs.loc[key, ["coef", "se", "p"]]53    return f"\\makecell{{{b:.3f}{stars(p)}\\\\\\scriptsize({se:.3f})}}"545556# --------------------------------------------------------------------------57def table_summary_stats(res_dir: Path) -> None:58    stats_file = res_dir / "summary_stats.csv"59    if stats_file.exists():60        stats = pd.read_csv(stats_file, index_col=0)61        n = int(stats.attrs.get("n", 0)) or int(stats["n"].iloc[0])62    else:  # reproduced tier: compute from the micro sample63        s = sample.load_sample()64        rows = {}65        for var, _, _ in SUMMARY_VARS:66            col = s[var]67            rows[var] = {"mean": col.mean(), "sd": col.std(), "p25": col.quantile(0.25),68                         "median": col.median(), "p75": col.quantile(0.75), "n": len(s)}69        stats = pd.DataFrame(rows).T70        stats.to_csv(res_dir / "summary_stats.csv")71        n = len(s)72    lines = []73    for var, label, fmt in SUMMARY_VARS:74        r = stats.loc[var]75        lines.append(f"{label} & {fmt.format(r['mean'])} & {fmt.format(r['sd'])} & "76                     f"{fmt.format(r['p25'])} & {fmt.format(r['median'])} & "77                     f"{fmt.format(r['p75'])} \\\\")78    write("summary_stats.tex",79          "\\begin{table}[t]\\centering\n\\caption{Descriptive statistics}\n"80          "\\label{tab:summary_stats}\n\\begin{threeparttable}\n"81          "\\begin{tabular}{lccccc}\n\\toprule\n"82          "Variable & Mean & SD & P25 & Median & P75 \\\\\n\\midrule\n"83          + "\n".join(lines) +84          "\n\\bottomrule\n\\end{tabular}\n\\begin{tablenotes}\\footnotesize\\item "85          f"Notes: Estimation sample of {n:,} residential dwellings (houses, "86          "condominiums, plexes, townhouses and apartments) from the Canadian MLS, "87          "after parsing and trimming the extreme 1\\% tails of price and living area.\n"88          "\\end{tablenotes}\n\\end{threeparttable}\n\\end{table}")899091# --------------------------------------------------------------------------92def table_regression(res_dir: Path) -> None:93    fit = json.load(open(res_dir / "fit.json"))94    coefs = {m: pd.read_csv(res_dir / f"coef_{m}.csv", index_col=0)95             for m in ("M1", "M2", "M3", "M5")}96    lines = [label + " & " + " & ".join(cell(coefs[m], key) for m in ("M1", "M2", "M3", "M5"))97             + " \\\\" for key, label in COEF_ORDER]98    footer = ["\\midrule",99              "Dwelling-type FE & No & Yes & Yes & Yes \\\\",100              "Ownership FE & No & Yes & Yes & Yes \\\\",101              "Province FE & No & No & Yes & --- \\\\",102              "FSA fixed effects & No & No & No & Yes \\\\",103              f"$R^2$ & {fit['M1']['r2']:.3f} & {fit['M2']['r2']:.3f} & "104              f"{fit['M3']['r2']:.3f} & {fit['M5']['r2']:.3f} \\\\",105              f"Observations & {fit['M1']['n']:,} & {fit['M2']['n']:,} & "106              f"{fit['M3']['n']:,} & {fit['M5']['n']:,} \\\\"]107    write("regression.tex",108          "\\begin{table}[t]\\centering\n\\caption{Hedonic regression estimates}\n"109          "\\label{tab:regression}\n\\begin{threeparttable}\n"110          "\\begin{tabular}{lcccc}\n\\toprule\n"111          " & (1) & (2) & (3) & (4) \\\\\n"112          " & Structural & +Type/Own. & +Province & Grand+FSA \\\\\n\\midrule\n"113          + "\n".join(lines + footer) +114          "\n\\bottomrule\n\\end{tabular}\n\\begin{tablenotes}\\footnotesize\\item "115          "Notes: Dependent variable is $\\ln(\\text{price})$. Columns (1)--(3) use the "116          "house subsample; column (4) is the grand model over all residential dwellings "117          "with 1{,}153 absorbed FSA fixed effects. Cluster-robust standard errors "118          "(by FSA) in parentheses. $^{*}p<0.1$, $^{**}p<0.05$, $^{***}p<0.01$.\n"119          "\\end{tablenotes}\n\\end{threeparttable}\n\\end{table}")120121122# --------------------------------------------------------------------------123def table_robustness(res_dir: Path) -> None:124    rob = pd.read_csv(res_dir / "robustness.csv")125    lines = []126    for _, r in rob.iterrows():127        lines.append(128            f"{r['label']} & {r['n']:,.0f} & {r['r2']:.3f} & "129            f"\\makecell{{{r['ln_living']:.3f}\\\\\\scriptsize({r['se_living']:.3f})}} & "130            f"\\makecell{{{r['bathrooms']:.3f}\\\\\\scriptsize({r['se_bath']:.3f})}} & "131            f"\\makecell{{{r['ln_lot']:.3f}\\\\\\scriptsize({r['se_lot']:.3f})}} \\\\")132    write("robustness.tex",133          "\\begin{table}[t]\\centering\n"134          "\\caption{Robustness of key implicit prices across samples}\n"135          "\\label{tab:robustness}\n\\begin{threeparttable}\n"136          "\\begin{tabular}{lccccc}\n\\toprule\n"137          "Sample / specification & $N$ & $R^2$ & $\\ln$ area & Full bath & $\\ln$ lot \\\\\n"138          "\\midrule\n" + "\n".join(lines) +139          "\n\\bottomrule\n\\end{tabular}\n\\begin{tablenotes}\\footnotesize\\item "140          "Notes: Each row re-estimates the grand FSA-fixed-effects model on a different "141          "sample. Cluster-robust (FSA) standard errors in parentheses. The size elasticity "142          "and bathroom premium are stable across all cuts.\n"143          "\\end{tablenotes}\n\\end{threeparttable}\n\\end{table}")144145146# --------------------------------------------------------------------------147def table_quantile(res_dir: Path) -> None:148    q = pd.read_csv(res_dir / "quantile.csv")149    taus = q[q["tau"].notna()]150    ols = q[q["tau"].isna()].iloc[0]151    lines = []152    for _, r in taus.iterrows():153        tau = f"{r['tau']:g}"154        lines.append(155            f"$\\tau={tau}$ & "156            f"\\makecell{{{r['ln_living']:.3f}\\\\\\scriptsize({r['se_living']:.3f})}} & "157            f"\\makecell{{{r['bathrooms']:.3f}\\\\\\scriptsize({r['se_bath']:.3f})}} & "158            f"\\makecell{{{r['ln_lot']:.3f}\\\\\\scriptsize({r['se_lot']:.3f})}} \\\\")159    write("quantile.tex",160          "\\begin{table}[t]\\centering"161          "\\caption{Quantile hedonic estimates across the price distribution}\n"162          "\\label{tab:quantile}\\begin{threeparttable}\\begin{tabular}{lccc}\n\\toprule\n"163          "Quantile & $\\ln$ area & Full bath & $\\ln$ lot \\\\\n\\midrule\n"164          + "\n".join(lines) + "\n\\midrule\nOLS & "165          f"{ols['ln_living']:.3f} & {ols['bathrooms']:.3f} & {ols['ln_lot']:.3f} \\\\\n"166          "\\bottomrule\\end{tabular}\\begin{tablenotes}\\footnotesize\\item Notes: House "167          "subsample, province fixed effects, structural controls. Analytical standard "168          "errors in parentheses. The size elasticity rises and the lot elasticity rises "169          "with price.\\end{tablenotes}\\end{threeparttable}\\end{table}")170171172# --------------------------------------------------------------------------173def table_lopo(res_dir: Path) -> None:174    lopo = pd.read_csv(res_dir / "lopo.csv")175    lines = [f"{r['prov']} & {r['n']:,.0f} & {r['r2_within']:.3f} \\\\"176             for _, r in lopo.iterrows()]177    write("lopo.tex",178          "\\begin{table}[t]\\centering"179          "\\caption{Leave-one-province-out spatial cross-validation}\n"180          "\\label{tab:lopo}\\begin{threeparttable}\\begin{tabular}{lcc}\n\\toprule\n"181          "Held-out province & $N$ & Within-province $R^2$ \\\\\n\\midrule\n"182          + "\n".join(lines) +183          f"\n\\midrule\nMean & --- & {lopo['r2_within'].mean():.3f} \\\\\n"184          "\\bottomrule\\end{tabular}\\begin{tablenotes}\\footnotesize\\item Notes: The "185          "structural hedonic model is estimated on all provinces except one and used to "186          "predict the held-out province; a province-specific intercept is allowed (the "187          "price \\emph{level} is not identified out of sample), so the metric captures "188          "whether the \\emph{structural} implicit prices transfer across space."189          "\\end{tablenotes}\\end{threeparttable}\\end{table}")190191192# --------------------------------------------------------------------------193def table_oos(res_dir: Path) -> None:194    oos = json.load(open(res_dir / "oos.json"))195    write("oos.tex",196          "\\begin{table}[t]\\centering\n"197          "\\caption{Out-of-sample valuation performance (80/20 split)}\n"198          "\\label{tab:oos}\n\\begin{threeparttable}\n\\begin{tabular}{lc}\n\\toprule\n"199          "Metric & Value \\\\\n\\midrule\n"200          f"Training listings & {oos['n_train']:,} \\\\\n"201          f"Test listings & {oos['n_test']:,} \\\\\n"202          f"Out-of-sample $R^2$ (log price) & {oos['oos_r2']:.3f} \\\\\n"203          f"RMSE (log points) & {oos['rmse_log']:.3f} \\\\\n"204          f"Median absolute \\% error & {oos['median_ape']:.1f}\\% \\\\\n"205          f"Mean absolute \\% error & {oos['mean_ape']:.1f}\\% \\\\\n"206          f"Share priced within $\\pm$10\\% & {oos['within10']:.1f}\\% \\\\\n"207          f"Share priced within $\\pm$20\\% & {oos['within20']:.1f}\\% \\\\\n"208          "\\bottomrule\n\\end{tabular}\n\\begin{tablenotes}\\footnotesize\\item Notes: "209          "Model trained on a random 80\\% of listings and scored on the held-out 20\\% "210          "(restricted to neighbourhoods observed in training). Prices back-transformed "211          "with Duan's smearing estimator.\n"212          "\\end{tablenotes}\n\\end{threeparttable}\n\\end{table}")213214215def main() -> None:216    parser = argparse.ArgumentParser()217    parser.add_argument("--results", choices=["reference", "reproduced"],218                        default="reference",219                        help="results tier (default: reference — published numbers)")220    args = parser.parse_args()221    res_dir = REFERENCE if args.results == "reference" else REPRODUCED222    ensure_dirs()223    table_summary_stats(res_dir)224    table_regression(res_dir)225    table_robustness(res_dir)226    table_quantile(res_dir)227    table_lopo(res_dir)228    table_oos(res_dir)229    print("all tables written to", TABLES)230231232if __name__ == "__main__":233    main()234