spb/wp5_uqo Public
UQO Working Paper No. 5 — Airbnb, residential rents and housing market pressure.
TeX 53.4%
Python 46.5%
1# Author: Simon-Pierre Boucher — contact@spboucher.ai2"""311_extended_robustness.py4-------------------------5Extended robustness checks for the preferred hedonic specification6(log_rent ~ airbnb_count_500m + bedrooms + bathrooms + building type + city FE):78 (R1) Baseline with HC1 standard errors (reference column)9 (R2) Same coefficients with standard errors clustered by city10 (R3) Adding interior size (in 100 sq ft) on the subsample reporting it11 (R4) log(1 + count) functional form for the exposure variable12 (R5) Ring exposure: count in the 500m-1km annulus alongside the 500m count1314plus a leave-one-city-out sensitivity figure (dropping each of the largest15cities in turn).1617Outputs:18 results/tables/extended_robustness.tex19 figures/leave_one_city_out.pdf20"""2122import sys23from pathlib import Path2425sys.path.insert(0, str(Path(__file__).resolve().parent.parent))2627import matplotlib28matplotlib.use("Agg")29import matplotlib.pyplot as plt # noqa: E40230import numpy as np # noqa: E40231import pandas as pd # noqa: E40232import statsmodels.api as sm # noqa: E4023334from src.config import MERGED_ANALYSIS, TABLE_DIR, FIG_DIR, require # noqa: E40235from src.latex_tables import significance_star as _star # noqa: E4023637PROCESSED_HINT = "Run scripts/04_merge_data.py first."3839CONTROL_LABELS = {40 "airbnb_count_500m": r"Airbnb count (500m)",41 "log1p_airbnb_500m": r"log(1 + Airbnb count 500m)",42 "airbnb_ring_500m_1km": r"Airbnb count (500m--1km ring)",43 "bedrooms": "Bedrooms",44 "bathrooms": "Bathrooms",45 "size_100sqft": r"Interior size (100 sq ft)",46}474849def prepare(df: pd.DataFrame) -> tuple[pd.DataFrame, list[str]]:50 """Add derived regressors and building-type dummies; return (frame, bt_cols)."""51 d = df.copy()52 d["log1p_airbnb_500m"] = np.log1p(d["airbnb_count_500m"])53 d["airbnb_ring_500m_1km"] = d["airbnb_count_1000m"] - d["airbnb_count_500m"]54 d["size_100sqft"] = d["size_interior_sqft"] / 100.05556 bt = pd.get_dummies(d["building_type"], prefix="bt", drop_first=True, dtype=float)57 d = pd.concat([d, bt], axis=1)58 return d, list(bt.columns)596061def fit(d: pd.DataFrame, x_main: list[str], bt_cols: list[str],62 label: str, cluster: bool = False, extra_controls: list[str] | None = None):63 """64 OLS of log_rent on x_main + bedrooms/bathrooms (+extras) + building type65 + city FE, with HC1 or city-clustered standard errors.66 """67 controls = ["bedrooms", "bathrooms"] + (extra_controls or [])68 needed = ["log_rent", "city"] + x_main + controls69 sub = d[needed + bt_cols].dropna().reset_index(drop=True)7071 city_fe = pd.get_dummies(sub["city"], prefix="city", drop_first=True, dtype=float)72 X = sm.add_constant(pd.concat(73 [sub[x_main + controls + bt_cols].astype(float), city_fe], axis=174 ).astype(float))75 y = sub["log_rent"].astype(float)7677 if cluster:78 res = sm.OLS(y, X).fit(cov_type="cluster",79 cov_kwds={"groups": sub["city"]})80 else:81 res = sm.OLS(y, X).fit(cov_type="HC1")8283 print(f" [{label}] N={int(res.nobs):,} R2={res.rsquared:.4f} "84 f"beta({x_main[0]})={res.params[x_main[0]]:.6f} "85 f"SE={res.bse[x_main[0]]:.6f}")86 return res878889def build_table(results: list, col_headers: list[str], se_row: list[str],90 display_vars: list[str]) -> None:91 """Write the extended-robustness table fragment."""92 n = len(results)93 lines: list[str] = []94 lines.append(r"\begin{tabular}{l" + "c" * n + "}")95 lines.append(r"\toprule")96 lines.append(" & ".join([""] + [f"\\textbf{{{h}}}" for h in col_headers]) + r" \\")97 lines.append(98 " & ".join(["Dep.\\ var:"] + [r"\textit{log\_rent}"] * n) + r" \\"99 )100 lines.append(r"\midrule")101102 for var in display_vars:103 cells_c, cells_s = [], []104 for res in results:105 if var in res.params.index:106 b, se, p = res.params[var], res.bse[var], res.pvalues[var]107 cells_c.append(f"{b:.4f}{_star(p)}")108 cells_s.append(f"({se:.4f})")109 else:110 cells_c.append("")111 cells_s.append("")112 label = CONTROL_LABELS.get(var, var.replace("_", r"\_"))113 lines.append(f"{label} & " + " & ".join(cells_c) + r" \\")114 lines.append(" & " + " & ".join(cells_s) + r" \\[4pt]")115116 lines.append(r"\midrule")117 lines.append("Standard errors & " + " & ".join(se_row) + r" \\")118 lines.append("Building type \\& city FE & " + " & ".join(["Yes"] * n) + r" \\")119 lines.append("Observations & "120 + " & ".join(f"{int(r.nobs):,}" for r in results) + r" \\")121 lines.append("R$^2$ & "122 + " & ".join(f"{r.rsquared:.4f}" for r in results) + r" \\")123 lines.append(r"\bottomrule")124 lines.append(r"\end{tabular}")125 lines.append(126 r"\parbox{\textwidth}{\footnotesize Standard errors in parentheses: "127 r"HC1 except column (R2), which clusters by city. All columns control "128 r"for bedrooms, bathrooms, building type, and city fixed effects; "129 r"column (R3) adds interior size (per 100 sq ft) on the subsample "130 r"reporting it. Column (R5) includes the 500m count and the count in "131 r"the 500m--1km annulus jointly. "132 r"$^{***}p<0.01$; $^{**}p<0.05$; $^{*}p<0.10$.}"133 )134135 out = TABLE_DIR / "extended_robustness.tex"136 out.write_text("\n".join(lines) + "\n", encoding="utf-8")137 print(f" -> saved {out}")138139140def leave_one_city_out(d: pd.DataFrame, bt_cols: list[str], n_cities: int = 8) -> None:141 """Coefficient on the 500m count when each large city is excluded in turn."""142 print("\n--- Leave-one-city-out ---")143 top = d["city"].value_counts().head(n_cities).index.tolist()144145 res_full = fit(d, ["airbnb_count_500m"], bt_cols, "Full sample")146 beta_full = res_full.params["airbnb_count_500m"]147 se_full = res_full.bse["airbnb_count_500m"]148149 labels, betas, ses = [], [], []150 for city in top:151 res = fit(d[d["city"] != city], ["airbnb_count_500m"], bt_cols,152 f"excl. {city}")153 labels.append(f"excl. {city}")154 betas.append(res.params["airbnb_count_500m"])155 ses.append(res.bse["airbnb_count_500m"])156157 betas = np.array(betas)158 ses = np.array(ses)159160 fig, ax = plt.subplots(figsize=(7, 0.45 * len(labels) + 2))161 y_pos = np.arange(len(labels))162 ax.errorbar(betas, y_pos, xerr=1.96 * ses,163 fmt="o", color="#2166ac", ecolor="#92c5de", capsize=4,164 markersize=6, elinewidth=1.5)165 ax.axvline(beta_full, color="firebrick", linestyle="--", linewidth=1.2,166 label=f"Full sample ({beta_full:.4f})")167 ax.axvspan(beta_full - 1.96 * se_full, beta_full + 1.96 * se_full,168 color="firebrick", alpha=0.08)169 ax.axvline(0, color="grey", linestyle=":", linewidth=0.8)170 ax.set_yticks(y_pos)171 ax.set_yticklabels(labels)172 ax.set_xlabel(r"Coefficient on Airbnb count within 500m ($\beta$)")173 ax.set_title("Leave-One-City-Out Sensitivity")174 ax.legend(fontsize=8, loc="best")175 ax.invert_yaxis()176 fig.tight_layout()177178 out = FIG_DIR / "leave_one_city_out.pdf"179 fig.savefig(out, dpi=300)180 plt.close(fig)181 print(f" -> saved {out}")182183184def main() -> None:185 print("=" * 72)186 print("11 EXTENDED ROBUSTNESS")187 print("=" * 72)188189 df = pd.read_parquet(require(MERGED_ANALYSIS, PROCESSED_HINT))190 d, bt_cols = prepare(df)191 print(f"\nSample: {len(d):,} rows | "192 f"interior size non-missing: {d['size_100sqft'].notna().sum():,}")193194 print("\n--- Specifications R1-R5 ---")195 res_r1 = fit(d, ["airbnb_count_500m"], bt_cols, "R1 baseline HC1")196 res_r2 = fit(d, ["airbnb_count_500m"], bt_cols, "R2 clustered", cluster=True)197 res_r3 = fit(d, ["airbnb_count_500m"], bt_cols, "R3 + size",198 extra_controls=["size_100sqft"])199 res_r4 = fit(d, ["log1p_airbnb_500m"], bt_cols, "R4 log(1+count)")200 res_r5 = fit(d, ["airbnb_count_500m", "airbnb_ring_500m_1km"], bt_cols,201 "R5 ring")202203 build_table(204 [res_r1, res_r2, res_r3, res_r4, res_r5],205 ["(R1)", "(R2)", "(R3)", "(R4)", "(R5)"],206 ["HC1", "City cluster", "HC1", "HC1", "HC1"],207 display_vars=[208 "airbnb_count_500m", "log1p_airbnb_500m", "airbnb_ring_500m_1km",209 "bedrooms", "bathrooms", "size_100sqft",210 ],211 )212213 leave_one_city_out(d, bt_cols)214215 print("\n" + "=" * 72)216 print("11 DONE")217 print("=" * 72)218219220if __name__ == "__main__":221 main()222