UQO Working Paper No. 5 — Airbnb, residential rents and housing market pressure.
1# Author: Simon-Pierre Boucher — contact@spboucher.ai2"""306_hedonic_models.py4--------------------5Hedonic regression models for the Airbnb-rent analysis.67Model 1 (1a-1e): Baseline hedonic rent model (OLS, HC1)8Model 2 (2a-2c): Hedonic Airbnb pricing model9Model 3: City-level interaction (forward + reverse)1011Outputs:12 results/tables/hedonic_rent_baseline.tex13 results/tables/hedonic_airbnb_pricing.tex14 results/tables/city_level_interaction.tex15"""1617import sys18from pathlib import Path1920sys.path.insert(0, str(Path(__file__).resolve().parent.parent))2122import numpy as np23import pandas as pd24import statsmodels.api as sm2526from src.config import AIRBNB_CLEAN, MERGED_ANALYSIS, TABLE_DIR, require27from src.latex_tables import significance_star, results_to_latex, label2829PROCESSED_HINT = "Run scripts/04_merge_data.py first."303132def run_ols(df: pd.DataFrame, y_col: str, x_cols: list[str], label: str):33 """Run OLS with HC1 robust SE, dropping NaN rows for relevant cols."""34 cols = [y_col] + x_cols35 sub = df[cols].dropna()36 Y = sub[y_col]37 X = sm.add_constant(sub[x_cols])38 model = sm.OLS(Y, X).fit(cov_type="HC1")39 print(f"\n [{label}] N={int(model.nobs):,} R2={model.rsquared:.4f} "40 f"Adj-R2={model.rsquared_adj:.4f}")41 return model424344# ═════════════════════════════════════════════════════════════════════════════45# MODEL 1: Baseline Hedonic Rent Model (1a – 1e)46# ═════════════════════════════════════════════════════════════════════════════4748def model1_hedonic_rent(rent: pd.DataFrame) -> None:49 print("\n" + "-" * 72)50 print("MODEL 1: Baseline Hedonic Rent Model")51 print("-" * 72)5253 # --- prepare variables -------------------------------------------------54 rent_m = rent.copy()5556 # Building-type dummies57 if "building_type" in rent_m.columns:58 bt_dummies = pd.get_dummies(rent_m["building_type"], prefix="bt",59 drop_first=True, dtype=float)60 rent_m = pd.concat([rent_m, bt_dummies], axis=1)61 bt_cols = list(bt_dummies.columns)62 else:63 bt_cols = []6465 # City dummies (fixed effects)66 if "city" in rent_m.columns:67 city_dummies = pd.get_dummies(rent_m["city"], prefix="city",68 drop_first=True, dtype=float)69 rent_m = pd.concat([rent_m, city_dummies], axis=1)70 city_fe_cols = list(city_dummies.columns)71 else:72 city_fe_cols = []7374 controls = ["bedrooms", "bathrooms"] + bt_cols7576 # 1a: No controls, no FE77 res_1a = run_ols(rent_m, "log_rent", ["airbnb_count_500m"], "1a")7879 # 1b: With controls, no FE80 res_1b = run_ols(rent_m, "log_rent", ["airbnb_count_500m"] + controls, "1b")8182 # 1c: With controls + city FE83 res_1c = run_ols(84 rent_m, "log_rent", ["airbnb_count_500m"] + controls + city_fe_cols, "1c"85 )8687 # 1d: airbnb_density_500m instead88 res_1d = run_ols(89 rent_m, "log_rent", ["airbnb_density_500m"] + controls + city_fe_cols, "1d"90 )9192 # 1e: share_entire_home_500m instead93 res_1e = run_ols(94 rent_m, "log_rent", ["share_entire_home_500m"] + controls + city_fe_cols, "1e"95 )9697 # -- Key display variables (not all city/building dummies) --------------98 display_rent = (99 ["const", "airbnb_count_500m", "airbnb_density_500m",100 "share_entire_home_500m", "bedrooms", "bathrooms"]101 + bt_cols102 )103 all_res = [res_1a, res_1b, res_1c, res_1d, res_1e]104105 results_to_latex(106 all_res,107 ["(1a)", "(1b)", "(1c)", "(1d)", "(1e)"],108 dep_var="log\\_rent",109 display_vars=[v for v in display_rent110 if any(v in r.params.index for r in all_res)],111 out_path=TABLE_DIR / "hedonic_rent_baseline.tex",112 note="Models (1c)-(1e) include city fixed effects (not shown).",113 )114115116# ═════════════════════════════════════════════════════════════════════════════117# MODEL 2: Hedonic Airbnb Pricing Model (2a – 2c)118# ═════════════════════════════════════════════════════════════════════════════119120def model2_airbnb_pricing(rent: pd.DataFrame, airbnb: pd.DataFrame) -> None:121 print("\n" + "-" * 72)122 print("MODEL 2: Hedonic Airbnb Pricing Model")123 print("-" * 72)124125 # Compute mean rent per city from rent data126 city_col_rent = "city"127 # Determine city column name in airbnb data128 city_col_ab = "city_clean" if "city_clean" in airbnb.columns else "city"129130 mean_rent_city = (131 rent.groupby(city_col_rent)["log_rent"]132 .mean()133 .rename("mean_rent_city")134 .reset_index()135 .rename(columns={city_col_rent: city_col_ab})136 )137 print(f"\n Mean rent computed for {len(mean_rent_city)} cities")138139 ab = airbnb.merge(mean_rent_city, on=city_col_ab, how="inner")140 print(f" Airbnb rows after merge: {len(ab):,}")141142 # Ensure log_price exists143 if "log_price" not in ab.columns and "price_numeric" in ab.columns:144 ab["log_price"] = np.log(ab["price_numeric"].clip(lower=1))145146 # City FE for Airbnb147 ab_city_dum = pd.get_dummies(ab[city_col_ab], prefix="acity",148 drop_first=True, dtype=float)149 ab = pd.concat([ab, ab_city_dum], axis=1)150 ab_city_fe = list(ab_city_dum.columns)151152 ab_controls = [153 c for c in ["bedrooms", "bathrooms", "guests_count",154 "amenities_count", "is_superhost", "is_entire_home"]155 if c in ab.columns156 ]157158 # Convert boolean controls to float159 for c in ab_controls:160 if ab[c].dtype == bool:161 ab[c] = ab[c].astype(float)162163 # 2a: No controls164 res_2a = run_ols(ab, "log_price", ["mean_rent_city"], "2a")165166 # 2b: With controls167 res_2b = run_ols(ab, "log_price", ["mean_rent_city"] + ab_controls, "2b")168169 # 2c: With controls + city FE.170 # 2026-10-17 correction: mean_rent_city is constant within city and hence171 # perfectly collinear with the city dummies; the previous version reported172 # a pseudo-inverse "coefficient" for it. It is now dropped in (2c).173 res_2c = run_ols(ab, "log_price", ab_controls + ab_city_fe, "2c")174175 # 2d-2e (added 2026-10-17): the nightly-price distribution is bimodal, with176 # 26% of listings above $1,000 "per night" (the scraper's own category177 # "Ultra-Luxury (>$1000)"), which most plausibly reflects total-stay rather178 # than nightly prices. Re-estimate on listings priced at or below $1,000.179 ab_r = ab[ab["price_numeric"] <= 1000].copy()180 print(f" Restricted sample (price <= $1,000): {len(ab_r):,} listings")181 res_2d = run_ols(ab_r, "log_price", ["mean_rent_city"] + ab_controls, "2d")182 res_2e = run_ols(ab_r, "log_price", ab_controls + ab_city_fe, "2e")183184 display_ab = ["const", "mean_rent_city"] + ab_controls185 all_res = [res_2a, res_2b, res_2c, res_2d, res_2e]186187 results_to_latex(188 all_res,189 ["(2a)", "(2b)", "(2c)", "(2d)", "(2e)"],190 dep_var="log\\_price",191 display_vars=[v for v in display_ab192 if any(v in r.params.index for r in all_res)],193 out_path=TABLE_DIR / "hedonic_airbnb_pricing.tex",194 note=("Columns (2c) and (2e) include city fixed effects (not shown), which "195 "absorb the city mean log rent. Columns (2a)--(2c) use all listings "196 "with a matched city; columns (2d)--(2e) restrict the sample to "197 "listings with a nightly price of at most \\$1,000 (see Section~3)."),198 )199200201# ═════════════════════════════════════════════════════════════════════════════202# MODEL 3: City-level Interaction Model203# ═════════════════════════════════════════════════════════════════════════════204205def model3_city_interaction(rent: pd.DataFrame) -> None:206 print("\n" + "-" * 72)207 print("MODEL 3: City-level Interaction")208 print("-" * 72)209210 # Aggregate rent data to city level211 city_agg = (212 rent.groupby("city")213 .agg(214 mean_log_rent=("log_rent", "mean"),215 mean_bedrooms=("bedrooms", "mean"),216 mean_bathrooms=("bathrooms", "mean"),217 n_rent_listings=("log_rent", "count"),218 )219 .reset_index()220 )221222 # Get airbnb_count_city from rent data (should be constant within a city)223 airbnb_city_vars = [c for c in rent.columns224 if c.startswith("airbnb_") and c.endswith("_city")]225 if airbnb_city_vars:226 city_airbnb = rent.groupby("city")[airbnb_city_vars].first().reset_index()227 city_agg = city_agg.merge(city_airbnb, on="city", how="left")228229 print(f" City-level rows: {len(city_agg)}")230 print(f" Columns: {list(city_agg.columns)}")231232 # Determine which airbnb count variable is available233 ab_count_col = "airbnb_count_city" if "airbnb_count_city" in city_agg.columns else None234 city_controls = ["mean_bedrooms", "mean_bathrooms"]235236 if ab_count_col is None:237 print(" WARNING: airbnb_count_city not found — skipping Model 3.")238 return239240 # 3-forward: mean_log_rent ~ airbnb_count_city + controls241 res_3fwd = run_ols(242 city_agg, "mean_log_rent", [ab_count_col] + city_controls, "3-fwd"243 )244 # 3-reverse: airbnb_count_city ~ mean_log_rent + controls245 res_3rev = run_ols(246 city_agg, ab_count_col, ["mean_log_rent"] + city_controls, "3-rev"247 )248249 # LaTeX table (fragment: the paper supplies the table environment)250 lines: list[str] = []251 lines.append(r"\begin{tabular}{lcc}")252 lines.append(r"\toprule")253 lines.append(r" & \textbf{(3-fwd)} & \textbf{(3-rev)} \\")254 lines.append(255 r"Dep.\ var: & \textit{mean log rent} & \textit{Airbnb listings in city} \\"256 )257 lines.append(r"\midrule")258259 # Show all variables for both models260 all_vars_3 = list(dict.fromkeys(261 list(res_3fwd.params.index) + list(res_3rev.params.index)262 ))263 for var in all_vars_3:264 cells_coef = []265 cells_se = []266 for res in [res_3fwd, res_3rev]:267 if var in res.params.index:268 b = res.params[var]269 se = res.bse[var]270 p = res.pvalues[var]271 cells_coef.append(f"{b:.4f}{significance_star(p)}")272 cells_se.append(f"({se:.4f})")273 else:274 cells_coef.append("")275 cells_se.append("")276 vn = label(var)277 lines.append(f"{vn} & {cells_coef[0]} & {cells_coef[1]} " + r"\\")278 lines.append(f" & {cells_se[0]} & {cells_se[1]} " + r"\\[4pt]")279280 lines.append(r"\midrule")281 lines.append(282 f"Observations & {int(res_3fwd.nobs)} & {int(res_3rev.nobs)} " + r"\\"283 )284 lines.append(285 f"R$^2$ & {res_3fwd.rsquared:.4f} & {res_3rev.rsquared:.4f} " + r"\\"286 )287 lines.append(r"\bottomrule")288 lines.append(r"\end{tabular}")289 lines.append(290 r"\parbox{\textwidth}{\footnotesize Robust (HC1) standard errors in "291 r"parentheses. $^{***}p<0.01$; $^{**}p<0.05$; $^{*}p<0.10$.}"292 )293294 tex3 = "\n".join(lines) + "\n"295 out3 = TABLE_DIR / "city_level_interaction.tex"296 out3.write_text(tex3, encoding="utf-8")297 print(f" -> saved {out3}")298299300def main() -> None:301 print("=" * 72)302 print("06 HEDONIC REGRESSION MODELS")303 print("=" * 72)304305 rent = pd.read_parquet(require(MERGED_ANALYSIS, PROCESSED_HINT))306 airbnb = pd.read_parquet(require(AIRBNB_CLEAN, PROCESSED_HINT))307308 print(f"\nRent data: {rent.shape[0]:,} rows, {rent.shape[1]} cols")309 print(f"Airbnb data: {airbnb.shape[0]:,} rows, {airbnb.shape[1]} cols")310311 model1_hedonic_rent(rent)312 model2_airbnb_pricing(rent, airbnb)313 model3_city_interaction(rent)314315 print("\n" + "=" * 72)316 print("06 DONE")317 print("=" * 72)318319320if __name__ == "__main__":321 main()322