spb/wp5_uqo Public
UQO Working Paper No. 5 — Airbnb, residential rents and housing market pressure.
TeX 53.4%
Python 46.5%
1# Author: Simon-Pierre Boucher — contact@spboucher.ai2"""305_descriptive_analysis.py4==========================5Generate descriptive statistics tables and figures for the Airbnb-rent6linkage analysis.78Tables -> results/tables/ (.tex + .csv)9Figures -> figures/ (.pdf)1011Inputs:12 data/processed/merged_analysis.parquet (preferred)13 data/processed/merged_spatial.parquet (fallback)14 data/processed/airbnb_clean.parquet15 data/processed/rent_clean.parquet16"""1718import sys19from pathlib import Path2021sys.path.insert(0, str(Path(__file__).resolve().parent.parent))2223from src.plotting import use_descriptive_style2425use_descriptive_style()2627import matplotlib.pyplot as plt # noqa: E40228import matplotlib.ticker as mticker # noqa: E40229import pandas as pd # noqa: E4023031from src.config import ( # noqa: E40232 AIRBNB_CLEAN,33 RENT_CLEAN,34 MERGED_ANALYSIS,35 MERGED_SPATIAL,36 FIG_DIR,37 TABLE_DIR,38 require,39)4041PROCESSED_HINT = "Run scripts/04_merge_data.py first."424344# ===================================================================45# Helper: summary statistics46# ===================================================================4748def summary_stats(df: pd.DataFrame, cols: list[str], labels: dict | None = None) -> pd.DataFrame:49 """50 Compute N, mean, sd, min, p25, median, p75, max for *cols*.51 Columns absent from *df* are silently skipped (matches the published52 tables, which omit variables not present in the processed data).53 """54 rows = []55 for c in cols:56 if c not in df.columns:57 continue58 s = df[c].dropna()59 label = (labels or {}).get(c, c)60 rows.append({61 "Variable": label,62 "N": int(len(s)),63 "Mean": s.mean(),64 "SD": s.std(),65 "Min": s.min(),66 "P25": s.quantile(0.25),67 "Median": s.median(),68 "P75": s.quantile(0.75),69 "Max": s.max(),70 })71 return pd.DataFrame(rows)727374def save_table(df: pd.DataFrame, name: str):75 """Save as .csv and .tex (a tabular fragment, wrapped by the paper)."""76 csv_path = TABLE_DIR / f"{name}.csv"77 tex_path = TABLE_DIR / f"{name}.tex"78 df.to_csv(csv_path, index=False)79 # LaTeX: format floats to 2 decimals except integers80 float_fmt = "%.2f"81 df.to_latex(tex_path, index=False, float_format=float_fmt)82 print(f" Saved {csv_path.name} + {tex_path.name}")838485# ===================================================================86# Tables87# ===================================================================8889def make_tables(airbnb: pd.DataFrame, rent: pd.DataFrame, merged: pd.DataFrame):90 print("\n--- Tables ---")9192 # 1. Summary statistics: Airbnb93 airbnb_vars = [94 "price_numeric", "rating", "bedrooms", "bathrooms",95 "guests_count", "amenities_count", "num_images", "num_reviews",96 "quality_score",97 ]98 airbnb_labels = {99 "price_numeric": "Nightly price (CAD)",100 "rating": "Rating",101 "bedrooms": "Bedrooms",102 "bathrooms": "Bathrooms",103 "guests_count": "Max guests",104 "amenities_count": "Amenities count",105 "num_images": "Number of images",106 "num_reviews": "Number of reviews",107 "quality_score": "Quality score",108 }109 tbl_airbnb = summary_stats(airbnb, airbnb_vars, airbnb_labels)110 save_table(tbl_airbnb, "summary_stats_airbnb")111112 # 2. Summary statistics: Rent113 rent_vars = [114 "monthly_rent", "bedrooms", "bathrooms", "size_sqft",115 ]116 rent_labels = {117 "monthly_rent": "Monthly rent (CAD)",118 "bedrooms": "Bedrooms",119 "bathrooms": "Bathrooms",120 "size_sqft": "Size (sq ft)",121 }122 tbl_rent = summary_stats(rent, rent_vars, rent_labels)123 save_table(tbl_rent, "summary_stats_rent")124125 # 3. Correlation matrix126 corr_vars = []127 # Rent outcome128 for v in ["log_rent", "monthly_rent"]:129 if v in merged.columns:130 corr_vars.append(v)131 break # prefer log_rent132133 # Airbnb exposure measures (500m buffer)134 for v in ["airbnb_count_500m", "airbnb_density_500m",135 "mean_airbnb_price_500m", "share_entire_home_500m"]:136 if v in merged.columns:137 corr_vars.append(v)138139 # Property characteristics140 for v in ["bedrooms", "bathrooms", "size_sqft"]:141 if v in merged.columns:142 corr_vars.append(v)143144 if len(corr_vars) >= 3:145 corr_df = merged[corr_vars].dropna()146 corr_mat = corr_df.corr()147 save_table(corr_mat.round(3).reset_index().rename(columns={"index": ""}),148 "correlation_matrix")149 else:150 print(" Skipping correlation matrix — not enough overlapping columns.")151152153# ===================================================================154# Figures155# ===================================================================156157def fig_dist_airbnb_price(airbnb: pd.DataFrame):158 """Histogram of Airbnb nightly prices."""159 prices = airbnb["price_numeric"].dropna()160 # Trim extreme outliers (above 99th percentile) for readability161 p99 = prices.quantile(0.99)162 prices_trim = prices[prices <= p99]163164 fig, ax = plt.subplots(figsize=(6, 4))165 ax.hist(prices_trim, bins=60, color="#4C72B0", edgecolor="white", linewidth=0.5)166 ax.set_xlabel("Nightly price (CAD)")167 ax.set_ylabel("Count")168 ax.set_title("Distribution of Airbnb Nightly Prices")169 ax.xaxis.set_major_formatter(mticker.StrMethodFormatter("${x:,.0f}"))170 fig.tight_layout()171 fig.savefig(FIG_DIR / "dist_airbnb_price.pdf")172 plt.close(fig)173 print(" dist_airbnb_price.pdf")174175176def fig_dist_rent(rent: pd.DataFrame):177 """Histogram of monthly rents."""178 rents = rent["monthly_rent"].dropna()179 p99 = rents.quantile(0.99)180 rents_trim = rents[rents <= p99]181182 fig, ax = plt.subplots(figsize=(6, 4))183 ax.hist(rents_trim, bins=60, color="#DD8452", edgecolor="white", linewidth=0.5)184 ax.set_xlabel("Monthly rent (CAD)")185 ax.set_ylabel("Count")186 ax.set_title("Distribution of Monthly Rents")187 ax.xaxis.set_major_formatter(mticker.StrMethodFormatter("${x:,.0f}"))188 fig.tight_layout()189 fig.savefig(FIG_DIR / "dist_rent.pdf")190 plt.close(fig)191 print(" dist_rent.pdf")192193194def fig_airbnb_by_city(airbnb: pd.DataFrame):195 """Bar chart of Airbnb listing count by top 15 cities."""196 city_col = "city_clean" if "city_clean" in airbnb.columns else "city"197 counts = airbnb[city_col].value_counts().head(15)198199 fig, ax = plt.subplots(figsize=(8, 5))200 counts.sort_values().plot.barh(ax=ax, color="#4C72B0", edgecolor="white")201 ax.set_xlabel("Number of listings")202 ax.set_ylabel("")203 ax.set_title("Airbnb Listings by City (Top 15)")204 fig.tight_layout()205 fig.savefig(FIG_DIR / "airbnb_by_city.pdf")206 plt.close(fig)207 print(" airbnb_by_city.pdf")208209210def fig_rent_by_city(rent: pd.DataFrame):211 """Bar chart of mean rent by top 15 cities."""212 city_counts = rent["city"].value_counts()213 top_cities = city_counts.head(15).index214 sub = rent[rent["city"].isin(top_cities)]215 means = sub.groupby("city")["monthly_rent"].mean().sort_values()216217 fig, ax = plt.subplots(figsize=(8, 5))218 means.plot.barh(ax=ax, color="#DD8452", edgecolor="white")219 ax.set_xlabel("Mean monthly rent (CAD)")220 ax.set_ylabel("")221 ax.set_title("Mean Monthly Rent by City (Top 15)")222 ax.xaxis.set_major_formatter(mticker.StrMethodFormatter("${x:,.0f}"))223 fig.tight_layout()224 fig.savefig(FIG_DIR / "rent_by_city.pdf")225 plt.close(fig)226 print(" rent_by_city.pdf")227228229def fig_scatter_airbnb_rent(merged: pd.DataFrame):230 """Scatter plot of mean Airbnb count (500m) vs log rent231 at a neighbourhood level (postal code or small-area average)."""232 if "airbnb_count_500m" not in merged.columns:233 print(" Skipping scatter_airbnb_rent.pdf — no airbnb_count_500m column.")234 return235236 rent_col = "log_rent" if "log_rent" in merged.columns else "monthly_rent"237238 # Aggregate at postal-code level if available, else use city239 group_col = None240 for candidate in ["postal_code", "city"]:241 if candidate in merged.columns:242 group_col = candidate243 break244245 if group_col is None:246 print(" Skipping scatter_airbnb_rent.pdf — no grouping column found.")247 return248249 agg = merged.groupby(group_col).agg(250 mean_airbnb_count=("airbnb_count_500m", "mean"),251 mean_rent=(rent_col, "mean"),252 n=("airbnb_count_500m", "size"),253 ).reset_index()254255 # Keep neighbourhoods with at least 5 observations256 agg = agg[agg["n"] >= 5]257258 fig, ax = plt.subplots(figsize=(6, 5))259 ax.scatter(agg["mean_airbnb_count"], agg["mean_rent"],260 alpha=0.6, s=20, color="#4C72B0", edgecolors="none")261 ax.set_xlabel("Mean Airbnb count within 500 m")262 y_label = "Log monthly rent" if rent_col == "log_rent" else "Mean monthly rent (CAD)"263 ax.set_ylabel(y_label)264 ax.set_title("Airbnb Density vs. Rent")265 fig.tight_layout()266 fig.savefig(FIG_DIR / "scatter_airbnb_rent.pdf")267 plt.close(fig)268 print(" scatter_airbnb_rent.pdf")269270271def fig_map_airbnb(airbnb: pd.DataFrame):272 """Scatter map of Airbnb listings coloured by price."""273 lon_col = "lon" if "lon" in airbnb.columns else "long"274 valid = airbnb.dropna(subset=["lat", lon_col, "price_numeric"])275276 # Cap at 99th percentile for colour scale277 p99 = valid["price_numeric"].quantile(0.99)278279 fig, ax = plt.subplots(figsize=(8, 7))280 sc = ax.scatter(281 valid[lon_col], valid["lat"],282 c=valid["price_numeric"].clip(upper=p99),283 cmap="YlOrRd", s=4, alpha=0.6, edgecolors="none",284 )285 cbar = fig.colorbar(sc, ax=ax, shrink=0.7)286 cbar.set_label("Nightly price (CAD)")287 ax.set_xlabel("Longitude")288 ax.set_ylabel("Latitude")289 ax.set_title("Airbnb Listings — Quebec")290 fig.tight_layout()291 fig.savefig(FIG_DIR / "map_airbnb.pdf")292 plt.close(fig)293 print(" map_airbnb.pdf")294295296def fig_map_rent(rent: pd.DataFrame):297 """Scatter map of rental listings coloured by rent."""298 valid = rent.dropna(subset=["lat", "lon", "monthly_rent"])299 p99 = valid["monthly_rent"].quantile(0.99)300301 fig, ax = plt.subplots(figsize=(8, 7))302 sc = ax.scatter(303 valid["lon"], valid["lat"],304 c=valid["monthly_rent"].clip(upper=p99),305 cmap="YlGnBu", s=4, alpha=0.6, edgecolors="none",306 )307 cbar = fig.colorbar(sc, ax=ax, shrink=0.7)308 cbar.set_label("Monthly rent (CAD)")309 ax.set_xlabel("Longitude")310 ax.set_ylabel("Latitude")311 ax.set_title("Rental Listings — Quebec")312 fig.tight_layout()313 fig.savefig(FIG_DIR / "map_rent.pdf")314 plt.close(fig)315 print(" map_rent.pdf")316317318# ===================================================================319# Main320# ===================================================================321322def main():323 print("=" * 70)324 print("05 DESCRIPTIVE ANALYSIS")325 print("=" * 70)326327 # ------------------------------------------------------------------328 # Load data329 # ------------------------------------------------------------------330 print("\n[1] Loading data ...")331332 airbnb = pd.read_parquet(require(AIRBNB_CLEAN, PROCESSED_HINT))333 if "long" in airbnb.columns and "lon" not in airbnb.columns:334 airbnb = airbnb.rename(columns={"long": "lon"})335336 rent = pd.read_parquet(require(RENT_CLEAN, PROCESSED_HINT))337338 merged_path = MERGED_ANALYSIS339 if not merged_path.exists():340 merged_path = MERGED_SPATIAL341 merged = pd.read_parquet(require(merged_path, PROCESSED_HINT))342 if "long" in merged.columns and "lon" not in merged.columns:343 merged = merged.rename(columns={"long": "lon"})344345 print(f" Airbnb : {airbnb.shape}")346 print(f" Rent : {rent.shape}")347 print(f" Merged : {merged.shape} ({merged_path.name})")348349 # ------------------------------------------------------------------350 # Tables351 # ------------------------------------------------------------------352 make_tables(airbnb, rent, merged)353354 # ------------------------------------------------------------------355 # Figures356 # ------------------------------------------------------------------357 print("\n--- Figures ---")358 fig_dist_airbnb_price(airbnb)359 fig_dist_rent(rent)360 fig_airbnb_by_city(airbnb)361 fig_rent_by_city(rent)362 fig_scatter_airbnb_rent(merged)363 fig_map_airbnb(airbnb)364 fig_map_rent(rent)365366 print("\nDone.\n")367368369if __name__ == "__main__":370 main()371