Python 64.6%
TeX 23.3%
HTML 11.6%
Makefile 0.5%
1"""Master feature engineering for hedonic pricing analysis of Airbnb listings."""23from __future__ import annotations45import logging6from typing import Any78import numpy as np9import pandas as pd1011logger = logging.getLogger(__name__)1213# ---------------------------------------------------------------------------14# Structural features15# ---------------------------------------------------------------------------1617def create_structural_features(df: pd.DataFrame) -> pd.DataFrame:18 """Derive structural / capacity features from bedrooms, bathrooms, guests_count, property_type."""19 out = df.copy()2021 # Log transforms (add 1 to avoid log(0))22 out["log_accommodates"] = np.log1p(out["guests_count"])23 out["log_bedrooms"] = np.log1p(out["bedrooms"])24 out["log_beds"] = np.log1p(out["bedrooms"]) # beds not available; proxy with bedrooms25 out["log_bathrooms"] = np.log1p(out["bathrooms"])2627 # Per-guest ratios (guard against division by zero)28 guests_safe = out["guests_count"].replace(0, np.nan)29 out["bathrooms_per_guest"] = out["bathrooms"] / guests_safe30 out["bedrooms_per_guest"] = out["bedrooms"] / guests_safe31 out["bathrooms_per_guest"] = out["bathrooms_per_guest"].fillna(0.0)32 out["bedrooms_per_guest"] = out["bedrooms_per_guest"].fillna(0.0)3334 # Property-type dummies35 pt_lower = out["property_type"].str.lower().fillna("")36 out["is_entire_home"] = (37 pt_lower.str.contains("house")38 | pt_lower.str.contains("condo")39 | pt_lower.str.contains("apartment")40 | pt_lower.str.contains("cabin")41 | pt_lower.str.contains("chalet")42 ).astype(int)43 out["is_private_room"] = pt_lower.str.contains("rental unit").astype(int)44 out["is_shared_room"] = pt_lower.str.contains("other").astype(int)4546 # Capacity flag47 out["large_capacity_listing"] = (out["guests_count"] >= 8).astype(int)4849 # Polynomial terms50 out["accommodates_sq"] = out["guests_count"] ** 251 out["bedrooms_sq"] = out["bedrooms"] ** 252 out["bathrooms_sq"] = out["bathrooms"] ** 253 out["accommodates_bedrooms"] = out["guests_count"] * out["bedrooms"]54 out["accommodates_bathrooms"] = out["guests_count"] * out["bathrooms"]5556 logger.info("Structural features created: %d new columns", out.shape[1] - df.shape[1])57 return out585960# ---------------------------------------------------------------------------61# Host features62# ---------------------------------------------------------------------------6364def create_host_features(df: pd.DataFrame) -> pd.DataFrame:65 """Derive host-quality features."""66 out = df.copy()6768 # is_superhost already boolean; cast to int for modelling69 out["superhost_flag"] = out["is_superhost"].astype(int)7071 # host_professional_dummy: not available in limited data72 out["host_professional_dummy"] = 07374 # multi_listing_host: proxy via name frequency (same host likely uses similar names)75 name_counts = out["name"].map(out["name"].value_counts())76 out["multi_listing_host"] = (name_counts > 1).astype(int)7778 # Interaction: superhost x entire home (requires structural features already present)79 if "is_entire_home" in out.columns:80 out["superhost_x_entire_home"] = out["superhost_flag"] * out["is_entire_home"]81 else:82 pt_lower = out["property_type"].str.lower().fillna("")83 is_entire = (84 pt_lower.str.contains("house")85 | pt_lower.str.contains("condo")86 | pt_lower.str.contains("apartment")87 | pt_lower.str.contains("cabin")88 | pt_lower.str.contains("chalet")89 ).astype(int)90 out["superhost_x_entire_home"] = out["superhost_flag"] * is_entire9192 # Guest favourite flag93 out["guest_favorite_flag"] = out["is_guest_favorite"].astype(int)9495 logger.info("Host features created: %d new columns", out.shape[1] - df.shape[1])96 return out979899# ---------------------------------------------------------------------------100# Review / reputation features101# ---------------------------------------------------------------------------102103def create_review_features(df: pd.DataFrame) -> pd.DataFrame:104 """Derive review and reputation features from rating and num_reviews."""105 out = df.copy()106107 # Log reviews (handle NaN)108 out["log_reviews"] = np.log1p(out["num_reviews"].fillna(0))109110 # Review quality bucket from rating111 rating_filled = out["rating"].fillna(0.0)112 out["review_quality"] = pd.cut(113 rating_filled,114 bins=[-np.inf, 0.0, 4.0, 4.5, 4.8, 5.01],115 labels=[0, 1, 2, 3, 4],116 ).astype(float)117118 # Reputation score: weighted combination of rating and log(reviews)119 rating_norm = rating_filled / 5.0120 review_norm = out["log_reviews"] / (out["log_reviews"].max() or 1.0)121 out["reputation_score"] = 0.6 * rating_norm + 0.4 * review_norm122123 # Cleanliness premium proxy: high rating (>= 4.8) with many reviews signals cleanliness124 out["cleanliness_premium"] = (125 (rating_filled >= 4.8) & (out["num_reviews"].fillna(0) >= 10)126 ).astype(int)127128 # Location rating premium proxy: high rating in a high-density area (use quality_score if helpful)129 out["location_rating_premium"] = (130 (rating_filled >= 4.7) & (out["quality_score"] >= out["quality_score"].median())131 ).astype(int)132133 # Has reviews flag134 out["has_reviews"] = (out["num_reviews"].fillna(0) > 0).astype(int)135136 # No rating flag (for imputation awareness)137 out["no_rating_flag"] = out["rating"].isna().astype(int)138139 logger.info("Review features created: %d new columns", out.shape[1] - df.shape[1])140 return out141142143# ---------------------------------------------------------------------------144# Price segment features145# ---------------------------------------------------------------------------146147def create_price_segment_features(df: pd.DataFrame) -> pd.DataFrame:148 """Categorize listings into price segments based on quantiles."""149 out = df.copy()150151 price = out["price_numeric"]152 valid_mask = price.notna()153154 # Quantile boundaries on valid prices155 q25, q50, q75 = price.quantile([0.25, 0.50, 0.75])156157 conditions = [158 price <= q25,159 (price > q25) & (price <= q50),160 (price > q50) & (price <= q75),161 price > q75,162 ]163 choices = ["budget", "moderate", "premium", "luxury"]164 out["price_segment"] = np.select(conditions, choices, default="unknown")165166 # Numeric encoding167 segment_map = {"budget": 0, "moderate": 1, "premium": 2, "luxury": 3, "unknown": -1}168 out["price_segment_code"] = out["price_segment"].map(segment_map).astype(int)169170 # Log price (primary dependent variable in hedonic models)171 out["log_price"] = np.log1p(price)172173 # Price per guest174 guests_safe = out["guests_count"].replace(0, np.nan)175 out["price_per_guest"] = price / guests_safe176177 # Price per bedroom178 bedrooms_safe = out["bedrooms"].replace(0, np.nan)179 out["price_per_bedroom"] = price / bedrooms_safe180181 logger.info("Price segment features created: %d new columns", out.shape[1] - df.shape[1])182 return out183184185# ---------------------------------------------------------------------------186# Master pipeline187# ---------------------------------------------------------------------------188189def create_all_features(df: pd.DataFrame) -> pd.DataFrame:190 """Run the full feature engineering pipeline in order."""191 logger.info("Starting feature engineering pipeline on %d rows", len(df))192 out = df.copy()193 out = create_structural_features(out)194 out = create_host_features(out)195 out = create_review_features(out)196 out = create_price_segment_features(out)197 logger.info("Pipeline complete: %d columns total", out.shape[1])198 return out199200201# ---------------------------------------------------------------------------202# Feature group registry203# ---------------------------------------------------------------------------204205def get_feature_groups() -> dict[str, list[str]]:206 """Return a dict mapping group names to lists of feature column names."""207 return {208 "structural": [209 "log_accommodates", "log_bedrooms", "log_beds", "log_bathrooms",210 "bathrooms_per_guest", "bedrooms_per_guest",211 "is_entire_home", "is_private_room", "is_shared_room",212 "large_capacity_listing",213 "accommodates_sq", "bedrooms_sq", "bathrooms_sq",214 "accommodates_bedrooms", "accommodates_bathrooms",215 ],216 "host": [217 "superhost_flag", "host_professional_dummy",218 "multi_listing_host", "superhost_x_entire_home",219 "guest_favorite_flag",220 ],221 "review": [222 "log_reviews", "review_quality", "reputation_score",223 "cleanliness_premium", "location_rating_premium",224 "has_reviews", "no_rating_flag",225 ],226 "price_segment": [227 "price_segment", "price_segment_code",228 "log_price", "price_per_guest", "price_per_bedroom",229 ],230 "amenity": [231 "amenity_richness_index", "luxury_index",232 "business_travel_index", "family_comfort_index",233 "leisure_index", "remote_work_index",234 ],235 "spatial": [236 "distance_to_center", "local_density_500m", "local_density_1km",237 "local_avg_price_500m", "local_avg_price_1km",238 "spatial_cluster",239 ],240 }241