"""Master feature engineering for hedonic pricing analysis of Airbnb listings.""" from __future__ import annotations import logging from typing import Any import numpy as np import pandas as pd logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- # Structural features # --------------------------------------------------------------------------- def create_structural_features(df: pd.DataFrame) -> pd.DataFrame: """Derive structural / capacity features from bedrooms, bathrooms, guests_count, property_type.""" out = df.copy() # Log transforms (add 1 to avoid log(0)) out["log_accommodates"] = np.log1p(out["guests_count"]) out["log_bedrooms"] = np.log1p(out["bedrooms"]) out["log_beds"] = np.log1p(out["bedrooms"]) # beds not available; proxy with bedrooms out["log_bathrooms"] = np.log1p(out["bathrooms"]) # Per-guest ratios (guard against division by zero) guests_safe = out["guests_count"].replace(0, np.nan) out["bathrooms_per_guest"] = out["bathrooms"] / guests_safe out["bedrooms_per_guest"] = out["bedrooms"] / guests_safe out["bathrooms_per_guest"] = out["bathrooms_per_guest"].fillna(0.0) out["bedrooms_per_guest"] = out["bedrooms_per_guest"].fillna(0.0) # Property-type dummies pt_lower = out["property_type"].str.lower().fillna("") out["is_entire_home"] = ( pt_lower.str.contains("house") | pt_lower.str.contains("condo") | pt_lower.str.contains("apartment") | pt_lower.str.contains("cabin") | pt_lower.str.contains("chalet") ).astype(int) out["is_private_room"] = pt_lower.str.contains("rental unit").astype(int) out["is_shared_room"] = pt_lower.str.contains("other").astype(int) # Capacity flag out["large_capacity_listing"] = (out["guests_count"] >= 8).astype(int) # Polynomial terms out["accommodates_sq"] = out["guests_count"] ** 2 out["bedrooms_sq"] = out["bedrooms"] ** 2 out["bathrooms_sq"] = out["bathrooms"] ** 2 out["accommodates_bedrooms"] = out["guests_count"] * out["bedrooms"] out["accommodates_bathrooms"] = out["guests_count"] * out["bathrooms"] logger.info("Structural features created: %d new columns", out.shape[1] - df.shape[1]) return out # --------------------------------------------------------------------------- # Host features # --------------------------------------------------------------------------- def create_host_features(df: pd.DataFrame) -> pd.DataFrame: """Derive host-quality features.""" out = df.copy() # is_superhost already boolean; cast to int for modelling out["superhost_flag"] = out["is_superhost"].astype(int) # host_professional_dummy: not available in limited data out["host_professional_dummy"] = 0 # multi_listing_host: proxy via name frequency (same host likely uses similar names) name_counts = out["name"].map(out["name"].value_counts()) out["multi_listing_host"] = (name_counts > 1).astype(int) # Interaction: superhost x entire home (requires structural features already present) if "is_entire_home" in out.columns: out["superhost_x_entire_home"] = out["superhost_flag"] * out["is_entire_home"] else: pt_lower = out["property_type"].str.lower().fillna("") is_entire = ( pt_lower.str.contains("house") | pt_lower.str.contains("condo") | pt_lower.str.contains("apartment") | pt_lower.str.contains("cabin") | pt_lower.str.contains("chalet") ).astype(int) out["superhost_x_entire_home"] = out["superhost_flag"] * is_entire # Guest favourite flag out["guest_favorite_flag"] = out["is_guest_favorite"].astype(int) logger.info("Host features created: %d new columns", out.shape[1] - df.shape[1]) return out # --------------------------------------------------------------------------- # Review / reputation features # --------------------------------------------------------------------------- def create_review_features(df: pd.DataFrame) -> pd.DataFrame: """Derive review and reputation features from rating and num_reviews.""" out = df.copy() # Log reviews (handle NaN) out["log_reviews"] = np.log1p(out["num_reviews"].fillna(0)) # Review quality bucket from rating rating_filled = out["rating"].fillna(0.0) out["review_quality"] = pd.cut( rating_filled, bins=[-np.inf, 0.0, 4.0, 4.5, 4.8, 5.01], labels=[0, 1, 2, 3, 4], ).astype(float) # Reputation score: weighted combination of rating and log(reviews) rating_norm = rating_filled / 5.0 review_norm = out["log_reviews"] / (out["log_reviews"].max() or 1.0) out["reputation_score"] = 0.6 * rating_norm + 0.4 * review_norm # Cleanliness premium proxy: high rating (>= 4.8) with many reviews signals cleanliness out["cleanliness_premium"] = ( (rating_filled >= 4.8) & (out["num_reviews"].fillna(0) >= 10) ).astype(int) # Location rating premium proxy: high rating in a high-density area (use quality_score if helpful) out["location_rating_premium"] = ( (rating_filled >= 4.7) & (out["quality_score"] >= out["quality_score"].median()) ).astype(int) # Has reviews flag out["has_reviews"] = (out["num_reviews"].fillna(0) > 0).astype(int) # No rating flag (for imputation awareness) out["no_rating_flag"] = out["rating"].isna().astype(int) logger.info("Review features created: %d new columns", out.shape[1] - df.shape[1]) return out # --------------------------------------------------------------------------- # Price segment features # --------------------------------------------------------------------------- def create_price_segment_features(df: pd.DataFrame) -> pd.DataFrame: """Categorize listings into price segments based on quantiles.""" out = df.copy() price = out["price_numeric"] valid_mask = price.notna() # Quantile boundaries on valid prices q25, q50, q75 = price.quantile([0.25, 0.50, 0.75]) conditions = [ price <= q25, (price > q25) & (price <= q50), (price > q50) & (price <= q75), price > q75, ] choices = ["budget", "moderate", "premium", "luxury"] out["price_segment"] = np.select(conditions, choices, default="unknown") # Numeric encoding segment_map = {"budget": 0, "moderate": 1, "premium": 2, "luxury": 3, "unknown": -1} out["price_segment_code"] = out["price_segment"].map(segment_map).astype(int) # Log price (primary dependent variable in hedonic models) out["log_price"] = np.log1p(price) # Price per guest guests_safe = out["guests_count"].replace(0, np.nan) out["price_per_guest"] = price / guests_safe # Price per bedroom bedrooms_safe = out["bedrooms"].replace(0, np.nan) out["price_per_bedroom"] = price / bedrooms_safe logger.info("Price segment features created: %d new columns", out.shape[1] - df.shape[1]) return out # --------------------------------------------------------------------------- # Master pipeline # --------------------------------------------------------------------------- def create_all_features(df: pd.DataFrame) -> pd.DataFrame: """Run the full feature engineering pipeline in order.""" logger.info("Starting feature engineering pipeline on %d rows", len(df)) out = df.copy() out = create_structural_features(out) out = create_host_features(out) out = create_review_features(out) out = create_price_segment_features(out) logger.info("Pipeline complete: %d columns total", out.shape[1]) return out # --------------------------------------------------------------------------- # Feature group registry # --------------------------------------------------------------------------- def get_feature_groups() -> dict[str, list[str]]: """Return a dict mapping group names to lists of feature column names.""" return { "structural": [ "log_accommodates", "log_bedrooms", "log_beds", "log_bathrooms", "bathrooms_per_guest", "bedrooms_per_guest", "is_entire_home", "is_private_room", "is_shared_room", "large_capacity_listing", "accommodates_sq", "bedrooms_sq", "bathrooms_sq", "accommodates_bedrooms", "accommodates_bathrooms", ], "host": [ "superhost_flag", "host_professional_dummy", "multi_listing_host", "superhost_x_entire_home", "guest_favorite_flag", ], "review": [ "log_reviews", "review_quality", "reputation_score", "cleanliness_premium", "location_rating_premium", "has_reviews", "no_rating_flag", ], "price_segment": [ "price_segment", "price_segment_code", "log_price", "price_per_guest", "price_per_bedroom", ], "amenity": [ "amenity_richness_index", "luxury_index", "business_travel_index", "family_comfort_index", "leisure_index", "remote_work_index", ], "spatial": [ "distance_to_center", "local_density_500m", "local_density_1km", "local_avg_price_500m", "local_avg_price_1km", "spatial_cluster", ], }