spb/wp5_uqo Public

1# Author: Simon-Pierre Boucher — contact@spboucher.ai2"""302_clean_airbnb.py4------------------5Clean and standardize the Airbnb dataset.67Steps8  1.  Standardize city names (merge accented / unaccented variants).9  2.  Clean price_numeric: drop null / non-positive, winsorize at p1/p99.10  3.  Create log_price = ln(price_numeric).11  4.  Clean rating and num_reviews (fill NaN with median for rating, 0 for12      num_reviews — rationale documented below).13  5.  Create is_entire_home indicator from property_type.14  6.  Save to data/processed/airbnb_clean.parquet.15"""1617import sys18from pathlib import Path1920sys.path.insert(0, str(Path(__file__).resolve().parent.parent))2122import numpy as np23import pandas as pd2425from src.config import AIRBNB_RAW, AIRBNB_CLEAN, require2627RAW_HINT = ("Place the original airbnb.csv in data/raw/ to rebuild "28            "airbnb_clean.parquet; data/processed/ already contains the "29            "committed version.")3031# Mapping for known duplicates / accent variants.32CITY_MAP = {33    "Montréal":                  "Montreal",34    "Québec City":               "Québec",35    "Quebec City":               "Québec",36    "Quebec":                    "Québec",37    "Levis":                     "Lévis",38    "Saint Come":                "Saint-Côme",39    "Sainte Adele":              "Sainte-Adèle",40    "Sainte-Adele":              "Sainte-Adèle",41    "Saint Sauveur":             "Saint-Sauveur",42    "Saint-Sauveur-des-Monts":   "Saint-Sauveur",43    "Ste-Adèle":                 "Sainte-Adèle",44    "Ste-Agathe-des-Monts":      "Sainte-Agathe-des-Monts",45}4647# Entire-home types: House, Cabin/Chalet, Condo — i.e. standalone units that48# are typically rented in their entirety.  "Rental unit" and "Apartment" are49# often private rooms inside a larger building; "Other" is ambiguous.50ENTIRE_HOME_TYPES = {"House", "Cabin/Chalet", "Condo"}515253def main() -> None:54    require(AIRBNB_RAW, RAW_HINT)5556    # 1. Load57    df = pd.read_csv(AIRBNB_RAW)58    print(f"[load] Raw shape: {df.shape}")5960    # 2. Standardize city names61    df["city"] = df["city"].str.strip()62    df["city"] = df["city"].replace(CITY_MAP)6364    print(f"[city] Unique cities after standardisation: {df['city'].nunique()}")65    print(f"[city] Top 10 cities:\n{df['city'].value_counts().head(10).to_string()}\n")6667    # 3. Clean price_numeric68    n_before = len(df)69    n_null_price = df["price_numeric"].isnull().sum()70    n_nonpos = (df["price_numeric"] <= 0).sum()71    print(f"[price] Null prices: {n_null_price}")72    print(f"[price] Non-positive prices: {n_nonpos}")7374    # Drop rows with null or non-positive price75    df = df.dropna(subset=["price_numeric"])76    df = df[df["price_numeric"] > 0].copy()77    print(f"[price] Rows dropped: {n_before - len(df)} -> remaining: {len(df)}")7879    # Winsorize at 1st and 99th percentile80    p01 = df["price_numeric"].quantile(0.01)81    p99 = df["price_numeric"].quantile(0.99)82    print(f"[price] Winsorize bounds: p1={p01:.2f}, p99={p99:.2f}")83    df["price_numeric"] = df["price_numeric"].clip(lower=p01, upper=p99)8485    # Log transform86    df["log_price"] = np.log(df["price_numeric"])87    print(f"[price] price_numeric  mean={df['price_numeric'].mean():.2f}  "88          f"median={df['price_numeric'].median():.2f}")8990    # 4. Clean rating and num_reviews91    # Rating:  Fill NaN with the *median* of observed ratings.92    # Rationale: missing ratings typically mean "not yet rated", which may not93    # be 0.  Using the median avoids pulling the distribution toward zero and94    # is a conservative imputation that preserves the central tendency.95    rating_median = df["rating"].median()96    n_rating_null = df["rating"].isnull().sum()97    df["rating"] = df["rating"].fillna(rating_median)98    print(f"[rating] Filled {n_rating_null} NaN with median={rating_median:.2f}")99100    # num_reviews:  Fill NaN with 0.101    # Rationale: missing review count almost certainly means zero reviews.102    n_reviews_null = df["num_reviews"].isnull().sum()103    df["num_reviews"] = df["num_reviews"].fillna(0)104    print(f"[num_reviews] Filled {n_reviews_null} NaN with 0")105106    # 5. Create is_entire_home indicator107    df["is_entire_home"] = df["property_type"].isin(ENTIRE_HOME_TYPES)108    print(f"[is_entire_home] True: {df['is_entire_home'].sum()}, "109          f"False: {(~df['is_entire_home']).sum()}")110111    # 6. Save112    df.to_parquet(AIRBNB_CLEAN, index=False)113    print(f"\n>>> Saved cleaned Airbnb data ({len(df)} rows) to {AIRBNB_CLEAN}")114    print(f"    Columns: {list(df.columns)}")115116117if __name__ == "__main__":118    main()119