Python 64.6%
TeX 23.3%
HTML 11.6%
Makefile 0.5%
1"""2cleaning.py -- Data cleaning and feature engineering for hedonic pricing analysis.34Provides deterministic, auditable transformations that convert raw5Airbnb listing data into analysis-ready features. Every function6logs what it changes so that a full cleaning report can be generated.7"""89from __future__ import annotations1011import logging12import re13from dataclasses import dataclass, field14from typing import Any, Dict, List, Optional, Tuple, Union1516import numpy as np17import pandas as pd1819from src.config import (20 DEFAULT_WINSORIZE,21 WINSORIZE_1_99,22 WINSORIZE_2_98,23 WINSORIZE_5_95,24 PRICE_FLOOR,25 PRICE_CEILING,26 QUEBEC_LAT_RANGE,27 QUEBEC_LON_RANGE,28 PROPERTY_TYPE_MAP,29 ROOM_TYPE_MAP,30 LOG_LEVEL,31 LOG_FORMAT,32 LOG_DATE_FORMAT,33)3435# ---------------------------------------------------------------------------36# Logger37# ---------------------------------------------------------------------------3839logging.basicConfig(format=LOG_FORMAT, datefmt=LOG_DATE_FORMAT, level=LOG_LEVEL)40logger = logging.getLogger(__name__)4142# ---------------------------------------------------------------------------43# Cleaning audit record44# ---------------------------------------------------------------------------454647@dataclass48class CleaningRecord:49 """Accumulates counts and descriptions of every cleaning action."""5051 actions: List[Dict[str, Any]] = field(default_factory=list)5253 def log(54 self,55 step: str,56 column: str,57 description: str,58 rows_affected: int,59 total_rows: int,60 ) -> None:61 pct = 100.0 * rows_affected / total_rows if total_rows > 0 else 0.062 entry = {63 "step": step,64 "column": column,65 "description": description,66 "rows_affected": rows_affected,67 "total_rows": total_rows,68 "pct_affected": round(pct, 2),69 }70 self.actions.append(entry)71 logger.info(72 "[%s] %s | %s rows (%.1f%%): %s",73 step, column, f"{rows_affected:,}", pct, description,74 )7576 def to_dataframe(self) -> pd.DataFrame:77 return pd.DataFrame(self.actions)7879 def summary_dict(self) -> Dict[str, Any]:80 return {81 "total_actions": len(self.actions),82 "total_rows_affected": sum(a["rows_affected"] for a in self.actions),83 "steps": [a["step"] for a in self.actions],84 }858687# Module-level record -- reset by ``run_full_cleaning``88_record = CleaningRecord()899091def _get_record() -> CleaningRecord:92 return _record939495# ---------------------------------------------------------------------------96# 1. Price cleaning97# ---------------------------------------------------------------------------9899_PRICE_STRIP_RE = re.compile(r"[^\d.]")100101102def clean_price(103 df: pd.DataFrame,104 *,105 price_col: str = "price_numeric",106 raw_price_col: str = "price",107 floor: float = PRICE_FLOOR,108 ceiling: float = PRICE_CEILING,109 record: Optional[CleaningRecord] = None,110) -> pd.DataFrame:111 """Clean and standardise the price column.112113 Handles both the pre-parsed ``price_numeric`` column in the cleaned114 sample and the raw ``$1,234.00``-style ``price`` string column in115 full Inside Airbnb data.116117 Steps118 -----119 1. If *raw_price_col* exists and *price_col* does not, parse the120 raw column (strip ``$``, commas, whitespace) to float.121 2. Replace non-positive values with ``NaN``.122 3. Replace values outside ``[floor, ceiling]`` with ``NaN``.123 """124 rec = record or _get_record()125 n = len(df)126 df = df.copy()127128 # Parse raw string price if needed129 if raw_price_col in df.columns and price_col not in df.columns:130 raw = df[raw_price_col].astype(str)131 parsed = raw.apply(lambda s: _PRICE_STRIP_RE.sub("", s))132 df[price_col] = pd.to_numeric(parsed, errors="coerce")133 failed = df[price_col].isnull().sum() - df[raw_price_col].isnull().sum()134 rec.log("clean_price", price_col, "Parsed raw price string to numeric", max(failed, 0), n)135 elif price_col not in df.columns:136 logger.warning("Neither '%s' nor '%s' found in DataFrame", price_col, raw_price_col)137 return df138139 before_missing = int(df[price_col].isnull().sum())140141 # Non-positive -> NaN142 mask_nonpos = df[price_col] <= 0143 n_nonpos = int(mask_nonpos.sum())144 if n_nonpos > 0:145 df.loc[mask_nonpos, price_col] = np.nan146 rec.log("clean_price", price_col, f"Set {n_nonpos} non-positive prices to NaN", n_nonpos, n)147148 # Below floor -> NaN149 mask_low = df[price_col] < floor150 n_low = int(mask_low.sum())151 if n_low > 0:152 df.loc[mask_low, price_col] = np.nan153 rec.log("clean_price", price_col, f"Set {n_low} prices below floor (${floor}) to NaN", n_low, n)154155 # Above ceiling -> NaN156 mask_high = df[price_col] > ceiling157 n_high = int(mask_high.sum())158 if n_high > 0:159 df.loc[mask_high, price_col] = np.nan160 rec.log("clean_price", price_col, f"Set {n_high} prices above ceiling (${ceiling:,}) to NaN", n_high, n)161162 after_missing = int(df[price_col].isnull().sum())163 total_set_nan = after_missing - before_missing164 rec.log(165 "clean_price", price_col,166 f"Total new NaN from price cleaning: {total_set_nan} "167 f"({after_missing} total missing)",168 total_set_nan, n,169 )170171 return df172173174def _winsorize_series(s: pd.Series, lower: float, upper: float) -> pd.Series:175 """Winsorize *s* at the given quantile bounds, ignoring NaN."""176 q_lo = s.quantile(lower)177 q_hi = s.quantile(upper)178 return s.clip(lower=q_lo, upper=q_hi)179180181def create_price_variants(182 df: pd.DataFrame,183 *,184 price_col: str = "price_numeric",185 record: Optional[CleaningRecord] = None,186) -> pd.DataFrame:187 """Create multiple price representations for modelling.188189 Columns added190 -------------191 ``price_raw``192 Copy of the cleaned numeric price.193 ``price_winsor_1_99``194 Winsorized at 1st / 99th percentile.195 ``price_winsor_2_98``196 Winsorized at 2nd / 98th percentile.197 ``price_winsor_5_95``198 Winsorized at 5th / 95th percentile.199 ``log_price_raw``200 Natural log of raw price (NaN for non-positive).201 ``log_price_winsor``202 Natural log of the 1-99 winsorized price.203 """204 rec = record or _get_record()205 n = len(df)206 df = df.copy()207208 if price_col not in df.columns:209 logger.warning("Price column '%s' not found; skipping variant creation", price_col)210 return df211212 prices = df[price_col]213214 # Raw copy215 df["price_raw"] = prices.copy()216217 # Winsorized variants218 df["price_winsor_1_99"] = _winsorize_series(prices, *WINSORIZE_1_99)219 df["price_winsor_2_98"] = _winsorize_series(prices, *WINSORIZE_2_98)220 df["price_winsor_5_95"] = _winsorize_series(prices, *WINSORIZE_5_95)221222 # Log transforms (only for positive values)223 df["log_price_raw"] = np.where(prices > 0, np.log(prices), np.nan)224 df["log_price_winsor"] = np.where(225 df["price_winsor_1_99"] > 0,226 np.log(df["price_winsor_1_99"]),227 np.nan,228 )229230 variants_created = [231 "price_raw", "price_winsor_1_99", "price_winsor_2_98",232 "price_winsor_5_95", "log_price_raw", "log_price_winsor",233 ]234 rec.log(235 "create_price_variants", price_col,236 f"Created {len(variants_created)} price variants: {variants_created}",237 n, n,238 )239240 # Log winsorization bounds for reproducibility241 valid_prices = prices.dropna()242 if len(valid_prices) > 0:243 for label, lo, hi in [244 ("1-99", *WINSORIZE_1_99),245 ("2-98", *WINSORIZE_2_98),246 ("5-95", *WINSORIZE_5_95),247 ]:248 q_lo = valid_prices.quantile(lo)249 q_hi = valid_prices.quantile(hi)250 logger.info(251 "Winsor %s bounds: $%.2f -- $%.2f", label, q_lo, q_hi252 )253254 return df255256257# ---------------------------------------------------------------------------258# 2. Rating cleaning259# ---------------------------------------------------------------------------260261262def clean_rating(263 df: pd.DataFrame,264 *,265 rating_col: str = "rating",266 raw_rating_col: str = "review_scores_rating",267 record: Optional[CleaningRecord] = None,268) -> pd.DataFrame:269 """Parse and validate rating values.270271 Handles both pre-parsed ``rating`` (0-5 scale) and the Inside272 Airbnb ``review_scores_rating`` column (may be 0-100 scale in older273 extracts or 0-5 in newer ones).274 """275 rec = record or _get_record()276 n = len(df)277 df = df.copy()278279 # If raw_rating_col exists but rating_col does not, create it280 if raw_rating_col in df.columns and rating_col not in df.columns:281 df[rating_col] = pd.to_numeric(df[raw_rating_col], errors="coerce")282 rec.log("clean_rating", rating_col, "Parsed raw rating column to numeric", n, n)283284 if rating_col not in df.columns:285 logger.info("No rating column found; skipping rating cleaning")286 return df287288 # Convert old 0-100 scale to 0-5289 if df[rating_col].max() > 5.5:290 mask_100 = df[rating_col] > 5.5291 n_converted = int(mask_100.sum())292 df.loc[mask_100, rating_col] = df.loc[mask_100, rating_col] / 20.0293 rec.log(294 "clean_rating", rating_col,295 f"Converted {n_converted} ratings from 0-100 scale to 0-5",296 n_converted, n,297 )298299 # Clip to valid range300 before_nan = int(df[rating_col].isnull().sum())301 mask_invalid = (df[rating_col] < 0) | (df[rating_col] > 5)302 n_invalid = int(mask_invalid.sum())303 if n_invalid > 0:304 df.loc[mask_invalid, rating_col] = np.nan305 rec.log(306 "clean_rating", rating_col,307 f"Set {n_invalid} out-of-range ratings to NaN",308 n_invalid, n,309 )310311 # Create rating category if not present312 if "rating_category" not in df.columns:313 df["rating_category"] = pd.cut(314 df[rating_col],315 bins=[-np.inf, 4.0, 4.5, 4.7, 4.9, 5.01],316 labels=["Average (<4.0)", "Good (4.0-4.5)", "Very Good (4.5-4.7)",317 "Excellent (4.7-4.9)", "Exceptional (4.9-5.0)"],318 right=False,319 ).astype("object")320 df.loc[df[rating_col].isnull(), "rating_category"] = "No rating"321 rec.log("clean_rating", "rating_category", "Created rating_category from rating", n, n)322323 after_nan = int(df[rating_col].isnull().sum())324 rec.log(325 "clean_rating", rating_col,326 f"Rating cleaning complete. Missing: {after_nan} ({100*after_nan/n:.1f}%)",327 after_nan - before_nan, n,328 )329330 return df331332333# ---------------------------------------------------------------------------334# 3. Text field cleaning335# ---------------------------------------------------------------------------336337_WHITESPACE_RE = re.compile(r"\s+")338_HTML_TAG_RE = re.compile(r"<[^>]+>")339_URL_RE = re.compile(r"https?://\S+")340341342def clean_text_fields(343 df: pd.DataFrame,344 *,345 text_cols: Optional[List[str]] = None,346 record: Optional[CleaningRecord] = None,347) -> pd.DataFrame:348 """Clean free-text columns (name, description, neighborhood_overview, etc.).349350 Operations: strip HTML tags, normalise whitespace, strip leading/351 trailing whitespace, replace empty strings with NaN.352 """353 rec = record or _get_record()354 n = len(df)355 df = df.copy()356357 if text_cols is None:358 text_cols = [359 c for c in ("name", "description", "neighborhood_overview",360 "host_about", "transit", "access", "interaction",361 "house_rules", "notes")362 if c in df.columns363 ]364365 for col in text_cols:366 if col not in df.columns:367 continue368369 original_missing = int(df[col].isnull().sum())370 s = df[col].astype(str)371372 # Remove HTML373 s = s.str.replace(_HTML_TAG_RE, " ", regex=True)374 # Remove URLs375 s = s.str.replace(_URL_RE, " ", regex=True)376 # Collapse whitespace377 s = s.str.replace(_WHITESPACE_RE, " ", regex=True)378 # Strip379 s = s.str.strip()380 # Empty / 'nan' -> actual NaN381 s = s.replace({"": np.nan, "nan": np.nan, "None": np.nan, "N/A": np.nan})382383 df[col] = s384 new_missing = int(df[col].isnull().sum())385 cleaned_count = new_missing - original_missing386387 rec.log(388 "clean_text", col,389 f"Stripped HTML/URLs/whitespace; {cleaned_count} new NaN",390 max(cleaned_count, 0), n,391 )392393 return df394395396# ---------------------------------------------------------------------------397# 4. Host field cleaning398# ---------------------------------------------------------------------------399400_PCT_RE = re.compile(r"[\d.]+")401402403def _parse_pct_to_float(s: pd.Series) -> pd.Series:404 """Convert percentage strings like '87%' or '0.87' to float [0, 1]."""405 out = pd.to_numeric(s.astype(str).str.replace("%", "", regex=False), errors="coerce")406 # If values are > 1, assume they are 0-100 scale407 mask_pct = out > 1.0408 out = out.where(~mask_pct, out / 100.0)409 return out410411412def clean_host_fields(413 df: pd.DataFrame,414 *,415 record: Optional[CleaningRecord] = None,416) -> pd.DataFrame:417 """Parse host_response_rate, host_acceptance_rate to float [0, 1].418419 Also converts host_is_superhost / host_identity_verified from420 't'/'f' strings to boolean if needed.421 """422 rec = record or _get_record()423 n = len(df)424 df = df.copy()425426 # Response / acceptance rates427 for col in ("host_response_rate", "host_acceptance_rate"):428 if col not in df.columns:429 continue430 before_valid = int(df[col].notna().sum())431 df[col] = _parse_pct_to_float(df[col])432 # Clip to [0, 1]433 df[col] = df[col].clip(0.0, 1.0)434 after_valid = int(df[col].notna().sum())435 rec.log(436 "clean_host", col,437 f"Parsed to float [0,1]. Valid: {before_valid} -> {after_valid}",438 abs(after_valid - before_valid), n,439 )440441 # Boolean-like host columns ('t'/'f' -> True/False)442 bool_cols = [443 "host_is_superhost", "host_identity_verified",444 "host_has_profile_pic",445 ]446 for col in bool_cols:447 if col not in df.columns:448 continue449 if df[col].dtype == "bool":450 continue451 mapping = {"t": True, "f": False, "True": True, "False": False}452 df[col] = df[col].map(mapping)453 rec.log("clean_host", col, "Converted t/f strings to boolean", n, n)454455 # is_superhost: unify from host_is_superhost if our column is missing456 if "is_superhost" not in df.columns and "host_is_superhost" in df.columns:457 df["is_superhost"] = df["host_is_superhost"].fillna(False).astype(bool)458 rec.log("clean_host", "is_superhost", "Created from host_is_superhost", n, n)459460 # host_listings_count -> numeric461 if "host_listings_count" in df.columns:462 df["host_listings_count"] = pd.to_numeric(463 df["host_listings_count"], errors="coerce"464 )465466 # host_total_listings_count -> numeric467 if "host_total_listings_count" in df.columns:468 df["host_total_listings_count"] = pd.to_numeric(469 df["host_total_listings_count"], errors="coerce"470 )471472 return df473474475# ---------------------------------------------------------------------------476# 5. Coordinate cleaning477# ---------------------------------------------------------------------------478479480def clean_coordinates(481 df: pd.DataFrame,482 *,483 lat_col: str = "lat",484 lon_col: str = "long",485 lat_range: Tuple[float, float] = QUEBEC_LAT_RANGE,486 lon_range: Tuple[float, float] = QUEBEC_LON_RANGE,487 record: Optional[CleaningRecord] = None,488) -> pd.DataFrame:489 """Validate and clean geographic coordinates for Quebec listings.490491 Also handles the Inside Airbnb column names ``latitude`` /492 ``longitude`` by renaming them to ``lat`` / ``long``.493 """494 rec = record or _get_record()495 n = len(df)496 df = df.copy()497498 # Rename full names to short aliases if needed499 rename_map: Dict[str, str] = {}500 if "latitude" in df.columns and lat_col not in df.columns:501 rename_map["latitude"] = lat_col502 if "longitude" in df.columns and lon_col not in df.columns:503 rename_map["longitude"] = lon_col504 if rename_map:505 df.rename(columns=rename_map, inplace=True)506 rec.log(507 "clean_coords", "rename",508 f"Renamed columns: {rename_map}", len(rename_map), n,509 )510511 if lat_col not in df.columns or lon_col not in df.columns:512 logger.warning("Coordinate columns not found; skipping coordinate cleaning")513 return df514515 # Ensure numeric516 df[lat_col] = pd.to_numeric(df[lat_col], errors="coerce")517 df[lon_col] = pd.to_numeric(df[lon_col], errors="coerce")518519 # Flag out-of-range520 lat_min, lat_max = lat_range521 lon_min, lon_max = lon_range522523 mask_bad_lat = (df[lat_col] < lat_min) | (df[lat_col] > lat_max)524 mask_bad_lon = (df[lon_col] < lon_min) | (df[lon_col] > lon_max)525 mask_bad = mask_bad_lat | mask_bad_lon526 n_bad = int(mask_bad.sum())527528 if n_bad > 0:529 df.loc[mask_bad, lat_col] = np.nan530 df.loc[mask_bad, lon_col] = np.nan531 rec.log(532 "clean_coords", f"{lat_col}/{lon_col}",533 f"Set {n_bad} out-of-bounds coordinates to NaN "534 f"(lat: [{lat_min}, {lat_max}], lon: [{lon_min}, {lon_max}])",535 n_bad, n,536 )537538 # Flag missing coordinates539 mask_missing = df[lat_col].isnull() | df[lon_col].isnull()540 n_missing = int(mask_missing.sum())541 rec.log(542 "clean_coords", f"{lat_col}/{lon_col}",543 f"Total missing coordinates after cleaning: {n_missing}",544 n_missing, n,545 )546547 return df548549550# ---------------------------------------------------------------------------551# 6. Property type standardisation552# ---------------------------------------------------------------------------553554555def clean_property_type(556 df: pd.DataFrame,557 *,558 col: str = "property_type",559 mapping: Optional[Dict[str, str]] = None,560 record: Optional[CleaningRecord] = None,561) -> pd.DataFrame:562 """Standardise property types to a canonical set.563564 Uses ``PROPERTY_TYPE_MAP`` from config by default. Unmapped values565 are assigned ``"Other"``.566 """567 rec = record or _get_record()568 n = len(df)569 df = df.copy()570571 if col not in df.columns:572 logger.info("Column '%s' not found; skipping property type cleaning", col)573 return df574575 mapping = mapping or PROPERTY_TYPE_MAP576 original_unique = df[col].nunique()577578 # Map known types; assign "Other" to unknowns579 df[col] = df[col].map(mapping).fillna("Other")580581 new_unique = df[col].nunique()582 mapped_to_other = int((df[col] == "Other").sum())583584 rec.log(585 "clean_property_type", col,586 f"Standardised {original_unique} types -> {new_unique} types. "587 f"{mapped_to_other} mapped to 'Other'",588 n, n,589 )590591 return df592593594# ---------------------------------------------------------------------------595# 7. Room type standardisation596# ---------------------------------------------------------------------------597598599def clean_room_type(600 df: pd.DataFrame,601 *,602 col: str = "room_type",603 mapping: Optional[Dict[str, str]] = None,604 record: Optional[CleaningRecord] = None,605) -> pd.DataFrame:606 """Standardise room types to the canonical four-level set.607608 Maps case-insensitive variants to:609 ``Entire home/apt``, ``Private room``, ``Shared room``, ``Hotel room``.610 """611 rec = record or _get_record()612 n = len(df)613 df = df.copy()614615 if col not in df.columns:616 logger.info("Column '%s' not found; skipping room type cleaning", col)617 return df618619 mapping = mapping or ROOM_TYPE_MAP620 original_unique = df[col].nunique()621622 # Try direct map first, then lower-case map623 mapped = df[col].map(mapping)624 still_null = mapped.isnull()625 if still_null.any():626 lower_mapped = df.loc[still_null, col].str.lower().str.strip().map(mapping)627 mapped = mapped.fillna(lower_mapped)628629 # Anything still unmapped -> try substring matching630 still_null = mapped.isnull()631 if still_null.any():632 for idx in df.loc[still_null].index:633 raw = str(df.at[idx, col]).lower()634 if "entire" in raw:635 mapped.at[idx] = "Entire home/apt"636 elif "private" in raw:637 mapped.at[idx] = "Private room"638 elif "shared" in raw:639 mapped.at[idx] = "Shared room"640 elif "hotel" in raw:641 mapped.at[idx] = "Hotel room"642 else:643 mapped.at[idx] = "Entire home/apt" # default644645 df[col] = mapped646 new_unique = df[col].nunique()647648 rec.log(649 "clean_room_type", col,650 f"Standardised {original_unique} room types -> {new_unique}",651 n, n,652 )653654 return df655656657# ---------------------------------------------------------------------------658# 8. Additional numeric cleaning helpers659# ---------------------------------------------------------------------------660661662def _ensure_numeric(663 df: pd.DataFrame,664 cols: List[str],665 record: Optional[CleaningRecord] = None,666) -> pd.DataFrame:667 """Coerce columns to numeric, logging conversion failures."""668 rec = record or _get_record()669 n = len(df)670 for col in cols:671 if col not in df.columns:672 continue673 before_na = int(df[col].isnull().sum())674 df[col] = pd.to_numeric(df[col], errors="coerce")675 after_na = int(df[col].isnull().sum())676 new_na = after_na - before_na677 if new_na > 0:678 rec.log(679 "ensure_numeric", col,680 f"Coerced to numeric; {new_na} new NaN from unparseable values",681 new_na, n,682 )683 return df684685686def _clean_boolean(687 df: pd.DataFrame,688 cols: List[str],689 record: Optional[CleaningRecord] = None,690) -> pd.DataFrame:691 """Convert t/f/True/False/1/0 columns to proper bool dtype."""692 rec = record or _get_record()693 n = len(df)694 truthy = {"t", "true", "1", "yes", "True"}695 for col in cols:696 if col not in df.columns:697 continue698 if df[col].dtype == "bool":699 continue700 df[col] = df[col].astype(str).str.strip().str.lower().isin(truthy)701 rec.log("clean_boolean", col, "Converted to boolean dtype", n, n)702 return df703704705# ---------------------------------------------------------------------------706# 9. Inside Airbnb amenities parsing707# ---------------------------------------------------------------------------708709710def parse_amenities(711 df: pd.DataFrame,712 *,713 col: str = "amenities",714 record: Optional[CleaningRecord] = None,715) -> pd.DataFrame:716 """Parse the JSON-like amenities string into count and flag columns.717718 Inside Airbnb stores amenities as: ``'["Wifi", "Kitchen", ...]'``.719 We extract the count and create binary flags for key amenities.720 """721 rec = record or _get_record()722 n = len(df)723 df = df.copy()724725 if col not in df.columns:726 logger.info("Column '%s' not found; skipping amenity parsing", col)727 return df728729 def _count_amenities(val: Any) -> int:730 if pd.isnull(val):731 return 0732 s = str(val)733 # Count quoted items734 items = re.findall(r'"([^"]+)"', s)735 return len(items) if items else 0736737 def _has_amenity(val: Any, keyword: str) -> bool:738 if pd.isnull(val):739 return False740 return keyword.lower() in str(val).lower()741742 if "amenities_count" not in df.columns:743 df["amenities_count"] = df[col].apply(_count_amenities)744 rec.log("parse_amenities", "amenities_count", "Counted amenities from raw string", n, n)745746 # Key amenity flags747 key_amenities = {748 "has_wifi": "wifi",749 "has_kitchen": "kitchen",750 "has_parking": "parking",751 "has_pool": "pool",752 "has_ac": "air conditioning",753 "has_heating": "heating",754 "has_washer": "washer",755 "has_dryer": "dryer",756 "has_tv": "tv",757 "has_hot_tub": "hot tub",758 "has_fireplace": "fireplace",759 "has_elevator": "elevator",760 "has_gym": "gym",761 }762763 created_flags: List[str] = []764 for flag_col, keyword in key_amenities.items():765 if flag_col not in df.columns:766 df[flag_col] = df[col].apply(lambda v, kw=keyword: _has_amenity(v, kw))767 created_flags.append(flag_col)768769 if created_flags:770 rec.log(771 "parse_amenities", "amenity_flags",772 f"Created {len(created_flags)} binary amenity flags",773 n, n,774 )775776 return df777778779# ---------------------------------------------------------------------------780# 10. Master cleaning pipeline781# ---------------------------------------------------------------------------782783784def run_full_cleaning(785 df: pd.DataFrame,786 *,787 price_floor: float = PRICE_FLOOR,788 price_ceiling: float = PRICE_CEILING,789 create_variants: bool = True,790 parse_amenities_col: bool = True,791 verbose: bool = True,792) -> pd.DataFrame:793 """Execute the full cleaning pipeline in the canonical order.794795 Parameters796 ----------797 df : pd.DataFrame798 Raw or semi-cleaned listings data.799 price_floor / price_ceiling : float800 Plausible nightly price bounds (CAD).801 create_variants : bool802 Whether to create winsorized / log price columns.803 parse_amenities_col : bool804 Whether to parse the raw amenities string (if present).805 verbose : bool806 Extra logging.807808 Returns809 -------810 pd.DataFrame811 Cleaned DataFrame with all new columns appended.812 """813 global _record814 _record = CleaningRecord()815 rec = _record816817 n_original = len(df)818 logger.info("=" * 70)819 logger.info("STARTING FULL CLEANING PIPELINE | %s rows | %s cols", f"{n_original:,}", len(df.columns))820 logger.info("=" * 70)821822 # 1. Text fields823 df = clean_text_fields(df, record=rec)824825 # 2. Host fields (before price so is_superhost is available)826 df = clean_host_fields(df, record=rec)827828 # 3. Prices829 df = clean_price(df, floor=price_floor, ceiling=price_ceiling, record=rec)830 if create_variants:831 df = create_price_variants(df, record=rec)832833 # 4. Ratings834 df = clean_rating(df, record=rec)835836 # 5. Coordinates837 df = clean_coordinates(df, record=rec)838839 # 6. Property type840 df = clean_property_type(df, record=rec)841842 # 7. Room type843 df = clean_room_type(df, record=rec)844845 # 8. Amenities (raw string parsing)846 if parse_amenities_col:847 df = parse_amenities(df, record=rec)848849 # 9. Ensure key numeric columns850 numeric_cols = [851 "bedrooms", "bathrooms", "guests_count", "accommodates",852 "beds", "amenities_count", "num_images", "num_reviews",853 "quality_score", "number_of_reviews",854 "minimum_nights", "maximum_nights",855 "availability_30", "availability_60", "availability_90", "availability_365",856 "reviews_per_month", "calculated_host_listings_count",857 ]858 df = _ensure_numeric(df, [c for c in numeric_cols if c in df.columns], record=rec)859860 # 10. Ensure boolean columns861 bool_cols = ["is_superhost", "is_guest_favorite", "pets_allowed",862 "instant_bookable"]863 df = _clean_boolean(df, [c for c in bool_cols if c in df.columns], record=rec)864865 # Summary866 n_final = len(df)867 n_cols_final = len(df.columns)868 logger.info("=" * 70)869 logger.info(870 "CLEANING COMPLETE | %s rows | %s cols | %s actions logged",871 f"{n_final:,}", n_cols_final, len(rec.actions),872 )873 logger.info("=" * 70)874875 return df876877878# ---------------------------------------------------------------------------879# 11. Cleaning report880# ---------------------------------------------------------------------------881882883def generate_cleaning_report(884 df_before: pd.DataFrame,885 df_after: pd.DataFrame,886 *,887 record: Optional[CleaningRecord] = None,888) -> Dict[str, Any]:889 """Generate a comprehensive report comparing pre- and post-cleaning data.890891 Parameters892 ----------893 df_before : pd.DataFrame894 The DataFrame *before* cleaning.895 df_after : pd.DataFrame896 The DataFrame *after* cleaning.897 record : CleaningRecord, optional898 The cleaning audit trail; defaults to the module-level record.899900 Returns901 -------902 dict903 A structured report suitable for logging or serialisation.904 """905 rec = record or _get_record()906907 report: Dict[str, Any] = {908 "rows_before": len(df_before),909 "rows_after": len(df_after),910 "rows_dropped": len(df_before) - len(df_after),911 "cols_before": len(df_before.columns),912 "cols_after": len(df_after.columns),913 "cols_added": sorted(set(df_after.columns) - set(df_before.columns)),914 "cols_removed": sorted(set(df_before.columns) - set(df_after.columns)),915 }916917 # Missing value comparison918 missing_before: Dict[str, int] = {}919 missing_after: Dict[str, int] = {}920 for col in df_before.columns:921 missing_before[col] = int(df_before[col].isnull().sum())922 for col in df_after.columns:923 missing_after[col] = int(df_after[col].isnull().sum())924 report["missing_before"] = missing_before925 report["missing_after"] = missing_after926927 # Price comparison928 if "price_numeric" in df_before.columns and "price_numeric" in df_after.columns:929 p_before = df_before["price_numeric"].dropna()930 p_after = df_after["price_numeric"].dropna()931 report["price_comparison"] = {932 "valid_before": len(p_before),933 "valid_after": len(p_after),934 "mean_before": float(p_before.mean()) if len(p_before) > 0 else None,935 "mean_after": float(p_after.mean()) if len(p_after) > 0 else None,936 "median_before": float(p_before.median()) if len(p_before) > 0 else None,937 "median_after": float(p_after.median()) if len(p_after) > 0 else None,938 "std_before": float(p_before.std()) if len(p_before) > 0 else None,939 "std_after": float(p_after.std()) if len(p_after) > 0 else None,940 "min_before": float(p_before.min()) if len(p_before) > 0 else None,941 "min_after": float(p_after.min()) if len(p_after) > 0 else None,942 "max_before": float(p_before.max()) if len(p_before) > 0 else None,943 "max_after": float(p_after.max()) if len(p_after) > 0 else None,944 }945946 # Cleaning actions log947 report["actions"] = rec.actions948 report["total_actions"] = len(rec.actions)949950 # Log the report highlights951 logger.info("-" * 70)952 logger.info("CLEANING REPORT")953 logger.info("-" * 70)954 logger.info("Rows: %s -> %s (dropped %s)", f"{report['rows_before']:,}",955 f"{report['rows_after']:,}", report["rows_dropped"])956 logger.info("Columns: %s -> %s", report["cols_before"], report["cols_after"])957 if report["cols_added"]:958 logger.info("Columns added: %s", report["cols_added"])959 if report["cols_removed"]:960 logger.info("Columns removed: %s", report["cols_removed"])961 logger.info("Total cleaning actions: %s", report["total_actions"])962 logger.info("-" * 70)963964 return report965