Python 64.6%
TeX 23.3%
HTML 11.6%
Makefile 0.5%
1"""2NLP Feature Extraction for Airbnb Hedonic Pricing Analysis.34Extracts linguistic, semantic, and structural features from listing text fields5(primarily 'name', but also 'description', 'neighborhood_overview', 'host_about'6when available in richer datasets).78Supports bilingual (French/English) Airbnb listings common in Quebec markets.910Functions11---------12extract_basic_text_features : Character-level and token-level text statistics.13extract_language_scores : Keyword-based thematic scoring (luxury, cozy, ...).14extract_property_info_from_name : Structured parsing of the Airbnb name template.15create_text_embeddings : TF-IDF or CountVectorizer with TruncatedSVD reduction.16find_price_associated_words : Correlation of TF-IDF terms with (log) price.17create_all_nlp_features : Master pipeline combining all of the above.18"""1920from __future__ import annotations2122import logging23import re24from typing import Dict, List, Optional, Tuple2526import numpy as np27import pandas as pd28from scipy import stats29from sklearn.decomposition import TruncatedSVD30from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer3132logger = logging.getLogger(__name__)3334# ---------------------------------------------------------------------------35# Keyword dictionaries (bilingual FR/EN)36# ---------------------------------------------------------------------------3738LANGUAGE_KEYWORDS: Dict[str, List[str]] = {39 "luxury_language_score": [40 "luxe", "luxury", "premium", "elegant", "chic", "spa",41 "designer", "upscale", "exclusive", "prestige", "haut de gamme",42 "somptueux", "raffiné", "opulent", "magnifique",43 ],44 "cozy_language_score": [45 "cozy", "cosy", "chaleureux", "douillet", "confortable",46 "warm", "charming", "intime", "accueillant", "nid",47 "mignon", "petit coin", "cocoon",48 ],49 "professional_tone_score": [50 "professional", "modern", "clean", "bright", "renovated",51 "new", "stylish", "design", "moderne", "lumineux",52 "rénové", "neuf", "propre", "contemporain", "minimaliste",53 ],54 "family_language_score": [55 "family", "famille", "kids", "enfants", "children",56 "spacious", "grand", "spacieux", "familial", "aire de jeux",57 "playground", "bébé", "baby",58 ],59 "business_language_score": [60 "business", "workspace", "bureau", "wifi", "downtown",61 "centre-ville", "corporate", "professionnel", "travail",62 "work", "desk", "office",63 ],64 "romantic_language_score": [65 "romantic", "romantique", "intimate", "couple",66 "retreat", "getaway", "escapade", "amoureux",67 "lune de miel", "honeymoon", "chandelle",68 ],69 "tourism_language_score": [70 "tourist", "touristique", "attractions", "view", "vue",71 "panoramic", "nature", "montagne", "lac", "ski",72 "mountain", "lake", "panoramique", "plein air",73 "randonnée", "hiking", "plage", "beach", "forêt", "forest",74 ],75}767778# ---------------------------------------------------------------------------79# 1. Basic text features80# ---------------------------------------------------------------------------8182def extract_basic_text_features(df: pd.DataFrame) -> pd.DataFrame:83 """Extract character-level and token-level statistics from listing text.8485 Parameters86 ----------87 df : pd.DataFrame88 Must contain a ``name`` column. Optionally ``description``,89 ``neighborhood_overview``, and ``host_about``.9091 Returns92 -------93 pd.DataFrame94 Copy of *df* with appended feature columns:95 ``title_length``, ``title_word_count``, ``exclamation_count``,96 ``uppercase_ratio``, ``has_star_rating``, ``extracted_star_rating``.97 """98 logger.info("Extracting basic text features ...")99 result = df.copy()100101 name_series: pd.Series = result["name"].fillna("")102103 # Character count104 result["title_length"] = name_series.str.len()105106 # Word count (split on whitespace)107 result["title_word_count"] = name_series.str.split().str.len().fillna(0).astype(int)108109 # Exclamation marks110 result["exclamation_count"] = name_series.str.count("!")111112 # Upper-case ratio (letters only; avoid division by zero)113 alpha_chars = name_series.str.replace(r"[^A-Za-zÀ-ÿ]", "", regex=True)114 upper_chars = name_series.str.replace(r"[^A-ZÀ-Ý]", "", regex=True)115 alpha_len = alpha_chars.str.len().replace(0, np.nan)116 result["uppercase_ratio"] = (upper_chars.str.len() / alpha_len).fillna(0.0)117118 # Star rating detection119 result["has_star_rating"] = name_series.str.contains("★", regex=False)120121 # Numeric star rating extraction (e.g. ★4.94 or ★5.0)122 extracted = name_series.str.extract(r"★(\d+\.?\d*)", expand=False)123 result["extracted_star_rating"] = pd.to_numeric(extracted, errors="coerce")124125 # --- Optional: description-level features (if column exists) -----------126 for col in ("description", "neighborhood_overview", "host_about"):127 if col in result.columns:128 text = result[col].fillna("")129 result[f"{col}_length"] = text.str.len()130 result[f"{col}_word_count"] = text.str.split().str.len().fillna(0).astype(int)131 logger.info(" Added length/word_count for '%s'.", col)132133 logger.info("Basic text features extracted: %d rows.", len(result))134 return result135136137# ---------------------------------------------------------------------------138# 2. Language / keyword scores139# ---------------------------------------------------------------------------140141def _score_keywords(text: str, keywords: List[str]) -> int:142 """Count how many distinct keywords appear in *text* (case-insensitive)."""143 text_lower = text.lower()144 return sum(1 for kw in keywords if kw in text_lower)145146147def extract_language_scores(148 df: pd.DataFrame,149 text_columns: Optional[List[str]] = None,150) -> pd.DataFrame:151 """Score each listing on thematic keyword dimensions.152153 For every theme defined in ``LANGUAGE_KEYWORDS`` the score equals the154 number of matching keywords found in the concatenated text of all155 *text_columns* (case-insensitive substring matching).156157 Parameters158 ----------159 df : pd.DataFrame160 Input dataframe.161 text_columns : list[str] | None162 Columns to scan. Defaults to ``['name']`` plus any of163 ``description``, ``neighborhood_overview``, ``host_about`` that164 exist in *df*.165166 Returns167 -------168 pd.DataFrame169 Copy with one new column per theme key in ``LANGUAGE_KEYWORDS``.170 """171 logger.info("Extracting language keyword scores ...")172 result = df.copy()173174 if text_columns is None:175 text_columns = ["name"]176 for optional in ("description", "neighborhood_overview", "host_about"):177 if optional in result.columns:178 text_columns.append(optional)179180 # Build a single concatenated text per row181 combined_text: pd.Series = (182 result[text_columns]183 .fillna("")184 .apply(lambda row: " ".join(row), axis=1)185 )186187 for score_name, keywords in LANGUAGE_KEYWORDS.items():188 result[score_name] = combined_text.apply(_score_keywords, keywords=keywords)189 logger.debug(" %s mean=%.3f", score_name, result[score_name].mean())190191 logger.info("Language scores computed for %d themes.", len(LANGUAGE_KEYWORDS))192 return result193194195# ---------------------------------------------------------------------------196# 3. Structured name parsing197# ---------------------------------------------------------------------------198199# Pre-compiled patterns for the Airbnb name template:200# "PropertyType in CityName · ★Rating · N bedroom(s) · N bed(s) · N bath(s)"201_RE_CITY = re.compile(202 r"^.+?\s+in\s+(.+?)(?:\s+·|$)", re.IGNORECASE203)204_RE_BEDROOMS = re.compile(205 r"(\d+)\s+bedroom", re.IGNORECASE206)207_RE_STUDIO = re.compile(208 r"\bStudio\b", re.IGNORECASE209)210_RE_BEDS = re.compile(211 r"(\d+)\s+beds?\b", re.IGNORECASE212)213_RE_BATHS = re.compile(214 r"([\d.]+)\s+(?:shared\s+)?(?:private\s+)?bath", re.IGNORECASE215)216_RE_HALF_BATH = re.compile(217 r"\bHalf-bath\b", re.IGNORECASE218)219_RE_NEW = re.compile(220 r"★New\b", re.IGNORECASE221)222_RE_PROPERTY_TYPE = re.compile(223 r"^(.+?)\s+in\s+", re.IGNORECASE224)225226227def extract_property_info_from_name(df: pd.DataFrame) -> pd.DataFrame:228 """Parse the structured Airbnb ``name`` field into discrete features.229230 Parameters231 ----------232 df : pd.DataFrame233 Must contain column ``name``.234235 Returns236 -------237 pd.DataFrame238 Copy with columns: ``parsed_property_type``, ``parsed_city``,239 ``parsed_bedrooms``, ``parsed_beds``, ``parsed_baths``,240 ``is_studio``, ``is_new_listing``, ``has_shared_bath``,241 ``has_private_bath``.242 """243 logger.info("Parsing property info from name field ...")244 result = df.copy()245246 names: pd.Series = result["name"].fillna("")247248 # --- Parsed property type ------------------------------------------------249 result["parsed_property_type"] = names.str.extract(250 _RE_PROPERTY_TYPE, expand=False251 )252253 # --- City ----------------------------------------------------------------254 result["parsed_city"] = names.str.extract(_RE_CITY, expand=False)255256 # --- Bedrooms (0 for Studio) --------------------------------------------257 bedroom_match = names.str.extract(_RE_BEDROOMS, expand=False)258 is_studio = names.str.contains(_RE_STUDIO, regex=True)259 result["parsed_bedrooms"] = pd.to_numeric(bedroom_match, errors="coerce")260 result.loc[is_studio & result["parsed_bedrooms"].isna(), "parsed_bedrooms"] = 0261 result["is_studio"] = is_studio262263 # --- Beds ----------------------------------------------------------------264 beds_match = names.str.extract(_RE_BEDS, expand=False)265 result["parsed_beds"] = pd.to_numeric(beds_match, errors="coerce")266267 # --- Baths (handles "1.5 baths", "Half-bath", "shared bath") ------------268 baths_match = names.str.extract(_RE_BATHS, expand=False)269 result["parsed_baths"] = pd.to_numeric(baths_match, errors="coerce")270 # Handle "Half-bath" => 0.5271 half_bath_mask = names.str.contains(_RE_HALF_BATH, regex=True)272 result.loc[half_bath_mask & result["parsed_baths"].isna(), "parsed_baths"] = 0.5273274 # --- Qualifiers ----------------------------------------------------------275 result["is_new_listing"] = names.str.contains(_RE_NEW, regex=True)276 result["has_shared_bath"] = names.str.contains(r"shared\s+bath", case=False, regex=True)277 result["has_private_bath"] = names.str.contains(r"private\s+bath", case=False, regex=True)278279 logger.info("Name parsing complete: %d rows.", len(result))280 return result281282283# ---------------------------------------------------------------------------284# 4. Text embeddings (TF-IDF / CountVectorizer + TruncatedSVD)285# ---------------------------------------------------------------------------286287def create_text_embeddings(288 df: pd.DataFrame,289 method: str = "tfidf",290 n_components: int = 10,291 text_columns: Optional[List[str]] = None,292 max_features: int = 5000,293 ngram_range: Tuple[int, int] = (1, 2),294 random_state: int = 42,295) -> pd.DataFrame:296 """Create dense low-dimensional text embeddings via TF-IDF/Count + SVD.297298 Parameters299 ----------300 df : pd.DataFrame301 Input data.302 method : {'tfidf', 'count'}303 Vectoriser method.304 n_components : int305 Number of SVD components to retain.306 text_columns : list[str] | None307 Text columns to concatenate. Defaults to ``['name']`` + optional308 ``description`` if present.309 max_features : int310 Maximum vocabulary size for the vectoriser.311 ngram_range : tuple[int, int]312 N-gram range passed to the vectoriser.313 random_state : int314 Seed for ``TruncatedSVD``.315316 Returns317 -------318 pd.DataFrame319 Copy with ``text_embed_0`` ... ``text_embed_{n_components-1}`` appended.320 """321 logger.info("Creating text embeddings (method=%s, n_components=%d) ...", method, n_components)322 result = df.copy()323324 if text_columns is None:325 text_columns = ["name"]326 if "description" in result.columns:327 text_columns.append("description")328329 # Concatenate and clean330 corpus: pd.Series = (331 result[text_columns]332 .fillna("")333 .apply(lambda row: " ".join(row), axis=1)334 )335 corpus = corpus.str.lower()336 corpus = corpus.str.replace(r"[^\w\sÀ-ÿ]", " ", regex=True) # strip punctuation337 corpus = corpus.str.replace(r"\s+", " ", regex=True).str.strip()338339 # Replace empty strings to avoid all-zero rows340 corpus = corpus.replace("", "empty_listing")341342 # Vectorise343 if method == "count":344 vectoriser = CountVectorizer(345 max_features=max_features,346 ngram_range=ngram_range,347 stop_words=None, # keep French words348 )349 else:350 vectoriser = TfidfVectorizer(351 max_features=max_features,352 ngram_range=ngram_range,353 stop_words=None,354 sublinear_tf=True,355 )356357 dtm = vectoriser.fit_transform(corpus)358 logger.info(" Vocabulary size: %d, document-term matrix shape: %s", len(vectoriser.vocabulary_), dtm.shape)359360 # Ensure n_components does not exceed matrix rank361 effective_components = min(n_components, dtm.shape[1] - 1, dtm.shape[0] - 1)362 if effective_components < n_components:363 logger.warning(364 "Reducing n_components from %d to %d (matrix rank constraint).",365 n_components, effective_components,366 )367368 if effective_components < 1:369 logger.warning("Not enough data for SVD; skipping embeddings.")370 return result371372 svd = TruncatedSVD(n_components=effective_components, random_state=random_state)373 embeddings = svd.fit_transform(dtm)374375 explained = svd.explained_variance_ratio_.sum()376 logger.info(" SVD explained variance: %.2f%%", explained * 100)377378 embed_cols = [f"text_embed_{i}" for i in range(effective_components)]379 embed_df = pd.DataFrame(embeddings, columns=embed_cols, index=result.index)380 result = pd.concat([result, embed_df], axis=1)381382 # Pad with zeros if we reduced components383 for i in range(effective_components, n_components):384 result[f"text_embed_{i}"] = 0.0385386 logger.info("Text embeddings added: %d components.", n_components)387 return result388389390# ---------------------------------------------------------------------------391# 5. Price-associated words392# ---------------------------------------------------------------------------393394def find_price_associated_words(395 df: pd.DataFrame,396 price_col: str = "log_price",397 top_n: int = 20,398 text_columns: Optional[List[str]] = None,399 max_features: int = 3000,400) -> pd.DataFrame:401 """Identify words most correlated with listing price.402403 Builds a TF-IDF matrix from listing text and computes Pearson404 correlation of each term column with the price variable.405406 Parameters407 ----------408 df : pd.DataFrame409 Must contain *price_col* and at least ``name``.410 price_col : str411 Column with (log-)price values. Rows with NaN are excluded.412 top_n : int413 Number of top positively and negatively correlated words.414 text_columns : list[str] | None415 Text columns to concatenate.416 max_features : int417 Maximum vocabulary size.418419 Returns420 -------421 pd.DataFrame422 Columns: ``word``, ``correlation``, ``abs_correlation``,423 ``direction`` ('positive' | 'negative').424 Sorted by absolute correlation descending.425 """426 logger.info("Finding price-associated words (price_col=%s, top_n=%d) ...", price_col, top_n)427428 if text_columns is None:429 text_columns = ["name"]430 if "description" in df.columns:431 text_columns.append("description")432433 # Work on rows with valid price434 if price_col not in df.columns:435 # Attempt to create log_price on the fly436 if "price_numeric" in df.columns:437 work = df.copy()438 work["log_price"] = np.log1p(pd.to_numeric(work["price_numeric"], errors="coerce"))439 price_col = "log_price"440 else:441 logger.error("Price column '%s' not found. Cannot compute word correlations.", price_col)442 return pd.DataFrame(columns=["word", "correlation", "abs_correlation", "direction"])443444 else:445 work = df.copy()446447 mask = work[price_col].notna()448 work = work.loc[mask].reset_index(drop=True)449450 if len(work) < 30:451 logger.warning("Too few rows with valid price (%d); skipping word-price correlation.", len(work))452 return pd.DataFrame(columns=["word", "correlation", "abs_correlation", "direction"])453454 # Concatenate text455 corpus = (456 work[text_columns]457 .fillna("")458 .apply(lambda row: " ".join(row), axis=1)459 .str.lower()460 .str.replace(r"[^\w\sÀ-ÿ]", " ", regex=True)461 .str.replace(r"\s+", " ", regex=True)462 .str.strip()463 )464465 vectoriser = TfidfVectorizer(466 max_features=max_features,467 ngram_range=(1, 2),468 stop_words=None,469 sublinear_tf=True,470 min_df=5,471 )472 dtm = vectoriser.fit_transform(corpus)473 feature_names = vectoriser.get_feature_names_out()474 price_values = work[price_col].values475476 # Compute correlations477 correlations = np.zeros(len(feature_names))478 dtm_dense = dtm.toarray()479 for i in range(len(feature_names)):480 col_vals = dtm_dense[:, i]481 if col_vals.std() == 0:482 correlations[i] = 0.0483 else:484 correlations[i], _ = stats.pearsonr(col_vals, price_values)485486 corr_df = pd.DataFrame({487 "word": feature_names,488 "correlation": correlations,489 })490 corr_df["abs_correlation"] = corr_df["correlation"].abs()491 corr_df["direction"] = np.where(corr_df["correlation"] >= 0, "positive", "negative")492493 # Select top positive and top negative494 top_positive = corr_df.nlargest(top_n, "correlation")495 top_negative = corr_df.nsmallest(top_n, "correlation")496 combined = (497 pd.concat([top_positive, top_negative])498 .drop_duplicates(subset=["word"])499 .sort_values("abs_correlation", ascending=False)500 .reset_index(drop=True)501 )502503 logger.info(504 "Top correlated word: '%s' (r=%.3f); bottom: '%s' (r=%.3f)",505 combined.iloc[0]["word"],506 combined.iloc[0]["correlation"],507 combined.iloc[-1]["word"],508 combined.iloc[-1]["correlation"],509 )510 return combined511512513# ---------------------------------------------------------------------------514# 6. Master function515# ---------------------------------------------------------------------------516517def create_all_nlp_features(518 df: pd.DataFrame,519 embedding_method: str = "tfidf",520 n_components: int = 10,521 compute_embeddings: bool = True,522 compute_price_words: bool = True,523 price_col: str = "log_price",524 top_n_words: int = 20,525) -> Tuple[pd.DataFrame, Optional[pd.DataFrame]]:526 """Run the full NLP feature-extraction pipeline.527528 Parameters529 ----------530 df : pd.DataFrame531 Raw listing data (must have at least ``name``).532 embedding_method : {'tfidf', 'count'}533 Vectorisation method for embeddings.534 n_components : int535 Number of SVD dimensions.536 compute_embeddings : bool537 Whether to compute text embeddings.538 compute_price_words : bool539 Whether to compute price-associated word analysis.540 price_col : str541 Price column for word-price correlation.542 top_n_words : int543 Number of top words per direction.544545 Returns546 -------547 tuple[pd.DataFrame, pd.DataFrame | None]548 - Enriched DataFrame with all NLP features.549 - Price-word correlation DataFrame (or ``None`` if skipped).550 """551 logger.info("=" * 60)552 logger.info("NLP FEATURE EXTRACTION PIPELINE")553 logger.info("=" * 60)554 logger.info("Input shape: %s", df.shape)555556 if "name" not in df.columns:557 logger.error("Column 'name' not found. Returning input unchanged.")558 return df.copy(), None559560 # --- Step 1: basic text features ----------------------------------------561 enriched = extract_basic_text_features(df)562563 # --- Step 2: language keyword scores ------------------------------------564 enriched = extract_language_scores(enriched)565566 # --- Step 3: structured name parsing ------------------------------------567 enriched = extract_property_info_from_name(enriched)568569 # --- Step 4: text embeddings --------------------------------------------570 if compute_embeddings:571 enriched = create_text_embeddings(572 enriched,573 method=embedding_method,574 n_components=n_components,575 )576577 # --- Step 5: price-associated words -------------------------------------578 price_words: Optional[pd.DataFrame] = None579 if compute_price_words:580 price_words = find_price_associated_words(581 enriched,582 price_col=price_col,583 top_n=top_n_words,584 )585586 # --- Summary ------------------------------------------------------------587 new_cols = [c for c in enriched.columns if c not in df.columns]588 logger.info("NLP pipeline complete. %d new features added.", len(new_cols))589 logger.info("New columns: %s", new_cols)590 logger.info("Output shape: %s", enriched.shape)591592 return enriched, price_words593594595# ---------------------------------------------------------------------------596# CLI quick-test597# ---------------------------------------------------------------------------598599if __name__ == "__main__":600 import sys601602 logging.basicConfig(603 level=logging.INFO,604 format="%(asctime)s %(levelname)-8s %(message)s",605 stream=sys.stdout,606 )607608 sample_path = "cleaned_dataset_sample.csv"609 logger.info("Loading sample data from '%s' ...", sample_path)610 data = pd.read_csv(sample_path)611 logger.info("Loaded %d rows, %d columns.", *data.shape)612613 enriched_df, price_word_df = create_all_nlp_features(data)614615 print("\n--- Enriched DataFrame (first 5 rows, NLP columns) ---")616 nlp_cols = [617 "title_length", "title_word_count", "exclamation_count",618 "uppercase_ratio", "has_star_rating", "extracted_star_rating",619 "luxury_language_score", "cozy_language_score",620 "professional_tone_score", "family_language_score",621 "business_language_score", "romantic_language_score",622 "tourism_language_score",623 "parsed_property_type", "parsed_city",624 "parsed_bedrooms", "parsed_beds", "parsed_baths",625 "is_studio", "is_new_listing",626 "has_shared_bath", "has_private_bath",627 ]628 existing_nlp_cols = [c for c in nlp_cols if c in enriched_df.columns]629 print(enriched_df[existing_nlp_cols].head().to_string())630631 embed_cols = [c for c in enriched_df.columns if c.startswith("text_embed_")]632 if embed_cols:633 print(f"\n--- Text Embedding columns ({len(embed_cols)}) ---")634 print(enriched_df[embed_cols].describe().to_string())635636 if price_word_df is not None and not price_word_df.empty:637 print("\n--- Top Price-Associated Words ---")638 print(price_word_df.to_string(index=False))639 else:640 print("\n(No price-word correlations computed.)")641