Python 64.6%
TeX 23.3%
HTML 11.6%
Makefile 0.5%
1"""2report.py -- Price decomposition and reporting for hedonic pricing analysis.34Provides functions to decompose predicted prices into interpretable5components (structural, amenity, host, review, location, text, spatial),6identify the strongest positive and negative price drivers for individual7listings, and orchestrate the full output-generation pipeline (tables,8figures, decomposition summaries).9"""1011from __future__ import annotations1213import logging14import time15from pathlib import Path16from typing import Any, Dict, List, Optional, Tuple1718import numpy as np19import pandas as pd2021from src import config2223logger = logging.getLogger(__name__)2425# ---------------------------------------------------------------------------26# 1. Single-listing price decomposition27# ---------------------------------------------------------------------------2829def explain_listing_price(30 model: Any,31 df: pd.DataFrame,32 listing_idx: int,33 feature_groups: Dict[str, List[str]],34 *,35 n_top: int = 10,36) -> Dict[str, Any]:37 """Decompose a single listing's predicted price into component contributions.3839 Parameters40 ----------41 model : fitted statsmodels-style regression result42 Must expose ``.params`` (Series of coefficients) and, when the model43 includes an intercept, a ``const`` / ``Intercept`` key.44 df : pd.DataFrame45 The design matrix used for estimation (same columns as model.params).46 listing_idx : int47 Row position (iloc index) of the listing to explain.48 feature_groups : dict[str, list[str]]49 Mapping of group name -> list of feature column names. Expected50 keys: ``"structural"``, ``"amenity"``, ``"host"``, ``"review"``,51 ``"location"``, ``"text"``, ``"spatial"``. Missing keys are52 treated as empty.53 n_top : int54 Number of top positive / negative drivers to return.5556 Returns57 -------58 dict with keys:59 actual_price, predicted_price, prediction_error,60 base_component, property_component, amenity_component,61 host_component, review_component, location_component,62 text_component, spatial_component,63 top_positive_drivers, top_negative_drivers64 """65 row = df.iloc[listing_idx]66 params = model.params6768 # Determine which features actually exist in both the model and the data69 model_features = set(params.index)70 data_features = set(df.columns)71 common_features = model_features & data_features7273 # --- intercept / base component ---74 intercept_keys = {"const", "Intercept", "intercept"}75 intercept_name = intercept_keys & model_features76 base_component = float(params[intercept_name.pop()]) if intercept_name else 0.07778 # --- compute individual feature contributions ---79 contributions: Dict[str, float] = {}80 for feat in common_features:81 val = row.get(feat, 0.0)82 if pd.isna(val):83 val = 0.084 contributions[feat] = float(params[feat]) * float(val)8586 # --- aggregate by group ---87 group_aliases = {88 "structural": "property_component",89 "amenity": "amenity_component",90 "host": "host_component",91 "review": "review_component",92 "location": "location_component",93 "text": "text_component",94 "spatial": "spatial_component",95 }9697 component_sums: Dict[str, float] = {}98 assigned_features: set = set()99100 for group_key, component_name in group_aliases.items():101 group_feats = feature_groups.get(group_key, [])102 total = 0.0103 for feat in group_feats:104 if feat in contributions:105 total += contributions[feat]106 assigned_features.add(feat)107 component_sums[component_name] = total108109 # Features present in the model but not assigned to any group are folded110 # into property_component (structural catch-all).111 for feat, contrib in contributions.items():112 if feat not in assigned_features:113 component_sums["property_component"] = (114 component_sums.get("property_component", 0.0) + contrib115 )116117 # --- predicted price ---118 predicted_price = base_component + sum(contributions.values())119120 # --- actual price ---121 # Try common dependent-variable column names122 actual_price = np.nan123 for dep_col in ("log_price", "price_numeric", "price"):124 if dep_col in df.columns:125 actual_price = float(row[dep_col])126 break127128 prediction_error = actual_price - predicted_price129130 # --- top drivers ---131 sorted_contribs = sorted(contributions.items(), key=lambda x: x[1], reverse=True)132 top_positive = [(f, c) for f, c in sorted_contribs if c > 0][:n_top]133 top_negative = [(f, c) for f, c in sorted_contribs if c < 0][-n_top:]134 # Sort negatives by magnitude (most negative first)135 top_negative = sorted(top_negative, key=lambda x: x[1])136137 return {138 "actual_price": actual_price,139 "predicted_price": predicted_price,140 "prediction_error": prediction_error,141 "base_component": base_component,142 **component_sums,143 "top_positive_drivers": top_positive,144 "top_negative_drivers": top_negative,145 }146147148# ---------------------------------------------------------------------------149# 2. Full-dataset decomposition150# ---------------------------------------------------------------------------151152def decompose_all_prices(153 model: Any,154 df: pd.DataFrame,155 feature_groups: Dict[str, List[str]],156) -> pd.DataFrame:157 """Apply :func:`explain_listing_price` to every row and return a DataFrame.158159 Each row corresponds to a listing; columns include all scalar components160 produced by the single-listing function. The ``top_positive_drivers``161 and ``top_negative_drivers`` lists are excluded from the DataFrame (use162 :func:`explain_listing_price` for individual inspection).163164 Parameters165 ----------166 model : fitted statsmodels result167 df : pd.DataFrame168 feature_groups : dict[str, list[str]]169170 Returns171 -------172 pd.DataFrame173 """174 logger.info("Decomposing prices for %d listings ...", len(df))175 t0 = time.perf_counter()176177 records: List[Dict[str, float]] = []178 scalar_keys = [179 "actual_price", "predicted_price", "prediction_error",180 "base_component", "property_component", "amenity_component",181 "host_component", "review_component", "location_component",182 "text_component", "spatial_component",183 ]184185 for idx in range(len(df)):186 try:187 expl = explain_listing_price(model, df, idx, feature_groups)188 records.append({k: expl.get(k, 0.0) for k in scalar_keys})189 except Exception:190 logger.debug("Decomposition failed for listing idx=%d", idx, exc_info=True)191 records.append({k: np.nan for k in scalar_keys})192193 elapsed = time.perf_counter() - t0194 logger.info("Price decomposition complete in %.1f s", elapsed)195 return pd.DataFrame(records, index=df.index)196197198# ---------------------------------------------------------------------------199# 3. Decomposition summary statistics200# ---------------------------------------------------------------------------201202def generate_decomposition_summary(decomp_df: pd.DataFrame) -> pd.DataFrame:203 """Compute summary statistics for each price component.204205 Parameters206 ----------207 decomp_df : pd.DataFrame208 Output of :func:`decompose_all_prices`.209210 Returns211 -------212 pd.DataFrame213 Index = component names, columns = mean, median, std, min, max.214 """215 component_cols = [216 c for c in decomp_df.columns217 if c.endswith("_component") or c in ("predicted_price", "actual_price", "prediction_error")218 ]219 summary = decomp_df[component_cols].agg(["mean", "median", "std", "min", "max"]).T220 summary.columns = ["mean", "median", "std", "min", "max"]221 summary = summary.round(4)222 logger.info("Decomposition summary:\n%s", summary.to_string())223 return summary224225226# ---------------------------------------------------------------------------227# 4. Pretty-print a single listing explanation228# ---------------------------------------------------------------------------229230def format_decomposition_report(listing_explanation: Dict[str, Any]) -> str:231 """Return a human-readable string summarising a listing's price decomposition.232233 Parameters234 ----------235 listing_explanation : dict236 Output of :func:`explain_listing_price`.237238 Returns239 -------240 str241 """242 e = listing_explanation243 lines: List[str] = []244 sep = "-" * 64245246 lines.append(sep)247 lines.append("LISTING PRICE DECOMPOSITION REPORT")248 lines.append(sep)249 lines.append(f" Actual price : {e.get('actual_price', np.nan):>12.4f}")250 lines.append(f" Predicted price : {e.get('predicted_price', np.nan):>12.4f}")251 lines.append(f" Prediction error : {e.get('prediction_error', np.nan):>12.4f}")252 lines.append("")253 lines.append(" COMPONENTS")254 lines.append(f" Base (intercept) : {e.get('base_component', 0.0):>12.4f}")255 lines.append(f" Property : {e.get('property_component', 0.0):>12.4f}")256 lines.append(f" Amenity : {e.get('amenity_component', 0.0):>12.4f}")257 lines.append(f" Host : {e.get('host_component', 0.0):>12.4f}")258 lines.append(f" Review : {e.get('review_component', 0.0):>12.4f}")259 lines.append(f" Location : {e.get('location_component', 0.0):>12.4f}")260 lines.append(f" Text / NLP : {e.get('text_component', 0.0):>12.4f}")261 lines.append(f" Spatial : {e.get('spatial_component', 0.0):>12.4f}")262 lines.append("")263264 # Top positive drivers265 pos = e.get("top_positive_drivers", [])266 if pos:267 lines.append(" TOP POSITIVE DRIVERS")268 for feat, contrib in pos:269 lines.append(f" {feat:<40s} +{contrib:>10.4f}")270 lines.append("")271272 # Top negative drivers273 neg = e.get("top_negative_drivers", [])274 if neg:275 lines.append(" TOP NEGATIVE DRIVERS")276 for feat, contrib in neg:277 lines.append(f" {feat:<40s} {contrib:>10.4f}")278 lines.append("")279280 lines.append(sep)281 return "\n".join(lines)282283284# ---------------------------------------------------------------------------285# 5. Master report / output generation pipeline286# ---------------------------------------------------------------------------287288def generate_full_report(289 df: pd.DataFrame,290 models_dict: Dict[str, Any],291 ml_models_dict: Dict[str, Any],292 spatial_results: Dict[str, Any],293 feature_groups: Dict[str, List[str]],294 output_dir: str | Path = config.OUTPUTS_DIR,295) -> Dict[str, Any]:296 """Orchestrate full output generation: tables, figures, decomposition.297298 Calls the ``tables`` and ``figures`` modules to produce every artefact299 required for the research paper, then runs the price-decomposition300 pipeline and saves the summary.301302 Parameters303 ----------304 df : pd.DataFrame305 Feature-engineered dataset.306 models_dict : dict[str, Any]307 Mapping of model name -> fitted hedonic / quantile result.308 ml_models_dict : dict[str, Any]309 Mapping of model name -> fitted ML model or results dict.310 spatial_results : dict[str, Any]311 Output from the spatial-models module.312 feature_groups : dict[str, list[str]]313 Feature group registry.314 output_dir : str or Path315 Root output directory (tables/ and figures/ subdirectories are created).316317 Returns318 -------319 dict with keys: ``decomp_df``, ``decomp_summary``, ``tables_generated``,320 ``figures_generated``, ``elapsed_seconds``.321 """322 t0 = time.perf_counter()323 output_dir = Path(output_dir)324 tables_dir = output_dir / "tables"325 figures_dir = output_dir / "figures"326 tables_dir.mkdir(parents=True, exist_ok=True)327 figures_dir.mkdir(parents=True, exist_ok=True)328329 report_artifacts: Dict[str, Any] = {330 "tables_generated": [],331 "figures_generated": [],332 }333334 # ---- Tables ----335 logger.info("Generating tables ...")336 try:337 from src import tables as tbl_mod338339 table_funcs = [340 ("descriptive_stats", lambda: tbl_mod.generate_descriptive_stats_table(df, tables_dir)),341 ("correlation_matrix", lambda: tbl_mod.generate_correlation_table(df, tables_dir)),342 ("hedonic_results", lambda: tbl_mod.generate_hedonic_results_table(models_dict, tables_dir)),343 ("quantile_results", lambda: tbl_mod.generate_quantile_results_table(models_dict, tables_dir)),344 ("ml_comparison", lambda: tbl_mod.generate_ml_comparison_table(ml_models_dict, tables_dir)),345 ("spatial_results", lambda: tbl_mod.generate_spatial_results_table(spatial_results, tables_dir)),346 ]347348 for name, func in table_funcs:349 try:350 func()351 report_artifacts["tables_generated"].append(name)352 logger.info(" Table '%s' generated.", name)353 except Exception:354 logger.warning(" Table '%s' failed.", name, exc_info=True)355356 except ImportError:357 logger.warning("tables module not available; skipping table generation.")358 except Exception:359 logger.warning("Table generation failed.", exc_info=True)360361 # ---- Figures ----362 logger.info("Generating figures ...")363 try:364 from src import figures as fig_mod365366 figure_funcs = [367 ("price_distribution", lambda: fig_mod.plot_price_distribution(df, figures_dir)),368 ("correlation_heatmap", lambda: fig_mod.plot_correlation_heatmap(df, figures_dir)),369 ("coefficient_plot", lambda: fig_mod.plot_coefficient_comparison(models_dict, figures_dir)),370 ("quantile_plot", lambda: fig_mod.plot_quantile_coefficients(models_dict, figures_dir)),371 ("spatial_map", lambda: fig_mod.plot_spatial_map(df, spatial_results, figures_dir)),372 ("residual_diagnostics", lambda: fig_mod.plot_residual_diagnostics(models_dict, df, figures_dir)),373 ("shap_summary", lambda: fig_mod.plot_shap_summary(ml_models_dict, df, figures_dir)),374 ]375376 for name, func in figure_funcs:377 try:378 func()379 report_artifacts["figures_generated"].append(name)380 logger.info(" Figure '%s' generated.", name)381 except Exception:382 logger.warning(" Figure '%s' failed.", name, exc_info=True)383384 except ImportError:385 logger.warning("figures module not available; skipping figure generation.")386 except Exception:387 logger.warning("Figure generation failed.", exc_info=True)388389 # ---- Price decomposition ----390 logger.info("Running price decomposition ...")391 decomp_df = pd.DataFrame()392 decomp_summary = pd.DataFrame()393394 # Pick the best available hedonic model for decomposition395 preferred_model_keys = [396 "ols_semilog", "ols_log", "ols_linear",397 "robust_huber", "robust_bisquare",398 ]399 decomp_model = None400 decomp_model_name = None401 for key in preferred_model_keys:402 if key in models_dict and models_dict[key] is not None:403 decomp_model = models_dict[key]404 decomp_model_name = key405 break406407 if decomp_model is not None:408 try:409 logger.info("Using model '%s' for price decomposition.", decomp_model_name)410 decomp_df = decompose_all_prices(decomp_model, df, feature_groups)411 decomp_summary = generate_decomposition_summary(decomp_df)412413 # Save artefacts414 decomp_csv = output_dir / "tables" / "price_decomposition.csv"415 decomp_df.to_csv(decomp_csv, index=False)416 logger.info("Decomposition saved to %s", decomp_csv)417418 summary_csv = output_dir / "tables" / "decomposition_summary.csv"419 decomp_summary.to_csv(summary_csv)420 logger.info("Summary saved to %s", summary_csv)421422 # Pretty-print a sample listing423 if len(decomp_df) > 0:424 sample_idx = 0425 sample_expl = explain_listing_price(426 decomp_model, df, sample_idx, feature_groups,427 )428 logger.info(429 "Sample decomposition:\n%s",430 format_decomposition_report(sample_expl),431 )432 except Exception:433 logger.warning("Price decomposition failed.", exc_info=True)434 else:435 logger.warning(436 "No suitable hedonic model found for decomposition (tried: %s).",437 ", ".join(preferred_model_keys),438 )439440 elapsed = time.perf_counter() - t0441 report_artifacts["decomp_df"] = decomp_df442 report_artifacts["decomp_summary"] = decomp_summary443 report_artifacts["elapsed_seconds"] = elapsed444 logger.info("Full report generation complete in %.1f s", elapsed)445 return report_artifacts446