""" tables.py - Publication-quality table generation for hedonic pricing analysis. Generates LaTeX (booktabs), CSV, and HTML tables suitable for academic publication. All tables follow APA / econometrics journal conventions: three-line tables, significance stars, proper formatting of coefficients and standard errors. Author: Simon-Pierre Boucher Date: 2026-05-20 """ from __future__ import annotations import logging import os from pathlib import Path from typing import Any, Dict, List, Optional, Sequence, Tuple, Union import numpy as np import pandas as pd logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- # Significance helpers # --------------------------------------------------------------------------- _STAR_THRESHOLDS: List[Tuple[float, str]] = [ (0.001, "***"), (0.01, "**"), (0.05, "*"), ] def _stars(pvalue: float) -> str: """Return significance stars for a p-value.""" for threshold, symbol in _STAR_THRESHOLDS: if pvalue <= threshold: return symbol return "" def _fmt_coef(value: float, se: Optional[float] = None, pvalue: Optional[float] = None, decimals: int = 3) -> str: """Format a coefficient with optional standard error and stars.""" s = f"{value:.{decimals}f}" if pvalue is not None: s += _stars(pvalue) if se is not None: s += f"\n({se:.{decimals}f})" return s # --------------------------------------------------------------------------- # Table 1 - Summary statistics # --------------------------------------------------------------------------- def summary_statistics_table( df: pd.DataFrame, cols: List[str], ) -> pd.DataFrame: """ Table 1 -- Descriptive statistics. Columns: Variable, Mean, Std. Dev., Min, P25, Median, P75, Max, N. Parameters ---------- df : pd.DataFrame Analysis dataset. cols : list[str] Numeric columns to summarise. Returns ------- pd.DataFrame Summary statistics table. """ logger.info("Building summary statistics table for %d variables.", len(cols)) records: List[Dict[str, Any]] = [] for col in cols: s = df[col].dropna() records.append({ "Variable": col, "Mean": round(s.mean(), 3), "Std. Dev.": round(s.std(), 3), "Min": round(s.min(), 3), "P25": round(s.quantile(0.25), 3), "Median": round(s.median(), 3), "P75": round(s.quantile(0.75), 3), "Max": round(s.max(), 3), "N": int(s.count()), }) return pd.DataFrame(records).set_index("Variable") # --------------------------------------------------------------------------- # Table 2 - Price distribution by room / property type # --------------------------------------------------------------------------- def price_by_room_type_table( df: pd.DataFrame, price_col: str = "price_numeric", room_col: str = "property_type", type_col: Optional[str] = None, ) -> pd.DataFrame: """ Table 2 -- Price distribution by room type (and optionally property type). Parameters ---------- df : pd.DataFrame Must contain *price_col* and *room_col*. price_col : str Numeric price column. room_col : str Room / property type column. type_col : str, optional Additional grouping column (e.g., property_type). Returns ------- pd.DataFrame Grouped statistics. """ logger.info("Building price-by-room-type table.") group_cols = [room_col] if type_col is not None: group_cols.append(type_col) agg = ( df.dropna(subset=[price_col]) .groupby(group_cols, observed=True)[price_col] .agg( Mean="mean", Median="median", Std="std", Min="min", Max="max", N="count", ) .round(2) .reset_index() ) return agg # --------------------------------------------------------------------------- # Table 3 - Correlation matrix # --------------------------------------------------------------------------- def correlation_matrix_table( df: pd.DataFrame, cols: List[str], method: str = "pearson", ) -> pd.DataFrame: """ Table 3 -- Pairwise correlation matrix. Lower triangle only, with significance stars computed via t-test approximation. Parameters ---------- df : pd.DataFrame cols : list[str] method : str 'pearson' or 'spearman'. Returns ------- pd.DataFrame Formatted correlation matrix (lower triangle with stars). """ from scipy import stats as sp_stats logger.info("Building correlation matrix table (%s).", method) sub = df[cols].dropna() n = len(sub) corr = sub.corr(method=method) formatted = corr.copy().astype(str) for i, row_var in enumerate(cols): for j, col_var in enumerate(cols): if j > i: formatted.iloc[i, j] = "" elif i == j: formatted.iloc[i, j] = "1.000" else: r = corr.iloc[i, j] # t-test for correlation significance if abs(r) < 1.0: t_stat = r * np.sqrt((n - 2) / (1 - r ** 2)) p = 2 * sp_stats.t.sf(abs(t_stat), df=n - 2) else: p = 0.0 formatted.iloc[i, j] = f"{r:.3f}{_stars(p)}" return formatted # --------------------------------------------------------------------------- # Table 4/5 - Regression comparison # --------------------------------------------------------------------------- def regression_comparison_table( results_dict: Dict[str, Any], stat_keys: Optional[List[str]] = None, ) -> pd.DataFrame: """ Table 4/5 -- Side-by-side regression results with significance stars. Parameters ---------- results_dict : dict ``{model_label: result}`` where each *result* is either: - a ``statsmodels`` results object (with ``.params``, ``.bse``, ``.pvalues``, ``.rsquared``, ``.nobs``), OR - a plain dict ``{variable: {'coef': float, 'se': float, 'pvalue': float}, ...}`` with optional top-level keys ``'R2'``, ``'Adj_R2'``, ``'N'``, ``'AIC'``, ``'BIC'``. stat_keys : list[str], optional Extra bottom-panel statistics to include. Returns ------- pd.DataFrame Variables in rows, models in columns. """ logger.info("Building regression comparison table (%d models).", len(results_dict)) all_vars: List[str] = [] parsed: Dict[str, Dict[str, Dict[str, float]]] = {} model_stats: Dict[str, Dict[str, Any]] = {} for label, res in results_dict.items(): if isinstance(res, dict): # Plain dict input coefs = { k: v for k, v in res.items() if isinstance(v, dict) and "coef" in v } parsed[label] = coefs model_stats[label] = { k: v for k, v in res.items() if not isinstance(v, dict) } for v in coefs: if v not in all_vars: all_vars.append(v) else: # statsmodels result object coefs = {} for var in res.params.index: coefs[var] = { "coef": res.params[var], "se": res.bse[var], "pvalue": res.pvalues[var], } if var not in all_vars: all_vars.append(var) parsed[label] = coefs ms: Dict[str, Any] = {"N": int(getattr(res, "nobs", 0))} if hasattr(res, "rsquared"): ms["R2"] = round(res.rsquared, 4) if hasattr(res, "rsquared_adj"): ms["Adj_R2"] = round(res.rsquared_adj, 4) if hasattr(res, "aic"): ms["AIC"] = round(res.aic, 1) if hasattr(res, "bic"): ms["BIC"] = round(res.bic, 1) model_stats[label] = ms # Build coefficient rows (coef + se rows interleaved) rows: List[Dict[str, str]] = [] for var in all_vars: coef_row: Dict[str, str] = {"Variable": var} se_row: Dict[str, str] = {"Variable": ""} for label in results_dict: if var in parsed[label]: c = parsed[label][var] coef_row[label] = f"{c['coef']:.3f}{_stars(c.get('pvalue', 1.0))}" se_row[label] = f"({c.get('se', 0.0):.3f})" else: coef_row[label] = "" se_row[label] = "" rows.append(coef_row) rows.append(se_row) # Bottom panel: model statistics default_stat_keys = ["N", "R2", "Adj_R2", "AIC", "BIC"] use_keys = stat_keys if stat_keys is not None else default_stat_keys for key in use_keys: stat_row: Dict[str, str] = {"Variable": key} for label in results_dict: val = model_stats.get(label, {}).get(key, "") if isinstance(val, float): stat_row[label] = f"{val:.4f}" if val < 10 else f"{val:.1f}" else: stat_row[label] = str(val) if val != "" else "" rows.append(stat_row) out = pd.DataFrame(rows) out = out.set_index("Variable") return out # --------------------------------------------------------------------------- # Table 6 - Amenity premiums # --------------------------------------------------------------------------- def amenity_premiums_table( results: Any, amenity_vars: Optional[List[str]] = None, ) -> pd.DataFrame: """ Table 6 -- Amenity coefficients converted to percentage premiums. For a semi-log model ln(price) = Xb, the percentage premium of a dummy variable is (exp(b) - 1) * 100. Parameters ---------- results : statsmodels result or dict Regression results. amenity_vars : list[str], optional If None, all variables containing 'amenity' or common amenity keywords are selected. Returns ------- pd.DataFrame Amenity, Coefficient, Std. Error, % Premium, Significance. """ logger.info("Building amenity premiums table.") if isinstance(results, dict): data = results else: params = results.params bse = results.bse pvals = results.pvalues data = { v: {"coef": params[v], "se": bse[v], "pvalue": pvals[v]} for v in params.index } _amenity_keywords = [ "amenity", "pool", "wifi", "parking", "kitchen", "gym", "air_conditioning", "ac", "washer", "dryer", "tv", "hot_tub", "fireplace", "elevator", "pet", "breakfast", ] if amenity_vars is None: amenity_vars = [ v for v in data if any(kw in v.lower() for kw in _amenity_keywords) ] records = [] for var in amenity_vars: info = data[var] coef = info["coef"] se = info.get("se", np.nan) pval = info.get("pvalue", np.nan) pct_premium = (np.exp(coef) - 1) * 100 records.append({ "Amenity": var, "Coefficient": round(coef, 4), "Std. Error": round(se, 4), "% Premium": round(pct_premium, 2), "Significance": _stars(pval) if not np.isnan(pval) else "", }) out = pd.DataFrame(records).sort_values("% Premium", ascending=False) return out.reset_index(drop=True) # --------------------------------------------------------------------------- # Table 7 - Host reputation # --------------------------------------------------------------------------- def host_reputation_table( results: Any, host_vars: Optional[List[str]] = None, ) -> pd.DataFrame: """ Table 7 -- Host and review coefficients. Parameters ---------- results : statsmodels result or dict host_vars : list[str], optional Returns ------- pd.DataFrame """ logger.info("Building host reputation table.") _host_keywords = [ "host", "superhost", "review", "rating", "response", "experience", "verified", "identity", "listing_count", "quality_score", "is_superhost", "num_reviews", "is_guest_favorite", ] if isinstance(results, dict): data = results else: params = results.params bse = results.bse pvals = results.pvalues data = { v: {"coef": params[v], "se": bse[v], "pvalue": pvals[v]} for v in params.index } if host_vars is None: host_vars = [ v for v in data if any(kw in v.lower() for kw in _host_keywords) ] records = [] for var in host_vars: info = data[var] coef = info["coef"] se = info.get("se", np.nan) pval = info.get("pvalue", np.nan) records.append({ "Variable": var, "Coefficient": round(coef, 4), "Std. Error": round(se, 4), "p-value": round(pval, 4) if not np.isnan(pval) else "", "Significance": _stars(pval) if not np.isnan(pval) else "", "% Effect": round((np.exp(coef) - 1) * 100, 2), }) return pd.DataFrame(records) # --------------------------------------------------------------------------- # Table 8 - Location gradient # --------------------------------------------------------------------------- def location_gradient_table( results: Any, location_vars: Optional[List[str]] = None, ) -> pd.DataFrame: """ Table 8 -- Distance-to-centre effects. Parameters ---------- results : statsmodels result or dict location_vars : list[str], optional Returns ------- pd.DataFrame """ logger.info("Building location gradient table.") _loc_keywords = [ "distance", "dist_", "km", "center", "centre", "cbd", "downtown", "lat", "long", "longitude", "latitude", "neighbourhood", "neighborhood", "city", ] if isinstance(results, dict): data = results else: params = results.params bse = results.bse pvals = results.pvalues data = { v: {"coef": params[v], "se": bse[v], "pvalue": pvals[v]} for v in params.index } if location_vars is None: location_vars = [ v for v in data if any(kw in v.lower() for kw in _loc_keywords) ] records = [] for var in location_vars: info = data[var] coef = info["coef"] se = info.get("se", np.nan) pval = info.get("pvalue", np.nan) records.append({ "Variable": var, "Coefficient": round(coef, 4), "Std. Error": round(se, 4), "p-value": round(pval, 4) if not np.isnan(pval) else "", "Significance": _stars(pval) if not np.isnan(pval) else "", "% Effect per unit": round((np.exp(coef) - 1) * 100, 2), }) return pd.DataFrame(records) # --------------------------------------------------------------------------- # Table 9 - Quantile regression # --------------------------------------------------------------------------- def quantile_table( qreg_results: Dict[float, Any], variables: Optional[List[str]] = None, ) -> pd.DataFrame: """ Table 9 -- Quantile regression coefficients side by side. Parameters ---------- qreg_results : dict ``{quantile: result}`` where *quantile* is e.g. 0.10, 0.25, ... Each *result* is a statsmodels ``QuantRegResults`` or a dict ``{variable: {'coef': float, 'se': float, 'pvalue': float}}``. variables : list[str], optional Variables to include; if None, use all from the first result. Returns ------- pd.DataFrame Variables as rows, quantiles as columns. """ logger.info("Building quantile regression table (%d quantiles).", len(qreg_results)) parsed: Dict[float, Dict[str, Dict[str, float]]] = {} for q, res in qreg_results.items(): if isinstance(res, dict): parsed[q] = res else: parsed[q] = { v: {"coef": res.params[v], "se": res.bse[v], "pvalue": res.pvalues[v]} for v in res.params.index } if variables is None: first_key = next(iter(parsed)) variables = list(parsed[first_key].keys()) quantiles_sorted = sorted(parsed.keys()) rows = [] for var in variables: coef_row: Dict[str, str] = {"Variable": var} se_row: Dict[str, str] = {"Variable": ""} for q in quantiles_sorted: col_label = f"Q{int(q * 100)}" if var in parsed[q]: info = parsed[q][var] coef_row[col_label] = ( f"{info['coef']:.3f}{_stars(info.get('pvalue', 1.0))}" ) se_row[col_label] = f"({info.get('se', 0.0):.3f})" else: coef_row[col_label] = "" se_row[col_label] = "" rows.append(coef_row) rows.append(se_row) return pd.DataFrame(rows).set_index("Variable") # --------------------------------------------------------------------------- # Table 10 - ML model comparison # --------------------------------------------------------------------------- def ml_comparison_table( ml_results: Dict[str, Dict[str, float]], ) -> pd.DataFrame: """ Table 10 -- Machine-learning model performance comparison. Parameters ---------- ml_results : dict ``{model_name: {'RMSE': float, 'MAE': float, 'R2': float, 'MAPE': float, ...}}``. Returns ------- pd.DataFrame """ logger.info("Building ML comparison table (%d models).", len(ml_results)) records = [] for model, metrics in ml_results.items(): row: Dict[str, Any] = {"Model": model} row.update({k: round(v, 4) if isinstance(v, float) else v for k, v in metrics.items()}) records.append(row) out = pd.DataFrame(records).set_index("Model") # Sort by R2 descending if present if "R2" in out.columns: out = out.sort_values("R2", ascending=False) return out # --------------------------------------------------------------------------- # Table 11 - SHAP importance # --------------------------------------------------------------------------- def shap_importance_table( shap_df: pd.DataFrame, top_n: int = 20, ) -> pd.DataFrame: """ Table 11 -- SHAP top variables. Parameters ---------- shap_df : pd.DataFrame Must contain 'Variable' and 'Mean_SHAP' (or 'mean_abs_shap') columns. top_n : int Number of top variables to display. Returns ------- pd.DataFrame """ logger.info("Building SHAP importance table (top %d).", top_n) df = shap_df.copy() # Normalise column names col_map = {} for c in df.columns: cl = c.lower() if "variable" in cl or "feature" in cl: col_map[c] = "Variable" elif "shap" in cl: col_map[c] = "Mean |SHAP|" df = df.rename(columns=col_map) if "Mean |SHAP|" not in df.columns: raise ValueError("shap_df must contain a column with 'shap' in its name.") df = df.sort_values("Mean |SHAP|", ascending=False).head(top_n).reset_index(drop=True) df["Rank"] = range(1, len(df) + 1) # Compute cumulative share if possible total = df["Mean |SHAP|"].sum() if total > 0: df["Share (%)"] = (df["Mean |SHAP|"] / total * 100).round(2) df["Cumul. (%)"] = df["Share (%)"].cumsum().round(2) return df[["Rank", "Variable", "Mean |SHAP|", "Share (%)", "Cumul. (%)"]] # --------------------------------------------------------------------------- # Table 12 - Spatial model comparison # --------------------------------------------------------------------------- def spatial_results_table( spatial_results: Dict[str, Dict[str, Any]], ) -> pd.DataFrame: """ Table 12 -- Spatial model comparison (OLS, SAR, SEM, SDM, GWR, etc.). Parameters ---------- spatial_results : dict ``{model_name: {'R2': float, 'AIC': float, 'Log-Lik': float, 'Lambda': float, 'Rho': float, 'Moran_I': float, ...}}``. Returns ------- pd.DataFrame """ logger.info("Building spatial results table (%d models).", len(spatial_results)) records = [] for model, stats in spatial_results.items(): row: Dict[str, Any] = {"Model": model} for k, v in stats.items(): if isinstance(v, float): row[k] = round(v, 4) else: row[k] = v records.append(row) return pd.DataFrame(records).set_index("Model") # --------------------------------------------------------------------------- # Table 13 - Robustness checks summary # --------------------------------------------------------------------------- def robustness_table( robustness_results: Dict[str, Dict[str, Any]], ) -> pd.DataFrame: """ Table 13 -- Robustness checks summary. Parameters ---------- robustness_results : dict ``{check_name: {variable: {'coef': float, 'se': float, ...}, ... 'R2': float, 'N': int}}``. Returns ------- pd.DataFrame """ logger.info("Building robustness table (%d checks).", len(robustness_results)) records = [] for check_name, content in robustness_results.items(): row: Dict[str, Any] = {"Check": check_name} # Extract model-level stats for key in ("R2", "Adj_R2", "N", "AIC", "BIC", "RMSE"): if key in content: val = content[key] row[key] = round(val, 4) if isinstance(val, float) else val # Count significant variables var_entries = { k: v for k, v in content.items() if isinstance(v, dict) and "coef" in v } n_sig = sum( 1 for v in var_entries.values() if v.get("pvalue", 1.0) < 0.05 ) row["Sig. vars (p<.05)"] = n_sig row["Total vars"] = len(var_entries) records.append(row) return pd.DataFrame(records).set_index("Check") # --------------------------------------------------------------------------- # Export utilities # --------------------------------------------------------------------------- def export_table( df: pd.DataFrame, name: str, formats: Optional[List[str]] = None, output_dir: Optional[str] = None, caption: Optional[str] = None, label: Optional[str] = None, ) -> Dict[str, str]: """ Export a DataFrame to results/tables/ in multiple formats. Parameters ---------- df : pd.DataFrame name : str Base filename (without extension). formats : list[str] Subset of ``['csv', 'tex', 'html']``. output_dir : str, optional Override for the output directory. caption : str, optional LaTeX caption. label : str, optional LaTeX label. Returns ------- dict ``{format: filepath}`` """ if formats is None: formats = ["csv", "tex", "html"] if output_dir is None: output_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "results", "tables", ) os.makedirs(output_dir, exist_ok=True) paths: Dict[str, str] = {} for fmt in formats: fpath = os.path.join(output_dir, f"{name}.{fmt}") if fmt == "csv": df.to_csv(fpath) elif fmt == "tex": latex_str = format_latex_table( df, caption=caption or name.replace("_", " ").title(), label=label or f"tab:{name}", ) with open(fpath, "w", encoding="utf-8") as fh: fh.write(latex_str) elif fmt == "html": df.to_html(fpath, border=0, classes="table table-sm") else: logger.warning("Unknown export format: %s", fmt) continue paths[fmt] = fpath logger.info("Exported %s -> %s", name, fpath) return paths def format_latex_table( df: pd.DataFrame, caption: str, label: str, note: Optional[str] = None, ) -> str: """ Format a DataFrame as a publication-ready LaTeX table with booktabs. Parameters ---------- df : pd.DataFrame caption : str label : str note : str, optional Table note placed below the table. Returns ------- str LaTeX source code. """ ncols = len(df.columns) + (1 if df.index.name else 0) col_spec = "l" + "c" * (ncols - 1) lines: List[str] = [ r"\begin{table}[htbp]", r"\centering", r"\caption{" + caption + "}", r"\label{" + label + "}", rf"\begin{{tabular}}{{{col_spec}}}", r"\toprule", ] # Header header_parts: List[str] = [] if df.index.name: header_parts.append(df.index.name) else: header_parts.append("") header_parts.extend(str(c) for c in df.columns) lines.append(" & ".join(header_parts) + r" \\") lines.append(r"\midrule") # Body for idx, row in df.iterrows(): row_parts = [str(idx)] for val in row: if isinstance(val, float): row_parts.append(f"{val:.4f}" if abs(val) < 100 else f"{val:.1f}") else: row_parts.append(str(val)) lines.append(" & ".join(row_parts) + r" \\") lines.append(r"\bottomrule") lines.append(r"\end{tabular}") if note: lines.append(r"\begin{tablenotes}") lines.append(r"\small") lines.append(rf"\item \textit{{Note:}} {note}") lines.append(r"\end{tablenotes}") # Default significance note lines.append(r"\par\smallskip\footnotesize") lines.append( r"\textit{Note:} $^{***}p<0.001$, $^{**}p<0.01$, $^{*}p<0.05$." ) lines.append(r"\end{table}") return "\n".join(lines)