spb/wp2_uqo Public

1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — contact@spboucher.ai3#4"""Step 5 — Generate all paper figures (fig1-fig11).56Faithful port of paper_figures_v2.py: same layouts, colors and parameters,7but reading the pipeline's analysis dataset instead of re-encoding the8embeddings, and writing into the repository's figures/ directory.910Inputs : data/processed/hedonic_maison_results.csv11Outputs: figures/fig{1..11}_*.pdf and .png12"""1314import sys15import warnings16from pathlib import Path1718import matplotlib19import numpy as np20import pandas as pd2122matplotlib.use("Agg")23import matplotlib.pyplot as plt24from matplotlib.patches import FancyBboxPatch25from scipy.stats import probplot2627sys.path.insert(0, str(Path(__file__).resolve().parents[1]))2829from src import config30from src.models import fit_all_models31from src.references import ENGLISH_LABELS, REFERENCES, SIM_COLS3233warnings.filterwarnings("ignore")3435plt.rcParams.update({36    "font.family": "serif",37    "font.serif": ["Times New Roman", "Times", "DejaVu Serif"],38    "font.size": 10,39    "axes.labelsize": 11,40    "axes.titlesize": 12,41    "xtick.labelsize": 9,42    "ytick.labelsize": 9,43    "legend.fontsize": 9,44    "figure.dpi": 300,45    "savefig.dpi": 300,46    "savefig.bbox": "tight",47    "savefig.pad_inches": 0.08,48    "axes.spines.top": False,49    "axes.spines.right": False,50})5152LABELS = [ENGLISH_LABELS[slug] for slug in REFERENCES]535455def save(fig_name):56    plt.tight_layout()57    for ext in ("pdf", "png"):58        plt.savefig(config.FIGURES_DIR / f"{fig_name}.{ext}")59    plt.close()60    print(f"  {fig_name}")616263def fig1_model_comparison(models):64    fig, ax = plt.subplots(figsize=(6.2, 3.8))65    names = ["Model A\nStructural", "Model B\n+ Length", "Model C\n+ Semantic",66             "Model D\nFull", "Model E\nParsim."]67    r2v = [m.rsquared for m in models.values()]68    r2a = [m.rsquared_adj for m in models.values()]69    x = np.arange(len(names))70    b1 = ax.bar(x - 0.15, r2v, 0.28, label=r"$R^2$", color="#1565C0", alpha=0.9)71    ax.bar(x + 0.15, r2a, 0.28, label=r"Adj. $R^2$", color="#FB8C00", alpha=0.9)72    for b, v in zip(b1, r2v):73        ax.text(b.get_x() + b.get_width() / 2, b.get_height() + 0.006,74                f"{v:.3f}", ha="center", fontsize=7.5)75    ax.set_xticks(x)76    ax.set_xticklabels(names, fontsize=8)77    ax.set_ylabel(r"$R^2$")78    ax.set_ylim(0, 0.58)79    ax.legend(loc="upper left", framealpha=0.9)80    ax.axhline(models["A"].rsquared, color="grey", ls="--", lw=0.7, alpha=0.5)81    save("fig1_model_comparison")828384def fig2_coefficient_plot(mD):85    fig, ax = plt.subplots(figsize=(6.2, 7))86    conf = mD.conf_int()87    sd = []88    for v, label in zip(SIM_COLS, LABELS):89        sd.append((label, mD.params[v], conf.loc[v, 0], conf.loc[v, 1],90                   mD.pvalues[v], (np.exp(mD.params[v]) - 1) * 100))91    sd.sort(key=lambda t: t[1])92    names = [d[0] for d in sd]93    coefs = [d[1] for d in sd]94    pvs = [d[4] for d in sd]95    yp = np.arange(len(names))96    cols = ["#C62828" if c < 0 and p < 0.05 else97            "#2E7D32" if c > 0 and p < 0.05 else "#BDBDBD"98            for c, p in zip(coefs, pvs)]99    ax.axvline(0, color="black", lw=0.8)100    ax.barh(yp, coefs, color=cols, height=0.55, alpha=0.85,101            edgecolor="white", lw=0.4)102    for i, (name, c, cl, ch, p, imp) in enumerate(sd):103        ax.plot([cl, ch], [yp[i], yp[i]], color="#333", lw=1)104        st = "***" if p < 0.001 else "**" if p < 0.01 else "*" if p < 0.05 else ""105        if st:106            off = 0.004 if c >= 0 else -0.004107            ha = "left" if c >= 0 else "right"108            ax.text(c + off, yp[i], f"{imp:+.1f}%{st}", va="center", ha=ha,109                    fontsize=7.5)110    ax.set_yticks(yp)111    ax.set_yticklabels(names, fontsize=9)112    ax.set_xlabel("Standardized coefficient (log-price)")113    save("fig2_coefficient_plot")114115116def fig3_similarity_distributions(df):117    fig, ax = plt.subplots(figsize=(6.2, 5.5))118    bd = [df[c].values for c in SIM_COLS]119    order = np.argsort([np.median(d) for d in bd])[::-1]120    bd = [bd[i] for i in order]121    bl = [LABELS[i] for i in order]122    bp = ax.boxplot(bd, vert=False, patch_artist=True, widths=0.55,123                    flierprops=dict(marker=".", markersize=1.5, alpha=0.2),124                    medianprops=dict(color="black", lw=1.2))125    cm = plt.cm.viridis126    for i, patch in enumerate(bp["boxes"]):127        patch.set_facecolor(cm(i / len(bp["boxes"])))128        patch.set_alpha(0.7)129    ax.set_yticklabels(bl, fontsize=8)130    ax.set_xlabel("Cosine similarity")131    save("fig3_similarity_distributions")132133134def fig4_quintile_heatmap(df):135    df = df.copy()136    df["pq"] = pd.qcut(df["price"], q=5,137                       labels=["Q1 (Low)", "Q2", "Q3 (Med)", "Q4", "Q5 (High)"])138    profile = df.groupby("pq", observed=False)[SIM_COLS].mean()139    fig, ax = plt.subplots(figsize=(6.2, 6.5))140    im = ax.imshow(profile.T.values, aspect="auto", cmap="RdYlGn",141                   interpolation="nearest")142    ax.set_xticks(range(5))143    ax.set_xticklabels(profile.index, fontsize=9)144    ax.set_yticks(range(len(LABELS)))145    ax.set_yticklabels(LABELS, fontsize=8)146    ax.set_xlabel("Price quintile")147    for i in range(len(LABELS)):148        for j in range(5):149            v = profile.T.values[i, j]150            ax.text(j, i, f"{v:.3f}", ha="center", va="center", fontsize=6.5,151                    color="white" if v < 0.25 or v > 0.40 else "black")152    plt.colorbar(im, ax=ax, shrink=0.7, label="Mean cosine similarity", pad=0.02)153    save("fig4_quintile_heatmap")154155156def fig5_correlation_matrix(df):157    corr = df[SIM_COLS].corr().values158    mask = np.triu(np.ones_like(corr, dtype=bool), k=1)159    corr_show = np.where(mask, np.nan, corr)160    fig, ax = plt.subplots(figsize=(6.5, 6))161    im = ax.imshow(corr_show, cmap="RdBu_r", vmin=-0.1, vmax=1.0,162                   interpolation="nearest")163    ax.set_xticks(range(len(LABELS)))164    ax.set_xticklabels(LABELS, rotation=55, ha="right", fontsize=7)165    ax.set_yticks(range(len(LABELS)))166    ax.set_yticklabels(LABELS, fontsize=7)167    for i in range(len(LABELS)):168        for j in range(i + 1):169            v = corr[i, j]170            ax.text(j, i, f"{v:.2f}", ha="center", va="center", fontsize=5.5,171                    color="white" if abs(v) > 0.65 else "black")172    plt.colorbar(im, ax=ax, shrink=0.7, label="Pearson r", pad=0.02)173    save("fig5_correlation_matrix")174175176def fig6_scatter_plots(df):177    fig, axes = plt.subplots(2, 2, figsize=(6.2, 5.5))178    pairs = [179        ("sim_luxe", "Luxury", "#7B1FA2"),180        ("sim_a_renover", "Needs Renovation", "#C62828"),181        ("sim_moderne_contemporain", "Modern/Contemporary", "#1565C0"),182        ("sim_urgence_motivation", "Motivated Seller", "#E65100"),183    ]184    samp = df.sample(min(2500, len(df)), random_state=config.SEED)185    for ax, (col, label, color) in zip(axes.flat, pairs):186        ax.scatter(samp[col], samp["log_price"], alpha=0.12, s=5, c=color,187                   edgecolors="none")188        z = np.polyfit(df[col], df["log_price"], 1)189        xr = np.linspace(df[col].min(), df[col].max(), 100)190        ax.plot(xr, np.poly1d(z)(xr), color="black", lw=1.8)191        r = df[col].corr(df["log_price"])192        ax.set_xlabel(f"Sim: {label}", fontsize=8)193        ax.set_ylabel("log(Price)", fontsize=8)194        ax.set_title(f"{label} (r={r:.3f})", fontsize=9)195        ax.tick_params(labelsize=7)196    plt.tight_layout(h_pad=1.2, w_pad=0.8)197    for ext in ("pdf", "png"):198        plt.savefig(config.FIGURES_DIR / f"fig6_scatter_plots.{ext}")199    plt.close()200    print("  fig6_scatter_plots")201202203def fig7_methodology(n_obs):204    fig, ax = plt.subplots(figsize=(6.2, 2.8))205    ax.set_xlim(0, 12)206    ax.set_ylim(0, 3.5)207    ax.axis("off")208    boxes = [209        (1.2, 1.75, f"Property\nListings\n(n={n_obs:,})", "#E3F2FD"),210        (3.6, 1.75, "Sentence\nEmbeddings\n(384-d)", "#E8F5E9"),211        (6.0, 1.75, "Cosine\nSimilarity\n(20 refs)", "#FFF3E0"),212        (8.4, 1.75, "Hedonic\nOLS Model", "#F3E5F5"),213        (10.8, 1.75, "Implicit\nPrice\nEstimates", "#FFEBEE"),214    ]215    for x, yy, text, color in boxes:216        ax.add_patch(FancyBboxPatch((x - 0.65, yy - 0.65), 1.3, 1.3,217                                    boxstyle="round,pad=0.08",218                                    facecolor=color, edgecolor="#444", lw=1.3))219        ax.text(x, yy, text, ha="center", va="center", fontsize=7.5,220                fontweight="bold")221    for i in range(len(boxes) - 1):222        ax.annotate("", xy=(boxes[i + 1][0] - 0.7, 1.75),223                    xytext=(boxes[i][0] + 0.7, 1.75),224                    arrowprops=dict(arrowstyle="->", color="#444", lw=1.8))225    labs = [226        (2.4, 0.55, "PublicRemarks\nextraction"),227        (4.8, 0.55, "all-MiniLM-L6-v2\ntransformer"),228        (7.2, 0.55, "Reference-based\nfeatures"),229        (9.6, 0.55, "log(P) = βX+γS+ε"),230    ]231    for x, yy, text in labs:232        ax.text(x, yy, text, ha="center", va="center", fontsize=7,233                style="italic", color="#666")234    save("fig7_methodology")235236237def fig8_r2_decomposition(models):238    fig, ax = plt.subplots(figsize=(4.5, 4))239    mA, mB, mD = models["A"], models["B"], models["D"]240    comps = [241        ("Structural variables", mA.rsquared, "#1565C0"),242        ("Text length", mB.rsquared - mA.rsquared, "#43A047"),243        ("Semantic similarities", mD.rsquared - mB.rsquared, "#FB8C00"),244    ]245    bot = 0246    for lab, val, col in comps:247        ax.bar(0, val, bottom=bot, color=col, width=0.5, edgecolor="white",248               lw=0.5, label=f"{lab}: {val:.4f}")249        if val > 0.008:250            ax.text(0, bot + val / 2, f"{val:.4f}\n({val / mD.rsquared * 100:.1f}%)",251                    ha="center", va="center", fontsize=8, fontweight="bold",252                    color="white")253        bot += val254    ax.set_ylabel(r"$R^2$")255    ax.set_ylim(0, 0.58)256    ax.set_xticks([0])257    ax.set_xticklabels(["Full Model (D)"], fontsize=9)258    ax.legend(loc="upper left", fontsize=8, framealpha=0.9)259    save("fig8_r2_decomposition")260261262def fig9_price_distribution(df):263    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(6.2, 3))264    ax1.hist(df["price"] / 1000, bins=80, color="#1565C0", alpha=0.8,265             edgecolor="white", lw=0.3)266    ax1.set_xlabel("Price (\\$000s)")267    ax1.set_ylabel("Frequency")268    ax1.set_title("(a) Price distribution", fontsize=10)269    ax1.set_xlim(0, 3000)270    ax2.hist(df["log_price"], bins=80, color="#2E7D32", alpha=0.8,271             edgecolor="white", lw=0.3)272    ax2.set_xlabel("log(Price)")273    ax2.set_ylabel("Frequency")274    ax2.set_title("(b) Log-price distribution", fontsize=10)275    plt.tight_layout(w_pad=1.5)276    for ext in ("pdf", "png"):277        plt.savefig(config.FIGURES_DIR / f"fig9_price_distribution.{ext}")278    plt.close()279    print("  fig9_price_distribution")280281282STRUCT_LABELS = {283    "bedrooms": "Bedrooms", "bathrooms": "Bathrooms",284    "half_baths": "Half-bathrooms", "parking": "Parking",285    "stories": "Stories", "land_size": "Lot size",286    "remarks_length": "Description length",287}288289290def fig10_structural_coefficients(mD):291    fig, ax = plt.subplots(figsize=(6.2, 3))292    sv = config.STRUCTURAL_VARS + ["remarks_length"]293    conf = mD.conf_int()294    sdata = sorted(295        [(v, mD.params[v], conf.loc[v, 0], conf.loc[v, 1], mD.pvalues[v])296         for v in sv],297        key=lambda t: t[1],298    )299    yy = np.arange(len(sdata))300    cols = ["#1565C0" if d[4] < 0.05 else "#BDBDBD" for d in sdata]301    ax.barh(yy, [d[1] for d in sdata], color=cols, height=0.5, alpha=0.85)302    for i, (v, c, cl, ch, p) in enumerate(sdata):303        ax.plot([cl, ch], [yy[i], yy[i]], color="#333", lw=1)304        st = "***" if p < 0.001 else "**" if p < 0.01 else "*" if p < 0.05 else ""305        if st:306            ax.text(c + 0.005, yy[i], f"{(np.exp(c) - 1) * 100:+.1f}%{st}",307                    va="center", fontsize=7.5)308    ax.axvline(0, color="black", lw=0.7)309    ax.set_yticks(yy)310    ax.set_yticklabels([STRUCT_LABELS[d[0]] for d in sdata], fontsize=9)311    ax.set_xlabel("Standardized coefficient")312    save("fig10_structural_coefficients")313314315def fig11_residual_diagnostics(mD):316    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(6.2, 3))317    ax1.scatter(mD.fittedvalues, mD.resid, s=2, alpha=0.1, c="#1565C0",318                edgecolors="none")319    ax1.axhline(0, color="red", lw=0.8, ls="--")320    ax1.set_xlabel("Fitted values")321    ax1.set_ylabel("Residuals")322    ax1.set_title("(a) Residuals vs. fitted", fontsize=10)323    probplot(mD.resid, dist="norm", plot=ax2)324    ax2.set_title("(b) Normal Q-Q plot", fontsize=10)325    ax2.get_lines()[0].set(markersize=2, alpha=0.15, color="#1565C0")326    ax2.get_lines()[1].set(color="red", lw=1)327    plt.tight_layout(w_pad=1.5)328    for ext in ("pdf", "png"):329        plt.savefig(config.FIGURES_DIR / f"fig11_residual_diagnostics.{ext}")330    plt.close()331    print("  fig11_residual_diagnostics")332333334def main():335    df = pd.read_csv(config.ANALYSIS_CSV)336    print(f"{len(df):,} observations — fitting models ...")337    models, _, _ = fit_all_models(df)338    mD = models["D"]339340    config.FIGURES_DIR.mkdir(parents=True, exist_ok=True)341    print("Generating figures ...")342    fig1_model_comparison(models)343    fig2_coefficient_plot(mD)344    fig3_similarity_distributions(df)345    fig4_quintile_heatmap(df)346    fig5_correlation_matrix(df)347    fig6_scatter_plots(df)348    fig7_methodology(len(df))349    fig8_r2_decomposition(models)350    fig9_price_distribution(df)351    fig10_structural_coefficients(mD)352    fig11_residual_diagnostics(mD)353    print("All figures saved to figures/")354355356if __name__ == "__main__":357    main()358