spb/wp3_uqo Public

1# Author: Simon-Pierre Boucher — contact@spboucher.ai2#3"""Loaders for the processed datasets and stored result pickles."""45import pickle67import pandas as pd89from . import config101112def load_analytical_sample() -> pd.DataFrame:13    """Return the analytical sample (788,842 listings x 68 engineered columns)."""14    return pd.read_pickle(config.ANALYTICAL_SAMPLE)151617def load_model_data() -> dict:18    """Return model-ready data: standardized X_const, y_clean, df_clean, OLS results."""19    with open(config.MODEL_DATA, "rb") as f:20        return pickle.load(f)212223def load_shap_data() -> dict:24    """Return SHAP values for the 10,000-observation XGBoost explanation sample."""25    with open(config.SHAP_DATA, "rb") as f:26        return pickle.load(f)272829def load_result(name: str) -> dict:30    """Load a stored result pickle from results/ by file name (with or without .pkl)."""31    if not name.endswith(".pkl"):32        name += ".pkl"33    with open(config.RESULTS_DIR / name, "rb") as f:34        return pickle.load(f)353637def build_feature_matrix(df: pd.DataFrame):38    """Build the 62-regressor design matrix used by the tree models and FE OLS.3940    Returns41    -------42    X_full : pd.DataFrame43        Base features plus one-hot categorical dummies (drop-first).44    parts : dict45        Column groups: 'region_dummies', 'pure_categorical_dummies',46        and the raw dummy frame under 'cat_dummies'.47    """48    cat_dummies = pd.get_dummies(df[config.CATEGORICAL_COLS], drop_first=True).astype(int)49    region_cols = [c for c in cat_dummies.columns if c.startswith("region_")]50    pure_cat_cols = [c for c in cat_dummies.columns if not c.startswith("region_")]5152    X_full = pd.concat([df[config.ALL_BASE_FEATS], cat_dummies], axis=1)53    parts = {54        "cat_dummies": cat_dummies,55        "region_dummies": region_cols,56        "pure_categorical_dummies": pure_cat_cols,57    }58    return X_full, parts59