spb/wp3_uqo Public

1#!/usr/bin/env python32# Author: Simon-Pierre Boucher — contact@spboucher.ai3#4"""Export the key result tables of the paper as CSV files into results/tables/.56Each CSV mirrors a table in the paper and is generated directly from the7stored result pickles, providing a machine-readable audit trail between the8pickles and the numbers reported in the LaTeX source.910Usage:  python scripts/04_export_tables.py11"""1213import sys14from pathlib import Path1516import numpy as np17import pandas as pd1819sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))20from wp3 import config, data2122OUT = config.RESULTS_DIR / "tables"232425def export_qr_coefficients():26    """Quantile regression coefficients across taus (paper Table: QR results)."""27    qr = data.load_result("qr_results.pkl")28    taus = sorted(qr.keys())29    rows = {}30    for t in taus:31        rows[f"tau_{t}"] = pd.Series(qr[t]["params"])32    df = pd.DataFrame(rows)33    df.index.name = "variable"34    df.to_csv(OUT / "qr_coefficients.csv")35    pd.DataFrame({f"tau_{t}": {"pseudo_r2": qr[t]["pseudo_r2"]} for t in taus}) \36        .to_csv(OUT / "qr_pseudo_r2.csv")373839def export_iqr_tests():40    """Inter-quantile Wald tests, tau=0.10 vs tau=0.90."""41    r = data.load_result("v3_qr_imputation_results.pkl")42    df = pd.DataFrame(r["iqr_test"]).T43    df.index.name = "variable"44    df.to_csv(OUT / "iqr_wald_tests.csv")454647def export_qr_stability():48    """QR subsample stability (CV, sign stability)."""49    r = data.load_result("v3_qr_imputation_results.pkl")50    df = pd.DataFrame(r["qr_stability"]).T51    df.index.name = "variable"52    df.to_csv(OUT / "qr_stability.csv")535455def export_imputation_winsorization():56    """Imputation and winsorization sensitivity."""57    r = data.load_result("v3_qr_imputation_results.pkl")58    imp = pd.DataFrame([59        {"scenario": s["label"], "N": s["N"], "R2": s["R2"],60         **{f"beta_{k}": v for k, v in s["key_coefficients"].items()}}61        for s in r["imputation_sensitivity"]62    ])63    imp.to_csv(OUT / "imputation_sensitivity.csv", index=False)6465    w = r["winsorization_sensitivity"]66    win = pd.DataFrame({67        "baseline": {"R2": w["baseline"]["R2"], **w["baseline"]["key_coefficients"]},68        "winsorized": {"R2": w["winsorized"]["R2"], **w["winsorized"]["key_coefficients"]},69    })70    win.index.name = "metric"71    win.to_csv(OUT / "winsorization_sensitivity.csv")727374def export_spatial_and_ablation():75    """ZIP3 FE, Moran robustness, XGBoost geography variants, ablation."""76    v3 = data.load_result("v3_spatial_results.pkl")7778    pd.Series(v3["zip3_fe_ols"]).to_csv(OUT / "zip3_fixed_effects.csv")7980    m = v3["morans_i_robustness"]81    pd.DataFrame({"morans_i": m["values"], "p_value": m["pvalues"]}) \82        .to_csv(OUT / "morans_i_robustness.csv", index_label="subsample")8384    rows = []85    for name in ["xgb_with_latlon", "xgb_no_geography"]:86        rows.append({"model": name, **{k: v for k, v in v3[name].items()}})87    for stage, res in v3["ablation"].items():88        rows.append({"model": f"ablation_{stage}",89                     **{k: v for k, v in res.items() if k != "features"}})90    pd.DataFrame(rows).to_csv(OUT / "xgb_geography_and_ablation.csv", index=False)919293def export_shap():94    """Mean |SHAP| importance and cross-model stability."""95    sd = data.load_shap_data()96    pd.Series(sd["mean_shap"], name="mean_abs_shap") \97        .sort_values(ascending=False) \98        .to_csv(OUT / "shap_importance_xgb.csv", index_label="variable")99100    st = data.load_result("v3_shap_stability.pkl")101    pd.DataFrame({102        "pair": ["xgb_lgb", "xgb_rf", "lgb_rf"],103        "spearman_rho": [float(st["rho_xgb_lgb"]), float(st["rho_xgb_rf"]),104                         float(st["rho_lgb_rf"])],105    }).to_csv(OUT / "shap_cross_model_stability.csv", index=False)106107108def export_ols():109    """Standardized and unstandardized OLS coefficients."""110    md = data.load_model_data()111    ols = md["ols_results"]112    df = pd.DataFrame({"coef": ols["params"], "se": ols["bse"],113                       "t": ols["tvalues"], "p": ols["pvalues"]})114    df.index.name = "variable"115    df.to_csv(OUT / "ols_standardized.csv")116117    ext = data.load_result("extended_results.pkl")118    df_u = pd.DataFrame({"coef": ext["unstd_params"], "se": ext["unstd_bse"],119                         "t": ext["unstd_tvalues"], "p": ext["unstd_pvalues"]})120    df_u.index.name = "variable"121    df_u.to_csv(OUT / "ols_unstandardized.csv")122123124def main():125    OUT.mkdir(parents=True, exist_ok=True)126    for fn in [export_ols, export_qr_coefficients, export_iqr_tests,127               export_qr_stability, export_imputation_winsorization,128               export_spatial_and_ablation, export_shap]:129        print(f"Exporting {fn.__name__.replace('export_', '')}...")130        fn()131    print(f"Done. CSVs written to {OUT}")132133134if __name__ == "__main__":135    main()136