#!/usr/bin/env python """Step 08 — Model F: prediction and economic value (markdown / exit within 30 days, Cox time-to-exit). Idempotent: every output under results/prediction/, the figure fig10_prediction and paper/tables/tab08_prediction.tex are rebuilt from scratch on each run (stale files from a previous run are removed first). Usage: .venv/bin/python scripts/08_prediction.py [--n-boot 200] """ import argparse import json import shutil from ucml import config as C from ucml import prediction ap = argparse.ArgumentParser() ap.add_argument("--n-boot", type=int, default=prediction.N_BOOT, help="bootstrap resamples for the CIs (default 200)") args = ap.parse_args() if prediction.OUT.exists(): shutil.rmtree(prediction.OUT) prediction.OUT.mkdir(parents=True, exist_ok=True) for stem in ("fig10_prediction.pdf", "fig10_prediction.png"): (C.FIGURES / stem).unlink(missing_ok=True) (C.ROOT / "paper" / "figures" / "fig10_prediction.pdf").unlink(missing_ok=True) (C.TABLES / "tab08_prediction.tex").unlink(missing_ok=True) stats = C.PROCESSED / "cleaning_stats.json" if stats.exists(): s = json.loads(stats.read_text()) print("cleaning_stats.json:", {k: v for k, v in s.items() if "S2" in k or "S1" in k}, flush=True) m = prediction.run_all(n_boot=args.n_boot) for target in ("markdown30", "markdown30cr", "exit30"): for split in ("chronological", "dealer_holdout", "region_holdout"): r = m["targets"][target][split] print(f"{target:13s} {split:16s} n_test {r['n_test']:6d} base {r['models']['logistic']['base_rate']:.3f} | " + " | ".join(f"{k} logloss {r['models'][k]['log_loss']:.4f} brier {r['models'][k]['brier']:.4f} auc {r['models'][k]['auc']:.3f}" for k in ("logistic", "lightgbm"))) print("survival:", {k: m["survival"][k] for k in ("n_train", "n_test", "events_test", "c_index_train", "c_index_test", "dynamic_auc_30d")})