Python 75.9%
TeX 24%
1#!/usr/bin/env python32"""3================================================================4Auteur : Simon-Pierre Boucher5Contact : contact@spboucher.ai6Projet : Prévision de volatilité réalisée multi-actifs7 (HAR-RV vs GARCH vs Machine Learning)8Fichier : 08_make_tables.py9Description : Étape 08 — Génération des tables LaTeX du papier10 (booktabs) à partir des CSV de results/reproduced.11 AUCUN chiffre n'est écrit à la main.12Usage : python scripts/08_make_tables.py13================================================================14"""1516from __future__ import annotations1718import logging19import sys20from pathlib import Path2122import numpy as np23import pandas as pd2425sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))2627from wp12 import config # noqa: E4022829logger = logging.getLogger("08_tables")3031OUT = config.path("reproduced")32TAB = config.path("tables")3334GROUPS: list[tuple[str, list[str]]] = [35 ("A. GARCH family", ["GARCH", "GJR", "EGARCH", "RealGARCH"]),36 ("B. HAR family", ["HAR", "HAR-J", "HAR-CJ", "SHAR", "HARQ", "LogHAR"]),37 ("C. Machine learning --- HAR information set",38 ["Ridge-H", "LASSO-H", "ElasticNet-H", "RF-H", "XGBoost-H", "LightGBM-H"]),39 ("D. Machine learning --- extended information set",40 ["Ridge-X", "LASSO-X", "ElasticNet-X", "RF-X", "XGBoost-X", "LightGBM-X"]),41 ("E. Deep learning and pooled", ["LSTM", "Transformer", "Pooled-LGBM"]),42 ("F. Forecast combinations", ["Comb-Mean", "Comb-InvMSE"]),43]4445CLS_LABEL = {"equity": "Equities/ETFs", "fx": "FX", "crypto": "Crypto",46 "futures": "Futures"}474849def _fmt(x: float, nd: int = 3, bold: bool = False) -> str:50 if not np.isfinite(x):51 return "---"52 s = f"{x:.{nd}f}"53 return f"\\textbf{{{s}}}" if bold else s545556def _write(name: str, lines: list[str]) -> None:57 f = TAB / f"{name}.tex"58 f.write_text("\n".join(lines) + "\n", encoding="utf-8")59 logger.info("wrote %s", f.name)606162def table_summary_stats() -> None:63 """Descriptive statistics of daily annualized RV by instrument."""64 panel = pd.read_parquet(config.path("processed") / "panel.parquet")65 lines = [66 "\\begin{tabular}{lrrrrrrr}", "\\toprule",67 "Ticker & Days & \\makecell{Mean vol.\\\\(\\%)} & \\makecell{Median vol.\\\\(\\%)}"68 " & \\makecell{P95 vol.\\\\(\\%)} & \\makecell{Skew.\\\\$\\ln$RV}"69 " & \\makecell{AC(1)\\\\$\\ln$RV} & \\makecell{Jump\\\\days (\\%)} \\\\",70 "\\midrule",71 ]72 for cls in ["equity", "fx", "crypto", "futures"]:73 sub = panel[panel["cls"] == cls]74 if sub.empty:75 continue76 lines.append(f"\\multicolumn{{8}}{{l}}{{\\itshape {CLS_LABEL[cls]}}}\\\\")77 for tk, df in sub.groupby("ticker"):78 vol = np.sqrt(df["rv5ss"] * 252) * 10079 lrv = np.log(df["rv5ss"].clip(lower=1e-12))80 lines.append(81 f"\\quad {tk} & {len(df):,} & {vol.mean():.1f} & {vol.median():.1f}"82 f" & {vol.quantile(0.95):.1f} & {lrv.skew():.2f}"83 f" & {lrv.autocorr(1):.2f} & {100 * (df['jump'] > 0).mean():.1f} \\\\"84 )85 lines += ["\\bottomrule", "\\end{tabular}"]86 _write("summary_stats", lines)878889def _grouped_metric_table(90 name: str, wide: pd.DataFrame, cols: list[tuple[str, str]],91 nd: int = 3, note_bench: str | None = "HAR", bold_min_cols: bool = True,92) -> None:93 """Generic grouped model table. ``wide`` is indexed by model with the94 metric columns; ``cols`` maps (column, header)."""95 align = "l" + "c" * len(cols)96 lines = [f"\\begin{{tabular}}{{{align}}}", "\\toprule",97 "Model & " + " & ".join(h for _, h in cols) + " \\\\", "\\midrule"]98 mins = {c: wide[c].min() for c, _ in cols} if bold_min_cols else {}99 for gname, models in GROUPS:100 avail = [m for m in models if m in wide.index]101 if not avail:102 continue103 lines.append(f"\\multicolumn{{{len(cols) + 1}}}{{l}}{{\\itshape {gname}}}\\\\")104 for m in avail:105 cells = []106 for c, _ in cols:107 v = wide.loc[m, c]108 bold = bold_min_cols and np.isfinite(v) and v == mins[c]109 cells.append(_fmt(v, nd, bold))110 lines.append(f"\\quad {m} & " + " & ".join(cells) + " \\\\")111 lines += ["\\bottomrule", "\\end{tabular}"]112 _write(name, lines)113114115def table_losses_main() -> None:116 """Headline QLIKE table: level for HAR, ratio for everything else."""117 tab = pd.read_csv(OUT / "losses_overall.csv")118 wide = tab.pivot_table(index="model", columns="h", values="qlike_ratio")119 wide.columns = [f"r{h}" for h in wide.columns]120 ql = tab.pivot_table(index="model", columns="h", values="qlike")121 ql.columns = [f"q{h}" for h in ql.columns]122 wide = wide.join(ql)123 cols = [("q1", "QLIKE $h{=}1$"), ("r1", "Ratio $h{=}1$"),124 ("q5", "QLIKE $h{=}5$"), ("r5", "Ratio $h{=}5$"),125 ("q22", "QLIKE $h{=}22$"), ("r22", "Ratio $h{=}22$")]126 _grouped_metric_table("losses_main", wide, cols)127128129def table_losses_class() -> None:130 """QLIKE ratio vs HAR by asset class (h=1 and h=22)."""131 tab = pd.read_csv(OUT / "losses_by_class.csv")132 parts = {}133 for h in (1, 22):134 sub = tab[tab["h"] == h].pivot_table(135 index="model", columns="cls", values="qlike_ratio")136 for cls in ["equity", "fx", "crypto", "futures"]:137 if cls in sub.columns:138 parts[f"{cls}_{h}"] = sub[cls]139 wide = pd.DataFrame(parts)140 cols = ([(f"{c}_1", CLS_LABEL[c].split("/")[0]) for c in141 ["equity", "fx", "crypto", "futures"]]142 + [(f"{c}_22", CLS_LABEL[c].split("/")[0]) for c in143 ["equity", "fx", "crypto", "futures"]])144 align = "l" + "cccc" + "cccc"145 lines = [f"\\begin{{tabular}}{{{align}}}", "\\toprule",146 " & \\multicolumn{4}{c}{$h = 1$} & \\multicolumn{4}{c}{$h = 22$} \\\\",147 "\\cmidrule(lr){2-5}\\cmidrule(lr){6-9}",148 "Model & " + " & ".join(h for _, h in cols) + " \\\\", "\\midrule"]149 mins = {c: wide[c].min() for c, _ in cols}150 for gname, models in GROUPS:151 avail = [m for m in models if m in wide.index and m != "HAR"]152 if not avail:153 continue154 lines.append(f"\\multicolumn{{9}}{{l}}{{\\itshape {gname}}}\\\\")155 for m in avail:156 cells = [_fmt(wide.loc[m, c], 3, np.isfinite(wide.loc[m, c])157 and wide.loc[m, c] == mins[c]) for c, _ in cols]158 lines.append(f"\\quad {m} & " + " & ".join(cells) + " \\\\")159 lines += ["\\bottomrule", "\\end{tabular}"]160 _write("losses_class", lines)161162163def table_dm_mcs() -> None:164 """DM outcomes vs HAR and MCS inclusion, side by side."""165 dm = pd.read_csv(OUT / "dm_summary.csv")166 mcs = pd.read_csv(OUT / "mcs_inclusion.csv")167 parts = {}168 for h in (1, 5, 22):169 d = dm[dm["h"] == h].set_index("model")170 parts[f"sb{h}"] = d["pct_sig_better"] * 100171 parts[f"sw{h}"] = d["pct_sig_worse"] * 100172 parts[f"mcs{h}"] = mcs[mcs["h"] == h].set_index("model")["mcs_inclusion"] * 100173 wide = pd.DataFrame(parts)174 lines = ["\\begin{tabular}{lrrr rrr rrr}", "\\toprule",175 " & \\multicolumn{3}{c}{Sig.\\ better than HAR (\\%)}"176 " & \\multicolumn{3}{c}{Sig.\\ worse than HAR (\\%)}"177 " & \\multicolumn{3}{c}{In 90\\% MCS (\\%)} \\\\",178 "\\cmidrule(lr){2-4}\\cmidrule(lr){5-7}\\cmidrule(lr){8-10}",179 "Model & $h{=}1$ & $h{=}5$ & $h{=}22$ & $h{=}1$ & $h{=}5$ & $h{=}22$"180 " & $h{=}1$ & $h{=}5$ & $h{=}22$ \\\\", "\\midrule"]181 for gname, models in GROUPS:182 avail = [m for m in models if m in wide.index]183 if gname.startswith("B."):184 avail = ["HAR"] + [m for m in avail if m != "HAR"] if "HAR" in wide.index else avail185 if not avail:186 continue187 lines.append(f"\\multicolumn{{10}}{{l}}{{\\itshape {gname}}}\\\\")188 for m in avail:189 def _c(key: str) -> str:190 v = wide.loc[m, key] if key in wide.columns else np.nan191 return f"{v:.0f}" if np.isfinite(v) else "---"192 row = [_c(f"sb{h}") for h in (1, 5, 22)] + \193 [_c(f"sw{h}") for h in (1, 5, 22)] + \194 [_c(f"mcs{h}") for h in (1, 5, 22)]195 lines.append(f"\\quad {m} & " + " & ".join(row) + " \\\\")196 lines += ["\\bottomrule", "\\end{tabular}"]197 _write("dm_mcs", lines)198199200def table_mz_encompassing() -> None:201 """Mincer-Zarnowitz medians and encompassing outcomes (h=1)."""202 mz = pd.read_csv(OUT / "mz_summary.csv")203 enc = pd.read_csv(OUT / "encompassing_summary.csv")204 m1 = mz[mz["h"] == 1].set_index("model")205 e1 = enc[enc["h"] == 1].set_index("model")206 wide = pd.DataFrame({207 "beta": m1["med_beta"], "r2": m1["med_r2"],208 "rej": m1["pct_reject"] * 100,209 "b2": e1["med_b2"], "adds": e1["pct_adds_info"] * 100,210 })211 lines = ["\\begin{tabular}{lccccc}", "\\toprule",212 " & \\multicolumn{3}{c}{Mincer--Zarnowitz}"213 " & \\multicolumn{2}{c}{Encompassing (vs HAR)} \\\\",214 "\\cmidrule(lr){2-4}\\cmidrule(lr){5-6}",215 "Model & Med.\\ $\\hat\\beta$ & Med.\\ $R^2$ & Rej.\\ (\\%)"216 " & Med.\\ $\\hat b_2$ & Adds info (\\%) \\\\", "\\midrule"]217 for gname, models in GROUPS:218 avail = [m for m in models if m in wide.index]219 if not avail:220 continue221 lines.append(f"\\multicolumn{{6}}{{l}}{{\\itshape {gname}}}\\\\")222 for m in avail:223 r = wide.loc[m]224 b2 = f"{r['b2']:.2f}" if np.isfinite(r["b2"]) else "---"225 adds = f"{r['adds']:.0f}" if np.isfinite(r["adds"]) else "---"226 lines.append(227 f"\\quad {m} & {r['beta']:.2f} & {r['r2']:.2f} & {r['rej']:.0f}"228 f" & {b2} & {adds} \\\\")229 lines += ["\\bottomrule", "\\end{tabular}"]230 _write("mz_encompassing", lines)231232233def table_subperiods() -> None:234 """QLIKE ratio vs HAR per sub-period (h=1)."""235 tab = pd.read_csv(OUT / "subperiod_losses.csv")236 tab = tab[tab["h"] == 1]237 bench = tab[tab["model"] == "HAR"].set_index("period")["qlike"]238 wide = tab.pivot_table(index="model", columns="period", values="qlike")239 wide = wide.div(bench, axis=1)240 periods = ["pre2020", "covid", "inflation", "recent"]241 heads = ["2013--19", "2020", "2021--22", "2024--26"]242 cols = [(p, h) for p, h in zip(periods, heads) if p in wide.columns]243 _grouped_metric_table("subperiods", wide, cols)244245246def table_sensitivity() -> None:247 """Frequency and window sensitivity (QLIKE, h=1)."""248 lines = ["\\begin{tabular}{lccc c ccc}", "\\toprule",249 " & \\multicolumn{3}{c}{RV measure} & &"250 " \\multicolumn{3}{c}{Estimation window (days)} \\\\",251 "\\cmidrule(lr){2-4}\\cmidrule(lr){6-8}",252 "Model & 1-min & 5-min ss & Kernel & & 500 & 1000 & 2000 \\\\",253 "\\midrule"]254 freq = pd.read_csv(OUT / "frequency_sensitivity.csv")255 freq = freq[freq["h"] == 1].pivot_table(index="model", columns="frequency",256 values="qlike")257 win = pd.read_csv(OUT / "window_sensitivity.csv")258 win = win[win["h"] == 1].pivot_table(index="model", columns="window",259 values="qlike")260 win.columns = [str(c) for c in win.columns]261 for m in ["HAR", "LogHAR", "LightGBM"]:262 f = freq.loc[m] if m in freq.index else pd.Series(dtype=float)263 w = win.loc[m] if m in win.index else pd.Series(dtype=float)264 cells = [265 _fmt(f.get("rv1min", np.nan)), _fmt(f.get("rv5min_ss", np.nan)),266 _fmt(f.get("rkernel", np.nan)), "",267 _fmt(w.get("500", np.nan)), _fmt(w.get("1000", np.nan)),268 _fmt(w.get("2000", np.nan)),269 ]270 lines.append(f"{m} & " + " & ".join(cells) + " \\\\")271 lines += ["\\bottomrule", "\\end{tabular}"]272 _write("sensitivity", lines)273274275def table_transferability() -> None:276 """Cross-asset transferability: QLIKE h=1 for the class representatives."""277 tab = pd.read_csv(OUT / "transferability.csv")278 tab = tab[tab["h"] == 1]279 wide = tab.pivot_table(index="model", columns="ticker", values="qlike")280 order = ["HAR", "LightGBM-X", "Pooled-LGBM", "Pooled-LGBM-LOO"]281 label = {"HAR": "HAR (own asset)", "LightGBM-X": "LightGBM (own asset)",282 "Pooled-LGBM": "Pooled LightGBM (asset in pool)",283 "Pooled-LGBM-LOO": "Pooled LightGBM (asset excluded)"}284 tickers = [t for t in ["NVDA", "GBPUSD", "ETH", "GC"] if t in wide.columns]285 lines = ["\\begin{tabular}{l" + "c" * len(tickers) + "}", "\\toprule",286 "Model & " + " & ".join(tickers) + " \\\\", "\\midrule"]287 for m in order:288 if m not in wide.index:289 continue290 cells = [_fmt(wide.loc[m, t]) for t in tickers]291 lines.append(f"{label[m]} & " + " & ".join(cells) + " \\\\")292 lines += ["\\bottomrule", "\\end{tabular}"]293 _write("transferability", lines)294295296def table_timing() -> None:297 """Volatility timing: net Sharpe, cost sensitivity, utility gains."""298 overall = pd.read_csv(OUT / "timing_overall.csv").set_index("model")299 util = pd.read_csv(OUT / "utility_summary.csv").set_index("model")300 byt = pd.read_csv(OUT / "timing_by_ticker.csv")301 sens = byt.groupby(["model", "tc_bps"], observed=True)["sharpe"].mean().unstack()302 wide = pd.DataFrame({303 "sharpe": overall["sharpe"], "ann_ret": overall["ann_ret"] * 100,304 "turn": overall["turnover"],305 "s0": sens.get(0), "s10": sens.get(10), "s20": sens.get(20),306 "ug": util["mean"],307 })308 lines = ["\\begin{tabular}{lccc ccc c}", "\\toprule",309 " & \\multicolumn{3}{c}{Net of 5 bps} &"310 " \\multicolumn{3}{c}{Sharpe at cost (bps)} & \\\\",311 "\\cmidrule(lr){2-4}\\cmidrule(lr){5-7}",312 "Model & Sharpe & \\makecell{Ann.\\ ret.\\\\(\\%)} & Turnover"313 " & 0 & 10 & 20 & \\makecell{Utility gain\\\\vs HAR (bps)} \\\\",314 "\\midrule"]315 smax = wide["sharpe"].max()316 for gname, models in GROUPS:317 avail = [m for m in models if m in wide.index]318 if not avail:319 continue320 lines.append(f"\\multicolumn{{8}}{{l}}{{\\itshape {gname}}}\\\\")321 for m in avail:322 r = wide.loc[m]323 ug = f"{r['ug']:.0f}" if np.isfinite(r["ug"]) else "---"324 sh = _fmt(r["sharpe"], 2, r["sharpe"] == smax)325 lines.append(326 f"\\quad {m} & {sh} & {r['ann_ret']:.1f} & {r['turn']:.3f}"327 f" & {_fmt(r['s0'], 2)} & {_fmt(r['s10'], 2)} & {_fmt(r['s20'], 2)}"328 f" & {ug} \\\\")329 lines += ["\\bottomrule", "\\end{tabular}"]330 _write("timing", lines)331332333def table_var() -> None:334 """VaR backtest outcomes at 1% and 5%."""335 tab = pd.read_csv(OUT / "var_summary.csv")336 parts = {}337 for lv in (0.01, 0.05):338 s = tab[tab["level"] == lv].set_index("model")339 parts[f"vr{lv}"] = s["mean_viol_rate"] * 100340 parts[f"ku{lv}"] = s["pct_pass_kupiec"] * 100341 parts[f"cc{lv}"] = s["pct_pass_cc"] * 100342 wide = pd.DataFrame(parts)343 lines = ["\\begin{tabular}{lccc ccc}", "\\toprule",344 " & \\multicolumn{3}{c}{VaR 1\\%} & \\multicolumn{3}{c}{VaR 5\\%} \\\\",345 "\\cmidrule(lr){2-4}\\cmidrule(lr){5-7}",346 "Model & \\makecell{Viol.\\\\(\\%)} & \\makecell{Kupiec\\\\pass (\\%)}"347 " & \\makecell{CC\\\\pass (\\%)}"348 " & \\makecell{Viol.\\\\(\\%)} & \\makecell{Kupiec\\\\pass (\\%)}"349 " & \\makecell{CC\\\\pass (\\%)} \\\\", "\\midrule"]350 for gname, models in GROUPS:351 avail = [m for m in models if m in wide.index]352 if not avail:353 continue354 lines.append(f"\\multicolumn{{7}}{{l}}{{\\itshape {gname}}}\\\\")355 for m in avail:356 r = wide.loc[m]357 lines.append(358 f"\\quad {m} & {r['vr0.01']:.2f} & {r['ku0.01']:.0f} & {r['cc0.01']:.0f}"359 f" & {r['vr0.05']:.2f} & {r['ku0.05']:.0f} & {r['cc0.05']:.0f} \\\\")360 lines += ["\\bottomrule", "\\end{tabular}"]361 _write("var", lines)362363364def table_by_ticker_appendix() -> None:365 """Appendix: QLIKE h=1 per ticker for headline models."""366 tab = pd.read_csv(OUT / "losses_by_ticker.csv")367 tab = tab[tab["h"] == 1]368 models = ["HAR", "LogHAR", "RealGARCH", "LightGBM-X", "Pooled-LGBM",369 "LSTM", "Comb-InvMSE"]370 wide = tab.pivot_table(index=["cls", "ticker"], columns="model",371 values="qlike")372 avail = [m for m in models if m in wide.columns]373 lines = ["\\begin{tabular}{l" + "c" * len(avail) + "}", "\\toprule",374 "Ticker & " + " & ".join(avail) + " \\\\", "\\midrule"]375 for cls in ["equity", "fx", "crypto", "futures"]:376 if cls not in wide.index.get_level_values(0):377 continue378 lines.append(f"\\multicolumn{{{len(avail) + 1}}}{{l}}"379 f"{{\\itshape {CLS_LABEL[cls]}}}\\\\")380 sub = wide.loc[cls]381 for tk, r in sub.iterrows():382 best = r[avail].min()383 cells = [_fmt(r[m], 3, np.isfinite(r[m]) and r[m] == best)384 for m in avail]385 lines.append(f"\\quad {tk} & " + " & ".join(cells) + " \\\\")386 lines += ["\\bottomrule", "\\end{tabular}"]387 _write("by_ticker_appendix", lines)388389390def main() -> None:391 """Generate every LaTeX table from the reproduced CSVs."""392 config.setup_logging()393 for fn in [table_summary_stats, table_losses_main, table_losses_class,394 table_dm_mcs, table_mz_encompassing, table_subperiods,395 table_sensitivity, table_transferability, table_timing,396 table_var, table_by_ticker_appendix]:397 try:398 fn()399 except Exception as exc: # noqa: BLE001400 logger.error("table %s FAILED: %s", fn.__name__, exc)401402403if __name__ == "__main__":404 main()405