#!/usr/bin/env python3 """ ================================================================ Auteur : Simon-Pierre Boucher Contact : contact@spboucher.ai Projet : Prévision de volatilité réalisée multi-actifs (HAR-RV vs GARCH vs Machine Learning) Fichier : 08_make_tables.py Description : Étape 08 — Génération des tables LaTeX du papier (booktabs) à partir des CSV de results/reproduced. AUCUN chiffre n'est écrit à la main. Usage : python scripts/08_make_tables.py ================================================================ """ from __future__ import annotations import logging import sys from pathlib import Path import numpy as np import pandas as pd sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from wp12 import config # noqa: E402 logger = logging.getLogger("08_tables") OUT = config.path("reproduced") TAB = config.path("tables") GROUPS: list[tuple[str, list[str]]] = [ ("A. GARCH family", ["GARCH", "GJR", "EGARCH", "RealGARCH"]), ("B. HAR family", ["HAR", "HAR-J", "HAR-CJ", "SHAR", "HARQ", "LogHAR"]), ("C. Machine learning --- HAR information set", ["Ridge-H", "LASSO-H", "ElasticNet-H", "RF-H", "XGBoost-H", "LightGBM-H"]), ("D. Machine learning --- extended information set", ["Ridge-X", "LASSO-X", "ElasticNet-X", "RF-X", "XGBoost-X", "LightGBM-X"]), ("E. Deep learning and pooled", ["LSTM", "Transformer", "Pooled-LGBM"]), ("F. Forecast combinations", ["Comb-Mean", "Comb-InvMSE"]), ] CLS_LABEL = {"equity": "Equities/ETFs", "fx": "FX", "crypto": "Crypto", "futures": "Futures"} def _fmt(x: float, nd: int = 3, bold: bool = False) -> str: if not np.isfinite(x): return "---" s = f"{x:.{nd}f}" return f"\\textbf{{{s}}}" if bold else s def _write(name: str, lines: list[str]) -> None: f = TAB / f"{name}.tex" f.write_text("\n".join(lines) + "\n", encoding="utf-8") logger.info("wrote %s", f.name) def table_summary_stats() -> None: """Descriptive statistics of daily annualized RV by instrument.""" panel = pd.read_parquet(config.path("processed") / "panel.parquet") lines = [ "\\begin{tabular}{lrrrrrrr}", "\\toprule", "Ticker & Days & \\makecell{Mean vol.\\\\(\\%)} & \\makecell{Median vol.\\\\(\\%)}" " & \\makecell{P95 vol.\\\\(\\%)} & \\makecell{Skew.\\\\$\\ln$RV}" " & \\makecell{AC(1)\\\\$\\ln$RV} & \\makecell{Jump\\\\days (\\%)} \\\\", "\\midrule", ] for cls in ["equity", "fx", "crypto", "futures"]: sub = panel[panel["cls"] == cls] if sub.empty: continue lines.append(f"\\multicolumn{{8}}{{l}}{{\\itshape {CLS_LABEL[cls]}}}\\\\") for tk, df in sub.groupby("ticker"): vol = np.sqrt(df["rv5ss"] * 252) * 100 lrv = np.log(df["rv5ss"].clip(lower=1e-12)) lines.append( f"\\quad {tk} & {len(df):,} & {vol.mean():.1f} & {vol.median():.1f}" f" & {vol.quantile(0.95):.1f} & {lrv.skew():.2f}" f" & {lrv.autocorr(1):.2f} & {100 * (df['jump'] > 0).mean():.1f} \\\\" ) lines += ["\\bottomrule", "\\end{tabular}"] _write("summary_stats", lines) def _grouped_metric_table( name: str, wide: pd.DataFrame, cols: list[tuple[str, str]], nd: int = 3, note_bench: str | None = "HAR", bold_min_cols: bool = True, ) -> None: """Generic grouped model table. ``wide`` is indexed by model with the metric columns; ``cols`` maps (column, header).""" align = "l" + "c" * len(cols) lines = [f"\\begin{{tabular}}{{{align}}}", "\\toprule", "Model & " + " & ".join(h for _, h in cols) + " \\\\", "\\midrule"] mins = {c: wide[c].min() for c, _ in cols} if bold_min_cols else {} for gname, models in GROUPS: avail = [m for m in models if m in wide.index] if not avail: continue lines.append(f"\\multicolumn{{{len(cols) + 1}}}{{l}}{{\\itshape {gname}}}\\\\") for m in avail: cells = [] for c, _ in cols: v = wide.loc[m, c] bold = bold_min_cols and np.isfinite(v) and v == mins[c] cells.append(_fmt(v, nd, bold)) lines.append(f"\\quad {m} & " + " & ".join(cells) + " \\\\") lines += ["\\bottomrule", "\\end{tabular}"] _write(name, lines) def table_losses_main() -> None: """Headline QLIKE table: level for HAR, ratio for everything else.""" tab = pd.read_csv(OUT / "losses_overall.csv") wide = tab.pivot_table(index="model", columns="h", values="qlike_ratio") wide.columns = [f"r{h}" for h in wide.columns] ql = tab.pivot_table(index="model", columns="h", values="qlike") ql.columns = [f"q{h}" for h in ql.columns] wide = wide.join(ql) cols = [("q1", "QLIKE $h{=}1$"), ("r1", "Ratio $h{=}1$"), ("q5", "QLIKE $h{=}5$"), ("r5", "Ratio $h{=}5$"), ("q22", "QLIKE $h{=}22$"), ("r22", "Ratio $h{=}22$")] _grouped_metric_table("losses_main", wide, cols) def table_losses_class() -> None: """QLIKE ratio vs HAR by asset class (h=1 and h=22).""" tab = pd.read_csv(OUT / "losses_by_class.csv") parts = {} for h in (1, 22): sub = tab[tab["h"] == h].pivot_table( index="model", columns="cls", values="qlike_ratio") for cls in ["equity", "fx", "crypto", "futures"]: if cls in sub.columns: parts[f"{cls}_{h}"] = sub[cls] wide = pd.DataFrame(parts) cols = ([(f"{c}_1", CLS_LABEL[c].split("/")[0]) for c in ["equity", "fx", "crypto", "futures"]] + [(f"{c}_22", CLS_LABEL[c].split("/")[0]) for c in ["equity", "fx", "crypto", "futures"]]) align = "l" + "cccc" + "cccc" lines = [f"\\begin{{tabular}}{{{align}}}", "\\toprule", " & \\multicolumn{4}{c}{$h = 1$} & \\multicolumn{4}{c}{$h = 22$} \\\\", "\\cmidrule(lr){2-5}\\cmidrule(lr){6-9}", "Model & " + " & ".join(h for _, h in cols) + " \\\\", "\\midrule"] mins = {c: wide[c].min() for c, _ in cols} for gname, models in GROUPS: avail = [m for m in models if m in wide.index and m != "HAR"] if not avail: continue lines.append(f"\\multicolumn{{9}}{{l}}{{\\itshape {gname}}}\\\\") for m in avail: cells = [_fmt(wide.loc[m, c], 3, np.isfinite(wide.loc[m, c]) and wide.loc[m, c] == mins[c]) for c, _ in cols] lines.append(f"\\quad {m} & " + " & ".join(cells) + " \\\\") lines += ["\\bottomrule", "\\end{tabular}"] _write("losses_class", lines) def table_dm_mcs() -> None: """DM outcomes vs HAR and MCS inclusion, side by side.""" dm = pd.read_csv(OUT / "dm_summary.csv") mcs = pd.read_csv(OUT / "mcs_inclusion.csv") parts = {} for h in (1, 5, 22): d = dm[dm["h"] == h].set_index("model") parts[f"sb{h}"] = d["pct_sig_better"] * 100 parts[f"sw{h}"] = d["pct_sig_worse"] * 100 parts[f"mcs{h}"] = mcs[mcs["h"] == h].set_index("model")["mcs_inclusion"] * 100 wide = pd.DataFrame(parts) lines = ["\\begin{tabular}{lrrr rrr rrr}", "\\toprule", " & \\multicolumn{3}{c}{Sig.\\ better than HAR (\\%)}" " & \\multicolumn{3}{c}{Sig.\\ worse than HAR (\\%)}" " & \\multicolumn{3}{c}{In 90\\% MCS (\\%)} \\\\", "\\cmidrule(lr){2-4}\\cmidrule(lr){5-7}\\cmidrule(lr){8-10}", "Model & $h{=}1$ & $h{=}5$ & $h{=}22$ & $h{=}1$ & $h{=}5$ & $h{=}22$" " & $h{=}1$ & $h{=}5$ & $h{=}22$ \\\\", "\\midrule"] for gname, models in GROUPS: avail = [m for m in models if m in wide.index] if gname.startswith("B."): avail = ["HAR"] + [m for m in avail if m != "HAR"] if "HAR" in wide.index else avail if not avail: continue lines.append(f"\\multicolumn{{10}}{{l}}{{\\itshape {gname}}}\\\\") for m in avail: def _c(key: str) -> str: v = wide.loc[m, key] if key in wide.columns else np.nan return f"{v:.0f}" if np.isfinite(v) else "---" row = [_c(f"sb{h}") for h in (1, 5, 22)] + \ [_c(f"sw{h}") for h in (1, 5, 22)] + \ [_c(f"mcs{h}") for h in (1, 5, 22)] lines.append(f"\\quad {m} & " + " & ".join(row) + " \\\\") lines += ["\\bottomrule", "\\end{tabular}"] _write("dm_mcs", lines) def table_mz_encompassing() -> None: """Mincer-Zarnowitz medians and encompassing outcomes (h=1).""" mz = pd.read_csv(OUT / "mz_summary.csv") enc = pd.read_csv(OUT / "encompassing_summary.csv") m1 = mz[mz["h"] == 1].set_index("model") e1 = enc[enc["h"] == 1].set_index("model") wide = pd.DataFrame({ "beta": m1["med_beta"], "r2": m1["med_r2"], "rej": m1["pct_reject"] * 100, "b2": e1["med_b2"], "adds": e1["pct_adds_info"] * 100, }) lines = ["\\begin{tabular}{lccccc}", "\\toprule", " & \\multicolumn{3}{c}{Mincer--Zarnowitz}" " & \\multicolumn{2}{c}{Encompassing (vs HAR)} \\\\", "\\cmidrule(lr){2-4}\\cmidrule(lr){5-6}", "Model & Med.\\ $\\hat\\beta$ & Med.\\ $R^2$ & Rej.\\ (\\%)" " & Med.\\ $\\hat b_2$ & Adds info (\\%) \\\\", "\\midrule"] for gname, models in GROUPS: avail = [m for m in models if m in wide.index] if not avail: continue lines.append(f"\\multicolumn{{6}}{{l}}{{\\itshape {gname}}}\\\\") for m in avail: r = wide.loc[m] b2 = f"{r['b2']:.2f}" if np.isfinite(r["b2"]) else "---" adds = f"{r['adds']:.0f}" if np.isfinite(r["adds"]) else "---" lines.append( f"\\quad {m} & {r['beta']:.2f} & {r['r2']:.2f} & {r['rej']:.0f}" f" & {b2} & {adds} \\\\") lines += ["\\bottomrule", "\\end{tabular}"] _write("mz_encompassing", lines) def table_subperiods() -> None: """QLIKE ratio vs HAR per sub-period (h=1).""" tab = pd.read_csv(OUT / "subperiod_losses.csv") tab = tab[tab["h"] == 1] bench = tab[tab["model"] == "HAR"].set_index("period")["qlike"] wide = tab.pivot_table(index="model", columns="period", values="qlike") wide = wide.div(bench, axis=1) periods = ["pre2020", "covid", "inflation", "recent"] heads = ["2013--19", "2020", "2021--22", "2024--26"] cols = [(p, h) for p, h in zip(periods, heads) if p in wide.columns] _grouped_metric_table("subperiods", wide, cols) def table_sensitivity() -> None: """Frequency and window sensitivity (QLIKE, h=1).""" lines = ["\\begin{tabular}{lccc c ccc}", "\\toprule", " & \\multicolumn{3}{c}{RV measure} & &" " \\multicolumn{3}{c}{Estimation window (days)} \\\\", "\\cmidrule(lr){2-4}\\cmidrule(lr){6-8}", "Model & 1-min & 5-min ss & Kernel & & 500 & 1000 & 2000 \\\\", "\\midrule"] freq = pd.read_csv(OUT / "frequency_sensitivity.csv") freq = freq[freq["h"] == 1].pivot_table(index="model", columns="frequency", values="qlike") win = pd.read_csv(OUT / "window_sensitivity.csv") win = win[win["h"] == 1].pivot_table(index="model", columns="window", values="qlike") win.columns = [str(c) for c in win.columns] for m in ["HAR", "LogHAR", "LightGBM"]: f = freq.loc[m] if m in freq.index else pd.Series(dtype=float) w = win.loc[m] if m in win.index else pd.Series(dtype=float) cells = [ _fmt(f.get("rv1min", np.nan)), _fmt(f.get("rv5min_ss", np.nan)), _fmt(f.get("rkernel", np.nan)), "", _fmt(w.get("500", np.nan)), _fmt(w.get("1000", np.nan)), _fmt(w.get("2000", np.nan)), ] lines.append(f"{m} & " + " & ".join(cells) + " \\\\") lines += ["\\bottomrule", "\\end{tabular}"] _write("sensitivity", lines) def table_transferability() -> None: """Cross-asset transferability: QLIKE h=1 for the class representatives.""" tab = pd.read_csv(OUT / "transferability.csv") tab = tab[tab["h"] == 1] wide = tab.pivot_table(index="model", columns="ticker", values="qlike") order = ["HAR", "LightGBM-X", "Pooled-LGBM", "Pooled-LGBM-LOO"] label = {"HAR": "HAR (own asset)", "LightGBM-X": "LightGBM (own asset)", "Pooled-LGBM": "Pooled LightGBM (asset in pool)", "Pooled-LGBM-LOO": "Pooled LightGBM (asset excluded)"} tickers = [t for t in ["NVDA", "GBPUSD", "ETH", "GC"] if t in wide.columns] lines = ["\\begin{tabular}{l" + "c" * len(tickers) + "}", "\\toprule", "Model & " + " & ".join(tickers) + " \\\\", "\\midrule"] for m in order: if m not in wide.index: continue cells = [_fmt(wide.loc[m, t]) for t in tickers] lines.append(f"{label[m]} & " + " & ".join(cells) + " \\\\") lines += ["\\bottomrule", "\\end{tabular}"] _write("transferability", lines) def table_timing() -> None: """Volatility timing: net Sharpe, cost sensitivity, utility gains.""" overall = pd.read_csv(OUT / "timing_overall.csv").set_index("model") util = pd.read_csv(OUT / "utility_summary.csv").set_index("model") byt = pd.read_csv(OUT / "timing_by_ticker.csv") sens = byt.groupby(["model", "tc_bps"], observed=True)["sharpe"].mean().unstack() wide = pd.DataFrame({ "sharpe": overall["sharpe"], "ann_ret": overall["ann_ret"] * 100, "turn": overall["turnover"], "s0": sens.get(0), "s10": sens.get(10), "s20": sens.get(20), "ug": util["mean"], }) lines = ["\\begin{tabular}{lccc ccc c}", "\\toprule", " & \\multicolumn{3}{c}{Net of 5 bps} &" " \\multicolumn{3}{c}{Sharpe at cost (bps)} & \\\\", "\\cmidrule(lr){2-4}\\cmidrule(lr){5-7}", "Model & Sharpe & \\makecell{Ann.\\ ret.\\\\(\\%)} & Turnover" " & 0 & 10 & 20 & \\makecell{Utility gain\\\\vs HAR (bps)} \\\\", "\\midrule"] smax = wide["sharpe"].max() for gname, models in GROUPS: avail = [m for m in models if m in wide.index] if not avail: continue lines.append(f"\\multicolumn{{8}}{{l}}{{\\itshape {gname}}}\\\\") for m in avail: r = wide.loc[m] ug = f"{r['ug']:.0f}" if np.isfinite(r["ug"]) else "---" sh = _fmt(r["sharpe"], 2, r["sharpe"] == smax) lines.append( f"\\quad {m} & {sh} & {r['ann_ret']:.1f} & {r['turn']:.3f}" f" & {_fmt(r['s0'], 2)} & {_fmt(r['s10'], 2)} & {_fmt(r['s20'], 2)}" f" & {ug} \\\\") lines += ["\\bottomrule", "\\end{tabular}"] _write("timing", lines) def table_var() -> None: """VaR backtest outcomes at 1% and 5%.""" tab = pd.read_csv(OUT / "var_summary.csv") parts = {} for lv in (0.01, 0.05): s = tab[tab["level"] == lv].set_index("model") parts[f"vr{lv}"] = s["mean_viol_rate"] * 100 parts[f"ku{lv}"] = s["pct_pass_kupiec"] * 100 parts[f"cc{lv}"] = s["pct_pass_cc"] * 100 wide = pd.DataFrame(parts) lines = ["\\begin{tabular}{lccc ccc}", "\\toprule", " & \\multicolumn{3}{c}{VaR 1\\%} & \\multicolumn{3}{c}{VaR 5\\%} \\\\", "\\cmidrule(lr){2-4}\\cmidrule(lr){5-7}", "Model & \\makecell{Viol.\\\\(\\%)} & \\makecell{Kupiec\\\\pass (\\%)}" " & \\makecell{CC\\\\pass (\\%)}" " & \\makecell{Viol.\\\\(\\%)} & \\makecell{Kupiec\\\\pass (\\%)}" " & \\makecell{CC\\\\pass (\\%)} \\\\", "\\midrule"] for gname, models in GROUPS: avail = [m for m in models if m in wide.index] if not avail: continue lines.append(f"\\multicolumn{{7}}{{l}}{{\\itshape {gname}}}\\\\") for m in avail: r = wide.loc[m] lines.append( f"\\quad {m} & {r['vr0.01']:.2f} & {r['ku0.01']:.0f} & {r['cc0.01']:.0f}" f" & {r['vr0.05']:.2f} & {r['ku0.05']:.0f} & {r['cc0.05']:.0f} \\\\") lines += ["\\bottomrule", "\\end{tabular}"] _write("var", lines) def table_by_ticker_appendix() -> None: """Appendix: QLIKE h=1 per ticker for headline models.""" tab = pd.read_csv(OUT / "losses_by_ticker.csv") tab = tab[tab["h"] == 1] models = ["HAR", "LogHAR", "RealGARCH", "LightGBM-X", "Pooled-LGBM", "LSTM", "Comb-InvMSE"] wide = tab.pivot_table(index=["cls", "ticker"], columns="model", values="qlike") avail = [m for m in models if m in wide.columns] lines = ["\\begin{tabular}{l" + "c" * len(avail) + "}", "\\toprule", "Ticker & " + " & ".join(avail) + " \\\\", "\\midrule"] for cls in ["equity", "fx", "crypto", "futures"]: if cls not in wide.index.get_level_values(0): continue lines.append(f"\\multicolumn{{{len(avail) + 1}}}{{l}}" f"{{\\itshape {CLS_LABEL[cls]}}}\\\\") sub = wide.loc[cls] for tk, r in sub.iterrows(): best = r[avail].min() cells = [_fmt(r[m], 3, np.isfinite(r[m]) and r[m] == best) for m in avail] lines.append(f"\\quad {tk} & " + " & ".join(cells) + " \\\\") lines += ["\\bottomrule", "\\end{tabular}"] _write("by_ticker_appendix", lines) def main() -> None: """Generate every LaTeX table from the reproduced CSVs.""" config.setup_logging() for fn in [table_summary_stats, table_losses_main, table_losses_class, table_dm_mcs, table_mz_encompassing, table_subperiods, table_sensitivity, table_transferability, table_timing, table_var, table_by_ticker_appendix]: try: fn() except Exception as exc: # noqa: BLE001 logger.error("table %s FAILED: %s", fn.__name__, exc) if __name__ == "__main__": main()