1"""P7/P8: aggregate saved experiment outputs into LaTeX tables and paper macros (results/tables, paper/tables).2Usage: evaluate.py --exps pilot,main [--ref naive_rw]"""3from __future__ import annotations45import argparse6import json7import sys8from pathlib import Path910import numpy as np11import pandas as pd1213ROOT = Path(__file__).resolve().parents[1]14sys.path.insert(0, str(ROOT / "src"))1516PRETTY = {"naive_rw": "Random walk (trailing RMS)", "seasonal_vol": "Seasonal volatility", "ewma": "EWMA-$t$", "har_rv": "HAR-RV",17 "garch_t": "GARCH(1,1)-$t$", "ridge": "Ridge", "elastic_net": "Elastic net", "lightgbm": "LightGBM", "mlp": "MLP (1m)",18 "tcn": "TCN (1m)", "mrpt_small": "MRPT-S (1m+5m+30m)", "mrpt_small_1m_only": "MRPT-S (1m only)",19 "mrpt_small_1m_5m": "MRPT-S (1m+5m)", "mrpt_base": "MRPT-B (1m+5m+30m)", "mrpt_base_1m_only": "MRPT-B (1m only)",20 "mrpt_small_cross": "MRPT-S cross-attention fusion", "mrpt_small_gaussian": "MRPT-S Gaussian head",21 "mrpt_small_noregime": "MRPT-S no regime", "mrpt_small_learnedregime": "MRPT-S learned regime",22 "mrpt_small_returns_only": "MRPT-S returns only", "mrpt_small_time_only": "MRPT-S time only",23 "mrpt_small_ohlcv": "MRPT-S OHLCV (no rel. volume)", "mrpt_small_shuffled": "MRPT-S shuffled minutes",24 "mrpt_small_ticker": "MRPT-S + ticker id", "mrpt_small_60m": "MRPT-S 60-min context", "mrpt_small_copula": "MRPT-S + copula paths",25 "mrpt_small_copula_indep": "MRPT-S copula model, independent sampling", "mrpt_spec": "MRPT-M (1m+5m+30m)"}262728def pretty(m: str) -> str:29 base = m.replace("_heldout", "")30 return PRETTY.get(base, base.replace("_", " ")) + (" [held-out]" if m.endswith("_heldout") else "")313233def tex_escape(s: str) -> str:34 return s.replace("_", "\\_").replace("%", "\\%")353637def leaderboard_table(lb: pd.DataFrame, horizons, metric="crps_scaled", label="", caption="") -> str:38 piv = lb.pivot(index="model", columns="H", values=metric)39 lo = lb.pivot(index="model", columns="H", values=f"{metric}_lo"); hi = lb.pivot(index="model", columns="H", values=f"{metric}_hi")40 order = [m for m in PRETTY if m in piv.index] + [m for m in piv.index if m not in PRETTY]41 lines = ["\\begin{table}[htbp]\\centering\\small", f"\\caption{{{caption}}}", f"\\label{{{label}}}",42 "\\begin{tabular}{l" + "c" * len(horizons) + "}", "\\toprule", "Model & " + " & ".join(f"$H={h}$" for h in horizons) + " \\\\", "\\midrule"]43 for m in order:44 cells = []45 for h in horizons:46 v = piv.loc[m, h]47 cells.append(f"{v:.4f}" + (f" {{\\scriptsize[{lo.loc[m, h]:.4f}, {hi.loc[m, h]:.4f}]}}" if not np.isnan(lo.loc[m, h]) else ""))48 lines.append(tex_escape(pretty(m)) + " & " + " & ".join(cells) + " \\\\")49 lines += ["\\bottomrule", "\\end{tabular}", "\\begin{minipage}{0.95\\textwidth}\\scriptsize Cluster-bootstrap (trading-day) 95\\% intervals in brackets.\\end{minipage}", "\\end{table}"]50 return "\n".join(lines)515253def compare_table(cmp: pd.DataFrame, horizons, label, caption, metric_name) -> str:54 lines = ["\\begin{table}[htbp]\\centering\\small", f"\\caption{{{caption}}}", f"\\label{{{label}}}",55 "\\begin{tabular}{l" + "c" * len(horizons) + "}", "\\toprule", "Model & " + " & ".join(f"$H={h}$" for h in horizons) + " \\\\", "\\midrule"]56 for m in [x for x in PRETTY if x in set(cmp.model)] + [x for x in cmp.model.unique() if x not in PRETTY]:57 cells = []58 for h in horizons:59 r = cmp[(cmp.model == m) & (cmp.H == h)]60 if r.empty:61 cells.append("--"); continue62 r = r.iloc[0]63 cells.append(f"{100 * r.rel_diff:+.2f}\\% ({r.dm_t:.1f})")64 lines.append(tex_escape(pretty(m)) + " & " + " & ".join(cells) + " \\\\")65 lines += ["\\bottomrule", "\\end{tabular}", f"\\begin{{minipage}}{{0.95\\textwidth}}\\scriptsize Relative change in mean {metric_name} versus the reference (negative = better); Diebold--Mariano $t$-statistic with HAC variance in parentheses.\\end{{minipage}}", "\\end{table}"]66 return "\n".join(lines)676869def main() -> None:70 ap = argparse.ArgumentParser(); ap.add_argument("--exps", default="pilot"); ap.add_argument("--ref", default="naive_rw")71 ap.add_argument("--split", default="test")72 a = ap.parse_args()73 tdir = ROOT / "paper" / "tables"; tdir.mkdir(parents=True, exist_ok=True)74 macros = []75 for exp in a.exps.split(","):76 d = ROOT / "results" / "metrics" / exp77 for suffix in ("", "_heldout"):78 f = d / f"leaderboard_{a.split}{suffix}.csv"79 if not f.exists():80 continue81 lb = pd.read_csv(f); H = sorted(lb.H.unique())82 for metric, nm in (("crps_scaled", "scaled CRPS"), ("wis", "WIS"), ("qlike", "QLIKE"), ("logscore", "log score")):83 if metric in lb.columns and lb[metric].notna().any():84 (tdir / f"{exp}_{metric}{suffix}.tex").write_text(leaderboard_table(lb, H, metric, f"tab:{exp}_{metric}{suffix}",85 f"{exp.capitalize()}{' (held-out instruments)' if suffix else ''}: mean {nm} of the cumulative-return forecast by horizon (minutes)"))86 cov = lb.pivot(index="model", columns="H", values="cov_95")87 lines = ["\\begin{table}[htbp]\\centering\\small", f"\\caption{{{exp.capitalize()}{' (held-out)' if suffix else ''}: empirical 50/80/95\\% coverage and PIT KS statistic}}", f"\\label{{tab:{exp}_calib{suffix}}}",88 "\\begin{tabular}{l" + "c" * (4 * len(H)) + "}", "\\toprule", "Model & " + " & ".join(f"\\multicolumn{{4}}{{c}}{{$H={h}$}}" for h in H) + " \\\\",89 " & " + " & ".join(["50 & 80 & 95 & KS"] * len(H)) + " \\\\", "\\midrule"]90 for m in [x for x in PRETTY if x in cov.index] + [x for x in cov.index if x not in PRETTY]:91 cells = []92 for h in H:93 r = lb[(lb.model == m) & (lb.H == h)].iloc[0]94 cells.append(f"{r.cov_50:.2f} & {r.cov_80:.2f} & {r.cov_95:.2f} & {r.pit_ks:.3f}")95 lines.append(tex_escape(pretty(m)) + " & " + " & ".join(cells) + " \\\\")96 lines += ["\\bottomrule", "\\end{tabular}", "\\end{table}"]97 (tdir / f"{exp}_calibration{suffix}.tex").write_text("\n".join(lines))98 for metric in ("crps", "wis", "qlike"):99 fc = d / f"compare_{metric}_vs_{a.ref}_{a.split}{suffix}.csv"100 if fc.exists():101 cmp = pd.read_csv(fc)102 (tdir / f"{exp}_compare_{metric}{suffix}.tex").write_text(compare_table(cmp, H, f"tab:{exp}_cmp_{metric}{suffix}",103 f"{exp.capitalize()}{' (held-out)' if suffix else ''}: {metric.upper()} relative to {pretty(a.ref)}", metric.upper()))104 for r in cmp.itertuples():105 key = f"{exp}{suffix}{r.model}{metric}H{r.H}".replace("_", "")106 macros.append(f"\\newcommand{{\\{''.join(c for c in key if c.isalnum())}}}{{{100 * r.rel_diff:+.2f}}}")107 for r in lb.itertuples():108 key = "".join(c for c in f"{exp}{suffix}{r.model}crpsH{r.H}" if c.isalnum())109 macros.append(f"\\newcommand{{\\{key}}}{{{r.crps_scaled:.4f}}}")110 (ROOT / "paper" / "results_macros.tex").write_text("\n".join(macros) + "\n")111 print(f"wrote {len(list(tdir.glob('*.tex')))} tables, {len(macros)} macros")112113114if __name__ == "__main__":115 main()116