1"""P7/P8: aggregate saved experiment outputs into LaTeX tables and paper macros (results/tables, paper/tables).2Usage: evaluate.py --exps pilot,main [--ref naive_rw]"""3from __future__ import annotations45import argparse6import json7import re8import sys9from pathlib import Path1011import numpy as np12import pandas as pd1314ROOT = Path(__file__).resolve().parents[1]15sys.path.insert(0, str(ROOT / "src"))1617PRETTY = {"naive_rw": "Random walk (trailing RMS)", "seasonal_vol": "Seasonal volatility", "ewma": "EWMA-$t$", "har_rv": "HAR-RV",18 "garch_t": "GARCH(1,1)-$t$", "ridge": "Ridge", "elastic_net": "Elastic net", "lightgbm": "LightGBM", "mlp": "MLP (1m)",19 "tcn": "TCN (1m)", "mrpt_small": "MRPT-S (1m+5m+30m)", "mrpt_small_1m_only": "MRPT-S (1m only)",20 "mrpt_small_1m_5m": "MRPT-S (1m+5m)", "mrpt_base": "MRPT-B (1m+5m+30m)", "mrpt_base_1m_only": "MRPT-B (1m only)",21 "mrpt_small_cross": "MRPT-S cross-attention fusion", "mrpt_small_gaussian": "MRPT-S Gaussian head",22 "mrpt_small_noregime": "MRPT-S no regime", "mrpt_small_learnedregime": "MRPT-S learned regime",23 "mrpt_small_returns_only": "MRPT-S returns only", "mrpt_small_time_only": "MRPT-S time only",24 "mrpt_small_ohlcv": "MRPT-S OHLCV (no rel. volume)", "mrpt_small_shuffled": "MRPT-S shuffled minutes",25 "mrpt_small_ticker": "MRPT-S + ticker id", "mrpt_small_60m": "MRPT-S 60-min context", "mrpt_small_copula": "MRPT-S + copula paths",26 "mrpt_small_copula_indep": "MRPT-S copula model, independent sampling", "mrpt_spec": "MRPT-M (1m+5m+30m)"}272829def pretty(m: str) -> str:30 base = m.replace("_heldout", "")31 return PRETTY.get(base, base.replace("_", " ")) + (" [held-out]" if m.endswith("_heldout") else "")323334def tex_escape(s: str) -> str:35 return s.replace("_", "\\_").replace("%", "\\%")363738def leaderboard_table(lb: pd.DataFrame, horizons, metric="crps_scaled", label="", caption="") -> str:39 piv = lb.pivot(index="model", columns="H", values=metric)40 lo = lb.pivot(index="model", columns="H", values=f"{metric}_lo"); hi = lb.pivot(index="model", columns="H", values=f"{metric}_hi")41 order = [m for m in PRETTY if m in piv.index] + [m for m in piv.index if m not in PRETTY]42 lines = ["\\begin{table}[htbp]\\centering\\small", f"\\caption{{{caption}}}", f"\\label{{{label}}}",43 "\\begin{tabular}{l" + "c" * len(horizons) + "}", "\\toprule", "Model & " + " & ".join(f"$H={h}$" for h in horizons) + " \\\\", "\\midrule"]44 for m in order:45 cells = []46 for h in horizons:47 v = piv.loc[m, h]48 cells.append(f"{v:.4f}" + (f" {{\\scriptsize[{lo.loc[m, h]:.4f}, {hi.loc[m, h]:.4f}]}}" if not np.isnan(lo.loc[m, h]) else ""))49 lines.append(tex_escape(pretty(m)) + " & " + " & ".join(cells) + " \\\\")50 lines += ["\\bottomrule", "\\end{tabular}", "\\begin{minipage}{0.95\\textwidth}\\scriptsize Cluster-bootstrap (trading-day) 95\\% intervals in brackets.\\end{minipage}", "\\end{table}"]51 return "\n".join(lines)525354def compare_table(cmp: pd.DataFrame, horizons, label, caption, metric_name) -> str:55 lines = ["\\begin{table}[htbp]\\centering\\small", f"\\caption{{{caption}}}", f"\\label{{{label}}}",56 "\\begin{tabular}{l" + "c" * len(horizons) + "}", "\\toprule", "Model & " + " & ".join(f"$H={h}$" for h in horizons) + " \\\\", "\\midrule"]57 for m in [x for x in PRETTY if x in set(cmp.model)] + [x for x in cmp.model.unique() if x not in PRETTY]:58 cells = []59 for h in horizons:60 r = cmp[(cmp.model == m) & (cmp.H == h)]61 if r.empty:62 cells.append("--"); continue63 r = r.iloc[0]64 cells.append(f"{100 * r.rel_diff:+.2f}\\% ({r.dm_t:.1f})")65 lines.append(tex_escape(pretty(m)) + " & " + " & ".join(cells) + " \\\\")66 lines += ["\\bottomrule", "\\end{tabular}", f"\\begin{{minipage}}{{0.95\\textwidth}}\\scriptsize Relative change in mean {metric_name} versus the reference (negative = better); Diebold--Mariano $t$-statistic with HAC variance in parentheses.\\end{{minipage}}", "\\end{table}"]67 return "\n".join(lines)686970def main() -> None:71 ap = argparse.ArgumentParser(); ap.add_argument("--exps", default="pilot"); ap.add_argument("--ref", default="naive_rw")72 ap.add_argument("--split", default="test")73 a = ap.parse_args()74 tdir = ROOT / "paper" / "tables"; tdir.mkdir(parents=True, exist_ok=True)75 macros = []76 for exp in a.exps.split(","):77 d = ROOT / "results" / "metrics" / exp78 for suffix in ("", "_heldout"):79 f = d / f"leaderboard_{a.split}{suffix}.csv"80 if not f.exists():81 continue82 lb = pd.read_csv(f); H = sorted(lb.H.unique())83 for metric, nm in (("crps_scaled", "scaled CRPS"), ("wis", "WIS"), ("qlike", "QLIKE"), ("logscore", "log score")):84 if metric in lb.columns and lb[metric].notna().any():85 (tdir / f"{exp}_{metric}{suffix}.tex").write_text(leaderboard_table(lb, H, metric, f"tab:{exp}_{metric}{suffix}",86 f"{exp.capitalize()}{' (held-out instruments)' if suffix else ''}: mean {nm} of the cumulative-return forecast by horizon (minutes)"))87 cov = lb.pivot(index="model", columns="H", values="cov_95")88 lines = ["\\begin{table}[htbp]\\centering\\small", f"\\caption{{{exp.capitalize()}{' (held-out)' if suffix else ''}: empirical 50/80/95\\% coverage and PIT KS statistic}}", f"\\label{{tab:{exp}_calib{suffix}}}",89 "\\begin{tabular}{l" + "c" * (4 * len(H)) + "}", "\\toprule", "Model & " + " & ".join(f"\\multicolumn{{4}}{{c}}{{$H={h}$}}" for h in H) + " \\\\",90 " & " + " & ".join(["50 & 80 & 95 & KS"] * len(H)) + " \\\\", "\\midrule"]91 for m in [x for x in PRETTY if x in cov.index] + [x for x in cov.index if x not in PRETTY]:92 cells = []93 for h in H:94 r = lb[(lb.model == m) & (lb.H == h)].iloc[0]95 cells.append(f"{r.cov_50:.2f} & {r.cov_80:.2f} & {r.cov_95:.2f} & {r.pit_ks:.3f}")96 lines.append(tex_escape(pretty(m)) + " & " + " & ".join(cells) + " \\\\")97 lines += ["\\bottomrule", "\\end{tabular}", "\\end{table}"]98 (tdir / f"{exp}_calibration{suffix}.tex").write_text("\n".join(lines))99 for metric in ("crps", "wis", "qlike"):100 fc = d / f"compare_{metric}_vs_{a.ref}_{a.split}{suffix}.csv"101 if fc.exists():102 cmp = pd.read_csv(fc)103 (tdir / f"{exp}_compare_{metric}{suffix}.tex").write_text(compare_table(cmp, H, f"tab:{exp}_cmp_{metric}{suffix}",104 f"{exp.capitalize()}{' (held-out)' if suffix else ''}: {metric.upper()} relative to {pretty(a.ref)}", metric.upper()))105 for r in cmp.itertuples():106 key = "".join(c for c in f"rel{exp}{suffix}{r.model}{metric}H{r.H}" if c.isalnum())107 macros.append(f"\\newcommand{{\\{key}}}{{{100 * r.rel_diff:+.2f}}}")108 macros.append(f"\\newcommand{{\\dm{key}}}{{{r.dm_t:.1f}}}")109 for r in lb.itertuples():110 key = "".join(c for c in f"{exp}{suffix}{r.model}H{r.H}" if c.isalnum())111 macros.append(f"\\newcommand{{\\crps{key}}}{{{r.crps_scaled:.4f}}}")112 macros.append(f"\\newcommand{{\\covfifty{key}}}{{{100 * r.cov_50:.0f}}}")113 macros.append(f"\\newcommand{{\\covninetyfive{key}}}{{{100 * r.cov_95:.1f}}}")114 # pairwise contrasts, if computed115 fp = d / f"pairwise_{a.split}{suffix}.csv"116 if fp.exists():117 pw = pd.read_csv(fp)118 for r in pw[pw.metric == "crps"].itertuples():119 key = "".join(c for c in f"pw{exp}{suffix}{r.model}vs{r.reference}H{r.H}" if c.isalnum())120 macros.append(f"\\newcommand{{\\{key}}}{{{100 * r.rel_diff:+.2f}}}")121 macros.append(f"\\newcommand{{\\t{key}}}{{{r.dm_t:.1f}}}")122 macros.append(f"\\newcommand{{\\p{key}}}{{{r.p_holm:.3g}}}")123 words = {"0": "zero", "1": "one", "2": "two", "3": "three", "4": "four", "5": "five", "6": "six", "7": "seven", "8": "eight", "9": "nine"}124125 def wordify(line: str) -> str:126 m = re.match(r"\\newcommand\{\\([A-Za-z0-9]+)\}", line)127 if not m:128 return line129 name = m.group(1)130 new = re.sub(r"\d", lambda x: words[x.group(0)], name)131 return line.replace("\\" + name + "}", "\\" + new + "}", 1)132 (ROOT / "paper" / "results_macros.tex").write_text("\n".join(wordify(l) for l in macros) + "\n")133 print(f"wrote {len(list(tdir.glob('*.tex')))} tables, {len(macros)} macros")134135136if __name__ == "__main__":137 main()138