1"""P7/P8: aggregate saved experiment outputs into LaTeX tables and paper macros (results/tables, paper/tables).2Usage: evaluate.py --exps pilot,main [--ref naive_rw]"""3from __future__ import annotations45import argparse6import json7import re8import sys9from pathlib import Path1011import numpy as np12import pandas as pd1314ROOT = Path(__file__).resolve().parents[1]15sys.path.insert(0, str(ROOT / "src"))1617PRETTY = {"naive_rw": "Random walk (trailing RMS)", "seasonal_vol": "Seasonal volatility", "ewma": "EWMA-$t$", "har_rv": "HAR-RV",18 "garch_t": "GARCH(1,1)-$t$", "ridge": "Ridge", "elastic_net": "Elastic net", "lightgbm": "LightGBM", "mlp": "MLP (1m)",19 "tcn": "TCN (1m)", "mrpt_small": "MRPT-S (1m+5m+30m)", "mrpt_small_1m_only": "MRPT-S (1m only)",20 "mrpt_small_1m_5m": "MRPT-S (1m+5m)", "mrpt_base": "MRPT-B (1m+5m+30m)", "mrpt_base_1m_only": "MRPT-B (1m only)",21 "mrpt_small_cross": "MRPT-S cross-attention fusion", "mrpt_small_gaussian": "MRPT-S Gaussian head",22 "mrpt_small_noregime": "MRPT-S no regime", "mrpt_small_learnedregime": "MRPT-S learned regime",23 "mrpt_small_returns_only": "MRPT-S returns only", "mrpt_small_time_only": "MRPT-S time only",24 "mrpt_small_ohlcv": "MRPT-S OHLCV (no rel. volume)", "mrpt_small_shuffled": "MRPT-S shuffled minutes",25 "mrpt_small_ticker": "MRPT-S + ticker id", "mrpt_small_60m": "MRPT-S 60-min context", "mrpt_small_copula": "MRPT-S + copula paths",26 "mrpt_spec": "MRPT-M (1m+5m+30m)",27 "mrpt_small_b4k": "MRPT-S reference (4k steps)", "mrpt_small_1m_only_b4k": "MRPT-S 1m only (4k)", "mrpt_small_seed2": "MRPT-S reference, seed 2",28 "mrpt_small_seed3": "MRPT-S reference, seed 3", "mrpt_small_copula": "MRPT-S + copula (joint sampling)", "mrpt_small_copula_indep": "MRPT-S + copula, independent sampling",}293031def pretty(m: str) -> str:32 base = m.replace("_heldout", "")33 return PRETTY.get(base, base.replace("_", " ")) + (" [held-out]" if m.endswith("_heldout") else "")343536def tex_escape(s: str) -> str:37 return s.replace("_", "\\_").replace("%", "\\%")383940def leaderboard_table(lb: pd.DataFrame, horizons, metric="crps_scaled", label="", caption="") -> str:41 piv = lb.pivot(index="model", columns="H", values=metric)42 lo = lb.pivot(index="model", columns="H", values=f"{metric}_lo"); hi = lb.pivot(index="model", columns="H", values=f"{metric}_hi")43 order = [m for m in PRETTY if m in piv.index] + [m for m in piv.index if m not in PRETTY]44 lines = ["\\begin{table}[htbp]\\centering\\small", f"\\caption{{{caption}}}", f"\\label{{{label}}}", "\\begin{adjustbox}{max width=\\textwidth}",45 "\\begin{tabular}{l" + "c" * len(horizons) + "}", "\\toprule", "Model & " + " & ".join(f"$H={h}$" for h in horizons) + " \\\\", "\\midrule"]46 for m in order:47 cells = []48 for h in horizons:49 v = piv.loc[m, h]50 cells.append(f"{v:.4f}" + (f" {{\\scriptsize[{lo.loc[m, h]:.4f}, {hi.loc[m, h]:.4f}]}}" if not np.isnan(lo.loc[m, h]) else ""))51 lines.append(tex_escape(pretty(m)) + " & " + " & ".join(cells) + " \\\\")52 lines += ["\\bottomrule", "\\end{tabular}", "\\end{adjustbox}", "\\begin{minipage}{0.95\\textwidth}\\scriptsize Cluster-bootstrap (trading-day) 95\\% intervals in brackets.\\end{minipage}", "\\end{table}"]53 return "\n".join(lines)545556def compare_table(cmp: pd.DataFrame, horizons, label, caption, metric_name) -> str:57 lines = ["\\begin{table}[htbp]\\centering\\small", f"\\caption{{{caption}}}", f"\\label{{{label}}}", "\\begin{adjustbox}{max width=\\textwidth}",58 "\\begin{tabular}{l" + "c" * len(horizons) + "}", "\\toprule", "Model & " + " & ".join(f"$H={h}$" for h in horizons) + " \\\\", "\\midrule"]59 for m in [x for x in PRETTY if x in set(cmp.model)] + [x for x in cmp.model.unique() if x not in PRETTY]:60 cells = []61 for h in horizons:62 r = cmp[(cmp.model == m) & (cmp.H == h)]63 if r.empty:64 cells.append("--"); continue65 r = r.iloc[0]66 cells.append(f"{100 * r.rel_diff:+.2f}\\% ({r.dm_t:.1f})")67 lines.append(tex_escape(pretty(m)) + " & " + " & ".join(cells) + " \\\\")68 lines += ["\\bottomrule", "\\end{tabular}", "\\end{adjustbox}", f"\\begin{{minipage}}{{0.95\\textwidth}}\\scriptsize Relative change in mean {metric_name} versus the reference (negative = better); Diebold--Mariano $t$-statistic with HAC variance in parentheses.\\end{{minipage}}", "\\end{table}"]69 return "\n".join(lines)707172def main() -> None:73 ap = argparse.ArgumentParser(); ap.add_argument("--exps", default="pilot"); ap.add_argument("--ref", default="naive_rw")74 ap.add_argument("--split", default="test")75 a = ap.parse_args()76 tdir = ROOT / "paper" / "tables"; tdir.mkdir(parents=True, exist_ok=True)77 macros = []78 for item in a.exps.split(","):79 exp, split = (item.split(":") + [a.split])[:2]80 d = ROOT / "results" / "metrics" / exp81 for suffix in ("", "_heldout"):82 f = d / f"leaderboard_{split}{suffix}.csv"83 if not f.exists():84 continue85 lb = pd.read_csv(f); H = sorted(lb.H.unique())86 for metric, nm in (("crps_scaled", "scaled CRPS"), ("wis", "WIS"), ("qlike", "QLIKE"), ("logscore", "log score")):87 if metric in lb.columns and lb[metric].notna().any():88 (tdir / f"{exp}_{metric}{suffix}.tex").write_text(leaderboard_table(lb, H, metric, f"tab:{exp}_{metric}{suffix}",89 f"{exp.capitalize()}{' (held-out instruments)' if suffix else ''}: mean {nm} of the cumulative-return forecast by horizon (minutes)"))90 cov = lb.pivot(index="model", columns="H", values="cov_95")91 lines = ["\\begin{table}[htbp]\\centering\\small", f"\\caption{{{exp.capitalize()}{' (held-out)' if suffix else ''}: empirical 50/80/95\\% coverage and PIT KS statistic}}", f"\\label{{tab:{exp}_calib{suffix}}}",92 "\\begin{adjustbox}{max width=\\textwidth}", "\\begin{tabular}{l" + "c" * (4 * len(H)) + "}", "\\toprule", "Model & " + " & ".join(f"\\multicolumn{{4}}{{c}}{{$H={h}$}}" for h in H) + " \\\\",93 " & " + " & ".join(["50 & 80 & 95 & KS"] * len(H)) + " \\\\", "\\midrule"]94 for m in [x for x in PRETTY if x in cov.index] + [x for x in cov.index if x not in PRETTY]:95 cells = []96 for h in H:97 r = lb[(lb.model == m) & (lb.H == h)].iloc[0]98 cells.append(f"{r.cov_50:.2f} & {r.cov_80:.2f} & {r.cov_95:.2f} & {r.pit_ks:.3f}")99 lines.append(tex_escape(pretty(m)) + " & " + " & ".join(cells) + " \\\\")100 lines += ["\\bottomrule", "\\end{tabular}", "\\end{adjustbox}", "\\end{table}"]101 (tdir / f"{exp}_calibration{suffix}.tex").write_text("\n".join(lines))102 for metric in ("crps", "wis", "qlike"):103 fc = d / f"compare_{metric}_vs_{a.ref}_{split}{suffix}.csv"104 if fc.exists():105 cmp = pd.read_csv(fc)106 (tdir / f"{exp}_compare_{metric}{suffix}.tex").write_text(compare_table(cmp, H, f"tab:{exp}_cmp_{metric}{suffix}",107 f"{exp.capitalize()}{' (held-out)' if suffix else ''}: {metric.upper()} relative to {pretty(a.ref)}", metric.upper()))108 for r in cmp.itertuples():109 key = "".join(c for c in f"rel{exp}{suffix}{r.model}{metric}H{r.H}" if c.isalnum())110 macros.append(f"\\newcommand{{\\{key}}}{{{100 * r.rel_diff:+.2f}}}")111 macros.append(f"\\newcommand{{\\dm{key}}}{{{r.dm_t:.1f}}}")112 for r in lb.itertuples():113 key = "".join(c for c in f"{exp}{suffix}{r.model}H{r.H}" if c.isalnum())114 macros.append(f"\\newcommand{{\\crps{key}}}{{{r.crps_scaled:.4f}}}")115 macros.append(f"\\newcommand{{\\covfifty{key}}}{{{100 * r.cov_50:.0f}}}")116 macros.append(f"\\newcommand{{\\covninetyfive{key}}}{{{100 * r.cov_95:.1f}}}")117 # pairwise contrasts, if computed118 fp = d / f"pairwise_{split}{suffix}.csv"119 if fp.exists():120 pw = pd.read_csv(fp)121 for r in pw[pw.metric == "crps"].itertuples():122 key = "".join(c for c in f"pw{exp}{suffix}{r.model}vs{r.reference}H{r.H}" if c.isalnum())123 macros.append(f"\\newcommand{{\\{key}}}{{{100 * r.rel_diff:+.2f}}}")124 macros.append(f"\\newcommand{{\\t{key}}}{{{r.dm_t:.1f}}}")125 macros.append(f"\\newcommand{{\\p{key}}}{{{r.p_holm:.3g}}}")126 words = {"0": "zero", "1": "one", "2": "two", "3": "three", "4": "four", "5": "five", "6": "six", "7": "seven", "8": "eight", "9": "nine"}127128 def wordify(line: str) -> str:129 m = re.match(r"\\newcommand\{\\([A-Za-z0-9]+)\}", line)130 if not m:131 return line132 name = m.group(1)133 new = re.sub(r"\d", lambda x: words[x.group(0)], name)134 return line.replace("\\" + name + "}", "\\" + new + "}", 1)135 (ROOT / "paper" / "results_macros.tex").write_text("\n".join(wordify(l) for l in macros) + "\n")136 print(f"wrote {len(list(tdir.glob('*.tex')))} tables, {len(macros)} macros")137138139if __name__ == "__main__":140 main()141