"""P7/P8: aggregate saved experiment outputs into LaTeX tables and paper macros (results/tables, paper/tables). Usage: evaluate.py --exps pilot,main [--ref naive_rw]""" from __future__ import annotations import argparse import json import re import sys from pathlib import Path import numpy as np import pandas as pd ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "src")) PRETTY = {"naive_rw": "Random walk (trailing RMS)", "seasonal_vol": "Seasonal volatility", "ewma": "EWMA-$t$", "har_rv": "HAR-RV", "garch_t": "GARCH(1,1)-$t$", "ridge": "Ridge", "elastic_net": "Elastic net", "lightgbm": "LightGBM", "mlp": "MLP (1m)", "tcn": "TCN (1m)", "mrpt_small": "MRPT-S (1m+5m+30m)", "mrpt_small_1m_only": "MRPT-S (1m only)", "mrpt_small_1m_5m": "MRPT-S (1m+5m)", "mrpt_base": "MRPT-B (1m+5m+30m)", "mrpt_base_1m_only": "MRPT-B (1m only)", "mrpt_small_cross": "MRPT-S cross-attention fusion", "mrpt_small_gaussian": "MRPT-S Gaussian head", "mrpt_small_noregime": "MRPT-S no regime", "mrpt_small_learnedregime": "MRPT-S learned regime", "mrpt_small_returns_only": "MRPT-S returns only", "mrpt_small_time_only": "MRPT-S time only", "mrpt_small_ohlcv": "MRPT-S OHLCV (no rel. volume)", "mrpt_small_shuffled": "MRPT-S shuffled minutes", "mrpt_small_ticker": "MRPT-S + ticker id", "mrpt_small_60m": "MRPT-S 60-min context", "mrpt_small_copula": "MRPT-S + copula paths", "mrpt_spec": "MRPT-M (1m+5m+30m)", "mrpt_small_b4k": "MRPT-S reference (4k steps)", "mrpt_small_1m_only_b4k": "MRPT-S 1m only (4k)", "mrpt_small_seed2": "MRPT-S reference, seed 2", "mrpt_small_seed3": "MRPT-S reference, seed 3", "mrpt_small_copula": "MRPT-S + copula (joint sampling)", "mrpt_small_copula_indep": "MRPT-S + copula, independent sampling",} def pretty(m: str) -> str: base = m.replace("_heldout", "") return PRETTY.get(base, base.replace("_", " ")) + (" [held-out]" if m.endswith("_heldout") else "") def tex_escape(s: str) -> str: return s.replace("_", "\\_").replace("%", "\\%") def leaderboard_table(lb: pd.DataFrame, horizons, metric="crps_scaled", label="", caption="") -> str: piv = lb.pivot(index="model", columns="H", values=metric) lo = lb.pivot(index="model", columns="H", values=f"{metric}_lo"); hi = lb.pivot(index="model", columns="H", values=f"{metric}_hi") order = [m for m in PRETTY if m in piv.index] + [m for m in piv.index if m not in PRETTY] lines = ["\\begin{table}[htbp]\\centering\\small", f"\\caption{{{caption}}}", f"\\label{{{label}}}", "\\begin{adjustbox}{max width=\\textwidth}", "\\begin{tabular}{l" + "c" * len(horizons) + "}", "\\toprule", "Model & " + " & ".join(f"$H={h}$" for h in horizons) + " \\\\", "\\midrule"] for m in order: cells = [] for h in horizons: v = piv.loc[m, h] cells.append(f"{v:.4f}" + (f" {{\\scriptsize[{lo.loc[m, h]:.4f}, {hi.loc[m, h]:.4f}]}}" if not np.isnan(lo.loc[m, h]) else "")) lines.append(tex_escape(pretty(m)) + " & " + " & ".join(cells) + " \\\\") lines += ["\\bottomrule", "\\end{tabular}", "\\end{adjustbox}", "\\begin{minipage}{0.95\\textwidth}\\scriptsize Cluster-bootstrap (trading-day) 95\\% intervals in brackets.\\end{minipage}", "\\end{table}"] return "\n".join(lines) def compare_table(cmp: pd.DataFrame, horizons, label, caption, metric_name) -> str: lines = ["\\begin{table}[htbp]\\centering\\small", f"\\caption{{{caption}}}", f"\\label{{{label}}}", "\\begin{adjustbox}{max width=\\textwidth}", "\\begin{tabular}{l" + "c" * len(horizons) + "}", "\\toprule", "Model & " + " & ".join(f"$H={h}$" for h in horizons) + " \\\\", "\\midrule"] for m in [x for x in PRETTY if x in set(cmp.model)] + [x for x in cmp.model.unique() if x not in PRETTY]: cells = [] for h in horizons: r = cmp[(cmp.model == m) & (cmp.H == h)] if r.empty: cells.append("--"); continue r = r.iloc[0] cells.append(f"{100 * r.rel_diff:+.2f}\\% ({r.dm_t:.1f})") lines.append(tex_escape(pretty(m)) + " & " + " & ".join(cells) + " \\\\") lines += ["\\bottomrule", "\\end{tabular}", "\\end{adjustbox}", f"\\begin{{minipage}}{{0.95\\textwidth}}\\scriptsize Relative change in mean {metric_name} versus the reference (negative = better); Diebold--Mariano $t$-statistic with HAC variance in parentheses.\\end{{minipage}}", "\\end{table}"] return "\n".join(lines) def main() -> None: ap = argparse.ArgumentParser(); ap.add_argument("--exps", default="pilot"); ap.add_argument("--ref", default="naive_rw") ap.add_argument("--split", default="test") a = ap.parse_args() tdir = ROOT / "paper" / "tables"; tdir.mkdir(parents=True, exist_ok=True) macros = [] for item in a.exps.split(","): exp, split = (item.split(":") + [a.split])[:2] d = ROOT / "results" / "metrics" / exp for suffix in ("", "_heldout"): f = d / f"leaderboard_{split}{suffix}.csv" if not f.exists(): continue lb = pd.read_csv(f); H = sorted(lb.H.unique()) for metric, nm in (("crps_scaled", "scaled CRPS"), ("wis", "WIS"), ("qlike", "QLIKE"), ("logscore", "log score")): if metric in lb.columns and lb[metric].notna().any(): (tdir / f"{exp}_{metric}{suffix}.tex").write_text(leaderboard_table(lb, H, metric, f"tab:{exp}_{metric}{suffix}", f"{exp.capitalize()}{' (held-out instruments)' if suffix else ''}: mean {nm} of the cumulative-return forecast by horizon (minutes)")) cov = lb.pivot(index="model", columns="H", values="cov_95") lines = ["\\begin{table}[htbp]\\centering\\small", f"\\caption{{{exp.capitalize()}{' (held-out)' if suffix else ''}: empirical 50/80/95\\% coverage and PIT KS statistic}}", f"\\label{{tab:{exp}_calib{suffix}}}", "\\begin{adjustbox}{max width=\\textwidth}", "\\begin{tabular}{l" + "c" * (4 * len(H)) + "}", "\\toprule", "Model & " + " & ".join(f"\\multicolumn{{4}}{{c}}{{$H={h}$}}" for h in H) + " \\\\", " & " + " & ".join(["50 & 80 & 95 & KS"] * len(H)) + " \\\\", "\\midrule"] for m in [x for x in PRETTY if x in cov.index] + [x for x in cov.index if x not in PRETTY]: cells = [] for h in H: r = lb[(lb.model == m) & (lb.H == h)].iloc[0] cells.append(f"{r.cov_50:.2f} & {r.cov_80:.2f} & {r.cov_95:.2f} & {r.pit_ks:.3f}") lines.append(tex_escape(pretty(m)) + " & " + " & ".join(cells) + " \\\\") lines += ["\\bottomrule", "\\end{tabular}", "\\end{adjustbox}", "\\end{table}"] (tdir / f"{exp}_calibration{suffix}.tex").write_text("\n".join(lines)) for metric in ("crps", "wis", "qlike"): fc = d / f"compare_{metric}_vs_{a.ref}_{split}{suffix}.csv" if fc.exists(): cmp = pd.read_csv(fc) (tdir / f"{exp}_compare_{metric}{suffix}.tex").write_text(compare_table(cmp, H, f"tab:{exp}_cmp_{metric}{suffix}", f"{exp.capitalize()}{' (held-out)' if suffix else ''}: {metric.upper()} relative to {pretty(a.ref)}", metric.upper())) for r in cmp.itertuples(): key = "".join(c for c in f"rel{exp}{suffix}{r.model}{metric}H{r.H}" if c.isalnum()) macros.append(f"\\newcommand{{\\{key}}}{{{100 * r.rel_diff:+.2f}}}") macros.append(f"\\newcommand{{\\dm{key}}}{{{r.dm_t:.1f}}}") for r in lb.itertuples(): key = "".join(c for c in f"{exp}{suffix}{r.model}H{r.H}" if c.isalnum()) macros.append(f"\\newcommand{{\\crps{key}}}{{{r.crps_scaled:.4f}}}") macros.append(f"\\newcommand{{\\covfifty{key}}}{{{100 * r.cov_50:.0f}}}") macros.append(f"\\newcommand{{\\covninetyfive{key}}}{{{100 * r.cov_95:.1f}}}") # pairwise contrasts, if computed fp = d / f"pairwise_{split}{suffix}.csv" if fp.exists(): pw = pd.read_csv(fp) for r in pw[pw.metric == "crps"].itertuples(): key = "".join(c for c in f"pw{exp}{suffix}{r.model}vs{r.reference}H{r.H}" if c.isalnum()) macros.append(f"\\newcommand{{\\{key}}}{{{100 * r.rel_diff:+.2f}}}") macros.append(f"\\newcommand{{\\t{key}}}{{{r.dm_t:.1f}}}") macros.append(f"\\newcommand{{\\p{key}}}{{{r.p_holm:.3g}}}") words = {"0": "zero", "1": "one", "2": "two", "3": "three", "4": "four", "5": "five", "6": "six", "7": "seven", "8": "eight", "9": "nine"} def wordify(line: str) -> str: m = re.match(r"\\newcommand\{\\([A-Za-z0-9]+)\}", line) if not m: return line name = m.group(1) new = re.sub(r"\d", lambda x: words[x.group(0)], name) return line.replace("\\" + name + "}", "\\" + new + "}", 1) (ROOT / "paper" / "results_macros.tex").write_text("\n".join(wordify(l) for l in macros) + "\n") print(f"wrote {len(list(tdir.glob('*.tex')))} tables, {len(macros)} macros") if __name__ == "__main__": main()