Commit
L1/L4 post-pull run 5 (P4 learning curve 12k/24k/36k + budget verdict, ablations_h3 complete): budget tables, learning-curve and H3-control records in inline_numbers.json, compute ledger rows (WP41-EDIT-08, WP41-AI_C-02, WP41-THEO-09, WP41-THEO-05, WP41-AI_C-04)
budget_verdict.json (P4): criterion (i) fails at H = 5 and 15 (the 36k H1 gap +0.05 % / -0.01 % falls outside the five-seed band of the 12k contrast) and holds at H = 30 and 60 (-0.09 / -0.25 %); criterion (ii) holds (MRPT-S vs HAR-RV keeps its sign at 30/60 min, +0.49 / +0.62 % at 36k vs +0.53 / +0.65 % at 12k) -> the budget condition stays in the abstract. Validation-NLL slope per 1,000 steps at 36k: -0.00001 (MRPT-S, converged) vs +0.00006 (1m-only). latexmk 42 p., 0 errors, 0 overfull. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
modified
paper/main.pdf
+0 −0
Binary file not shown.
modified
paper/results_macros.tex
+80 −0
@@ -11674,4 +11674,84 @@ | ||
| 11674 | 11674 | \newcommand{\pwunseedsmrptsmallsfivevsmrptsmallonemonlysfiveHsixzero}{-0.14} |
| 11675 | 11675 | \newcommand{\tpwunseedsmrptsmallsfivevsmrptsmallonemonlysfiveHsixzero}{-4.8} |
| 11676 | 11676 | \newcommand{\ppwunseedsmrptsmallsfivevsmrptsmallonemonlysfiveHsixzero}{1.58e-06} |
| 11677 | +\newcommand{\crpsbudgetmrptsmallonemonlyctwofourkHfive}{0.4532} | |
| 11678 | +\newcommand{\crpsrawbudgetmrptsmallonemonlyctwofourkHfive}{7.936e-04} | |
| 11679 | +\newcommand{\covfiftybudgetmrptsmallonemonlyctwofourkHfive}{55} | |
| 11680 | +\newcommand{\covninetyfivebudgetmrptsmallonemonlyctwofourkHfive}{95.7} | |
| 11681 | +\newcommand{\diraccbudgetmrptsmallonemonlyctwofourkHfive}{0.501} | |
| 11682 | +\newcommand{\crpsbudgetmrptsmallonemonlyctwofourkHonefive}{0.4418} | |
| 11683 | +\newcommand{\crpsrawbudgetmrptsmallonemonlyctwofourkHonefive}{1.338e-03} | |
| 11684 | +\newcommand{\covfiftybudgetmrptsmallonemonlyctwofourkHonefive}{57} | |
| 11685 | +\newcommand{\covninetyfivebudgetmrptsmallonemonlyctwofourkHonefive}{96.0} | |
| 11686 | +\newcommand{\diraccbudgetmrptsmallonemonlyctwofourkHonefive}{0.500} | |
| 11687 | +\newcommand{\crpsbudgetmrptsmallonemonlyctwofourkHthreezero}{0.4319} | |
| 11688 | +\newcommand{\crpsrawbudgetmrptsmallonemonlyctwofourkHthreezero}{1.847e-03} | |
| 11689 | +\newcommand{\covfiftybudgetmrptsmallonemonlyctwofourkHthreezero}{57} | |
| 11690 | +\newcommand{\covninetyfivebudgetmrptsmallonemonlyctwofourkHthreezero}{96.2} | |
| 11691 | +\newcommand{\diraccbudgetmrptsmallonemonlyctwofourkHthreezero}{0.500} | |
| 11692 | +\newcommand{\crpsbudgetmrptsmallonemonlyctwofourkHsixzero}{0.4199} | |
| 11693 | +\newcommand{\crpsrawbudgetmrptsmallonemonlyctwofourkHsixzero}{2.533e-03} | |
| 11694 | +\newcommand{\covfiftybudgetmrptsmallonemonlyctwofourkHsixzero}{58} | |
| 11695 | +\newcommand{\covninetyfivebudgetmrptsmallonemonlyctwofourkHsixzero}{96.3} | |
| 11696 | +\newcommand{\diraccbudgetmrptsmallonemonlyctwofourkHsixzero}{0.499} | |
| 11697 | +\newcommand{\crpsbudgetmrptsmallonemonlycthreesixkHfive}{0.4526} | |
| 11698 | +\newcommand{\crpsrawbudgetmrptsmallonemonlycthreesixkHfive}{7.923e-04} | |
| 11699 | +\newcommand{\covfiftybudgetmrptsmallonemonlycthreesixkHfive}{54} | |
| 11700 | +\newcommand{\covninetyfivebudgetmrptsmallonemonlycthreesixkHfive}{95.2} | |
| 11701 | +\newcommand{\diraccbudgetmrptsmallonemonlycthreesixkHfive}{0.500} | |
| 11702 | +\newcommand{\crpsbudgetmrptsmallonemonlycthreesixkHonefive}{0.4411} | |
| 11703 | +\newcommand{\crpsrawbudgetmrptsmallonemonlycthreesixkHonefive}{1.335e-03} | |
| 11704 | +\newcommand{\covfiftybudgetmrptsmallonemonlycthreesixkHonefive}{56} | |
| 11705 | +\newcommand{\covninetyfivebudgetmrptsmallonemonlycthreesixkHonefive}{95.6} | |
| 11706 | +\newcommand{\diraccbudgetmrptsmallonemonlycthreesixkHonefive}{0.501} | |
| 11707 | +\newcommand{\crpsbudgetmrptsmallonemonlycthreesixkHthreezero}{0.4313} | |
| 11708 | +\newcommand{\crpsrawbudgetmrptsmallonemonlycthreesixkHthreezero}{1.843e-03} | |
| 11709 | +\newcommand{\covfiftybudgetmrptsmallonemonlycthreesixkHthreezero}{57} | |
| 11710 | +\newcommand{\covninetyfivebudgetmrptsmallonemonlycthreesixkHthreezero}{95.8} | |
| 11711 | +\newcommand{\diraccbudgetmrptsmallonemonlycthreesixkHthreezero}{0.501} | |
| 11712 | +\newcommand{\crpsbudgetmrptsmallonemonlycthreesixkHsixzero}{0.4195} | |
| 11713 | +\newcommand{\crpsrawbudgetmrptsmallonemonlycthreesixkHsixzero}{2.529e-03} | |
| 11714 | +\newcommand{\covfiftybudgetmrptsmallonemonlycthreesixkHsixzero}{58} | |
| 11715 | +\newcommand{\covninetyfivebudgetmrptsmallonemonlycthreesixkHsixzero}{96.1} | |
| 11716 | +\newcommand{\diraccbudgetmrptsmallonemonlycthreesixkHsixzero}{0.502} | |
| 11717 | +\newcommand{\crpsbudgetmrptsmallctwofourkHfive}{0.4529} | |
| 11718 | +\newcommand{\crpsrawbudgetmrptsmallctwofourkHfive}{7.929e-04} | |
| 11719 | +\newcommand{\covfiftybudgetmrptsmallctwofourkHfive}{55} | |
| 11720 | +\newcommand{\covninetyfivebudgetmrptsmallctwofourkHfive}{95.6} | |
| 11721 | +\newcommand{\diraccbudgetmrptsmallctwofourkHfive}{0.501} | |
| 11722 | +\newcommand{\crpsbudgetmrptsmallctwofourkHonefive}{0.4413} | |
| 11723 | +\newcommand{\crpsrawbudgetmrptsmallctwofourkHonefive}{1.336e-03} | |
| 11724 | +\newcommand{\covfiftybudgetmrptsmallctwofourkHonefive}{56} | |
| 11725 | +\newcommand{\covninetyfivebudgetmrptsmallctwofourkHonefive}{95.9} | |
| 11726 | +\newcommand{\diraccbudgetmrptsmallctwofourkHonefive}{0.500} | |
| 11727 | +\newcommand{\crpsbudgetmrptsmallctwofourkHthreezero}{0.4312} | |
| 11728 | +\newcommand{\crpsrawbudgetmrptsmallctwofourkHthreezero}{1.843e-03} | |
| 11729 | +\newcommand{\covfiftybudgetmrptsmallctwofourkHthreezero}{57} | |
| 11730 | +\newcommand{\covninetyfivebudgetmrptsmallctwofourkHthreezero}{96.0} | |
| 11731 | +\newcommand{\diraccbudgetmrptsmallctwofourkHthreezero}{0.501} | |
| 11732 | +\newcommand{\crpsbudgetmrptsmallctwofourkHsixzero}{0.4188} | |
| 11733 | +\newcommand{\crpsrawbudgetmrptsmallctwofourkHsixzero}{2.525e-03} | |
| 11734 | +\newcommand{\covfiftybudgetmrptsmallctwofourkHsixzero}{58} | |
| 11735 | +\newcommand{\covninetyfivebudgetmrptsmallctwofourkHsixzero}{96.1} | |
| 11736 | +\newcommand{\diraccbudgetmrptsmallctwofourkHsixzero}{0.502} | |
| 11737 | +\newcommand{\crpsbudgetmrptsmallcthreesixkHfive}{0.4528} | |
| 11738 | +\newcommand{\crpsrawbudgetmrptsmallcthreesixkHfive}{7.926e-04} | |
| 11739 | +\newcommand{\covfiftybudgetmrptsmallcthreesixkHfive}{54} | |
| 11740 | +\newcommand{\covninetyfivebudgetmrptsmallcthreesixkHfive}{95.5} | |
| 11741 | +\newcommand{\diraccbudgetmrptsmallcthreesixkHfive}{0.500} | |
| 11742 | +\newcommand{\crpsbudgetmrptsmallcthreesixkHonefive}{0.4411} | |
| 11743 | +\newcommand{\crpsrawbudgetmrptsmallcthreesixkHonefive}{1.335e-03} | |
| 11744 | +\newcommand{\covfiftybudgetmrptsmallcthreesixkHonefive}{56} | |
| 11745 | +\newcommand{\covninetyfivebudgetmrptsmallcthreesixkHonefive}{95.8} | |
| 11746 | +\newcommand{\diraccbudgetmrptsmallcthreesixkHonefive}{0.501} | |
| 11747 | +\newcommand{\crpsbudgetmrptsmallcthreesixkHthreezero}{0.4310} | |
| 11748 | +\newcommand{\crpsrawbudgetmrptsmallcthreesixkHthreezero}{1.842e-03} | |
| 11749 | +\newcommand{\covfiftybudgetmrptsmallcthreesixkHthreezero}{57} | |
| 11750 | +\newcommand{\covninetyfivebudgetmrptsmallcthreesixkHthreezero}{96.0} | |
| 11751 | +\newcommand{\diraccbudgetmrptsmallcthreesixkHthreezero}{0.502} | |
| 11752 | +\newcommand{\crpsbudgetmrptsmallcthreesixkHsixzero}{0.4185} | |
| 11753 | +\newcommand{\crpsrawbudgetmrptsmallcthreesixkHsixzero}{2.523e-03} | |
| 11754 | +\newcommand{\covfiftybudgetmrptsmallcthreesixkHsixzero}{57} | |
| 11755 | +\newcommand{\covninetyfivebudgetmrptsmallcthreesixkHsixzero}{96.1} | |
| 11756 | +\newcommand{\diraccbudgetmrptsmallcthreesixkHsixzero}{0.503} | |
| 11677 | 11757 | \newcommand{\pairwiseConvention}{scaled CRPS} |
added
paper/tables/budget_calibration.tex
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +\begin{table}[htbp]\centering\small | |
| 2 | +\caption{Learning curve (continued training): empirical 50/80/95\% coverage and PIT KS statistic} | |
| 3 | +\label{tab:budget_calib} | |
| 4 | +\begin{adjustbox}{max width=\textwidth} | |
| 5 | +\begin{tabular}{lcccccccccccccccc} | |
| 6 | +\toprule | |
| 7 | +Model & \multicolumn{4}{c}{$H=5$} & \multicolumn{4}{c}{$H=15$} & \multicolumn{4}{c}{$H=30$} & \multicolumn{4}{c}{$H=60$} \\ | |
| 8 | + & 50 & 80 & 95 & KS & 50 & 80 & 95 & KS & 50 & 80 & 95 & KS & 50 & 80 & 95 & KS \\ | |
| 9 | +\midrule | |
| 10 | +MRPT-S continued to 24k steps & 0.55 & 0.83 & 0.96 & 0.029 & 0.56 & 0.84 & 0.96 & 0.033 & 0.57 & 0.85 & 0.96 & 0.037 & 0.58 & 0.85 & 0.96 & 0.040 \\ | |
| 11 | +MRPT-S continued to 36k steps & 0.54 & 0.83 & 0.96 & 0.029 & 0.56 & 0.84 & 0.96 & 0.034 & 0.57 & 0.84 & 0.96 & 0.039 & 0.57 & 0.85 & 0.96 & 0.043 \\ | |
| 12 | +MRPT-S 1m only, 24k steps & 0.55 & 0.83 & 0.96 & 0.026 & 0.57 & 0.84 & 0.96 & 0.037 & 0.57 & 0.85 & 0.96 & 0.043 & 0.58 & 0.85 & 0.96 & 0.050 \\ | |
| 13 | +MRPT-S 1m only, 36k steps & 0.54 & 0.82 & 0.95 & 0.024 & 0.56 & 0.83 & 0.96 & 0.030 & 0.57 & 0.84 & 0.96 & 0.036 & 0.58 & 0.85 & 0.96 & 0.042 \\ | |
| 14 | +\bottomrule | |
| 15 | +\end{tabular} | |
| 16 | +\end{adjustbox} | |
| 17 | +\begin{minipage}{0.95\textwidth}\scriptsize \emph{Notes.} development test 2024, \num{986468} evaluation origins over 251 sessions. Empirical coverage of the central 50/80/95\,\% prediction intervals of the cumulative return (nominal 0.50/0.80/0.95) and the Kolmogorov--Smirnov distance of the PIT values from uniformity; no scaling is involved.\end{minipage} | |
| 18 | +\end{table} | |
| \ No newline at end of file | ||
added
paper/tables/budget_crps_scaled.tex
+17 −0
@@ -0,0 +1,17 @@ | ||
| 1 | +\begin{table}[htbp]\centering\small | |
| 2 | +\caption{Learning curve (continued training): mean scaled CRPS of the cumulative-return forecast by horizon (minutes)} | |
| 3 | +\label{tab:budget_crps_scaled} | |
| 4 | +\begin{adjustbox}{max width=\textwidth} | |
| 5 | +\begin{tabular}{lrcccc} | |
| 6 | +\toprule | |
| 7 | +Model & Parameters & $H=5$ & $H=15$ & $H=30$ & $H=60$ \\ | |
| 8 | +\midrule | |
| 9 | +MRPT-S continued to 24k steps & 3{,}455{,}619 & 0.4529 {\scriptsize[0.4474, 0.4588]} & 0.4413 {\scriptsize[0.4350, 0.4478]} & 0.4312 {\scriptsize[0.4249, 0.4381]} & 0.4188 {\scriptsize[0.4106, 0.4284]} \\ | |
| 10 | +MRPT-S continued to 36k steps & 3{,}455{,}619 & 0.4528 {\scriptsize[0.4473, 0.4587]} & 0.4411 {\scriptsize[0.4348, 0.4476]} & 0.4310 {\scriptsize[0.4247, 0.4380]} & 0.4185 {\scriptsize[0.4104, 0.4281]} \\ | |
| 11 | +MRPT-S 1m only, 24k steps & 3{,}200{,}259 & 0.4532 {\scriptsize[0.4478, 0.4591]} & 0.4418 {\scriptsize[0.4355, 0.4483]} & 0.4319 {\scriptsize[0.4257, 0.4387]} & 0.4199 {\scriptsize[0.4119, 0.4297]} \\ | |
| 12 | +MRPT-S 1m only, 36k steps & 3{,}200{,}259 & 0.4526 {\scriptsize[0.4471, 0.4585]} & 0.4411 {\scriptsize[0.4348, 0.4477]} & 0.4313 {\scriptsize[0.4250, 0.4381]} & 0.4195 {\scriptsize[0.4113, 0.4291]} \\ | |
| 13 | +\bottomrule | |
| 14 | +\end{tabular} | |
| 15 | +\end{adjustbox} | |
| 16 | +\begin{minipage}{0.95\textwidth}\scriptsize \emph{Notes.} development test 2024, \num{986468} evaluation origins over 251 sessions. Brackets: 95\,\% trading-day cluster-bootstrap intervals (all instruments and origins of one session form one cluster). Convention: scaled CRPS, i.e.\ CRPS divided by $\sigma_t\sqrt{H}$. Parameters: trainable parameters of the trained models (\texttt{train\_summary.json}); baselines have none in this sense.\end{minipage} | |
| 17 | +\end{table} | |
| \ No newline at end of file | ||
added
paper/tables/budget_qlike.tex
+17 −0
@@ -0,0 +1,17 @@ | ||
| 1 | +\begin{table}[htbp]\centering\small | |
| 2 | +\caption{Learning curve (continued training): mean QLIKE of the realised-variance forecast by horizon (minutes)} | |
| 3 | +\label{tab:budget_qlike} | |
| 4 | +\begin{adjustbox}{max width=\textwidth} | |
| 5 | +\begin{tabular}{lcccc} | |
| 6 | +\toprule | |
| 7 | +Model & $H=5$ & $H=15$ & $H=30$ & $H=60$ \\ | |
| 8 | +\midrule | |
| 9 | +MRPT-S continued to 24k steps & -12.475 {\scriptsize[-12.507, -12.439]} & -11.408 {\scriptsize[-11.443, -11.369]} & -10.737 {\scriptsize[-10.772, -10.697]} & -10.080 {\scriptsize[-10.117, -10.038]} \\ | |
| 10 | +MRPT-S continued to 36k steps & -12.476 {\scriptsize[-12.508, -12.439]} & -11.408 {\scriptsize[-11.443, -11.370]} & -10.736 {\scriptsize[-10.772, -10.697]} & -10.079 {\scriptsize[-10.117, -10.038]} \\ | |
| 11 | +MRPT-S 1m only, 24k steps & -12.469 {\scriptsize[-12.501, -12.431]} & -11.399 {\scriptsize[-11.437, -11.356]} & -10.727 {\scriptsize[-10.766, -10.682]} & -10.069 {\scriptsize[-10.108, -10.019]} \\ | |
| 12 | +MRPT-S 1m only, 36k steps & -12.469 {\scriptsize[-12.501, -12.432]} & -11.400 {\scriptsize[-11.438, -11.357]} & -10.729 {\scriptsize[-10.767, -10.685]} & -10.070 {\scriptsize[-10.108, -10.023]} \\ | |
| 13 | +\bottomrule | |
| 14 | +\end{tabular} | |
| 15 | +\end{adjustbox} | |
| 16 | +\begin{minipage}{0.95\textwidth}\scriptsize \emph{Notes.} development test 2024, \num{986468} evaluation origins over 251 sessions. Brackets: 95\,\% trading-day cluster-bootstrap intervals (all instruments and origins of one session form one cluster). Convention: QLIKE of the realised variance, $\log v + \mathrm{RV}/v$ without the data-only constant (levels can be negative). The LightGBM $H=5$ QLIKE is dominated by a few near-zero variance forecasts (floored at $10^{-14}$ in \texttt{eval/scores.py}); the floor share is tabulated once \texttt{recal/lightgbm\_floor.csv} is in.\end{minipage} | |
| 17 | +\end{table} | |
| \ No newline at end of file | ||
added
paper/tables/budget_wis.tex
+17 −0
@@ -0,0 +1,17 @@ | ||
| 1 | +\begin{table}[htbp]\centering\small | |
| 2 | +\caption{Learning curve (continued training): mean WIS ($\times 10^{4}$) of the cumulative-return forecast by horizon (minutes)} | |
| 3 | +\label{tab:budget_wis} | |
| 4 | +\begin{adjustbox}{max width=\textwidth} | |
| 5 | +\begin{tabular}{lcccc} | |
| 6 | +\toprule | |
| 7 | +Model & $H=5$ & $H=15$ & $H=30$ & $H=60$ \\ | |
| 8 | +\midrule | |
| 9 | +MRPT-S continued to 24k steps & 7.138 {\scriptsize[7.003, 7.279]} & 12.022 {\scriptsize[11.780, 12.281]} & 16.589 {\scriptsize[16.218, 16.970]} & 22.733 {\scriptsize[22.165, 23.355]} \\ | |
| 10 | +MRPT-S continued to 36k steps & 7.135 {\scriptsize[7.000, 7.276]} & 12.015 {\scriptsize[11.773, 12.275]} & 16.578 {\scriptsize[16.209, 16.958]} & 22.711 {\scriptsize[22.144, 23.333]} \\ | |
| 11 | +MRPT-S 1m only, 24k steps & 7.144 {\scriptsize[7.011, 7.286]} & 12.043 {\scriptsize[11.801, 12.303]} & 16.629 {\scriptsize[16.259, 17.006]} & 22.814 {\scriptsize[22.242, 23.431]} \\ | |
| 12 | +MRPT-S 1m only, 36k steps & 7.133 {\scriptsize[6.999, 7.274]} & 12.019 {\scriptsize[11.777, 12.281]} & 16.596 {\scriptsize[16.224, 16.976]} & 22.776 {\scriptsize[22.209, 23.397]} \\ | |
| 13 | +\bottomrule | |
| 14 | +\end{tabular} | |
| 15 | +\end{adjustbox} | |
| 16 | +\begin{minipage}{0.95\textwidth}\scriptsize \emph{Notes.} development test 2024, \num{986468} evaluation origins over 251 sessions. Brackets: 95\,\% trading-day cluster-bootstrap intervals (all instruments and origins of one session form one cluster). Convention: weighted interval score of the cumulative return (unscaled). Values are multiplied by $10^{4}$ (raw WIS of order $10^{-3}$).\end{minipage} | |
| 17 | +\end{table} | |
| \ No newline at end of file | ||
modified
paper/tables/compute.tex
+5 −1
@@ -36,11 +36,15 @@ seeds & MRPT-S seed 4 & MacStudnpierre4 & Apple M4 Max & 12{,}000 & 3{,}455{,}61 | ||
| 36 | 36 | seeds & MRPT-S seed 5 & MacStudnpierre4 & Apple M4 Max & 12{,}000 & 3{,}455{,}619 & 2{,}024 & 379 \\ |
| 37 | 37 | budget & MRPT-S 1m only, 24k steps & M5M36 & Apple M5 Max & 20{,}000 & 3{,}200{,}259 & 743 & 689 \\ |
| 38 | 38 | budget & MRPT-S 1m only, 36k steps & M5M36 & Apple M5 Max & 30{,}000 & 3{,}200{,}259 & 912 & 702 \\ |
| 39 | +budget & MRPT-S continued to 24k steps & MacStudnpierre2 & Apple M3 Ultra & 24{,}000 & 3{,}455{,}619 & 1{,}928 & 398 \\ | |
| 40 | +budget & MRPT-S continued to 36k steps & MacStudnpierre2 & Apple M3 Ultra & 34{,}000 & 3{,}455{,}619 & 1{,}564 & 409 \\ | |
| 39 | 41 | transfer300 & MRPT-S trained on all 300 instruments & m3u96b & Apple M3 Ultra & 12{,}000 & 3{,}455{,}619 & 1{,}902 & 404 \\ |
| 42 | +ablations\_h3 & MRPT-S no regime, reduced global token & MacStudnpierre2 & Apple M3 Ultra & 4{,}000 & 3{,}416{,}643 & 640 & 400 \\ | |
| 43 | +ablations\_h3 & MRPT-S regime stats, reduced global token & MacStudnpierre2 & Apple M3 Ultra & 4{,}000 & 3{,}455{,}619 & 640 & 400 \\ | |
| 40 | 44 | ablations\_h3 & MRPT-S shuffled price/volume columns & M5M36 & Apple M5 Max & 4{,}000 & 3{,}455{,}619 & 933 & 274 \\ |
| 41 | 45 | ablations\_h3 & MRPT-S time only, reduced global token & M5M36 & Apple M5 Max & 4{,}000 & 3{,}455{,}619 & 936 & 273 \\ |
| 42 | 46 | \midrule |
| 43 | −Total & 33 runs & & & & & 38{,}868 (10.8 GPU-h; baseline study 6.0 GPU-h) & \\ | |
| 47 | +Total & 37 runs & & & & & 43{,}639 (12.1 GPU-h; baseline study 6.0 GPU-h) & \\ | |
| 44 | 48 | \bottomrule |
| 45 | 49 | \end{tabular} |
| 46 | 50 | \end{adjustbox} |
modified
results/inline_numbers.json
+472 −0
@@ -6944,5 +6944,477 @@ | ||
| 6944 | 6944 | } |
| 6945 | 6945 | } |
| 6946 | 6946 | } |
| 6947 | + }, | |
| 6948 | + "budget": { | |
| 6949 | + "source": "results/metrics/budget/learning_curve.csv + budget_verdict.json (P4, scripts/review/learning_curve.py)", | |
| 6950 | + "learning_curve": [ | |
| 6951 | + { | |
| 6952 | + "budget": 12000, | |
| 6953 | + "model": "mrpt_small", | |
| 6954 | + "name": "mrpt_small", | |
| 6955 | + "host": null, | |
| 6956 | + "steps": 12000.0, | |
| 6957 | + "wall_s": 1759.3876690865, | |
| 6958 | + "ex_per_s": 436.5155067835, | |
| 6959 | + "val_subset": null, | |
| 6960 | + "val_nll_best": -6.4534611303, | |
| 6961 | + "val_nll_last": -6.4534611303, | |
| 6962 | + "val_nll_slope_per_1k": -0.0001691746, | |
| 6963 | + "n_val_evals": 12.0, | |
| 6964 | + "lr_at_resume": null, | |
| 6965 | + "crps_scaled_5": 0.4530927718, | |
| 6966 | + "crps_5": 0.0007932828, | |
| 6967 | + "rel_har_5_pct": -1.9056344036, | |
| 6968 | + "t_cl_har_5": -51.6870464186, | |
| 6969 | + "rel_rw_5_pct": -4.0537189763, | |
| 6970 | + "t_cl_rw_5": -41.4739939006, | |
| 6971 | + "crps_scaled_15": 0.4413554209, | |
| 6972 | + "crps_15": 0.0013358037, | |
| 6973 | + "rel_har_15_pct": 0.3419293208, | |
| 6974 | + "t_cl_har_15": 8.813362543, | |
| 6975 | + "rel_rw_15_pct": -4.0316649432, | |
| 6976 | + "t_cl_rw_15": -39.7797357585, | |
| 6977 | + "crps_scaled_30": 0.4311301909, | |
| 6978 | + "crps_30": 0.0018423706, | |
| 6979 | + "rel_har_30_pct": 0.5292370008, | |
| 6980 | + "t_cl_har_30": 9.2709593131, | |
| 6981 | + "rel_rw_30_pct": -4.0527658699, | |
| 6982 | + "t_cl_rw_30": -35.6420338188, | |
| 6983 | + "crps_scaled_60": 0.4187017316, | |
| 6984 | + "crps_60": 0.002523578, | |
| 6985 | + "rel_har_60_pct": 0.6525650454, | |
| 6986 | + "t_cl_har_60": 8.9204972459, | |
| 6987 | + "rel_rw_60_pct": -4.2698639812, | |
| 6988 | + "t_cl_rw_60": -32.1737574342, | |
| 6989 | + "gap_rel_pct_5": null, | |
| 6990 | + "gap_t_cl_5": null, | |
| 6991 | + "gap_rel_pct_15": null, | |
| 6992 | + "gap_t_cl_15": null, | |
| 6993 | + "gap_rel_pct_30": null, | |
| 6994 | + "gap_t_cl_30": null, | |
| 6995 | + "gap_rel_pct_60": null, | |
| 6996 | + "gap_t_cl_60": null | |
| 6997 | + }, | |
| 6998 | + { | |
| 6999 | + "budget": 12000, | |
| 7000 | + "model": "mrpt_small_1m_only", | |
| 7001 | + "name": "mrpt_small_1m_only", | |
| 7002 | + "host": null, | |
| 7003 | + "steps": 12000.0, | |
| 7004 | + "wall_s": 1087.6674129963, | |
| 7005 | + "ex_per_s": 706.0981976874, | |
| 7006 | + "val_subset": null, | |
| 7007 | + "val_nll_best": -6.4520200365, | |
| 7008 | + "val_nll_last": -6.4520200365, | |
| 7009 | + "val_nll_slope_per_1k": -0.0001421914, | |
| 7010 | + "n_val_evals": 12.0, | |
| 7011 | + "lr_at_resume": null, | |
| 7012 | + "crps_scaled_5": 0.4532714872, | |
| 7013 | + "crps_5": 0.0007935916, | |
| 7014 | + "rel_har_5_pct": -1.8674466033, | |
| 7015 | + "t_cl_har_5": -51.8908135155, | |
| 7016 | + "rel_rw_5_pct": -4.016367418, | |
| 7017 | + "t_cl_rw_5": -41.8341982228, | |
| 7018 | + "crps_scaled_15": 0.4415638994, | |
| 7019 | + "crps_15": 0.0013363016, | |
| 7020 | + "rel_har_15_pct": 0.3793291671, | |
| 7021 | + "t_cl_har_15": 9.6283302875, | |
| 7022 | + "rel_rw_15_pct": -3.9958952405, | |
| 7023 | + "t_cl_rw_15": -40.6351208841, | |
| 7024 | + "crps_scaled_30": 0.4314122831, | |
| 7025 | + "crps_30": 0.0018432955, | |
| 7026 | + "rel_har_30_pct": 0.5797083842, | |
| 7027 | + "t_cl_har_30": 10.2181128163, | |
| 7028 | + "rel_rw_30_pct": -4.0045949121, | |
| 7029 | + "t_cl_rw_30": -36.3956022031, | |
| 7030 | + "crps_scaled_60": 0.4192359584, | |
| 7031 | + "crps_60": 0.0025261483, | |
| 7032 | + "rel_har_60_pct": 0.7550823259, | |
| 7033 | + "t_cl_har_60": 10.9749079299, | |
| 7034 | + "rel_rw_60_pct": -4.172360324, | |
| 7035 | + "t_cl_rw_60": -32.5223804654, | |
| 7036 | + "gap_rel_pct_5": null, | |
| 7037 | + "gap_t_cl_5": null, | |
| 7038 | + "gap_rel_pct_15": null, | |
| 7039 | + "gap_t_cl_15": null, | |
| 7040 | + "gap_rel_pct_30": null, | |
| 7041 | + "gap_t_cl_30": null, | |
| 7042 | + "gap_rel_pct_60": null, | |
| 7043 | + "gap_t_cl_60": null | |
| 7044 | + }, | |
| 7045 | + { | |
| 7046 | + "budget": 24000, | |
| 7047 | + "model": "mrpt_small", | |
| 7048 | + "name": "mrpt_small_c24k", | |
| 7049 | + "host": "MacStudnpierre2", | |
| 7050 | + "steps": 24000.0, | |
| 7051 | + "wall_s": 1927.6599280834, | |
| 7052 | + "ex_per_s": 398.4105229409, | |
| 7053 | + "val_subset": 50000.0, | |
| 7054 | + "val_nll_best": -6.1220689368, | |
| 7055 | + "val_nll_last": -6.1218395602, | |
| 7056 | + "val_nll_slope_per_1k": -0.0001630566, | |
| 7057 | + "n_val_evals": 12.0, | |
| 7058 | + "lr_at_resume": 0.0001685938, | |
| 7059 | + "crps_scaled_5": 0.452924514, | |
| 7060 | + "crps_5": 0.0007929011, | |
| 7061 | + "rel_har_5_pct": -1.9528320004, | |
| 7062 | + "t_cl_har_5": -52.3988358765, | |
| 7063 | + "rel_rw_5_pct": -4.0998830333, | |
| 7064 | + "t_cl_rw_5": -41.3339362172, | |
| 7065 | + "crps_scaled_15": 0.4413040312, | |
| 7066 | + "crps_15": 0.0013357356, | |
| 7067 | + "rel_har_15_pct": 0.3368106517, | |
| 7068 | + "t_cl_har_15": 8.3123049256, | |
| 7069 | + "rel_rw_15_pct": -4.0365605053, | |
| 7070 | + "t_cl_rw_15": -39.3052422262, | |
| 7071 | + "crps_scaled_30": 0.4311654587, | |
| 7072 | + "crps_30": 0.0018429486, | |
| 7073 | + "rel_har_30_pct": 0.560775491, | |
| 7074 | + "t_cl_har_30": 9.5829338584, | |
| 7075 | + "rel_rw_30_pct": -4.0226648666, | |
| 7076 | + "t_cl_rw_30": -35.1506776245, | |
| 7077 | + "crps_scaled_60": 0.4187689452, | |
| 7078 | + "crps_60": 0.002525162, | |
| 7079 | + "rel_har_60_pct": 0.715742186, | |
| 7080 | + "t_cl_har_60": 9.2621874546, | |
| 7081 | + "rel_rw_60_pct": -4.2097765283, | |
| 7082 | + "t_cl_rw_60": -31.1260754817, | |
| 7083 | + "gap_rel_pct_5": null, | |
| 7084 | + "gap_t_cl_5": null, | |
| 7085 | + "gap_rel_pct_15": null, | |
| 7086 | + "gap_t_cl_15": null, | |
| 7087 | + "gap_rel_pct_30": null, | |
| 7088 | + "gap_t_cl_30": null, | |
| 7089 | + "gap_rel_pct_60": null, | |
| 7090 | + "gap_t_cl_60": null | |
| 7091 | + }, | |
| 7092 | + { | |
| 7093 | + "budget": 24000, | |
| 7094 | + "model": "mrpt_small_1m_only", | |
| 7095 | + "name": "mrpt_small_1m_only_c24k", | |
| 7096 | + "host": "M5M36", | |
| 7097 | + "steps": 20000.0, | |
| 7098 | + "wall_s": 743.2170901299, | |
| 7099 | + "ex_per_s": 688.8969680589, | |
| 7100 | + "val_subset": 50000.0, | |
| 7101 | + "val_nll_best": -6.1198707079, | |
| 7102 | + "val_nll_last": -6.1194296813, | |
| 7103 | + "val_nll_slope_per_1k": 5.22885e-05, | |
| 7104 | + "n_val_evals": 8.0, | |
| 7105 | + "lr_at_resume": 0.0001685938, | |
| 7106 | + "crps_scaled_5": 0.4532151677, | |
| 7107 | + "crps_5": 0.0007935599, | |
| 7108 | + "rel_har_5_pct": -1.8713624941, | |
| 7109 | + "t_cl_har_5": -54.2244520434, | |
| 7110 | + "rel_rw_5_pct": -4.020197558, | |
| 7111 | + "t_cl_rw_5": -41.3971915353, | |
| 7112 | + "crps_scaled_15": 0.4418304208, | |
| 7113 | + "crps_15": 0.0013378214, | |
| 7114 | + "rel_har_15_pct": 0.4934911362, | |
| 7115 | + "t_cl_har_15": 12.4285828702, | |
| 7116 | + "rel_rw_15_pct": -3.8867092385, | |
| 7117 | + "t_cl_rw_15": -39.8689997176, | |
| 7118 | + "crps_scaled_30": 0.4319130602, | |
| 7119 | + "crps_30": 0.0018469954, | |
| 7120 | + "rel_har_30_pct": 0.7815930931, | |
| 7121 | + "t_cl_har_30": 12.5976496918, | |
| 7122 | + "rel_rw_30_pct": -3.8119118677, | |
| 7123 | + "t_cl_rw_30": -35.964718943, | |
| 7124 | + "crps_scaled_60": 0.4199447727, | |
| 7125 | + "crps_60": 0.0025334924, | |
| 7126 | + "rel_har_60_pct": 1.047999968, | |
| 7127 | + "t_cl_har_60": 12.5309568992, | |
| 7128 | + "rel_rw_60_pct": -3.8937678638, | |
| 7129 | + "t_cl_rw_60": -31.5502836674, | |
| 7130 | + "gap_rel_pct_5": null, | |
| 7131 | + "gap_t_cl_5": null, | |
| 7132 | + "gap_rel_pct_15": null, | |
| 7133 | + "gap_t_cl_15": null, | |
| 7134 | + "gap_rel_pct_30": null, | |
| 7135 | + "gap_t_cl_30": null, | |
| 7136 | + "gap_rel_pct_60": null, | |
| 7137 | + "gap_t_cl_60": null | |
| 7138 | + }, | |
| 7139 | + { | |
| 7140 | + "budget": 36000, | |
| 7141 | + "model": "mrpt_small", | |
| 7142 | + "name": "mrpt_small_c36k", | |
| 7143 | + "host": "MacStudnpierre2", | |
| 7144 | + "steps": 34000.0, | |
| 7145 | + "wall_s": 1563.6283888817, | |
| 7146 | + "ex_per_s": 409.3044130887, | |
| 7147 | + "val_subset": 50000.0, | |
| 7148 | + "val_nll_best": -6.1223114292, | |
| 7149 | + "val_nll_last": -6.1218243954, | |
| 7150 | + "val_nll_slope_per_1k": -9.3733e-06, | |
| 7151 | + "n_val_evals": 10.0, | |
| 7152 | + "lr_at_resume": 9.88803e-05, | |
| 7153 | + "crps_scaled_5": 0.4527753305, | |
| 7154 | + "crps_5": 0.0007926279, | |
| 7155 | + "rel_har_5_pct": -1.9866196764, | |
| 7156 | + "t_cl_har_5": -49.5143744088, | |
| 7157 | + "rel_rw_5_pct": -4.1329308219, | |
| 7158 | + "t_cl_rw_5": -41.4659723695, | |
| 7159 | + "crps_scaled_15": 0.4411368498, | |
| 7160 | + "crps_15": 0.0013349884, | |
| 7161 | + "rel_har_15_pct": 0.2806854891, | |
| 7162 | + "t_cl_har_15": 6.722016244, | |
| 7163 | + "rel_rw_15_pct": -4.0902393458, | |
| 7164 | + "t_cl_rw_15": -38.9222677398, | |
| 7165 | + "crps_scaled_30": 0.4309795153, | |
| 7166 | + "crps_30": 0.0018417289, | |
| 7167 | + "rel_har_30_pct": 0.4942246442, | |
| 7168 | + "t_cl_har_30": 8.0308731555, | |
| 7169 | + "rel_rw_30_pct": -4.086182405, | |
| 7170 | + "t_cl_rw_30": -34.8617265841, | |
| 7171 | + "crps_scaled_60": 0.4185283397, | |
| 7172 | + "crps_60": 0.0025228662, | |
| 7173 | + "rel_har_60_pct": 0.6241739458, | |
| 7174 | + "t_cl_har_60": 7.3565662921, | |
| 7175 | + "rel_rw_60_pct": -4.2968666098, | |
| 7176 | + "t_cl_rw_60": -30.6639632503, | |
| 7177 | + "gap_rel_pct_5": null, | |
| 7178 | + "gap_t_cl_5": null, | |
| 7179 | + "gap_rel_pct_15": null, | |
| 7180 | + "gap_t_cl_15": null, | |
| 7181 | + "gap_rel_pct_30": null, | |
| 7182 | + "gap_t_cl_30": null, | |
| 7183 | + "gap_rel_pct_60": null, | |
| 7184 | + "gap_t_cl_60": null | |
| 7185 | + }, | |
| 7186 | + { | |
| 7187 | + "budget": 36000, | |
| 7188 | + "model": "mrpt_small_1m_only", | |
| 7189 | + "name": "mrpt_small_1m_only_c36k", | |
| 7190 | + "host": "M5M36", | |
| 7191 | + "steps": 30000.0, | |
| 7192 | + "wall_s": 912.2271790504, | |
| 7193 | + "ex_per_s": 701.579622596, | |
| 7194 | + "val_subset": 50000.0, | |
| 7195 | + "val_nll_best": -6.120694511, | |
| 7196 | + "val_nll_last": -6.1205803238, | |
| 7197 | + "val_nll_slope_per_1k": 5.78628e-05, | |
| 7198 | + "n_val_evals": 10.0, | |
| 7199 | + "lr_at_resume": 0.000143646, | |
| 7200 | + "crps_scaled_5": 0.4526041256, | |
| 7201 | + "crps_5": 0.0007922578, | |
| 7202 | + "rel_har_5_pct": -2.0323796092, | |
| 7203 | + "t_cl_har_5": -51.7236941458, | |
| 7204 | + "rel_rw_5_pct": -4.1776886971, | |
| 7205 | + "t_cl_rw_5": -41.7867386688, | |
| 7206 | + "crps_scaled_15": 0.4411428699, | |
| 7207 | + "crps_15": 0.001335139, | |
| 7208 | + "rel_har_15_pct": 0.291998869, | |
| 7209 | + "t_cl_har_15": 7.9942337119, | |
| 7210 | + "rel_rw_15_pct": -4.0794190811, | |
| 7211 | + "t_cl_rw_15": -39.3623285293, | |
| 7212 | + "crps_scaled_30": 0.4312809373, | |
| 7213 | + "crps_30": 0.0018433191, | |
| 7214 | + "rel_har_30_pct": 0.5809963109, | |
| 7215 | + "t_cl_har_30": 10.6489541594, | |
| 7216 | + "rel_rw_30_pct": -4.0033656875, | |
| 7217 | + "t_cl_rw_30": -35.0803124675, | |
| 7218 | + "crps_scaled_60": 0.4194674603, | |
| 7219 | + "crps_60": 0.0025292926, | |
| 7220 | + "rel_har_60_pct": 0.8804899075, | |
| 7221 | + "t_cl_har_60": 12.4698191559, | |
| 7222 | + "rel_rw_60_pct": -4.0530858192, | |
| 7223 | + "t_cl_rw_60": -30.3938767976, | |
| 7224 | + "gap_rel_pct_5": null, | |
| 7225 | + "gap_t_cl_5": null, | |
| 7226 | + "gap_rel_pct_15": null, | |
| 7227 | + "gap_t_cl_15": null, | |
| 7228 | + "gap_rel_pct_30": null, | |
| 7229 | + "gap_t_cl_30": null, | |
| 7230 | + "gap_rel_pct_60": null, | |
| 7231 | + "gap_t_cl_60": null | |
| 7232 | + }, | |
| 7233 | + { | |
| 7234 | + "budget": 12000, | |
| 7235 | + "model": "H1_gap", | |
| 7236 | + "name": "full_minus_1m_12000", | |
| 7237 | + "host": null, | |
| 7238 | + "steps": null, | |
| 7239 | + "wall_s": null, | |
| 7240 | + "ex_per_s": null, | |
| 7241 | + "val_subset": null, | |
| 7242 | + "val_nll_best": null, | |
| 7243 | + "val_nll_last": null, | |
| 7244 | + "val_nll_slope_per_1k": null, | |
| 7245 | + "n_val_evals": null, | |
| 7246 | + "lr_at_resume": null, | |
| 7247 | + "crps_scaled_5": null, | |
| 7248 | + "crps_5": -3.088e-07, | |
| 7249 | + "rel_har_5_pct": null, | |
| 7250 | + "t_cl_har_5": null, | |
| 7251 | + "rel_rw_5_pct": -0.0389145079, | |
| 7252 | + "t_cl_rw_5": -4.3973787059, | |
| 7253 | + "crps_scaled_15": null, | |
| 7254 | + "crps_15": -4.979e-07, | |
| 7255 | + "rel_har_15_pct": null, | |
| 7256 | + "t_cl_har_15": null, | |
| 7257 | + "rel_rw_15_pct": -0.0372585138, | |
| 7258 | + "t_cl_rw_15": -3.6705851182, | |
| 7259 | + "crps_scaled_30": null, | |
| 7260 | + "crps_30": -9.25e-07, | |
| 7261 | + "rel_har_30_pct": null, | |
| 7262 | + "t_cl_har_30": null, | |
| 7263 | + "rel_rw_30_pct": -0.0501804829, | |
| 7264 | + "t_cl_rw_30": -3.3917796089, | |
| 7265 | + "crps_scaled_60": null, | |
| 7266 | + "crps_60": -2.5703e-06, | |
| 7267 | + "rel_har_60_pct": null, | |
| 7268 | + "t_cl_har_60": null, | |
| 7269 | + "rel_rw_60_pct": -0.1017489918, | |
| 7270 | + "t_cl_rw_60": -4.1229218089, | |
| 7271 | + "gap_rel_pct_5": -0.0389145079, | |
| 7272 | + "gap_t_cl_5": -4.3973787059, | |
| 7273 | + "gap_rel_pct_15": -0.0372585138, | |
| 7274 | + "gap_t_cl_15": -3.6705851182, | |
| 7275 | + "gap_rel_pct_30": -0.0501804829, | |
| 7276 | + "gap_t_cl_30": -3.3917796089, | |
| 7277 | + "gap_rel_pct_60": -0.1017489918, | |
| 7278 | + "gap_t_cl_60": -4.1229218089 | |
| 7279 | + }, | |
| 7280 | + { | |
| 7281 | + "budget": 24000, | |
| 7282 | + "model": "H1_gap", | |
| 7283 | + "name": "full_minus_1m_24000", | |
| 7284 | + "host": null, | |
| 7285 | + "steps": null, | |
| 7286 | + "wall_s": null, | |
| 7287 | + "ex_per_s": null, | |
| 7288 | + "val_subset": null, | |
| 7289 | + "val_nll_best": null, | |
| 7290 | + "val_nll_last": null, | |
| 7291 | + "val_nll_slope_per_1k": null, | |
| 7292 | + "n_val_evals": null, | |
| 7293 | + "lr_at_resume": null, | |
| 7294 | + "crps_scaled_5": null, | |
| 7295 | + "crps_5": -6.588e-07, | |
| 7296 | + "rel_har_5_pct": null, | |
| 7297 | + "t_cl_har_5": null, | |
| 7298 | + "rel_rw_5_pct": -0.0830231708, | |
| 7299 | + "t_cl_rw_5": -5.7955317545, | |
| 7300 | + "crps_scaled_15": null, | |
| 7301 | + "crps_15": -2.0858e-06, | |
| 7302 | + "rel_har_15_pct": null, | |
| 7303 | + "t_cl_har_15": null, | |
| 7304 | + "rel_rw_15_pct": -0.1559110771, | |
| 7305 | + "t_cl_rw_15": -8.4853718992, | |
| 7306 | + "crps_scaled_30": null, | |
| 7307 | + "crps_30": -4.0469e-06, | |
| 7308 | + "rel_har_30_pct": null, | |
| 7309 | + "t_cl_har_30": null, | |
| 7310 | + "rel_rw_30_pct": -0.2191050918, | |
| 7311 | + "t_cl_rw_30": -7.0374665828, | |
| 7312 | + "crps_scaled_60": null, | |
| 7313 | + "crps_60": -8.3304e-06, | |
| 7314 | + "rel_har_60_pct": null, | |
| 7315 | + "t_cl_har_60": null, | |
| 7316 | + "rel_rw_60_pct": -0.3288118341, | |
| 7317 | + "t_cl_rw_60": -6.0666476581, | |
| 7318 | + "gap_rel_pct_5": -0.0830231708, | |
| 7319 | + "gap_t_cl_5": -5.7955317545, | |
| 7320 | + "gap_rel_pct_15": -0.1559110771, | |
| 7321 | + "gap_t_cl_15": -8.4853718992, | |
| 7322 | + "gap_rel_pct_30": -0.2191050918, | |
| 7323 | + "gap_t_cl_30": -7.0374665828, | |
| 7324 | + "gap_rel_pct_60": -0.3288118341, | |
| 7325 | + "gap_t_cl_60": -6.0666476581 | |
| 7326 | + }, | |
| 7327 | + { | |
| 7328 | + "budget": 36000, | |
| 7329 | + "model": "H1_gap", | |
| 7330 | + "name": "full_minus_1m_36000", | |
| 7331 | + "host": null, | |
| 7332 | + "steps": null, | |
| 7333 | + "wall_s": null, | |
| 7334 | + "ex_per_s": null, | |
| 7335 | + "val_subset": null, | |
| 7336 | + "val_nll_best": null, | |
| 7337 | + "val_nll_last": null, | |
| 7338 | + "val_nll_slope_per_1k": null, | |
| 7339 | + "n_val_evals": null, | |
| 7340 | + "lr_at_resume": null, | |
| 7341 | + "crps_scaled_5": null, | |
| 7342 | + "crps_5": 3.701e-07, | |
| 7343 | + "rel_har_5_pct": null, | |
| 7344 | + "t_cl_har_5": null, | |
| 7345 | + "rel_rw_5_pct": 0.0467092419, | |
| 7346 | + "t_cl_rw_5": 3.8427215052, | |
| 7347 | + "crps_scaled_15": null, | |
| 7348 | + "crps_15": -1.506e-07, | |
| 7349 | + "rel_har_15_pct": null, | |
| 7350 | + "t_cl_har_15": null, | |
| 7351 | + "rel_rw_15_pct": -0.0112804412, | |
| 7352 | + "t_cl_rw_15": -0.909494338, | |
| 7353 | + "crps_scaled_30": null, | |
| 7354 | + "crps_30": -1.5902e-06, | |
| 7355 | + "rel_har_30_pct": null, | |
| 7356 | + "t_cl_har_30": null, | |
| 7357 | + "rel_rw_30_pct": -0.0862704386, | |
| 7358 | + "t_cl_rw_30": -4.4859784999, | |
| 7359 | + "crps_scaled_60": null, | |
| 7360 | + "crps_60": -6.4264e-06, | |
| 7361 | + "rel_har_60_pct": null, | |
| 7362 | + "t_cl_har_60": null, | |
| 7363 | + "rel_rw_60_pct": -0.2540788233, | |
| 7364 | + "t_cl_rw_60": -7.4893913719, | |
| 7365 | + "gap_rel_pct_5": 0.0467092419, | |
| 7366 | + "gap_t_cl_5": 3.8427215052, | |
| 7367 | + "gap_rel_pct_15": -0.0112804412, | |
| 7368 | + "gap_t_cl_15": -0.909494338, | |
| 7369 | + "gap_rel_pct_30": -0.0862704386, | |
| 7370 | + "gap_t_cl_30": -4.4859784999, | |
| 7371 | + "gap_rel_pct_60": -0.2540788233, | |
| 7372 | + "gap_t_cl_60": -7.4893913719 | |
| 7373 | + } | |
| 7374 | + ], | |
| 7375 | + "verdict": { | |
| 7376 | + "criteria": "(i) H1 gap at 36k within the five-seed band of the 12k contrast; (ii) sign of MRPT-S vs HAR-RV at H=30 and 60 unchanged between 12k and 36k", | |
| 7377 | + "i_gap_inside_band": { | |
| 7378 | + "5": { | |
| 7379 | + "gap36k_rel_pct": 0.046709241853904265, | |
| 7380 | + "band_min_pct": -0.18256088703242176, | |
| 7381 | + "band_max_pct": -0.038915392667973274, | |
| 7382 | + "inside": false | |
| 7383 | + }, | |
| 7384 | + "15": { | |
| 7385 | + "gap36k_rel_pct": -0.011280441152003365, | |
| 7386 | + "band_min_pct": -0.34177789642559, | |
| 7387 | + "band_max_pct": -0.03723313038141836, | |
| 7388 | + "inside": false | |
| 7389 | + }, | |
| 7390 | + "30": { | |
| 7391 | + "gap36k_rel_pct": -0.08627043863992985, | |
| 7392 | + "band_min_pct": -0.4501766913992362, | |
| 7393 | + "band_max_pct": -0.050118279111809695, | |
| 7394 | + "inside": true | |
| 7395 | + }, | |
| 7396 | + "60": { | |
| 7397 | + "gap36k_rel_pct": -0.2540788233422957, | |
| 7398 | + "band_min_pct": -0.5488716027403494, | |
| 7399 | + "band_max_pct": -0.10156903627846983, | |
| 7400 | + "inside": true | |
| 7401 | + } | |
| 7402 | + }, | |
| 7403 | + "i_pass": false, | |
| 7404 | + "ii_sign_vs_har": { | |
| 7405 | + "30": { | |
| 7406 | + "rel_har_12k_pct": 0.5292370008204252, | |
| 7407 | + "rel_har_36k_pct": 0.49422464418555406, | |
| 7408 | + "same_sign": true | |
| 7409 | + }, | |
| 7410 | + "60": { | |
| 7411 | + "rel_har_12k_pct": 0.6525650454198514, | |
| 7412 | + "rel_har_36k_pct": 0.6241739457910326, | |
| 7413 | + "same_sign": true | |
| 7414 | + } | |
| 7415 | + }, | |
| 7416 | + "ii_pass": true, | |
| 7417 | + "budget_condition_leaves_abstract": false | |
| 7418 | + } | |
| 6947 | 7419 | } |
| 6948 | 7420 | } |
| \ No newline at end of file | ||
modified
scripts/review/headline_numbers.py
+17 −0
@@ -219,6 +219,23 @@ def main() -> None: | ||
| 219 | 219 | for r in q[(q.model == m) & (q.reference == ref)].itertuples()} for m, ref in q[["model", "reference"]].drop_duplicates().itertuples(index=False)} |
| 220 | 220 | e["pairwise"] = pc |
| 221 | 221 | out["lookahead66"][tag] = e |
| 222 | + # learning curve (P4): records of budget/learning_curve.csv and the budget verdict, copied verbatim | |
| 223 | + bd = M / "budget" | |
| 224 | + if (bd / "learning_curve.csv").exists(): | |
| 225 | + lc = pd.read_csv(bd / "learning_curve.csv") | |
| 226 | + out["budget"] = {"source": "results/metrics/budget/learning_curve.csv + budget_verdict.json (P4, scripts/review/learning_curve.py)", | |
| 227 | + "learning_curve": json.loads(lc.drop(columns=[c for c in lc.columns if c == "val_curve"]).to_json(orient="records"))} | |
| 228 | + if (bd / "budget_verdict.json").exists(): | |
| 229 | + out["budget"]["verdict"] = json.loads((bd / "budget_verdict.json").read_text()) | |
| 230 | + # H3 control (P10): stress-day contrasts of the reduced-global-token twins, copied verbatim | |
| 231 | + h3 = M / "ablations_h3" | |
| 232 | + if (h3 / "stress_contrast_test.csv").exists(): | |
| 233 | + sc = pd.read_csv(h3 / "stress_contrast_test.csv") | |
| 234 | + out["ablations_h3"] = {"source": "results/metrics/ablations_h3/{stress_contrast_test,pairwise_test}.csv (P10)", | |
| 235 | + "stress_contrast": json.loads(sc.to_json(orient="records"))} | |
| 236 | + if (h3 / "pairwise_test.csv").exists(): | |
| 237 | + pw3 = pd.read_csv(h3 / "pairwise_test.csv") | |
| 238 | + out["ablations_h3"]["pairwise_crps_scaled"] = json.loads(pw3[pw3.metric == "crps_scaled"].to_json(orient="records")) | |
| 222 | 239 | (ROOT / "results").mkdir(exist_ok=True) |
| 223 | 240 | (ROOT / "results" / "inline_numbers.json").write_text(json.dumps(out, indent=1, default=lambda x: x.item() if hasattr(x, "item") else str(x))) |
| 224 | 241 | # macros for the writing editor (letters only; percentages with two decimals; ranges as "a--b") |
| 225 | 242 | |