\begin{table}[t]\centering\small
\caption{103M pilot: measured decode latency per generated token on mps (fp32, prompt 256 tokens, 32 generated, three
repetitions, median and p95), batch 1 and 8.}
\label{tab:pilotlatency}
\begin{tabular}{llrrrr}\toprule
Policy & budget & batch & median ms & p95 ms & cache bytes\\\midrule
budget0 & 0 & 1 & 7.55 & 8.27 & 5,898,240\\
forced single & 1 & 1 & 10.20 & 11.00 & 5,898,240\\
forced pair & 1 & 1 & 10.23 & 10.95 & 5,898,240\\
forced single & 2 & 1 & 12.38 & 12.70 & 5,898,240\\
forced pair & 2 & 1 & 11.51 & 12.13 & 5,898,240\\
forced single & 4 & 1 & 15.87 & 54.16 & 5,898,240\\
forced pair & 4 & 1 & 15.46 & 27.41 & 5,898,240\\
forced single & 8 & 1 & 22.88 & 80.61 & 5,898,240\\
forced pair & 8 & 1 & 23.29 & 24.80 & 5,898,240\\
controller & 8 & 1 & 11.01 & 27.54 & 5,898,240\\
dense forced & 1 & 1 & 10.65 & 17.74 & 5,898,240\\
dense forced & 4 & 1 & 17.85 & 65.12 & 5,898,240\\
dense forced & 8 & 1 & 26.91 & 27.86 & 5,898,240\\
budget0 & 0 & 8 & 10.27 & 18.75 & 47,185,920\\
forced single & 1 & 8 & 13.08 & 23.51 & 47,185,920\\
forced pair & 1 & 8 & 13.04 & 57.55 & 47,185,920\\
forced single & 2 & 8 & 15.26 & 16.93 & 47,185,920\\
forced pair & 2 & 8 & 15.20 & 16.37 & 47,185,920\\
forced single & 4 & 8 & 18.81 & 20.15 & 47,185,920\\
forced pair & 4 & 8 & 20.04 & 21.58 & 47,185,920\\
forced single & 8 & 8 & 28.97 & 29.76 & 47,185,920\\
forced pair & 8 & 8 & 29.06 & 33.15 & 47,185,920\\
controller & 8 & 8 & 32.67 & 77.22 & 47,185,920\\
dense forced & 1 & 8 & 13.03 & 26.24 & 47,185,920\\
dense forced & 4 & 8 & 21.36 & 88.63 & 47,185,920\\
dense forced & 8 & 8 & 32.10 & 34.51 & 47,185,920\\
\bottomrule\end{tabular}\end{table}