\begin{table}[t]\centering\small
\caption{Measured decode latency per generated token (fp32, prompt 64 tokens, 32 generated, warm-up and
synchronisation), device mps. Median and p95 over generated tokens of three repetitions.}
\label{tab:latency}
\begin{tabular}{llrrrr}\toprule
Policy & budget & batch & median ms & p95 ms & cache bytes\\\midrule
budget0 & 0 & 1 & 2.80 & 3.18 & 245,760\\
forced single & 1 & 1 & 5.25 & 6.08 & 245,760\\
forced pair & 1 & 1 & 5.22 & 6.36 & 245,760\\
forced single & 2 & 1 & 7.24 & 9.02 & 245,760\\
forced pair & 2 & 1 & 7.46 & 12.41 & 245,760\\
forced single & 4 & 1 & 11.29 & 12.40 & 245,760\\
forced pair & 4 & 1 & 11.35 & 12.38 & 245,760\\
forced single & 8 & 1 & 19.33 & 21.23 & 245,760\\
forced pair & 8 & 1 & 18.95 & 20.54 & 245,760\\
controller & 8 & 1 & 5.72 & 23.91 & 245,760\\
dense forced & 1 & 1 & 4.75 & 8.08 & 245,760\\
dense forced & 4 & 1 & 10.02 & 11.02 & 245,760\\
dense forced & 8 & 1 & 17.33 & 27.34 & 245,760\\
budget0 & 0 & 8 & 3.37 & 3.92 & 1,966,080\\
forced single & 1 & 8 & 6.34 & 10.40 & 1,966,080\\
forced pair & 1 & 8 & 6.17 & 7.16 & 1,966,080\\
forced single & 2 & 8 & 8.59 & 14.24 & 1,966,080\\
forced pair & 2 & 8 & 8.38 & 9.85 & 1,966,080\\
forced single & 4 & 8 & 12.96 & 14.20 & 1,966,080\\
forced pair & 4 & 8 & 13.35 & 20.81 & 1,966,080\\
forced single & 8 & 8 & 22.17 & 26.03 & 1,966,080\\
forced pair & 8 & 8 & 22.19 & 24.25 & 1,966,080\\
controller & 8 & 8 & 24.37 & 26.04 & 1,966,080\\
dense forced & 1 & 8 & 5.20 & 6.06 & 1,966,080\\
dense forced & 4 & 8 & 10.36 & 11.75 & 1,966,080\\
dense forced & 8 & 8 & 17.04 & 19.35 & 1,966,080\\
\bottomrule\end{tabular}\end{table}