1\begin{table}[t]\centering\small2\caption{Measured decode latency per generated token (fp32, prompt 64 tokens, 32 generated, warm-up and3synchronisation), device mps. Median and p95 over generated tokens of three repetitions.}4\label{tab:latency}5\begin{tabular}{llrrrr}\toprule6Policy & budget & batch & median ms & p95 ms & cache bytes\\\midrule7budget0 & 0 & 1 & 2.80 & 3.18 & 245,760\\8forced single & 1 & 1 & 5.25 & 6.08 & 245,760\\9forced pair & 1 & 1 & 5.22 & 6.36 & 245,760\\10forced single & 2 & 1 & 7.24 & 9.02 & 245,760\\11forced pair & 2 & 1 & 7.46 & 12.41 & 245,760\\12forced single & 4 & 1 & 11.29 & 12.40 & 245,760\\13forced pair & 4 & 1 & 11.35 & 12.38 & 245,760\\14forced single & 8 & 1 & 19.33 & 21.23 & 245,760\\15forced pair & 8 & 1 & 18.95 & 20.54 & 245,760\\16controller & 8 & 1 & 5.72 & 23.91 & 245,760\\17dense forced & 1 & 1 & 4.75 & 8.08 & 245,760\\18dense forced & 4 & 1 & 10.02 & 11.02 & 245,760\\19dense forced & 8 & 1 & 17.33 & 27.34 & 245,760\\20budget0 & 0 & 8 & 3.37 & 3.92 & 1,966,080\\21forced single & 1 & 8 & 6.34 & 10.40 & 1,966,080\\22forced pair & 1 & 8 & 6.17 & 7.16 & 1,966,080\\23forced single & 2 & 8 & 8.59 & 14.24 & 1,966,080\\24forced pair & 2 & 8 & 8.38 & 9.85 & 1,966,080\\25forced single & 4 & 8 & 12.96 & 14.20 & 1,966,080\\26forced pair & 4 & 8 & 13.35 & 20.81 & 1,966,080\\27forced single & 8 & 8 & 22.17 & 26.03 & 1,966,080\\28forced pair & 8 & 8 & 22.19 & 24.25 & 1,966,080\\29controller & 8 & 8 & 24.37 & 26.04 & 1,966,080\\30dense forced & 1 & 8 & 5.20 & 6.06 & 1,966,080\\31dense forced & 4 & 8 & 10.36 & 11.75 & 1,966,080\\32dense forced & 8 & 8 & 17.04 & 19.35 & 1,966,080\\33\bottomrule\end{tabular}\end{table}34