1\begin{table}[t]\centering\small2\caption{103M pilot: measured decode latency per generated token on mps (fp32, prompt 256 tokens, 32 generated, three3repetitions, median and p95), batch 1 and 8.}4\label{tab:pilotlatency}5\begin{tabular}{llrrrr}\toprule6Policy & budget & batch & median ms & p95 ms & cache bytes\\\midrule7budget0 & 0 & 1 & 7.55 & 8.27 & 5,898,240\\8forced single & 1 & 1 & 10.20 & 11.00 & 5,898,240\\9forced pair & 1 & 1 & 10.23 & 10.95 & 5,898,240\\10forced single & 2 & 1 & 12.38 & 12.70 & 5,898,240\\11forced pair & 2 & 1 & 11.51 & 12.13 & 5,898,240\\12forced single & 4 & 1 & 15.87 & 54.16 & 5,898,240\\13forced pair & 4 & 1 & 15.46 & 27.41 & 5,898,240\\14forced single & 8 & 1 & 22.88 & 80.61 & 5,898,240\\15forced pair & 8 & 1 & 23.29 & 24.80 & 5,898,240\\16controller & 8 & 1 & 11.01 & 27.54 & 5,898,240\\17dense forced & 1 & 1 & 10.65 & 17.74 & 5,898,240\\18dense forced & 4 & 1 & 17.85 & 65.12 & 5,898,240\\19dense forced & 8 & 1 & 26.91 & 27.86 & 5,898,240\\20budget0 & 0 & 8 & 10.27 & 18.75 & 47,185,920\\21forced single & 1 & 8 & 13.08 & 23.51 & 47,185,920\\22forced pair & 1 & 8 & 13.04 & 57.55 & 47,185,920\\23forced single & 2 & 8 & 15.26 & 16.93 & 47,185,920\\24forced pair & 2 & 8 & 15.20 & 16.37 & 47,185,920\\25forced single & 4 & 8 & 18.81 & 20.15 & 47,185,920\\26forced pair & 4 & 8 & 20.04 & 21.58 & 47,185,920\\27forced single & 8 & 8 & 28.97 & 29.76 & 47,185,920\\28forced pair & 8 & 8 & 29.06 & 33.15 & 47,185,920\\29controller & 8 & 8 & 32.67 & 77.22 & 47,185,920\\30dense forced & 1 & 8 & 13.03 & 26.24 & 47,185,920\\31dense forced & 4 & 8 & 21.36 & 88.63 & 47,185,920\\32dense forced & 8 & 8 & 32.10 & 34.51 & 47,185,920\\33\bottomrule\end{tabular}\end{table}34