1{2 "config": "configs/experiments/latency_103m.yaml",3 "checkpoint_sha256": "64d08e9120608983b261bd6d108eb2421da45c84ebc733fa5fb2b88780dae427",4 "device": "mps",5 "torch": "2.15.0",6 "precision": "fp32",7 "prompt_len": 256,8 "n_new": 32,9 "code_commit": "unknown",10 "peak_memory_bytes": 3273342976,11 "analytic_ops_per_token": {12 "budget0": 209622528,13 "4_k1": 339414528,14 "4_k2": 355540480,15 "8_k2": 435656192,16 "8_dense": 88718284817 },18 "records": [19 {20 "policy": "budget0",21 "batch": 1,22 "budget": 0,23 "median_ms": 7.554021023679525,24 "p95_ms": 8.273145940620452,25 "mean_ms": 7.569623681774829,26 "n": 96,27 "cache_bytes": 589824028 },29 {30 "policy": "forced_single",31 "batch": 1,32 "budget": 1,33 "forced_k": 1,34 "median_ms": 10.197187482845038,35 "p95_ms": 10.996218799846247,36 "mean_ms": 10.16590669072078,37 "n": 96,38 "cache_bytes": 589824039 },40 {41 "policy": "forced_pair",42 "batch": 1,43 "budget": 1,44 "forced_k": 2,45 "median_ms": 10.232354514300823,46 "p95_ms": 10.949291929136962,47 "mean_ms": 10.210288580613755,48 "n": 96,49 "cache_bytes": 589824050 },51 {52 "policy": "forced_single",53 "batch": 1,54 "budget": 2,55 "forced_k": 1,56 "median_ms": 12.383270543068647,57 "p95_ms": 12.69960420904681,58 "mean_ms": 11.684006077606076,59 "n": 96,60 "cache_bytes": 589824061 },62 {63 "policy": "forced_pair",64 "batch": 1,65 "budget": 2,66 "forced_k": 2,67 "median_ms": 11.509437521453947,68 "p95_ms": 12.131708004744723,69 "mean_ms": 11.05400784460168,70 "n": 96,71 "cache_bytes": 589824072 },73 {74 "policy": "forced_single",75 "batch": 1,76 "budget": 4,77 "forced_k": 1,78 "median_ms": 15.866020519752055,79 "p95_ms": 54.16153179248795,80 "mean_ms": 19.282480483525433,81 "n": 96,82 "cache_bytes": 589824083 },84 {85 "policy": "forced_pair",86 "batch": 1,87 "budget": 4,88 "forced_k": 2,89 "median_ms": 15.460958005860448,90 "p95_ms": 27.407583547756076,91 "mean_ms": 16.31527776650425,92 "n": 96,93 "cache_bytes": 589824094 },95 {96 "policy": "forced_single",97 "batch": 1,98 "budget": 8,99 "forced_k": 1,100 "median_ms": 22.882541525177658,101 "p95_ms": 80.60674997977912,102 "mean_ms": 28.589312497691328,103 "n": 96,104 "cache_bytes": 5898240105 },106 {107 "policy": "forced_pair",108 "batch": 1,109 "budget": 8,110 "forced_k": 2,111 "median_ms": 23.291583522222936,112 "p95_ms": 24.80255250702612,113 "mean_ms": 23.393341953730367,114 "n": 96,115 "cache_bytes": 5898240116 },117 {118 "policy": "controller",119 "batch": 1,120 "budget": 8,121 "median_ms": 11.006999993696809,122 "p95_ms": 27.535593806533143,123 "mean_ms": 12.026253437700992,124 "n": 96,125 "cache_bytes": 5898240126 },127 {128 "policy": "dense_forced",129 "batch": 1,130 "budget": 1,131 "forced_k": 1,132 "median_ms": 10.64931252039969,133 "p95_ms": 17.744635028066114,134 "mean_ms": 11.291276908499034,135 "n": 96,136 "cache_bytes": 5898240137 },138 {139 "policy": "dense_forced",140 "batch": 1,141 "budget": 4,142 "forced_k": 1,143 "median_ms": 17.851791460998356,144 "p95_ms": 65.12488526641391,145 "mean_ms": 22.8273315927557,146 "n": 96,147 "cache_bytes": 5898240148 },149 {150 "policy": "dense_forced",151 "batch": 1,152 "budget": 8,153 "forced_k": 1,154 "median_ms": 26.906583516392857,155 "p95_ms": 27.862822695169598,156 "mean_ms": 27.034480945682542,157 "n": 96,158 "cache_bytes": 5898240159 },160 {161 "policy": "budget0",162 "batch": 8,163 "budget": 0,164 "median_ms": 10.26502053719014,165 "p95_ms": 18.752072588540614,166 "mean_ms": 10.654087217214206,167 "n": 96,168 "cache_bytes": 47185920169 },170 {171 "policy": "forced_single",172 "batch": 8,173 "budget": 1,174 "forced_k": 1,175 "median_ms": 13.082749966997653,176 "p95_ms": 23.506614292273298,177 "mean_ms": 13.859784298498804,178 "n": 96,179 "cache_bytes": 47185920180 },181 {182 "policy": "forced_pair",183 "batch": 8,184 "budget": 1,185 "forced_k": 2,186 "median_ms": 13.039500452578068,187 "p95_ms": 57.54836479900405,188 "mean_ms": 17.062923668466585,189 "n": 96,190 "cache_bytes": 47185920191 },192 {193 "policy": "forced_single",194 "batch": 8,195 "budget": 2,196 "forced_k": 1,197 "median_ms": 15.261041990015656,198 "p95_ms": 16.932114987866953,199 "mean_ms": 16.069337232086884,200 "n": 96,201 "cache_bytes": 47185920202 },203 {204 "policy": "forced_pair",205 "batch": 8,206 "budget": 2,207 "forced_k": 2,208 "median_ms": 15.204499941319227,209 "p95_ms": 16.368166747270152,210 "mean_ms": 14.876095493188283,211 "n": 96,212 "cache_bytes": 47185920213 },214 {215 "policy": "forced_single",216 "batch": 8,217 "budget": 4,218 "forced_k": 1,219 "median_ms": 18.81216699257493,220 "p95_ms": 20.145801798207685,221 "mean_ms": 18.852500479018392,222 "n": 96,223 "cache_bytes": 47185920224 },225 {226 "policy": "forced_pair",227 "batch": 8,228 "budget": 4,229 "forced_k": 2,230 "median_ms": 20.03912499640137,231 "p95_ms": 21.57740603433922,232 "mean_ms": 19.88851258768894,233 "n": 96,234 "cache_bytes": 47185920235 },236 {237 "policy": "forced_single",238 "batch": 8,239 "budget": 8,240 "forced_k": 1,241 "median_ms": 28.970604529604316,242 "p95_ms": 29.755865019978955,243 "mean_ms": 28.583446644915966,244 "n": 96,245 "cache_bytes": 47185920246 },247 {248 "policy": "forced_pair",249 "batch": 8,250 "budget": 8,251 "forced_k": 2,252 "median_ms": 29.06233351677656,253 "p95_ms": 33.15305270371027,254 "mean_ms": 28.535953611329507,255 "n": 96,256 "cache_bytes": 47185920257 },258 {259 "policy": "controller",260 "batch": 8,261 "budget": 8,262 "median_ms": 32.673625042662024,263 "p95_ms": 77.21810429939069,264 "mean_ms": 30.23307767701529,265 "n": 96,266 "cache_bytes": 47185920267 },268 {269 "policy": "dense_forced",270 "batch": 8,271 "budget": 1,272 "forced_k": 1,273 "median_ms": 13.034250470809639,274 "p95_ms": 26.244479202432558,275 "mean_ms": 14.254568138615772,276 "n": 96,277 "cache_bytes": 47185920278 },279 {280 "policy": "dense_forced",281 "batch": 8,282 "budget": 4,283 "forced_k": 1,284 "median_ms": 21.35618799366057,285 "p95_ms": 88.62939599202946,286 "mean_ms": 26.533539952652063,287 "n": 96,288 "cache_bytes": 47185920289 },290 {291 "policy": "dense_forced",292 "batch": 8,293 "budget": 8,294 "forced_k": 1,295 "median_ms": 32.09874941967428,296 "p95_ms": 34.511145524447784,297 "mean_ms": 32.247447521513095,298 "n": 96,299 "cache_bytes": 47185920300 }301 ]302}