SPB Git
spboucher.ai

spb/wp33_uqo

Public MIT

Cold sweep complete, host-time attribution, Gate D profiling (latency-bound), remote cost breakdown; thresholded predictors; LOG/DECISIONS D-010

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 1, 2026) parent d3deca4 Browse files

25 changed files +806 −6

25 changed files
Unified Split
added benchmarks/processed/gateC/sim_olmoe_v2_ssd043_m8_probe_fixed.json +341 −0
@@ -0,0 +1,341 @@
1 +{
2 + "cmd": "benchmarks/scripts/simulate_cache.py --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --capacities 16,24,32,48 --policies lru --predictors probe --m 8 --t-read-ms 0.43 --out benchmarks/processed/gateC/sim_olmoe_v2_ssd043_m8_probe_fixed.json",
3 + "ts": "2026-11-01T01:03:19",
4 + "traces": "benchmarks/raw/gateC/traces_olmoe_v2_laptop",
5 + "L": 16,
6 + "E": 64,
7 + "K": 8,
8 + "t_read_ms": 0.43,
9 + "t_layer_ms": 0.128,
10 + "loaders": 1,
11 + "results": [
12 + {
13 + "capacity": 16,
14 + "policy": "lru",
15 + "predictor": "probe",
16 + "m": 8,
17 + "hit_rate": 0.8134218505477309,
18 + "misses_per_token": 23.882003129890453,
19 + "stall_ms_per_token": 9.836425039123652,
20 + "prefetch_loads_per_token": 53.71596244131455,
21 + "prefetch_precision": 0.7587539512593046,
22 + "loads_per_token": 77.59796557120501,
23 + "pred_recall_at_k": 0.7792273082942097,
24 + "tokens": 12780,
25 + "families": {
26 + "general": {
27 + "hit_rate": 0.802437477750089,
28 + "stall_ms_per_token": 10.394331078675686
29 + },
30 + "code": {
31 + "hit_rate": 0.8304392880364109,
32 + "stall_ms_per_token": 8.95694408322492
33 + },
34 + "reasoning": {
35 + "hit_rate": 0.8113862683438156,
36 + "stall_ms_per_token": 9.931153039832292
37 + },
38 + "multilingual": {
39 + "hit_rate": 0.8074926380991064,
40 + "stall_ms_per_token": 10.152024370430548
41 + },
42 + "longctx": {
43 + "hit_rate": 0.8017371762099523,
44 + "stall_ms_per_token": 10.446455351056592
45 + },
46 + "drift": {
47 + "hit_rate": 0.8127011263073209,
48 + "stall_ms_per_token": 9.878384553499592
49 + }
50 + },
51 + "split": "fit"
52 + },
53 + {
54 + "capacity": 16,
55 + "policy": "lru",
56 + "predictor": "probe",
57 + "m": 8,
58 + "hit_rate": 0.8107280287984111,
59 + "misses_per_token": 24.226812313803375,
60 + "stall_ms_per_token": 10.039172989076453,
61 + "prefetch_loads_per_token": 53.51976166832175,
62 + "prefetch_precision": 0.7396946621541385,
63 + "loads_per_token": 77.74657398212513,
64 + "pred_recall_at_k": 0.761458126448196,
65 + "tokens": 5035,
66 + "families": {
67 + "general": {
68 + "hit_rate": 0.7883979628305933,
69 + "stall_ms_per_token": 11.246839170836292
70 + },
71 + "code": {
72 + "hit_rate": 0.8312998982467126,
73 + "stall_ms_per_token": 8.94341264871635
74 + },
75 + "reasoning": {
76 + "hit_rate": 0.8172567437379576,
77 + "stall_ms_per_token": 9.647799614643532
78 + },
79 + "multilingual": {
80 + "hit_rate": 0.8011740986717267,
81 + "stall_ms_per_token": 10.597753320683125
82 + },
83 + "longctx": {
84 + "hit_rate": 0.8135106519507187,
85 + "stall_ms_per_token": 9.833852156057503
86 + },
87 + "drift": {
88 + "hit_rate": 0.8081151185770751,
89 + "stall_ms_per_token": 10.175826086956512
90 + }
91 + },
92 + "split": "heldout"
93 + },
94 + {
95 + "capacity": 24,
96 + "policy": "lru",
97 + "predictor": "probe",
98 + "m": 8,
99 + "hit_rate": 0.8682762372848201,
100 + "misses_per_token": 16.860641627543036,
101 + "stall_ms_per_token": 7.022730672926475,
102 + "prefetch_loads_per_token": 36.59170579029734,
103 + "prefetch_precision": 0.7842216909516254,
104 + "loads_per_token": 53.45234741784038,
105 + "pred_recall_at_k": 0.7792273082942097,
106 + "tokens": 12780,
107 + "families": {
108 + "general": {
109 + "hit_rate": 0.8568663225347098,
110 + "stall_ms_per_token": 7.590407974368118
111 + },
112 + "code": {
113 + "hit_rate": 0.8836354031209362,
114 + "stall_ms_per_token": 6.252072821846548
115 + },
116 + "reasoning": {
117 + "hit_rate": 0.8658690382599581,
118 + "stall_ms_per_token": 7.147633123689735
119 + },
120 + "multilingual": {
121 + "hit_rate": 0.86384291226645,
122 + "stall_ms_per_token": 7.249991876523144
123 + },
124 + "longctx": {
125 + "hit_rate": 0.8567122528970689,
126 + "stall_ms_per_token": 7.596267211997278
127 + },
128 + "drift": {
129 + "hit_rate": 0.8708266291230893,
130 + "stall_ms_per_token": 6.900865647626699
131 + }
132 + },
133 + "split": "fit"
134 + },
135 + {
136 + "capacity": 24,
137 + "policy": "lru",
138 + "predictor": "probe",
139 + "m": 8,
140 + "hit_rate": 0.8636714870903675,
141 + "misses_per_token": 17.45004965243297,
142 + "stall_ms_per_token": 7.27823118172787,
143 + "prefetch_loads_per_token": 37.06554121151937,
144 + "prefetch_precision": 0.7664514400535833,
145 + "loads_per_token": 54.51559086395233,
146 + "pred_recall_at_k": 0.761458126448196,
147 + "tokens": 5035,
148 + "families": {
149 + "general": {
150 + "hit_rate": 0.8398688795568263,
151 + "stall_ms_per_token": 8.499243745532524
152 + },
153 + "code": {
154 + "hit_rate": 0.8832283187226049,
155 + "stall_ms_per_token": 6.283484032561061
156 + },
157 + "reasoning": {
158 + "hit_rate": 0.8712969653179191,
159 + "stall_ms_per_token": 6.859876685934493
160 + },
161 + "multilingual": {
162 + "hit_rate": 0.8551648481973435,
163 + "stall_ms_per_token": 7.757745730550289
164 + },
165 + "longctx": {
166 + "hit_rate": 0.8699948665297741,
167 + "stall_ms_per_token": 6.927868583162222
168 + },
169 + "drift": {
170 + "hit_rate": 0.8627099802371542,
171 + "stall_ms_per_token": 7.308790513833989
172 + }
173 + },
174 + "split": "heldout"
175 + },
176 + {
177 + "capacity": 32,
178 + "policy": "lru",
179 + "predictor": "probe",
180 + "m": 8,
181 + "hit_rate": 0.90824469385759,
182 + "misses_per_token": 11.744679186228481,
183 + "stall_ms_per_token": 4.924405477308244,
184 + "prefetch_loads_per_token": 23.853442879499216,
185 + "prefetch_precision": 0.8062536288695641,
186 + "loads_per_token": 35.5981220657277,
187 + "pred_recall_at_k": 0.7792273082942097,
188 + "tokens": 12780,
189 + "families": {
190 + "general": {
191 + "hit_rate": 0.8983512815948737,
192 + "stall_ms_per_token": 5.417714489142024
193 + },
194 + "code": {
195 + "hit_rate": 0.9207330949284785,
196 + "stall_ms_per_token": 4.2883807542262495
197 + },
198 + "reasoning": {
199 + "hit_rate": 0.9015166404612159,
200 + "stall_ms_per_token": 5.270654088050317
201 + },
202 + "multilingual": {
203 + "hit_rate": 0.9051901401299757,
204 + "stall_ms_per_token": 5.091374492282687
205 + },
206 + "longctx": {
207 + "hit_rate": 0.8993534850034083,
208 + "stall_ms_per_token": 5.378602590320401
209 + },
210 + "drift": {
211 + "hit_rate": 0.9136791029766693,
212 + "stall_ms_per_token": 4.644521319388589
213 + }
214 + },
215 + "split": "fit"
216 + },
217 + {
218 + "capacity": 32,
219 + "policy": "lru",
220 + "predictor": "probe",
221 + "m": 8,
222 + "hit_rate": 0.9042980387288977,
223 + "misses_per_token": 12.249851042701092,
224 + "stall_ms_per_token": 5.1351904667328165,
225 + "prefetch_loads_per_token": 24.360476663356504,
226 + "prefetch_precision": 0.7871020341608577,
227 + "loads_per_token": 36.610327706057596,
228 + "pred_recall_at_k": 0.761458126448196,
229 + "tokens": 5035,
230 + "families": {
231 + "general": {
232 + "hit_rate": 0.8843761168691923,
233 + "stall_ms_per_token": 6.165396711937117
234 + },
235 + "code": {
236 + "hit_rate": 0.9198203663118347,
237 + "stall_ms_per_token": 4.33534126487165
238 + },
239 + "reasoning": {
240 + "hit_rate": 0.9087186897880539,
241 + "stall_ms_per_token": 4.888231213872839
242 + },
243 + "multilingual": {
244 + "hit_rate": 0.8971181214421252,
245 + "stall_ms_per_token": 5.538747628083496
246 + },
247 + "longctx": {
248 + "hit_rate": 0.9107417864476386,
249 + "stall_ms_per_token": 4.783983572895287
250 + },
251 + "drift": {
252 + "hit_rate": 0.907130064229249,
253 + "stall_ms_per_token": 4.982296442687752
254 + }
255 + },
256 + "split": "heldout"
257 + },
258 + {
259 + "capacity": 48,
260 + "policy": "lru",
261 + "predictor": "probe",
262 + "m": 8,
263 + "hit_rate": 0.9654024843505478,
264 + "misses_per_token": 4.428482003129891,
265 + "stall_ms_per_token": 1.8745909233176232,
266 + "prefetch_loads_per_token": 7.827934272300469,
267 + "prefetch_precision": 0.8526004338221329,
268 + "loads_per_token": 12.25641627543036,
269 + "pred_recall_at_k": 0.7792273082942097,
270 + "tokens": 12780,
271 + "families": {
272 + "general": {
273 + "hit_rate": 0.9605842826628693,
274 + "stall_ms_per_token": 2.1284300462798225
275 + },
276 + "code": {
277 + "hit_rate": 0.9724357932379714,
278 + "stall_ms_per_token": 1.5022200260078151
279 + },
280 + "reasoning": {
281 + "hit_rate": 0.9616417714884696,
282 + "stall_ms_per_token": 2.083463312368981
283 + },
284 + "multilingual": {
285 + "hit_rate": 0.9631302041023558,
286 + "stall_ms_per_token": 1.9939082047116292
287 + },
288 + "longctx": {
289 + "hit_rate": 0.9590096710974778,
290 + "stall_ms_per_token": 2.2097764144512735
291 + },
292 + "drift": {
293 + "hit_rate": 0.9694665124698311,
294 + "stall_ms_per_token": 1.660585679806928
295 + }
296 + },
297 + "split": "fit"
298 + },
299 + {
300 + "capacity": 48,
301 + "policy": "lru",
302 + "predictor": "probe",
303 + "m": 8,
304 + "hit_rate": 0.9650524453823237,
305 + "misses_per_token": 4.473286991062562,
306 + "stall_ms_per_token": 1.8950407149949926,
307 + "prefetch_loads_per_token": 7.836544190665343,
308 + "prefetch_precision": 0.8325012038421573,
309 + "loads_per_token": 12.309831181727905,
310 + "pred_recall_at_k": 0.761458126448196,
311 + "tokens": 5035,
312 + "families": {
313 + "general": {
314 + "hit_rate": 0.9567213187991422,
315 + "stall_ms_per_token": 2.3366776268763556
316 + },
317 + "code": {
318 + "hit_rate": 0.9724679085785849,
319 + "stall_ms_per_token": 1.499576706324367
320 + },
321 + "reasoning": {
322 + "hit_rate": 0.9659802504816956,
323 + "stall_ms_per_token": 1.8477842003853637
324 + },
325 + "multilingual": {
326 + "hit_rate": 0.9567569971537002,
327 + "stall_ms_per_token": 2.3383187855787524
328 + },
329 + "longctx": {
330 + "hit_rate": 0.9713648613963038,
331 + "stall_ms_per_token": 1.5582053388090409
332 + },
333 + "drift": {
334 + "hit_rate": 0.9662950839920948,
335 + "stall_ms_per_token": 1.8331106719367645
336 + }
337 + },
338 + "split": "heldout"
339 + }
340 + ]
341 +}
\ No newline at end of file
added benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl +30 −0
@@ -0,0 +1,30 @@
1 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:02:22", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 517.0783339999616, "prompt_tps": 35.20778092655101, "gen_tps": 33.4326627330368, "decode_tps_wall": 32.911857674670465, "itl_ms_p50": 30.212124926038086, "itl_ms_p95": 39.193958044052124, "itl_ms_p99": 40.8845420461148, "itl_ms_mean": 30.384185842223584, "total_s": 2.4312820420600474, "peak_mem_gb": 1.829203036, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.4996987346856799, "cache_misses": 4982, "cache_hits": 4976, "stall_s": 1.3353543087141588, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 0.9596480529289693, "prefetch": 0.0, "ensure": 1.3742589412722737, "build": 0.01710272755008191}, "ssd_bytes": 17631019008, "ssd_read_s": 3.426499897032045}
2 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:02:25", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 348.18725008517504, "prompt_tps": 40.253284123634536, "gen_tps": 32.55307470525428, "decode_tps_wall": 32.04460133325126, "itl_ms_p50": 30.876416945829988, "itl_ms_p95": 39.19358307030052, "itl_ms_p99": 43.04183297790587, "itl_ms_mean": 31.206504633975406, "total_s": 2.3141970420256257, "peak_mem_gb": 1.747773596, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.4795243019648397, "cache_misses": 5033, "cache_hits": 4637, "stall_s": 1.2445547764655203, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 0.9447881018277258, "prefetch": 0.0, "ensure": 1.2828235330525786, "build": 0.016037469962611794}, "ssd_bytes": 17811505152, "ssd_read_s": 3.1572994944872335}
3 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:02:27", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 378.3321250230074, "prompt_tps": 47.62164073300397, "gen_tps": 33.273626374779, "decode_tps_wall": 32.75394520951318, "itl_ms_p50": 29.696666984818876, "itl_ms_p95": 38.782707997597754, "itl_ms_p99": 40.73108301963657, "itl_ms_mean": 30.530673285414064, "total_s": 2.3017645420040935, "peak_mem_gb": 1.812821016, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.5051215103434424, "cache_misses": 4928, "cache_hits": 5030, "stall_s": 1.2373295988654718, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 0.9343643408501521, "prefetch": 0.0, "ensure": 1.2764358408749104, "build": 0.01720138662494719}, "ssd_bytes": 17439916032, "ssd_read_s": 3.0876483746105805}
4 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:02:29", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 343.4790410101414, "prompt_tps": 40.801873390380386, "gen_tps": 32.914169802976595, "decode_tps_wall": 32.400085155163254, "itl_ms_p50": 30.86812503170222, "itl_ms_p95": 39.35583296697587, "itl_ms_p99": 43.849790934473276, "itl_ms_mean": 30.864116412380504, "total_s": 2.287918374990113, "peak_mem_gb": 1.747773596, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.4795243019648397, "cache_misses": 5033, "cache_hits": 4637, "stall_s": 1.2332852088147774, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 0.9296224870486185, "prefetch": 0.0, "ensure": 1.271775528904982, "build": 0.016064595663920045}, "ssd_bytes": 17811505152, "ssd_read_s": 3.1249096140963957}
5 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:02:31", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 342.86870807409286, "prompt_tps": 40.86731643441979, "gen_tps": 32.636451601945915, "decode_tps_wall": 32.12670637617493, "itl_ms_p50": 31.254375004209578, "itl_ms_p95": 39.06466602347791, "itl_ms_p99": 41.918667033314705, "itl_ms_mean": 31.12675131682957, "total_s": 2.3038540410343558, "peak_mem_gb": 1.741482148, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.4792140641158221, "cache_misses": 5036, "cache_hits": 4634, "stall_s": 1.2375774497631937, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 0.9419836424058303, "prefetch": 0.0, "ensure": 1.2754085791530088, "build": 0.016049782163463533}, "ssd_bytes": 17822121984, "ssd_read_s": 3.1362346468959004}
6 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:02:34", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 381.7877920810133, "prompt_tps": 47.18251065755433, "gen_tps": 33.0810152124823, "decode_tps_wall": 32.56430248431398, "itl_ms_p50": 29.244499979540706, "itl_ms_p95": 39.76695891469717, "itl_ms_p99": 42.16379206627607, "itl_ms_mean": 30.708472889345433, "total_s": 2.3164215841097757, "peak_mem_gb": 1.832006824, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.5051215103434424, "cache_misses": 4928, "cache_hits": 5030, "stall_s": 1.2385562813142315, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 0.9488227724796161, "prefetch": 0.0, "ensure": 1.2772458243416622, "build": 0.016906953300349414}, "ssd_bytes": 17439916032, "ssd_read_s": 3.0955757402116433}
7 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:02:41", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 543.5582080390304, "prompt_tps": 33.29991009188411, "gen_tps": 28.225582238301115, "decode_tps_wall": 27.78593305891577, "itl_ms_p50": 36.18829196784645, "itl_ms_p95": 44.08766608685255, "itl_ms_p99": 49.420999945141375, "itl_ms_mean": 35.98943385776014, "total_s": 2.8108925410779193, "peak_mem_gb": 1.849931992, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6504318136171922, "cache_misses": 3481, "cache_hits": 6477, "stall_s": 1.2660493151051924, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3315, "installed": 3313, "late": 110, "dup": 5333, "predictor": "transition2", "m": 8}, "host_time_s": {"sync": 1.3203314587008208, "prefetch": 0.1151198975276202, "ensure": 1.2682464926037937, "build": 0.019508084165863693}, "ssd_bytes": 24050663424, "ssd_read_s": 4.82381077145692}
8 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:02:44", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 363.3501670556143, "prompt_tps": 38.570176025229074, "gen_tps": 26.121693316338536, "decode_tps_wall": 25.713670423271733, "itl_ms_p50": 38.63416600506753, "itl_ms_p95": 45.95149995293468, "itl_ms_p99": 52.62233305256814, "itl_ms_mean": 38.889819443861526, "total_s": 2.8134087920188904, "peak_mem_gb": 1.766271896, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6188210961737332, "cache_misses": 3686, "cache_hits": 5984, "stall_s": 1.2340706296963617, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4201, "installed": 4195, "late": 107, "dup": 4351, "predictor": "transition2", "m": 8}, "host_time_s": {"sync": 1.3461572570959106, "prefetch": 0.12368838163092732, "ensure": 1.2391801627818495, "build": 0.01933034381363541}, "ssd_bytes": 27911651328, "ssd_read_s": 5.115074231871404}
9 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:02:46", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 395.5610409611836, "prompt_tps": 45.556328182596026, "gen_tps": 28.245109293687104, "decode_tps_wall": 27.803938979960005, "itl_ms_p50": 36.07900009956211, "itl_ms_p95": 42.57133405189961, "itl_ms_p99": 49.7471250128001, "itl_ms_mean": 35.96612698368965, "total_s": 2.6614270409336314, "peak_mem_gb": 1.864631956, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6567583852179153, "cache_misses": 3418, "cache_hits": 6540, "stall_s": 1.1613253159448504, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3257, "installed": 3263, "late": 128, "dup": 5455, "predictor": "transition2", "m": 8}, "host_time_s": {"sync": 1.2785086863441393, "prefetch": 0.12367042782716453, "ensure": 1.1563438426237553, "build": 0.0191463737282902}, "ssd_bytes": 23622451200, "ssd_read_s": 4.365179446875118}
10 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:02:49", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 360.393957933411, "prompt_tps": 38.880581717523405, "gen_tps": 26.03441502874237, "decode_tps_wall": 25.627780626094925, "itl_ms_p50": 39.00966595392674, "itl_ms_p95": 46.859833993948996, "itl_ms_p99": 49.87420805264264, "itl_ms_mean": 39.02015607944497, "total_s": 2.8186637909384444, "peak_mem_gb": 1.762732952, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6191313340227508, "cache_misses": 3683, "cache_hits": 5987, "stall_s": 1.246894485782832, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4204, "installed": 4200, "late": 116, "dup": 4348, "predictor": "transition2", "m": 8}, "host_time_s": {"sync": 1.3361439587315544, "prefetch": 0.1258306608069688, "ensure": 1.2519114535534754, "build": 0.019310060422867537}, "ssd_bytes": 27911651328, "ssd_read_s": 5.152700936654583}
11 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:02:52", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 359.2003750381991, "prompt_tps": 39.013927651294985, "gen_tps": 26.024585035633084, "decode_tps_wall": 25.618071984511023, "itl_ms_p50": 39.175417041406035, "itl_ms_p95": 47.21666604746133, "itl_ms_p99": 49.61366706993431, "itl_ms_mean": 39.034943792983775, "total_s": 2.8184018339961767, "peak_mem_gb": 1.766140932, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6191313340227508, "cache_misses": 3683, "cache_hits": 5987, "stall_s": 1.243524986319244, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4199, "installed": 4199, "late": 110, "dup": 4353, "predictor": "transition2", "m": 8}, "host_time_s": {"sync": 1.3395317263202742, "prefetch": 0.12579591968096793, "ensure": 1.247619368485175, "build": 0.019359704107046127}, "ssd_bytes": 27893956608, "ssd_read_s": 5.161090494017117}
12 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:02:55", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 392.5358339911327, "prompt_tps": 45.89750167336509, "gen_tps": 28.221729007624088, "decode_tps_wall": 27.780911509162685, "itl_ms_p50": 36.173832952044904, "itl_ms_p95": 44.07958302181214, "itl_ms_p99": 46.510166954249144, "itl_ms_mean": 35.995939142248105, "total_s": 2.6602799999527633, "peak_mem_gb": 1.842349792, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6565575416750351, "cache_misses": 3420, "cache_hits": 6538, "stall_s": 1.16297057911288, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3258, "installed": 3260, "late": 118, "dup": 5454, "predictor": "transition2", "m": 8}, "host_time_s": {"sync": 1.2777759277960286, "prefetch": 0.11877476540394127, "ensure": 1.1620669818948954, "build": 0.01921978371683508}, "ssd_bytes": 23633068032, "ssd_read_s": 4.389560426585376}
13 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:03:52", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 529.1794170625508, "prompt_tps": 34.192393477150624, "gen_tps": 29.86195784549739, "decode_tps_wall": 29.39630203157457, "itl_ms_p50": 34.54083402175456, "itl_ms_p95": 41.63816594518721, "itl_ms_p99": 46.15004197694361, "itl_ms_mean": 34.01788425380512, "total_s": 2.6723061250522733, "peak_mem_gb": 1.842854104, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.641594697730468, "cache_misses": 3569, "cache_hits": 6389, "stall_s": 1.1812305508647114, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2982, "installed": 2982, "late": 68, "dup": 4886, "predictor": "transition2t50", "m": 8}, "host_time_s": {"sync": 1.2894892855547369, "prefetch": 0.093443124438636, "ensure": 1.1950585045851767, "build": 0.01706448826007545}, "ssd_bytes": 23183622144, "ssd_read_s": 4.405051150824875}
14 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:03:54", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 347.15437493287027, "prompt_tps": 40.366358260015424, "gen_tps": 27.84939114644391, "decode_tps_wall": 27.41434630785206, "itl_ms_p50": 36.282457993365824, "itl_ms_p95": 43.52620802819729, "itl_ms_p99": 46.807541977614164, "itl_ms_mean": 36.4772513183573, "total_s": 2.6452212079893798, "peak_mem_gb": 1.75919402, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.612306101344364, "cache_misses": 3749, "cache_hits": 5921, "stall_s": 1.1427990465890616, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3922, "installed": 3917, "late": 64, "dup": 4031, "predictor": "transition2t50", "m": 8}, "host_time_s": {"sync": 1.2900060514220968, "prefetch": 0.10061502200551331, "ensure": 1.1604060878744349, "build": 0.01761058170814067}, "ssd_bytes": 27147239424, "ssd_read_s": 4.670730734593235}
15 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:03:57", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 371.8323749490082, "prompt_tps": 48.44667837686298, "gen_tps": 29.41238423146309, "decode_tps_wall": 28.952942132932755, "itl_ms_p50": 34.52545800246298, "itl_ms_p95": 41.58550000283867, "itl_ms_p99": 44.41204201430082, "itl_ms_mean": 34.53880422268181, "total_s": 2.5477770409779623, "peak_mem_gb": 1.85047618, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6476200040168708, "cache_misses": 3509, "cache_hits": 6449, "stall_s": 1.0857952203368768, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2935, "installed": 2937, "late": 75, "dup": 4997, "predictor": "transition2t50", "m": 8}, "host_time_s": {"sync": 1.26257005892694, "prefetch": 0.09576272440608591, "ensure": 1.098084571887739, "build": 0.017263506189920008}, "ssd_bytes": 22804955136, "ssd_read_s": 4.011304949061014}
16 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:03:59", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 338.45895901322365, "prompt_tps": 41.400091973073614, "gen_tps": 27.448277821131576, "decode_tps_wall": 27.01951678370005, "itl_ms_p50": 36.77250002510846, "itl_ms_p95": 43.173833983018994, "itl_ms_p99": 48.3146250480786, "itl_ms_mean": 37.010284380928155, "total_s": 2.6701068750116974, "peak_mem_gb": 1.75919402, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6126163391933815, "cache_misses": 3746, "cache_hits": 5924, "stall_s": 1.1472254098625854, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3926, "installed": 3924, "late": 57, "dup": 4027, "predictor": "transition2t50", "m": 8}, "host_time_s": {"sync": 1.3082013352541253, "prefetch": 0.1002028794027865, "ensure": 1.166193835902959, "build": 0.01799292885698378}, "ssd_bytes": 27150778368, "ssd_read_s": 4.7154672087635845}
17 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:04:02", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 344.41329108085483, "prompt_tps": 40.68289139770562, "gen_tps": 26.832091171562322, "decode_tps_wall": 26.412956008529626, "itl_ms_p50": 37.27545903529972, "itl_ms_p95": 45.126124983653426, "itl_ms_p99": 51.0092080803588, "itl_ms_mean": 37.86020768281546, "total_s": 2.7296063750982285, "peak_mem_gb": 1.776888728, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6120992761116856, "cache_misses": 3751, "cache_hits": 5919, "stall_s": 1.167463383404538, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3923, "installed": 3922, "late": 66, "dup": 4030, "predictor": "transition2t50", "m": 8}, "host_time_s": {"sync": 1.3267823717324063, "prefetch": 0.11268862476572394, "ensure": 1.1873078583739698, "build": 0.01936343580018729}, "ssd_bytes": 27157856256, "ssd_read_s": 4.747054176754318}
18 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:04:05", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 385.8761249575764, "prompt_tps": 46.6842459752629, "gen_tps": 28.40039964078331, "decode_tps_wall": 27.95676331223442, "itl_ms_p50": 36.158874980174005, "itl_ms_p95": 42.71891701500863, "itl_ms_p99": 46.233917004428804, "itl_ms_mean": 35.769519841461076, "total_s": 2.639355874969624, "peak_mem_gb": 1.83548528, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6477204257883109, "cache_misses": 3508, "cache_hits": 6450, "stall_s": 1.1170780754182488, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2934, "installed": 2937, "late": 85, "dup": 4998, "predictor": "transition2t50", "m": 8}, "host_time_s": {"sync": 1.2881446726387367, "prefetch": 0.11748608236666769, "ensure": 1.1284848203649744, "build": 0.020568106789141893}, "ssd_bytes": 22797877248, "ssd_read_s": 4.073544128099456}
19 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t70", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t70 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:04:12", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 520.4024999402463, "prompt_tps": 34.79124767995126, "gen_tps": 29.01467632571752, "decode_tps_wall": 28.562278602486078, "itl_ms_p50": 34.93270801845938, "itl_ms_p95": 43.403832940384746, "itl_ms_p99": 43.99354197084904, "itl_ms_mean": 35.0112123026823, "total_s": 2.7261088750092313, "peak_mem_gb": 1.83970846, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.5900783289817232, "cache_misses": 4082, "cache_hits": 5876, "stall_s": 1.2609232818940654, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1737, "installed": 1737, "late": 8, "dup": 3275, "predictor": "transition2t70", "m": 8}, "host_time_s": {"sync": 1.2682854104787111, "prefetch": 0.0691060513490811, "ensure": 1.2916151109384373, "build": 0.01749010500498116}, "ssd_bytes": 20593115136, "ssd_read_s": 4.132487543975003}
20 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t70", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t70 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:04:14", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 363.68562490679324, "prompt_tps": 38.5290569285975, "gen_tps": 27.939468551384746, "decode_tps_wall": 27.50304593514092, "itl_ms_p50": 36.37795790564269, "itl_ms_p95": 43.44491695519537, "itl_ms_p99": 47.161499969661236, "itl_ms_mean": 36.359609126867284, "total_s": 2.654340999899432, "peak_mem_gb": 1.751334372, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.5699069286452947, "cache_misses": 4159, "cache_hits": 5511, "stall_s": 1.186734238639474, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2312, "installed": 2312, "late": 10, "dup": 2668, "predictor": "transition2t70", "m": 8}, "host_time_s": {"sync": 1.2761520012281835, "prefetch": 0.07300603704061359, "ensure": 1.2156566720223054, "build": 0.016580969910137355}, "ssd_bytes": 22900506624, "ssd_read_s": 4.225527738803066}
21 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t70", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t70 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:04:17", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 375.0706249848008, "prompt_tps": 48.03028032344766, "gen_tps": 29.388223861291042, "decode_tps_wall": 28.92916348541071, "itl_ms_p50": 34.04141601640731, "itl_ms_p95": 42.17079107183963, "itl_ms_p99": 44.80775003321469, "itl_ms_mean": 34.567193776768235, "total_s": 2.5528038329211995, "peak_mem_gb": 1.823409816, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.5956015264109259, "cache_misses": 4027, "cache_hits": 5931, "stall_s": 1.1404099247884005, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1707, "installed": 1707, "late": 12, "dup": 3369, "predictor": "transition2t70", "m": 8}, "host_time_s": {"sync": 1.224560545757413, "prefetch": 0.06828455091454089, "ensure": 1.170226154383272, "build": 0.016853181179612875}, "ssd_bytes": 20292304896, "ssd_read_s": 3.6672123356256634}
22 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t70", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t70 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:04:20", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 344.3355839699507, "prompt_tps": 40.69267655882825, "gen_tps": 28.418940554095958, "decode_tps_wall": 27.975041089827275, "itl_ms_p50": 35.822125035338104, "itl_ms_p95": 42.88266703952104, "itl_ms_p99": 46.16183403413743, "itl_ms_mean": 35.746149461908594, "total_s": 2.596343000070192, "peak_mem_gb": 1.734421476, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.569493278179938, "cache_misses": 4163, "cache_hits": 5507, "stall_s": 1.1820058053126559, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2316, "installed": 2316, "late": 12, "dup": 2664, "predictor": "transition2t70", "m": 8}, "host_time_s": {"sync": 1.226772032561712, "prefetch": 0.07226992608048022, "ensure": 1.210089394939132, "build": 0.015987955266609788}, "ssd_bytes": 22928818176, "ssd_read_s": 4.200161305372603}
23 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t70", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t70 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:04:22", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 355.18995800521225, "prompt_tps": 39.448185592933086, "gen_tps": 27.225612276316657, "decode_tps_wall": 26.800377864463773, "itl_ms_p50": 37.167792092077434, "itl_ms_p95": 43.930709012784064, "itl_ms_p99": 47.71245794836432, "itl_ms_mean": 37.31290674546645, "total_s": 2.7059030829695985, "peak_mem_gb": 1.761951132, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.5695966907962772, "cache_misses": 4162, "cache_hits": 5508, "stall_s": 1.1938807795522735, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2315, "installed": 2315, "late": 12, "dup": 2665, "predictor": "transition2t70", "m": 8}, "host_time_s": {"sync": 1.3114323328481987, "prefetch": 0.07715808448847383, "ensure": 1.22335886687506, "build": 0.0173785729566589}, "ssd_bytes": 22921740288, "ssd_read_s": 4.247205002582632}
24 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t70", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t70 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:04:25", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 385.1542919874191, "prompt_tps": 46.77025411947176, "gen_tps": 26.652845588205146, "decode_tps_wall": 26.236517797397315, "itl_ms_p50": 38.85137499310076, "itl_ms_p95": 46.687541995197535, "itl_ms_p99": 51.674208021722734, "itl_ms_mean": 38.114814157967295, "total_s": 2.7863875839393586, "peak_mem_gb": 1.816463, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.5955011046394858, "cache_misses": 4028, "cache_hits": 5930, "stall_s": 1.2055370619054884, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1707, "installed": 1707, "late": 11, "dup": 3369, "predictor": "transition2t70", "m": 8}, "host_time_s": {"sync": 1.3609302886761725, "prefetch": 0.08145119482651353, "ensure": 1.2395877146627754, "build": 0.020705119939520955}, "ssd_bytes": 20295843840, "ssd_read_s": 3.862109358306043}
25 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:04:32", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 518.8054169993848, "prompt_tps": 34.87292168084887, "gen_tps": 29.724905866314426, "decode_tps_wall": 29.261436697346053, "itl_ms_p50": 34.55199999734759, "itl_ms_p95": 41.06304200831801, "itl_ms_p99": 47.81104193534702, "itl_ms_mean": 34.17467195281966, "total_s": 2.6718097500270233, "peak_mem_gb": 1.833023704, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6617794737899176, "cache_misses": 3368, "cache_hits": 6590, "stall_s": 1.1574277263134718, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3704, "installed": 3698, "late": 153, "dup": 5818, "predictor": "transition2t50", "m": 16}, "host_time_s": {"sync": 1.2977434222120792, "prefetch": 0.13386894366703928, "ensure": 1.1420442478265613, "build": 0.017283538007177413}, "ssd_bytes": 25027411968, "ssd_read_s": 4.566733709187247}
26 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:04:35", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 350.45979102142155, "prompt_tps": 39.98399691941206, "gen_tps": 26.99611487877608, "decode_tps_wall": 26.574413138665893, "itl_ms_p50": 37.45154105126858, "itl_ms_p95": 45.125292032025754, "itl_ms_p99": 48.94620890263468, "itl_ms_mean": 37.630181888946225, "total_s": 2.721161250025034, "peak_mem_gb": 1.77610698, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6412616339193382, "cache_misses": 3469, "cache_hits": 6201, "stall_s": 1.1320953140966594, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 5175, "installed": 5172, "late": 193, "dup": 5317, "predictor": "transition2t50", "m": 16}, "host_time_s": {"sync": 1.3525510855251923, "prefetch": 0.16570083075203001, "ensure": 1.1026974058477208, "build": 0.017763708136044443}, "ssd_bytes": 30590631936, "ssd_read_s": 4.9587827558862045}
27 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:04:37", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 364.8281251080334, "prompt_tps": 49.38202172368022, "gen_tps": 29.477932572770538, "decode_tps_wall": 29.017480210847932, "itl_ms_p50": 34.84945907257497, "itl_ms_p95": 41.88662499655038, "itl_ms_p99": 45.62962497584522, "itl_ms_mean": 34.46198611091527, "total_s": 2.5359332500956953, "peak_mem_gb": 1.839221404, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6680056236192007, "cache_misses": 3306, "cache_hits": 6652, "stall_s": 1.0438875223044306, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3642, "installed": 3645, "late": 161, "dup": 5944, "predictor": "transition2t50", "m": 16}, "host_time_s": {"sync": 1.2794099297607318, "prefetch": 0.137731593567878, "ensure": 1.0257551883114502, "build": 0.01715429697651416}, "ssd_bytes": 24588582912, "ssd_read_s": 4.099930818192661}
28 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:04:40", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 341.52458293829113, "prompt_tps": 41.02854651024846, "gen_tps": 27.240125215225166, "decode_tps_wall": 26.81462332201316, "itl_ms_p50": 37.03995898831636, "itl_ms_p95": 44.634749996475875, "itl_ms_p99": 48.89829200692475, "itl_ms_mean": 37.29308400088773, "total_s": 2.6909888749942183, "peak_mem_gb": 1.77610698, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6412616339193382, "cache_misses": 3469, "cache_hits": 6201, "stall_s": 1.1144671511137858, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 5172, "installed": 5169, "late": 201, "dup": 5320, "predictor": "transition2t50", "m": 16}, "host_time_s": {"sync": 1.34039991220925, "prefetch": 0.16478171071503311, "ensure": 1.0864357101963833, "build": 0.017624183907173574}, "ssd_bytes": 30580015104, "ssd_read_s": 4.9008175508352}
29 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:04:42", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 325.52800001576543, "prompt_tps": 43.04411021589692, "gen_tps": 27.72629489769962, "decode_tps_wall": 27.293189284377878, "itl_ms_p50": 36.34737501852214, "itl_ms_p95": 43.429667013697326, "itl_ms_p99": 47.101125004701316, "itl_ms_mean": 36.63917725336634, "total_s": 2.633796166977845, "peak_mem_gb": 1.78672374, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6413650465356774, "cache_misses": 3468, "cache_hits": 6202, "stall_s": 1.099088135175407, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 5178, "installed": 5178, "late": 193, "dup": 5314, "predictor": "transition2t50", "m": 16}, "host_time_s": {"sync": 1.3210817894432694, "prefetch": 0.15844549681060016, "ensure": 1.0648446546401829, "build": 0.015979969641193748}, "ssd_bytes": 30597709824, "ssd_read_s": 4.831111097242683}
30 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:04:45", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 349.48175004683435, "prompt_tps": 51.5479356880377, "gen_tps": 31.327212216405787, "decode_tps_wall": 30.83788490730665, "itl_ms_p50": 32.85600000526756, "itl_ms_p95": 38.74312504194677, "itl_ms_p99": 43.36958297062665, "itl_ms_mean": 32.427645508303414, "total_s": 2.392423417069949, "peak_mem_gb": 1.835944588, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6676039365334404, "cache_misses": 3310, "cache_hits": 6648, "stall_s": 0.9897855184972286, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3640, "installed": 3640, "late": 148, "dup": 5946, "predictor": "transition2t50", "m": 16}, "host_time_s": {"sync": 1.2337870801566169, "prefetch": 0.11556095094420016, "ensure": 0.9695895125623792, "build": 0.01276098215021193}, "ssd_bytes": 24595660800, "ssd_read_s": 3.909353203023784}
modified benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl +19 −0
@@ -195,3 +195,22 @@
195 195 {"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "5edaf3e"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:59:08", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 261.61037501879036, "prompt_tps": 68.9416892824878, "gen_tps": 27.464052531956753, "decode_tps_wall": 27.0350746300655, "itl_ms_p50": 36.600374965928495, "itl_ms_p95": 46.41845799051225, "itl_ms_p99": 47.61833290103823, "itl_ms_mean": 36.98898611095039, "total_s": 2.591916500008665, "peak_mem_gb": 2.727021948, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.7674767155550798, "cache_misses": 2172, "cache_hits": 7169, "stall_s": 0.9160170473624021, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 6864, "installed": 6853, "late": 224, "dup": 9776, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 31938969600, "ssd_read_s": 4.686025946168229}
196 196 {"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "5edaf3e"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:59:10", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 261.63404202088714, "prompt_tps": 91.9208585718617, "gen_tps": 33.073144288813886, "decode_tps_wall": 32.556587418479594, "itl_ms_p50": 29.125166009180248, "itl_ms_p95": 38.984958082437515, "itl_ms_p99": 46.841374947689474, "itl_ms_mean": 30.715750000025665, "total_s": 2.196726292022504, "peak_mem_gb": 2.8326179, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.8581856100104275, "cache_misses": 1360, "cache_hits": 8230, "stall_s": 0.5971727652940899, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 5856, "installed": 5895, "late": 114, "dup": 10784, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 25653805056, "ssd_read_s": 3.7076285743387416}
197 197 {"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "5edaf3e"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:59:13", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 246.18675000965595, "prompt_tps": 56.988426417342126, "gen_tps": 26.874429803645274, "decode_tps_wall": 26.45488250478592, "itl_ms_p50": 37.13241696823388, "itl_ms_p95": 47.65595798380673, "itl_ms_p99": 52.26895899977535, "itl_ms_mean": 37.800205682980874, "total_s": 2.627599708037451, "peak_mem_gb": 2.603629484, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.7534276152434416, "cache_misses": 2284, "cache_hits": 6979, "stall_s": 0.9120199183234945, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 7093, "installed": 7056, "late": 134, "dup": 9547, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 33067892736, "ssd_read_s": 4.923615432111546}
198 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "5edaf3e"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:59:15", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 322.21449993085116, "prompt_tps": 99.48828294265009, "gen_tps": 27.120569903421792, "decode_tps_wall": 26.697235232092478, "itl_ms_p50": 35.92433303128928, "itl_ms_p95": 48.65050001535565, "itl_ms_p99": 57.47608304955065, "itl_ms_mean": 37.457062175408716, "total_s": 2.6820094169816002, "peak_mem_gb": 3.033357056, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.7722249388753056, "cache_misses": 2329, "cache_hits": 7896, "stall_s": 0.9441881097154692, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 6843, "installed": 6829, "late": 129, "dup": 9797, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 32604291072, "ssd_read_s": 4.862698620767333}
199 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:59:23", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 535.2712909225374, "prompt_tps": 60.273155196191404, "gen_tps": 35.65712803254787, "decode_tps_wall": 35.10161836662931, "itl_ms_p50": 27.24037505686283, "itl_ms_p95": 35.15112504828721, "itl_ms_p99": 49.79716590605676, "itl_ms_mean": 28.48871495197749, "total_s": 2.3300603328971192, "peak_mem_gb": 2.857885252, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.807921760391198, "cache_misses": 1964, "cache_hits": 8261, "stall_s": 0.8874180053826421, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3138, "installed": 3138, "late": 206, "dup": 12838, "predictor": "transition2", "m": 16}, "ssd_bytes": 18055692288, "ssd_read_s": 3.668634384754114}
200 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:59:25", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 323.38604202959687, "prompt_tps": 74.31246454899245, "gen_tps": 41.313832577221554, "decode_tps_wall": 40.66811035832868, "itl_ms_p50": 22.526916000060737, "itl_ms_p95": 36.096416995860636, "itl_ms_p99": 40.55079095996916, "itl_ms_mean": 24.589290999482195, "total_s": 1.872511374996975, "peak_mem_gb": 2.491276696, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.8741397288842544, "cache_misses": 1207, "cache_hits": 8383, "stall_s": 0.5873838003026322, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1184, "installed": 1184, "late": 59, "dup": 14568, "predictor": "transition2", "m": 16}, "ssd_bytes": 8461615104, "ssd_read_s": 1.914486086810939}
201 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:59:27", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 257.2485829005018, "prompt_tps": 54.51359587139584, "gen_tps": 32.58839225386014, "decode_tps_wall": 32.079404163241, "itl_ms_p50": 30.457665910944343, "itl_ms_p95": 39.518165984191, "itl_ms_p99": 53.99866704829037, "itl_ms_mean": 31.172648809539776, "total_s": 2.2211254579015076, "peak_mem_gb": 2.52823389, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.8275936521645255, "cache_misses": 1597, "cache_hits": 7666, "stall_s": 0.7541046148398891, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4355, "installed": 4329, "late": 334, "dup": 11205, "predictor": "transition2", "m": 16}, "ssd_bytes": 21063794688, "ssd_read_s": 3.3201080777216703}
202 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:59:29", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 247.3491249838844, "prompt_tps": 72.88057963132758, "gen_tps": 38.63121945347461, "decode_tps_wall": 38.027869656437936, "itl_ms_p50": 25.281209032982588, "itl_ms_p95": 35.500874975696206, "itl_ms_p99": 40.43662501499057, "itl_ms_mean": 26.296503302301208, "total_s": 1.9040288330288604, "peak_mem_gb": 2.575077004, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.8572957927416764, "cache_misses": 1333, "cache_hits": 8008, "stall_s": 0.5783263321500272, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2955, "installed": 2959, "late": 237, "dup": 12645, "predictor": "transition2", "m": 16}, "ssd_bytes": 15174991872, "ssd_read_s": 2.4115369338542223}
203 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:59:31", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 269.37645801808685, "prompt_tps": 107.81151768384234, "gen_tps": 38.007079972890885, "decode_tps_wall": 37.413449841781464, "itl_ms_p50": 26.186582981608808, "itl_ms_p95": 34.237375017255545, "itl_ms_p99": 38.784332922659814, "itl_ms_mean": 26.728355824681266, "total_s": 1.9532628749730065, "peak_mem_gb": 2.692917728, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.8428950592078399, "cache_misses": 1539, "cache_hits": 8257, "stall_s": 0.6016018842346966, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2482, "installed": 2487, "late": 127, "dup": 13318, "predictor": "transition2", "m": 16}, "ssd_bytes": 14230093824, "ssd_read_s": 2.329985384712927}
204 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:59:33", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 220.89533298276365, "prompt_tps": 86.15119533674667, "gen_tps": 44.60681013316751, "decode_tps_wall": 43.910217652120465, "itl_ms_p50": 22.50249998178333, "itl_ms_p95": 29.182667029090226, "itl_ms_p99": 31.45670797675848, "itl_ms_mean": 22.773742729369257, "total_s": 1.6556411249330267, "peak_mem_gb": 2.540226668, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.8970619544390036, "cache_misses": 967, "cache_hits": 8427, "stall_s": 0.38502774434164166, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1200, "installed": 1209, "late": 41, "dup": 14448, "predictor": "transition2", "m": 16}, "ssd_bytes": 7668891648, "ssd_read_s": 1.3222372271120548}
205 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:59:35", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 311.58033397514373, "prompt_tps": 102.85514074426068, "gen_tps": 34.6949340690122, "decode_tps_wall": 34.15302090529882, "itl_ms_p50": 28.45316706225276, "itl_ms_p95": 37.236333009786904, "itl_ms_p99": 49.54491602256894, "itl_ms_mean": 29.27998676230865, "total_s": 2.1562195000005886, "peak_mem_gb": 2.778785212, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.8255256723716381, "cache_misses": 1784, "cache_hits": 8441, "stall_s": 0.7007565473904833, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2929, "installed": 2937, "late": 197, "dup": 13079, "predictor": "transition2", "m": 16}, "ssd_bytes": 16679043072, "ssd_read_s": 2.7620964454254135}
206 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:59:37", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 255.35870902240276, "prompt_tps": 70.5861705885171, "gen_tps": 36.12829949468941, "decode_tps_wall": 35.564010633101745, "itl_ms_p50": 27.851958060637116, "itl_ms_p95": 35.9834999544546, "itl_ms_p99": 39.681707974523306, "itl_ms_mean": 28.11831349159576, "total_s": 2.0268124589929357, "peak_mem_gb": 2.481241124, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.8545123648431645, "cache_misses": 1359, "cache_hits": 7982, "stall_s": 0.5864012060919777, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2937, "installed": 2913, "late": 233, "dup": 12663, "predictor": "transition2", "m": 16}, "ssd_bytes": 15203303424, "ssd_read_s": 2.4893748553004116}
207 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:59:39", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 245.4188329866156, "prompt_tps": 57.13558902793146, "gen_tps": 33.350650641387794, "decode_tps_wall": 32.82979121022096, "itl_ms_p50": 30.520499916747212, "itl_ms_p95": 39.363207994028926, "itl_ms_p99": 51.52833310421556, "itl_ms_mean": 30.460138890212253, "total_s": 2.1644075830699876, "peak_mem_gb": 2.531807918, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.8282413904782467, "cache_misses": 1591, "cache_hits": 7672, "stall_s": 0.7381878941087052, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4366, "installed": 4368, "late": 332, "dup": 11194, "predictor": "transition2", "m": 16}, "ssd_bytes": 21081489408, "ssd_read_s": 3.2596225037705153}
208 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:59:41", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 317.87662499118596, "prompt_tps": 91.33051624775473, "gen_tps": 41.627584354211805, "decode_tps_wall": 40.97748539083085, "itl_ms_p50": 24.385959026403725, "itl_ms_p95": 30.50545894075185, "itl_ms_p99": 37.26725000888109, "itl_ms_mean": 24.403644842095673, "total_s": 1.8553062500432134, "peak_mem_gb": 2.693093168, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.8405471621069824, "cache_misses": 1562, "cache_hits": 8234, "stall_s": 0.5801819945918396, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2482, "installed": 2490, "late": 105, "dup": 13318, "predictor": "transition2", "m": 16}, "ssd_bytes": 14311489536, "ssd_read_s": 2.302620130358264}
209 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:59:43", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 234.9707919638604, "prompt_tps": 102.27526981129552, "gen_tps": 45.06857507328316, "decode_tps_wall": 44.36469752953406, "itl_ms_p50": 22.080166963860393, "itl_ms_p95": 32.08154102321714, "itl_ms_p99": 35.39329208433628, "itl_ms_mean": 22.54044444536761, "total_s": 1.6550187920220196, "peak_mem_gb": 2.544360856, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.8794577685088634, "cache_misses": 1156, "cache_hits": 8434, "stall_s": 0.3957794235320762, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1237, "installed": 1251, "late": 45, "dup": 14515, "predictor": "transition2", "m": 16}, "ssd_bytes": 8468692992, "ssd_read_s": 1.3409272423014045}
210 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:59:44", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 208.75837502535433, "prompt_tps": 91.15956165861617, "gen_tps": 47.000010171472624, "decode_tps_wall": 46.265963464491485, "itl_ms_p50": 21.05712506454438, "itl_ms_p95": 28.216041042469442, "itl_ms_p99": 31.784582999534905, "itl_ms_mean": 21.614161364379385, "total_s": 1.5704505409812555, "peak_mem_gb": 2.48014654, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.8946135831381733, "cache_misses": 990, "cache_hits": 8404, "stall_s": 0.3688524183817208, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1162, "installed": 1156, "late": 48, "dup": 14486, "predictor": "transition2", "m": 16}, "ssd_bytes": 7615807488, "ssd_read_s": 1.2374950614757836}
211 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:59:46", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 244.17691701091826, "prompt_tps": 118.91345867928221, "gen_tps": 40.70535381982427, "decode_tps_wall": 40.069606632004536, "itl_ms_p50": 25.789916049689054, "itl_ms_p95": 31.336250016465783, "itl_ms_p99": 32.150042010471225, "itl_ms_mean": 24.956571427913058, "total_s": 1.8164409169694409, "peak_mem_gb": 2.600462816, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.842486729277256, "cache_misses": 1543, "cache_hits": 8253, "stall_s": 0.5462719945935532, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2467, "installed": 2459, "late": 112, "dup": 13333, "predictor": "transition2", "m": 16}, "ssd_bytes": 14191165440, "ssd_read_s": 2.1599057024577633}
212 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:59:48", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 200.30516700353473, "prompt_tps": 95.01252977045793, "gen_tps": 48.30449034823297, "decode_tps_wall": 47.55004226391123, "itl_ms_p50": 20.732083008624613, "itl_ms_p95": 26.532790972851217, "itl_ms_p99": 28.708333964459598, "itl_ms_mean": 21.03047552407675, "total_s": 1.5252251250203699, "peak_mem_gb": 2.517359468, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.8967426016606345, "cache_misses": 970, "cache_hits": 8424, "stall_s": 0.351440568221733, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1210, "installed": 1218, "late": 40, "dup": 14438, "predictor": "transition2", "m": 16}, "ssd_bytes": 7714897920, "ssd_read_s": 1.212078574928455}
213 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:59:49", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 230.96304200589657, "prompt_tps": 78.0410117482381, "gen_tps": 41.26642908079333, "decode_tps_wall": 40.621904175654606, "itl_ms_p50": 24.360541021451354, "itl_ms_p95": 32.594584045000374, "itl_ms_p99": 35.16958293039352, "itl_ms_mean": 24.61726057143616, "total_s": 1.7818504580063745, "peak_mem_gb": 2.482294668, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.8580451771758912, "cache_misses": 1326, "cache_hits": 8015, "stall_s": 0.5254195804009214, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2937, "installed": 2919, "late": 227, "dup": 12663, "predictor": "transition2", "m": 16}, "ssd_bytes": 15086518272, "ssd_read_s": 2.2348477555206046}
214 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:59:51", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 242.7965000970289, "prompt_tps": 98.97924191302036, "gen_tps": 47.77664866967458, "decode_tps_wall": 47.030472032844315, "itl_ms_p50": 20.501999999396503, "itl_ms_p95": 29.22224998474121, "itl_ms_p99": 31.38637507800013, "itl_ms_mean": 21.262810190415216, "total_s": 1.5823535420931876, "peak_mem_gb": 2.578455916, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.8765380604796663, "cache_misses": 1184, "cache_hits": 8406, "stall_s": 0.39839902927633375, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1222, "installed": 1246, "late": 48, "dup": 14530, "predictor": "transition2", "m": 16}, "ssd_bytes": 8514699264, "ssd_read_s": 1.3423358012223616}
215 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:59:53", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 221.3424580404535, "prompt_tps": 63.33984220635816, "gen_tps": 33.33601584236507, "decode_tps_wall": 32.81538558375421, "itl_ms_p50": 30.22895904723555, "itl_ms_p95": 39.32141698896885, "itl_ms_p99": 44.8081661015749, "itl_ms_mean": 30.47351058690794, "total_s": 2.1411736250156537, "peak_mem_gb": 2.521191086, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.8275936521645255, "cache_misses": 1597, "cache_hits": 7666, "stall_s": 0.7109807091765106, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4383, "installed": 4352, "late": 324, "dup": 11177, "predictor": "transition2", "m": 16}, "ssd_bytes": 21162885120, "ssd_read_s": 3.1957702171057463}
216 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:59:55", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 314.20337490271777, "prompt_tps": 101.95785628273087, "gen_tps": 38.10694992833497, "decode_tps_wall": 37.511737309435766, "itl_ms_p50": 25.707249995321035, "itl_ms_p95": 33.5518749197945, "itl_ms_p99": 45.18720810301602, "itl_ms_mean": 26.65832274711676, "total_s": 1.9936777079710737, "peak_mem_gb": 2.842027452, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.8253300733496333, "cache_misses": 1786, "cache_hits": 8439, "stall_s": 0.6469127177260816, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2965, "installed": 2996, "late": 180, "dup": 13043, "predictor": "transition2", "m": 16}, "ssd_bytes": 16813522944, "ssd_read_s": 2.59259042784106}
added benchmarks/raw/gateE/remote_cost_breakdown_m3u96a_m3u96b.txt +9 −0
@@ -0,0 +1,9 @@
1 +ping rtt ms: [1.174, 1.014, 0.989, 0.864, 0.741, 0.873, 0.81, 0.95, 0.834, 0.874]
2 +exec rows= 1 pairs= 1: round trip median 2.09 ms, p90 2.91 ms, bytes out 4108 in 4096
3 +exec rows= 1 pairs= 4: round trip median 1.99 ms, p90 2.56 ms, bytes out 4144 in 16384
4 +exec rows= 1 pairs= 8: round trip median 2.59 ms, p90 2.66 ms, bytes out 4192 in 32768
5 +exec rows= 8 pairs= 64: round trip median 7.27 ms, p90 7.83 ms, bytes out 33536 in 262144
6 +exec rows=32 pairs=256: round trip median 19.09 ms, p90 19.69 ms, bytes out 134144 in 1048576
7 +Traceback (most recent call last):
8 + File "<stdin>", line 21, in <module>
9 +NameError: name 'exec_ms' is not defined
modified benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl +16 −0
@@ -54,3 +54,19 @@
54 54 {"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": true, "repeat": -1, "family": "general", "prompt": "What should a visitor know before hiking in Gatineau Park in", "ts": "2026-11-01T01:57:57", "prompt_tokens": 16, "gen_tokens": 64, "ttft_ms": 287.85133291967213, "prompt_tps": 55.84103800491083, "gen_tps": 16.437401480807562, "decode_tps_wall": 16.180715476516337, "itl_ms_p50": 62.47966596856713, "itl_ms_p95": 69.74233302753419, "itl_ms_p99": 72.38837506156415, "itl_ms_mean": 61.80196428590173, "total_s": 4.181375082931481, "peak_mem_gb": 3.929057676, "text": "\n\n1. **Temperature and Weather Conditions**: The temperature in Gatineau Park can drop to as low as -20\u00b0C (-4\u00b0F) during winter. Visitors should dress warmly, including layers of clothing, a hat, glove", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5195, "local_pairs": 5045, "layers_remote": 1056, "remote_wait_s": 2.3621108037186787, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 6338, "exec_wall_s": 14.095058789243922, "bytes_sent": 33266207, "bytes_recv": 128852601}}}}
55 55 {"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a Go function that reads lines from stdin and prints t", "ts": "2026-11-01T01:58:01", "prompt_tokens": 21, "gen_tokens": 64, "ttft_ms": 328.16816703416407, "prompt_tps": 64.29895954419706, "gen_tps": 16.441890008158396, "decode_tps_wall": 16.18513256726093, "itl_ms_p50": 61.68937508482486, "itl_ms_p95": 70.47816691920161, "itl_ms_p99": 72.9299170197919, "itl_ms_mean": 61.78509788809433, "total_s": 4.220629333984107, "peak_mem_gb": 3.929732608, "text": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\nfunc main() {\n\tfmt.Println(\"Enter your lines (press Ctrl+D to finish):\")\n\tvar line string\n\tfmt.Scan(&line)\n\n\tif len(line) >", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5303, "local_pairs": 5577, "layers_remote": 1056, "remote_wait_s": 2.3896616504061967, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 7394, "exec_wall_s": 16.524280771496706, "bytes_sent": 38973417, "bytes_recv": 150678220}}}}
56 56 {"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Implement Dijkstra's shortest path in Python using a binary ", "ts": "2026-11-01T01:58:05", "prompt_tokens": 20, "gen_tokens": 64, "ttft_ms": 318.29787499736995, "prompt_tps": 63.094882802830774, "gen_tps": 16.187676379146705, "decode_tps_wall": 15.934791127721224, "itl_ms_p50": 62.68637499306351, "itl_ms_p95": 69.6389579679817, "itl_ms_p99": 81.47662505507469, "itl_ms_mean": 62.75576453966399, "total_s": 4.271911040996201, "peak_mem_gb": 3.928974552, "text": "import heapq\n\ndef dijkstra(graph, start):\n # Create a priority queue (binary heap)\n pq = []\n for node, distance in graph.items():\n heapq.heappush(pq, (distance, node))\n\n # Initializ", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5276, "local_pairs": 5476, "layers_remote": 1053, "remote_wait_s": 2.444692069548182, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 8447, "exec_wall_s": 19.00972998223733, "bytes_sent": 44602299, "bytes_recv": 172392932}}}}
57 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:58:09", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 277.1625410532579, "prompt_tps": 50.710122831968036, "gen_tps": 17.370990202125057, "decode_tps_wall": 17.099712548657298, "itl_ms_p50": 58.65108291618526, "itl_ms_p95": 67.12662510108203, "itl_ms_p99": 71.53516705147922, "itl_ms_mean": 58.480515222375594, "total_s": 3.96143500006292, "peak_mem_gb": 3.928853312, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 4766, "local_pairs": 5218, "layers_remote": 1055, "remote_wait_s": 2.2757996366126463, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 9502, "exec_wall_s": 21.324912759126164, "bytes_sent": 49842466, "bytes_recv": 192018873}}}}
58 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:58:13", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 314.9471250362694, "prompt_tps": 57.41954955116638, "gen_tps": 16.72422787875299, "decode_tps_wall": 16.463147000736527, "itl_ms_p50": 60.21262507420033, "itl_ms_p95": 68.67741700261831, "itl_ms_p99": 72.74716708343476, "itl_ms_mean": 60.74172817355406, "total_s": 4.141675999970175, "peak_mem_gb": 3.929440136, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5185, "local_pairs": 5311, "layers_remote": 1050, "remote_wait_s": 2.3272604587255046, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 10552, "exec_wall_s": 23.692372957943007, "bytes_sent": 55326901, "bytes_recv": 213360450}}}}
59 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:58:18", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 366.462167003192, "prompt_tps": 65.76356701026846, "gen_tps": 16.912153663335577, "decode_tps_wall": 16.648152388791885, "itl_ms_p50": 58.397875051014125, "itl_ms_p95": 70.92670793645084, "itl_ms_p99": 73.38308403268456, "itl_ms_mean": 60.06672552284149, "total_s": 4.150665874942206, "peak_mem_gb": 3.930389652, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5498, "local_pairs": 5766, "layers_remote": 1055, "remote_wait_s": 2.301105487626046, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 11607, "exec_wall_s": 26.03322983160615, "bytes_sent": 61228131, "bytes_recv": 235984255}}}}
60 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:58:22", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 307.99845792353153, "prompt_tps": 62.01355569123096, "gen_tps": 17.05873644203296, "decode_tps_wall": 16.79229550635324, "itl_ms_p50": 58.62408294342458, "itl_ms_p95": 68.79629206378013, "itl_ms_p99": 73.7650420051068, "itl_ms_mean": 59.55111971568493, "total_s": 4.0597190000116825, "peak_mem_gb": 3.929555316, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5080, "local_pairs": 5544, "layers_remote": 1053, "remote_wait_s": 2.281695583485998, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 12660, "exec_wall_s": 28.357330945320427, "bytes_sent": 66790038, "bytes_recv": 256895756}}}}
61 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 0, "family": "general", "prompt": "What should a visitor know before hiking in Gatineau Park in", "ts": "2026-11-01T01:58:26", "prompt_tokens": 16, "gen_tokens": 64, "ttft_ms": 303.25433297548443, "prompt_tps": 52.96033073170715, "gen_tps": 16.747381561926478, "decode_tps_wall": 16.486004623569183, "itl_ms_p50": 59.50295797083527, "itl_ms_p95": 70.59470796957612, "itl_ms_p99": 71.61116600036621, "itl_ms_mean": 60.657510587516875, "total_s": 4.124677499989048, "peak_mem_gb": 3.929051516, "text": "\n\n1. **Temperature and Weather Conditions**: The temperature in Gatineau Park can drop to as low as -20\u00b0C (-4\u00b0F) during winter. Visitors should dress warmly, including layers of clothing, a hat, glove", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5195, "local_pairs": 5045, "layers_remote": 1056, "remote_wait_s": 2.3054669718258083, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 13716, "exec_wall_s": 30.70366168045439, "bytes_sent": 72168478, "bytes_recv": 278278592}}}}
62 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Write a short motivational note for someone starting to lear", "ts": "2026-11-01T01:58:30", "prompt_tokens": 16, "gen_tokens": 64, "ttft_ms": 339.3256670096889, "prompt_tps": 47.350487491347074, "gen_tps": 17.32043353474051, "decode_tps_wall": 17.04994537966174, "itl_ms_p50": 58.83241700939834, "itl_ms_p95": 71.87708304263651, "itl_ms_p99": 73.41812504455447, "itl_ms_mean": 58.651214284408425, "total_s": 4.03435216692742, "peak_mem_gb": 3.929057664, "text": "\n\nDear [Name],\n\nWelcome to the wonderful world of piano playing! You've made an exciting decision to learn this beautiful instrument, and I'm thrilled to be a part of your journey.\n\nAs an adult, you b", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5268, "local_pairs": 4972, "layers_remote": 1053, "remote_wait_s": 2.3013280048035085, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 14769, "exec_wall_s": 33.041625296114944, "bytes_sent": 77534914, "bytes_recv": 299960109}}}}
63 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a Go function that reads lines from stdin and prints t", "ts": "2026-11-01T01:58:34", "prompt_tokens": 21, "gen_tokens": 64, "ttft_ms": 342.00699999928474, "prompt_tps": 61.66631944653354, "gen_tps": 16.925835625670608, "decode_tps_wall": 16.661501270359107, "itl_ms_p50": 60.39737502578646, "itl_ms_p95": 66.95083295926452, "itl_ms_p99": 71.25345803797245, "itl_ms_mean": 60.018601191658824, "total_s": 4.1231788750737906, "peak_mem_gb": 3.9297326, "text": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\nfunc main() {\n\tfmt.Println(\"Enter your lines (press Ctrl+D to finish):\")\n\tvar line string\n\tfmt.Scan(&line)\n\n\tif len(line) >", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5303, "local_pairs": 5577, "layers_remote": 1056, "remote_wait_s": 2.3457581215770915, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 15825, "exec_wall_s": 35.428386667976156, "bytes_sent": 83242124, "bytes_recv": 321785293}}}}
64 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Implement Dijkstra's shortest path in Python using a binary ", "ts": "2026-11-01T01:58:38", "prompt_tokens": 20, "gen_tokens": 64, "ttft_ms": 337.675292044878, "prompt_tps": 59.44652320122906, "gen_tps": 16.39640005206339, "decode_tps_wall": 16.140335004911005, "itl_ms_p50": 62.394042033702135, "itl_ms_p95": 69.84700006432831, "itl_ms_p99": 75.79287502449006, "itl_ms_mean": 61.956582666699966, "total_s": 4.240940000046976, "peak_mem_gb": 3.928987864, "text": "import heapq\n\ndef dijkstra(graph, start):\n # Create a priority queue (binary heap)\n pq = []\n for node, distance in graph.items():\n heapq.heappush(pq, (distance, node))\n\n # Initializ", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5276, "local_pairs": 5476, "layers_remote": 1053, "remote_wait_s": 2.3813273230334744, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 16878, "exec_wall_s": 37.85123591427691, "bytes_sent": 88871006, "bytes_recv": 343499635}}}}
65 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:58:42", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 314.2438749782741, "prompt_tps": 60.74697206931925, "gen_tps": 16.335709722923635, "decode_tps_wall": 16.080592695876064, "itl_ms_p50": 64.47133293841034, "itl_ms_p95": 70.98191697150469, "itl_ms_p99": 73.02016601897776, "itl_ms_mean": 62.18676257228095, "total_s": 4.232009917031974, "peak_mem_gb": 3.929555312, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5080, "local_pairs": 5544, "layers_remote": 1053, "remote_wait_s": 2.396930565824732, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 17931, "exec_wall_s": 40.29035819589626, "bytes_sent": 94432913, "bytes_recv": 364411110}}}}
66 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:58:47", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 368.5920829884708, "prompt_tps": 65.3798340795785, "gen_tps": 15.89720568717961, "decode_tps_wall": 15.648938987322436, "itl_ms_p50": 65.70179096888751, "itl_ms_p95": 72.5217079743743, "itl_ms_p99": 78.81049998104572, "itl_ms_mean": 63.90209590631818, "total_s": 4.394424125086516, "peak_mem_gb": 3.930396948, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5498, "local_pairs": 5766, "layers_remote": 1055, "remote_wait_s": 2.4686616924591362, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 18986, "exec_wall_s": 42.800328896264546, "bytes_sent": 100334143, "bytes_recv": 387034930}}}}
67 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 1, "family": "general", "prompt": "What should a visitor know before hiking in Gatineau Park in", "ts": "2026-11-01T01:58:51", "prompt_tokens": 16, "gen_tokens": 64, "ttft_ms": 304.0867920499295, "prompt_tps": 52.87881787496036, "gen_tps": 17.123550718786106, "decode_tps_wall": 16.85612057902145, "itl_ms_p50": 58.13645804300904, "itl_ms_p95": 67.32404208742082, "itl_ms_p99": 126.80229207035154, "itl_ms_mean": 59.32563161920933, "total_s": 4.041601584060118, "peak_mem_gb": 3.929051532, "text": "\n\n1. **Temperature and Weather Conditions**: The temperature in Gatineau Park can drop to as low as -20\u00b0C (-4\u00b0F) during winter. Visitors should dress warmly, including layers of clothing, a hat, glove", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5195, "local_pairs": 5045, "layers_remote": 1056, "remote_wait_s": 2.2295505325309932, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 20042, "exec_wall_s": 45.06953572505154, "bytes_sent": 105712583, "bytes_recv": 408417736}}}}
68 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Implement Dijkstra's shortest path in Python using a binary ", "ts": "2026-11-01T01:58:55", "prompt_tokens": 20, "gen_tokens": 64, "ttft_ms": 333.642499987036, "prompt_tps": 60.21527714334043, "gen_tps": 16.801364717181464, "decode_tps_wall": 16.539100105883325, "itl_ms_p50": 59.55762509256601, "itl_ms_p95": 68.51629097945988, "itl_ms_p99": 71.15379197057337, "itl_ms_mean": 60.462781747374386, "total_s": 4.142797750071622, "peak_mem_gb": 3.9289766, "text": "import heapq\n\ndef dijkstra(graph, start):\n # Create a priority queue (binary heap)\n pq = []\n for node, distance in graph.items():\n heapq.heappush(pq, (distance, node))\n\n # Initializ", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5276, "local_pairs": 5476, "layers_remote": 1053, "remote_wait_s": 2.302055887179449, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 21095, "exec_wall_s": 47.41107607888989, "bytes_sent": 111341465, "bytes_recv": 430132044}}}}
69 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:58:59", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 309.8848749650642, "prompt_tps": 58.36380308768657, "gen_tps": 16.627315176570754, "decode_tps_wall": 16.367629074619177, "itl_ms_p50": 60.280082980170846, "itl_ms_p95": 70.00595796853304, "itl_ms_p99": 75.6408330053091, "itl_ms_mean": 61.096203698229694, "total_s": 4.158945707953535, "peak_mem_gb": 3.930145552, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5185, "local_pairs": 5311, "layers_remote": 1050, "remote_wait_s": 2.299758543027565, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 22145, "exec_wall_s": 49.75092511344701, "bytes_sent": 116825900, "bytes_recv": 451473335}}}}
70 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:59:03", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 304.7085839789361, "prompt_tps": 46.15351647580348, "gen_tps": 16.97438091815592, "decode_tps_wall": 16.70929710991178, "itl_ms_p50": 59.820333030074835, "itl_ms_p95": 69.97345807030797, "itl_ms_p99": 72.18454196117818, "itl_ms_mean": 59.846921951421315, "total_s": 4.075064666918479, "peak_mem_gb": 3.928859872, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 4766, "local_pairs": 5218, "layers_remote": 1055, "remote_wait_s": 2.386137642432004, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 23200, "exec_wall_s": 52.17577198485378, "bytes_sent": 122066067, "bytes_recv": 471098872}}}}
71 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a Go function that reads lines from stdin and prints t", "ts": "2026-11-01T01:59:07", "prompt_tokens": 21, "gen_tokens": 64, "ttft_ms": 346.0335830459371, "prompt_tps": 60.91614982825395, "gen_tps": 16.835513384106463, "decode_tps_wall": 16.57275148401759, "itl_ms_p50": 60.098999994806945, "itl_ms_p95": 70.96566702239215, "itl_ms_p99": 72.5581250153482, "itl_ms_mean": 60.34001058691906, "total_s": 4.147454250021838, "peak_mem_gb": 3.92973056, "text": "package main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\nfunc main() {\n\tfmt.Println(\"Enter your lines (press Ctrl+D to finish):\")\n\tvar line string\n\tfmt.Scan(&line)\n\n\tif len(line) >", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5303, "local_pairs": 5577, "layers_remote": 1056, "remote_wait_s": 2.3132648391183466, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 24256, "exec_wall_s": 54.527921771397814, "bytes_sent": 127773277, "bytes_recv": 492924006}}}}
72 +{"backend": "titan-distributed", "capacity": 64, "placement": "odd", "workers": ["w0"], "rtt_ms": {"w0": [0.5988340126350522, 0.534667051397264, 0.5446670111268759, 0.547999981790781, 0.5813329480588436, 0.5235420539975166, 0.634583062492311, 0.5202499451115727, 0.6310830358415842, 0.5090000340715051]}, "n_remote_experts": 512, "predictor": "none", "m": 8, "max_tokens": 64, "split": "heldout", "tag": "", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_distributed.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --workers w0=192.168.2.85:9411 --placement odd --capacity 64 --repeats 2 --out benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl", "env": {"host": "MacStudnpierre2", "chip": "Apple M3 Ultra", "ram_gb": 96, "macos": "26.6", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": ""}, "seed": 0, "dlogit_vs_mlx_lm": 0.0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Write a short motivational note for someone starting to lear", "ts": "2026-11-01T01:59:11", "prompt_tokens": 16, "gen_tokens": 64, "ttft_ms": 314.44320792797953, "prompt_tps": 51.12232824472741, "gen_tps": 16.591369647323315, "decode_tps_wall": 16.33231737588295, "itl_ms_p50": 60.72887510526925, "itl_ms_p95": 69.88466694019735, "itl_ms_p99": 74.59004106931388, "itl_ms_mean": 61.22829828647867, "total_s": 4.171825999976136, "peak_mem_gb": 3.92905152, "text": "\n\nDear [Name],\n\nWelcome to the wonderful world of piano playing! You've made an exciting decision to learn this beautiful instrument, and I'm thrilled to be a part of your journey.\n\nAs an adult, you b", "cache_hit_rate": 1.0, "cache_misses": 0, "stall_s": 0.0, "remote": {"remote_pairs": 5268, "local_pairs": 4972, "layers_remote": 1053, "remote_wait_s": 2.3272019665455446, "failures": 0, "workers": {"w0": {"worker": "w0", "exec_calls": 25309, "exec_wall_s": 56.893484837026335, "bytes_sent": 133139713, "bytes_recv": 514605555}}}}
added benchmarks/raw/tests/distributed_localhost_olmoe_20261101T010211.json +56 −0
@@ -0,0 +1,56 @@
1 +{
2 + "rtt_ms_localhost": [
3 + 0.08658308070152998,
4 + 0.07870804984122515,
5 + 0.07054198067635298,
6 + 0.06937491707503796,
7 + 0.0678749056532979
8 + ],
9 + "prompts": [
10 + {
11 + "prompt": "The capital of Canada is",
12 + "max_abs_dlogit": 0.0,
13 + "argmax_equal": true,
14 + "remote": {
15 + "remote_pairs": 312,
16 + "local_pairs": 328,
17 + "layers_remote": 16,
18 + "remote_wait_s": 0.0033775841584429145,
19 + "failures": 0,
20 + "workers": {
21 + "w0": {
22 + "worker": "w0",
23 + "exec_calls": 16,
24 + "exec_wall_s": 0.04540304176043719,
25 + "bytes_sent": 333317,
26 + "bytes_recv": 1281714
27 + }
28 + }
29 + }
30 + },
31 + {
32 + "prompt": "def fibonacci(n):\n \"\"\"Retur",
33 + "max_abs_dlogit": 0.0,
34 + "argmax_equal": true,
35 + "remote": {
36 + "remote_pairs": 1660,
37 + "local_pairs": 1668,
38 + "layers_remote": 56,
39 + "remote_wait_s": 0.011338463518768549,
40 + "failures": 0,
41 + "workers": {
42 + "w0": {
43 + "worker": "w0",
44 + "exec_calls": 56,
45 + "exec_wall_s": 0.09497450070921332,
46 + "bytes_sent": 1722764,
47 + "bytes_recv": 6807110
48 + }
49 + }
50 + }
51 + }
52 + ],
53 + "greedy_equal": true,
54 + "greedy_ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is",
55 + "greedy_titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is"
56 +}
\ No newline at end of file
added benchmarks/raw/tests/greedy_olmoe_cap16_20261101T010217.json +14 −0
@@ -0,0 +1,14 @@
1 +[
2 + {
3 + "prompt": "The capital of Canada is",
4 + "ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
5 + "titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
6 + "equal": true
7 + },
8 + {
9 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
10 + "ref": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
11 + "titan": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
12 + "equal": true
13 + }
14 +]
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap16_2026-11-01T010212.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 16,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.2580101180438449,
10 + "misses": 440,
11 + "stall_s": 0.059322625398635864,
12 + "layers_routed": 46
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.42512908777969016,
20 + "misses": 1002,
21 + "stall_s": 0.13415837637148798,
22 + "layers_routed": 127
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.41904344010530936,
30 + "misses": 1324,
31 + "stall_s": 0.17012629297096282,
32 + "layers_routed": 161
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.45278969957081544,
40 + "misses": 1275,
41 + "stall_s": 0.16569862724281847,
42 + "layers_routed": 164
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:02:12",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap32_2026-11-01T010212.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 32,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.0,
10 + "misses": 430,
11 + "stall_s": 0.05754941806662828,
12 + "layers_routed": 16
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.6060606060606061,
20 + "misses": 429,
21 + "stall_s": 0.058471832890063524,
22 + "layers_routed": 40
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.6090458488228006,
30 + "misses": 631,
31 + "stall_s": 0.0811612062389031,
32 + "layers_routed": 58
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.6224188790560472,
40 + "misses": 512,
41 + "stall_s": 0.06696591933723539,
42 + "layers_routed": 49
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:02:12",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap64_2026-11-01T010211.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 64,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.0,
10 + "misses": 430,
11 + "stall_s": 0.05780649953521788,
12 + "layers_routed": 16
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.4637904468412943,
20 + "misses": 348,
21 + "stall_s": 0.046038166736252606,
22 + "layers_routed": 16
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.823076923076923,
30 + "misses": 138,
31 + "stall_s": 0.02237904106732458,
32 + "layers_routed": 16
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.9234828496042217,
40 + "misses": 58,
41 + "stall_s": 0.010341043351218104,
42 + "layers_routed": 16
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:02:11",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap8_2026-11-01T010214.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 8,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.1625,
10 + "misses": 536,
11 + "stall_s": 0.07156783423852175,
12 + "layers_routed": 80
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.4425981873111782,
20 + "misses": 1476,
21 + "stall_s": 0.20007904060184956,
22 + "layers_routed": 331
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.375,
30 + "misses": 1995,
31 + "stall_s": 0.27012933895457536,
32 + "layers_routed": 399
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.3878048780487805,
40 + "misses": 2008,
41 + "stall_s": 0.27310475008562207,
42 + "layers_routed": 410
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:02:14",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
modified benchmarks/scripts/bench_decode.py +1 −1
@@ -107,7 +107,7 @@ def main():
107 107 if tm:
108 108 cs = tm.cache_stats()
109 109 rec.update(cache_hit_rate=cs["hit_rate"], cache_misses=cs["misses"], cache_hits=cs["hits"], stall_s=cs["stall_time_s"],
110 − hits_device=sum(p["hits_device"] for p in cs["layers"]), layers_complete=sum(1 for t in tm.tables if t.complete()), prefetch=cs["prefetch"],
110 + hits_device=sum(p["hits_device"] for p in cs["layers"]), layers_complete=sum(1 for t in tm.tables if t.complete()), prefetch=cs["prefetch"], host_time_s=cs["host_time_s"],
111 111 ssd_bytes=cs["store"]["bytes_read"], ssd_read_s=cs["store"]["read_time_s"])
112 112 f.write(json.dumps(rec) + "\n"); f.flush()
113 113 print(f"{a.backend} cap={a.capacity} {a.predictor} rep={rep} {fam:12s} ttft={r['ttft_ms']:.0f}ms prefill={r['prompt_tps']:.0f} decode={r['gen_tps']:.1f} tok/s"
added benchmarks/scripts/profile_layer.py +60 −0
@@ -0,0 +1,60 @@
1 +#!/usr/bin/env python3
2 +"""Gate D profiling: where does a decode step spend its time with all experts resident? Times (synchronous, median of N) the
3 +full step, one layer's attention, one layer's MoE block (3 gather_qmm + swiglu, top-8), and the three gather_qmm kernels alone,
4 +and computes the effective bytes moved per second against the expert bytes touched (bandwidth-boundedness). Writes JSON.
5 +"""
6 +import argparse, json, os, platform, socket, statistics, subprocess, sys, time
7 +import mlx.core as mx
8 +from pathlib import Path
9 +sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
10 +from mlx_lm.utils import load_model, load_tokenizer
11 +from mlx_lm.models.cache import make_prompt_cache
12 +from mlx_lm.models.activations import swiglu
13 +from titan.registry import load_spec
14 +
15 +def timeit(f, n=100, warm=10):
16 + for _ in range(warm): f()
17 + mx.synchronize(); ts = []
18 + for _ in range(n):
19 + t = time.perf_counter(); f(); mx.synchronize(); ts.append((time.perf_counter() - t) * 1e3)
20 + return {"median_ms": statistics.median(ts), "p10_ms": sorted(ts)[n // 10], "p90_ms": sorted(ts)[9 * n // 10]}
21 +
22 +def main():
23 + ap = argparse.ArgumentParser(); ap.add_argument("--model", required=True); ap.add_argument("--out", required=True); ap.add_argument("--n", type=int, default=100)
24 + a = ap.parse_args()
25 + spec = load_spec(a.model); model, cfg = load_model(Path(a.model)); tok = load_tokenizer(Path(a.model))
26 + L, E, K, D = spec.num_layers, spec.num_experts, spec.top_k, spec.hidden_size
27 + ids = mx.array(tok.encode("The capital of Canada is"))[None]
28 + cache = make_prompt_cache(model); y = mx.argmax(model(ids, cache=cache)[:, -1], -1); mx.eval(y)
29 + rec = {"host": socket.gethostname(), "chip": subprocess.run(["sysctl", "-n", "machdep.cpu.brand_string"], capture_output=True, text=True).stdout.strip(),
30 + "macos": platform.mac_ver()[0], "mlx": mx.__version__, "model": os.path.realpath(a.model), "L": L, "E": E, "K": K, "D": D,
31 + "bytes_per_expert": spec.bytes_per_expert, "expert_bytes_per_token": L * K * spec.bytes_per_expert, "n": a.n, "cmd": " ".join(sys.argv)}
32 + # 1. full decode step (sync)
33 + def step():
34 + out = model(y[None], cache=cache); mx.eval(out)
35 + rec["decode_step"] = timeit(step, a.n)
36 + # 2. one layer's blocks in isolation (layer 4), on a realistic hidden state
37 + layer = model.model.layers[4]
38 + h = mx.random.normal((1, 1, D)).astype(mx.float16); mx.eval(h)
39 + from mlx_lm.models.base import create_attention_mask
40 + c2 = make_prompt_cache(model); model(ids, cache=c2); mx.eval(c2[4].state)
41 + rec["attention_layer"] = timeit(lambda: mx.eval(layer.self_attn(layer.input_layernorm(h), None, c2[4])), a.n)
42 + rec["moe_block_layer"] = timeit(lambda: mx.eval(layer.mlp(layer.post_attention_layernorm(h))), a.n)
43 + sw = layer.mlp.switch_mlp
44 + xf = h.reshape(-1, D); idx = mx.array([[list(range(K))]], dtype=mx.uint32)
45 + xe = mx.expand_dims(xf, (-2, -3))
46 + def qmm3():
47 + up = sw.up_proj(xe, idx); g = sw.gate_proj(xe, idx); d = sw.down_proj(swiglu(g, up), idx); mx.eval(d)
48 + rec["gather_qmm_x3_layer"] = timeit(qmm3, a.n)
49 + rec["gather_qmm_up_only"] = timeit(lambda: mx.eval(sw.up_proj(xe, idx)), a.n)
50 + rec["router_layer"] = timeit(lambda: mx.eval(mx.argpartition(-mx.softmax(layer.mlp.gate(xf), axis=1, precise=True), kth=K - 1, axis=-1)[..., :K]), a.n)
51 + # 3. derived
52 + t_step = rec["decode_step"]["median_ms"]; t_moe = rec["moe_block_layer"]["median_ms"] * L; t_att = rec["attention_layer"]["median_ms"] * L
53 + rec["derived"] = {"moe_fraction_of_step_est": t_moe / t_step, "attention_fraction_est": t_att / t_step,
54 + "expert_GBps_effective_in_moe_block": rec["expert_bytes_per_token"] / (t_moe / 1e3) / 1e9,
55 + "expert_GBps_effective_full_step": rec["expert_bytes_per_token"] / (t_step / 1e3) / 1e9,
56 + "tok_s_sync": 1e3 / t_step}
57 + os.makedirs(os.path.dirname(a.out), exist_ok=True); json.dump(rec, open(a.out, "w"), indent=2); print(json.dumps(rec, indent=1))
58 +
59 +if __name__ == "__main__":
60 + main()
modified figures/capacity_sweep.pdf +0 −0

Binary file not shown.

modified figures/sim_hit_vs_capacity.pdf +0 −0

Binary file not shown.

modified figures/two_node.pdf +0 −0

Binary file not shown.

modified paper/figures/capacity_sweep.pdf +0 −0

Binary file not shown.

modified paper/figures/sim_hit_vs_capacity.pdf +0 −0

Binary file not shown.

modified paper/figures/two_node.pdf +0 −0

Binary file not shown.

modified paper/tables/cold.tex +12 −1
@@ -6,7 +6,18 @@
6 6 \toprule
7 7 Backend & cfg & $n$ & decode tok/s (median) & p95 ITL ms & hit & stall s & TTFT ms \\
8 8 \midrule
9 −titan & C=16, none, m=8, cold & 1 & 9.9 & 139.6 & 0.490 & 7.89 & 2849 \\
9 +titan & C=16, none, m=8, cold & 12 & 36.4 & 35.2 & 0.526 & 1.15 & 371 \\
10 +titan & C=16, prev\_rank, m=16, cold & 12 & 19.1 & 62.2 & 0.647 & 1.69 & 467 \\
11 +titan & C=16, transition, m=8, cold & 12 & 27.9 & 45.2 & 0.670 & 1.24 & 446 \\
12 +titan & C=16, transition2, m=16, cold & 12 & 25.9 & 50.8 & 0.736 & 1.21 & 423 \\
13 +titan & C=16, transition2, m=8, cold & 12 & 35.0 & 35.7 & 0.654 & 0.77 & 339 \\
14 +titan & C=16, transition3, m=8, cold & 12 & 29.1 & 43.4 & 0.639 & 1.17 & 393 \\
15 +titan & C=32, none, m=8, cold & 12 & 50.6 & 25.9 & 0.788 & 0.62 & 236 \\
16 +titan & C=32, prev\_rank, m=16, cold & 12 & 28.4 & 45.7 & 0.785 & 0.91 & 263 \\
17 +titan & C=32, transition, m=8, cold & 12 & 41.3 & 30.4 & 0.825 & 0.63 & 249 \\
18 +titan & C=32, transition2, m=16, cold & 12 & 41.0 & 32.3 & 0.848 & 0.56 & 243 \\
19 +titan & C=32, transition2, m=8, cold & 12 & 42.0 & 30.3 & 0.820 & 0.59 & 239 \\
20 +titan & C=32, transition3, m=8, cold & 12 & 41.0 & 31.2 & 0.813 & 0.60 & 242 \\
10 21 \bottomrule
11 22 \end{tabular}
12 23 \begin{tablenotes}\footnotesize
modified research/DECISIONS.md +6 −0
@@ -60,3 +60,9 @@ generated once from the checkpoint by `scripts/convert_layout.py`; the source ch
60 60 source of dense weights. Rationale: measured 9 scattered reads per expert in the HF safetensors (expert span 1.5 GB) → cold
61 61 16-slot decode 8–10 tok/s; with parallel reads 18; with the contiguous layout 36 tok/s (4.5× over v1). Colibri performs an
62 62 equivalent conversion to its own container; we keep safetensors for everything else so mlx-lm remains the exact reference.
63 +
64 +## D-010 · 2026-11-01 · H1 verdict for the small model and the remote tier as capacity only
65 +Decision: report H1 as *not supported* for OLMoE on the laptop NVMe tier (transition prefetch at depth 2: +0.14 hit rate, −4 to −15 %
66 +throughput; mechanism = install cost on the critical path × 50 % precision) and keep the prefetcher as an instrumented component;
67 +report the remote tier as a capacity mechanism (2.5 ms per layer over 1 GbE vs 1.2 ms per layer of cold-NVMe misses). Both verdicts
68 +are limited to the measured regime (small model, experts fit on local NVMe, 1 GbE) and the paper says so.
modified research/LOG.md +19 −0
@@ -111,3 +111,22 @@ Chronological, append-only. Times are local (America/Toronto). Every entry names
111 111 than a cold-NVMe demand miss (≈ 1.2 ms per layer at 16 slots) and far more than local compute (0.13 ms): as the formulation
112 112 predicts, the remote tier cannot be a latency win for a model whose experts fit on local NVMe; it can only add *capacity*.
113 113 Worker-side cost breakdown to be measured once the sweep finishes.
114 +- **Cold-NVMe sweep v3 complete** (`benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl`, 12 runs per cell, held-out general/code/reasoning,
115 + 64 tokens, purge + F_NOCACHE, 4 readers, contiguous layout): 16 slots — none **36.4 tok/s** (hit 0.53, 267 MB/token from SSD),
116 + transition d=1 27.9, **d=2 35.0** (hit 0.65, stall −33 %: 0.77 s vs 1.15 s per 64 tokens), d=3 29.1, transition2 m=16 25.9,
117 + prev_rank m=16 19.1; 32 slots — none **50.6**, transition 41.3, transition2 42.0, transition2 m=16 41.0, transition3 41.0, prev_rank 28.4.
118 +- **Host-time attribution** (`attribution.jsonl`, per token, 16 slots, cold): none = sync 14.7 ms (GPU graph incl. the installs of missed
119 + experts) + reads 19.9 ms; transition2 = sync 20.4 + prefetch bookkeeping 1.9 + reads 18.8 ms with 58 prefetches/token (1.8 late).
120 + Confidence thresholds (`transition2t50`, `t70`: 54 and 31 prefetches/token) leave sync at 20.0–20.3 ms. Mechanism: every installed
121 + expert costs ≈ 9 lazy slice writes (≈ 0.1 ms of graph work on the critical path); with ≈ 50 % prefetch precision the installs double
122 + while demand reads fall only 30 % → **H1 not supported on OLMoE/laptop NVMe**: hit +0.14, throughput −4 to −15 %. The oracle
123 + (hit 0.94) shows the ceiling; the lever is install cost per expert and prediction precision, not lead time (d = 2 already suffices).
124 +- **Gate D profiling** on M4M64a (`benchmarks/raw/gateD/profile_olmoe_m4m64a.json`, `launch_overhead_m4m64a.txt`): decode step 2.99 ms
125 + sync (334 tok/s), effective 151 GB/s on 452 MB of expert bytes per token (≈ 28 % of the M4 Max's nominal bandwidth); isolated
126 + per-layer timings are dominated by the ≈ 0.15–0.17 ms synchronisation floor; in-graph elementwise ops cost 4.7–7.2 µs each; a
127 + `gather_qmm` for k = 1…16 experts takes the same 0.17 ms (latency-bound), 250 GB/s only at k = 64. Decode at batch 1 is therefore
128 + launch/latency-bound: fusing up+gate into one gather_qmm saves one dispatch per layer (≈ 16 × ~10 µs per token, ≤ 5 %); larger
129 + gains need token batching, not kernels. H4 is tested only with the cheap fusion (next entry).
130 +- **Remote cost breakdown** M3U96a→M3U96b (`benchmarks/raw/gateE/remote_cost_breakdown_m3u96a_m3u96b.txt`): application ping 0.74–1.17 ms;
131 + exec round trip 2.09 ms (1 pair), 1.99 (4), 2.59 (8 pairs = one decode layer), 7.27 (8 rows × 64 pairs), 19.1 ms (32 rows × 256 pairs,
132 + 1 MB returned ≈ 9 ms on the wire) → ≈ 1.2–1.7 ms of worker-side overhead per call on top of the RTT.
modified titan/moe.py +10 −0
@@ -9,6 +9,7 @@ from __future__ import annotations
9 9
10 10 from typing import Callable, List, Optional
11 11
12 +import time
12 13 import mlx.core as mx
13 14 import numpy as np
14 15 from mlx_lm.models.activations import swiglu
@@ -25,6 +26,7 @@ class TitanSwitchGLU:
25 26 self.prefetcher = prefetcher
26 27 self.current_probs = None
27 28 self.remote = None # RemoteExecutor (Gate E); when set, experts placed on workers are executed remotely
29 + self.t_sync = 0.0; self.t_prefetch = 0.0; self.t_ensure = 0.0; self.t_build = 0.0 # per-layer host-time attribution (seconds)
28 30 self.group_size = group_size
29 31 self.bits = bits
30 32 self.on_route = on_route # hook(layer, unique_experts) used by predictors/prefetchers (Gate C)
@@ -49,15 +51,21 @@ class TitanSwitchGLU:
49 51 self.on_route(self.table.layer, uniq)
50 52 pf = self.prefetcher
51 53 if pf is not None:
54 + _t = time.perf_counter()
52 55 pf.before_layer(self.table.layer, uniq) # install completed prefetches (wait for late ones we need)
53 56 pf.after_layer(self.table.layer, uniq, probs_np) # issue prefetch for the next layer BEFORE this layer's compute
57 + self.t_prefetch += time.perf_counter() - _t
58 + _t = time.perf_counter()
54 59 slot_of = dict(zip(uniq, self.table.ensure(uniq)))
60 + self.t_ensure += time.perf_counter() - _t
61 + _t = time.perf_counter()
55 62 slot_idx = mx.array([slot_of[e] for e in flat], dtype=mx.uint32).reshape(shape)
56 63 x = mx.expand_dims(x, (-2, -3))
57 64 x_up = self._qmm(x, "up", slot_idx, False)
58 65 x_gate = self._qmm(x, "gate", slot_idx, False)
59 66 h = swiglu(x_gate, x_up) # identical compiled activation to mlx-lm SwitchGLU
60 67 y = self._qmm(h, "down", slot_idx, False)
68 + self.t_build += time.perf_counter() - _t
61 69 return y.squeeze(-2)
62 70
63 71 def __call__(self, x: mx.array, indices: mx.array) -> mx.array:
@@ -69,11 +77,13 @@ class TitanSwitchGLU:
69 77 t.hits_device += indices.size
70 78 return self._run_device(x, t.slot_map[indices])
71 79 # Materialise the router's choice: this is the unavoidable host sync of any demand-loading design.
80 + _t0 = time.perf_counter()
72 81 probs_np = None
73 82 if self.prefetcher is not None and self.current_probs is not None:
74 83 mx.eval(indices, self.current_probs)
75 84 probs_np = np.array(self.current_probs[-1]) # last row: decode token (prefill rows are not used for prediction)
76 85 flat = indices.reshape(-1).tolist()
86 + self.t_sync += time.perf_counter() - _t0
77 87 self.last_indices = flat
78 88 k = indices.shape[-1]
79 89 n_rows = len(flat) // k
modified titan/predictors.py +8 −3
@@ -54,10 +54,11 @@ class Transition(Predictor):
54 54 name = "transition"
55 55 horizon = "layer"
56 56
57 − def __init__(self, L, E, depth=1):
57 + def __init__(self, L, E, depth=1, threshold=0.0):
58 58 self.M = np.zeros((L, E, E), dtype=np.float64)
59 59 self.depth = depth
60 − self.name = "transition" if depth == 1 else f"transition{depth}"
60 + self.threshold = threshold # keep only experts whose score >= threshold * max score (precision/recall knob)
61 + self.name = ("transition" if depth == 1 else f"transition{depth}") + (f"t{int(threshold*100)}" if threshold else "")
61 62
62 63 def fit(self, traces):
63 64 d = self.depth
@@ -73,7 +74,10 @@ class Transition(Predictor):
73 74 if l < d or (l - d) not in cur_routes:
74 75 return list(prev_topk[l][:m]) if prev_topk is not None else []
75 76 s = self.M[l][list(cur_routes[l - d])].sum(0)
76 − return list(np.argsort(-s)[:m])
77 + order = np.argsort(-s)[:m]
78 + if self.threshold:
79 + order = [e for e in order if s[e] >= self.threshold * s[order[0]]]
80 + return list(order)
77 81
78 82
79 83 class Probe(Predictor):
@@ -121,5 +125,6 @@ class Oracle(Predictor):
121 125 def make(name, L, E):
122 126 reg = {"none": Predictor, "prev_token": PrevToken, "prev_rank": PrevTokenRank, "transition": lambda: Transition(L, E),
123 127 "transition2": lambda: Transition(L, E, 2), "transition3": lambda: Transition(L, E, 3), "transition4": lambda: Transition(L, E, 4),
128 + "transition2t50": lambda: Transition(L, E, 2, 0.5), "transition2t70": lambda: Transition(L, E, 2, 0.7), "transition1t50": lambda: Transition(L, E, 1, 0.5),
124 129 "probe": lambda: Probe(L, E), "oracle": Oracle}
125 130 return reg[name]()
modified titan/runtime.py +5 −1
@@ -94,11 +94,15 @@ class TitanModel:
94 94 misses = sum(p["misses"] for p in per)
95 95 return {"layers": per, "hits": hits, "misses": misses,
96 96 "hit_rate": hits / (hits + misses) if hits + misses else None,
97 − "stall_time_s": sum(p["stall_time_s"] for p in per), "store": self.store.stats(), "prefetch": per_pf}
97 + "stall_time_s": sum(p["stall_time_s"] for p in per), "store": self.store.stats(), "prefetch": per_pf,
98 + "host_time_s": {"sync": sum(b.t_sync for b in self.blocks), "prefetch": sum(b.t_prefetch for b in self.blocks),
99 + "ensure": sum(b.t_ensure for b in self.blocks), "build": sum(b.t_build for b in self.blocks)}}
98 100
99 101 def reset_stats(self):
100 102 for t in self.tables:
101 103 t.reset_stats()
104 + for b in self.blocks:
105 + b.t_sync = b.t_prefetch = b.t_ensure = b.t_build = 0.0
102 106 self.store.reset_stats()
103 107 if self.prefetcher is not None:
104 108 self.prefetcher.reset_stats()
105 109
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere