SPB Git
spboucher.ai

spb/wp33_uqo

Public MIT

Gate E: authenticated TCP transport, expert worker, remote executor with local fallback (localhost bit-identical), distributed bench, power sampler, LOG

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 1, 2026) parent 2574612 Browse files

17 changed files +869 −0

17 changed files
Unified Split
modified benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl +107 −0
@@ -5,3 +5,110 @@
5 5 {"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:50:57", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 420.1389999361709, "prompt_tps": 69.07719609252912, "gen_tps": 37.35810759487954, "decode_tps_wall": 36.77462865794749, "itl_ms_p50": 28.77229091245681, "itl_ms_p95": 33.97729201242328, "itl_ms_p99": 35.62474995851517, "itl_ms_mean": 27.192660714573567, "total_s": 2.1332766249543056, "peak_mem_gb": 1.905393932, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.5451856093138658, "cache_misses": 4766, "cache_hits": 5713, "stall_s": 1.1352890465641394, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 16866607104, "ssd_read_s": 2.85799129109364}
6 6 {"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:50:58", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 325.22700005210936, "prompt_tps": 58.47830967620645, "gen_tps": 44.22672637332204, "decode_tps_wall": 43.53605355216225, "itl_ms_p50": 21.73804200720042, "itl_ms_p95": 29.404125059954822, "itl_ms_p99": 31.615916988812387, "itl_ms_mean": 22.96946825466991, "total_s": 1.7723035000963137, "peak_mem_gb": 1.773794588, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.6699147381242387, "cache_misses": 3252, "cache_hits": 6600, "stall_s": 0.8593332751188427, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 11508645888, "ssd_read_s": 1.9043039631797}
7 7 {"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:51:01", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 958.3374169887975, "prompt_tps": 33.402911106697594, "gen_tps": 33.07673168156339, "decode_tps_wall": 32.560083043862754, "itl_ms_p50": 29.210791923105717, "itl_ms_p95": 35.54279205854982, "itl_ms_p99": 116.7389169568196, "itl_ms_mean": 30.712452380814486, "total_s": 2.89322191698011, "peak_mem_gb": 2.094958152, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.49030716119323714, "cache_misses": 5758, "cache_hits": 5539, "stall_s": 1.7997499047778547, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 20377239552, "ssd_read_s": 4.840675379149616}
8 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:51:03", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 368.4726250357926, "prompt_tps": 48.89591995154156, "gen_tps": 35.25698277643246, "decode_tps_wall": 34.70629636717931, "itl_ms_p50": 28.44312507659197, "itl_ms_p95": 35.80216702539474, "itl_ms_p99": 36.164957913570106, "itl_ms_mean": 28.81321560273627, "total_s": 2.1837052080081776, "peak_mem_gb": 1.839953, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.5069291022293633, "cache_misses": 4910, "cache_hits": 5048, "stall_s": 1.1720648780465126, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17376215040, "ssd_read_s": 2.986359332106076}
9 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:51:06", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 335.31462494283915, "prompt_tps": 41.79292133374972, "gen_tps": 34.24103356449932, "decode_tps_wall": 33.70621428467988, "itl_ms_p50": 29.802417033351958, "itl_ms_p95": 36.68737504631281, "itl_ms_p99": 40.050665964372456, "itl_ms_mean": 29.66811969905855, "total_s": 2.204406165983528, "peak_mem_gb": 1.747773596, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.4795243019648397, "cache_misses": 5033, "cache_hits": 4637, "stall_s": 1.1822265810333192, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17811505152, "ssd_read_s": 3.05972489528358}
10 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:51:08", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 418.8025000039488, "prompt_tps": 69.29881355560856, "gen_tps": 37.537740267744965, "decode_tps_wall": 36.95148489956471, "itl_ms_p50": 28.28020800370723, "itl_ms_p95": 33.74383400660008, "itl_ms_p99": 34.99762492720038, "itl_ms_mean": 27.062511904948643, "total_s": 2.1237407500157133, "peak_mem_gb": 1.905393956, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.5457581830327322, "cache_misses": 4760, "cache_hits": 5719, "stall_s": 1.127805772703141, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 16845373440, "ssd_read_s": 2.838174633914605}
11 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:51:10", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 374.44504199083894, "prompt_tps": 64.15261188613987, "gen_tps": 42.52224456988057, "decode_tps_wall": 41.858079039022705, "itl_ms_p50": 24.035749956965446, "itl_ms_p95": 30.22029099520296, "itl_ms_p99": 36.48670797701925, "itl_ms_mean": 23.890250650722358, "total_s": 1.8795308329863474, "peak_mem_gb": 1.925774224, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.6497722647543367, "cache_misses": 3614, "cache_hits": 6705, "stall_s": 0.9495816181879491, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 12789743616, "ssd_read_s": 2.1544746939325705}
12 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:51:11", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 316.6579579701647, "prompt_tps": 60.06138905385063, "gen_tps": 43.74587372657144, "decode_tps_wall": 43.062635113920706, "itl_ms_p50": 21.923959022387862, "itl_ms_p95": 30.937374918721616, "itl_ms_p99": 31.97908296715468, "itl_ms_mean": 23.221988096049735, "total_s": 1.779643208021298, "peak_mem_gb": 1.783100776, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.6759033698741372, "cache_misses": 3193, "cache_hits": 6659, "stall_s": 0.8607746257912368, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 11299848192, "ssd_read_s": 1.8981057128403336}
13 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:51:14", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 416.65312508121133, "prompt_tps": 69.65812977727899, "gen_tps": 37.280073973973195, "decode_tps_wall": 36.69780081939727, "itl_ms_p50": 28.502625063993037, "itl_ms_p95": 34.91549997124821, "itl_ms_p99": 36.375541938468814, "itl_ms_mean": 27.249589285236738, "total_s": 2.133377250051126, "peak_mem_gb": 1.903411428, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.5446130355949995, "cache_misses": 4772, "cache_hits": 5707, "stall_s": 1.1383883256930858, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 16887840768, "ssd_read_s": 2.8613588492153212}
14 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:51:15", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 341.64862509351224, "prompt_tps": 55.65981600473538, "gen_tps": 43.30100596604308, "decode_tps_wall": 42.624710140036576, "itl_ms_p50": 22.45845808647573, "itl_ms_p95": 29.735749936662614, "itl_ms_p99": 32.036874908953905, "itl_ms_mean": 23.460570094545208, "total_s": 1.8196645410498604, "peak_mem_gb": 1.773794588, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.6699147381242387, "cache_misses": 3252, "cache_hits": 6600, "stall_s": 0.8717616148060188, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 11508645888, "ssd_read_s": 1.923264330253005}
15 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:51:18", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 362.4877498950809, "prompt_tps": 49.698369966583876, "gen_tps": 35.44949672632509, "decode_tps_wall": 34.895773922080814, "itl_ms_p50": 28.025666950270534, "itl_ms_p95": 36.12537495791912, "itl_ms_p99": 37.00979193672538, "itl_ms_mean": 28.65676520695348, "total_s": 2.1678639579331502, "peak_mem_gb": 1.843491944, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.5079333199437638, "cache_misses": 4900, "cache_hits": 5058, "stall_s": 1.1607553774956614, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17340825600, "ssd_read_s": 2.963632360799238}
16 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:51:20", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 401.48825000505894, "prompt_tps": 59.82431586713207, "gen_tps": 41.56776955586064, "decode_tps_wall": 40.91847359488166, "itl_ms_p50": 23.810542072169483, "itl_ms_p95": 29.90987489465624, "itl_ms_p99": 36.111374967731535, "itl_ms_mean": 24.438839285664024, "total_s": 1.9411351250018924, "peak_mem_gb": 1.934097296, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.6497722647543367, "cache_misses": 3614, "cache_hits": 6705, "stall_s": 0.9638368771411479, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 12789743616, "ssd_read_s": 2.177788548753597}
17 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:51:22", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 334.01695801876485, "prompt_tps": 41.95431694931566, "gen_tps": 33.55691134043318, "decode_tps_wall": 33.0327910023831, "itl_ms_p50": 30.482040951028466, "itl_ms_p95": 37.04166691750288, "itl_ms_p99": 41.51424998417497, "itl_ms_mean": 30.272949080441204, "total_s": 2.241212750086561, "peak_mem_gb": 1.742661784, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.48355739400206826, "cache_misses": 4994, "cache_hits": 4676, "stall_s": 1.1990491144824773, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17673486336, "ssd_read_s": 3.0536379632540047}
18 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:51:24", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 575.3412500489503, "prompt_tps": 55.65209726540678, "gen_tps": 34.034027323611205, "decode_tps_wall": 33.502500274535386, "itl_ms_p50": 29.327333089895546, "itl_ms_p95": 36.18929197546095, "itl_ms_p99": 44.5929579436779, "itl_ms_mean": 29.848518522663248, "total_s": 2.455797916976735, "peak_mem_gb": 2.079360592, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.49048419934495885, "cache_misses": 5756, "cache_hits": 5541, "stall_s": 1.38695521466434, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 20370161664, "ssd_read_s": 3.5395298274233937}
19 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:51:33", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 772.1267499728128, "prompt_tps": 41.608416670316196, "gen_tps": 28.47543362667651, "decode_tps_wall": 28.03176881410257, "itl_ms_p50": 34.94879195932299, "itl_ms_p95": 42.67483309376985, "itl_ms_p99": 57.45962506625801, "itl_ms_mean": 35.67381019127511, "total_s": 3.0195767920231447, "peak_mem_gb": 2.160502028, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.6128175621846508, "cache_misses": 4374, "cache_hits": 6923, "stall_s": 1.482811972266063, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3581, "installed": 3580, "late": 46, "dup": 5803, "predictor": "transition", "m": 8}, "ssd_bytes": 28152299520, "ssd_read_s": 5.722053766134195}
20 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:51:35", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 408.1629170104861, "prompt_tps": 58.852348604539685, "gen_tps": 35.390164693827245, "decode_tps_wall": 34.83739886083086, "itl_ms_p50": 28.134417021647096, "itl_ms_p95": 37.53758396487683, "itl_ms_p99": 40.30858399346471, "itl_ms_mean": 28.704783729543642, "total_s": 2.2165642919717357, "peak_mem_gb": 1.9614018, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7241011725942437, "cache_misses": 2847, "cache_hits": 7472, "stall_s": 0.9345974458847195, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1557, "installed": 1558, "late": 20, "dup": 7339, "predictor": "transition", "m": 8}, "ssd_bytes": 15585509376, "ssd_read_s": 2.9250047012465075}
21 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:51:38", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 378.2986250007525, "prompt_tps": 37.04016045882731, "gen_tps": 27.031827230294134, "decode_tps_wall": 26.609577156162334, "itl_ms_p50": 37.20854106359184, "itl_ms_p95": 44.75649993401021, "itl_ms_p99": 47.99770808313042, "itl_ms_mean": 37.58045436540944, "total_s": 2.745867250021547, "peak_mem_gb": 1.789864856, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6477766287487073, "cache_misses": 3406, "cache_hits": 6264, "stall_s": 1.2254173471592367, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4572, "installed": 4565, "late": 112, "dup": 3980, "predictor": "transition", "m": 8}, "ssd_bytes": 28233695232, "ssd_read_s": 5.215782183106057}
22 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:51:40", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 395.0674580410123, "prompt_tps": 45.60141609241971, "gen_tps": 28.743164707500142, "decode_tps_wall": 28.294193060592328, "itl_ms_p50": 35.50791600719094, "itl_ms_p95": 42.84283402375877, "itl_ms_p99": 45.48420803621411, "itl_ms_mean": 35.34294114196821, "total_s": 2.6216727499850094, "peak_mem_gb": 1.89117404, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6738300863627235, "cache_misses": 3248, "cache_hits": 6710, "stall_s": 1.164204154862091, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3562, "installed": 3568, "late": 123, "dup": 5150, "predictor": "transition", "m": 8}, "ssd_bytes": 24100208640, "ssd_read_s": 4.4939437442226335}
23 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:51:43", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 459.74879094865173, "prompt_tps": 63.12747100100931, "gen_tps": 30.265578685281362, "decode_tps_wall": 29.79284338060757, "itl_ms_p50": 34.34629202820361, "itl_ms_p95": 42.33533306978643, "itl_ms_p99": 44.85229204874486, "itl_ms_mean": 33.5651078087736, "total_s": 2.5743505829013884, "peak_mem_gb": 1.962727656, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.6650443744632122, "cache_misses": 3510, "cache_hits": 6969, "stall_s": 1.1666903716977686, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2836, "installed": 2837, "late": 39, "dup": 6084, "predictor": "transition", "m": 8}, "ssd_bytes": 22458138624, "ssd_read_s": 4.246741865412332}
24 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:51:45", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 365.00933300703764, "prompt_tps": 52.09831087984164, "gen_tps": 34.83227387710879, "decode_tps_wall": 34.28816965332164, "itl_ms_p50": 27.327749994583428, "itl_ms_p95": 37.82312502153218, "itl_ms_p99": 43.85545803233981, "itl_ms_mean": 29.164578048660168, "total_s": 2.2023777500726283, "peak_mem_gb": 1.811433372, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.749187982135607, "cache_misses": 2471, "cache_hits": 7381, "stall_s": 0.8500924549298361, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1530, "installed": 1530, "late": 15, "dup": 7134, "predictor": "transition", "m": 8}, "ssd_bytes": 14159314944, "ssd_read_s": 2.5876708430005237}
25 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:51:48", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 609.2141249682754, "prompt_tps": 52.55432495384122, "gen_tps": 27.046209505045038, "decode_tps_wall": 26.623845944181046, "itl_ms_p50": 37.319499999284744, "itl_ms_p95": 44.7193750878796, "itl_ms_p99": 57.728291023522615, "itl_ms_mean": 37.56031349101769, "total_s": 2.97551387490239, "peak_mem_gb": 2.134008904, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.6143223864742852, "cache_misses": 4357, "cache_hits": 6940, "stall_s": 1.4290780789451674, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3428, "installed": 3427, "late": 59, "dup": 5956, "predictor": "transition", "m": 8}, "ssd_bytes": 27550679040, "ssd_read_s": 5.290976330987178}
26 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:51:51", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 416.3769589504227, "prompt_tps": 43.264303739765175, "gen_tps": 27.988423868178053, "decode_tps_wall": 27.55124934041564, "itl_ms_p50": 36.89020802266896, "itl_ms_p95": 44.10533409100026, "itl_ms_p99": 46.77029198501259, "itl_ms_mean": 36.29599469861696, "total_s": 2.703024624963291, "peak_mem_gb": 1.896056532, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6744326169913637, "cache_misses": 3242, "cache_hits": 6716, "stall_s": 1.2099400870501995, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3566, "installed": 3566, "late": 130, "dup": 5146, "predictor": "transition", "m": 8}, "ssd_bytes": 24093130752, "ssd_read_s": 4.651791002950631}
27 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:51:54", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 381.9474159972742, "prompt_tps": 36.684246394513124, "gen_tps": 25.94018958681683, "decode_tps_wall": 25.53496770643889, "itl_ms_p50": 38.30345894675702, "itl_ms_p95": 46.70962505042553, "itl_ms_p99": 50.887249992229044, "itl_ms_mean": 39.161984126881826, "total_s": 2.8491524159908295, "peak_mem_gb": 1.79723612, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6458117890382626, "cache_misses": 3425, "cache_hits": 6245, "stall_s": 1.2707051890902221, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4557, "installed": 4550, "late": 119, "dup": 3995, "predictor": "transition", "m": 8}, "ssd_bytes": 28247851008, "ssd_read_s": 5.366749788634479}
28 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:51:56", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 481.4576250500977, "prompt_tps": 60.28089337608887, "gen_tps": 28.671537883760458, "decode_tps_wall": 28.223687879672383, "itl_ms_p50": 35.76179197989404, "itl_ms_p95": 44.94012508075684, "itl_ms_p99": 48.35233394987881, "itl_ms_mean": 35.43123082509116, "total_s": 2.713625167030841, "peak_mem_gb": 1.956036108, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.6653306613226453, "cache_misses": 3507, "cache_hits": 6972, "stall_s": 1.233517144457437, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2849, "installed": 2857, "late": 41, "dup": 6071, "predictor": "transition", "m": 8}, "ssd_bytes": 22493528064, "ssd_read_s": 4.465901253512129}
29 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:51:59", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 442.5310830119997, "prompt_tps": 54.27066306043648, "gen_tps": 33.787125666716776, "decode_tps_wall": 33.25937302301062, "itl_ms_p50": 29.701624996960163, "itl_ms_p95": 38.44520903658122, "itl_ms_p99": 39.874125039204955, "itl_ms_mean": 30.06671230116534, "total_s": 2.336733957985416, "peak_mem_gb": 1.94344492, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7230351778273089, "cache_misses": 2858, "cache_hits": 7461, "stall_s": 0.9875611962052062, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1567, "installed": 1567, "late": 19, "dup": 7329, "predictor": "transition", "m": 8}, "ssd_bytes": 15659827200, "ssd_read_s": 3.0826816788176075}
30 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:52:01", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 367.9271249566227, "prompt_tps": 51.68762347875283, "gen_tps": 36.96160404975749, "decode_tps_wall": 36.384369678357146, "itl_ms_p50": 26.697875000536442, "itl_ms_p95": 36.270500044338405, "itl_ms_p99": 39.05266697984189, "itl_ms_mean": 27.484329365607763, "total_s": 2.099439874989912, "peak_mem_gb": 1.8149887, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.750812017864393, "cache_misses": 2455, "cache_hits": 7397, "stall_s": 0.8192729116417468, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1505, "installed": 1505, "late": 20, "dup": 7159, "predictor": "transition", "m": 8}, "ssd_bytes": 14014218240, "ssd_read_s": 2.4920683407690376}
31 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:52:04", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 453.00583308562636, "prompt_tps": 64.06463609999683, "gen_tps": 26.82671878297458, "decode_tps_wall": 26.407680900724202, "itl_ms_p50": 35.80704203341156, "itl_ms_p95": 63.98283294402063, "itl_ms_p99": 102.152582956478, "itl_ms_mean": 37.8677705081091, "total_s": 2.8386753750965, "peak_mem_gb": 1.947228324, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.6648535165569234, "cache_misses": 3512, "cache_hits": 6967, "stall_s": 1.2396353124640882, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2843, "installed": 2843, "late": 44, "dup": 6077, "predictor": "transition", "m": 8}, "ssd_bytes": 22489989120, "ssd_read_s": 4.380332345608622}
32 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:52:06", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 431.1944170622155, "prompt_tps": 44.09987648009074, "gen_tps": 28.18513959548602, "decode_tps_wall": 27.744889843902232, "itl_ms_p50": 35.49208294134587, "itl_ms_p95": 45.85787502583116, "itl_ms_p99": 49.1723750019446, "itl_ms_mean": 36.042673286006206, "total_s": 2.7018828340806067, "peak_mem_gb": 1.809139612, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.7490864799025578, "cache_misses": 2472, "cache_hits": 7380, "stall_s": 1.046185417799279, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1530, "installed": 1530, "late": 20, "dup": 7134, "predictor": "transition", "m": 8}, "ssd_bytes": 14162853888, "ssd_read_s": 3.140761493239552}
33 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:52:09", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 450.0240000197664, "prompt_tps": 40.031146461605665, "gen_tps": 26.194949735952612, "decode_tps_wall": 25.785786902003856, "itl_ms_p50": 37.66541602090001, "itl_ms_p95": 47.37641604151577, "itl_ms_p99": 52.02454107347876, "itl_ms_mean": 38.78105422186236, "total_s": 2.8932304159970954, "peak_mem_gb": 1.86305908, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6752359911628841, "cache_misses": 3234, "cache_hits": 6724, "stall_s": 1.2523563622962683, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3532, "installed": 3531, "late": 126, "dup": 5180, "predictor": "transition", "m": 8}, "ssd_bytes": 23944495104, "ssd_read_s": 4.7414319039089605}
34 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:52:12", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 449.3287919322029, "prompt_tps": 53.458818051430605, "gen_tps": 32.35508969994482, "decode_tps_wall": 31.849744706017475, "itl_ms_p50": 30.898042023181915, "itl_ms_p95": 41.62845900282264, "itl_ms_p99": 43.35575003642589, "itl_ms_mean": 31.397425920687734, "total_s": 2.42736662493553, "peak_mem_gb": 1.98237332, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7226475433666053, "cache_misses": 2862, "cache_hits": 7457, "stall_s": 0.9958835493307561, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1564, "installed": 1565, "late": 22, "dup": 7332, "predictor": "transition", "m": 8}, "ssd_bytes": 15663366144, "ssd_read_s": 3.0712371243862435}
35 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:52:15", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 391.460333019495, "prompt_tps": 35.791297177265804, "gen_tps": 25.000425221607767, "decode_tps_wall": 24.609897719801413, "itl_ms_p50": 40.51454202271998, "itl_ms_p95": 48.93241706304252, "itl_ms_p99": 52.02154198195785, "itl_ms_mean": 40.634057540003035, "total_s": 2.951405958039686, "peak_mem_gb": 1.789864856, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6473629782833505, "cache_misses": 3410, "cache_hits": 6260, "stall_s": 1.2819557231850922, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4557, "installed": 4552, "late": 119, "dup": 3995, "predictor": "transition", "m": 8}, "ssd_bytes": 28194766848, "ssd_read_s": 5.415862058289349}
36 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:52:18", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 619.5360409328714, "prompt_tps": 51.67940227960384, "gen_tps": 27.820224544948196, "decode_tps_wall": 27.385726982564265, "itl_ms_p50": 35.614707972854376, "itl_ms_p95": 45.39079195819795, "itl_ms_p99": 54.132417077198625, "itl_ms_mean": 36.515371698427884, "total_s": 2.9200044579338282, "peak_mem_gb": 2.13448616, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.6143223864742852, "cache_misses": 4357, "cache_hits": 6940, "stall_s": 1.4040574849350378, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3430, "installed": 3434, "late": 63, "dup": 5954, "predictor": "transition", "m": 8}, "ssd_bytes": 27557756928, "ssd_read_s": 5.245501340832561}
37 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:52:26", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 771.1540409363806, "prompt_tps": 41.65372223002491, "gen_tps": 27.094916876170924, "decode_tps_wall": 26.672521919988778, "itl_ms_p50": 37.9937500692904, "itl_ms_p95": 46.87191604170948, "itl_ms_p99": 64.66595910023898, "itl_ms_mean": 37.49176785756376, "total_s": 3.1331354159628972, "peak_mem_gb": 2.110217936, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.5922811365849341, "cache_misses": 4606, "cache_hits": 6691, "stall_s": 1.4938332651508972, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3016, "installed": 3016, "late": 29, "dup": 6256, "predictor": "transition2", "m": 8}, "ssd_bytes": 26973831168, "ssd_read_s": 5.189640132477507}
38 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:52:28", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 353.465290972963, "prompt_tps": 67.97521311404653, "gen_tps": 39.38169105789814, "decode_tps_wall": 38.76664831243123, "itl_ms_p50": 25.737374904565513, "itl_ms_p95": 36.12379194237292, "itl_ms_p99": 38.14187506213784, "itl_ms_mean": 25.795369048691576, "total_s": 1.9785735410405323, "peak_mem_gb": 1.811898128, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7109216009303228, "cache_misses": 2983, "cache_hits": 7336, "stall_s": 0.7793248305097222, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1337, "installed": 1337, "late": 0, "dup": 7559, "predictor": "transition2", "m": 8}, "ssd_bytes": 15288238080, "ssd_read_s": 1.9663502274779603}
39 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:52:31", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 328.02633300889283, "prompt_tps": 42.735298295847464, "gen_tps": 28.216706367450364, "decode_tps_wall": 27.77589533673794, "itl_ms_p50": 36.09183302614838, "itl_ms_p95": 46.18437506724149, "itl_ms_p99": 49.58812496624887, "itl_ms_mean": 36.0024398089283, "total_s": 2.596180040971376, "peak_mem_gb": 1.740456936, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6220268872802482, "cache_misses": 3655, "cache_hits": 6015, "stall_s": 1.1207358334213495, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4249, "installed": 4246, "late": 22, "dup": 4303, "predictor": "transition2", "m": 8}, "ssd_bytes": 27971813376, "ssd_read_s": 3.889851111569442}
40 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:52:33", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 381.94245903287083, "prompt_tps": 47.18371134498646, "gen_tps": 32.91625621517676, "decode_tps_wall": 32.402164007691304, "itl_ms_p50": 30.359124997630715, "itl_ms_p95": 41.354916989803314, "itl_ms_p99": 42.30225004721433, "itl_ms_mean": 30.862136237648507, "total_s": 2.326257042004727, "peak_mem_gb": 1.754967608, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6565575416750351, "cache_misses": 3420, "cache_hits": 6538, "stall_s": 0.9807824203744531, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3271, "installed": 3274, "late": 24, "dup": 5441, "predictor": "transition2", "m": 8}, "ssd_bytes": 23679074304, "ssd_read_s": 3.1471330774948}
41 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:52:35", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 387.1852499432862, "prompt_tps": 74.96652363611523, "gen_tps": 34.813347879089356, "decode_tps_wall": 34.26958505075751, "itl_ms_p50": 29.05166696291417, "itl_ms_p95": 37.13020891882479, "itl_ms_p99": 38.455875008367, "itl_ms_mean": 29.180394175151985, "total_s": 2.225550082977861, "peak_mem_gb": 1.912478948, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.6512071762572764, "cache_misses": 3655, "cache_hits": 6824, "stall_s": 0.7946433289907873, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2456, "installed": 2456, "late": 0, "dup": 6464, "predictor": "transition2", "m": 8}, "ssd_bytes": 21626486784, "ssd_read_s": 2.294017414445989}
42 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:52:37", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 305.5981670040637, "prompt_tps": 62.255984819796424, "gen_tps": 42.333203310080826, "decode_tps_wall": 41.67207088920581, "itl_ms_p50": 23.547915974631906, "itl_ms_p95": 29.626416973769665, "itl_ms_p99": 34.785165917128325, "itl_ms_mean": 23.9968875715036, "total_s": 1.8174020840087906, "peak_mem_gb": 1.774123092, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.7386317498984978, "cache_misses": 2575, "cache_hits": 7277, "stall_s": 0.5245005063479766, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1242, "installed": 1242, "late": 0, "dup": 7422, "predictor": "transition2", "m": 8}, "ssd_bytes": 13508149248, "ssd_read_s": 1.3246690002270043}
43 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:52:39", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 504.75587497930974, "prompt_tps": 63.43570765636268, "gen_tps": 33.63231791437441, "decode_tps_wall": 33.10700722765589, "itl_ms_p50": 29.965166002511978, "itl_ms_p95": 35.564167075790465, "itl_ms_p99": 45.27879203669727, "itl_ms_mean": 30.205085984475556, "total_s": 2.4076762920012698, "peak_mem_gb": 2.090230864, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.5951137470124812, "cache_misses": 4574, "cache_hits": 6723, "stall_s": 0.9355781902559102, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2981, "installed": 2981, "late": 0, "dup": 6403, "predictor": "transition2", "m": 8}, "ssd_bytes": 26736721920, "ssd_read_s": 2.909290548763238}
44 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:52:42", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 323.07720894459635, "prompt_tps": 55.77347093293254, "gen_tps": 34.76556916605975, "decode_tps_wall": 34.222504322196336, "itl_ms_p50": 29.257082962431014, "itl_ms_p95": 35.35625000949949, "itl_ms_p99": 37.514208001084626, "itl_ms_mean": 29.220538350590875, "total_s": 2.1639711250318214, "peak_mem_gb": 1.83861416, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6556537457320747, "cache_misses": 3429, "cache_hits": 6529, "stall_s": 0.7441158297006041, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3262, "installed": 3262, "late": 1, "dup": 5450, "predictor": "transition2", "m": 8}, "ssd_bytes": 23679074304, "ssd_read_s": 2.4573283984791487}
45 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:52:44", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 303.9848329499364, "prompt_tps": 46.10153863994922, "gen_tps": 32.26179979669117, "decode_tps_wall": 31.757873934920628, "itl_ms_p50": 31.53604199178517, "itl_ms_p95": 38.562958012335, "itl_ms_p99": 41.18674993515015, "itl_ms_mean": 31.48825396968436, "total_s": 2.287744833040051, "peak_mem_gb": 1.762732952, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6189245087900724, "cache_misses": 3685, "cache_hits": 5985, "stall_s": 0.812436431995593, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4272, "installed": 4272, "late": 3, "dup": 4280, "predictor": "transition2", "m": 8}, "ssd_bytes": 28159377408, "ssd_read_s": 2.9446414752164856}
46 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:52:46", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 369.5938750170171, "prompt_tps": 78.53176300418164, "gen_tps": 35.86073790447289, "decode_tps_wall": 35.30060591962893, "itl_ms_p50": 28.571082977578044, "itl_ms_p95": 35.74508405290544, "itl_ms_p99": 37.75250003673136, "itl_ms_mean": 28.328125649649238, "total_s": 2.154265790944919, "peak_mem_gb": 1.925994748, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.6514934631167096, "cache_misses": 3652, "cache_hits": 6827, "stall_s": 0.7625655947485939, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2452, "installed": 2452, "late": 0, "dup": 6468, "predictor": "transition2", "m": 8}, "ssd_bytes": 21601714176, "ssd_read_s": 2.2692865426652133}
47 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:52:48", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 354.6893330058083, "prompt_tps": 67.72682667127343, "gen_tps": 41.44801108449971, "decode_tps_wall": 40.80068428436811, "itl_ms_p50": 23.674833006225526, "itl_ms_p95": 30.851582996547222, "itl_ms_p99": 33.191584050655365, "itl_ms_mean": 24.509392857980277, "total_s": 1.8987810830585659, "peak_mem_gb": 1.936432592, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7095648803178602, "cache_misses": 2997, "cache_hits": 7322, "stall_s": 0.5869833705946803, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1354, "installed": 1354, "late": 0, "dup": 7542, "predictor": "transition2", "m": 8}, "ssd_bytes": 15397945344, "ssd_read_s": 1.528933696099557}
48 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:52:50", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 293.2850420475006, "prompt_tps": 64.85182984578265, "gen_tps": 42.035485166208865, "decode_tps_wall": 41.378936643178235, "itl_ms_p50": 22.92745898012072, "itl_ms_p95": 29.90829199552536, "itl_ms_p99": 33.40404201298952, "itl_ms_mean": 24.166884920781566, "total_s": 1.8157987920567393, "peak_mem_gb": 1.77871062, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.7416768168899716, "cache_misses": 2545, "cache_hits": 7307, "stall_s": 0.5172337950207293, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1220, "installed": 1220, "late": 0, "dup": 7444, "predictor": "transition2", "m": 8}, "ssd_bytes": 13324124160, "ssd_read_s": 1.2893915047170594}
49 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:52:52", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 378.07437498122454, "prompt_tps": 76.7743733148134, "gen_tps": 35.166428437289746, "decode_tps_wall": 34.61712498277362, "itl_ms_p50": 29.142415965907276, "itl_ms_p95": 36.59095801413059, "itl_ms_p99": 38.46429195255041, "itl_ms_mean": 28.887436507151477, "total_s": 2.1979828749317676, "peak_mem_gb": 1.913297764, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.6513980341635652, "cache_misses": 3653, "cache_hits": 6826, "stall_s": 0.7939919838681817, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2486, "installed": 2486, "late": 0, "dup": 6434, "predictor": "transition2", "m": 8}, "ssd_bytes": 21725577216, "ssd_read_s": 2.304772370494902}
50 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:52:54", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 302.420417079702, "prompt_tps": 62.89671670966046, "gen_tps": 42.07464256237184, "decode_tps_wall": 41.41748720225738, "itl_ms_p50": 23.17712502554059, "itl_ms_p95": 29.35124991927296, "itl_ms_p99": 35.00116697978228, "itl_ms_mean": 24.144390873270968, "total_s": 1.823517042095773, "peak_mem_gb": 1.774123092, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.7386317498984978, "cache_misses": 2575, "cache_hits": 7277, "stall_s": 0.5174110381631181, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1242, "installed": 1242, "late": 0, "dup": 7422, "predictor": "transition2", "m": 8}, "ssd_bytes": 13508149248, "ssd_read_s": 1.2985591932665557}
51 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:52:56", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 323.702875059098, "prompt_tps": 55.65837616338047, "gen_tps": 33.72026162156194, "decode_tps_wall": 33.19346415416032, "itl_ms_p50": 29.05595803167671, "itl_ms_p95": 37.15220803860575, "itl_ms_p99": 38.89787499792874, "itl_ms_mean": 30.12641269846686, "total_s": 2.2216668750625104, "peak_mem_gb": 1.832781476, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6555533239606347, "cache_misses": 3430, "cache_hits": 6528, "stall_s": 0.7704306307714432, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3225, "installed": 3224, "late": 0, "dup": 5487, "predictor": "transition2", "m": 8}, "ssd_bytes": 23551672320, "ssd_read_s": 2.4809985145693645}
52 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:52:58", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 381.26549997832626, "prompt_tps": 62.9984231902653, "gen_tps": 41.532063736529814, "decode_tps_wall": 40.88342042071837, "itl_ms_p50": 23.87808298226446, "itl_ms_p95": 30.905375024303794, "itl_ms_p99": 33.17204106133431, "itl_ms_mean": 24.4597929847678, "total_s": 1.9222324580186978, "peak_mem_gb": 1.94711496, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7108246923151468, "cache_misses": 2984, "cache_hits": 7335, "stall_s": 0.6018826133804396, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1374, "installed": 1375, "late": 0, "dup": 7522, "predictor": "transition2", "m": 8}, "ssd_bytes": 15422717952, "ssd_read_s": 1.5850066980347037}
53 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:53:00", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 296.582916053012, "prompt_tps": 47.25632718283334, "gen_tps": 31.66000322566727, "decode_tps_wall": 31.165460211841552, "itl_ms_p50": 31.57820808701217, "itl_ms_p95": 40.186041966080666, "itl_ms_p99": 41.83170909527689, "itl_ms_mean": 32.08680357044888, "total_s": 2.3180515409912914, "peak_mem_gb": 1.765490084, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6216132368148914, "cache_misses": 3659, "cache_hits": 6011, "stall_s": 0.8197586905444041, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4262, "installed": 4262, "late": 2, "dup": 4290, "predictor": "transition2", "m": 8}, "ssd_bytes": 28031975424, "ssd_read_s": 2.9535520400386304}
54 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:53:03", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 469.61216698400676, "prompt_tps": 68.18820478879253, "gen_tps": 34.05711275171166, "decode_tps_wall": 33.52519782793929, "itl_ms_p50": 29.667209018953145, "itl_ms_p95": 36.3955000648275, "itl_ms_p99": 45.34125002101064, "itl_ms_mean": 29.828310190212157, "total_s": 2.3487957089673728, "peak_mem_gb": 2.1066804, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.5950252279366204, "cache_misses": 4575, "cache_hits": 6722, "stall_s": 0.9184405328705907, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2953, "installed": 2953, "late": 0, "dup": 6431, "predictor": "transition2", "m": 8}, "ssd_bytes": 26641170432, "ssd_read_s": 2.8397510712966323}
55 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:53:11", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 753.2049170695245, "prompt_tps": 42.64084524581707, "gen_tps": 27.340811820146158, "decode_tps_wall": 26.91468525848031, "itl_ms_p50": 36.52970807161182, "itl_ms_p95": 46.35674995370209, "itl_ms_p99": 52.91012499947101, "itl_ms_mean": 37.1544378244185, "total_s": 3.09393450000789, "peak_mem_gb": 2.110217936, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.5786491989023634, "cache_misses": 4760, "cache_hits": 6537, "stall_s": 1.5305998735129833, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2771, "installed": 2771, "late": 36, "dup": 6381, "predictor": "transition3", "m": 8}, "ssd_bytes": 26651787264, "ssd_read_s": 5.400361988926306}
56 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:53:13", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 420.8150419872254, "prompt_tps": 57.087344311843424, "gen_tps": 34.54763082527328, "decode_tps_wall": 34.00806503975473, "itl_ms_p50": 28.49091694224626, "itl_ms_p95": 38.611458032391965, "itl_ms_p99": 40.60274991206825, "itl_ms_mean": 29.40478968241858, "total_s": 2.273316791979596, "peak_mem_gb": 1.943576016, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7041379978680105, "cache_misses": 3053, "cache_hits": 7266, "stall_s": 0.9600879641948268, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1204, "installed": 1203, "late": 6, "dup": 7692, "predictor": "transition3", "m": 8}, "ssd_bytes": 15065284608, "ssd_read_s": 2.7973526313435286}
57 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:53:16", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 365.8355000661686, "prompt_tps": 38.30144935160632, "gen_tps": 25.917288957690737, "decode_tps_wall": 25.512444964646704, "itl_ms_p50": 39.969999925233424, "itl_ms_p95": 46.09837499447167, "itl_ms_p99": 51.19445803575218, "itl_ms_mean": 39.19655687197866, "total_s": 2.835218583000824, "peak_mem_gb": 1.75032922, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6042399172699069, "cache_misses": 3827, "cache_hits": 5843, "stall_s": 1.285978739731945, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3844, "installed": 3843, "late": 91, "dup": 4708, "predictor": "transition3", "m": 8}, "ssd_bytes": 27147239424, "ssd_read_s": 5.100391939282417}
58 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:53:18", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 383.7739999871701, "prompt_tps": 46.95035189643236, "gen_tps": 28.602661656534913, "decode_tps_wall": 28.15585883307409, "itl_ms_p50": 36.13400005269796, "itl_ms_p95": 43.767041992396116, "itl_ms_p99": 47.90783300995827, "itl_ms_mean": 35.51658665177427, "total_s": 2.621318959048949, "peak_mem_gb": 1.85630888, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6438039767021491, "cache_misses": 3547, "cache_hits": 6411, "stall_s": 1.1678438034141436, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3008, "installed": 3006, "late": 108, "dup": 5704, "predictor": "transition3", "m": 8}, "ssd_bytes": 23197777920, "ssd_read_s": 4.267102239537053}
59 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:53:21", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 444.09462506882846, "prompt_tps": 65.35459799267639, "gen_tps": 29.238124163006884, "decode_tps_wall": 28.781462001861236, "itl_ms_p50": 35.20095895510167, "itl_ms_p95": 44.54370797611773, "itl_ms_p99": 45.570999966003, "itl_ms_mean": 34.74458663480445, "total_s": 2.633003583061509, "peak_mem_gb": 1.919294376, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.6341253936444318, "cache_misses": 3834, "cache_hits": 6645, "stall_s": 1.2079252713592723, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2304, "installed": 2308, "late": 34, "dup": 6616, "predictor": "transition3", "m": 8}, "ssd_bytes": 21722038272, "ssd_read_s": 4.141525521525182}
60 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:53:23", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 349.29945797193795, "prompt_tps": 54.44807314196366, "gen_tps": 35.810574934728145, "decode_tps_wall": 35.25124427394701, "itl_ms_p50": 26.920875068753958, "itl_ms_p95": 36.960417055524886, "itl_ms_p99": 37.821333040483296, "itl_ms_mean": 28.3677929842342, "total_s": 2.1364704159786925, "peak_mem_gb": 1.785985096, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.7290905399918798, "cache_misses": 2669, "cache_hits": 7183, "stall_s": 0.860856703016907, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1155, "installed": 1155, "late": 9, "dup": 7509, "predictor": "transition3", "m": 8}, "ssd_bytes": 13532921856, "ssd_read_s": 2.4766576415859163}
61 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:53:26", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 613.1644169799984, "prompt_tps": 52.21898744345239, "gen_tps": 27.392022640473126, "decode_tps_wall": 26.96422280617183, "itl_ms_p50": 36.512749968096614, "itl_ms_p95": 44.91145897191018, "itl_ms_p99": 56.61400000099093, "itl_ms_mean": 37.086179237886675, "total_s": 2.9495937089668587, "peak_mem_gb": 2.097308752, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.5817473665574932, "cache_misses": 4725, "cache_hits": 6572, "stall_s": 1.4236707100644708, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2720, "installed": 2720, "late": 39, "dup": 6664, "predictor": "transition3", "m": 8}, "ssd_bytes": 26347438080, "ssd_read_s": 4.9407663334859535}
62 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:53:29", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 371.72820803243667, "prompt_tps": 48.46693041974209, "gen_tps": 28.816363362515276, "decode_tps_wall": 28.36622317108597, "itl_ms_p50": 35.94166599214077, "itl_ms_p95": 43.90633397269994, "itl_ms_p99": 47.56720794830471, "itl_ms_mean": 35.25319511056065, "total_s": 2.5926794999977574, "peak_mem_gb": 1.842153112, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6442056637879092, "cache_misses": 3543, "cache_hits": 6415, "stall_s": 1.1450044836383313, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2992, "installed": 2989, "late": 97, "dup": 5720, "predictor": "transition3", "m": 8}, "ssd_bytes": 23126999040, "ssd_read_s": 4.172673489316367}
63 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:53:32", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 354.6910840086639, "prompt_tps": 39.50963831692955, "gen_tps": 25.957192597923797, "decode_tps_wall": 25.551733243549446, "itl_ms_p50": 39.379875059239566, "itl_ms_p95": 46.66274995543063, "itl_ms_p99": 49.57350005861372, "itl_ms_mean": 39.13628834758013, "total_s": 2.820277249906212, "peak_mem_gb": 1.766499812, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6010341261633919, "cache_misses": 3858, "cache_hits": 5812, "stall_s": 1.2766682531218976, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3841, "installed": 3839, "late": 84, "dup": 4711, "predictor": "transition3", "m": 8}, "ssd_bytes": 27246329856, "ssd_read_s": 5.046303864219226}
64 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:53:34", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 467.11124991998076, "prompt_tps": 62.131434986674485, "gen_tps": 29.35151327800442, "decode_tps_wall": 28.893030604599495, "itl_ms_p50": 35.02037504222244, "itl_ms_p95": 42.79870796017349, "itl_ms_p99": 44.56633294466883, "itl_ms_mean": 34.61042262007675, "total_s": 2.6475678749848157, "peak_mem_gb": 1.923137056, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.6344116805038649, "cache_misses": 3831, "cache_hits": 6648, "stall_s": 1.2043944289907813, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2302, "installed": 2307, "late": 33, "dup": 6618, "predictor": "transition3", "m": 8}, "ssd_bytes": 21704343552, "ssd_read_s": 4.134974777814932}
65 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:53:36", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 410.85666709113866, "prompt_tps": 58.46602798684686, "gen_tps": 34.395164607205125, "decode_tps_wall": 33.857986572159255, "itl_ms_p50": 28.66954100318253, "itl_ms_p95": 38.134667091071606, "itl_ms_p99": 40.19066703040153, "itl_ms_mean": 29.5351289678365, "total_s": 2.271569792064838, "peak_mem_gb": 1.939971536, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7022967341796685, "cache_misses": 3072, "cache_hits": 7247, "stall_s": 0.96135191002395, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1209, "installed": 1208, "late": 8, "dup": 7687, "predictor": "transition3", "m": 8}, "ssd_bytes": 15150219264, "ssd_read_s": 2.8135463966755196}
66 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:53:39", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 323.7640829756856, "prompt_tps": 58.73757487780616, "gen_tps": 37.34104801238553, "decode_tps_wall": 36.757786275582944, "itl_ms_p50": 26.66179195512086, "itl_ms_p95": 35.13124992605299, "itl_ms_p99": 36.5643750410527, "itl_ms_mean": 27.20512036559364, "total_s": 2.037686666008085, "peak_mem_gb": 1.784448924, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.7333536337799431, "cache_misses": 2627, "cache_hits": 7225, "stall_s": 0.8013249097857624, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1137, "installed": 1138, "late": 7, "dup": 7527, "predictor": "transition3", "m": 8}, "ssd_bytes": 13320585216, "ssd_read_s": 2.287489281501621}
67 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:53:41", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 440.6553329899907, "prompt_tps": 65.85454881905765, "gen_tps": 29.704000209209045, "decode_tps_wall": 29.24005841482512, "itl_ms_p50": 34.30370893329382, "itl_ms_p95": 42.71358298137784, "itl_ms_p99": 44.67908293008804, "itl_ms_mean": 34.1996580791024, "total_s": 2.595233791973442, "peak_mem_gb": 1.92016208, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.6337436778318541, "cache_misses": 3838, "cache_hits": 6641, "stall_s": 1.181948950397782, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2326, "installed": 2326, "late": 33, "dup": 6594, "predictor": "transition3", "m": 8}, "ssd_bytes": 21814050816, "ssd_read_s": 4.06036287418101}
68 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:53:43", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 352.86379198078066, "prompt_tps": 53.89972802061477, "gen_tps": 36.22907587479348, "decode_tps_wall": 35.663209385743905, "itl_ms_p50": 26.90387493930757, "itl_ms_p95": 35.17370799090713, "itl_ms_p99": 39.51866598799825, "itl_ms_mean": 28.04010119178288, "total_s": 2.119390167063102, "peak_mem_gb": 1.787987868, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.729192042224929, "cache_misses": 2668, "cache_hits": 7184, "stall_s": 0.8518375136191025, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1154, "installed": 1154, "late": 10, "dup": 7510, "predictor": "transition3", "m": 8}, "ssd_bytes": 13525843968, "ssd_read_s": 2.446215729578398}
69 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:53:46", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 378.37908300571144, "prompt_tps": 47.60749737789905, "gen_tps": 28.540004306655902, "decode_tps_wall": 28.09421289530375, "itl_ms_p50": 35.74779094196856, "itl_ms_p95": 45.13195808976889, "itl_ms_p99": 48.81591710727662, "itl_ms_mean": 35.594519188938044, "total_s": 2.620833791908808, "peak_mem_gb": 1.832715924, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.643904398473589, "cache_misses": 3546, "cache_hits": 6412, "stall_s": 1.1633029583608732, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2958, "installed": 2954, "late": 106, "dup": 5754, "predictor": "transition3", "m": 8}, "ssd_bytes": 23017291776, "ssd_read_s": 4.207141849328764}
70 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:53:48", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 406.8406249862164, "prompt_tps": 59.0398097534521, "gen_tps": 35.0150041351082, "decode_tps_wall": 34.46810605404178, "itl_ms_p50": 28.314916999079287, "itl_ms_p95": 38.30704092979431, "itl_ms_p99": 38.786791963502765, "itl_ms_mean": 29.012328047039254, "total_s": 2.2346172919496894, "peak_mem_gb": 1.916378552, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7030720031010756, "cache_misses": 3064, "cache_hits": 7255, "stall_s": 0.9412726262817159, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 1228, "installed": 1232, "late": 8, "dup": 7668, "predictor": "transition3", "m": 8}, "ssd_bytes": 15189147648, "ssd_read_s": 2.764597364817746}
71 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:53:51", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 347.69745799712837, "prompt_tps": 40.29696456872564, "gen_tps": 26.761941039115683, "decode_tps_wall": 26.343887607006273, "itl_ms_p50": 38.27670798636973, "itl_ms_p95": 45.40520906448364, "itl_ms_p99": 50.906124990433455, "itl_ms_mean": 37.95946957100006, "total_s": 2.7391440409701318, "peak_mem_gb": 1.748074372, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6040330920372285, "cache_misses": 3829, "cache_hits": 5841, "stall_s": 1.2457460204605013, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3847, "installed": 3842, "late": 90, "dup": 4705, "predictor": "transition3", "m": 8}, "ssd_bytes": 27164934144, "ssd_read_s": 4.955624026013538}
72 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition3", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:53:54", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 589.582083048299, "prompt_tps": 54.30586079041611, "gen_tps": 27.95022901670601, "decode_tps_wall": 27.5137635278525, "itl_ms_p50": 36.157459020614624, "itl_ms_p95": 44.21754099894315, "itl_ms_p99": 51.41895799897611, "itl_ms_mean": 36.34544576163448, "total_s": 2.879345166031271, "peak_mem_gb": 2.08730024, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.5817473665574932, "cache_misses": 4725, "cache_hits": 6572, "stall_s": 1.3919748768676072, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 2695, "installed": 2700, "late": 37, "dup": 6689, "predictor": "transition3", "m": 8}, "ssd_bytes": 26258964480, "ssd_read_s": 4.873588268761523}
73 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:54:01", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 738.4890830144286, "prompt_tps": 43.654311526418894, "gen_tps": 19.634206300194876, "decode_tps_wall": 19.32782107607674, "itl_ms_p50": 51.67612503282726, "itl_ms_p95": 67.43425002787262, "itl_ms_p99": 68.57829191721976, "itl_ms_mean": 51.738889555313754, "total_s": 3.9980391249991953, "peak_mem_gb": 2.090229456, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.5930778082676816, "cache_misses": 4597, "cache_hits": 6700, "stall_s": 1.8813947818707675, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 11228, "installed": 10981, "late": 500, "dup": 5412, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 55377395712, "ssd_read_s": 8.49053135048598}
74 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:54:05", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 488.09512495063245, "prompt_tps": 49.23585268689506, "gen_tps": 21.883011480968356, "decode_tps_wall": 21.54128737238935, "itl_ms_p50": 46.18499998468906, "itl_ms_p95": 56.120374938473105, "itl_ms_p99": 59.813000028952956, "itl_ms_mean": 46.42248082543827, "total_s": 3.4127114169532433, "peak_mem_gb": 2.604126676, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7457117937784669, "cache_misses": 2624, "cache_hits": 7695, "stall_s": 1.4655210030032322, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 10649, "installed": 10695, "late": 639, "dup": 5991, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 46926397440, "ssd_read_s": 6.642568752751686}
75 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:54:08", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 396.3068750454113, "prompt_tps": 35.381113498651956, "gen_tps": 19.234880529914122, "decode_tps_wall": 18.93443605876646, "itl_ms_p50": 53.189291968010366, "itl_ms_p95": 62.700584065169096, "itl_ms_p99": 68.87941702734679, "itl_ms_mean": 52.813825397086994, "total_s": 3.723577875061892, "peak_mem_gb": 2.188492224, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.609100310237849, "cache_misses": 3780, "cache_hits": 5890, "stall_s": 1.6934111309237778, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 11309, "installed": 11200, "late": 522, "dup": 5331, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 53455749120, "ssd_read_s": 8.21669495117385}
76 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:54:12", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 491.03087501134723, "prompt_tps": 36.704659129798834, "gen_tps": 20.09872583298128, "decode_tps_wall": 19.784749257624057, "itl_ms_p50": 51.59587494563311, "itl_ms_p95": 60.76370901428163, "itl_ms_p99": 66.465082927607, "itl_ms_mean": 50.54398147677559, "total_s": 3.6753017080482095, "peak_mem_gb": 2.802191004, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6431010243020687, "cache_misses": 3554, "cache_hits": 6404, "stall_s": 1.6476899560075253, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 10812, "installed": 10927, "late": 570, "dup": 5828, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 50773229568, "ssd_read_s": 7.653688264661469}
77 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:54:16", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 472.8152919560671, "prompt_tps": 61.41383647956028, "gen_tps": 18.834992443979438, "decode_tps_wall": 18.540842788427884, "itl_ms_p50": 54.68833306804299, "itl_ms_p95": 64.29741601459682, "itl_ms_p99": 75.55237493943423, "itl_ms_mean": 53.9349808102651, "total_s": 3.8707190830027685, "peak_mem_gb": 2.420217636, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.6525431816012979, "cache_misses": 3641, "cache_hits": 6838, "stall_s": 1.6587911994429305, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 11375, "installed": 11344, "late": 481, "dup": 5265, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 53162016768, "ssd_read_s": 7.974729008390568}
78 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:54:19", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 415.41041689924896, "prompt_tps": 45.806256416292655, "gen_tps": 20.88992012176414, "decode_tps_wall": 20.563628660998237, "itl_ms_p50": 49.125874997116625, "itl_ms_p95": 58.533707982860506, "itl_ms_p99": 59.69133402686566, "itl_ms_mean": 48.62954960359881, "total_s": 3.479072041925974, "peak_mem_gb": 2.352155628, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.7481729598051157, "cache_misses": 2481, "cache_hits": 7371, "stall_s": 1.4357508576940745, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 10667, "installed": 10665, "late": 554, "dup": 5973, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 46438023168, "ssd_read_s": 6.680345768225379}
79 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:54:24", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 649.8539160238579, "prompt_tps": 49.28818678012778, "gen_tps": 18.667066904812263, "decode_tps_wall": 18.37546700941863, "itl_ms_p50": 54.8915839754045, "itl_ms_p95": 64.01470908895135, "itl_ms_p99": 74.58949997089803, "itl_ms_mean": 54.4203855873396, "total_s": 4.078338208026253, "peak_mem_gb": 2.717506984, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.6018411967779056, "cache_misses": 4498, "cache_hits": 6799, "stall_s": 1.8833149742567912, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 11217, "installed": 11188, "late": 535, "dup": 5423, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 55741906944, "ssd_read_s": 8.682453730027191}
80 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:54:27", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 485.24133302271366, "prompt_tps": 37.14527554034433, "gen_tps": 19.30410680855894, "decode_tps_wall": 19.002615310633526, "itl_ms_p50": 54.14712498895824, "itl_ms_p95": 60.98545901477337, "itl_ms_p99": 64.66695899143815, "itl_ms_mean": 52.624335316645485, "total_s": 3.800574457971379, "peak_mem_gb": 2.607663772, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6432014460735087, "cache_misses": 3553, "cache_hits": 6405, "stall_s": 1.6771054364508018, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 10876, "installed": 10940, "late": 549, "dup": 5764, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 50989105152, "ssd_read_s": 7.8780766791896895}
81 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:54:31", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 406.9164589745924, "prompt_tps": 34.45566857049648, "gen_tps": 18.40166272644055, "decode_tps_wall": 18.114272594287947, "itl_ms_p50": 55.84937508683652, "itl_ms_p95": 64.01233398355544, "itl_ms_p99": 73.2377499807626, "itl_ms_mean": 55.205087303110055, "total_s": 3.884836959070526, "peak_mem_gb": 2.196848, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6103412616339193, "cache_misses": 3768, "cache_hits": 5902, "stall_s": 1.6982223885133862, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 11411, "installed": 11382, "late": 513, "dup": 5229, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 53664546816, "ssd_read_s": 8.464784750132822}
82 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:54:35", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 493.30812506377697, "prompt_tps": 58.85286129871006, "gen_tps": 18.823585699780406, "decode_tps_wall": 18.529541651272485, "itl_ms_p50": 53.91395802143961, "itl_ms_p95": 62.34887498430908, "itl_ms_p99": 69.93033294565976, "itl_ms_mean": 53.96787566687202, "total_s": 3.8932842920767143, "peak_mem_gb": 2.48598714, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.6518751789292871, "cache_misses": 3648, "cache_hits": 6831, "stall_s": 1.72785309352912, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 11293, "installed": 11307, "late": 529, "dup": 5347, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 52882440192, "ssd_read_s": 8.228081482113339}
83 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:54:39", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 467.34316705260426, "prompt_tps": 51.4308995639455, "gen_tps": 21.603713406163724, "decode_tps_wall": 21.266237944446882, "itl_ms_p50": 46.558125060983, "itl_ms_p95": 56.56345805618912, "itl_ms_p99": 59.992791968397796, "itl_ms_mean": 47.02289152469131, "total_s": 3.429785333108157, "peak_mem_gb": 2.412978448, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7453241593177633, "cache_misses": 2628, "cache_hits": 7691, "stall_s": 1.4848567594308406, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 10669, "installed": 10670, "late": 620, "dup": 5971, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 47067955200, "ssd_read_s": 6.713389698998071}
84 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:54:42", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 406.449917005375, "prompt_tps": 46.81849823455439, "gen_tps": 19.356718769709314, "decode_tps_wall": 19.054455895962505, "itl_ms_p50": 52.89687495678663, "itl_ms_p95": 64.27420896943659, "itl_ms_p99": 66.87500001862645, "itl_ms_mean": 52.4811626981116, "total_s": 3.712763166986406, "peak_mem_gb": 2.26417756, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.7502030044660982, "cache_misses": 2461, "cache_hits": 7391, "stall_s": 1.4805924523388967, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 10686, "installed": 10673, "late": 527, "dup": 5954, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 46484029440, "ssd_read_s": 7.116914416081272}
85 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:54:46", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 526.9134580157697, "prompt_tps": 55.12156562908391, "gen_tps": 18.86589467621387, "decode_tps_wall": 18.571189200660164, "itl_ms_p50": 54.11991698201746, "itl_ms_p95": 61.992709059268236, "itl_ms_p99": 73.37999995797873, "itl_ms_mean": 53.846847888688366, "total_s": 3.9192648750031367, "peak_mem_gb": 2.463901516, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.6499665998663995, "cache_misses": 3668, "cache_hits": 6811, "stall_s": 1.7454534453572705, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 11277, "installed": 11278, "late": 534, "dup": 5363, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 52917829632, "ssd_read_s": 8.160517507814802}
86 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:54:50", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 396.0263750050217, "prompt_tps": 48.05188077740327, "gen_tps": 21.062078045370882, "decode_tps_wall": 20.73320511655182, "itl_ms_p50": 48.925124923698604, "itl_ms_p95": 58.50166699383408, "itl_ms_p99": 62.86554201506078, "itl_ms_mean": 48.2318095238288, "total_s": 3.434630375006236, "peak_mem_gb": 2.314738664, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.747563946406821, "cache_misses": 2487, "cache_hits": 7365, "stall_s": 1.4233429500600323, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 10604, "installed": 10594, "late": 553, "dup": 6036, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 46278770688, "ssd_read_s": 6.7789444053778425}
87 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:54:53", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 445.88733301497996, "prompt_tps": 40.42741134778498, "gen_tps": 18.957355326548118, "decode_tps_wall": 18.661272635469416, "itl_ms_p50": 55.01899996306747, "itl_ms_p95": 64.2057079821825, "itl_ms_p99": 71.09279197175056, "itl_ms_mean": 53.58691336513157, "total_s": 3.821862875018269, "peak_mem_gb": 2.357238344, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6404900582446275, "cache_misses": 3580, "cache_hits": 6378, "stall_s": 1.6816431327024475, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 10842, "installed": 10872, "late": 565, "dup": 5798, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 51074039808, "ssd_read_s": 7.943893982679583}
88 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:54:57", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 467.50991605222225, "prompt_tps": 51.400003202950664, "gen_tps": 20.63356794043737, "decode_tps_wall": 20.311353708294387, "itl_ms_p50": 50.07370898965746, "itl_ms_p95": 59.30404202081263, "itl_ms_p99": 69.94429090991616, "itl_ms_mean": 49.23354761881961, "total_s": 3.5692234160378575, "peak_mem_gb": 2.388152764, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7461963368543464, "cache_misses": 2619, "cache_hits": 7700, "stall_s": 1.5268688319483772, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 10645, "installed": 10638, "late": 624, "dup": 5995, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 46929936384, "ssd_read_s": 6.952907785656862}
89 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:55:01", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 401.94941696245223, "prompt_tps": 34.885749172632515, "gen_tps": 18.349128659023847, "decode_tps_wall": 18.062554279328474, "itl_ms_p50": 55.842416011728346, "itl_ms_p95": 64.86345804296434, "itl_ms_p99": 80.04754199646413, "itl_ms_mean": 55.3631554283793, "total_s": 3.889828208950348, "peak_mem_gb": 2.192671264, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6107549120992761, "cache_misses": 3764, "cache_hits": 5906, "stall_s": 1.7141088997013867, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 11404, "installed": 11396, "late": 496, "dup": 5236, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 53625618432, "ssd_read_s": 8.446497321594507}
90 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "prev_rank", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor prev_rank --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:55:05", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 672.1777920611203, "prompt_tps": 47.64901139788459, "gen_tps": 19.377153200391433, "decode_tps_wall": 19.07446627689362, "itl_ms_p50": 53.06133395060897, "itl_ms_p95": 61.16104102693498, "itl_ms_p99": 80.60945908073336, "itl_ms_mean": 52.42610647572234, "total_s": 3.9750225000316277, "peak_mem_gb": 2.717506952, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.6066212268743915, "cache_misses": 4444, "cache_hits": 6853, "stall_s": 1.8803486177930608, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 11176, "installed": 11143, "late": 555, "dup": 5464, "predictor": "prev_rank", "m": 16}, "ssd_bytes": 55363239936, "ssd_read_s": 8.55122381716501}
91 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:55:13", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 776.9622500054538, "prompt_tps": 41.34613986780918, "gen_tps": 25.967484486226184, "decode_tps_wall": 25.56300531981259, "itl_ms_p50": 38.845291011966765, "itl_ms_p95": 48.301166971214116, "itl_ms_p99": 68.34666698705405, "itl_ms_mean": 39.119031095492936, "total_s": 3.2414612090215087, "peak_mem_gb": 2.148032208, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.6855802425422679, "cache_misses": 3552, "cache_hits": 7745, "stall_s": 1.4738099195528775, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 6898, "installed": 6866, "late": 655, "dup": 11126, "predictor": "transition2", "m": 16}, "ssd_bytes": 36981964800, "ssd_read_s": 6.262643188936636}
92 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:55:16", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 414.5083329640329, "prompt_tps": 57.9639732617623, "gen_tps": 31.666907672355048, "decode_tps_wall": 31.172245270645256, "itl_ms_p50": 31.223458936437964, "itl_ms_p95": 41.79754201322794, "itl_ms_p99": 47.1361669478938, "itl_ms_mean": 32.079819445720034, "total_s": 2.435536958044395, "peak_mem_gb": 1.964744136, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7671285977323384, "cache_misses": 2403, "cache_hits": 7916, "stall_s": 0.8962502006907016, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4647, "installed": 4677, "late": 282, "dup": 12545, "predictor": "transition2", "m": 16}, "ssd_bytes": 24949555200, "ssd_read_s": 4.040312738739885}
93 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:55:19", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 369.6352079277858, "prompt_tps": 37.916168027005675, "gen_tps": 22.871121466101666, "decode_tps_wall": 22.51386579064535, "itl_ms_p50": 44.827249948866665, "itl_ms_p95": 54.21820795163512, "itl_ms_p99": 68.7379160663113, "itl_ms_mean": 44.41707209676563, "total_s": 3.1679107500240207, "peak_mem_gb": 1.781939684, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.7026887280248191, "cache_misses": 2875, "cache_hits": 6795, "stall_s": 1.3573551709996536, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 8061, "installed": 7991, "late": 741, "dup": 8475, "predictor": "transition2", "m": 16}, "ssd_bytes": 38701891584, "ssd_read_s": 6.064778626896441}
94 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:55:22", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 423.7950840033591, "prompt_tps": 42.51287290038875, "gen_tps": 26.07871480406025, "decode_tps_wall": 25.671433636822613, "itl_ms_p50": 39.0990839805454, "itl_ms_p95": 49.68541697598994, "itl_ms_p99": 55.28849991969764, "itl_ms_mean": 38.953804222512105, "total_s": 2.8778847500216216, "peak_mem_gb": 2.011277752, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.747941353685479, "cache_misses": 2510, "cache_hits": 7448, "stall_s": 1.1847929114010185, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 6685, "installed": 6714, "late": 727, "dup": 10155, "predictor": "transition2", "m": 16}, "ssd_bytes": 32540590080, "ssd_read_s": 5.085203237365931}
95 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:55:25", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 467.8872500080615, "prompt_tps": 62.04282407348515, "gen_tps": 26.945721894644805, "decode_tps_wall": 26.524822949824582, "itl_ms_p50": 38.14550000242889, "itl_ms_p95": 47.59520792867988, "itl_ms_p99": 51.14270804915577, "itl_ms_mean": 37.70053439721879, "total_s": 2.8430209170328453, "peak_mem_gb": 2.01832226, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.7263097623819067, "cache_misses": 2868, "cache_hits": 7611, "stall_s": 1.177170201903209, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 6216, "installed": 6228, "late": 504, "dup": 11024, "predictor": "transition2", "m": 16}, "ssd_bytes": 32147767296, "ssd_read_s": 5.041412454214878}
96 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:55:27", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 368.53945802431554, "prompt_tps": 51.6192014373395, "gen_tps": 32.01257493891522, "decode_tps_wall": 31.51259846961477, "itl_ms_p50": 30.93004203401506, "itl_ms_p95": 42.08649997599423, "itl_ms_p99": 46.378000057302415, "itl_ms_mean": 31.733339951772773, "total_s": 2.3677398749860004, "peak_mem_gb": 1.894558416, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.7973000406008932, "cache_misses": 1997, "cache_hits": 7855, "stall_s": 0.8341774244327098, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4644, "installed": 4642, "late": 369, "dup": 12100, "predictor": "transition2", "m": 16}, "ssd_bytes": 23502127104, "ssd_read_s": 3.8159793339436874}
97 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:55:30", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 636.0643749358132, "prompt_tps": 50.34029512861154, "gen_tps": 24.651099877657906, "decode_tps_wall": 24.266084157941314, "itl_ms_p50": 41.11083399038762, "itl_ms_p95": 51.034624921157956, "itl_ms_p99": 73.67904204875231, "itl_ms_mean": 41.20978042815945, "total_s": 3.2322805419098586, "peak_mem_gb": 2.200612016, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.6885013720456759, "cache_misses": 3519, "cache_hits": 7778, "stall_s": 1.4014906310476363, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 6816, "installed": 6815, "late": 646, "dup": 11320, "predictor": "transition2", "m": 16}, "ssd_bytes": 36574986240, "ssd_read_s": 5.90382578340359}
98 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:55:33", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 420.7969999406487, "prompt_tps": 42.82554626658236, "gen_tps": 25.505216667393704, "decode_tps_wall": 25.106821885091115, "itl_ms_p50": 38.832667050883174, "itl_ms_p95": 51.45812500268221, "itl_ms_p99": 61.62699998822063, "itl_ms_mean": 39.829812175224696, "total_s": 2.9300751669798046, "peak_mem_gb": 1.87839452, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.7467362924281984, "cache_misses": 2522, "cache_hits": 7436, "stall_s": 1.146633548894897, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 6759, "installed": 6746, "late": 663, "dup": 10081, "predictor": "transition2", "m": 16}, "ssd_bytes": 32844939264, "ssd_read_s": 5.040094618801959}
99 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:55:37", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 400.48387506976724, "prompt_tps": 34.99180972369275, "gen_tps": 21.264402708984, "decode_tps_wall": 20.932234221907454, "itl_ms_p50": 48.412750009447336, "itl_ms_p95": 56.89112504478544, "itl_ms_p99": 75.21420798730105, "itl_ms_mean": 47.77320898470602, "total_s": 3.4101960411062464, "peak_mem_gb": 1.874141156, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.703722854188211, "cache_misses": 2865, "cache_hits": 6805, "stall_s": 1.42131144576706, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 8055, "installed": 8027, "late": 726, "dup": 8481, "predictor": "transition2", "m": 16}, "ssd_bytes": 38645268480, "ssd_read_s": 6.298445483553223}
100 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:55:40", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 488.34279098082334, "prompt_tps": 59.44349613376352, "gen_tps": 25.35182381699112, "decode_tps_wall": 24.955831723044728, "itl_ms_p50": 39.084124960936606, "itl_ms_p95": 51.17345799226314, "itl_ms_p99": 53.56079200282693, "itl_ms_mean": 40.070794317649586, "total_s": 3.0128028329927474, "peak_mem_gb": 2.12796398, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.7270731940070617, "cache_misses": 2860, "cache_hits": 7619, "stall_s": 1.2197487407829612, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 6221, "installed": 6261, "late": 486, "dup": 11019, "predictor": "transition2", "m": 16}, "ssd_bytes": 32137150464, "ssd_read_s": 5.259236756595783}
101 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:55:42", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 425.2993749687448, "prompt_tps": 56.48295363041655, "gen_tps": 32.138579546005204, "decode_tps_wall": 31.636594953866712, "itl_ms_p50": 30.74816707521677, "itl_ms_p95": 42.22845798358321, "itl_ms_p99": 44.47245807386935, "itl_ms_mean": 31.608964285133258, "total_s": 2.41666412493214, "peak_mem_gb": 2.00596628, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7669347805019866, "cache_misses": 2405, "cache_hits": 7914, "stall_s": 0.9113926404388621, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4672, "installed": 4705, "late": 304, "dup": 12520, "predictor": "transition2", "m": 16}, "ssd_bytes": 25045106688, "ssd_read_s": 4.0977403610013425}
102 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:55:44", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 346.1302499054, "prompt_tps": 54.953424612251396, "gen_tps": 33.33982331112324, "decode_tps_wall": 32.81931171593335, "itl_ms_p50": 29.924332979135215, "itl_ms_p95": 39.25083298236132, "itl_ms_p99": 42.250624974258244, "itl_ms_mean": 30.469865079910036, "total_s": 2.2657317499397323, "peak_mem_gb": 1.804486556, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.798721071863581, "cache_misses": 1983, "cache_hits": 7869, "stall_s": 0.8119838304119185, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4591, "installed": 4557, "late": 371, "dup": 12153, "predictor": "transition2", "m": 16}, "ssd_bytes": 23265017856, "ssd_read_s": 3.6946225438732654}
103 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:55:47", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 476.76983301062137, "prompt_tps": 60.87839121917163, "gen_tps": 27.02849420181791, "decode_tps_wall": 26.606332233733795, "itl_ms_p50": 37.03683300409466, "itl_ms_p95": 47.84154205117375, "itl_ms_p99": 52.01483308337629, "itl_ms_mean": 37.585037697609216, "total_s": 2.844627207960002, "peak_mem_gb": 2.019850032, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.7278366256322168, "cache_misses": 2852, "cache_hits": 7627, "stall_s": 1.1982156806625426, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 6246, "installed": 6233, "late": 507, "dup": 10994, "predictor": "transition2", "m": 16}, "ssd_bytes": 32197312512, "ssd_read_s": 5.143610044615343}
104 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:55:50", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 368.6952500138432, "prompt_tps": 51.58245006719815, "gen_tps": 32.463807034247445, "decode_tps_wall": 31.95682615395422, "itl_ms_p50": 30.910708010196686, "itl_ms_p95": 40.77341593801975, "itl_ms_p99": 43.03674993570894, "itl_ms_mean": 31.292218920065178, "total_s": 2.3401050419779494, "peak_mem_gb": 1.968876252, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.7978075517661388, "cache_misses": 1992, "cache_hits": 7860, "stall_s": 0.8308984115719795, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4606, "installed": 4615, "late": 358, "dup": 12138, "predictor": "transition2", "m": 16}, "ssd_bytes": 23349952512, "ssd_read_s": 3.80275591800455}
105 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:55:52", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 410.11187504045665, "prompt_tps": 43.93609203570996, "gen_tps": 26.751296903221434, "decode_tps_wall": 26.33350143768132, "itl_ms_p50": 36.49824997410178, "itl_ms_p95": 50.55545805953443, "itl_ms_p99": 55.68374996073544, "itl_ms_mean": 37.97444112650636, "total_s": 2.8025016660103574, "peak_mem_gb": 1.95638236, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.7444265916850773, "cache_misses": 2545, "cache_hits": 7413, "stall_s": 1.1414121487177908, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 6657, "installed": 6643, "late": 710, "dup": 10183, "predictor": "transition2", "m": 16}, "ssd_bytes": 32565362688, "ssd_read_s": 4.980593756656162}
106 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:55:55", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 646.0756659507751, "prompt_tps": 37.1689855004326, "gen_tps": 26.376589670662668, "decode_tps_wall": 25.96460213401613, "itl_ms_p50": 38.04433310870081, "itl_ms_p95": 56.77058396395296, "itl_ms_p99": 65.424875007011, "itl_ms_mean": 38.51397355671026, "total_s": 3.0724560000235215, "peak_mem_gb": 2.026643728, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.7680007752689214, "cache_misses": 2394, "cache_hits": 7925, "stall_s": 1.486848161672242, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4463, "installed": 4517, "late": 405, "dup": 12729, "predictor": "transition2", "m": 16}, "ssd_bytes": 24266539008, "ssd_read_s": 6.124055169057101}
107 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:55:59", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 378.0507919145748, "prompt_tps": 37.06682700288372, "gen_tps": 23.243584398691333, "decode_tps_wall": 22.880552968713843, "itl_ms_p50": 43.70020900387317, "itl_ms_p95": 53.16062504425645, "itl_ms_p99": 66.26566592603922, "itl_ms_mean": 43.70523742880554, "total_s": 3.131480749929324, "peak_mem_gb": 1.798916584, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.703205791106515, "cache_misses": 2870, "cache_hits": 6800, "stall_s": 1.3516874810447916, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 8046, "installed": 7980, "late": 742, "dup": 8490, "predictor": "transition2", "m": 16}, "ssd_bytes": 38631112704, "ssd_read_s": 5.961565906996839}
108 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:56:02", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 595.8154579857364, "prompt_tps": 53.74817590743959, "gen_tps": 25.37077504697424, "decode_tps_wall": 24.97453118122204, "itl_ms_p50": 40.18462507519871, "itl_ms_p95": 47.956916969269514, "itl_ms_p99": 67.65291700139642, "itl_ms_mean": 40.040791666667374, "total_s": 3.118385332985781, "peak_mem_gb": 2.294852752, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.6883243338939542, "cache_misses": 3521, "cache_hits": 7776, "stall_s": 1.3618296253262088, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 6736, "installed": 6772, "late": 653, "dup": 11400, "predictor": "transition2", "m": 16}, "ssd_bytes": 36298948608, "ssd_read_s": 5.666648270911537}
109 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:56:07", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 478.6368330242112, "prompt_tps": 67.70281952994452, "gen_tps": 46.02429908057159, "decode_tps_wall": 45.30772847460189, "itl_ms_p50": 21.44587505608797, "itl_ms_p95": 29.769583023153245, "itl_ms_p99": 33.46587496344, "itl_ms_mean": 22.07128968208082, "total_s": 1.869128082995303, "peak_mem_gb": 2.805416836, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.71559902200489, "cache_misses": 2908, "cache_hits": 7317, "stall_s": 0.9139544410863891, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 10291249152, "ssd_read_s": 2.1015102174133062}
110 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:56:09", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 245.16537506133318, "prompt_tps": 98.05817563791334, "gen_tps": 57.37723932649922, "decode_tps_wall": 56.48113558477519, "itl_ms_p50": 17.299875034950674, "itl_ms_p95": 23.77570909447968, "itl_ms_p99": 26.005167048424482, "itl_ms_mean": 17.70502645965843, "total_s": 1.3605820420198143, "peak_mem_gb": 2.480622872, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.8336809176225235, "cache_misses": 1595, "cache_hits": 7995, "stall_s": 0.4802855326561257, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 5644615680, "ssd_read_s": 0.864837720640935}
111 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:56:10", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 212.01699995435774, "prompt_tps": 66.18112848361841, "gen_tps": 44.76336658404608, "decode_tps_wall": 44.064319079396974, "itl_ms_p50": 22.347333026118577, "itl_ms_p95": 32.07187505904585, "itl_ms_p99": 34.36987497843802, "itl_ms_mean": 22.694098556207287, "total_s": 1.6417452089954168, "peak_mem_gb": 2.385389948, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.7323761200475009, "cache_misses": 2479, "cache_hits": 6784, "stall_s": 0.7234980964567512, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 8773042176, "ssd_read_s": 1.447061560349539}
112 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:56:12", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 251.57787499483675, "prompt_tps": 71.65770317737694, "gen_tps": 48.42080705339218, "decode_tps_wall": 47.664664722850894, "itl_ms_p50": 20.775957964360714, "itl_ms_p95": 26.101749972440302, "itl_ms_p99": 30.63241601921618, "itl_ms_mean": 20.979902110180802, "total_s": 1.5733117079362273, "peak_mem_gb": 2.472523792, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.7912429076116048, "cache_misses": 1950, "cache_hits": 7391, "stall_s": 0.6353897843509912, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 6900940800, "ssd_read_s": 1.1909447448560968}
113 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:56:13", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 289.09962507896125, "prompt_tps": 100.43671230577584, "gen_tps": 49.18196558897762, "decode_tps_wall": 48.41389732634681, "itl_ms_p50": 20.339249982498586, "itl_ms_p95": 26.802208041772246, "itl_ms_p99": 34.39366596285254, "itl_ms_mean": 20.655226189687493, "total_s": 1.5903788750292733, "peak_mem_gb": 2.597377892, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.7793997550020416, "cache_misses": 2161, "cache_hits": 7635, "stall_s": 0.6567125713918358, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 7647657984, "ssd_read_s": 1.286987041006796}
114 +{"backend": "titan", "capacity": 32, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 32 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "2574612"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:56:15", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 199.78074997197837, "prompt_tps": 95.28834680741699, "gen_tps": 56.62403522263462, "decode_tps_wall": 55.73988672118333, "itl_ms_p50": 17.17858307529241, "itl_ms_p95": 24.96608393266797, "itl_ms_p99": 26.742333080619574, "itl_ms_mean": 17.940474206596495, "total_s": 1.3300306249875575, "peak_mem_gb": 2.469248884, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.8563977006599958, "cache_misses": 1349, "cache_hits": 8045, "stall_s": 0.4672505249036476, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 4774035456, "ssd_read_s": 0.7979402884375304}
added benchmarks/raw/tests/greedy_olmoe_cap16_20261101T015233.json +14 −0
@@ -0,0 +1,14 @@
1 +[
2 + {
3 + "prompt": "The capital of Canada is",
4 + "ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
5 + "titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
6 + "equal": true
7 + },
8 + {
9 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
10 + "ref": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
11 + "titan": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
12 + "equal": true
13 + }
14 +]
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap16_2026-11-01T015227.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 16,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.2580101180438449,
10 + "misses": 440,
11 + "stall_s": 0.07488429301884025,
12 + "layers_routed": 46
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.42512908777969016,
20 + "misses": 1002,
21 + "stall_s": 0.18303883913904428,
22 + "layers_routed": 127
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.41904344010530936,
30 + "misses": 1324,
31 + "stall_s": 0.2338036282453686,
32 + "layers_routed": 161
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.45278969957081544,
40 + "misses": 1275,
41 + "stall_s": 0.23169296083506197,
42 + "layers_routed": 164
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:52:27",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap32_2026-11-01T015226.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 32,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.0,
10 + "misses": 430,
11 + "stall_s": 0.07700599753297865,
12 + "layers_routed": 16
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.6060606060606061,
20 + "misses": 429,
21 + "stall_s": 0.07513291679788381,
22 + "layers_routed": 40
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.6090458488228006,
30 + "misses": 631,
31 + "stall_s": 0.10690541681833565,
32 + "layers_routed": 58
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.6224188790560472,
40 + "misses": 512,
41 + "stall_s": 0.08529358741361648,
42 + "layers_routed": 49
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:52:26",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap64_2026-11-01T015225.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 64,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.0,
10 + "misses": 430,
11 + "stall_s": 0.08607849897816777,
12 + "layers_routed": 16
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.4637904468412943,
20 + "misses": 348,
21 + "stall_s": 0.0754558329936117,
22 + "layers_routed": 16
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.823076923076923,
30 + "misses": 138,
31 + "stall_s": 0.03212125087156892,
32 + "layers_routed": 16
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.9234828496042217,
40 + "misses": 58,
41 + "stall_s": 0.018905332079157233,
42 + "layers_routed": 16
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:52:25",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap8_2026-11-01T015229.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 8,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.1625,
10 + "misses": 536,
11 + "stall_s": 0.0982387486146763,
12 + "layers_routed": 80
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.4425981873111782,
20 + "misses": 1476,
21 + "stall_s": 0.28289528854656965,
22 + "layers_routed": 331
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.375,
30 + "misses": 1995,
31 + "stall_s": 0.3788431251887232,
32 + "layers_routed": 399
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.3878048780487805,
40 + "misses": 2008,
41 + "stall_s": 0.377490884042345,
42 + "layers_routed": 410
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:52:29",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/scripts/bench_distributed.py +102 −0
@@ -0,0 +1,102 @@
1 +#!/usr/bin/env python3
2 +"""Coordinator-side distributed benchmark (Gate E): compares local bounded residency vs remote execution of a placed subset.
3 +--workers name=host:port[,name=host:port] --placement {none,odd,half_layers,popular:<traces>} --capacity C
4 +Writes JSONL records (same schema as bench_decode + remote stats) and a correctness check vs mlx-lm on the first prompt.
5 +"""
6 +import argparse, json, os, random, socket, subprocess, sys, time, statistics
7 +import mlx.core as mx, numpy as np, yaml
8 +from pathlib import Path
9 +sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
10 +from mlx_lm import stream_generate
11 +from mlx_lm.utils import load_model, load_tokenizer
12 +from titan.runtime import TitanModel
13 +from transport.client import WorkerClient
14 +sys.path.insert(0, str(Path(__file__).resolve().parent))
15 +from bench_decode import run_one, env_record
16 +
17 +
18 +def make_placement(kind, spec, workers, traces=None):
19 + L, E = spec.num_layers, spec.num_experts
20 + names = list(workers)
21 + if kind == "none":
22 + return {}
23 + if kind == "odd":
24 + return {(l, e): names[0] for l in range(L) for e in range(E) if e % 2 == 1}
25 + if kind == "all_remote":
26 + return {(l, e): names[(l * E + e) % len(names)] for l in range(L) for e in range(E)}
27 + if kind.startswith("popular:"): # least popular experts go remote (coordinator keeps the hot set), split round-robin over workers
28 + d = kind.split(":", 1)[1]; idx = json.load(open(os.path.join(d, "index.json"))); cnt = np.zeros((L, E))
29 + for p in idx["prompts"]:
30 + if p["split"] != "fit": continue
31 + z = np.load(os.path.join(d, p["name"] + ".npz")); t = z["topk"]
32 + for l in range(L):
33 + cnt[l] += np.bincount(t[:, l].reshape(-1), minlength=E)
34 + pl = {}
35 + for l in range(L):
36 + order = np.argsort(cnt[l]) # ascending popularity
37 + for i, e in enumerate(order[: E // 2]):
38 + pl[(l, int(e))] = names[i % len(names)]
39 + return pl
40 + raise ValueError(kind)
41 +
42 +
43 +def main():
44 + ap = argparse.ArgumentParser()
45 + ap.add_argument("--model", required=True)
46 + ap.add_argument("--workers", default="")
47 + ap.add_argument("--token-file", default="~/.titan_token")
48 + ap.add_argument("--placement", default="none")
49 + ap.add_argument("--capacity", type=int, required=True)
50 + ap.add_argument("--predictor", default="none"); ap.add_argument("--m", type=int, default=8); ap.add_argument("--traces", default=None)
51 + ap.add_argument("--workloads", default=str(Path(__file__).resolve().parents[1] / "configs" / "workloads_v2.yaml"))
52 + ap.add_argument("--families", default="general,code"); ap.add_argument("--split", default="heldout")
53 + ap.add_argument("--max-tokens", type=int, default=64); ap.add_argument("--repeats", type=int, default=2); ap.add_argument("--warmup", type=int, default=1)
54 + ap.add_argument("--seed", type=int, default=0); ap.add_argument("--tag", default=""); ap.add_argument("--out", required=True)
55 + a = ap.parse_args()
56 + tm = TitanModel(a.model, capacity=a.capacity, predictor=a.predictor, m=a.m, traces_dir=a.traces)
57 + token = open(os.path.expanduser(a.token_file), "rb").read().strip()
58 + workers = {}
59 + for spec in [w for w in a.workers.split(",") if w]:
60 + name, hp = spec.split("="); host, port = hp.rsplit(":", 1)
61 + workers[name] = WorkerClient(host, int(port), token, tm.spec.model_type, name=name)
62 + placement = make_placement(a.placement, tm.spec, workers, a.traces)
63 + rtts = {n: w.ping(10) for n, w in workers.items()}
64 + if placement:
65 + assign = {}
66 + for (l, e), n in placement.items():
67 + assign.setdefault(n, {}).setdefault(l, []).append(e)
68 + for n, asg in assign.items():
69 + t = time.perf_counter(); r = workers[n].load(asg); print(f"worker {n}: loaded {r['loaded']} experts in {time.perf_counter()-t:.1f}s")
70 + rx = tm.attach_workers(workers, placement)
71 + else:
72 + rx = None
73 + # correctness vs mlx-lm on one prompt
74 + refm, _ = load_model(Path(a.model)); tok = tm.tokenizer
75 + ids = mx.array(tok.encode("The capital of Canada is"))[None]
76 + ref = refm(ids); mx.eval(ref); out = tm(ids); mx.eval(out)
77 + dlogit = float(mx.max(mx.abs(out.astype(mx.float32) - ref.astype(mx.float32))))
78 + del refm
79 + wl = yaml.safe_load(open(a.workloads)); prompts = [(f, p) for f in a.families.split(",") for p in wl["families"][f][a.split]]
80 + rng = random.Random(a.seed); order = []
81 + for rep in range(a.warmup + a.repeats):
82 + ps = prompts[:]; rng.shuffle(ps); order += [(rep, f, p) for f, p in ps]
83 + base = {"backend": "titan-distributed", "capacity": a.capacity, "placement": a.placement, "workers": list(workers), "rtt_ms": rtts,
84 + "n_remote_experts": len(placement), "predictor": a.predictor, "m": a.m, "max_tokens": a.max_tokens, "split": a.split, "tag": a.tag,
85 + "model": os.path.realpath(a.model), "cmd": " ".join(sys.argv), "env": env_record(), "seed": a.seed, "dlogit_vs_mlx_lm": dlogit}
86 + os.makedirs(os.path.dirname(a.out), exist_ok=True)
87 + with open(a.out, "a") as f:
88 + for rep, fam, p in order:
89 + tm.reset_stats(); mx.reset_peak_memory()
90 + if rx: rx.remote_pairs = rx.local_pairs = rx.layers_remote = rx.failures = 0; rx.remote_wait = 0.0
91 + r = run_one(tm.model, tok, p, a.max_tokens)
92 + cs = tm.cache_stats()
93 + rec = dict(base, warmup=rep < a.warmup, repeat=rep - a.warmup, family=fam, prompt=p[:60], ts=time.strftime("%Y-%m-%dT%H:%M:%S"), **r,
94 + cache_hit_rate=cs["hit_rate"], cache_misses=cs["misses"], stall_s=cs["stall_time_s"], remote=rx.stats() if rx else None)
95 + f.write(json.dumps(rec) + "\n"); f.flush()
96 + print(f"dist cap={a.capacity} {a.placement:10s} rep={rep} {fam:10s} ttft={r['ttft_ms']:.0f}ms decode={r['gen_tps']:.1f} tok/s hit={cs['hit_rate']:.3f} stall={cs['stall_time_s']:.2f}s"
97 + + (f" remote_wait={rx.remote_wait:.2f}s remote_pairs={rx.remote_pairs}" if rx else ""))
98 + for w in workers.values(): w.close()
99 +
100 +
101 +if __name__ == "__main__":
102 + main()
added benchmarks/scripts/power_sampler.py +40 −0
@@ -0,0 +1,40 @@
1 +#!/usr/bin/env python3
2 +"""Sample package power with powermetrics (requires sudo) at 1 s intervals into a JSONL file; a companion function integrates
3 +energy over a benchmark's wall-clock window to produce joules per token. Usage:
4 + sudo powermetrics --samplers cpu_power,gpu_power,thermal -i 1000 -f plist | python3 power_sampler.py --out power.jsonl
5 +"""
6 +import argparse, json, plistlib, sys, time
7 +
8 +def main():
9 + ap = argparse.ArgumentParser(); ap.add_argument("--out", required=True); a = ap.parse_args()
10 + buf = b""; out = open(a.out, "a")
11 + for chunk in iter(lambda: sys.stdin.buffer.read(65536), b""):
12 + buf += chunk
13 + while b"</plist>" in buf:
14 + i = buf.index(b"</plist>") + len(b"</plist>")
15 + doc, buf = buf[:i], buf[i:]
16 + s = doc.find(b"<?xml")
17 + if s < 0: continue
18 + try:
19 + d = plistlib.loads(doc[s:])
20 + except Exception:
21 + continue
22 + rec = {"t": time.time(), "elapsed_ns": d.get("elapsed_ns"),
23 + "cpu_mW": (d.get("processor") or {}).get("cpu_power"), "gpu_mW": (d.get("processor") or {}).get("gpu_power"),
24 + "ane_mW": (d.get("processor") or {}).get("ane_power"), "combined_mW": (d.get("processor") or {}).get("combined_power"),
25 + "thermal_pressure": (d.get("thermal_pressure"))}
26 + out.write(json.dumps(rec) + "\n"); out.flush()
27 +
28 +def energy_joules(power_jsonl, t0, t1, field="combined_mW"):
29 + """Integrate power (mW) over [t0, t1] (epoch seconds) with the trapezoid rule; returns joules."""
30 + pts = [json.loads(l) for l in open(power_jsonl)]
31 + pts = [(p["t"], p[field]) for p in pts if p.get(field) is not None and t0 - 1.5 <= p["t"] <= t1 + 1.5]
32 + if len(pts) < 2: return None
33 + e = 0.0
34 + for (ta, pa), (tb, pb) in zip(pts, pts[1:]):
35 + lo, hi = max(ta, t0), min(tb, t1)
36 + if hi > lo: e += (pa + pb) / 2 * 1e-3 * (hi - lo)
37 + return e
38 +
39 +if __name__ == "__main__":
40 + main()
modified research/LOG.md +11 −0
@@ -92,3 +92,14 @@ Chronological, append-only. Times are local (America/Toronto). Every entry names
92 92 expert-contiguous blob (one 4 KiB-aligned 3.4 MB record per expert, 3.6 GB written in 3.2 s; `~/.cache/titan/<hash>/`), the registry
93 93 points at it transparently, tests verify byte equality with the original (`tests/test_registry_store.py`, one pread per expert) →
94 94 **36 tok/s**, stall 1.2 s / 64 tokens (`quick_v3.jsonl`), vs 51–73 tok/s from the page cache. Cold sweep restarted with v3.
95 +- **Gate E (transport)** implemented: `transport/protocol.py` (TX1 frames, HMAC-SHA256 on every frame with a pre-shared token kept in
96 + `~/.titan_token`, mode 600, never in the repo), `transport/worker.py` (resident expert subset, exact gather_qmm + swiglu per (row, k)
97 + pair, IP allow-list, timeouts), `transport/client.py`, `titan/distributed.py` (dispatch to every worker before waiting on any, local
98 + execution of the rest, assembly in original (row, k) order, local fallback on failure). Localhost two-process test on M3U96a
99 + (`benchmarks/raw/tests/m3u96a/pytest_distributed_localhost_*.txt`, `distributed_localhost_olmoe_*.json`): odd experts of every layer
100 + executed by the worker → logits **max |Δ| = 0.0** vs mlx-lm, greedy 24 tokens identical, 16 layers remote per token, localhost RTT
101 + 0.11–0.14 ms. Two-node runs (coordinator M3U96a ↔ worker M3U96b over 1 GbE) launched: `benchmarks/raw/gateE/two_node_m3u96a_m3u96b.jsonl`
102 + (local control at 32 slots: 68–88 tok/s page-cached; `odd` placement at 32 and 64 slots pending).
103 +- Cold sweep v3 (in progress, 16 slots): none ≈ 36 tok/s; transition2 31.7–34.1 (stall 0.82–0.92 s vs ≈ 1.2 s), transition3 26.8–28.0,
104 + prev_rank (m = 16) 18–19 tok/s → deeper horizons cut the stall by ~30 % but the host-side cost of prefetch (installs, GIL contention of
105 + reader threads, eviction churn) cancels the gain on this small model; to be attributed with per-token timers after the sweep.
added tests/test_distributed_local.py +58 −0
@@ -0,0 +1,58 @@
1 +"""Two-process exactness test: a worker on localhost holds half of every layer's experts; the coordinator executes those
2 +remotely and the rest locally. Logits must equal mlx-lm's bit-for-bit (same kernels, same bytes, same aggregation order)."""
3 +import os, secrets, socket, subprocess, sys, time, json
4 +import mlx.core as mx
5 +import pytest
6 +from pathlib import Path
7 +from mlx_lm.utils import load_model, load_tokenizer
8 +from titan.runtime import TitanModel
9 +from transport.client import WorkerClient
10 +
11 +PROMPTS = ["The capital of Canada is", "def fibonacci(n):\n \"\"\"Return the n-th Fibonacci number.\"\"\"\n"]
12 +
13 +@pytest.fixture(scope="module")
14 +def worker(model_dir, tmp_path_factory):
15 + tok = secrets.token_hex(32).encode()
16 + tf = tmp_path_factory.mktemp("tok") / "token"; tf.write_bytes(tok)
17 + port = 9400 + os.getpid() % 100
18 + p = subprocess.Popen([sys.executable, "-m", "transport.worker", "--model", model_dir, "--listen", f"127.0.0.1:{port}", "--token-file", str(tf), "--allow", "127.0.0.1"],
19 + cwd=str(Path(__file__).resolve().parents[1]), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
20 + for _ in range(100):
21 + try:
22 + s = socket.create_connection(("127.0.0.1", port), timeout=0.5); s.close(); break
23 + except OSError:
24 + time.sleep(0.2)
25 + yield {"port": port, "token": tok, "proc": p}
26 + p.terminate()
27 +
28 +def test_remote_half_experts_bit_identical(model_dir, worker):
29 + refm, _ = load_model(Path(model_dir)); tok = load_tokenizer(Path(model_dir))
30 + tm = TitanModel(model_dir, capacity=max(8, tm_cap(model_dir)))
31 + E = tm.spec.num_experts; L = tm.spec.num_layers
32 + placement = {(l, e): ("w0" if e % 2 == 1 else None) for l in range(L) for e in range(E)} # odd experts remote
33 + w = WorkerClient("127.0.0.1", worker["port"], worker["token"], tm.spec.model_type, name="w0")
34 + rtt = w.ping(5)
35 + w.load({l: [e for e in range(E) if e % 2 == 1] for l in range(L)})
36 + rx = tm.attach_workers({"w0": w}, placement)
37 + rec = {"rtt_ms_localhost": rtt, "prompts": []}
38 + for p in PROMPTS:
39 + ids = mx.array(tok.encode(p))[None]
40 + ref = refm(ids); mx.eval(ref)
41 + tm.reset_stats()
42 + out = tm(ids); mx.eval(out)
43 + d = float(mx.max(mx.abs(out.astype(mx.float32) - ref.astype(mx.float32))))
44 + rec["prompts"].append({"prompt": p[:30], "max_abs_dlogit": d, "argmax_equal": bool(mx.array_equal(mx.argmax(out, -1), mx.argmax(ref, -1))), "remote": rx.stats()})
45 + assert d <= 1e-3, (p, d)
46 + # greedy decode token-identical
47 + from mlx_lm import generate
48 + a = generate(refm, tok, prompt=PROMPTS[0], max_tokens=24, verbose=False)
49 + b = generate(tm.model, tok, prompt=PROMPTS[0], max_tokens=24, verbose=False)
50 + rec["greedy_equal"] = a == b; rec["greedy_ref"] = a; rec["greedy_titan"] = b
51 + assert a == b
52 + os.makedirs("benchmarks/raw/tests", exist_ok=True)
53 + json.dump(rec, open(f"benchmarks/raw/tests/distributed_localhost_{tm.spec.model_type}_{time.strftime('%Y%m%dT%H%M%S')}.json", "w"), indent=2)
54 + w.close()
55 +
56 +def tm_cap(model_dir):
57 + from titan.registry import load_spec
58 + return load_spec(model_dir).num_experts // 2
added titan/distributed.py +94 −0
@@ -0,0 +1,94 @@
1 +"""Distributed expert execution: a placement maps (layer, expert) -> worker name (or None = local). For each layer the
2 +execution block splits the (row, k) pairs between local slots and remote workers, sends every worker its batch BEFORE waiting
3 +for any reply (so a layer waits for its slowest worker, not for each in turn), then assembles y[row, k, :] in the original
4 +order so the weighted sum is bit-identical to the single-node path. Workers that time out or fail raise; the caller may
5 +retry locally (fallback) — the expert exists on local NVMe too.
6 +"""
7 +from __future__ import annotations
8 +import time
9 +from typing import Dict, List, Optional, Tuple
10 +import numpy as np
11 +import mlx.core as mx
12 +from mlx_lm.models.activations import swiglu
13 +from transport.client import WorkerClient
14 +
15 +
16 +class RemoteExecutor:
17 + def __init__(self, workers: Dict[str, WorkerClient], placement: Dict[Tuple[int, int], Optional[str]], fallback_local: bool = True):
18 + self.workers = workers
19 + self.placement = placement
20 + self.fallback_local = fallback_local
21 + self.remote_pairs = 0; self.local_pairs = 0; self.layers_remote = 0; self.remote_wait = 0.0; self.failures = 0
22 +
23 + def where(self, layer: int, expert: int) -> Optional[str]:
24 + return self.placement.get((layer, expert))
25 +
26 + def split(self, layer: int, flat: List[int], k: int):
27 + """flat: expert ids in (row, k) order. Returns local pairs and per-worker pairs as (position, row, expert)."""
28 + local, remote = [], {}
29 + for pos, e in enumerate(flat):
30 + w = self.where(layer, e)
31 + if w is None or w not in self.workers:
32 + local.append((pos, pos // k, e))
33 + else:
34 + remote.setdefault(w, []).append((pos, pos // k, e))
35 + return local, remote
36 +
37 + def run_layer(self, block, x_flat: mx.array, flat: List[int], shape, probs_np=None):
38 + """Execute one layer's (row, k) pairs locally and remotely; returns y of shape (*shape, D)."""
39 + k = shape[-1]
40 + local, remote = self.split(block.table.layer, flat, k)
41 + R, D = x_flat.shape
42 + outs = [None] * len(flat)
43 + # 1. dispatch every remote batch first (activations as float16 rows)
44 + if remote:
45 + x_np = np.array(x_flat.astype(mx.float16))
46 + for w, pairs in remote.items():
47 + self.workers[w].exec_async(block.table.layer, x_np, [(r, e) for _, r, e in pairs])
48 + self.layers_remote += 1
49 + # 2. local experts through the slot table (same code path as single node)
50 + if local:
51 + lflat = [e for _, _, e in local]
52 + rows = [r for _, r, _ in local]
53 + uniq = list(dict.fromkeys(lflat))
54 + if block.prefetcher is not None:
55 + block.prefetcher.before_layer(block.table.layer, uniq); block.prefetcher.after_layer(block.table.layer, uniq, probs_np)
56 + slot_of = dict(zip(uniq, block.table.ensure(uniq)))
57 + sidx = mx.array([slot_of[e] for e in lflat], dtype=mx.uint32).reshape(-1, 1)
58 + xs = mx.expand_dims(x_flat[mx.array(rows, dtype=mx.uint32)], (-2, -3))
59 + up = block._qmm(xs, "up", sidx, False); gate = block._qmm(xs, "gate", sidx, False)
60 + yl = block._qmm(swiglu(gate, up), "down", sidx, False).squeeze(-2).squeeze(-2) # [P, D]
61 + for i, (pos, _, _) in enumerate(local):
62 + outs[pos] = yl[i]
63 + self.local_pairs += len(local)
64 + # 3. collect remote replies
65 + if remote:
66 + t = time.perf_counter()
67 + for w, pairs in remote.items():
68 + try:
69 + y, _ = self.workers[w].exec_wait()
70 + except Exception as exc:
71 + self.failures += 1
72 + if not self.fallback_local:
73 + raise
74 + # local fallback: execute the failed pairs locally
75 + lflat = [e for _, _, e in pairs]; rows = [r for _, r, _ in pairs]
76 + uniq = list(dict.fromkeys(lflat)); slot_of = dict(zip(uniq, block.table.ensure(uniq)))
77 + sidx = mx.array([slot_of[e] for e in lflat], dtype=mx.uint32).reshape(-1, 1)
78 + xs = mx.expand_dims(x_flat[mx.array(rows, dtype=mx.uint32)], (-2, -3))
79 + up = block._qmm(xs, "up", sidx, False); gate = block._qmm(xs, "gate", sidx, False)
80 + yf = block._qmm(swiglu(gate, up), "down", sidx, False).squeeze(-2).squeeze(-2)
81 + for i, (pos, _, _) in enumerate(pairs):
82 + outs[pos] = yf[i]
83 + continue
84 + ym = mx.array(y)
85 + for i, (pos, _, _) in enumerate(pairs):
86 + outs[pos] = ym[i]
87 + self.remote_pairs += len(pairs)
88 + self.remote_wait += time.perf_counter() - t
89 + y = mx.stack(outs, 0).reshape(tuple(shape) + (D,))
90 + return y
91 +
92 + def stats(self):
93 + return {"remote_pairs": self.remote_pairs, "local_pairs": self.local_pairs, "layers_remote": self.layers_remote,
94 + "remote_wait_s": self.remote_wait, "failures": self.failures, "workers": {n: w.stats() for n, w in self.workers.items()}}
modified titan/moe.py +7 −0
@@ -24,6 +24,7 @@ class TitanSwitchGLU:
24 24 self.table = table
25 25 self.prefetcher = prefetcher
26 26 self.current_probs = None
27 + self.remote = None # RemoteExecutor (Gate E); when set, experts placed on workers are executed remotely
27 28 self.group_size = group_size
28 29 self.bits = bits
29 30 self.on_route = on_route # hook(layer, unique_experts) used by predictors/prefetchers (Gate C)
@@ -88,6 +89,12 @@ class TitanSwitchGLU:
88 89 if cur:
89 90 chunks.append(cur)
90 91 self.n_chunks = len(chunks)
92 + if self.remote is not None:
93 + outs = []
94 + for ci, rows in enumerate(chunks):
95 + f = [flat[r * k + j] for r in rows for j in range(k)]
96 + outs.append(self.remote.run_layer(self, x2[rows[0]:rows[-1] + 1], f, (len(rows), k), probs_np if ci == len(chunks) - 1 else None))
97 + return mx.concatenate(outs, axis=0).reshape(indices.shape + (x.shape[-1],))
91 98 if len(chunks) == 1:
92 99 return self._run(x, flat, indices.shape, probs_np)
93 100 outs = []
modified titan/runtime.py +10 −0
@@ -67,6 +67,16 @@ class TitanModel:
67 67 def __call__(self, inputs, cache=None):
68 68 return self.model(inputs, cache=cache)
69 69
70 + def attach_workers(self, workers: Dict[str, "WorkerClient"], placement: Dict, fallback_local: bool = True):
71 + """Enable distributed execution (Gate E). placement: {(layer, expert): worker_name or None}."""
72 + from .distributed import RemoteExecutor
73 + self.remote = RemoteExecutor(workers, placement, fallback_local)
74 + for b in self.blocks:
75 + b.remote = self.remote
76 + # a layer with every expert resident locally would take the sync-free fast path and bypass the remote split:
77 + # the capacity is bounded by construction in distributed runs (capacity < num_experts), so no change is needed here.
78 + return self.remote
79 +
70 80 def warm_all(self):
71 81 """Load every expert (only meaningful when capacity == num_experts)."""
72 82 for t in self.tables:
added transport/__init__.py +1 −0
@@ -0,0 +1 @@
1 +"""Authenticated TCP transport for remote expert execution (Gate E)."""
added transport/client.py +50 −0
@@ -0,0 +1,50 @@
1 +"""Coordinator-side connection to one expert worker."""
2 +from __future__ import annotations
3 +import os, socket, time
4 +import numpy as np
5 +from transport.protocol import Framer
6 +
7 +
8 +class WorkerClient:
9 + def __init__(self, host: str, port: int, token: bytes, model_type: str, timeout: float = 30.0, name: str = None):
10 + self.host, self.port, self.name = host, port, name or host
11 + self.sock = socket.create_connection((host, port), timeout=timeout)
12 + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
13 + self.f = Framer(self.sock, token)
14 + self.f.send("hello", {"proto": "TX1", "model_type": model_type})
15 + kind, h, _ = self.f.recv()
16 + if kind != "hello_ack":
17 + raise ConnectionError(f"{host}: {h}")
18 + self.info = h
19 + self.exec_calls = 0; self.exec_wall = 0.0; self.rows_sent = 0
20 +
21 + def ping(self, n=5):
22 + rtts = []
23 + for _ in range(n):
24 + t = time.perf_counter(); self.f.send("ping", {}); self.f.recv(); rtts.append((time.perf_counter() - t) * 1e3)
25 + return rtts
26 +
27 + def load(self, assignments):
28 + self.f.send("load", {"assignments": {str(l): list(map(int, es)) for l, es in assignments.items()}})
29 + kind, h, _ = self.f.recv()
30 + if kind != "load_reply":
31 + raise RuntimeError(h)
32 + return h
33 +
34 + def exec_async(self, layer: int, rows: np.ndarray, pairs):
35 + """Send an exec request (non-blocking on the reply)."""
36 + self.f.send("exec", {"layer": int(layer), "rows": int(rows.shape[0]), "dim": int(rows.shape[1]), "pairs": [[int(r), int(e)] for r, e in pairs]}, rows.tobytes())
37 + self._t0 = time.perf_counter(); self.rows_sent += rows.shape[0]
38 +
39 + def exec_wait(self):
40 + kind, h, payload = self.f.recv()
41 + if kind != "exec_reply":
42 + raise RuntimeError(h)
43 + self.exec_calls += 1; self.exec_wall += time.perf_counter() - self._t0
44 + return np.frombuffer(payload, dtype=np.float16).reshape(h["n"], -1), h
45 +
46 + def stats(self):
47 + return {"worker": self.name, "exec_calls": self.exec_calls, "exec_wall_s": self.exec_wall, "bytes_sent": self.f.bytes_sent, "bytes_recv": self.f.bytes_recv}
48 +
49 + def close(self):
50 + self.sock.close()
added transport/protocol.py +55 −0
@@ -0,0 +1,55 @@
1 +"""Wire protocol TITAN-X1: length-prefixed frames, HMAC-SHA256 authentication of every frame with a pre-shared token.
2 +
3 +Frame = magic(4) | kind(1) | seq(4) | header_len(4) | payload_len(8) | header(JSON) | payload(bytes) | mac(32)
4 +mac = HMAC(token, magic..payload). A worker only answers frames whose MAC verifies; a bad MAC closes the connection.
5 +Kinds: 0 hello, 1 hello-ack, 2 exec request, 3 exec reply, 4 load request (install experts), 5 load reply, 6 ping, 7 pong, 9 error.
6 +Activations are sent as raw float16 rows; results come back as raw float16 rows (one per (row, k) slot), so the coordinator
7 +performs the weighted sum in exactly the local order (bit-for-bit identical aggregation).
8 +"""
9 +from __future__ import annotations
10 +import hashlib, hmac, json, socket, struct
11 +
12 +MAGIC = b"TX1\0"
13 +HDR = struct.Struct("<4sBIIQ")
14 +KIND = {"hello": 0, "hello_ack": 1, "exec": 2, "exec_reply": 3, "load": 4, "load_reply": 5, "ping": 6, "pong": 7, "error": 9}
15 +KIND_NAME = {v: k for k, v in KIND.items()}
16 +
17 +
18 +def _recv_exact(sock: socket.socket, n: int) -> bytes:
19 + buf = bytearray(n); view = memoryview(buf); got = 0
20 + while got < n:
21 + r = sock.recv_into(view[got:], n - got)
22 + if r == 0:
23 + raise ConnectionError("peer closed")
24 + got += r
25 + return bytes(buf)
26 +
27 +
28 +class Framer:
29 + def __init__(self, sock: socket.socket, token: bytes):
30 + self.sock = sock
31 + self.token = token
32 + self.seq = 0
33 + self.bytes_sent = 0
34 + self.bytes_recv = 0
35 +
36 + def send(self, kind: str, header: dict, payload: bytes = b""):
37 + h = json.dumps(header, separators=(",", ":")).encode()
38 + head = HDR.pack(MAGIC, KIND[kind], self.seq, len(h), len(payload))
39 + mac = hmac.new(self.token, head + h + payload, hashlib.sha256).digest()
40 + self.sock.sendall(b"".join([head, h, payload, mac]))
41 + self.seq += 1
42 + self.bytes_sent += HDR.size + len(h) + len(payload) + 32
43 +
44 + def recv(self):
45 + head = _recv_exact(self.sock, HDR.size)
46 + magic, kind, seq, hl, pl = HDR.unpack(head)
47 + if magic != MAGIC:
48 + raise ConnectionError("bad magic")
49 + h = _recv_exact(self.sock, hl)
50 + payload = _recv_exact(self.sock, pl)
51 + mac = _recv_exact(self.sock, 32)
52 + if not hmac.compare_digest(mac, hmac.new(self.token, head + h + payload, hashlib.sha256).digest()):
53 + raise ConnectionError("bad MAC")
54 + self.bytes_recv += HDR.size + hl + pl + 32
55 + return KIND_NAME[kind], json.loads(h), payload
added transport/worker.py +120 −0
@@ -0,0 +1,120 @@
1 +#!/usr/bin/env python3
2 +"""Expert worker: holds a set of (layer, expert) pairs resident (loaded from its local copy of the checkpoint) and executes them
3 +on activations sent by the coordinator. Exact: the same gather_qmm + swiglu sequence as the local block, per (row, k) slot.
4 +
5 + python -m transport.worker --model <dir> --listen 0.0.0.0:9411 --token-file ~/.titan_token [--allow 192.168.2.80]
6 +"""
7 +from __future__ import annotations
8 +import argparse, json, os, socket, sys, threading, time
9 +import numpy as np
10 +import mlx.core as mx
11 +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
12 +from mlx_lm.models.activations import swiglu
13 +from titan.registry import load_spec
14 +from titan.store import ExpertStore
15 +from titan.slots import SlotTable
16 +from transport.protocol import Framer
17 +
18 +
19 +class Worker:
20 + def __init__(self, model_dir, capacity_per_layer=None):
21 + self.spec = load_spec(model_dir)
22 + self.store = ExpertStore(self.spec)
23 + q = self.spec.quantization
24 + self.gs, self.bits = q["group_size"], q["bits"]
25 + self.tables = {}
26 + self.cap = capacity_per_layer or self.spec.num_experts
27 + self.lock = threading.Lock()
28 + self.exec_count = 0; self.exec_time = 0.0
29 +
30 + def table(self, l):
31 + if l not in self.tables:
32 + self.tables[l] = SlotTable(l, self.cap, self.store)
33 + return self.tables[l]
34 +
35 + def load(self, assignments):
36 + """assignments: {layer: [experts]} -> make them resident."""
37 + n = 0
38 + for l, es in assignments.items():
39 + t = self.table(int(l)); t.ensure([int(e) for e in es]); n += len(es)
40 + mx.eval(*[b for t in self.tables.values() for b in t.buf.values()])
41 + return n
42 +
43 + def execute(self, layer, rows_f16: np.ndarray, pairs):
44 + """rows_f16: [R, D] activations; pairs: list of (row, expert). Returns [len(pairs), D] float16 outputs."""
45 + t0 = time.perf_counter()
46 + tab = self.table(layer)
47 + experts = [e for _, e in pairs]
48 + slots = tab.ensure(list(dict.fromkeys(experts)))
49 + slot_of = dict(zip(dict.fromkeys(experts), slots))
50 + x = mx.array(rows_f16) # [R, D]
51 + ridx = mx.array([r for r, _ in pairs], dtype=mx.uint32)
52 + sidx = mx.array([slot_of[e] for e in experts], dtype=mx.uint32)
53 + xs = x[ridx] # [P, D]
54 + xe = mx.expand_dims(xs, (-2, -3)) # [P, 1, 1, D]
55 + b = tab.buf
56 + def qmm(h, m):
57 + return mx.gather_qmm(h, b[f"{m}.weight"], b[f"{m}.scales"], b.get(f"{m}.biases"), rhs_indices=sidx.reshape(-1, 1),
58 + transpose=True, group_size=self.gs, bits=self.bits)
59 + up = qmm(xe, "up"); gate = qmm(xe, "gate")
60 + y = qmm(swiglu(gate, up), "down").squeeze(-2).squeeze(-2) # [P, D]
61 + out = np.array(y.astype(mx.float16))
62 + self.exec_count += 1; self.exec_time += time.perf_counter() - t0
63 + return out
64 +
65 +
66 +def serve(args):
67 + token = open(os.path.expanduser(args.token_file), "rb").read().strip()
68 + allow = set(args.allow.split(",")) if args.allow else None
69 + w = Worker(args.model, args.capacity)
70 + host, port = args.listen.rsplit(":", 1)
71 + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM); srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
72 + srv.bind((host, int(port))); srv.listen(4)
73 + print(f"[worker] {socket.gethostname()} listening on {args.listen}, model {w.spec.model_type}, cap {w.cap}", flush=True)
74 + while True:
75 + conn, addr = srv.accept()
76 + if allow and addr[0] not in allow:
77 + print(f"[worker] refused {addr}", flush=True); conn.close(); continue
78 + conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
79 + conn.settimeout(args.timeout)
80 + threading.Thread(target=handle, args=(conn, addr, token, w), daemon=True).start()
81 +
82 +
83 +def handle(conn, addr, token, w: Worker):
84 + f = Framer(conn, token)
85 + try:
86 + kind, h, _ = f.recv()
87 + if kind != "hello" or h.get("proto") != "TX1" or h.get("model_type") != w.spec.model_type:
88 + f.send("error", {"msg": "bad hello"}); conn.close(); return
89 + f.send("hello_ack", {"host": socket.gethostname(), "num_layers": w.spec.num_layers, "num_experts": w.spec.num_experts,
90 + "cap": w.cap, "resident": {str(l): t.resident() for l, t in w.tables.items()}})
91 + print(f"[worker] coordinator {addr} connected", flush=True)
92 + while True:
93 + kind, h, payload = f.recv()
94 + if kind == "ping":
95 + f.send("pong", {"t": time.time()})
96 + elif kind == "load":
97 + n = w.load(h["assignments"])
98 + f.send("load_reply", {"loaded": n, "resident": {str(l): t.resident() for l, t in w.tables.items()}})
99 + elif kind == "exec":
100 + R, D = h["rows"], h["dim"]
101 + x = np.frombuffer(payload, dtype=np.float16).reshape(R, D)
102 + out = w.execute(h["layer"], x, [tuple(p) for p in h["pairs"]])
103 + f.send("exec_reply", {"layer": h["layer"], "n": out.shape[0], "exec_ms": w.exec_time * 1e3}, out.tobytes())
104 + else:
105 + f.send("error", {"msg": f"unexpected {kind}"})
106 + except (ConnectionError, socket.timeout, OSError) as e:
107 + print(f"[worker] {addr} closed: {e}", flush=True)
108 + finally:
109 + conn.close()
110 +
111 +
112 +if __name__ == "__main__":
113 + ap = argparse.ArgumentParser()
114 + ap.add_argument("--model", required=True)
115 + ap.add_argument("--listen", default="0.0.0.0:9411")
116 + ap.add_argument("--token-file", default="~/.titan_token")
117 + ap.add_argument("--allow", default=None, help="comma-separated coordinator IPs allowed to connect")
118 + ap.add_argument("--capacity", type=int, default=None)
119 + ap.add_argument("--timeout", type=float, default=600.0)
120 + serve(ap.parse_args())
121
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere