SPB Git
spboucher.ai

spb/wp33_uqo

Public MIT

Probe predictor (fixed) results, depth variants, attribution figure/table, H4 table; LOG

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 1, 2026) parent 7b5afc7 Browse files

18 changed files +135 −16

18 changed files
Unified Split
modified benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl +24 −0
@@ -28,3 +28,27 @@
28 28 {"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:04:40", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 341.52458293829113, "prompt_tps": 41.02854651024846, "gen_tps": 27.240125215225166, "decode_tps_wall": 26.81462332201316, "itl_ms_p50": 37.03995898831636, "itl_ms_p95": 44.634749996475875, "itl_ms_p99": 48.89829200692475, "itl_ms_mean": 37.29308400088773, "total_s": 2.6909888749942183, "peak_mem_gb": 1.77610698, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6412616339193382, "cache_misses": 3469, "cache_hits": 6201, "stall_s": 1.1144671511137858, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 5172, "installed": 5169, "late": 201, "dup": 5320, "predictor": "transition2t50", "m": 16}, "host_time_s": {"sync": 1.34039991220925, "prefetch": 0.16478171071503311, "ensure": 1.0864357101963833, "build": 0.017624183907173574}, "ssd_bytes": 30580015104, "ssd_read_s": 4.9008175508352}
29 29 {"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:04:42", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 325.52800001576543, "prompt_tps": 43.04411021589692, "gen_tps": 27.72629489769962, "decode_tps_wall": 27.293189284377878, "itl_ms_p50": 36.34737501852214, "itl_ms_p95": 43.429667013697326, "itl_ms_p99": 47.101125004701316, "itl_ms_mean": 36.63917725336634, "total_s": 2.633796166977845, "peak_mem_gb": 1.78672374, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.6413650465356774, "cache_misses": 3468, "cache_hits": 6202, "stall_s": 1.099088135175407, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 5178, "installed": 5178, "late": 193, "dup": 5314, "predictor": "transition2t50", "m": 16}, "host_time_s": {"sync": 1.3210817894432694, "prefetch": 0.15844549681060016, "ensure": 1.0648446546401829, "build": 0.015979969641193748}, "ssd_bytes": 30597709824, "ssd_read_s": 4.831111097242683}
30 30 {"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "transition2t50", "m": 16, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor transition2t50 --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "d3deca4"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:04:45", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 349.48175004683435, "prompt_tps": 51.5479356880377, "gen_tps": 31.327212216405787, "decode_tps_wall": 30.83788490730665, "itl_ms_p50": 32.85600000526756, "itl_ms_p95": 38.74312504194677, "itl_ms_p99": 43.36958297062665, "itl_ms_mean": 32.427645508303414, "total_s": 2.392423417069949, "peak_mem_gb": 1.835944588, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.6676039365334404, "cache_misses": 3310, "cache_hits": 6648, "stall_s": 0.9897855184972286, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3640, "installed": 3640, "late": 148, "dup": 5946, "predictor": "transition2t50", "m": 16}, "host_time_s": {"sync": 1.2337870801566169, "prefetch": 0.11556095094420016, "ensure": 0.9695895125623792, "build": 0.01276098215021193}, "ssd_bytes": 24595660800, "ssd_read_s": 3.909353203023784}
31 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:11:42", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 502.4365830468014, "prompt_tps": 36.03773749574263, "gen_tps": 32.921306889228994, "decode_tps_wall": 32.40808744423553, "itl_ms_p50": 30.22433293517679, "itl_ms_p95": 39.29099999368191, "itl_ms_p99": 41.23612493276596, "itl_ms_mean": 30.85649536464304, "total_s": 2.446395791019313, "peak_mem_gb": 1.84980092, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.7703354087166098, "cache_misses": 2287, "cache_hits": 7671, "stall_s": 0.9387649646960199, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3996, "installed": 3989, "late": 373, "dup": 3924, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 1.2815006269374862, "prefetch": 0.18630608334206045, "ensure": 0.8789908715989441, "build": 0.017593756318092346}, "ssd_bytes": 22235185152, "ssd_read_s": 3.9021120477700606}
32 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:11:44", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 373.23191692121327, "prompt_tps": 37.54289365640219, "gen_tps": 31.40608155351881, "decode_tps_wall": 30.915513876587053, "itl_ms_p50": 32.02120796777308, "itl_ms_p95": 39.44387496449053, "itl_ms_p99": 46.47145897615701, "itl_ms_mean": 32.34621957092295, "total_s": 2.411043749889359, "peak_mem_gb": 1.760650756, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.774870734229576, "cache_misses": 2177, "cache_hits": 7493, "stall_s": 0.8995586994569749, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4130, "installed": 4131, "late": 419, "dup": 4422, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 1.2754781130934134, "prefetch": 0.20737914938945323, "ensure": 0.8304450281430036, "build": 0.01828774716705084}, "ssd_bytes": 22320119808, "ssd_read_s": 3.8726607081480324}
33 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:11:47", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 408.2109589362517, "prompt_tps": 44.1305906364464, "gen_tps": 32.024858639512054, "decode_tps_wall": 31.524639137267656, "itl_ms_p50": 31.12475003581494, "itl_ms_p95": 39.56354199908674, "itl_ms_p99": 43.24475000612438, "itl_ms_mean": 31.72121957195775, "total_s": 2.40664779196959, "peak_mem_gb": 1.849231, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.7755573408314923, "cache_misses": 2235, "cache_hits": 7723, "stall_s": 0.9101867240387946, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3949, "installed": 3941, "late": 422, "dup": 4763, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 1.263371414737776, "prefetch": 0.21923305245582014, "ensure": 0.8266534864669666, "build": 0.01808950351551175}, "ssd_bytes": 21884829696, "ssd_read_s": 3.7697257244726643}
34 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:11:49", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 356.31954204291105, "prompt_tps": 39.327094646264484, "gen_tps": 31.596870661689568, "decode_tps_wall": 31.10333208308868, "itl_ms_p50": 31.790749984793365, "itl_ms_p95": 38.842375040985644, "itl_ms_p99": 45.3370000468567, "itl_ms_mean": 32.15089615892678, "total_s": 2.381826000055298, "peak_mem_gb": 1.770332132, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.77435367114788, "cache_misses": 2182, "cache_hits": 7488, "stall_s": 0.8886842440115288, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4122, "installed": 4131, "late": 420, "dup": 4430, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 1.2639832042623311, "prefetch": 0.2039724417263642, "ensure": 0.8186647708062083, "build": 0.017605758039280772}, "ssd_bytes": 22309502976, "ssd_read_s": 3.8125432294327766}
35 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:11:52", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 352.57408302277327, "prompt_tps": 39.74218675299524, "gen_tps": 31.65769523405388, "decode_tps_wall": 31.163191468369526, "itl_ms_p50": 32.34412509482354, "itl_ms_p95": 39.54333404544741, "itl_ms_p99": 44.45024998858571, "itl_ms_mean": 32.089139554753075, "total_s": 2.374189874972217, "peak_mem_gb": 1.760503268, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.7731127197518097, "cache_misses": 2194, "cache_hits": 7476, "stall_s": 0.8860083669424057, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 4126, "installed": 4126, "late": 418, "dup": 4426, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 1.2627789897378534, "prefetch": 0.19917341321706772, "ensure": 0.8180779997492209, "build": 0.01713948172982782}, "ssd_bytes": 22366126080, "ssd_read_s": 3.8371908410917968}
36 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:11:54", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 412.27854101452976, "prompt_tps": 43.690897816037726, "gen_tps": 32.84528995900421, "decode_tps_wall": 32.33228557038855, "itl_ms_p50": 30.302374972961843, "itl_ms_p95": 39.13604200351983, "itl_ms_p99": 40.90400005225092, "itl_ms_mean": 30.928837301741755, "total_s": 2.3607952910242602, "peak_mem_gb": 1.868302056, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.7755573408314923, "cache_misses": 2235, "cache_hits": 7723, "stall_s": 0.8902485313592479, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3947, "installed": 3940, "late": 439, "dup": 4765, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 1.2474988406756893, "prefetch": 0.20962574891746044, "ensure": 0.8094144120113924, "build": 0.017095185932703316}, "ssd_bytes": 21877751808, "ssd_read_s": 3.686413793475367}
37 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 16, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:14:14", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 895.8566249348223, "prompt_tps": 20.20077702500302, "gen_tps": 25.055397954692612, "decode_tps_wall": 24.66505483870544, "itl_ms_p50": 38.97029091604054, "itl_ms_p95": 55.66262500360608, "itl_ms_p99": 60.62483391724527, "itl_ms_mean": 40.54318981001242, "total_s": 3.450077582965605, "peak_mem_gb": 1.822079188, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.8403293834103234, "cache_misses": 1590, "cache_hits": 8368, "stall_s": 1.4188859943533316, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 8008, "installed": 7942, "late": 1628, "dup": 7832, "predictor": "probe", "m": 16}, "host_time_s": {"sync": 1.6609962891088799, "prefetch": 0.8776716555003077, "ensure": 0.7751008678460494, "build": 0.022822543629445136}, "ssd_bytes": 33966784512, "ssd_read_s": 6.307455720496364}
38 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 16, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:14:17", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 443.58262489549816, "prompt_tps": 31.606154157510538, "gen_tps": 24.469822890051635, "decode_tps_wall": 24.087597415190032, "itl_ms_p50": 40.289958007633686, "itl_ms_p95": 52.425541914999485, "itl_ms_p99": 75.69891691673547, "itl_ms_mean": 41.51514087367566, "total_s": 3.059036499937065, "peak_mem_gb": 1.884732324, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.8466390899689762, "cache_misses": 1483, "cache_hits": 8187, "stall_s": 1.1316436724737287, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 8306, "installed": 8290, "late": 1643, "dup": 8166, "predictor": "probe", "m": 16}, "host_time_s": {"sync": 1.553105992032215, "prefetch": 0.8503941696835682, "ensure": 0.5249539959477261, "build": 0.02193720976356417}, "ssd_bytes": 34642722816, "ssd_read_s": 5.387152876704931}
39 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 16, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:14:20", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 544.7171670384705, "prompt_tps": 33.0735097997283, "gen_tps": 25.842125208025504, "decode_tps_wall": 25.43849308673247, "itl_ms_p50": 37.868833052925766, "itl_ms_p95": 53.40204108506441, "itl_ms_p99": 58.49604192189872, "itl_ms_mean": 39.31050461953476, "total_s": 3.0212789580691606, "peak_mem_gb": 2.003913312, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.8472584856396866, "cache_misses": 1521, "cache_hits": 8437, "stall_s": 1.1443978425813839, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 7934, "installed": 7950, "late": 1592, "dup": 8698, "predictor": "probe", "m": 16}, "host_time_s": {"sync": 1.5227128986734897, "prefetch": 0.826282728696242, "ensure": 0.5458953622728586, "build": 0.021513799438253045}, "ssd_bytes": 33460715520, "ssd_read_s": 5.2902446574298665}
40 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 16, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:14:23", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 430.2754160016775, "prompt_tps": 32.568850599949556, "gen_tps": 24.76517223527836, "decode_tps_wall": 24.378319392562172, "itl_ms_p50": 40.20533396396786, "itl_ms_p95": 52.76508303359151, "itl_ms_p99": 59.0739999897778, "itl_ms_mean": 41.020054906044926, "total_s": 3.0145388750825077, "peak_mem_gb": 1.88860096, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.8464322647362978, "cache_misses": 1485, "cache_hits": 8185, "stall_s": 1.1297015901654959, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 8301, "installed": 8288, "late": 1629, "dup": 8171, "predictor": "probe", "m": 16}, "host_time_s": {"sync": 1.5254358507227153, "prefetch": 0.8407498337328434, "ensure": 0.5250253080157563, "build": 0.020758838509209454}, "ssd_bytes": 34632105984, "ssd_read_s": 5.336683586472645}
41 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 16, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:14:26", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 451.5320840291679, "prompt_tps": 31.036623221476617, "gen_tps": 24.792616062079944, "decode_tps_wall": 24.405261377279007, "itl_ms_p50": 41.400167043320835, "itl_ms_p95": 51.83545802719891, "itl_ms_p99": 59.27916697692126, "itl_ms_mean": 40.97477115860711, "total_s": 3.032942667021416, "peak_mem_gb": 1.94522206, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.8471561530506722, "cache_misses": 1478, "cache_hits": 8192, "stall_s": 1.1289817214710638, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 8313, "installed": 8313, "late": 1632, "dup": 8159, "predictor": "probe", "m": 16}, "host_time_s": {"sync": 1.5312410079641268, "prefetch": 0.8357784831896424, "ensure": 0.5366304551716894, "build": 0.02172651037108153}, "ssd_bytes": 34649800704, "ssd_read_s": 5.371664452250116}
42 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 16, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe --m 16 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:14:29", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 504.2627080110833, "prompt_tps": 35.728608423431574, "gen_tps": 26.203663395431743, "decode_tps_wall": 25.79470949171051, "itl_ms_p50": 37.67937491647899, "itl_ms_p95": 53.558959043584764, "itl_ms_p99": 58.460749918594956, "itl_ms_mean": 38.76763955497789, "total_s": 2.9466239999746904, "peak_mem_gb": 1.9931818, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.8467563767824865, "cache_misses": 1526, "cache_hits": 8432, "stall_s": 1.1152964674402028, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 7921, "installed": 7932, "late": 1617, "dup": 8711, "predictor": "probe", "m": 16}, "host_time_s": {"sync": 1.487131759757176, "prefetch": 0.8097676509059966, "ensure": 0.5279155237367377, "build": 0.020733201992698014}, "ssd_bytes": 33432403968, "ssd_read_s": 5.181995438179001}
43 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe2", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:17:17", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 1001.3276670360938, "prompt_tps": 18.062219831373454, "gen_tps": 30.189756179784066, "decode_tps_wall": 29.71969025036037, "itl_ms_p50": 32.692833920009434, "itl_ms_p95": 43.190291966311634, "itl_ms_p99": 56.555208982899785, "itl_ms_mean": 33.647726190143395, "total_s": 3.121134417015128, "peak_mem_gb": 1.846261988, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.7363928499698734, "cache_misses": 2625, "cache_hits": 7333, "stall_s": 1.3216708386316895, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3701, "installed": 3695, "late": 361, "dup": 4947, "predictor": "probe2", "m": 8}, "host_time_s": {"sync": 1.5726819206029177, "prefetch": 0.21285899879876524, "ensure": 1.2344930110266432, "build": 0.018278107163496315}, "ssd_bytes": 22387359744, "ssd_read_s": 5.210404163459316}
44 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe2", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:17:20", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 399.63033294770867, "prompt_tps": 35.083029868527234, "gen_tps": 30.869702613755237, "decode_tps_wall": 30.387516808679806, "itl_ms_p50": 33.09150005225092, "itl_ms_p95": 39.47520803194493, "itl_ms_p99": 40.891999960877, "itl_ms_mean": 32.90824999936694, "total_s": 2.4728500829078257, "peak_mem_gb": 1.76247242, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.7378490175801448, "cache_misses": 2535, "cache_hits": 7135, "stall_s": 0.9910381255904213, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3868, "installed": 3869, "late": 344, "dup": 4684, "predictor": "probe2", "m": 8}, "host_time_s": {"sync": 1.2793637560680509, "prefetch": 0.18615518277511, "ensure": 0.9213677381630987, "build": 0.01609112205915153}, "ssd_bytes": 22659858432, "ssd_read_s": 4.111017375951633}
45 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe2", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:17:22", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 427.46583302505314, "prompt_tps": 42.14546568691707, "gen_tps": 31.554680624149604, "decode_tps_wall": 31.061781672296963, "itl_ms_p50": 32.138042035512626, "itl_ms_p95": 39.94012507610023, "itl_ms_p99": 45.22000008728355, "itl_ms_mean": 32.193903445399236, "total_s": 2.455681750085205, "peak_mem_gb": 1.830553232, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.7414139385418759, "cache_misses": 2575, "cache_hits": 7383, "stall_s": 1.0046712778275833, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3647, "installed": 3644, "late": 350, "dup": 5065, "predictor": "probe2", "m": 8}, "host_time_s": {"sync": 1.253214834840037, "prefetch": 0.1883800079813227, "ensure": 0.9295253350865096, "build": 0.015667336992919445}, "ssd_bytes": 22019309568, "ssd_read_s": 4.02383333875332}
46 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe2", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:17:24", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 396.5723329456523, "prompt_tps": 35.327768118162375, "gen_tps": 30.97991967772089, "decode_tps_wall": 30.49599742967766, "itl_ms_p50": 32.31929091271013, "itl_ms_p95": 41.01816599722952, "itl_ms_p99": 45.103374985046685, "itl_ms_mean": 32.79118849304579, "total_s": 2.462417208007537, "peak_mem_gb": 1.755394532, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.7381592554291624, "cache_misses": 2532, "cache_hits": 7138, "stall_s": 0.9832507125101984, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3866, "installed": 3869, "late": 358, "dup": 4686, "predictor": "probe2", "m": 8}, "host_time_s": {"sync": 1.2777193939546123, "prefetch": 0.18355376191902906, "ensure": 0.916660810704343, "build": 0.015516375424340367}, "ssd_bytes": 22642163712, "ssd_read_s": 4.108548927935772}
47 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe2", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:17:27", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 401.68699994683266, "prompt_tps": 34.87923066595009, "gen_tps": 31.12611108532413, "decode_tps_wall": 30.639938206164455, "itl_ms_p50": 31.847332953475416, "itl_ms_p95": 40.61833396553993, "itl_ms_p99": 45.17837497405708, "itl_ms_mean": 32.63714153962653, "total_s": 2.4578269169433042, "peak_mem_gb": 1.762732952, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.7381592554291624, "cache_misses": 2532, "cache_hits": 7138, "stall_s": 0.9817315798718482, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3873, "installed": 3874, "late": 347, "dup": 4679, "predictor": "probe2", "m": 8}, "host_time_s": {"sync": 1.2711185896769166, "prefetch": 0.18561373895499855, "ensure": 0.9145922709722072, "build": 0.016176007338799536}, "ssd_bytes": 22666936320, "ssd_read_s": 4.091072851326317}
48 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe2", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe2 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:17:29", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 466.777375084348, "prompt_tps": 38.59071709355048, "gen_tps": 32.02471239861806, "decode_tps_wall": 31.524491235271036, "itl_ms_p50": 31.066416995599866, "itl_ms_p95": 38.839290966279805, "itl_ms_p99": 43.48008299712092, "itl_ms_mean": 31.7213683969356, "total_s": 2.465223584091291, "peak_mem_gb": 1.828194016, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.741715203856196, "cache_misses": 2572, "cache_hits": 7386, "stall_s": 0.9836312148254365, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3648, "installed": 3644, "late": 356, "dup": 5064, "predictor": "probe2", "m": 8}, "host_time_s": {"sync": 1.288077262463048, "prefetch": 0.18259123258758336, "ensure": 0.9126184373162687, "build": 0.01517882663756609}, "ssd_bytes": 22012231680, "ssd_read_s": 3.9626905703917146}
49 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe3", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:19:01", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 1018.3106670156121, "prompt_tps": 17.776447098219712, "gen_tps": 31.891249500646396, "decode_tps_wall": 31.3950959073452, "itl_ms_p50": 30.84979101549834, "itl_ms_p95": 40.79133295454085, "itl_ms_p99": 43.997166911140084, "itl_ms_mean": 31.852108461501462, "total_s": 3.024993500090204, "peak_mem_gb": 1.8370214, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.7214300060253063, "cache_misses": 2774, "cache_hits": 7184, "stall_s": 1.2297115926630795, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3490, "installed": 3482, "late": 341, "dup": 5086, "predictor": "probe3", "m": 8}, "host_time_s": {"sync": 1.5910293539054692, "prefetch": 0.1771570744458586, "ensure": 1.1629536031978205, "build": 0.015928100096061826}, "ssd_bytes": 22167945216, "ssd_read_s": 4.735749578336254}
50 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe3", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:19:03", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 417.7632079226896, "prompt_tps": 33.55570644273465, "gen_tps": 30.655205930220156, "decode_tps_wall": 30.176362270562443, "itl_ms_p50": 33.24174997396767, "itl_ms_p95": 40.57654202915728, "itl_ms_p99": 42.76920796837658, "itl_ms_mean": 33.138520509329815, "total_s": 2.505490000010468, "peak_mem_gb": 1.76902902, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.7127197518097208, "cache_misses": 2778, "cache_hits": 6892, "stall_s": 1.019743283977732, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3569, "installed": 3575, "late": 290, "dup": 4983, "predictor": "probe3", "m": 8}, "host_time_s": {"sync": 1.2866521296091378, "prefetch": 0.16383414575830102, "ensure": 0.9688517367467284, "build": 0.015919991652481258}, "ssd_bytes": 22461677568, "ssd_read_s": 4.080189808388241}
51 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe3", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:19:06", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 513.5413750540465, "prompt_tps": 35.07477991648297, "gen_tps": 31.377521603349404, "decode_tps_wall": 30.88737086696359, "itl_ms_p50": 31.350291916169226, "itl_ms_p95": 40.156333008781075, "itl_ms_p99": 47.554834047332406, "itl_ms_mean": 32.3756918096767, "total_s": 2.553209959063679, "peak_mem_gb": 1.821116128, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.7267523599116288, "cache_misses": 2721, "cache_hits": 7237, "stall_s": 1.0283256021793932, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3440, "installed": 3432, "late": 342, "dup": 5272, "predictor": "probe3", "m": 8}, "host_time_s": {"sync": 1.3280688052764162, "prefetch": 0.17529725469648838, "ensure": 0.9640046572312713, "build": 0.015673740301281214}, "ssd_bytes": 21803433984, "ssd_read_s": 4.0124820349738}
52 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe3", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:19:08", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 445.1344159897417, "prompt_tps": 31.47338430791281, "gen_tps": 29.277490438874107, "decode_tps_wall": 28.820232896597577, "itl_ms_p50": 34.61512504145503, "itl_ms_p95": 43.48645801655948, "itl_ms_p99": 49.46470900904387, "itl_ms_mean": 34.69784590526528, "total_s": 2.6310987080214545, "peak_mem_gb": 1.76902902, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.7127197518097208, "cache_misses": 2778, "cache_hits": 6892, "stall_s": 1.0517092497320846, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3566, "installed": 3574, "late": 288, "dup": 4986, "predictor": "probe3", "m": 8}, "host_time_s": {"sync": 1.36907018115744, "prefetch": 0.1682968153618276, "ensure": 1.001935888431035, "build": 0.017286069924011827}, "ssd_bytes": 22451060736, "ssd_read_s": 4.230282494449057}
53 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe3", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:19:11", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 422.8654580656439, "prompt_tps": 33.132170410624965, "gen_tps": 30.11652802794099, "decode_tps_wall": 29.646075272016446, "itl_ms_p50": 33.69879210367799, "itl_ms_p95": 42.30249999091029, "itl_ms_p99": 46.123499982059, "itl_ms_mean": 33.73127777705945, "total_s": 2.547935958020389, "peak_mem_gb": 1.76177274, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.7116856256463289, "cache_misses": 2788, "cache_hits": 6882, "stall_s": 1.032603504951112, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3573, "installed": 3572, "late": 301, "dup": 4979, "predictor": "probe3", "m": 8}, "host_time_s": {"sync": 1.3089445441728458, "prefetch": 0.1695250024786219, "ensure": 0.9806344367098063, "build": 0.01697311399038881}, "ssd_bytes": 22511222784, "ssd_read_s": 4.126012323773466}
54 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe3", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "attribution", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor probe3 --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general --split heldout --repeats 2 --max-tokens 64 --tag attribution --out benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "7b5afc7"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:19:13", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 487.9956250078976, "prompt_tps": 36.9404389279317, "gen_tps": 32.56493375331503, "decode_tps_wall": 32.05627861299624, "itl_ms_p50": 30.27679189108312, "itl_ms_p95": 37.66083298251033, "itl_ms_p99": 42.33550000935793, "itl_ms_mean": 31.195136905086063, "total_s": 2.4532892500283197, "peak_mem_gb": 1.820984976, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.7265515163687487, "cache_misses": 2723, "cache_hits": 7235, "stall_s": 1.0000087718944997, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 3441, "installed": 3438, "late": 337, "dup": 5271, "predictor": "probe3", "m": 8}, "host_time_s": {"sync": 1.2687757798703387, "prefetch": 0.17196745437104255, "ensure": 0.9326372838113457, "build": 0.014673922210931778}, "ssd_bytes": 21814050816, "ssd_read_s": 3.9007015310926363}
modified benchmarks/scripts/simulate_cache.py +7 −2
@@ -60,10 +60,15 @@ class LayerSim:
60 60 return miss
61 61
62 62
63 +_PRED_CACHE = {}
64 +
63 65 def simulate(traces, L, E, K, cap, policy, pred_name, m, t_read, t_layer, loaders, fit_traces):
64 − pred = make_predictor(pred_name, L, E)
66 + if pred_name not in _PRED_CACHE: # fit each predictor once (on the fit split) and reuse across capacities/splits
67 + pred = make_predictor(pred_name, L, E)
68 + if fit_traces is not None: pred.fit([(t["topk"], t["probs"]) for t in fit_traces])
69 + _PRED_CACHE[pred_name] = pred
70 + pred = _PRED_CACHE[pred_name]
65 71 depth = getattr(pred, "depth", 1)
66 − if fit_traces is not None: pred.fit([(t["topk"], t["probs"]) for t in fit_traces])
67 72 tot = {"tokens": 0, "hits": 0, "misses": 0, "pf_loads": 0, "pf_used": 0, "demand_loads": 0, "stall_s": 0.0, "pred_hits": 0, "pred_total": 0, "evictions": 0}
68 73 per_family = {}
69 74 for tr in traces:
added figures/attribution.pdf +0 −0

Binary file not shown.

modified figures/capacity_sweep.pdf +0 −0

Binary file not shown.

modified figures/sim_hit_vs_capacity.pdf +0 −0

Binary file not shown.

modified figures/two_node.pdf +0 −0

Binary file not shown.

added paper/figures/attribution.pdf +0 −0

Binary file not shown.

modified paper/figures/capacity_sweep.pdf +0 −0

Binary file not shown.

modified paper/figures/sim_hit_vs_capacity.pdf +0 −0

Binary file not shown.

modified paper/figures/two_node.pdf +0 −0

Binary file not shown.

added paper/tables/attr.tex +23 −0
@@ -0,0 +1,23 @@
1 +\begin{table}[t]
2 +\centering\small
3 +\caption{Predictor comparison with host-time attribution, cold NVMe, 16 slots, held-out general prompts (4 runs per row); see Figure~\ref{fig:attribution} for the breakdown.}
4 +\label{tab:attr}
5 +\begin{tabular}{llrrrrrr}
6 +\toprule
7 +Backend & cfg & $n$ & decode tok/s (median) & p95 ITL ms & hit & stall s & TTFT ms \\
8 +\midrule
9 +titan & C=16, none, m=8, cold & 4 & 33.0 & 39.2 & 0.492 & 1.24 & 361 \\
10 +titan & C=16, probe, m=16, cold & 4 & 25.3 & 53.1 & 0.847 & 1.13 & 478 \\
11 +titan & C=16, probe, m=8, cold & 4 & 31.8 & 39.3 & 0.775 & 0.89 & 382 \\
12 +titan & C=16, probe2, m=8, cold & 4 & 31.3 & 40.3 & 0.740 & 0.98 & 415 \\
13 +titan & C=16, probe3, m=8, cold & 4 & 30.7 & 41.2 & 0.720 & 1.03 & 467 \\
14 +titan & C=16, transition2, m=8, cold & 4 & 27.1 & 45.5 & 0.638 & 1.20 & 376 \\
15 +titan & C=16, transition2t50, m=16, cold & 4 & 28.6 & 42.7 & 0.654 & 1.07 & 346 \\
16 +titan & C=16, transition2t50, m=8, cold & 4 & 27.9 & 42.9 & 0.630 & 1.13 & 358 \\
17 +titan & C=16, transition2t70, m=8, cold & 4 & 27.8 & 43.4 & 0.583 & 1.19 & 365 \\
18 +\bottomrule
19 +\end{tabular}
20 +\begin{tablenotes}\footnotesize
21 +\item Sources: \code{benchmarks/raw/gateC/cold\_nvme\_laptop/attribution.jsonl}.
22 +\end{tablenotes}
23 +\end{table}
modified paper/tables/cold.tex +1 −1
@@ -1,6 +1,6 @@
1 1 \begin{table}[t]
2 2 \centering\small
3 −\caption{Cold-NVMe tier on the M5 Max laptop (purge + F\_NOCACHE), OLMoE 4-bit, held-out prompts, 64 tokens.}
3 +\caption{Cold-NVMe tier on the M5 Max laptop (purge + F\_NOCACHE), OLMoE 4-bit, held-out general/code/reasoning prompts, 64 tokens, 12 runs per row.}
4 4 \label{tab:cold}
5 5 \begin{tabular}{llrrrrrr}
6 6 \toprule
added paper/tables/fused.tex +18 −0
@@ -0,0 +1,18 @@
1 +\begin{table}[t]
2 +\centering\small
3 +\caption{H4: fused up+gate gather\_qmm vs two kernels (laptop; 64 slots resident, 128 tokens, 12 runs; 16 slots cold, 64 tokens, 4 runs).}
4 +\label{tab:fused}
5 +\begin{tabular}{llrrrrrr}
6 +\toprule
7 +Backend & cfg & $n$ & decode tok/s (median) & p95 ITL ms & hit & stall s & TTFT ms \\
8 +\midrule
9 +titan & C=16, fused=False, cold & 4 & 35.0 & 36.9 & 0.492 & 1.17 & 348 \\
10 +titan & C=16, fused=True, cold & 4 & 35.9 & 35.0 & 0.492 & 1.10 & 342 \\
11 +titan & C=64, fused=False & 12 & 476.7 & 2.2 & 1.000 & 0.00 & 17 \\
12 +titan & C=64, fused=True & 12 & 475.6 & 2.2 & 1.000 & 0.00 & 18 \\
13 +\bottomrule
14 +\end{tabular}
15 +\begin{tablenotes}\footnotesize
16 +\item Sources: \code{benchmarks/raw/gateD/fused\_upgate\_laptop.jsonl}.
17 +\end{tablenotes}
18 +\end{table}
modified paper/tables/sim.tex +2 −2
@@ -17,10 +17,10 @@ transition2 & fit & 0.663 & 18.0 & 0.758 & 13.0 & 0.832 & 9.1 & 0.938 & 3.4 \\
17 17 transition2 & heldout & 0.672 & 17.6 & 0.761 & 12.9 & 0.833 & 9.0 & 0.940 & 3.3 \\
18 18 transition3 & fit & 0.651 & 18.5 & 0.749 & 13.5 & 0.826 & 9.4 & 0.937 & 3.4 \\
19 19 transition3 & heldout & 0.660 & 18.1 & 0.752 & 13.3 & 0.828 & 9.3 & 0.939 & 3.3 \\
20 −probe & fit & 0.495 & 27.7 & 0.626 & 20.5 & 0.737 & 14.4 & 0.903 & 5.3 \\
21 −probe & heldout & 0.504 & 27.2 & 0.630 & 20.3 & 0.738 & 14.4 & 0.903 & 5.3 \\
22 20 oracle & fit & 0.938 & 3.0 & 0.959 & 2.0 & 0.973 & 1.4 & 0.991 & 0.5 \\
23 21 oracle & heldout & 0.940 & 2.9 & 0.959 & 2.0 & 0.972 & 1.4 & 0.991 & 0.5 \\
22 +probe & fit & 0.813 & 9.8 & 0.868 & 7.0 & 0.908 & 4.9 & 0.965 & 1.9 \\
23 +probe & heldout & 0.811 & 10.0 & 0.864 & 7.3 & 0.904 & 5.1 & 0.965 & 1.9 \\
24 24 \bottomrule
25 25 \end{tabular}
26 26 \begin{tablenotes}\footnotesize
modified research/LOG.md +9 −0
@@ -135,3 +135,12 @@ Chronological, append-only. Times are local (America/Toronto). Every entry names
135 135 all-resident 475.6 vs 476.7 tok/s (12 runs, −0.2 %), cold 16 slots 35.9 vs 35.0 tok/s (4 runs, +2.6 %, within the run-to-run range).
136 136 As the profiling predicted, saving one dispatch per layer is immaterial at batch 1 → **H4 not supported** for this fusion; no custom
137 137 Metal kernel is justified by the profile (decode is launch/latency-bound, 28 % of nominal bandwidth).
138 +- **Linear probe fixed and evaluated** (`sim_olmoe_v2_ssd043_m8_probe_fixed.json`): one-layer-ahead logistic probe on the previous
139 + layer's router log-probabilities → recall@8 **0.761 held-out** (0.779 fit), prefetch precision 0.74–0.83, simulated hit rate 0.81 /
140 + 0.86 / 0.90 / 0.97 at C = 16 / 24 / 32 / 48 (none: 0.58 / 0.71 / 0.81 / 0.93; oracle 0.94 / 0.96 / 0.97 / 0.99); modelled stall 10.0 ms vs
141 + 23.1 ms per token at C = 16. Depth sweep (probe2–4) running (`sim_olmoe_v2_ssd043_m8_probe_depths.json`).
142 +- **Online probe on the cold tier** (16 slots, held-out general, `attribution.jsonl`): probe d=1 hit 0.775, reads 12.8 ms/token (none 19.9)
143 + but sync 19.7 + prefetch 3.3 ms (6.6 late waits/token) → **31.8 tok/s vs 33.0 none**; probe m=16 hit 0.847, reads 8.3 ms, but 127
144 + prefetches/token and 25 late waits → 25.3; probe2 31.3 (hit 0.74), probe3 30.7 (hit 0.72). Every predictor converts 7–11 ms of saved
145 + reads into ≈ 5 ms of extra GPU-side install work plus 1–3 ms of late waits: on this model and tier the install path (9 lazy slice
146 + writes ≈ 0.13 ms per expert) is the bottleneck, not prediction accuracy. Identified fix for v0.2: single-copy record install.
modified scripts/make_figures.py +27 −2
@@ -19,7 +19,11 @@ def save(fig, name):
19 19 def fig_sim_hit_vs_capacity():
20 20 p = os.path.join(ROOT, "benchmarks/processed/gateC/sim_olmoe_v2_ssd043_m8.json")
21 21 if not os.path.exists(p): return
22 − d = json.load(open(p)); rows = [r for r in d["results"] if r["split"] == "heldout" and r["policy"] == "lru"]
22 + d = json.load(open(p)); rows = [r for r in d["results"] if r["split"] == "heldout" and r["policy"] == "lru" and r["predictor"] != "probe"]
23 + for extra in ("sim_olmoe_v2_ssd043_m8_probe_fixed.json", "sim_olmoe_v2_ssd043_m8_probe_depths.json"):
24 + q = os.path.join(ROOT, "benchmarks/processed/gateC", extra)
25 + if os.path.exists(q):
26 + rows += [r for r in json.load(open(q))["results"] if r["split"] == "heldout" and r["policy"] == "lru" and r["predictor"] not in {x["predictor"] for x in rows}]
23 27 preds = []
24 28 for r in rows:
25 29 if r["predictor"] not in preds: preds.append(r["predictor"])
@@ -75,5 +79,26 @@ def fig_two_node():
75 79 ax.set_title("OLMoE: M3U96a local vs odd experts on M3U96b (1 GbE)", fontsize=8)
76 80 save(fig, "two_node")
77 81
82 +def fig_attribution():
83 + p = os.path.join(ROOT, "benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl")
84 + if not os.path.exists(p): return
85 + recs = [json.loads(l) for l in open(p) if l.strip()]; recs = [r for r in recs if not r["warmup"]]
86 + groups = {}
87 + for r in recs: groups.setdefault((r["predictor"], r["m"]), []).append(r)
88 + order = sorted(groups, key=lambda k: (k[0] != "none", k[0], k[1]))
89 + comps = ["sync", "prefetch", "ensure"]; names = {"sync": "GPU sync (graph incl. installs)", "prefetch": "prefetch bookkeeping + late waits", "ensure": "demand reads (SSD)"}
90 + fig, ax = plt.subplots(figsize=(7.2, 3.0)); bottom = [0.0] * len(order)
91 + for ci, c in enumerate(comps):
92 + vals = []
93 + for k in order:
94 + v = groups[k]; n = sum(x["gen_tokens"] for x in v); vals.append(sum(x["host_time_s"][c] for x in v) / n * 1e3)
95 + ax.bar(range(len(order)), vals, bottom=bottom, color=PAL[ci], label=names[c]); bottom = [b + v for b, v in zip(bottom, vals)]
96 + for i, k in enumerate(order):
97 + v = groups[k]; ax.text(i, bottom[i] + 0.5, f"{statistics.median(x['gen_tps'] for x in v):.0f} tok/s\nhit {statistics.median(x['cache_hit_rate'] for x in v):.2f}", ha="center", fontsize=6)
98 + ax.set_xticks(range(len(order))); ax.set_xticklabels([f"{k[0]}\nm={k[1]}" for k in order], fontsize=6.5); ax.set_ylabel("host time per token (ms)")
99 + ax.set_ylim(0, max(bottom) * 1.25); ax.legend(frameon=False, fontsize=7, loc="upper left")
100 + ax.set_title("OLMoE, 16 slots, cold NVMe (M5 Max): where the decode step waits", fontsize=9)
101 + save(fig, "attribution")
102 +
78 103 if __name__ == "__main__":
79 − fig_sim_hit_vs_capacity(); fig_capacity_sweep(); fig_two_node()
104 + fig_sim_hit_vs_capacity(); fig_capacity_sweep(); fig_two_node(); fig_attribution()
modified scripts/make_tables.py +14 −3
@@ -5,7 +5,12 @@ ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
5 5 OUT = os.path.join(ROOT, "paper", "tables"); os.makedirs(OUT, exist_ok=True)
6 6
7 7 def sim_table(path, label, caption):
8 − d = json.load(open(path)); rows = d["results"]
8 + d = json.load(open(path)); rows = [r for r in d["results"] if r["predictor"] != "probe"] # buggy probe rows replaced below
9 + for extra in ("sim_olmoe_v2_ssd043_m8_probe_fixed.json", "sim_olmoe_v2_ssd043_m8_probe_depths.json"):
10 + q = os.path.join(os.path.dirname(path), extra)
11 + if os.path.exists(q):
12 + have = {r["predictor"] for r in rows}
13 + rows += [r for r in json.load(open(q))["results"] if r["predictor"] not in have]
9 14 caps = sorted({r["capacity"] for r in rows}); preds = []
10 15 for r in rows:
11 16 if r["predictor"] not in preds: preds.append(r["predictor"])
@@ -38,7 +43,7 @@ def bench_summary(paths, label, caption, keys=("backend", "capacity", "predictor
38 43 for k in sorted(groups, key=lambda x: tuple(str(v) for v in x)):
39 44 g = groups[k]
40 45 med = lambda f: statistics.median([r[f] for r in g if r.get(f) is not None]) if any(r.get(f) is not None for r in g) else None
41 − cfg = f"C={k[1]}, {k[2]}, m={k[3]}" + (", cold" if k[4] else "") if k[0] == "titan" else "--"
46 + cfg = (f"C={k[1]}, {k[2]}, m={k[3]}" if keys[2] == "predictor" else f"C={k[1]}, fused={k[2]}") + (", cold" if k[4] else "") if k[0] == "titan" else "--"
42 47 f = lambda v, fmt: (fmt % v) if v is not None else "--"
43 48 lines.append(f"{k[0].replace('_', '\\_')} & {cfg.replace('_', '\\_')} & {len(g)} & {f(med('gen_tps'), '%.1f')} & {f(med('itl_ms_p95'), '%.1f')} & {f(med('cache_hit_rate'), '%.3f')} & {f(med('stall_s'), '%.2f')} & {f(med('ttft_ms'), '%.0f')} \\\\")
44 49 lines += ["\\bottomrule", "\\end{tabular}", "\\begin{tablenotes}\\footnotesize", "\\item Sources: " + ", ".join(f"\\code{{{os.path.relpath(p, ROOT).replace('_', '\\_')}}}" for p in paths) + ".", "\\end{tablenotes}", "\\end{table}"]
@@ -51,7 +56,13 @@ if __name__ == "__main__":
51 56 sim_table(v2 if os.path.exists(v2) else v1, "tab:sim", "Trace-driven simulation (OLMoE, per-layer LRU tables, cold per prompt): hit rate and modelled stall per token by predictor, split and capacity $C$.")
52 57 cold = os.path.join(ROOT, "benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl")
53 58 if os.path.exists(cold):
54 − bench_summary([cold], "tab:cold", "Cold-NVMe tier on the M5 Max laptop (purge + F\\_NOCACHE), OLMoE 4-bit, held-out prompts, 64 tokens.")
59 + bench_summary([cold], "tab:cold", "Cold-NVMe tier on the M5 Max laptop (purge + F\\_NOCACHE), OLMoE 4-bit, held-out general/code/reasoning prompts, 64 tokens, 12 runs per row.")
60 + attr = os.path.join(ROOT, "benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl")
61 + if os.path.exists(attr):
62 + bench_summary([attr], "tab:attr", "Predictor comparison with host-time attribution, cold NVMe, 16 slots, held-out general prompts (4 runs per row); see Figure~\\ref{fig:attribution} for the breakdown.")
63 + fused = os.path.join(ROOT, "benchmarks/raw/gateD/fused_upgate_laptop.jsonl")
64 + if os.path.exists(fused):
65 + bench_summary([fused], "tab:fused", "H4: fused up+gate gather\\_qmm vs two kernels (laptop; 64 slots resident, 128 tokens, 12 runs; 16 slots cold, 64 tokens, 4 runs).", keys=("backend", "capacity", "fused_upgate", "m", "nocache", "split"))
55 66 q = os.path.join(ROOT, "benchmarks/raw/gateC/qwen3_m3u96a/baseline.jsonl")
56 67 if os.path.exists(q):
57 68 bench_summary([q], "tab:qwen3", "Qwen3-30B-A3B-Instruct-2507 4-bit on M3U96a (M3 Ultra, 96 GB): mlx-lm vs TITAN all-resident (128 slots, preloaded), 64 tokens, 3 repeats.")
modified titan/predictors.py +10 −6
@@ -84,13 +84,16 @@ class Probe(Predictor):
84 84 name = "probe"
85 85 horizon = "layer"
86 86
87 − def __init__(self, L, E, epochs=2000, lr=0.05, l2=1e-5, seed=0):
87 + def __init__(self, L, E, epochs=2000, lr=0.05, l2=1e-5, seed=0, depth=1):
88 88 self.L, self.E, self.epochs, self.lr, self.l2 = L, E, epochs, lr, l2
89 + self.depth = depth
90 + self.name = "probe" if depth == 1 else f"probe{depth}"
89 91 self.W = np.zeros((L, E + 1, E)); self.mu = np.zeros((L, E)); self.sd = np.ones((L, E)); self.rng = np.random.default_rng(seed)
90 92
91 93 def fit(self, traces):
92 − for l in range(1, self.L):
93 − X = np.concatenate([p[:, l - 1].astype(np.float32) for _, p in traces], 0)
94 + d = self.depth
95 + for l in range(d, self.L):
96 + X = np.concatenate([p[:, l - d].astype(np.float32) for _, p in traces], 0)
94 97 Y = np.zeros((X.shape[0], self.E), np.float32)
95 98 rows, cols, off = [], [], 0
96 99 for t, _ in traces: # row offsets accumulate across traces (bug fixed 2026-11-01: labels were misaligned)
@@ -108,9 +111,10 @@ class Probe(Predictor):
108 111 return self
109 112
110 113 def predict(self, l, prev_topk, prev_probs, cur_routes, m):
111 − if l == 0 or (l - 1) not in cur_routes or cur_routes.get(("probs", l - 1)) is None:
114 + d = self.depth
115 + if l < d or cur_routes.get(("probs", l - d)) is None:
112 116 return list(prev_topk[l][:m]) if prev_topk is not None else []
113 − x = (np.log(np.clip(cur_routes[("probs", l - 1)].astype(np.float32), 1e-6, None)) - self.mu[l]) / self.sd[l]
117 + x = (np.log(np.clip(cur_routes[("probs", l - d)].astype(np.float32), 1e-6, None)) - self.mu[l]) / self.sd[l]
114 118 return list(np.argsort(-(np.append(x, 1.0) @ self.W[l]))[:m])
115 119
116 120
@@ -126,5 +130,5 @@ def make(name, L, E):
126 130 reg = {"none": Predictor, "prev_token": PrevToken, "prev_rank": PrevTokenRank, "transition": lambda: Transition(L, E),
127 131 "transition2": lambda: Transition(L, E, 2), "transition3": lambda: Transition(L, E, 3), "transition4": lambda: Transition(L, E, 4),
128 132 "transition2t50": lambda: Transition(L, E, 2, 0.5), "transition2t70": lambda: Transition(L, E, 2, 0.7), "transition1t50": lambda: Transition(L, E, 1, 0.5),
129 − "probe": lambda: Probe(L, E), "oracle": Oracle}
133 + "probe": lambda: Probe(L, E), "probe2": lambda: Probe(L, E, depth=2), "probe3": lambda: Probe(L, E, depth=3), "oracle": Oracle}
130 134 return reg[name]()
131 135
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere