SPB Git
spboucher.ai

spb/wp33_uqo

Public MIT

SSD tier: parallel demand reads + expert-contiguous TITAN layout (cold 16-slot 8-10 -> 36 tok/s); sim v2 results; probe fix; LOG/DECISIONS

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 1, 2026) parent 8a2444c Browse files

30 changed files +827 −18

30 changed files
Unified Split
added benchmarks/raw/gateC/cold_nvme_laptop/quick_v2.jsonl +2 −0
@@ -0,0 +1,2 @@
1+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_v2_contig_parallel", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --nocache --families general --split heldout --repeats 1 --warmup 0 --max-tokens 64 --tag cold_v2_contig_parallel --out benchmarks/raw/gateC/cold_nvme_laptop/quick_v2.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:49:10", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 803.916750010103, "prompt_tps": 22.535348808701784, "gen_tps": 18.323417113767114, "decode_tps_wall": 18.03752965713207, "itl_ms_p50": 53.5679169697687, "itl_ms_p95": 71.37474999763072, "itl_ms_p99": 74.9241670127958, "itl_ms_mean": 55.43996428605169, "total_s": 4.2966345000313595, "peak_mem_gb": 1.829203036, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.4996987346856799, "cache_misses": 4982, "cache_hits": 4976, "stall_s": 2.954887881525792, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17631019008, "ssd_read_s": 8.434191456297413}
2+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_v2_contig_parallel", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --nocache --families general --split heldout --repeats 1 --warmup 0 --max-tokens 64 --tag cold_v2_contig_parallel --out benchmarks/raw/gateC/cold_nvme_laptop/quick_v2.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:49:14", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 663.5876669315621, "prompt_tps": 21.10723001459196, "gen_tps": 17.82187991563441, "decode_tps_wall": 17.5434704435222, "itl_ms_p50": 56.79500009864569, "itl_ms_p95": 73.50704108830541, "itl_ms_p99": 78.22308293543756, "itl_ms_mean": 57.00126455705021, "total_s": 4.254667334025726, "peak_mem_gb": 1.747773596, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.4795243019648397, "cache_misses": 5033, "cache_hits": 4637, "stall_s": 2.9577913424000144, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17811505152, "ssd_read_s": 8.498102859477513}
added benchmarks/raw/gateC/cold_nvme_laptop/quick_v3.jsonl +2 −0
@@ -0,0 +1,2 @@
1+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_v3_contig_layout_parallel4", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --nocache --families general --split heldout --repeats 1 --warmup 0 --max-tokens 64 --tag cold_v3_contig_layout_parallel4 --out benchmarks/raw/gateC/cold_nvme_laptop/quick_v3.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:50:34", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 488.5364999063313, "prompt_tps": 37.24041945787185, "gen_tps": 36.406036799921125, "decode_tps_wall": 35.83867900614282, "itl_ms_p50": 27.031624922528863, "itl_ms_p95": 34.49200000613928, "itl_ms_p99": 36.87300009187311, "itl_ms_mean": 27.90281415865239, "total_s": 2.246413791901432, "peak_mem_gb": 1.829203036, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.4996987346856799, "cache_misses": 4982, "cache_hits": 4976, "stall_s": 1.2363767947535962, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17631019008, "ssd_read_s": 3.280213507357985}
2+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_v3_contig_layout_parallel4", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --nocache --families general --split heldout --repeats 1 --warmup 0 --max-tokens 64 --tag cold_v3_contig_layout_parallel4 --out benchmarks/raw/gateC/cold_nvme_laptop/quick_v3.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:50:36", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 327.6590419700369, "prompt_tps": 42.770810288913786, "gen_tps": 35.63977846912582, "decode_tps_wall": 35.08310472207398, "itl_ms_p50": 28.59441703185439, "itl_ms_p95": 35.45754204969853, "itl_ms_p99": 37.86633303388953, "itl_ms_mean": 28.503748682504963, "total_s": 2.1233952089678496, "peak_mem_gb": 1.747773596, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.4795243019648397, "cache_misses": 5033, "cache_hits": 4637, "stall_s": 1.1357120022876188, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17811505152, "ssd_read_s": 2.9532211498590186}
modified benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl +7 −3
@@ -1,3 +1,7 @@
1−{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:47:04", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 3041.2979170214385, "prompt_tps": 10.538982502473754, "gen_tps": 9.615565496623349, "decode_tps_wall": 9.465435344104892, "itl_ms_p50": 102.56666608620435, "itl_ms_p95": 147.70795905496925, "itl_ms_p99": 167.03716607298702, "itl_ms_mean": 105.64754431742051, "total_s": 9.69709320901893, "peak_mem_gb": 2.090173904, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.4844649021864212, "cache_misses": 5824, "cache_hits": 5473, "stall_s": 8.191949911764823, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 20610809856, "ssd_read_s": 7.725407627178356} 1+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:50:48", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 692.0957081019878, "prompt_tps": 46.59402246111043, "gen_tps": 36.57738284532069, "decode_tps_wall": 36.007635327202976, "itl_ms_p50": 27.320333058014512, "itl_ms_p95": 33.130957977846265, "itl_ms_p99": 39.143749978393316, "itl_ms_mean": 27.77188757087089, "total_s": 2.441724625066854, "peak_mem_gb": 2.090173904, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.4844649021864212, "cache_misses": 5824, "cache_hits": 5473, "stall_s": 1.3842638342175633, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 20610809856, "ssd_read_s": 3.7224394695367664}
2−{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:47:11", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 1772.822791012004, "prompt_tps": 13.540661227523396, "gen_tps": 14.592826924160933, "decode_tps_wall": 14.364850714221733, "itl_ms_p50": 64.1560829244554, "itl_ms_p95": 111.17766599636525, "itl_ms_p99": 147.35658396966755, "itl_ms_mean": 69.61436772955552, "total_s": 6.158527957974002, "peak_mem_gb": 1.930492816, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.6523887973640856, "cache_misses": 3587, "cache_hits": 6732, "stall_s": 4.940353738376871, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 12694192128, "ssd_read_s": 4.6701676497468725} 2+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:50:50", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 369.74300001747906, "prompt_tps": 64.96787686611094, "gen_tps": 44.10237031807938, "decode_tps_wall": 43.4135612400075, "itl_ms_p50": 22.59599999524653, "itl_ms_p95": 29.346165945753455, "itl_ms_p99": 34.49933393858373, "itl_ms_mean": 23.034277111513628, "total_s": 1.8209024580428377, "peak_mem_gb": 1.9139122, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.6523887973640856, "cache_misses": 3587, "cache_hits": 6732, "stall_s": 0.9014140029903501, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 12694192128, "ssd_read_s": 2.055849175201729}
3−{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:47:19", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 1448.2218329794705, "prompt_tps": 9.668972925355584, "gen_tps": 9.486046026204507, "decode_tps_wall": 9.337841435614223, "itl_ms_p50": 106.86812503263354, "itl_ms_p95": 156.7806249950081, "itl_ms_p99": 170.41387502104044, "itl_ms_mean": 107.09113095303081, "total_s": 8.194963083020411, "peak_mem_gb": 1.742661784, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.48355739400206826, "cache_misses": 4994, "cache_hits": 4676, "stall_s": 6.844891174114309, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17673486336, "ssd_read_s": 6.502651879563928} 3+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:50:52", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 316.1524999886751, "prompt_tps": 44.3261007825923, "gen_tps": 37.43535274314916, "decode_tps_wall": 36.85073790399236, "itl_ms_p50": 26.99137502349913, "itl_ms_p95": 33.23575004469603, "itl_ms_p99": 36.866707960143685, "itl_ms_mean": 27.136498666737996, "total_s": 2.025751915993169, "peak_mem_gb": 1.737943192, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.48355739400206826, "cache_misses": 4994, "cache_hits": 4676, "stall_s": 1.0538058636011556, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17673486336, "ssd_read_s": 2.6701927126850933}
4+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:50:55", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 345.18750000279397, "prompt_tps": 52.19889665094133, "gen_tps": 35.34670805642188, "decode_tps_wall": 34.79462232635829, "itl_ms_p50": 28.336417046375573, "itl_ms_p95": 35.60750000178814, "itl_ms_p99": 37.13658300694078, "itl_ms_mean": 28.74007341193242, "total_s": 2.1558121249545366, "peak_mem_gb": 1.814131732, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.5051215103434424, "cache_misses": 4928, "cache_hits": 5030, "stall_s": 1.1506640871521086, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17439916032, "ssd_read_s": 2.9321806946536526}
5+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:50:57", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 420.1389999361709, "prompt_tps": 69.07719609252912, "gen_tps": 37.35810759487954, "decode_tps_wall": 36.77462865794749, "itl_ms_p50": 28.77229091245681, "itl_ms_p95": 33.97729201242328, "itl_ms_p99": 35.62474995851517, "itl_ms_mean": 27.192660714573567, "total_s": 2.1332766249543056, "peak_mem_gb": 1.905393932, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.5451856093138658, "cache_misses": 4766, "cache_hits": 5713, "stall_s": 1.1352890465641394, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 16866607104, "ssd_read_s": 2.85799129109364}
6+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:50:58", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 325.22700005210936, "prompt_tps": 58.47830967620645, "gen_tps": 44.22672637332204, "decode_tps_wall": 43.53605355216225, "itl_ms_p50": 21.73804200720042, "itl_ms_p95": 29.404125059954822, "itl_ms_p99": 31.615916988812387, "itl_ms_mean": 22.96946825466991, "total_s": 1.7723035000963137, "peak_mem_gb": 1.773794588, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.6699147381242387, "cache_misses": 3252, "cache_hits": 6600, "stall_s": 0.8593332751188427, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 11508645888, "ssd_read_s": 1.9043039631797}
7+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "8a2444c"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:51:01", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 958.3374169887975, "prompt_tps": 33.402911106697594, "gen_tps": 33.07673168156339, "decode_tps_wall": 32.560083043862754, "itl_ms_p50": 29.210791923105717, "itl_ms_p95": 35.54279205854982, "itl_ms_p99": 116.7389169568196, "itl_ms_mean": 30.712452380814486, "total_s": 2.89322191698011, "peak_mem_gb": 2.094958152, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.49030716119323714, "cache_misses": 5758, "cache_hits": 5539, "stall_s": 1.7997499047778547, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 20377239552, "ssd_read_s": 4.840675379149616}
added benchmarks/raw/gateC/cold_nvme_laptop/sweep_v1_9preads_serial.jsonl +15 −0
@@ -0,0 +1,15 @@
1+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:47:04", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 3041.2979170214385, "prompt_tps": 10.538982502473754, "gen_tps": 9.615565496623349, "decode_tps_wall": 9.465435344104892, "itl_ms_p50": 102.56666608620435, "itl_ms_p95": 147.70795905496925, "itl_ms_p99": 167.03716607298702, "itl_ms_mean": 105.64754431742051, "total_s": 9.69709320901893, "peak_mem_gb": 2.090173904, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.4844649021864212, "cache_misses": 5824, "cache_hits": 5473, "stall_s": 8.191949911764823, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 20610809856, "ssd_read_s": 7.725407627178356}
2+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:47:11", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 1772.822791012004, "prompt_tps": 13.540661227523396, "gen_tps": 14.592826924160933, "decode_tps_wall": 14.364850714221733, "itl_ms_p50": 64.1560829244554, "itl_ms_p95": 111.17766599636525, "itl_ms_p99": 147.35658396966755, "itl_ms_mean": 69.61436772955552, "total_s": 6.158527957974002, "peak_mem_gb": 1.930492816, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.6523887973640856, "cache_misses": 3587, "cache_hits": 6732, "stall_s": 4.940353738376871, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 12694192128, "ssd_read_s": 4.6701676497468725}
3+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:47:19", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 1448.2218329794705, "prompt_tps": 9.668972925355584, "gen_tps": 9.486046026204507, "decode_tps_wall": 9.337841435614223, "itl_ms_p50": 106.86812503263354, "itl_ms_p95": 156.7806249950081, "itl_ms_p99": 170.41387502104044, "itl_ms_mean": 107.09113095303081, "total_s": 8.194963083020411, "peak_mem_gb": 1.742661784, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.48355739400206826, "cache_misses": 4994, "cache_hits": 4676, "stall_s": 6.844891174114309, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17673486336, "ssd_read_s": 6.502651879563928}
4+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:47:27", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 1678.9133329875767, "prompt_tps": 10.7232358439087, "gen_tps": 10.018922261039506, "decode_tps_wall": 9.862394227537603, "itl_ms_p50": 97.25870797410607, "itl_ms_p95": 146.2174579501152, "itl_ms_p99": 158.16250001080334, "itl_ms_mean": 101.3952572700671, "total_s": 8.066814541001804, "peak_mem_gb": 1.832743964, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.5051215103434424, "cache_misses": 4928, "cache_hits": 5030, "stall_s": 6.716066468274221, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17439916032, "ssd_read_s": 6.371192621882074}
5+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": true, "repeat": -1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:47:35", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 2036.6630000062287, "prompt_tps": 14.241123672657547, "gen_tps": 11.004827989819676, "decode_tps_wall": 10.832897886006371, "itl_ms_p50": 100.70604097563773, "itl_ms_p95": 135.63829089980572, "itl_ms_p99": 152.31495804619044, "itl_ms_mean": 92.31140277725423, "total_s": 7.852281374973245, "peak_mem_gb": 1.905393952, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.5451856093138658, "cache_misses": 4766, "cache_hits": 5713, "stall_s": 6.519519609515555, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 16866607104, "ssd_read_s": 6.186666315770708}
6+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:47:40", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 1480.4692500038072, "prompt_tps": 12.836366279140407, "gen_tps": 15.293047498172099, "decode_tps_wall": 15.054134848034861, "itl_ms_p50": 56.96449999231845, "itl_ms_p95": 114.80837501585484, "itl_ms_p99": 126.13425008021295, "itl_ms_mean": 66.4269325401013, "total_s": 5.66536600003019, "peak_mem_gb": 1.773794588, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.6699147381242387, "cache_misses": 3252, "cache_hits": 6600, "stall_s": 4.478065619710833, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 11508645888, "ssd_read_s": 4.2451598775805905}
7+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: A tank fills in 6 hours with pipe A and 9 hours with pipe", "ts": "2026-11-01T01:47:50", "prompt_tokens": 32, "gen_tokens": 64, "ttft_ms": 2848.683707998134, "prompt_tps": 11.23438886125646, "gen_tps": 9.901486845998365, "decode_tps_wall": 9.746792386317114, "itl_ms_p50": 98.78045890945941, "itl_ms_p95": 139.5513330353424, "itl_ms_p99": 161.6568339522928, "itl_ms_mean": 102.59785582422323, "total_s": 9.312348624924198, "peak_mem_gb": 2.094958152, "text": "If pipe A fills the tank in 6 hours, then the rate of pipe A is 1 / 6 of the tank per hour. If pipe B fills the tank in 9 hours, then the rate of pipe B is 1 / 9 of the tank per hour. The combined rat", "cache_hit_rate": 0.49030716119323714, "cache_misses": 5758, "cache_hits": 5539, "stall_s": 7.887103883433156, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 20377239552, "ssd_read_s": 7.472977954312228}
8+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:47:58", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 1657.9446670366451, "prompt_tps": 10.85866666318932, "gen_tps": 9.96284746620274, "decode_tps_wall": 9.807193241289706, "itl_ms_p50": 99.73604197148234, "itl_ms_p95": 146.3351669954136, "itl_ms_p99": 159.94337503798306, "itl_ms_mean": 101.96597287283531, "total_s": 8.08180095802527, "peak_mem_gb": 1.839953, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.5069291022293633, "cache_misses": 4910, "cache_hits": 5048, "stall_s": 6.72081563831307, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17376215040, "ssd_read_s": 6.373155025416054}
9+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:48:07", "prompt_tokens": 14, "gen_tokens": 64, "ttft_ms": 1504.7367500374094, "prompt_tps": 9.305675828688162, "gen_tps": 7.954694823209586, "decode_tps_wall": 7.830415043880024, "itl_ms_p50": 125.9298330405727, "itl_ms_p95": 180.7774999178946, "itl_ms_p99": 192.8413329878822, "itl_ms_mean": 127.70715145955957, "total_s": 9.550287291989662, "peak_mem_gb": 1.747773596, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "cache_hit_rate": 0.4795243019648397, "cache_misses": 5033, "cache_hits": 4637, "stall_s": 8.067980261519551, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17811505152, "ssd_read_s": 7.641573060769588}
10+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": false, "repeat": 0, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:48:16", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 2396.1164170177653, "prompt_tps": 12.10512571237578, "gen_tps": 10.396180431335797, "decode_tps_wall": 10.23376338708268, "itl_ms_p50": 106.55224998481572, "itl_ms_p95": 146.57462493050843, "itl_ms_p99": 172.6577919907868, "itl_ms_mean": 97.71576322179051, "total_s": 8.552209499990568, "peak_mem_gb": 1.90539394, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.5457581830327322, "cache_misses": 4760, "cache_hits": 5719, "stall_s": 7.159492473700084, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 16845373440, "ssd_read_s": 6.763606808730401}
11+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:48:22", "prompt_tokens": 24, "gen_tokens": 64, "ttft_ms": 1804.6072499128059, "prompt_tps": 13.30151745591803, "gen_tps": 14.163653287680651, "decode_tps_wall": 13.942378732507121, "itl_ms_p50": 66.09183398541063, "itl_ms_p95": 115.31287501566112, "itl_ms_p99": 150.83458297885954, "itl_ms_mean": 71.72377247710727, "total_s": 6.323204915970564, "peak_mem_gb": 1.925970816, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "cache_hit_rate": 0.6497722647543367, "cache_misses": 3614, "cache_hits": 6705, "stall_s": 5.050725456792861, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 12789743616, "ssd_read_s": 4.767083190614358}
12+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:48:28", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 1421.2836669757962, "prompt_tps": 13.370988297603747, "gen_tps": 14.862207311364184, "decode_tps_wall": 14.630028638893691, "itl_ms_p50": 57.85045796073973, "itl_ms_p95": 115.3351670363918, "itl_ms_p99": 126.18254090193659, "itl_ms_mean": 68.35256612837492, "total_s": 5.727495333063416, "peak_mem_gb": 1.783100804, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.6759033698741372, "cache_misses": 3193, "cache_hits": 6659, "stall_s": 4.475894676987082, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 11299848192, "ssd_read_s": 4.227993114502169}
13+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": false, "repeat": 1, "family": "reasoning", "prompt": "Q: What is the smallest positive integer divisible by 2, 3, ", "ts": "2026-11-01T01:48:36", "prompt_tokens": 29, "gen_tokens": 64, "ttft_ms": 2034.2665419448167, "prompt_tps": 14.257879813570804, "gen_tps": 10.601058847905994, "decode_tps_wall": 10.435437038113436, "itl_ms_p50": 103.03058300632983, "itl_ms_p95": 140.08616702631116, "itl_ms_p99": 154.924959060736, "itl_ms_mean": 95.82732341230093, "total_s": 8.071387916919775, "peak_mem_gb": 1.90341144, "text": "The least common multiple (LCM) of 2, 3, 4, 5, and 6 is the smallest positive integer that is divisible by all of these numbers. The prime factorization of 2 is 2. The prime factorization of 3 is 3. T", "cache_hit_rate": 0.5446130355949995, "cache_misses": 4772, "cache_hits": 5707, "stall_s": 6.683604668593034, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 16887840768, "ssd_read_s": 6.316679805400781}
14+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:48:42", "prompt_tokens": 19, "gen_tokens": 64, "ttft_ms": 1547.3737920401618, "prompt_tps": 12.281128276714327, "gen_tps": 14.79498984013145, "decode_tps_wall": 14.563857265509588, "itl_ms_p50": 58.28862497583032, "itl_ms_p95": 116.0697090672329, "itl_ms_p99": 129.21441695652902, "itl_ms_mean": 68.6631283024326, "total_s": 5.873150875093415, "peak_mem_gb": 1.773794588, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array", "cache_hit_rate": 0.6699147381242387, "cache_misses": 3252, "cache_hits": 6600, "stall_s": 4.632669512648135, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 11508645888, "ssd_read_s": 4.369992908556014}
15+{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "nocache": true, "max_tokens": 64, "split": "heldout", "tag": "cold_nvme", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --predictor none --m 8 --nocache --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code,reasoning --split heldout --repeats 2 --max-tokens 64 --tag cold_nvme --out benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "258ba81"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:48:50", "prompt_tokens": 18, "gen_tokens": 64, "ttft_ms": 1817.269332939759, "prompt_tps": 9.906602292857812, "gen_tps": 9.587975958629395, "decode_tps_wall": 9.438182157713433, "itl_ms_p50": 102.33312495984137, "itl_ms_p95": 156.4575000666082, "itl_ms_p99": 179.1806248947978, "itl_ms_mean": 105.95260647546854, "total_s": 8.492283540894277, "peak_mem_gb": 1.843491944, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "cache_hit_rate": 0.5079333199437638, "cache_misses": 4900, "cache_hits": 5058, "stall_s": 7.061306995572522, "hits_device": 0, "layers_complete": 0, "prefetch": null, "ssd_bytes": 17340825600, "ssd_read_s": 6.670634448295459}
added benchmarks/raw/tests/greedy_olmoe_cap16_20261101T014903.json +14 −0
@@ -0,0 +1,14 @@
1+[
2+ {
3+ "prompt": "The capital of Canada is",
4+ "ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
5+ "titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
6+ "equal": true
7+ },
8+ {
9+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
10+ "ref": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
11+ "titan": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
12+ "equal": true
13+ }
14+]
\ No newline at end of file
added benchmarks/raw/tests/greedy_olmoe_cap16_20261101T015010.json +14 −0
@@ -0,0 +1,14 @@
1+[
2+ {
3+ "prompt": "The capital of Canada is",
4+ "ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
5+ "titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
6+ "equal": true
7+ },
8+ {
9+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
10+ "ref": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
11+ "titan": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
12+ "equal": true
13+ }
14+]
\ No newline at end of file
added benchmarks/raw/tests/greedy_olmoe_cap16_20261101T015029.json +14 −0
@@ -0,0 +1,14 @@
1+[
2+ {
3+ "prompt": "The capital of Canada is",
4+ "ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
5+ "titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
6+ "equal": true
7+ },
8+ {
9+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
10+ "ref": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
11+ "titan": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
12+ "equal": true
13+ }
14+]
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap16_2026-11-01T014859.json +50 −0
@@ -0,0 +1,50 @@
1+{
2+ "capacity": 16,
3+ "prompts": [
4+ {
5+ "prompt": "The capital of Canada is",
6+ "tokens": 5,
7+ "max_abs_dlogit": 0.0,
8+ "argmax_equal": true,
9+ "hit_rate": 0.2580101180438449,
10+ "misses": 440,
11+ "stall_s": 0.06682696042116731,
12+ "layers_routed": 46
13+ },
14+ {
15+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16+ "tokens": 21,
17+ "max_abs_dlogit": 0.0,
18+ "argmax_equal": true,
19+ "hit_rate": 0.42512908777969016,
20+ "misses": 1002,
21+ "stall_s": 0.15063788229599595,
22+ "layers_routed": 127
23+ },
24+ {
25+ "prompt": "Q: If a train travels 60 km in 45 minute",
26+ "tokens": 25,
27+ "max_abs_dlogit": 0.0,
28+ "argmax_equal": true,
29+ "hit_rate": 0.41904344010530936,
30+ "misses": 1324,
31+ "stall_s": 0.18718991475179791,
32+ "layers_routed": 161
33+ },
34+ {
35+ "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36+ "tokens": 26,
37+ "max_abs_dlogit": 0.0,
38+ "argmax_equal": true,
39+ "hit_rate": 0.45278969957081544,
40+ "misses": 1275,
41+ "stall_s": 0.18172545277047902,
42+ "layers_routed": 164
43+ }
44+ ],
45+ "tol": 0.001,
46+ "ts": "2026-11-01T01:48:59",
47+ "max_abs_dlogit_worst": 0.0,
48+ "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49+ "model_type": "olmoe"
50+}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap16_2026-11-01T015006.json +50 −0
@@ -0,0 +1,50 @@
1+{
2+ "capacity": 16,
3+ "prompts": [
4+ {
5+ "prompt": "The capital of Canada is",
6+ "tokens": 5,
7+ "max_abs_dlogit": 0.0,
8+ "argmax_equal": true,
9+ "hit_rate": 0.2580101180438449,
10+ "misses": 440,
11+ "stall_s": 0.055347624816931784,
12+ "layers_routed": 46
13+ },
14+ {
15+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16+ "tokens": 21,
17+ "max_abs_dlogit": 0.0,
18+ "argmax_equal": true,
19+ "hit_rate": 0.42512908777969016,
20+ "misses": 1002,
21+ "stall_s": 0.1264939100947231,
22+ "layers_routed": 127
23+ },
24+ {
25+ "prompt": "Q: If a train travels 60 km in 45 minute",
26+ "tokens": 25,
27+ "max_abs_dlogit": 0.0,
28+ "argmax_equal": true,
29+ "hit_rate": 0.41904344010530936,
30+ "misses": 1324,
31+ "stall_s": 0.1644190422957763,
32+ "layers_routed": 161
33+ },
34+ {
35+ "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36+ "tokens": 26,
37+ "max_abs_dlogit": 0.0,
38+ "argmax_equal": true,
39+ "hit_rate": 0.45278969957081544,
40+ "misses": 1275,
41+ "stall_s": 0.15638224757276475,
42+ "layers_routed": 164
43+ }
44+ ],
45+ "tol": 0.001,
46+ "ts": "2026-11-01T01:50:06",
47+ "max_abs_dlogit_worst": 0.0,
48+ "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49+ "model_type": "olmoe"
50+}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap16_2026-11-01T015025.json +50 −0
@@ -0,0 +1,50 @@
1+{
2+ "capacity": 16,
3+ "prompts": [
4+ {
5+ "prompt": "The capital of Canada is",
6+ "tokens": 5,
7+ "max_abs_dlogit": 0.0,
8+ "argmax_equal": true,
9+ "hit_rate": 0.2580101180438449,
10+ "misses": 440,
11+ "stall_s": 0.054807875072583556,
12+ "layers_routed": 46
13+ },
14+ {
15+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16+ "tokens": 21,
17+ "max_abs_dlogit": 0.0,
18+ "argmax_equal": true,
19+ "hit_rate": 0.42512908777969016,
20+ "misses": 1002,
21+ "stall_s": 0.1280115059344098,
22+ "layers_routed": 127
23+ },
24+ {
25+ "prompt": "Q: If a train travels 60 km in 45 minute",
26+ "tokens": 25,
27+ "max_abs_dlogit": 0.0,
28+ "argmax_equal": true,
29+ "hit_rate": 0.41904344010530936,
30+ "misses": 1324,
31+ "stall_s": 0.1614124964689836,
32+ "layers_routed": 161
33+ },
34+ {
35+ "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36+ "tokens": 26,
37+ "max_abs_dlogit": 0.0,
38+ "argmax_equal": true,
39+ "hit_rate": 0.45278969957081544,
40+ "misses": 1275,
41+ "stall_s": 0.15579284261912107,
42+ "layers_routed": 164
43+ }
44+ ],
45+ "tol": 0.001,
46+ "ts": "2026-11-01T01:50:25",
47+ "max_abs_dlogit_worst": 0.0,
48+ "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49+ "model_type": "olmoe"
50+}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap32_2026-11-01T014858.json +50 −0
@@ -0,0 +1,50 @@
1+{
2+ "capacity": 32,
3+ "prompts": [
4+ {
5+ "prompt": "The capital of Canada is",
6+ "tokens": 5,
7+ "max_abs_dlogit": 0.0,
8+ "argmax_equal": true,
9+ "hit_rate": 0.0,
10+ "misses": 430,
11+ "stall_s": 0.06809187470935285,
12+ "layers_routed": 16
13+ },
14+ {
15+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16+ "tokens": 21,
17+ "max_abs_dlogit": 0.0,
18+ "argmax_equal": true,
19+ "hit_rate": 0.6060606060606061,
20+ "misses": 429,
21+ "stall_s": 0.06508483353536576,
22+ "layers_routed": 40
23+ },
24+ {
25+ "prompt": "Q: If a train travels 60 km in 45 minute",
26+ "tokens": 25,
27+ "max_abs_dlogit": 0.0,
28+ "argmax_equal": true,
29+ "hit_rate": 0.6090458488228006,
30+ "misses": 631,
31+ "stall_s": 0.09457516169641167,
32+ "layers_routed": 58
33+ },
34+ {
35+ "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36+ "tokens": 26,
37+ "max_abs_dlogit": 0.0,
38+ "argmax_equal": true,
39+ "hit_rate": 0.6224188790560472,
40+ "misses": 512,
41+ "stall_s": 0.08756570424884558,
42+ "layers_routed": 49
43+ }
44+ ],
45+ "tol": 0.001,
46+ "ts": "2026-11-01T01:48:58",
47+ "max_abs_dlogit_worst": 0.0,
48+ "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49+ "model_type": "olmoe"
50+}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap32_2026-11-01T015005.json +50 −0
@@ -0,0 +1,50 @@
1+{
2+ "capacity": 32,
3+ "prompts": [
4+ {
5+ "prompt": "The capital of Canada is",
6+ "tokens": 5,
7+ "max_abs_dlogit": 0.0,
8+ "argmax_equal": true,
9+ "hit_rate": 0.0,
10+ "misses": 430,
11+ "stall_s": 0.05290862487163395,
12+ "layers_routed": 16
13+ },
14+ {
15+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16+ "tokens": 21,
17+ "max_abs_dlogit": 0.0,
18+ "argmax_equal": true,
19+ "hit_rate": 0.6060606060606061,
20+ "misses": 429,
21+ "stall_s": 0.057208749698475,
22+ "layers_routed": 40
23+ },
24+ {
25+ "prompt": "Q: If a train travels 60 km in 45 minute",
26+ "tokens": 25,
27+ "max_abs_dlogit": 0.0,
28+ "argmax_equal": true,
29+ "hit_rate": 0.6090458488228006,
30+ "misses": 631,
31+ "stall_s": 0.07817478699143976,
32+ "layers_routed": 58
33+ },
34+ {
35+ "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36+ "tokens": 26,
37+ "max_abs_dlogit": 0.0,
38+ "argmax_equal": true,
39+ "hit_rate": 0.6224188790560472,
40+ "misses": 512,
41+ "stall_s": 0.06320704496465623,
42+ "layers_routed": 49
43+ }
44+ ],
45+ "tol": 0.001,
46+ "ts": "2026-11-01T01:50:05",
47+ "max_abs_dlogit_worst": 0.0,
48+ "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49+ "model_type": "olmoe"
50+}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap32_2026-11-01T015024.json +50 −0
@@ -0,0 +1,50 @@
1+{
2+ "capacity": 32,
3+ "prompts": [
4+ {
5+ "prompt": "The capital of Canada is",
6+ "tokens": 5,
7+ "max_abs_dlogit": 0.0,
8+ "argmax_equal": true,
9+ "hit_rate": 0.0,
10+ "misses": 430,
11+ "stall_s": 0.05366220697760582,
12+ "layers_routed": 16
13+ },
14+ {
15+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16+ "tokens": 21,
17+ "max_abs_dlogit": 0.0,
18+ "argmax_equal": true,
19+ "hit_rate": 0.6060606060606061,
20+ "misses": 429,
21+ "stall_s": 0.05273353960365057,
22+ "layers_routed": 40
23+ },
24+ {
25+ "prompt": "Q: If a train travels 60 km in 45 minute",
26+ "tokens": 25,
27+ "max_abs_dlogit": 0.0,
28+ "argmax_equal": true,
29+ "hit_rate": 0.6090458488228006,
30+ "misses": 631,
31+ "stall_s": 0.07555628998670727,
32+ "layers_routed": 58
33+ },
34+ {
35+ "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36+ "tokens": 26,
37+ "max_abs_dlogit": 0.0,
38+ "argmax_equal": true,
39+ "hit_rate": 0.6224188790560472,
40+ "misses": 512,
41+ "stall_s": 0.062176456209272146,
42+ "layers_routed": 49
43+ }
44+ ],
45+ "tol": 0.001,
46+ "ts": "2026-11-01T01:50:24",
47+ "max_abs_dlogit_worst": 0.0,
48+ "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49+ "model_type": "olmoe"
50+}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap64_2026-11-01T014858.json +50 −0
@@ -0,0 +1,50 @@
1+{
2+ "capacity": 64,
3+ "prompts": [
4+ {
5+ "prompt": "The capital of Canada is",
6+ "tokens": 5,
7+ "max_abs_dlogit": 0.0,
8+ "argmax_equal": true,
9+ "hit_rate": 0.0,
10+ "misses": 430,
11+ "stall_s": 0.06326587486546487,
12+ "layers_routed": 16
13+ },
14+ {
15+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16+ "tokens": 21,
17+ "max_abs_dlogit": 0.0,
18+ "argmax_equal": true,
19+ "hit_rate": 0.4637904468412943,
20+ "misses": 348,
21+ "stall_s": 0.05040391813963652,
22+ "layers_routed": 16
23+ },
24+ {
25+ "prompt": "Q: If a train travels 60 km in 45 minute",
26+ "tokens": 25,
27+ "max_abs_dlogit": 0.0,
28+ "argmax_equal": true,
29+ "hit_rate": 0.823076923076923,
30+ "misses": 138,
31+ "stall_s": 0.02257291681598872,
32+ "layers_routed": 16
33+ },
34+ {
35+ "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36+ "tokens": 26,
37+ "max_abs_dlogit": 0.0,
38+ "argmax_equal": true,
39+ "hit_rate": 0.9234828496042217,
40+ "misses": 58,
41+ "stall_s": 0.011926291976124048,
42+ "layers_routed": 16
43+ }
44+ ],
45+ "tol": 0.001,
46+ "ts": "2026-11-01T01:48:58",
47+ "max_abs_dlogit_worst": 0.0,
48+ "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49+ "model_type": "olmoe"
50+}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap64_2026-11-01T015005.json +50 −0
@@ -0,0 +1,50 @@
1+{
2+ "capacity": 64,
3+ "prompts": [
4+ {
5+ "prompt": "The capital of Canada is",
6+ "tokens": 5,
7+ "max_abs_dlogit": 0.0,
8+ "argmax_equal": true,
9+ "hit_rate": 0.0,
10+ "misses": 430,
11+ "stall_s": 0.052728913724422455,
12+ "layers_routed": 16
13+ },
14+ {
15+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16+ "tokens": 21,
17+ "max_abs_dlogit": 0.0,
18+ "argmax_equal": true,
19+ "hit_rate": 0.4637904468412943,
20+ "misses": 348,
21+ "stall_s": 0.04487437487114221,
22+ "layers_routed": 16
23+ },
24+ {
25+ "prompt": "Q: If a train travels 60 km in 45 minute",
26+ "tokens": 25,
27+ "max_abs_dlogit": 0.0,
28+ "argmax_equal": true,
29+ "hit_rate": 0.823076923076923,
30+ "misses": 138,
31+ "stall_s": 0.019109665183350444,
32+ "layers_routed": 16
33+ },
34+ {
35+ "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36+ "tokens": 26,
37+ "max_abs_dlogit": 0.0,
38+ "argmax_equal": true,
39+ "hit_rate": 0.9234828496042217,
40+ "misses": 58,
41+ "stall_s": 0.009046415099874139,
42+ "layers_routed": 16
43+ }
44+ ],
45+ "tol": 0.001,
46+ "ts": "2026-11-01T01:50:05",
47+ "max_abs_dlogit_worst": 0.0,
48+ "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49+ "model_type": "olmoe"
50+}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap64_2026-11-01T015024.json +50 −0
@@ -0,0 +1,50 @@
1+{
2+ "capacity": 64,
3+ "prompts": [
4+ {
5+ "prompt": "The capital of Canada is",
6+ "tokens": 5,
7+ "max_abs_dlogit": 0.0,
8+ "argmax_equal": true,
9+ "hit_rate": 0.0,
10+ "misses": 430,
11+ "stall_s": 0.05269420798867941,
12+ "layers_routed": 16
13+ },
14+ {
15+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16+ "tokens": 21,
17+ "max_abs_dlogit": 0.0,
18+ "argmax_equal": true,
19+ "hit_rate": 0.4637904468412943,
20+ "misses": 348,
21+ "stall_s": 0.04307745990809053,
22+ "layers_routed": 16
23+ },
24+ {
25+ "prompt": "Q: If a train travels 60 km in 45 minute",
26+ "tokens": 25,
27+ "max_abs_dlogit": 0.0,
28+ "argmax_equal": true,
29+ "hit_rate": 0.823076923076923,
30+ "misses": 138,
31+ "stall_s": 0.018091750680468976,
32+ "layers_routed": 16
33+ },
34+ {
35+ "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36+ "tokens": 26,
37+ "max_abs_dlogit": 0.0,
38+ "argmax_equal": true,
39+ "hit_rate": 0.9234828496042217,
40+ "misses": 58,
41+ "stall_s": 0.008422082057222724,
42+ "layers_routed": 16
43+ }
44+ ],
45+ "tol": 0.001,
46+ "ts": "2026-11-01T01:50:24",
47+ "max_abs_dlogit_worst": 0.0,
48+ "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49+ "model_type": "olmoe"
50+}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap8_2026-11-01T014900.json +50 −0
@@ -0,0 +1,50 @@
1+{
2+ "capacity": 8,
3+ "prompts": [
4+ {
5+ "prompt": "The capital of Canada is",
6+ "tokens": 5,
7+ "max_abs_dlogit": 0.0,
8+ "argmax_equal": true,
9+ "hit_rate": 0.1625,
10+ "misses": 536,
11+ "stall_s": 0.07683816191274673,
12+ "layers_routed": 80
13+ },
14+ {
15+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16+ "tokens": 21,
17+ "max_abs_dlogit": 0.0,
18+ "argmax_equal": true,
19+ "hit_rate": 0.4425981873111782,
20+ "misses": 1476,
21+ "stall_s": 0.20861312677152455,
22+ "layers_routed": 331
23+ },
24+ {
25+ "prompt": "Q: If a train travels 60 km in 45 minute",
26+ "tokens": 25,
27+ "max_abs_dlogit": 0.0,
28+ "argmax_equal": true,
29+ "hit_rate": 0.375,
30+ "misses": 1995,
31+ "stall_s": 0.2837240828666836,
32+ "layers_routed": 399
33+ },
34+ {
35+ "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36+ "tokens": 26,
37+ "max_abs_dlogit": 0.0,
38+ "argmax_equal": true,
39+ "hit_rate": 0.3878048780487805,
40+ "misses": 2008,
41+ "stall_s": 0.2809703811071813,
42+ "layers_routed": 410
43+ }
44+ ],
45+ "tol": 0.001,
46+ "ts": "2026-11-01T01:49:00",
47+ "max_abs_dlogit_worst": 0.0,
48+ "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49+ "model_type": "olmoe"
50+}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap8_2026-11-01T015007.json +50 −0
@@ -0,0 +1,50 @@
1+{
2+ "capacity": 8,
3+ "prompts": [
4+ {
5+ "prompt": "The capital of Canada is",
6+ "tokens": 5,
7+ "max_abs_dlogit": 0.0,
8+ "argmax_equal": true,
9+ "hit_rate": 0.1625,
10+ "misses": 536,
11+ "stall_s": 0.06772437284234911,
12+ "layers_routed": 80
13+ },
14+ {
15+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16+ "tokens": 21,
17+ "max_abs_dlogit": 0.0,
18+ "argmax_equal": true,
19+ "hit_rate": 0.4425981873111782,
20+ "misses": 1476,
21+ "stall_s": 0.19338525040075183,
22+ "layers_routed": 331
23+ },
24+ {
25+ "prompt": "Q: If a train travels 60 km in 45 minute",
26+ "tokens": 25,
27+ "max_abs_dlogit": 0.0,
28+ "argmax_equal": true,
29+ "hit_rate": 0.375,
30+ "misses": 1995,
31+ "stall_s": 0.2538081571692601,
32+ "layers_routed": 399
33+ },
34+ {
35+ "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36+ "tokens": 26,
37+ "max_abs_dlogit": 0.0,
38+ "argmax_equal": true,
39+ "hit_rate": 0.3878048780487805,
40+ "misses": 2008,
41+ "stall_s": 0.25273924763314426,
42+ "layers_routed": 410
43+ }
44+ ],
45+ "tol": 0.001,
46+ "ts": "2026-11-01T01:50:07",
47+ "max_abs_dlogit_worst": 0.0,
48+ "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49+ "model_type": "olmoe"
50+}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap8_2026-11-01T015026.json +50 −0
@@ -0,0 +1,50 @@
1+{
2+ "capacity": 8,
3+ "prompts": [
4+ {
5+ "prompt": "The capital of Canada is",
6+ "tokens": 5,
7+ "max_abs_dlogit": 0.0,
8+ "argmax_equal": true,
9+ "hit_rate": 0.1625,
10+ "misses": 536,
11+ "stall_s": 0.06717508309520781,
12+ "layers_routed": 80
13+ },
14+ {
15+ "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16+ "tokens": 21,
17+ "max_abs_dlogit": 0.0,
18+ "argmax_equal": true,
19+ "hit_rate": 0.4425981873111782,
20+ "misses": 1476,
21+ "stall_s": 0.18977228936273605,
22+ "layers_routed": 331
23+ },
24+ {
25+ "prompt": "Q: If a train travels 60 km in 45 minute",
26+ "tokens": 25,
27+ "max_abs_dlogit": 0.0,
28+ "argmax_equal": true,
29+ "hit_rate": 0.375,
30+ "misses": 1995,
31+ "stall_s": 0.2510305915493518,
32+ "layers_routed": 399
33+ },
34+ {
35+ "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36+ "tokens": 26,
37+ "max_abs_dlogit": 0.0,
38+ "argmax_equal": true,
39+ "hit_rate": 0.3878048780487805,
40+ "misses": 2008,
41+ "stall_s": 0.250745591474697,
42+ "layers_routed": 410
43+ }
44+ ],
45+ "tol": 0.001,
46+ "ts": "2026-11-01T01:50:26",
47+ "max_abs_dlogit_worst": 0.0,
48+ "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49+ "model_type": "olmoe"
50+}
\ No newline at end of file
modified paper/main.pdf +0 −0

Binary file not shown.

modified paper/sections/results.tex +13 −4
@@ -48,8 +48,9 @@ model does 53 misses per token (Prop.~\ref{prop:capacity}), i.e.\ about 11~ms of
48 From the v1 traces (18 prompts, 3\,686 tokens), per-layer LRU hit rates are 0.57 at 16 slots, 0.80 at 32 and 0.93 at 48 (LFU 48 From the v1 traces (18 prompts, 3\,686 tokens), per-layer LRU hit rates are 0.57 at 16 slots, 0.80 at 32 and 0.93 at 48 (LFU
49 within 0.01), nearly identical on fit and held-out prompts (\code{benchmarks/processed/gateC/sim\_olmoe\_laptop\_ssd\_v1.json}); 49 within 0.01), nearly identical on fit and held-out prompts (\code{benchmarks/processed/gateC/sim\_olmoe\_laptop\_ssd\_v1.json});
50 a one-layer-ahead transition predictor has recall@8 of 0.50 on held-out prompts; a linear probe on the previous layer's 50 a one-layer-ahead transition predictor has recall@8 of 0.50 on held-out prompts; a linear probe on the previous layer's
51−log-probabilities only 0.16 with these few traces. The v2 traces (65 prompts, 6 families) and the depth-$d$ predictors: 51+log-probabilities only 0.16 with these few traces. Table~\ref{tab:sim} is generated from the simulation JSON (v2 traces when available, else v1).
52−\textsc{pending} (simulation running at the time of writing; Table~\ref{tab:sim} is generated from its JSON). 52+
53+\input{tables/sim}
53 54
54 \subsection{Prefetching (H1)} 55 \subsection{Prefetching (H1)}
55 \label{sec:results-prefetch} 56 \label{sec:results-prefetch}
@@ -57,10 +58,18 @@ With the one-layer horizon the online prefetcher raised the hit rate from 0.48 t
57 (code) but reduced throughput from 51 to 35 and from 73 to 56~\tps{} (Table~\ref{tab:capacity}): the lead time 58 (code) but reduced throughput from 51 to 35 and from 73 to 56~\tps{} (Table~\ref{tab:capacity}): the lead time
58 $d\,\tau_\ell = 0.13$~ms is shorter than the 0.21~ms fetch, so predicted experts arrive late and are waited for, and the 59 $d\,\tau_\ell = 0.13$~ms is shorter than the 0.21~ms fetch, so predicted experts arrive late and are waited for, and the
59 50\,\% of wrong predictions add traffic. This is exactly the regime excluded by condition~\eqref{eq:condition}. 60 50\,\% of wrong predictions add traffic. This is exactly the regime excluded by condition~\eqref{eq:condition}.
60−Depth-$d$ and token-horizon prefetching, and the cold-NVMe tier: \textsc{pending} (next experiments of Gate C). 61+Depth-$d$ and token-horizon prefetching on the cold-NVMe tier are reported in Table~\ref{tab:cold} (generated from
62+\code{benchmarks/raw/gateC/cold\_nvme\_laptop/sweep.jsonl}; rows present only for configurations that have run).
63+
64+\input{tables/cold}
61 65
62 \subsection{Medium model (Qwen3-30B-A3B)} 66 \subsection{Medium model (Qwen3-30B-A3B)}
63−mlx-lm and \titan{} all-resident baselines on M3U96a (96~GB): \textsc{pending} (running). 67+On M3U96a (M3 Ultra, 96~GB) the stacked-layout registry reads byte-equal slices, logits are bit-identical to mlx-lm at 128, 64,
68+32 and 16 slots per layer and greedy decoding is token-identical at 32 slots
69+(\code{benchmarks/raw/tests/m3u96a/}). Table~\ref{tab:qwen3} gives the baselines: mlx-lm 121.4~\tps{} and \titan{} all-resident
70+121.7~\tps{} (medians of 3 repeats), i.e.\ parity on a model whose decode step touches 0.97~GiB of expert weights.
71+
72+\input{tables/qwen3}
64 73
65 \subsection{Distributed execution (H2, H3), Metal kernels (H4), energy frontier (H5)} 74 \subsection{Distributed execution (H2, H3), Metal kernels (H4), energy frontier (H5)}
66 \textsc{pending}: Gates D--F have protocols (Section~\ref{sec:algorithms}, \code{research/protocol.md}) and no logs in this version. 75 \textsc{pending}: Gates D--F have protocols (Section~\ref{sec:algorithms}, \code{research/protocol.md}) and no logs in this version.
modified paper/tables/cold.tex +1 −0
@@ -6,6 +6,7 @@
6 \toprule 6 \toprule
7 Backend & cfg & $n$ & decode tok/s (median) & p95 ITL ms & hit & stall s & TTFT ms \\ 7 Backend & cfg & $n$ & decode tok/s (median) & p95 ITL ms & hit & stall s & TTFT ms \\
8 \midrule 8 \midrule
9+titan & C=16, none, m=8, cold & 1 & 9.9 & 139.6 & 0.490 & 7.89 & 2849 \\
9 \bottomrule 10 \bottomrule
10 \end{tabular} 11 \end{tabular}
11 \begin{tablenotes}\footnotesize 12 \begin{tablenotes}\footnotesize
modified research/DECISIONS.md +7 −0
@@ -53,3 +53,10 @@ on M3U96a (cluster coordinator for Gates E/F), then rsync to worker nodes over t
53 OLMoE (3.6 GB of experts) fits in every node's RAM, so the memory-budget regimes of H1/H3 need a model whose experts exceed 53 OLMoE (3.6 GB of experts) fits in every node's RAM, so the memory-budget regimes of H1/H3 need a model whose experts exceed
54 the budget of the smaller nodes (36–64 GB) and whose expert count (6 144) gives a realistic routing problem. gpt-oss-120b (62 GB) 54 the budget of the smaller nodes (36–64 GB) and whose expert count (6 144) gives a realistic routing problem. gpt-oss-120b (62 GB)
55 is deferred until explicitly authorized. 55 is deferred until explicitly authorized.
56+
57+## D-009 · 2026-11-01 · Expert-contiguous storage layout for the SSD tier
58+Decision: the SSD tier reads from a TITAN blob (`experts.titan`, one 4 KiB-aligned contiguous record per expert, index JSON)
59+generated once from the checkpoint by `scripts/convert_layout.py`; the source checkpoint is never modified and remains the
60+source of dense weights. Rationale: measured 9 scattered reads per expert in the HF safetensors (expert span 1.5 GB) → cold
61+16-slot decode 8–10 tok/s; with parallel reads 18; with the contiguous layout 36 tok/s (4.5× over v1). Colibri performs an
62+equivalent conversion to its own container; we keep safetensors for everything else so mlx-lm remains the exact reference.
modified research/LOG.md +13 −0
@@ -79,3 +79,16 @@ Chronological, append-only. Times are local (America/Toronto). Every entry names
79 - Cluster venvs ready on M3U96a, M3U96b, M4M64a, M2U64, M4M64c (`scripts/cluster_setup.sh`, mlx 0.32.4 / mlx-lm 0.32.2 everywhere). 79 - Cluster venvs ready on M3U96a, M3U96b, M4M64a, M2U64, M4M64c (`scripts/cluster_setup.sh`, mlx 0.32.4 / mlx-lm 0.32.2 everywhere).
80 - Paper skeleton v0.1 compiles (24 pages; `tab:sim` pending the v2 simulation). Cold-NVMe prefetch sweep launched on the laptop 80 - Paper skeleton v0.1 compiles (24 pages; `tab:sim` pending the v2 simulation). Cold-NVMe prefetch sweep launched on the laptop
81 (`benchmarks/scripts/run_cold_nvme_laptop.sh`: purge + F_NOCACHE, 16/32 slots, none/transition{1,2,3}/prev_rank, held-out). 81 (`benchmarks/scripts/run_cold_nvme_laptop.sh`: purge + F_NOCACHE, 16/32 slots, none/transition{1,2,3}/prev_rank, held-out).
82+- Offline simulation v2 (`benchmarks/processed/gateC/sim_olmoe_v2_ssd043_m8.json`, 65 prompts, per-layer LRU, t_read 0.43 ms, 1 loader),
83+ held-out: LRU hit 0.581 / 0.706 / 0.806 / 0.933 at C = 16 / 24 / 32 / 48. Transition predictor recall@8 **0.534 at depth 1, 0.525
84+ at depth 2, 0.517 at depth 3** → a 3-layer lead costs almost no accuracy, which is what the lead-time condition needs on a cold NVMe
85+ (d·τ_layer ≥ λ → d ≥ 4 at 0.43 ms). Hit rate with transition prefetch 0.686 / 0.771 / 0.840 / 0.942 (+0.01–0.10) at +60–100 % loads;
86+ prev_rank (m = 16) *hurts* (prefetch precision 0.18–0.47, evicts useful experts); the linear probe was broken by a label-alignment
87+ bug (rows not offset across traces) → fixed, re-simulated (`sim_olmoe_v2_ssd043_m8_probe_fixed.json`); oracle 0.94–0.99.
88+- Cold NVMe tier (purge + F_NOCACHE), 16 slots, held-out general prompts, laptop: **v1** (9 `pread` per expert, serial) 8–10 tok/s,
89+ stall 7–8 s / 64 tokens (`benchmarks/raw/gateC/cold_nvme_laptop/sweep_v1_9preads_serial.jsonl`); **v2** parallel demand reads
90+ (4 reader threads) 18 tok/s, stall 3.0 s (`quick_v2.jsonl`); finding: in the mlx-community OLMoE file the 9 tensors of one expert are
91+ scattered over a 1.5 GB span (hash order of keys), so every miss costs 9 random reads. **v3**: `scripts/convert_layout.py` writes an
92+ expert-contiguous blob (one 4 KiB-aligned 3.4 MB record per expert, 3.6 GB written in 3.2 s; `~/.cache/titan/<hash>/`), the registry
93+ points at it transparently, tests verify byte equality with the original (`tests/test_registry_store.py`, one pread per expert) →
94+ **36 tok/s**, stall 1.2 s / 64 tokens (`quick_v3.jsonl`), vs 51–73 tok/s from the page cache. Cold sweep restarted with v3.
added scripts/convert_layout.py +48 −0
@@ -0,0 +1,48 @@
1+#!/usr/bin/env python3
2+"""Convert a checkpoint's expert tensors into the TITAN expert-contiguous layout: one 4 KiB-aligned blob per (layer, expert)
3+holding its tensors back to back, so that the SSD tier reads an expert with a single pread. Dense tensors stay in the
4+original safetensors. Output: <out_dir>/experts.titan (blob) + <out_dir>/experts.titan.json (index). Never modifies the source.
5+"""
6+import argparse, hashlib, json, os, sys, time
7+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
8+from titan.registry import load_spec
9+
10+ALIGN = 4096
11+
12+def main():
13+ ap = argparse.ArgumentParser()
14+ ap.add_argument("model_dir")
15+ ap.add_argument("--out", default=None, help="default: ~/.cache/titan/<sha1 of realpath>/")
16+ a = ap.parse_args()
17+ spec = load_spec(a.model_dir)
18+ out = a.out or os.path.expanduser(f"~/.cache/titan/{hashlib.sha1(os.path.realpath(a.model_dir).encode()).hexdigest()[:12]}")
19+ os.makedirs(out, exist_ok=True)
20+ blob = os.path.join(out, "experts.titan")
21+ index = {"source": os.path.realpath(a.model_dir), "model_type": spec.model_type, "align": ALIGN, "experts": {}}
22+ fds = {}
23+ t0 = time.perf_counter(); written = 0
24+ with open(blob, "wb") as w:
25+ pos = 0
26+ for l in range(spec.num_layers):
27+ for e in range(spec.num_experts):
28+ es = spec.experts[(l, e)]
29+ parts = {}; off = 0
30+ chunks = []
31+ for name, loc in es.tensors.items(): # fixed order: as in the registry (gate, up, down × weight, scales, biases)
32+ fd = fds.get(loc.file) or fds.setdefault(loc.file, os.open(loc.file, os.O_RDONLY))
33+ b = os.pread(fd, loc.nbytes, loc.offset)
34+ assert len(b) == loc.nbytes
35+ parts[name] = {"off": off, "n": loc.nbytes, "dtype": loc.dtype, "shape": list(loc.shape)}
36+ chunks.append(b); off += loc.nbytes
37+ data = b"".join(chunks)
38+ pad = (-len(data)) % ALIGN
39+ w.write(data); w.write(b"\0" * pad)
40+ index["experts"][f"{l},{e}"] = {"offset": pos, "nbytes": len(data), "tensors": parts}
41+ pos += len(data) + pad; written += len(data)
42+ print(f"layer {l} done, {pos/1e9:.2f} GB", file=sys.stderr)
43+ index["blob_bytes"] = pos
44+ json.dump(index, open(blob + ".json", "w"))
45+ print(json.dumps({"out": out, "experts": len(index["experts"]), "bytes": written, "blob_bytes": pos, "seconds": time.perf_counter() - t0}))
46+
47+if __name__ == "__main__":
48+ main()
modified tests/test_registry_store.py +4 −1
@@ -15,8 +15,9 @@ def test_registry_olmoe(model_dir):
15 def test_store_reads_match_mx_load(model_dir): 15 def test_store_reads_match_mx_load(model_dir):
16 """Bytes read with pread at registry offsets must equal what mx.load returns for the same tensor.""" 16 """Bytes read with pread at registry offsets must equal what mx.load returns for the same tensor."""
17 s = load_spec(model_dir) 17 s = load_spec(model_dir)
18− if s.layout != "per_expert": 18+ if s.layout.startswith("stacked"):
19 pytest.skip("stacked layout covered by test_registry_stacked.py") 19 pytest.skip("stacked layout covered by test_registry_stacked.py")
20+ # with the TITAN contiguous layout present, this also verifies the conversion byte-for-byte against the original file
20 st = ExpertStore(s) 21 st = ExpertStore(s)
21 ref = mx.load(f"{model_dir}/model.safetensors") 22 ref = mx.load(f"{model_dir}/model.safetensors")
22 for (l, e) in [(0, 0), (7, 33), (15, 63)]: 23 for (l, e) in [(0, 0), (7, 33), (15, 63)]:
@@ -27,4 +28,6 @@ def test_store_reads_match_mx_load(model_dir):
27 assert arr.shape == r.shape and arr.dtype == r.dtype 28 assert arr.shape == r.shape and arr.dtype == r.dtype
28 assert bool(mx.array_equal(arr, r)), (l, e, name) 29 assert bool(mx.array_equal(arr, r)), (l, e, name)
29 assert st.stats()["bytes_read"] == 3 * s.bytes_per_expert 30 assert st.stats()["bytes_read"] == 3 * s.bytes_per_expert
31+ if "titan_contiguous" in s.layout:
32+ assert st.stats()["reads"] == 3 # one pread per expert
30 st.close() 33 st.close()
modified titan/predictors.py +4 −5
@@ -88,11 +88,10 @@ class Probe(Predictor):
88 for l in range(1, self.L): 88 for l in range(1, self.L):
89 X = np.concatenate([p[:, l - 1].astype(np.float32) for _, p in traces], 0) 89 X = np.concatenate([p[:, l - 1].astype(np.float32) for _, p in traces], 0)
90 Y = np.zeros((X.shape[0], self.E), np.float32) 90 Y = np.zeros((X.shape[0], self.E), np.float32)
91− rows = np.concatenate([np.repeat(np.arange(t.shape[0]), t.shape[2]) for t, _ in traces]); off = 0; cols = [] 91+ rows, cols, off = [], [], 0
92− for t, _ in traces: 92+ for t, _ in traces: # row offsets accumulate across traces (bug fixed 2026-11-01: labels were misaligned)
93− cols.append(t[:, l].reshape(-1)); 93+ rows.append(off + np.repeat(np.arange(t.shape[0]), t.shape[2])); cols.append(t[:, l].reshape(-1)); off += t.shape[0]
94− cols = np.concatenate(cols) 94+ Y[np.concatenate(rows), np.concatenate(cols)] = 1.0
95− Y[rows, cols] = 1.0
96 X = np.log(np.clip(X, 1e-6, None)) # log-probabilities as features (scale-free) 95 X = np.log(np.clip(X, 1e-6, None)) # log-probabilities as features (scale-free)
97 self.mu[l] = X.mean(0); self.sd[l] = X.std(0) + 1e-6 96 self.mu[l] = X.mean(0); self.sd[l] = X.std(0) + 1e-6
98 Xb = np.concatenate([(X - self.mu[l]) / self.sd[l], np.ones((X.shape[0], 1), np.float32)], 1) 97 Xb = np.concatenate([(X - self.mu[l]) / self.sd[l], np.ones((X.shape[0], 1), np.float32)], 1)
modified titan/registry.py +24 −1
@@ -123,7 +123,26 @@ def read_safetensors_header(path: str) -> Dict[str, TensorLoc]:
123 return locs 123 return locs
124 124
125 125
126−def load_spec(model_dir: str) -> ModelSpec: 126+def titan_layout_dir(model_dir: str):
127+ import hashlib
128+ d = os.path.expanduser(f"~/.cache/titan/{hashlib.sha1(os.path.realpath(model_dir).encode()).hexdigest()[:12]}")
129+ return d if os.path.exists(os.path.join(d, "experts.titan.json")) else None
130+
131+
132+def apply_titan_layout(spec: "ModelSpec", layout_dir: str) -> "ModelSpec":
133+ """Point every expert's tensors at the contiguous blob produced by scripts/convert_layout.py."""
134+ idx = json.load(open(os.path.join(layout_dir, "experts.titan.json")))
135+ blob = os.path.join(layout_dir, "experts.titan")
136+ for key, ent in idx["experts"].items():
137+ l, e = map(int, key.split(","))
138+ es = spec.experts[(l, e)]
139+ es.tensors = {name: TensorLoc(f"titan:{l},{e}:{name}", p["dtype"], tuple(p["shape"]), blob, ent["offset"] + p["off"], p["n"])
140+ for name, p in ent["tensors"].items()}
141+ spec.layout = spec.layout + "+titan_contiguous"
142+ return spec
143+
144+
145+def load_spec(model_dir: str, use_titan_layout: bool = True) -> ModelSpec:
127 model_dir = os.path.realpath(model_dir) 146 model_dir = os.path.realpath(model_dir)
128 with open(os.path.join(model_dir, "config.json")) as f: 147 with open(os.path.join(model_dir, "config.json")) as f:
129 cfg = json.load(f) 148 cfg = json.load(f)
@@ -170,6 +189,10 @@ def load_spec(model_dir: str) -> ModelSpec:
170 raise ValueError(f"no expert tensors found for layer {l} expert {e} (neither per-expert nor stacked keys)") 189 raise ValueError(f"no expert tensors found for layer {l} expert {e} (neither per-expert nor stacked keys)")
171 spec.experts[(l, e)] = ExpertSpec(l, e, tensors) 190 spec.experts[(l, e)] = ExpertSpec(l, e, tensors)
172 spec.dense = {k: v for k, v in locs.items() if k not in per_expert_keys} 191 spec.dense = {k: v for k, v in locs.items() if k not in per_expert_keys}
192+ if use_titan_layout:
193+ d = titan_layout_dir(model_dir)
194+ if d:
195+ apply_titan_layout(spec, d)
173 return spec 196 return spec
174 197
175 198
modified titan/slots.py +1 −1
@@ -142,7 +142,7 @@ class SlotTable:
142 self.misses += len(missing) 142 self.misses += len(missing)
143 if missing: 143 if missing:
144 t = time.perf_counter() 144 t = time.perf_counter()
145− tensors = {e: self.store.read_expert(self.layer, e) for e in missing} 145+ tensors = self.store.read_experts(self.layer, missing)
146 self.stall_time += time.perf_counter() - t 146 self.stall_time += time.perf_counter() - t
147 for e in missing: 147 for e in missing:
148 self._install(e, tensors[e], needed) 148 self._install(e, tensors[e], needed)
modified titan/store.py +44 −3
@@ -15,6 +15,7 @@ from typing import Dict, Tuple
15 import mlx.core as mx 15 import mlx.core as mx
16 import numpy as np 16 import numpy as np
17 17
18+from concurrent.futures import ThreadPoolExecutor
18 from .registry import ModelSpec, TensorLoc 19 from .registry import ModelSpec, TensorLoc
19 20
20 F_NOCACHE = 48 21 F_NOCACHE = 48
@@ -23,13 +24,21 @@ MX_DTYPE = {"F32": mx.float32, "F16": mx.float16, "U32": mx.uint32, "I32": mx.in
23 24
24 25
25 class ExpertStore: 26 class ExpertStore:
26− def __init__(self, spec: ModelSpec, nocache: bool = False): 27+ def __init__(self, spec: ModelSpec, nocache: bool = False, readers: int = 4):
27 self.spec = spec 28 self.spec = spec
28 self.nocache = nocache 29 self.nocache = nocache
29 self._fds: Dict[str, int] = {} 30 self._fds: Dict[str, int] = {}
30 self.bytes_read = 0 31 self.bytes_read = 0
31 self.reads = 0 32 self.reads = 0
32 self.read_time = 0.0 33 self.read_time = 0.0
34+ self.readers = readers
35+ self._pool = ThreadPoolExecutor(max_workers=readers, thread_name_prefix="titan-store") if readers > 1 else None
36+ # an expert whose tensors are contiguous in the file is read with ONE pread (OLMoE: all 1 024 experts are contiguous)
37+ self._contig: Dict[Tuple[int, int], Tuple[str, int, int, list]] = {}
38+ for key, es in spec.experts.items():
39+ locs = sorted(es.tensors.items(), key=lambda kv: kv[1].offset)
40+ if all(a[1].file == b[1].file and a[1].offset + a[1].nbytes == b[1].offset for a, b in zip(locs, locs[1:])):
41+ self._contig[key] = (locs[0][1].file, locs[0][1].offset, sum(t.nbytes for _, t in locs), locs)
33 42
34 def _fd(self, path: str) -> int: 43 def _fd(self, path: str) -> int:
35 fd = self._fds.get(path) 44 fd = self._fds.get(path)
@@ -54,9 +63,41 @@ class ExpertStore:
54 return mx.array(arr).view(mx.bfloat16) 63 return mx.array(arr).view(mx.bfloat16)
55 return mx.array(arr) 64 return mx.array(arr)
56 65
66+ def _wrap(self, loc: TensorLoc, buf) -> mx.array:
67+ arr = np.frombuffer(buf, dtype=NP_DTYPE[loc.dtype]).reshape(loc.shape)
68+ if loc.dtype == "BF16":
69+ return mx.array(arr).view(mx.bfloat16)
70+ return mx.array(arr)
71+
57 def read_expert(self, layer: int, expert: int) -> Dict[str, mx.array]: 72 def read_expert(self, layer: int, expert: int) -> Dict[str, mx.array]:
58− es = self.spec.experts[(layer, expert)] 73+ c = self._contig.get((layer, expert))
59− return {name: self.read_tensor(loc) for name, loc in es.tensors.items()} 74+ if c is None:
75+ es = self.spec.experts[(layer, expert)]
76+ return {name: self.read_tensor(loc) for name, loc in es.tensors.items()}
77+ path, off, n, locs = c
78+ fd = self._fd(path)
79+ t = time.perf_counter()
80+ buf = os.pread(fd, n, off)
81+ self.read_time += time.perf_counter() - t
82+ self.bytes_read += len(buf); self.reads += 1
83+ if len(buf) != n:
84+ raise IOError(f"short read for expert ({layer},{expert}): {len(buf)} != {n}")
85+ out = {}
86+ for name, loc in locs:
87+ a = loc.offset - off
88+ out[name] = self._wrap(loc, memoryview(buf)[a:a + loc.nbytes])
89+ return out
90+
91+ def read_experts(self, layer: int, experts) -> Dict[int, Dict[str, mx.array]]:
92+ """Read several experts, in parallel when a reader pool exists (NVMe benefits from queue depth)."""
93+ experts = list(experts)
94+ if self._pool is None or len(experts) == 1:
95+ return {e: self.read_expert(layer, e) for e in experts}
96+ t = time.perf_counter()
97+ res = dict(zip(experts, self._pool.map(lambda e: self.read_expert(layer, e), experts)))
98+ self.read_time += 0 # per-read times already accumulated inside read_expert (sum of thread times, not wall)
99+ self.wall_batch_time = getattr(self, "wall_batch_time", 0.0) + (time.perf_counter() - t)
100+ return res
60 101
61 def read_expert_bytes(self, layer: int, expert: int) -> Dict[str, bytes]: 102 def read_expert_bytes(self, layer: int, expert: int) -> Dict[str, bytes]:
62 """Raw bytes (for the network transport).""" 103 """Raw bytes (for the network transport)."""
63 104
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere