SPB Git
spboucher.ai

spb/wp33_uqo

Public MIT

Fix: MLX arrays created on the consuming thread only (worker/stacked-layout stream error)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 1, 2026) parent e0179f6 Browse files

18 changed files +889 −11

18 changed files
Unified Split
modified benchmarks/raw/gateG/energy_laptop/bench.jsonl +18 −0
@@ -34,3 +34,21 @@
34 34 {"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:25:58", "prompt_tokens": 18, "gen_tokens": 256, "ttft_ms": 473.99025002960116, "prompt_tps": 38.02749282251309, "gen_tps": 30.149703436329805, "decode_tps_wall": 30.03199408681774, "itl_ms_p50": 32.685291953384876, "itl_ms_p95": 43.34554204251617, "itl_ms_p99": 49.13512500934303, "itl_ms_mean": 33.29782221950225, "total_s": 8.964934916002676, "peak_mem_gb": 1.779299288, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "t_start_epoch": 1793514349.968512, "t_end_epoch": 1793514358.9347358, "t_first_epoch": 1793514350.4425023, "cache_hit_rate": 0.5059072218683037, "cache_misses": 17063, "cache_hits": 17471, "stall_s": 4.963466331246309, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.586632123333402, "prefetch": 0.0, "ensure": 5.089648996130563, "build": 0.053854094934649765}, "ssd_bytes": 60385001472, "ssd_read_s": 12.836250820546411}
35 35 {"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:26:07", "prompt_tokens": 14, "gen_tokens": 256, "ttft_ms": 449.9966250732541, "prompt_tps": 31.15616929910054, "gen_tps": 32.46190672778869, "decode_tps_wall": 32.33514887274489, "itl_ms_p50": 31.00487496703863, "itl_ms_p95": 41.70525004155934, "itl_ms_p99": 44.96754193678498, "itl_ms_mean": 30.926098529358995, "total_s": 8.336151750059798, "peak_mem_gb": 1.74502108, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "t_start_epoch": 1793514358.934883, "t_end_epoch": 1793514367.2724721, "t_first_epoch": 1793514359.3848798, "cache_hit_rate": 0.49322548618816797, "cache_misses": 17355, "cache_hits": 16891, "stall_s": 4.965477578341961, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.010114970151335, "prefetch": 0.0, "ensure": 5.079476987943053, "build": 0.04450405645184219}, "ssd_bytes": 61418373120, "ssd_read_s": 13.378405682509765}
36 36 {"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:26:14", "prompt_tokens": 19, "gen_tokens": 256, "ttft_ms": 430.6972920894623, "prompt_tps": 44.19175033284113, "gen_tps": 35.12275901609846, "decode_tps_wall": 34.985638138366845, "itl_ms_p50": 27.491708053275943, "itl_ms_p95": 44.86395907588303, "itl_ms_p99": 47.699542017653584, "itl_ms_mean": 28.583157352884026, "total_s": 7.719402417074889, "peak_mem_gb": 1.775334664, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array in descending ord", "t_start_epoch": 1793514367.272643, "t_end_epoch": 1793514374.9930902, "t_first_epoch": 1793514367.7033403, "cache_hit_rate": 0.6598117811084001, "cache_misses": 11712, "cache_hits": 22716, "stall_s": 4.332302771508694, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.0599256032146513, "prefetch": 0.0, "ensure": 4.420602378551848, "build": 0.04536858550272882}, "ssd_bytes": 41448112128, "ssd_read_s": 10.390336600947194}
37 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:26:26", "prompt_tokens": 18, "gen_tokens": 256, "ttft_ms": 639.3235409632325, "prompt_tps": 28.174881999836646, "gen_tps": 23.48477154172949, "decode_tps_wall": 23.3930878035731, "itl_ms_p50": 42.089167051017284, "itl_ms_p95": 57.280458975583315, "itl_ms_p99": 63.42966703232378, "itl_ms_mean": 42.74767009797049, "total_s": 11.539979415945709, "peak_mem_gb": 1.816294424, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "t_start_epoch": 1793514374.993252, "t_end_epoch": 1793514386.534442, "t_first_epoch": 1793514375.6325755, "cache_hit_rate": 0.5050095558000811, "cache_misses": 17094, "cache_hits": 17440, "stall_s": 6.836809101747349, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 4.1504173673456535, "prefetch": 0.0, "ensure": 6.992573518422432, "build": 0.08305123588070273}, "ssd_bytes": 60494708736, "ssd_read_s": 18.493526841746643}
38 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:26:37", "prompt_tokens": 14, "gen_tokens": 256, "ttft_ms": 531.4388750120997, "prompt_tps": 26.44934124112812, "gen_tps": 24.711482283116222, "decode_tps_wall": 24.61498107386689, "itl_ms_p50": 39.87624996807426, "itl_ms_p95": 57.44204192887992, "itl_ms_p99": 77.63004198204726, "itl_ms_mean": 40.62566601205617, "total_s": 10.890983708086424, "peak_mem_gb": 1.74502108, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "t_start_epoch": 1793514386.535053, "t_end_epoch": 1793514397.427111, "t_first_epoch": 1793514387.0664918, "cache_hit_rate": 0.49322548618816797, "cache_misses": 17355, "cache_hits": 16891, "stall_s": 6.635692013427615, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.8223506945651025, "prefetch": 0.0, "ensure": 6.766858409275301, "build": 0.056402937159873545}, "ssd_bytes": 61418373120, "ssd_read_s": 18.161584803368896}
39 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:26:45", "prompt_tokens": 19, "gen_tokens": 256, "ttft_ms": 458.79750000312924, "prompt_tps": 41.46472689933416, "gen_tps": 35.435561883947024, "decode_tps_wall": 35.29719546445759, "itl_ms_p50": 27.789458981715143, "itl_ms_p95": 39.23400002531707, "itl_ms_p99": 43.578875018283725, "itl_ms_mean": 28.330862745368744, "total_s": 7.683167500072159, "peak_mem_gb": 1.775334664, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array in descending ord", "t_start_epoch": 1793514397.427271, "t_end_epoch": 1793514405.111447, "t_first_epoch": 1793514397.8860683, "cache_hit_rate": 0.6598117811084001, "cache_misses": 11712, "cache_hits": 22716, "stall_s": 4.178970686276443, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.186878505628556, "prefetch": 0.0, "ensure": 4.265643556253053, "build": 0.042900146334432065}, "ssd_bytes": 41448112128, "ssd_read_s": 10.073308067861944}
40 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:26:53", "prompt_tokens": 24, "gen_tokens": 256, "ttft_ms": 623.6247919732705, "prompt_tps": 38.612191977894824, "gen_tps": 33.16532823969986, "decode_tps_wall": 33.03582378968533, "itl_ms_p50": 29.62212497368455, "itl_ms_p95": 43.53091702796519, "itl_ms_p99": 48.3604590408504, "itl_ms_mean": 30.27016993329002, "total_s": 8.342518124962226, "peak_mem_gb": 1.88188166, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "t_start_epoch": 1793514405.111955, "t_end_epoch": 1793514413.4554849, "t_first_epoch": 1793514405.7355797, "cache_hit_rate": 0.6277690213497635, "cache_misses": 12989, "cache_hits": 21906, "stall_s": 4.633197326445952, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.355849905521609, "prefetch": 0.0, "ensure": 4.731186131131835, "build": 0.047959326999261975}, "ssd_bytes": 45967343616, "ssd_read_s": 11.483838297543116}
41 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:27:04", "prompt_tokens": 18, "gen_tokens": 256, "ttft_ms": 682.5923330616206, "prompt_tps": 26.453586285389044, "gen_tps": 25.752552973495998, "decode_tps_wall": 25.651980179231074, "itl_ms_p50": 37.54041704814881, "itl_ms_p95": 61.23966700397432, "itl_ms_p99": 66.31250004284084, "itl_ms_mean": 38.9833452627428, "total_s": 10.623345375061035, "peak_mem_gb": 1.823634456, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "t_start_epoch": 1793514413.4557571, "t_end_epoch": 1793514424.080147, "t_first_epoch": 1793514414.1383495, "cache_hit_rate": 0.5059072218683037, "cache_misses": 17063, "cache_hits": 17471, "stall_s": 6.51750842097681, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.7390375823015347, "prefetch": 0.0, "ensure": 6.631261183996685, "build": 0.04644273582380265}, "ssd_bytes": 60385001472, "ssd_read_s": 18.00621229421813}
42 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:27:12", "prompt_tokens": 24, "gen_tokens": 256, "ttft_ms": 609.8394999280572, "prompt_tps": 39.39834350012532, "gen_tps": 33.499659360229174, "decode_tps_wall": 33.36883843156555, "itl_ms_p50": 29.542250093072653, "itl_ms_p95": 43.06070797611028, "itl_ms_p99": 46.94349993951619, "itl_ms_mean": 29.968079411899488, "total_s": 8.251699749962427, "peak_mem_gb": 1.917582228, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "t_start_epoch": 1793514424.0803132, "t_end_epoch": 1793514432.333015, "t_first_epoch": 1793514424.6901526, "cache_hit_rate": 0.6274537899412523, "cache_misses": 13000, "cache_hits": 21895, "stall_s": 4.565376511425711, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.3453963060164824, "prefetch": 0.0, "ensure": 4.660446377354674, "build": 0.04569679545238614}, "ssd_bytes": 46006272000, "ssd_read_s": 11.44613619998563}
43 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:27:22", "prompt_tokens": 14, "gen_tokens": 256, "ttft_ms": 501.20487494859844, "prompt_tps": 27.96523686232182, "gen_tps": 27.75032254346846, "decode_tps_wall": 27.641952809705582, "itl_ms_p50": 36.01766692008823, "itl_ms_p95": 47.68933297600597, "itl_ms_p99": 52.69720801152289, "itl_ms_mean": 36.176894117585, "total_s": 9.726312874932773, "peak_mem_gb": 1.730718152, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "t_start_epoch": 1793514432.33316, "t_end_epoch": 1793514442.060885, "t_first_epoch": 1793514432.834365, "cache_hit_rate": 0.49538632249021786, "cache_misses": 17281, "cache_hits": 16965, "stall_s": 5.6758743927348405, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.65540738531854, "prefetch": 0.0, "ensure": 5.796999507001601, "build": 0.051514498074539006}, "ssd_bytes": 61156491264, "ssd_read_s": 15.6160844755359}
44 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:27:29", "prompt_tokens": 19, "gen_tokens": 256, "ttft_ms": 502.0306670339778, "prompt_tps": 37.89478830273463, "gen_tps": 34.92529584914807, "decode_tps_wall": 34.78891696746336, "itl_ms_p50": 28.797582956030965, "itl_ms_p95": 38.754915934987366, "itl_ms_p99": 43.934584013186395, "itl_ms_mean": 28.74478676456812, "total_s": 7.831951291998848, "peak_mem_gb": 1.766782292, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array in descending ord", "t_start_epoch": 1793514442.0610409, "t_end_epoch": 1793514449.894342, "t_first_epoch": 1793514442.5630715, "cache_hit_rate": 0.6598117811084001, "cache_misses": 11712, "cache_hits": 22716, "stall_s": 4.281426511122845, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.2393267332809046, "prefetch": 0.0, "ensure": 4.365906663937494, "build": 0.04135329520795494}, "ssd_bytes": 41448112128, "ssd_read_s": 10.341405491228215}
45 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 2, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:27:39", "prompt_tokens": 14, "gen_tokens": 256, "ttft_ms": 482.9200420062989, "prompt_tps": 29.02662330914426, "gen_tps": 29.608985443323636, "decode_tps_wall": 29.49336102167945, "itl_ms_p50": 33.19170803297311, "itl_ms_p95": 47.35679202713072, "itl_ms_p99": 53.26833401340991, "itl_ms_mean": 33.90593561937339, "total_s": 9.128933624946512, "peak_mem_gb": 1.73301178, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "t_start_epoch": 1793514449.894492, "t_end_epoch": 1793514459.024576, "t_first_epoch": 1793514450.3774118, "cache_hit_rate": 0.49383869649010104, "cache_misses": 17334, "cache_hits": 16912, "stall_s": 5.228863131022081, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.534531174809672, "prefetch": 0.0, "ensure": 5.34318076109048, "build": 0.045298933633603156}, "ssd_bytes": 61344055296, "ssd_read_s": 14.219157688086852}
46 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 2, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:27:47", "prompt_tokens": 18, "gen_tokens": 256, "ttft_ms": 502.13170901406556, "prompt_tps": 35.90810218990763, "gen_tps": 30.72349003220354, "decode_tps_wall": 30.603541595917655, "itl_ms_p50": 31.901582959108055, "itl_ms_p95": 46.237667091190815, "itl_ms_p99": 53.17033396568149, "itl_ms_mean": 32.675956698207585, "total_s": 8.834500667057, "peak_mem_gb": 1.816294424, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "t_start_epoch": 1793514459.024839, "t_end_epoch": 1793514467.862941, "t_first_epoch": 1793514459.5269706, "cache_hit_rate": 0.5046910291307117, "cache_misses": 17105, "cache_hits": 17429, "stall_s": 4.990589527995326, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.485891460906714, "prefetch": 0.0, "ensure": 5.102543488028459, "build": 0.04451872012577951}, "ssd_bytes": 60533637120, "ssd_read_s": 13.334814880625345}
47 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 2, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:27:55", "prompt_tokens": 19, "gen_tokens": 256, "ttft_ms": 493.39433398563415, "prompt_tps": 38.57391560044966, "gen_tps": 37.846937735738834, "decode_tps_wall": 37.6991578148187, "itl_ms_p50": 26.25908306799829, "itl_ms_p95": 37.12208301294595, "itl_ms_p99": 40.99449992645532, "itl_ms_mean": 26.525791502082903, "total_s": 7.257471167016774, "peak_mem_gb": 1.773794588, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array in descending ord", "t_start_epoch": 1793514467.863209, "t_end_epoch": 1793514475.1218529, "t_first_epoch": 1793514468.3566034, "cache_hit_rate": 0.659840827233647, "cache_misses": 11711, "cache_hits": 22717, "stall_s": 3.7971609133528545, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.158985341200605, "prefetch": 0.0, "ensure": 3.880476161488332, "build": 0.03890139679424465}, "ssd_bytes": 41444573184, "ssd_read_s": 8.934044936555438}
48 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 2, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:28:02", "prompt_tokens": 24, "gen_tokens": 256, "ttft_ms": 660.2665830869228, "prompt_tps": 36.401945768739516, "gen_tps": 36.0720181380442, "decode_tps_wall": 35.93119493066561, "itl_ms_p50": 27.531874948181212, "itl_ms_p95": 41.34691599756479, "itl_ms_p99": 46.20891599915922, "itl_ms_mean": 27.830969772356397, "total_s": 7.757163875037804, "peak_mem_gb": 1.924660096, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "t_start_epoch": 1793514475.1230562, "t_end_epoch": 1793514482.88139, "t_first_epoch": 1793514475.7833228, "cache_hit_rate": 0.6277690213497635, "cache_misses": 12989, "cache_hits": 21906, "stall_s": 4.170646916842088, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.277813452645205, "prefetch": 0.0, "ensure": 4.25988938997034, "build": 0.03849247260950506}, "ssd_bytes": 45967343616, "ssd_read_s": 10.19479018333368}
49 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --predictor probe --m 8 --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "e0179f6"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:30:09", "prompt_tokens": 24, "gen_tokens": 256, "ttft_ms": 2217.1235830755904, "prompt_tps": 10.9491894139912, "gen_tps": 45.25487565647324, "decode_tps_wall": 45.08706810413277, "itl_ms_p50": 22.148250020109117, "itl_ms_p95": 27.636207989417017, "itl_ms_p99": 29.460499994456768, "itl_ms_mean": 22.179308658757922, "total_s": 7.872847291058861, "peak_mem_gb": 1.9434972, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n // TODO: Implement the dot product computation using NEON intrinsics\n // Start by loading the arrays int", "t_start_epoch": 1793514601.242044, "t_end_epoch": 1793514609.116669, "t_first_epoch": 1793514603.4591675, "cache_hit_rate": 0.837569852414386, "cache_misses": 5668, "cache_hits": 29227, "stall_s": 1.7648288635537028, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 10497, "installed": 10497, "late": 231, "dup": 20463, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 5.376388295902871, "prefetch": 0.38092932023573667, "ensure": 1.7893478660844266, "build": 0.05343569233082235}, "ssd_bytes": 57207029760, "ssd_read_s": 6.779934052960016}
50 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --predictor probe --m 8 --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "e0179f6"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:30:15", "prompt_tokens": 18, "gen_tokens": 256, "ttft_ms": 298.4869999345392, "prompt_tps": 60.80249156492609, "gen_tps": 39.53517597853951, "decode_tps_wall": 39.380788319710305, "itl_ms_p50": 25.2036249730736, "itl_ms_p95": 30.1873330026865, "itl_ms_p99": 32.05770906060934, "itl_ms_mean": 25.393092486660418, "total_s": 6.773725584032945, "peak_mem_gb": 1.838679744, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "t_start_epoch": 1793514609.117152, "t_end_epoch": 1793514615.891936, "t_first_epoch": 1793514609.415639, "cache_hit_rate": 0.8176869172409799, "cache_misses": 6296, "cache_hits": 28238, "stall_s": 1.6834739480400458, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 15936, "installed": 15936, "late": 107, "dup": 15816, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 4.424398881732486, "prefetch": 0.3463717408012599, "ensure": 1.7266047825105488, "build": 0.05043836368713528}, "ssd_bytes": 78677803008, "ssd_read_s": 7.841062449966557}
51 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --predictor probe --m 8 --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "e0179f6"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:30:22", "prompt_tokens": 14, "gen_tokens": 256, "ttft_ms": 297.41445800755173, "prompt_tps": 47.21940919709833, "gen_tps": 38.782262704355546, "decode_tps_wall": 38.63083508335882, "itl_ms_p50": 26.01779205724597, "itl_ms_p95": 30.615541036240757, "itl_ms_p99": 32.95816702302545, "itl_ms_mean": 25.88605702781648, "total_s": 6.898359000100754, "peak_mem_gb": 1.751351044, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "t_start_epoch": 1793514615.892053, "t_end_epoch": 1793514622.791529, "t_first_epoch": 1793514616.1894674, "cache_hit_rate": 0.8065175494948316, "cache_misses": 6626, "cache_hits": 27620, "stall_s": 1.7857510162284598, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 16251, "installed": 16251, "late": 103, "dup": 15341, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 4.414945824421011, "prefetch": 0.3581021197605878, "ensure": 1.8337656208314002, "build": 0.05403699423186481}, "ssd_bytes": 80960421888, "ssd_read_s": 8.22376422479283}
52 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --predictor probe --m 8 --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "e0179f6"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:30:29", "prompt_tokens": 19, "gen_tokens": 256, "ttft_ms": 390.1689579943195, "prompt_tps": 48.74155925723839, "gen_tps": 37.77430336988103, "decode_tps_wall": 37.626785667837176, "itl_ms_p50": 24.885875056497753, "itl_ms_p95": 40.63329100608826, "itl_ms_p99": 71.47024990990758, "itl_ms_mean": 26.57681176457189, "total_s": 7.167255957960151, "peak_mem_gb": 1.784851132, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array in descending ord", "t_start_epoch": 1793514622.7916439, "t_end_epoch": 1793514629.960074, "t_first_epoch": 1793514623.1818128, "cache_hit_rate": 0.848495410712211, "cache_misses": 5216, "cache_hits": 29212, "stall_s": 2.3131685683038086, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 9759, "installed": 9758, "late": 341, "dup": 21945, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 4.139780909870751, "prefetch": 0.5597494770772755, "ensure": 2.1483880928717554, "build": 0.059471825836226344}, "ssd_bytes": 52995686400, "ssd_read_s": 7.924561083083972}
53 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --predictor probe --m 8 --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "e0179f6"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:30:38", "prompt_tokens": 18, "gen_tokens": 256, "ttft_ms": 838.4084589779377, "prompt_tps": 21.496361739417928, "gen_tps": 32.63569056171431, "decode_tps_wall": 32.50862027189841, "itl_ms_p50": 27.306250063702464, "itl_ms_p95": 54.80616702698171, "itl_ms_p99": 118.00695804413408, "itl_ms_mean": 30.76107172916333, "total_s": 8.682481749914587, "peak_mem_gb": 1.792476808, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "t_start_epoch": 1793514629.960502, "t_end_epoch": 1793514638.643897, "t_first_epoch": 1793514630.7989104, "cache_hit_rate": 0.8172815196617826, "cache_misses": 6310, "cache_hits": 28224, "stall_s": 3.7183112679049373, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 15840, "installed": 15836, "late": 1217, "dup": 15912, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 4.124537539551966, "prefetch": 1.1234240679768845, "ensure": 3.083217271254398, "build": 0.06652727141045034}, "ssd_bytes": 78387609600, "ssd_read_s": 13.75253405154217}
54 +{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --predictor probe --m 8 --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "e0179f6"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:30:45", "prompt_tokens": 14, "gen_tokens": 256, "ttft_ms": 307.55045800469816, "prompt_tps": 45.69629934715006, "gen_tps": 36.686457870160076, "decode_tps_wall": 36.54326812941098, "itl_ms_p50": 27.61320793069899, "itl_ms_p95": 35.31170799396932, "itl_ms_p99": 41.34791693650186, "itl_ms_mean": 27.36482124309987, "total_s": 7.285579874995165, "peak_mem_gb": 1.753644552, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "t_start_epoch": 1793514638.644236, "t_end_epoch": 1793514645.9310138, "t_first_epoch": 1793514638.9517865, "cache_hit_rate": 0.8069263563627869, "cache_misses": 6612, "cache_hits": 27634, "stall_s": 2.490456422092393, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 16192, "installed": 16197, "late": 740, "dup": 15400, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 3.88708005938679, "prefetch": 0.6125109859276563, "ensure": 2.402447963017039, "build": 0.06990473286714405}, "ssd_bytes": 80702078976, "ssd_read_s": 10.351799859432504}
modified benchmarks/raw/gateG/energy_laptop/power.jsonl +257 −0
@@ -85,3 +85,260 @@
85 85 {"t": 1793514382.3597271, "elapsed_ns": 1020930916, "cpu_mW": 33065.9, "gpu_mW": 921.708, "ane_mW": 0.0, "combined_mW": 33987.6, "thermal_pressure": "Nominal"}
86 86 {"t": 1793514383.381633, "elapsed_ns": 1022480000, "cpu_mW": 33172.3, "gpu_mW": 973.124, "ane_mW": 0.0, "combined_mW": 34145.4, "thermal_pressure": "Nominal"}
87 87 {"t": 1793514384.4036071, "elapsed_ns": 1021607958, "cpu_mW": 31626.6, "gpu_mW": 989.616, "ane_mW": 0.0, "combined_mW": 32616.2, "thermal_pressure": "Nominal"}
88 +{"t": 1793514385.4265668, "elapsed_ns": 1021478833, "cpu_mW": 32083.9, "gpu_mW": 1013.24, "ane_mW": 0.0, "combined_mW": 33097.1, "thermal_pressure": "Nominal"}
89 +{"t": 1793514386.4477959, "elapsed_ns": 1022218583, "cpu_mW": 29769.6, "gpu_mW": 969.46, "ane_mW": 0.0, "combined_mW": 30739.0, "thermal_pressure": "Nominal"}
90 +{"t": 1793514388.4907782, "elapsed_ns": 1022043125, "cpu_mW": 24989.2, "gpu_mW": 1.95686, "ane_mW": 0.0, "combined_mW": 24991.1, "thermal_pressure": "Nominal"}
91 +{"t": 1793514388.4927402, "elapsed_ns": 1022142625, "cpu_mW": 22316.8, "gpu_mW": 1948.85, "ane_mW": 0.0, "combined_mW": 24265.7, "thermal_pressure": "Nominal"}
92 +{"t": 1793514389.5131981, "elapsed_ns": 1021520708, "cpu_mW": 22422.5, "gpu_mW": 1148.29, "ane_mW": 0.0, "combined_mW": 23570.7, "thermal_pressure": "Nominal"}
93 +{"t": 1793514390.537643, "elapsed_ns": 1021666750, "cpu_mW": 23418.6, "gpu_mW": 1075.69, "ane_mW": 0.0, "combined_mW": 24494.3, "thermal_pressure": "Nominal"}
94 +{"t": 1793514391.564992, "elapsed_ns": 1022320541, "cpu_mW": 26876.1, "gpu_mW": 1052.51, "ane_mW": 0.0, "combined_mW": 27928.6, "thermal_pressure": "Nominal"}
95 +{"t": 1793514392.58288, "elapsed_ns": 1023332375, "cpu_mW": 20500.7, "gpu_mW": 1114.01, "ane_mW": 10.7492, "combined_mW": 21625.4, "thermal_pressure": "Nominal"}
96 +{"t": 1793514393.605533, "elapsed_ns": 1024985291, "cpu_mW": 17832.5, "gpu_mW": 937.574, "ane_mW": 0.0, "combined_mW": 18770.0, "thermal_pressure": "Nominal"}
97 +{"t": 1793514394.627129, "elapsed_ns": 1021616375, "cpu_mW": 16964.3, "gpu_mW": 1036.59, "ane_mW": 0.978841, "combined_mW": 18001.9, "thermal_pressure": "Nominal"}
98 +{"t": 1793514395.64869, "elapsed_ns": 1022121958, "cpu_mW": 20260.8, "gpu_mW": 1011.62, "ane_mW": 942.158, "combined_mW": 22214.6, "thermal_pressure": "Nominal"}
99 +{"t": 1793514397.691945, "elapsed_ns": 1021705166, "cpu_mW": 20638.0, "gpu_mW": 983.65, "ane_mW": 326.904, "combined_mW": 21948.6, "thermal_pressure": "Nominal"}
100 +{"t": 1793514397.693907, "elapsed_ns": 1021223958, "cpu_mW": 18150.8, "gpu_mW": 865.628, "ane_mW": 74.4205, "combined_mW": 19090.8, "thermal_pressure": "Nominal"}
101 +{"t": 1793514398.7125518, "elapsed_ns": 1021437250, "cpu_mW": 18890.0, "gpu_mW": 1095.52, "ane_mW": 60.6988, "combined_mW": 20046.3, "thermal_pressure": "Nominal"}
102 +{"t": 1793514399.733614, "elapsed_ns": 1021598916, "cpu_mW": 14279.6, "gpu_mW": 1336.14, "ane_mW": 0.978858, "combined_mW": 15616.7, "thermal_pressure": "Nominal"}
103 +{"t": 1793514400.755388, "elapsed_ns": 1021273958, "cpu_mW": 11530.7, "gpu_mW": 1264.11, "ane_mW": 0.979169, "combined_mW": 12795.8, "thermal_pressure": "Nominal"}
104 +{"t": 1793514401.77793, "elapsed_ns": 1021503583, "cpu_mW": 13151.2, "gpu_mW": 1193.34, "ane_mW": 0.0, "combined_mW": 14344.5, "thermal_pressure": "Nominal"}
105 +{"t": 1793514402.7992268, "elapsed_ns": 1021795875, "cpu_mW": 14334.6, "gpu_mW": 1358.39, "ane_mW": 0.978669, "combined_mW": 15693.9, "thermal_pressure": "Nominal"}
106 +{"t": 1793514403.821048, "elapsed_ns": 1021719499, "cpu_mW": 15825.3, "gpu_mW": 1177.43, "ane_mW": 0.978742, "combined_mW": 17003.7, "thermal_pressure": "Nominal"}
107 +{"t": 1793514405.862755, "elapsed_ns": 1021391750, "cpu_mW": 15437.8, "gpu_mW": 1190.53, "ane_mW": 0.0, "combined_mW": 16628.3, "thermal_pressure": "Nominal"}
108 +{"t": 1793514405.864445, "elapsed_ns": 1021392541, "cpu_mW": 17193.2, "gpu_mW": 1105.35, "ane_mW": 0.979056, "combined_mW": 18299.5, "thermal_pressure": "Nominal"}
109 +{"t": 1793514406.884599, "elapsed_ns": 1021353083, "cpu_mW": 17691.2, "gpu_mW": 1013.36, "ane_mW": 0.979093, "combined_mW": 18705.6, "thermal_pressure": "Nominal"}
110 +{"t": 1793514407.907895, "elapsed_ns": 1021093416, "cpu_mW": 12793.1, "gpu_mW": 1262.37, "ane_mW": 0.0, "combined_mW": 14055.5, "thermal_pressure": "Nominal"}
111 +{"t": 1793514408.928425, "elapsed_ns": 1022192500, "cpu_mW": 15325.9, "gpu_mW": 1120.14, "ane_mW": 0.978289, "combined_mW": 16447.0, "thermal_pressure": "Nominal"}
112 +{"t": 1793514409.950219, "elapsed_ns": 1022046958, "cpu_mW": 16094.2, "gpu_mW": 1130.09, "ane_mW": 0.0, "combined_mW": 17224.3, "thermal_pressure": "Nominal"}
113 +{"t": 1793514410.9723232, "elapsed_ns": 1021888666, "cpu_mW": 14387.1, "gpu_mW": 1100.9, "ane_mW": 0.0, "combined_mW": 15488.0, "thermal_pressure": "Nominal"}
114 +{"t": 1793514411.995709, "elapsed_ns": 1021708958, "cpu_mW": 16325.6, "gpu_mW": 1093.27, "ane_mW": 0.0, "combined_mW": 17418.9, "thermal_pressure": "Nominal"}
115 +{"t": 1793514413.016373, "elapsed_ns": 1022038208, "cpu_mW": 21400.4, "gpu_mW": 5.87062, "ane_mW": 0.0, "combined_mW": 21406.2, "thermal_pressure": "Nominal"}
116 +{"t": 1793514415.059744, "elapsed_ns": 1022024208, "cpu_mW": 16490.8, "gpu_mW": 2297.4, "ane_mW": 0.0, "combined_mW": 18788.2, "thermal_pressure": "Nominal"}
117 +{"t": 1793514415.061377, "elapsed_ns": 1021963916, "cpu_mW": 15053.4, "gpu_mW": 835.646, "ane_mW": 0.0, "combined_mW": 15889.0, "thermal_pressure": "Nominal"}
118 +{"t": 1793514416.081883, "elapsed_ns": 1021660875, "cpu_mW": 10158.9, "gpu_mW": 911.261, "ane_mW": 0.0, "combined_mW": 11070.2, "thermal_pressure": "Nominal"}
119 +{"t": 1793514417.102023, "elapsed_ns": 1021549083, "cpu_mW": 13667.5, "gpu_mW": 738.095, "ane_mW": 0.0, "combined_mW": 14405.6, "thermal_pressure": "Nominal"}
120 +{"t": 1793514418.123428, "elapsed_ns": 1021050916, "cpu_mW": 13254.0, "gpu_mW": 760.001, "ane_mW": 0.0, "combined_mW": 14014.0, "thermal_pressure": "Nominal"}
121 +{"t": 1793514419.145301, "elapsed_ns": 1021496166, "cpu_mW": 14147.9, "gpu_mW": 754.775, "ane_mW": 0.0, "combined_mW": 14902.7, "thermal_pressure": "Nominal"}
122 +{"t": 1793514420.166976, "elapsed_ns": 1021468083, "cpu_mW": 15206.5, "gpu_mW": 1065.13, "ane_mW": 0.0, "combined_mW": 16271.7, "thermal_pressure": "Nominal"}
123 +{"t": 1793514422.2113018, "elapsed_ns": 1021702500, "cpu_mW": 15594.6, "gpu_mW": 1156.89, "ane_mW": 0.0, "combined_mW": 16751.5, "thermal_pressure": "Nominal"}
124 +{"t": 1793514422.2133288, "elapsed_ns": 1021771000, "cpu_mW": 16176.8, "gpu_mW": 1088.31, "ane_mW": 0.0, "combined_mW": 17265.1, "thermal_pressure": "Nominal"}
125 +{"t": 1793514423.232272, "elapsed_ns": 1021953249, "cpu_mW": 17117.2, "gpu_mW": 1056.8, "ane_mW": 0.0, "combined_mW": 18174.0, "thermal_pressure": "Nominal"}
126 +{"t": 1793514424.25409, "elapsed_ns": 1021522708, "cpu_mW": 16538.1, "gpu_mW": 1007.32, "ane_mW": 0.0, "combined_mW": 17545.4, "thermal_pressure": "Nominal"}
127 +{"t": 1793514425.275063, "elapsed_ns": 1021337083, "cpu_mW": 15063.6, "gpu_mW": 949.735, "ane_mW": 0.0, "combined_mW": 16013.3, "thermal_pressure": "Nominal"}
128 +{"t": 1793514426.2970428, "elapsed_ns": 1021887916, "cpu_mW": 11892.7, "gpu_mW": 1264.33, "ane_mW": 0.0, "combined_mW": 13157.0, "thermal_pressure": "Nominal"}
129 +{"t": 1793514427.3191779, "elapsed_ns": 1021075500, "cpu_mW": 11608.3, "gpu_mW": 1129.2, "ane_mW": 0.0, "combined_mW": 12737.5, "thermal_pressure": "Nominal"}
130 +{"t": 1793514428.341182, "elapsed_ns": 1022061000, "cpu_mW": 15656.6, "gpu_mW": 1096.8, "ane_mW": 0.0, "combined_mW": 16753.4, "thermal_pressure": "Nominal"}
131 +{"t": 1793514430.3845818, "elapsed_ns": 1022117250, "cpu_mW": 15581.4, "gpu_mW": 1116.31, "ane_mW": 0.0, "combined_mW": 16697.7, "thermal_pressure": "Nominal"}
132 +{"t": 1793514430.386417, "elapsed_ns": 1021989500, "cpu_mW": 15313.3, "gpu_mW": 1125.26, "ane_mW": 0.0, "combined_mW": 16438.5, "thermal_pressure": "Nominal"}
133 +{"t": 1793514431.406562, "elapsed_ns": 1021725541, "cpu_mW": 16490.7, "gpu_mW": 1092.27, "ane_mW": 0.0, "combined_mW": 17583.0, "thermal_pressure": "Nominal"}
134 +{"t": 1793514432.4275572, "elapsed_ns": 1021943125, "cpu_mW": 17140.9, "gpu_mW": 1175.21, "ane_mW": 0.0, "combined_mW": 18316.1, "thermal_pressure": "Nominal"}
135 +{"t": 1793514433.44819, "elapsed_ns": 1020863916, "cpu_mW": 18794.9, "gpu_mW": 1031.48, "ane_mW": 0.0, "combined_mW": 19826.3, "thermal_pressure": "Nominal"}
136 +{"t": 1793514434.470166, "elapsed_ns": 1021013916, "cpu_mW": 17423.9, "gpu_mW": 916.736, "ane_mW": 0.0, "combined_mW": 18340.6, "thermal_pressure": "Nominal"}
137 +{"t": 1793514435.492258, "elapsed_ns": 1021749208, "cpu_mW": 17873.3, "gpu_mW": 998.288, "ane_mW": 0.0, "combined_mW": 18871.6, "thermal_pressure": "Nominal"}
138 +{"t": 1793514436.5140748, "elapsed_ns": 1021631791, "cpu_mW": 20340.0, "gpu_mW": 1009.17, "ane_mW": 0.0, "combined_mW": 21349.2, "thermal_pressure": "Nominal"}
139 +{"t": 1793514437.535533, "elapsed_ns": 1022006916, "cpu_mW": 19875.6, "gpu_mW": 1043.05, "ane_mW": 0.0, "combined_mW": 20918.6, "thermal_pressure": "Nominal"}
140 +{"t": 1793514439.579565, "elapsed_ns": 1021884250, "cpu_mW": 16247.4, "gpu_mW": 1093.08, "ane_mW": 0.0, "combined_mW": 17340.5, "thermal_pressure": "Nominal"}
141 +{"t": 1793514439.581311, "elapsed_ns": 1021793583, "cpu_mW": 17301.9, "gpu_mW": 1060.88, "ane_mW": 0.0, "combined_mW": 18362.8, "thermal_pressure": "Nominal"}
142 +{"t": 1793514440.6016781, "elapsed_ns": 1021849083, "cpu_mW": 16753.9, "gpu_mW": 1061.8, "ane_mW": 0.0, "combined_mW": 17815.7, "thermal_pressure": "Nominal"}
143 +{"t": 1793514441.6226392, "elapsed_ns": 1021883249, "cpu_mW": 15540.9, "gpu_mW": 1078.4, "ane_mW": 0.0, "combined_mW": 16619.3, "thermal_pressure": "Nominal"}
144 +{"t": 1793514442.644178, "elapsed_ns": 1021513249, "cpu_mW": 14733.0, "gpu_mW": 1062.15, "ane_mW": 0.0, "combined_mW": 15795.2, "thermal_pressure": "Nominal"}
145 +{"t": 1793514443.665327, "elapsed_ns": 1021821499, "cpu_mW": 15329.5, "gpu_mW": 984.516, "ane_mW": 0.0, "combined_mW": 16314.0, "thermal_pressure": "Nominal"}
146 +{"t": 1793514444.68732, "elapsed_ns": 1020946833, "cpu_mW": 14259.3, "gpu_mW": 1212.6, "ane_mW": 0.0, "combined_mW": 15471.9, "thermal_pressure": "Nominal"}
147 +{"t": 1793514445.709019, "elapsed_ns": 1021898750, "cpu_mW": 14109.0, "gpu_mW": 1228.11, "ane_mW": 0.0, "combined_mW": 15337.1, "thermal_pressure": "Nominal"}
148 +{"t": 1793514447.7523189, "elapsed_ns": 1021779916, "cpu_mW": 14862.3, "gpu_mW": 1190.08, "ane_mW": 0.0, "combined_mW": 16052.4, "thermal_pressure": "Nominal"}
149 +{"t": 1793514447.754268, "elapsed_ns": 1021237291, "cpu_mW": 13904.7, "gpu_mW": 1178.96, "ane_mW": 0.0, "combined_mW": 15083.7, "thermal_pressure": "Nominal"}
150 +{"t": 1793514448.7734919, "elapsed_ns": 1021718458, "cpu_mW": 13841.4, "gpu_mW": 1134.36, "ane_mW": 0.0, "combined_mW": 14975.7, "thermal_pressure": "Nominal"}
151 +{"t": 1793514449.79485, "elapsed_ns": 1021626958, "cpu_mW": 16962.2, "gpu_mW": 1123.7, "ane_mW": 0.0, "combined_mW": 18085.9, "thermal_pressure": "Nominal"}
152 +{"t": 1793514450.817858, "elapsed_ns": 1021582166, "cpu_mW": 16915.9, "gpu_mW": 1081.66, "ane_mW": 0.0, "combined_mW": 17997.6, "thermal_pressure": "Nominal"}
153 +{"t": 1793514451.8394592, "elapsed_ns": 1021603541, "cpu_mW": 14946.1, "gpu_mW": 932.847, "ane_mW": 0.0, "combined_mW": 15879.0, "thermal_pressure": "Nominal"}
154 +{"t": 1793514452.861504, "elapsed_ns": 1022583500, "cpu_mW": 12901.6, "gpu_mW": 1180.34, "ane_mW": 0.0, "combined_mW": 14082.0, "thermal_pressure": "Nominal"}
155 +{"t": 1793514453.8831072, "elapsed_ns": 1022009000, "cpu_mW": 14385.4, "gpu_mW": 1157.52, "ane_mW": 0.0, "combined_mW": 15542.9, "thermal_pressure": "Nominal"}
156 +{"t": 1793514455.927351, "elapsed_ns": 1021453708, "cpu_mW": 16403.1, "gpu_mW": 1107.25, "ane_mW": 0.0, "combined_mW": 17510.3, "thermal_pressure": "Nominal"}
157 +{"t": 1793514455.929131, "elapsed_ns": 1021713708, "cpu_mW": 22752.9, "gpu_mW": 1042.37, "ane_mW": 0.0, "combined_mW": 23795.3, "thermal_pressure": "Nominal"}
158 +{"t": 1793514456.950163, "elapsed_ns": 1022238875, "cpu_mW": 20376.8, "gpu_mW": 1113.24, "ane_mW": 0.0, "combined_mW": 21490.1, "thermal_pressure": "Nominal"}
159 +{"t": 1793514457.971405, "elapsed_ns": 1022243583, "cpu_mW": 13198.4, "gpu_mW": 1191.5, "ane_mW": 0.0, "combined_mW": 14389.9, "thermal_pressure": "Nominal"}
160 +{"t": 1793514458.993207, "elapsed_ns": 1022400000, "cpu_mW": 12345.5, "gpu_mW": 1033.84, "ane_mW": 0.0, "combined_mW": 13379.3, "thermal_pressure": "Nominal"}
161 +{"t": 1793514460.0145972, "elapsed_ns": 1021578166, "cpu_mW": 15596.5, "gpu_mW": 1049.36, "ane_mW": 0.0, "combined_mW": 16645.8, "thermal_pressure": "Nominal"}
162 +{"t": 1793514461.038023, "elapsed_ns": 1021601208, "cpu_mW": 15055.8, "gpu_mW": 1111.0, "ane_mW": 0.0, "combined_mW": 16166.8, "thermal_pressure": "Nominal"}
163 +{"t": 1793514462.059385, "elapsed_ns": 1021782875, "cpu_mW": 13829.7, "gpu_mW": 1145.06, "ane_mW": 0.0, "combined_mW": 14974.8, "thermal_pressure": "Nominal"}
164 +{"t": 1793514464.1031199, "elapsed_ns": 1022869250, "cpu_mW": 17346.3, "gpu_mW": 1100.82, "ane_mW": 0.0, "combined_mW": 18447.1, "thermal_pressure": "Nominal"}
165 +{"t": 1793514464.104785, "elapsed_ns": 1021913666, "cpu_mW": 16754.8, "gpu_mW": 1151.76, "ane_mW": 0.0, "combined_mW": 17906.6, "thermal_pressure": "Nominal"}
166 +{"t": 1793514465.1260831, "elapsed_ns": 1021606291, "cpu_mW": 12847.4, "gpu_mW": 1182.45, "ane_mW": 0.0, "combined_mW": 14029.9, "thermal_pressure": "Nominal"}
167 +{"t": 1793514466.1469, "elapsed_ns": 1022077458, "cpu_mW": 13046.0, "gpu_mW": 1218.11, "ane_mW": 0.0, "combined_mW": 14264.1, "thermal_pressure": "Nominal"}
168 +{"t": 1793514467.170579, "elapsed_ns": 1022134000, "cpu_mW": 16310.0, "gpu_mW": 1088.9, "ane_mW": 0.0, "combined_mW": 17398.9, "thermal_pressure": "Nominal"}
169 +{"t": 1793514468.19178, "elapsed_ns": 1021845625, "cpu_mW": 16206.0, "gpu_mW": 1111.71, "ane_mW": 0.0, "combined_mW": 17317.7, "thermal_pressure": "Nominal"}
170 +{"t": 1793514469.213527, "elapsed_ns": 1022946916, "cpu_mW": 16437.8, "gpu_mW": 1133.98, "ane_mW": 0.0, "combined_mW": 17571.8, "thermal_pressure": "Nominal"}
171 +{"t": 1793514470.2353349, "elapsed_ns": 1021579958, "cpu_mW": 13671.0, "gpu_mW": 1229.47, "ane_mW": 0.0, "combined_mW": 14900.4, "thermal_pressure": "Nominal"}
172 +{"t": 1793514472.277964, "elapsed_ns": 1021693624, "cpu_mW": 14884.1, "gpu_mW": 1229.33, "ane_mW": 0.0, "combined_mW": 16113.4, "thermal_pressure": "Nominal"}
173 +{"t": 1793514472.279568, "elapsed_ns": 1021562708, "cpu_mW": 14678.5, "gpu_mW": 1228.51, "ane_mW": 0.0, "combined_mW": 15907.0, "thermal_pressure": "Nominal"}
174 +{"t": 1793514473.2998478, "elapsed_ns": 1021710333, "cpu_mW": 13312.0, "gpu_mW": 1293.91, "ane_mW": 0.0, "combined_mW": 14605.9, "thermal_pressure": "Nominal"}
175 +{"t": 1793514474.321727, "elapsed_ns": 1021494416, "cpu_mW": 11549.7, "gpu_mW": 1273.62, "ane_mW": 0.0, "combined_mW": 12823.4, "thermal_pressure": "Nominal"}
176 +{"t": 1793514475.344286, "elapsed_ns": 1021627666, "cpu_mW": 16016.6, "gpu_mW": 1178.51, "ane_mW": 0.0, "combined_mW": 17195.1, "thermal_pressure": "Nominal"}
177 +{"t": 1793514476.3647418, "elapsed_ns": 1022014208, "cpu_mW": 13622.1, "gpu_mW": 1021.51, "ane_mW": 0.0, "combined_mW": 14643.6, "thermal_pressure": "Nominal"}
178 +{"t": 1793514477.387168, "elapsed_ns": 1021841708, "cpu_mW": 12423.6, "gpu_mW": 1263.41, "ane_mW": 0.0, "combined_mW": 13687.1, "thermal_pressure": "Nominal"}
179 +{"t": 1793514478.408648, "elapsed_ns": 1021330000, "cpu_mW": 13194.6, "gpu_mW": 1225.85, "ane_mW": 0.0, "combined_mW": 14420.4, "thermal_pressure": "Nominal"}
180 +{"t": 1793514480.4519072, "elapsed_ns": 1021926666, "cpu_mW": 17705.8, "gpu_mW": 1127.28, "ane_mW": 0.0, "combined_mW": 18833.1, "thermal_pressure": "Nominal"}
181 +{"t": 1793514480.453556, "elapsed_ns": 1021187875, "cpu_mW": 14033.7, "gpu_mW": 1217.21, "ane_mW": 0.0, "combined_mW": 15250.9, "thermal_pressure": "Nominal"}
182 +{"t": 1793514481.4732, "elapsed_ns": 1021968833, "cpu_mW": 11436.7, "gpu_mW": 1264.23, "ane_mW": 0.0, "combined_mW": 12701.0, "thermal_pressure": "Nominal"}
183 +{"t": 1793514482.494974, "elapsed_ns": 1021597291, "cpu_mW": 12738.9, "gpu_mW": 1126.67, "ane_mW": 0.0, "combined_mW": 13865.5, "thermal_pressure": "Nominal"}
184 +{"t": 1793514483.515433, "elapsed_ns": 1021494083, "cpu_mW": 11437.2, "gpu_mW": 1239.36, "ane_mW": 0.0, "combined_mW": 12676.5, "thermal_pressure": "Nominal"}
185 +{"t": 1793514484.536913, "elapsed_ns": 1020892958, "cpu_mW": 7864.68, "gpu_mW": 594.578, "ane_mW": 0.0, "combined_mW": 8459.26, "thermal_pressure": "Nominal"}
186 +{"t": 1793514485.557188, "elapsed_ns": 1021537041, "cpu_mW": 5633.67, "gpu_mW": 57.7561, "ane_mW": 0.0, "combined_mW": 5691.42, "thermal_pressure": "Nominal"}
187 +{"t": 1793514487.6002078, "elapsed_ns": 1020647625, "cpu_mW": 4620.6, "gpu_mW": 51.9278, "ane_mW": 0.0, "combined_mW": 4672.52, "thermal_pressure": "Nominal"}
188 +{"t": 1793514487.601894, "elapsed_ns": 1021108541, "cpu_mW": 8648.44, "gpu_mW": 47.9871, "ane_mW": 0.0, "combined_mW": 8696.43, "thermal_pressure": "Nominal"}
189 +{"t": 1793514488.628555, "elapsed_ns": 1021436208, "cpu_mW": 5524.57, "gpu_mW": 42.0976, "ane_mW": 0.0, "combined_mW": 5566.67, "thermal_pressure": "Nominal"}
190 +{"t": 1793514489.644935, "elapsed_ns": 1022073458, "cpu_mW": 8911.3, "gpu_mW": 45.985, "ane_mW": 0.0, "combined_mW": 8957.28, "thermal_pressure": "Nominal"}
191 +{"t": 1793514490.6656032, "elapsed_ns": 1021647666, "cpu_mW": 5522.45, "gpu_mW": 51.877, "ane_mW": 0.0, "combined_mW": 5574.33, "thermal_pressure": "Nominal"}
192 +{"t": 1793514491.687382, "elapsed_ns": 1021523958, "cpu_mW": 5109.03, "gpu_mW": 48.9465, "ane_mW": 0.0, "combined_mW": 5157.98, "thermal_pressure": "Nominal"}
193 +{"t": 1793514492.7088158, "elapsed_ns": 1021930875, "cpu_mW": 6269.5, "gpu_mW": 41.0987, "ane_mW": 0.0, "combined_mW": 6310.6, "thermal_pressure": "Nominal"}
194 +{"t": 1793514493.7309928, "elapsed_ns": 1021636666, "cpu_mW": 9869.46, "gpu_mW": 56.7717, "ane_mW": 0.0, "combined_mW": 9926.23, "thermal_pressure": "Nominal"}
195 +{"t": 1793514495.773961, "elapsed_ns": 1021476875, "cpu_mW": 13784.9, "gpu_mW": 49.9277, "ane_mW": 0.0, "combined_mW": 13834.9, "thermal_pressure": "Nominal"}
196 +{"t": 1793514495.7756412, "elapsed_ns": 1022070541, "cpu_mW": 14115.5, "gpu_mW": 64.5748, "ane_mW": 0.0, "combined_mW": 14180.0, "thermal_pressure": "Nominal"}
197 +{"t": 1793514496.795663, "elapsed_ns": 1021417375, "cpu_mW": 14237.1, "gpu_mW": 58.7419, "ane_mW": 0.0, "combined_mW": 14295.8, "thermal_pressure": "Nominal"}
198 +{"t": 1793514497.817284, "elapsed_ns": 1021693500, "cpu_mW": 13863.3, "gpu_mW": 56.7685, "ane_mW": 0.0, "combined_mW": 13920.0, "thermal_pressure": "Nominal"}
199 +{"t": 1793514498.8393018, "elapsed_ns": 1021698541, "cpu_mW": 13827.0, "gpu_mW": 43.0655, "ane_mW": 0.0, "combined_mW": 13870.0, "thermal_pressure": "Nominal"}
200 +{"t": 1793514499.861644, "elapsed_ns": 1021565458, "cpu_mW": 19521.0, "gpu_mW": 56.7756, "ane_mW": 0.0, "combined_mW": 19577.8, "thermal_pressure": "Nominal"}
201 +{"t": 1793514500.88336, "elapsed_ns": 1022008291, "cpu_mW": 19885.4, "gpu_mW": 69.4711, "ane_mW": 0.0, "combined_mW": 19954.8, "thermal_pressure": "Nominal"}
202 +{"t": 1793514501.9054248, "elapsed_ns": 1022264291, "cpu_mW": 19776.7, "gpu_mW": 58.6932, "ane_mW": 0.0, "combined_mW": 19835.4, "thermal_pressure": "Nominal"}
203 +{"t": 1793514503.94885, "elapsed_ns": 1021672541, "cpu_mW": 19442.6, "gpu_mW": 68.5151, "ane_mW": 0.0, "combined_mW": 19511.1, "thermal_pressure": "Nominal"}
204 +{"t": 1793514503.9505222, "elapsed_ns": 1022086083, "cpu_mW": 18984.7, "gpu_mW": 62.617, "ane_mW": 0.0, "combined_mW": 19047.3, "thermal_pressure": "Nominal"}
205 +{"t": 1793514504.970506, "elapsed_ns": 1021910041, "cpu_mW": 18686.6, "gpu_mW": 54.7993, "ane_mW": 0.0, "combined_mW": 18741.4, "thermal_pressure": "Nominal"}
206 +{"t": 1793514505.992467, "elapsed_ns": 1021381416, "cpu_mW": 18623.8, "gpu_mW": 62.6602, "ane_mW": 0.0, "combined_mW": 18686.5, "thermal_pressure": "Nominal"}
207 +{"t": 1793514507.014581, "elapsed_ns": 1021875249, "cpu_mW": 19227.4, "gpu_mW": 60.6728, "ane_mW": 0.0, "combined_mW": 19288.1, "thermal_pressure": "Nominal"}
208 +{"t": 1793514508.03702, "elapsed_ns": 1021972249, "cpu_mW": 18336.1, "gpu_mW": 62.624, "ane_mW": 0.0, "combined_mW": 18398.7, "thermal_pressure": "Nominal"}
209 +{"t": 1793514509.058808, "elapsed_ns": 1022147291, "cpu_mW": 18492.4, "gpu_mW": 52.83, "ane_mW": 0.0, "combined_mW": 18545.3, "thermal_pressure": "Nominal"}
210 +{"t": 1793514510.0810182, "elapsed_ns": 1022380666, "cpu_mW": 18438.3, "gpu_mW": 68.4676, "ane_mW": 0.0, "combined_mW": 18506.8, "thermal_pressure": "Nominal"}
211 +{"t": 1793514512.123785, "elapsed_ns": 1021721874, "cpu_mW": 19911.5, "gpu_mW": 66.5543, "ane_mW": 0.0, "combined_mW": 19978.0, "thermal_pressure": "Nominal"}
212 +{"t": 1793514512.1259232, "elapsed_ns": 1022148541, "cpu_mW": 21309.0, "gpu_mW": 76.3098, "ane_mW": 0.0, "combined_mW": 21385.3, "thermal_pressure": "Nominal"}
213 +{"t": 1793514513.14463, "elapsed_ns": 1020759374, "cpu_mW": 22892.8, "gpu_mW": 70.5357, "ane_mW": 0.0, "combined_mW": 22963.3, "thermal_pressure": "Nominal"}
214 +{"t": 1793514514.166074, "elapsed_ns": 1020783583, "cpu_mW": 25540.2, "gpu_mW": 96.9843, "ane_mW": 0.0, "combined_mW": 25637.2, "thermal_pressure": "Nominal"}
215 +{"t": 1793514515.187936, "elapsed_ns": 1021559666, "cpu_mW": 26020.0, "gpu_mW": 75.3749, "ane_mW": 0.0, "combined_mW": 26095.4, "thermal_pressure": "Nominal"}
216 +{"t": 1793514516.20992, "elapsed_ns": 1021531708, "cpu_mW": 25808.3, "gpu_mW": 89.0819, "ane_mW": 0.0, "combined_mW": 25897.4, "thermal_pressure": "Nominal"}
217 +{"t": 1793514517.232925, "elapsed_ns": 1022469333, "cpu_mW": 26328.4, "gpu_mW": 89.9782, "ane_mW": 0.0, "combined_mW": 26418.4, "thermal_pressure": "Nominal"}
218 +{"t": 1793514518.254474, "elapsed_ns": 1021788375, "cpu_mW": 26758.0, "gpu_mW": 82.2088, "ane_mW": 0.0, "combined_mW": 26840.2, "thermal_pressure": "Nominal"}
219 +{"t": 1793514520.297753, "elapsed_ns": 1022129750, "cpu_mW": 27162.9, "gpu_mW": 90.0081, "ane_mW": 0.0, "combined_mW": 27252.9, "thermal_pressure": "Nominal"}
220 +{"t": 1793514520.299909, "elapsed_ns": 1021922000, "cpu_mW": 25562.6, "gpu_mW": 83.1766, "ane_mW": 0.0, "combined_mW": 25645.8, "thermal_pressure": "Nominal"}
221 +{"t": 1793514521.318976, "elapsed_ns": 1020875500, "cpu_mW": 25476.2, "gpu_mW": 82.2823, "ane_mW": 0.0, "combined_mW": 25558.5, "thermal_pressure": "Nominal"}
222 +{"t": 1793514522.340845, "elapsed_ns": 1021650583, "cpu_mW": 25642.8, "gpu_mW": 82.2199, "ane_mW": 0.0, "combined_mW": 25725.0, "thermal_pressure": "Nominal"}
223 +{"t": 1793514523.362674, "elapsed_ns": 1021897833, "cpu_mW": 25603.3, "gpu_mW": 85.1357, "ane_mW": 0.0, "combined_mW": 25688.5, "thermal_pressure": "Nominal"}
224 +{"t": 1793514524.3848588, "elapsed_ns": 1022047666, "cpu_mW": 26711.1, "gpu_mW": 76.3174, "ane_mW": 0.0, "combined_mW": 26787.4, "thermal_pressure": "Nominal"}
225 +{"t": 1793514525.4065568, "elapsed_ns": 1022037625, "cpu_mW": 26660.5, "gpu_mW": 76.3181, "ane_mW": 0.0, "combined_mW": 26736.8, "thermal_pressure": "Nominal"}
226 +{"t": 1793514526.429485, "elapsed_ns": 1021562041, "cpu_mW": 28996.8, "gpu_mW": 124.319, "ane_mW": 0.0, "combined_mW": 29121.1, "thermal_pressure": "Nominal"}
227 +{"t": 1793514527.450005, "elapsed_ns": 1022163208, "cpu_mW": 27119.9, "gpu_mW": 216.208, "ane_mW": 0.0, "combined_mW": 27336.1, "thermal_pressure": "Nominal"}
228 +{"t": 1793514529.493999, "elapsed_ns": 1021304166, "cpu_mW": 25635.8, "gpu_mW": 96.9349, "ane_mW": 0.0, "combined_mW": 25732.8, "thermal_pressure": "Nominal"}
229 +{"t": 1793514529.495698, "elapsed_ns": 1022108083, "cpu_mW": 25182.3, "gpu_mW": 96.8586, "ane_mW": 0.0, "combined_mW": 25279.1, "thermal_pressure": "Nominal"}
230 +{"t": 1793514530.516442, "elapsed_ns": 1022056333, "cpu_mW": 24693.4, "gpu_mW": 81.2088, "ane_mW": 0.0, "combined_mW": 24774.6, "thermal_pressure": "Nominal"}
231 +{"t": 1793514531.5377269, "elapsed_ns": 1022262875, "cpu_mW": 25314.4, "gpu_mW": 85.1053, "ane_mW": 0.0, "combined_mW": 25399.5, "thermal_pressure": "Nominal"}
232 +{"t": 1793514532.559656, "elapsed_ns": 1021201666, "cpu_mW": 25962.6, "gpu_mW": 78.3391, "ane_mW": 0.0, "combined_mW": 26040.9, "thermal_pressure": "Nominal"}
233 +{"t": 1793514533.581193, "elapsed_ns": 1021733333, "cpu_mW": 26784.9, "gpu_mW": 81.2345, "ane_mW": 0.0, "combined_mW": 26866.1, "thermal_pressure": "Nominal"}
234 +{"t": 1793514534.604172, "elapsed_ns": 1021031625, "cpu_mW": 26311.6, "gpu_mW": 96.9608, "ane_mW": 0.0, "combined_mW": 26408.6, "thermal_pressure": "Nominal"}
235 +{"t": 1793514535.625111, "elapsed_ns": 1022633458, "cpu_mW": 25383.5, "gpu_mW": 94.8531, "ane_mW": 0.0, "combined_mW": 25478.3, "thermal_pressure": "Nominal"}
236 +{"t": 1793514537.667336, "elapsed_ns": 1022133458, "cpu_mW": 25844.0, "gpu_mW": 89.0295, "ane_mW": 0.0, "combined_mW": 25933.0, "thermal_pressure": "Nominal"}
237 +{"t": 1793514537.669191, "elapsed_ns": 1020990416, "cpu_mW": 25004.2, "gpu_mW": 95.9852, "ane_mW": 0.0, "combined_mW": 25100.1, "thermal_pressure": "Nominal"}
238 +{"t": 1793514538.69014, "elapsed_ns": 1021334083, "cpu_mW": 21614.9, "gpu_mW": 60.7049, "ane_mW": 0.0, "combined_mW": 21675.6, "thermal_pressure": "Nominal"}
239 +{"t": 1793514539.711041, "elapsed_ns": 1022098916, "cpu_mW": 22790.4, "gpu_mW": 78.2703, "ane_mW": 0.0, "combined_mW": 22868.6, "thermal_pressure": "Nominal"}
240 +{"t": 1793514540.734891, "elapsed_ns": 1021523374, "cpu_mW": 22407.7, "gpu_mW": 93.9773, "ane_mW": 0.0, "combined_mW": 22501.7, "thermal_pressure": "Nominal"}
241 +{"t": 1793514541.75415, "elapsed_ns": 1022498833, "cpu_mW": 22451.9, "gpu_mW": 79.2177, "ane_mW": 0.0, "combined_mW": 22531.1, "thermal_pressure": "Nominal"}
242 +{"t": 1793514542.7761018, "elapsed_ns": 1021429083, "cpu_mW": 22847.4, "gpu_mW": 93.986, "ane_mW": 0.0, "combined_mW": 22941.4, "thermal_pressure": "Nominal"}
243 +{"t": 1793514543.798282, "elapsed_ns": 1021647958, "cpu_mW": 22011.5, "gpu_mW": 82.2201, "ane_mW": 0.0, "combined_mW": 22093.7, "thermal_pressure": "Nominal"}
244 +{"t": 1793514544.820381, "elapsed_ns": 1022020166, "cpu_mW": 23311.7, "gpu_mW": 96.867, "ane_mW": 0.0, "combined_mW": 23408.5, "thermal_pressure": "Nominal"}
245 +{"t": 1793514546.863437, "elapsed_ns": 1022159541, "cpu_mW": 21955.5, "gpu_mW": 85.1139, "ane_mW": 0.0, "combined_mW": 22040.6, "thermal_pressure": "Nominal"}
246 +{"t": 1793514546.865845, "elapsed_ns": 1021968875, "cpu_mW": 22660.2, "gpu_mW": 80.2373, "ane_mW": 0.0, "combined_mW": 22740.4, "thermal_pressure": "Nominal"}
247 +{"t": 1793514547.8850272, "elapsed_ns": 1021076041, "cpu_mW": 22223.6, "gpu_mW": 84.2249, "ane_mW": 0.0, "combined_mW": 22307.8, "thermal_pressure": "Nominal"}
248 +{"t": 1793514548.9071202, "elapsed_ns": 1021508083, "cpu_mW": 23728.6, "gpu_mW": 78.3156, "ane_mW": 0.0, "combined_mW": 23807.0, "thermal_pressure": "Nominal"}
249 +{"t": 1793514549.9292119, "elapsed_ns": 1022199083, "cpu_mW": 22779.3, "gpu_mW": 81.1975, "ane_mW": 0.0, "combined_mW": 22860.5, "thermal_pressure": "Nominal"}
250 +{"t": 1793514550.950944, "elapsed_ns": 1021774541, "cpu_mW": 21533.1, "gpu_mW": 73.4017, "ane_mW": 0.0, "combined_mW": 21606.5, "thermal_pressure": "Nominal"}
251 +{"t": 1793514551.971725, "elapsed_ns": 1021894958, "cpu_mW": 23978.0, "gpu_mW": 75.3502, "ane_mW": 0.0, "combined_mW": 24053.4, "thermal_pressure": "Nominal"}
252 +{"t": 1793514552.993418, "elapsed_ns": 1021023375, "cpu_mW": 21493.1, "gpu_mW": 88.1469, "ane_mW": 0.0, "combined_mW": 21581.3, "thermal_pressure": "Nominal"}
253 +{"t": 1793514555.037471, "elapsed_ns": 1020970624, "cpu_mW": 22527.6, "gpu_mW": 69.5417, "ane_mW": 0.0, "combined_mW": 22597.1, "thermal_pressure": "Nominal"}
254 +{"t": 1793514555.03964, "elapsed_ns": 1022603208, "cpu_mW": 22440.8, "gpu_mW": 65.5191, "ane_mW": 0.0, "combined_mW": 22506.3, "thermal_pressure": "Nominal"}
255 +{"t": 1793514556.058659, "elapsed_ns": 1021719166, "cpu_mW": 23632.7, "gpu_mW": 74.3844, "ane_mW": 0.0, "combined_mW": 23707.1, "thermal_pressure": "Nominal"}
256 +{"t": 1793514557.0797818, "elapsed_ns": 1021654958, "cpu_mW": 22088.7, "gpu_mW": 69.4951, "ane_mW": 0.0, "combined_mW": 22158.2, "thermal_pressure": "Nominal"}
257 +{"t": 1793514558.10219, "elapsed_ns": 1021193541, "cpu_mW": 23452.0, "gpu_mW": 83.2359, "ane_mW": 0.0, "combined_mW": 23535.2, "thermal_pressure": "Nominal"}
258 +{"t": 1793514559.122941, "elapsed_ns": 1021772874, "cpu_mW": 22866.1, "gpu_mW": 79.274, "ane_mW": 0.0, "combined_mW": 22945.4, "thermal_pressure": "Nominal"}
259 +{"t": 1793514560.144566, "elapsed_ns": 1020782083, "cpu_mW": 22559.2, "gpu_mW": 85.2288, "ane_mW": 0.0, "combined_mW": 22644.4, "thermal_pressure": "Nominal"}
260 +{"t": 1793514561.166204, "elapsed_ns": 1021794249, "cpu_mW": 23635.9, "gpu_mW": 90.0377, "ane_mW": 0.0, "combined_mW": 23725.9, "thermal_pressure": "Nominal"}
261 +{"t": 1793514563.209589, "elapsed_ns": 1022047541, "cpu_mW": 25811.9, "gpu_mW": 94.9075, "ane_mW": 0.0, "combined_mW": 25906.8, "thermal_pressure": "Nominal"}
262 +{"t": 1793514563.2113411, "elapsed_ns": 1021205916, "cpu_mW": 22840.6, "gpu_mW": 78.3388, "ane_mW": 0.0, "combined_mW": 22919.0, "thermal_pressure": "Nominal"}
263 +{"t": 1793514564.230853, "elapsed_ns": 1021970541, "cpu_mW": 23941.0, "gpu_mW": 91.9792, "ane_mW": 0.0, "combined_mW": 24033.0, "thermal_pressure": "Nominal"}
264 +{"t": 1793514565.252658, "elapsed_ns": 1020941416, "cpu_mW": 22888.7, "gpu_mW": 76.4001, "ane_mW": 0.0, "combined_mW": 22965.1, "thermal_pressure": "Nominal"}
265 +{"t": 1793514566.273998, "elapsed_ns": 1021302541, "cpu_mW": 22260.8, "gpu_mW": 95.9559, "ane_mW": 0.0, "combined_mW": 22356.7, "thermal_pressure": "Nominal"}
266 +{"t": 1793514567.295889, "elapsed_ns": 1022173416, "cpu_mW": 23539.1, "gpu_mW": 98.8091, "ane_mW": 0.0, "combined_mW": 23637.9, "thermal_pressure": "Nominal"}
267 +{"t": 1793514568.319764, "elapsed_ns": 1022106166, "cpu_mW": 21662.1, "gpu_mW": 92.9453, "ane_mW": 0.0, "combined_mW": 21755.1, "thermal_pressure": "Nominal"}
268 +{"t": 1793514569.3413289, "elapsed_ns": 1021949708, "cpu_mW": 23942.5, "gpu_mW": 78.2817, "ane_mW": 0.0, "combined_mW": 24020.8, "thermal_pressure": "Nominal"}
269 +{"t": 1793514570.363188, "elapsed_ns": 1023308958, "cpu_mW": 19676.4, "gpu_mW": 88.9272, "ane_mW": 0.0, "combined_mW": 19765.3, "thermal_pressure": "Nominal"}
270 +{"t": 1793514572.405995, "elapsed_ns": 1021976416, "cpu_mW": 22405.6, "gpu_mW": 91.9786, "ane_mW": 0.0, "combined_mW": 22497.6, "thermal_pressure": "Nominal"}
271 +{"t": 1793514572.4076219, "elapsed_ns": 1020872708, "cpu_mW": 22462.2, "gpu_mW": 80.3234, "ane_mW": 0.0, "combined_mW": 22542.5, "thermal_pressure": "Nominal"}
272 +{"t": 1793514573.4276311, "elapsed_ns": 1022055208, "cpu_mW": 23252.2, "gpu_mW": 89.0363, "ane_mW": 0.0, "combined_mW": 23341.2, "thermal_pressure": "Nominal"}
273 +{"t": 1793514574.449006, "elapsed_ns": 1021025458, "cpu_mW": 23627.2, "gpu_mW": 84.229, "ane_mW": 0.0, "combined_mW": 23711.5, "thermal_pressure": "Nominal"}
274 +{"t": 1793514575.471272, "elapsed_ns": 1021720250, "cpu_mW": 21855.3, "gpu_mW": 92.0017, "ane_mW": 0.0, "combined_mW": 21947.3, "thermal_pressure": "Nominal"}
275 +{"t": 1793514576.493875, "elapsed_ns": 1022116208, "cpu_mW": 22630.5, "gpu_mW": 73.3772, "ane_mW": 0.0, "combined_mW": 22703.9, "thermal_pressure": "Nominal"}
276 +{"t": 1793514577.514996, "elapsed_ns": 1022019250, "cpu_mW": 24012.3, "gpu_mW": 87.0825, "ane_mW": 0.0, "combined_mW": 24099.4, "thermal_pressure": "Nominal"}
277 +{"t": 1793514578.5362408, "elapsed_ns": 1021976124, "cpu_mW": 28597.5, "gpu_mW": 82.1937, "ane_mW": 0.0, "combined_mW": 28679.7, "thermal_pressure": "Nominal"}
278 +{"t": 1793514579.557139, "elapsed_ns": 1021521458, "cpu_mW": 27223.1, "gpu_mW": 96.9143, "ane_mW": 0.0, "combined_mW": 27320.0, "thermal_pressure": "Nominal"}
279 +{"t": 1793514581.5996008, "elapsed_ns": 1021027625, "cpu_mW": 29088.3, "gpu_mW": 256.604, "ane_mW": 0.0, "combined_mW": 29344.9, "thermal_pressure": "Nominal"}
280 +{"t": 1793514581.603251, "elapsed_ns": 1020728166, "cpu_mW": 26106.9, "gpu_mW": 471.232, "ane_mW": 0.0, "combined_mW": 26578.1, "thermal_pressure": "Nominal"}
281 +{"t": 1793514582.622281, "elapsed_ns": 1021810750, "cpu_mW": 26676.2, "gpu_mW": 311.212, "ane_mW": 0.0, "combined_mW": 26987.4, "thermal_pressure": "Nominal"}
282 +{"t": 1793514583.6432698, "elapsed_ns": 1020786958, "cpu_mW": 24951.3, "gpu_mW": 154.783, "ane_mW": 0.0, "combined_mW": 25106.1, "thermal_pressure": "Nominal"}
283 +{"t": 1793514584.665556, "elapsed_ns": 1021888458, "cpu_mW": 29507.1, "gpu_mW": 654.67, "ane_mW": 0.0, "combined_mW": 30161.8, "thermal_pressure": "Nominal"}
284 +{"t": 1793514585.688908, "elapsed_ns": 1021924916, "cpu_mW": 24335.4, "gpu_mW": 7978.08, "ane_mW": 0.0, "combined_mW": 32313.5, "thermal_pressure": "Nominal"}
285 +{"t": 1793514586.709344, "elapsed_ns": 1021219041, "cpu_mW": 25011.3, "gpu_mW": 394.626, "ane_mW": 0.0, "combined_mW": 25405.9, "thermal_pressure": "Nominal"}
286 +{"t": 1793514587.731756, "elapsed_ns": 1023015250, "cpu_mW": 24907.7, "gpu_mW": 101.66, "ane_mW": 0.0, "combined_mW": 25009.4, "thermal_pressure": "Nominal"}
287 +{"t": 1793514588.753751, "elapsed_ns": 1021932499, "cpu_mW": 29129.1, "gpu_mW": 2399.38, "ane_mW": 0.0, "combined_mW": 31528.5, "thermal_pressure": "Nominal"}
288 +{"t": 1793514590.7995431, "elapsed_ns": 1022791000, "cpu_mW": 28231.6, "gpu_mW": 3005.5, "ane_mW": 0.0, "combined_mW": 31237.1, "thermal_pressure": "Nominal"}
289 +{"t": 1793514590.801867, "elapsed_ns": 1021153583, "cpu_mW": 27854.8, "gpu_mW": 697.251, "ane_mW": 0.0, "combined_mW": 28552.0, "thermal_pressure": "Nominal"}
290 +{"t": 1793514591.819339, "elapsed_ns": 1022362333, "cpu_mW": 26885.8, "gpu_mW": 1284.28, "ane_mW": 0.0, "combined_mW": 28170.1, "thermal_pressure": "Nominal"}
291 +{"t": 1793514592.841732, "elapsed_ns": 1021977666, "cpu_mW": 25647.3, "gpu_mW": 591.011, "ane_mW": 0.0, "combined_mW": 26238.3, "thermal_pressure": "Nominal"}
292 +{"t": 1793514593.864361, "elapsed_ns": 1021584708, "cpu_mW": 26843.6, "gpu_mW": 805.611, "ane_mW": 0.0, "combined_mW": 27649.2, "thermal_pressure": "Nominal"}
293 +{"t": 1793514594.885059, "elapsed_ns": 1022235833, "cpu_mW": 28981.6, "gpu_mW": 402.06, "ane_mW": 0.0, "combined_mW": 29383.6, "thermal_pressure": "Nominal"}
294 +{"t": 1793514595.908468, "elapsed_ns": 1022254791, "cpu_mW": 25501.5, "gpu_mW": 466.616, "ane_mW": 0.0, "combined_mW": 25968.1, "thermal_pressure": "Nominal"}
295 +{"t": 1793514596.929143, "elapsed_ns": 1021760666, "cpu_mW": 25533.4, "gpu_mW": 1393.67, "ane_mW": 0.0, "combined_mW": 26927.0, "thermal_pressure": "Nominal"}
296 +{"t": 1793514598.973845, "elapsed_ns": 1022274375, "cpu_mW": 26681.7, "gpu_mW": 4379.45, "ane_mW": 0.0, "combined_mW": 31061.1, "thermal_pressure": "Nominal"}
297 +{"t": 1793514598.9761338, "elapsed_ns": 1021519666, "cpu_mW": 27560.9, "gpu_mW": 581.487, "ane_mW": 0.0, "combined_mW": 28142.4, "thermal_pressure": "Nominal"}
298 +{"t": 1793514599.996589, "elapsed_ns": 1022049916, "cpu_mW": 26450.8, "gpu_mW": 2766.01, "ane_mW": 0.0, "combined_mW": 29216.8, "thermal_pressure": "Nominal"}
299 +{"t": 1793514601.017697, "elapsed_ns": 1021806541, "cpu_mW": 23362.5, "gpu_mW": 2889.0, "ane_mW": 0.0, "combined_mW": 26251.5, "thermal_pressure": "Nominal"}
300 +{"t": 1793514602.038503, "elapsed_ns": 1021938250, "cpu_mW": 26732.5, "gpu_mW": 6007.21, "ane_mW": 0.0, "combined_mW": 32739.7, "thermal_pressure": "Nominal"}
301 +{"t": 1793514603.062836, "elapsed_ns": 1022131750, "cpu_mW": 21267.3, "gpu_mW": 4849.67, "ane_mW": 0.0, "combined_mW": 26117.0, "thermal_pressure": "Nominal"}
302 +{"t": 1793514604.084095, "elapsed_ns": 1020771708, "cpu_mW": 23382.3, "gpu_mW": 5966.07, "ane_mW": 0.0, "combined_mW": 29348.4, "thermal_pressure": "Nominal"}
303 +{"t": 1793514605.108299, "elapsed_ns": 1024470583, "cpu_mW": 19908.8, "gpu_mW": 1785.31, "ane_mW": 0.0, "combined_mW": 21694.1, "thermal_pressure": "Nominal"}
304 +{"t": 1793514606.130435, "elapsed_ns": 1022076166, "cpu_mW": 21146.2, "gpu_mW": 3732.6, "ane_mW": 0.0, "combined_mW": 24878.8, "thermal_pressure": "Nominal"}
305 +{"t": 1793514608.172689, "elapsed_ns": 1023282291, "cpu_mW": 23688.5, "gpu_mW": 9962.06, "ane_mW": 0.0, "combined_mW": 33650.5, "thermal_pressure": "Nominal"}
306 +{"t": 1793514608.174447, "elapsed_ns": 1022077375, "cpu_mW": 23678.2, "gpu_mW": 9913.14, "ane_mW": 0.0, "combined_mW": 33591.4, "thermal_pressure": "Nominal"}
307 +{"t": 1793514609.1942122, "elapsed_ns": 1022102583, "cpu_mW": 16287.0, "gpu_mW": 7.827, "ane_mW": 0.0, "combined_mW": 16294.8, "thermal_pressure": "Nominal"}
308 +{"t": 1793514610.2164102, "elapsed_ns": 1021012166, "cpu_mW": 14079.2, "gpu_mW": 3377.04, "ane_mW": 0.0, "combined_mW": 17456.2, "thermal_pressure": "Nominal"}
309 +{"t": 1793514611.238346, "elapsed_ns": 1021651958, "cpu_mW": 15655.0, "gpu_mW": 1541.62, "ane_mW": 0.0, "combined_mW": 17196.7, "thermal_pressure": "Nominal"}
310 +{"t": 1793514612.259666, "elapsed_ns": 1022117541, "cpu_mW": 15438.5, "gpu_mW": 1417.65, "ane_mW": 0.0, "combined_mW": 16856.2, "thermal_pressure": "Nominal"}
311 +{"t": 1793514613.281673, "elapsed_ns": 1021662666, "cpu_mW": 15223.2, "gpu_mW": 1414.36, "ane_mW": 0.0, "combined_mW": 16637.6, "thermal_pressure": "Nominal"}
312 +{"t": 1793514614.304944, "elapsed_ns": 1021745249, "cpu_mW": 15284.6, "gpu_mW": 1417.18, "ane_mW": 0.0, "combined_mW": 16701.8, "thermal_pressure": "Nominal"}
313 +{"t": 1793514616.347766, "elapsed_ns": 1022016083, "cpu_mW": 15099.6, "gpu_mW": 1411.92, "ane_mW": 0.0, "combined_mW": 16511.5, "thermal_pressure": "Nominal"}
314 +{"t": 1793514616.3495219, "elapsed_ns": 1022353708, "cpu_mW": 17499.8, "gpu_mW": 1395.8, "ane_mW": 0.0, "combined_mW": 18895.6, "thermal_pressure": "Nominal"}
315 +{"t": 1793514617.369737, "elapsed_ns": 1021157625, "cpu_mW": 18766.9, "gpu_mW": 1329.86, "ane_mW": 0.0, "combined_mW": 20096.8, "thermal_pressure": "Nominal"}
316 +{"t": 1793514618.391596, "elapsed_ns": 1022621416, "cpu_mW": 14450.1, "gpu_mW": 1391.52, "ane_mW": 0.0, "combined_mW": 15841.6, "thermal_pressure": "Nominal"}
317 +{"t": 1793514619.413666, "elapsed_ns": 1021993666, "cpu_mW": 15231.0, "gpu_mW": 1398.25, "ane_mW": 0.0, "combined_mW": 16629.3, "thermal_pressure": "Nominal"}
318 +{"t": 1793514620.435717, "elapsed_ns": 1021807500, "cpu_mW": 13641.5, "gpu_mW": 1400.46, "ane_mW": 0.0, "combined_mW": 15042.0, "thermal_pressure": "Nominal"}
319 +{"t": 1793514621.457668, "elapsed_ns": 1022103500, "cpu_mW": 17531.5, "gpu_mW": 1374.62, "ane_mW": 0.0, "combined_mW": 18906.1, "thermal_pressure": "Nominal"}
320 +{"t": 1793514622.479394, "elapsed_ns": 1021577833, "cpu_mW": 16720.2, "gpu_mW": 1716.95, "ane_mW": 0.0, "combined_mW": 18437.2, "thermal_pressure": "Nominal"}
321 +{"t": 1793514624.523325, "elapsed_ns": 1022127541, "cpu_mW": 19708.9, "gpu_mW": 1397.09, "ane_mW": 0.0, "combined_mW": 21106.0, "thermal_pressure": "Nominal"}
322 +{"t": 1793514624.5270631, "elapsed_ns": 1021298208, "cpu_mW": 20269.3, "gpu_mW": 1353.18, "ane_mW": 0.0, "combined_mW": 21622.5, "thermal_pressure": "Nominal"}
323 +{"t": 1793514625.544904, "elapsed_ns": 1021931583, "cpu_mW": 21425.1, "gpu_mW": 1422.8, "ane_mW": 0.0, "combined_mW": 22847.9, "thermal_pressure": "Nominal"}
324 +{"t": 1793514626.567263, "elapsed_ns": 1021341250, "cpu_mW": 20476.0, "gpu_mW": 1429.49, "ane_mW": 0.0, "combined_mW": 21905.5, "thermal_pressure": "Nominal"}
325 +{"t": 1793514627.589171, "elapsed_ns": 1022137583, "cpu_mW": 20095.1, "gpu_mW": 1182.82, "ane_mW": 0.0, "combined_mW": 21278.0, "thermal_pressure": "Nominal"}
326 +{"t": 1793514628.613048, "elapsed_ns": 1021251625, "cpu_mW": 19996.1, "gpu_mW": 1330.72, "ane_mW": 0.0, "combined_mW": 21326.8, "thermal_pressure": "Nominal"}
327 +{"t": 1793514629.633249, "elapsed_ns": 1022837041, "cpu_mW": 21384.6, "gpu_mW": 3196.99, "ane_mW": 0.0, "combined_mW": 24581.6, "thermal_pressure": "Nominal"}
328 +{"t": 1793514630.6989892, "elapsed_ns": 1023160541, "cpu_mW": 22916.2, "gpu_mW": 10281.9, "ane_mW": 0.0, "combined_mW": 33198.1, "thermal_pressure": "Nominal"}
329 +{"t": 1793514631.687009, "elapsed_ns": 1021780791, "cpu_mW": 22109.4, "gpu_mW": 863.199, "ane_mW": 0.0, "combined_mW": 22972.6, "thermal_pressure": "Nominal"}
330 +{"t": 1793514633.731264, "elapsed_ns": 1030373458, "cpu_mW": 23249.8, "gpu_mW": 1782.85, "ane_mW": 0.0, "combined_mW": 25032.7, "thermal_pressure": "Nominal"}
331 +{"t": 1793514633.7334049, "elapsed_ns": 1022465458, "cpu_mW": 22906.4, "gpu_mW": 6529.32, "ane_mW": 0.0, "combined_mW": 29435.7, "thermal_pressure": "Nominal"}
332 +{"t": 1793514634.753105, "elapsed_ns": 1022569666, "cpu_mW": 23174.9, "gpu_mW": 3640.83, "ane_mW": 0.0, "combined_mW": 26815.8, "thermal_pressure": "Nominal"}
333 +{"t": 1793514635.776225, "elapsed_ns": 1022206750, "cpu_mW": 20553.6, "gpu_mW": 3307.55, "ane_mW": 0.0, "combined_mW": 23861.1, "thermal_pressure": "Nominal"}
334 +{"t": 1793514636.801394, "elapsed_ns": 1021804333, "cpu_mW": 21215.4, "gpu_mW": 2.93598, "ane_mW": 0.0, "combined_mW": 21218.3, "thermal_pressure": "Nominal"}
335 +{"t": 1793514637.8185432, "elapsed_ns": 1022193791, "cpu_mW": 23387.0, "gpu_mW": 9600.92, "ane_mW": 0.0, "combined_mW": 32987.9, "thermal_pressure": "Nominal"}
336 +{"t": 1793514638.842475, "elapsed_ns": 1021944916, "cpu_mW": 21267.3, "gpu_mW": 8376.19, "ane_mW": 0.0, "combined_mW": 29643.5, "thermal_pressure": "Nominal"}
337 +{"t": 1793514639.864485, "elapsed_ns": 1020976916, "cpu_mW": 23274.8, "gpu_mW": 5490.82, "ane_mW": 0.0, "combined_mW": 28765.6, "thermal_pressure": "Nominal"}
338 +{"t": 1793514640.8875241, "elapsed_ns": 1022163250, "cpu_mW": 22014.1, "gpu_mW": 10267.4, "ane_mW": 0.0, "combined_mW": 32281.5, "thermal_pressure": "Nominal"}
339 +{"t": 1793514642.934209, "elapsed_ns": 1023654875, "cpu_mW": 27086.3, "gpu_mW": 9947.69, "ane_mW": 0.0, "combined_mW": 37034.0, "thermal_pressure": "Nominal"}
340 +{"t": 1793514642.9371352, "elapsed_ns": 1023348750, "cpu_mW": 25185.9, "gpu_mW": 9436.67, "ane_mW": 0.0, "combined_mW": 34622.6, "thermal_pressure": "Nominal"}
341 +{"t": 1793514643.956979, "elapsed_ns": 1023807208, "cpu_mW": 25876.9, "gpu_mW": 8864.95, "ane_mW": 0.0, "combined_mW": 34741.9, "thermal_pressure": "Nominal"}
342 +{"t": 1793514644.979489, "elapsed_ns": 1023149708, "cpu_mW": 24534.0, "gpu_mW": 5829.06, "ane_mW": 0.0, "combined_mW": 30363.1, "thermal_pressure": "Nominal"}
343 +{"t": 1793514646.002538, "elapsed_ns": 1022825125, "cpu_mW": 22665.7, "gpu_mW": 1668.91, "ane_mW": 0.0, "combined_mW": 24334.6, "thermal_pressure": "Nominal"}
344 +{"t": 1793514647.0245628, "elapsed_ns": 1021656541, "cpu_mW": 26014.6, "gpu_mW": 9672.53, "ane_mW": 0.0, "combined_mW": 35687.1, "thermal_pressure": "Nominal"}
added benchmarks/raw/tests/distributed_localhost_olmoe_20261101T012944.json +56 −0
@@ -0,0 +1,56 @@
1 +{
2 + "rtt_ms_localhost": [
3 + 0.08566700853407383,
4 + 0.05250005051493645,
5 + 0.05170807708054781,
6 + 0.04841701593250036,
7 + 0.038207974284887314
8 + ],
9 + "prompts": [
10 + {
11 + "prompt": "The capital of Canada is",
12 + "max_abs_dlogit": 0.0,
13 + "argmax_equal": true,
14 + "remote": {
15 + "remote_pairs": 312,
16 + "local_pairs": 328,
17 + "layers_remote": 16,
18 + "remote_wait_s": 0.04496862518135458,
19 + "failures": 0,
20 + "workers": {
21 + "w0": {
22 + "worker": "w0",
23 + "exec_calls": 16,
24 + "exec_wall_s": 0.20909150002989918,
25 + "bytes_sent": 333317,
26 + "bytes_recv": 1281716
27 + }
28 + }
29 + }
30 + },
31 + {
32 + "prompt": "def fibonacci(n):\n \"\"\"Retur",
33 + "max_abs_dlogit": 0.0,
34 + "argmax_equal": true,
35 + "remote": {
36 + "remote_pairs": 1660,
37 + "local_pairs": 1668,
38 + "layers_remote": 56,
39 + "remote_wait_s": 0.05559412541333586,
40 + "failures": 0,
41 + "workers": {
42 + "w0": {
43 + "worker": "w0",
44 + "exec_calls": 56,
45 + "exec_wall_s": 0.30413495854008943,
46 + "bytes_sent": 1722764,
47 + "bytes_recv": 6807094
48 + }
49 + }
50 + }
51 + }
52 + ],
53 + "greedy_equal": true,
54 + "greedy_ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is",
55 + "greedy_titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is"
56 +}
\ No newline at end of file
added benchmarks/raw/tests/distributed_localhost_olmoe_20261101T013029.json +56 −0
@@ -0,0 +1,56 @@
1 +{
2 + "rtt_ms_localhost": [
3 + 0.08600007276982069,
4 + 0.0826249597594142,
5 + 0.06912497337907553,
6 + 0.08879194501787424,
7 + 0.06387499161064625
8 + ],
9 + "prompts": [
10 + {
11 + "prompt": "The capital of Canada is",
12 + "max_abs_dlogit": 0.0,
13 + "argmax_equal": true,
14 + "remote": {
15 + "remote_pairs": 312,
16 + "local_pairs": 328,
17 + "layers_remote": 16,
18 + "remote_wait_s": 0.3690654179081321,
19 + "failures": 0,
20 + "workers": {
21 + "w0": {
22 + "worker": "w0",
23 + "exec_calls": 16,
24 + "exec_wall_s": 0.5654866262339056,
25 + "bytes_sent": 333317,
26 + "bytes_recv": 1281719
27 + }
28 + }
29 + }
30 + },
31 + {
32 + "prompt": "def fibonacci(n):\n \"\"\"Retur",
33 + "max_abs_dlogit": 0.0,
34 + "argmax_equal": true,
35 + "remote": {
36 + "remote_pairs": 1660,
37 + "local_pairs": 1668,
38 + "layers_remote": 56,
39 + "remote_wait_s": 0.3791747959330678,
40 + "failures": 0,
41 + "workers": {
42 + "w0": {
43 + "worker": "w0",
44 + "exec_calls": 56,
45 + "exec_wall_s": 0.6461280437652022,
46 + "bytes_sent": 1722764,
47 + "bytes_recv": 6807116
48 + }
49 + }
50 + }
51 + }
52 + ],
53 + "greedy_equal": true,
54 + "greedy_ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is",
55 + "greedy_titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is"
56 +}
\ No newline at end of file
added benchmarks/raw/tests/fused_upgate_olmoe_2026-11-01T012945.json +26 −0
@@ -0,0 +1,26 @@
1 +{
2 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
3 + "ts": "2026-11-01T01:29:45",
4 + "cases": [
5 + {
6 + "capacity": 64,
7 + "prompt": "The capital of Canada is",
8 + "max_abs_dlogit": 0.0
9 + },
10 + {
11 + "capacity": 64,
12 + "prompt": "def fibonacci(n):\n \"\"\"Retur",
13 + "max_abs_dlogit": 0.0
14 + },
15 + {
16 + "capacity": 16,
17 + "prompt": "The capital of Canada is",
18 + "max_abs_dlogit": 0.0
19 + },
20 + {
21 + "capacity": 16,
22 + "prompt": "def fibonacci(n):\n \"\"\"Retur",
23 + "max_abs_dlogit": 0.0
24 + }
25 + ]
26 +}
\ No newline at end of file
added benchmarks/raw/tests/fused_upgate_olmoe_2026-11-01T013029.json +26 −0
@@ -0,0 +1,26 @@
1 +{
2 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
3 + "ts": "2026-11-01T01:30:29",
4 + "cases": [
5 + {
6 + "capacity": 64,
7 + "prompt": "The capital of Canada is",
8 + "max_abs_dlogit": 0.0
9 + },
10 + {
11 + "capacity": 64,
12 + "prompt": "def fibonacci(n):\n \"\"\"Retur",
13 + "max_abs_dlogit": 0.0
14 + },
15 + {
16 + "capacity": 16,
17 + "prompt": "The capital of Canada is",
18 + "max_abs_dlogit": 0.0
19 + },
20 + {
21 + "capacity": 16,
22 + "prompt": "def fibonacci(n):\n \"\"\"Retur",
23 + "max_abs_dlogit": 0.0
24 + }
25 + ]
26 +}
\ No newline at end of file
added benchmarks/raw/tests/greedy_olmoe_cap16_20261101T013007.json +14 −0
@@ -0,0 +1,14 @@
1 +[
2 + {
3 + "prompt": "The capital of Canada is",
4 + "ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
5 + "titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
6 + "equal": true
7 + },
8 + {
9 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
10 + "ref": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
11 + "titan": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
12 + "equal": true
13 + }
14 +]
\ No newline at end of file
added benchmarks/raw/tests/greedy_olmoe_cap16_20261101T013047.json +14 −0
@@ -0,0 +1,14 @@
1 +[
2 + {
3 + "prompt": "The capital of Canada is",
4 + "ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
5 + "titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
6 + "equal": true
7 + },
8 + {
9 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
10 + "ref": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
11 + "titan": " if n <= 0:\n return \"Invalid input\"\n elif n == 1:\n return 0\n elif n == 2:\n",
12 + "equal": true
13 + }
14 +]
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap16_2026-11-01T012959.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 16,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.2580101180438449,
10 + "misses": 440,
11 + "stall_s": 0.09684595465660095,
12 + "layers_routed": 46
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.42512908777969016,
20 + "misses": 1002,
21 + "stall_s": 0.20027187676168978,
22 + "layers_routed": 127
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.41904344010530936,
30 + "misses": 1324,
31 + "stall_s": 0.2887474615126848,
32 + "layers_routed": 161
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.45278969957081544,
40 + "misses": 1275,
41 + "stall_s": 0.2674082911107689,
42 + "layers_routed": 164
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:29:59",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap16_2026-11-01T013039.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 16,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.2580101180438449,
10 + "misses": 440,
11 + "stall_s": 0.09993178956210613,
12 + "layers_routed": 46
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.42512908777969016,
20 + "misses": 1002,
21 + "stall_s": 0.2170187613228336,
22 + "layers_routed": 127
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.41904344010530936,
30 + "misses": 1324,
31 + "stall_s": 0.3052907877136022,
32 + "layers_routed": 161
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.45278969957081544,
40 + "misses": 1275,
41 + "stall_s": 0.2862527520628646,
42 + "layers_routed": 164
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:30:39",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap32_2026-11-01T012957.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 32,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.0,
10 + "misses": 430,
11 + "stall_s": 0.42347062879707664,
12 + "layers_routed": 16
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.6060606060606061,
20 + "misses": 429,
21 + "stall_s": 0.46165179007221013,
22 + "layers_routed": 40
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.6090458488228006,
30 + "misses": 631,
31 + "stall_s": 0.32894921104889363,
32 + "layers_routed": 58
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.6224188790560472,
40 + "misses": 512,
41 + "stall_s": 0.18187279207631946,
42 + "layers_routed": 49
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:29:57",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap32_2026-11-01T013038.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 32,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.0,
10 + "misses": 430,
11 + "stall_s": 0.0899640420684591,
12 + "layers_routed": 16
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.6060606060606061,
20 + "misses": 429,
21 + "stall_s": 0.28829907963518053,
22 + "layers_routed": 40
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.6090458488228006,
30 + "misses": 631,
31 + "stall_s": 0.14359562494792044,
32 + "layers_routed": 58
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.6224188790560472,
40 + "misses": 512,
41 + "stall_s": 0.1363065792247653,
42 + "layers_routed": 49
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:30:38",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap64_2026-11-01T012954.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 64,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.0,
10 + "misses": 430,
11 + "stall_s": 0.4835599169600755,
12 + "layers_routed": 16
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.4637904468412943,
20 + "misses": 348,
21 + "stall_s": 0.42762762669008225,
22 + "layers_routed": 16
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.823076923076923,
30 + "misses": 138,
31 + "stall_s": 0.1677537460345775,
32 + "layers_routed": 16
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.9234828496042217,
40 + "misses": 58,
41 + "stall_s": 0.0731603738386184,
42 + "layers_routed": 16
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:29:54",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap64_2026-11-01T013035.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 64,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.0,
10 + "misses": 430,
11 + "stall_s": 0.3988114590756595,
12 + "layers_routed": 16
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.4637904468412943,
20 + "misses": 348,
21 + "stall_s": 0.3186756670475006,
22 + "layers_routed": 16
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.823076923076923,
30 + "misses": 138,
31 + "stall_s": 0.09854987578000873,
32 + "layers_routed": 16
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.9234828496042217,
40 + "misses": 58,
41 + "stall_s": 0.0453380401013419,
42 + "layers_routed": 16
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:30:35",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap8_2026-11-01T013001.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 8,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.1625,
10 + "misses": 536,
11 + "stall_s": 0.09062562719918787,
12 + "layers_routed": 80
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.4425981873111782,
20 + "misses": 1476,
21 + "stall_s": 0.24081315705552697,
22 + "layers_routed": 331
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.375,
30 + "misses": 1995,
31 + "stall_s": 0.3570274900412187,
32 + "layers_routed": 399
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.3878048780487805,
40 + "misses": 2008,
41 + "stall_s": 0.39880511723458767,
42 + "layers_routed": 410
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:30:01",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
added benchmarks/raw/tests/numerical_olmoe_cap8_2026-11-01T013041.json +50 −0
@@ -0,0 +1,50 @@
1 +{
2 + "capacity": 8,
3 + "prompts": [
4 + {
5 + "prompt": "The capital of Canada is",
6 + "tokens": 5,
7 + "max_abs_dlogit": 0.0,
8 + "argmax_equal": true,
9 + "hit_rate": 0.1625,
10 + "misses": 536,
11 + "stall_s": 0.11774066393263638,
12 + "layers_routed": 80
13 + },
14 + {
15 + "prompt": "def fibonacci(n):\n \"\"\"Return the n-th",
16 + "tokens": 21,
17 + "max_abs_dlogit": 0.0,
18 + "argmax_equal": true,
19 + "hit_rate": 0.4425981873111782,
20 + "misses": 1476,
21 + "stall_s": 0.34902799827978015,
22 + "layers_routed": 331
23 + },
24 + {
25 + "prompt": "Q: If a train travels 60 km in 45 minute",
26 + "tokens": 25,
27 + "max_abs_dlogit": 0.0,
28 + "argmax_equal": true,
29 + "hit_rate": 0.375,
30 + "misses": 1995,
31 + "stall_s": 0.4580805116565898,
32 + "layers_routed": 399
33 + },
34 + {
35 + "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
36 + "tokens": 26,
37 + "max_abs_dlogit": 0.0,
38 + "argmax_equal": true,
39 + "hit_rate": 0.3878048780487805,
40 + "misses": 2008,
41 + "stall_s": 0.4504705760627985,
42 + "layers_routed": 410
43 + }
44 + ],
45 + "tol": 0.001,
46 + "ts": "2026-11-01T01:30:41",
47 + "max_abs_dlogit_worst": 0.0,
48 + "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
49 + "model_type": "olmoe"
50 +}
\ No newline at end of file
modified titan/prefetch.py +2 −1
@@ -38,7 +38,7 @@ class Prefetcher:
38 38 # ---- reading ---------------------------------------------------------------
39 39 def _fetch(self, layer: int, expert: int):
40 40 t = time.perf_counter()
41 − tensors = self.store.read_expert(layer, expert) # pread + np -> mx (CPU side, no GPU work)
41 + tensors = self.store.read_expert(layer, expert, as_mx=False) # pread + numpy only; MLX arrays are created in drain()
42 42 return tensors, time.perf_counter() - t
43 43
44 44 def issue(self, layer: int, experts: List[int]):
@@ -63,6 +63,7 @@ class Prefetcher:
63 63 self.late[layer] += 1
64 64 if fut.done():
65 65 tensors, _ = fut.result()
66 + tensors = {n: self.store.to_mx(v) for n, v in tensors.items()}
66 67 del inf[e]
67 68 if tab.insert_prefetched(e, tensors, protected) is not None:
68 69 self.installed[layer] += 1
modified titan/store.py +20 −10
@@ -49,7 +49,7 @@ class ExpertStore:
49 49 self._fds[path] = fd
50 50 return fd
51 51
52 − def read_tensor(self, loc: TensorLoc) -> mx.array:
52 + def read_tensor(self, loc: TensorLoc, as_mx: bool = True):
53 53 fd = self._fd(loc.file)
54 54 t = time.perf_counter()
55 55 buf = os.pread(fd, loc.nbytes, loc.offset)
@@ -58,22 +58,31 @@ class ExpertStore:
58 58 self.reads += 1
59 59 if len(buf) != loc.nbytes:
60 60 raise IOError(f"short read for {loc.key}: {len(buf)} != {loc.nbytes}")
61 − arr = np.frombuffer(buf, dtype=NP_DTYPE[loc.dtype]).reshape(loc.shape)
62 − if loc.dtype == "BF16":
63 − return mx.array(arr).view(mx.bfloat16)
64 − return mx.array(arr)
61 + return self._wrap(loc, buf, as_mx)
65 62
66 − def _wrap(self, loc: TensorLoc, buf) -> mx.array:
63 + def _wrap(self, loc: TensorLoc, buf, as_mx: bool = True):
67 64 arr = np.frombuffer(buf, dtype=NP_DTYPE[loc.dtype]).reshape(loc.shape)
65 + if not as_mx:
66 + return (arr, loc.dtype)
68 67 if loc.dtype == "BF16":
69 68 return mx.array(arr).view(mx.bfloat16)
70 69 return mx.array(arr)
71 70
72 − def read_expert(self, layer: int, expert: int) -> Dict[str, mx.array]:
71 + @staticmethod
72 + def to_mx(t):
73 + """Convert a numpy (array, dtype) pair produced by a reader thread into an MLX array ON THE CALLING THREAD.
74 + MLX arrays must be created in the thread that evaluates them (streams are thread-local): creating them in reader
75 + threads raised 'There is no Stream(gpu, N) in current thread' on the worker (2026-11-01)."""
76 + if isinstance(t, mx.array):
77 + return t
78 + arr, dt = t
79 + return mx.array(arr).view(mx.bfloat16) if dt == "BF16" else mx.array(arr)
80 +
81 + def read_expert(self, layer: int, expert: int, as_mx: bool = True):
73 82 c = self._contig.get((layer, expert))
74 83 if c is None:
75 84 es = self.spec.experts[(layer, expert)]
76 − return {name: self.read_tensor(loc) for name, loc in es.tensors.items()}
85 + return {name: self.read_tensor(loc, as_mx) for name, loc in es.tensors.items()}
77 86 path, off, n, locs = c
78 87 fd = self._fd(path)
79 88 t = time.perf_counter()
@@ -85,7 +94,7 @@ class ExpertStore:
85 94 out = {}
86 95 for name, loc in locs:
87 96 a = loc.offset - off
88 − out[name] = self._wrap(loc, memoryview(buf)[a:a + loc.nbytes])
97 + out[name] = self._wrap(loc, memoryview(buf)[a:a + loc.nbytes], as_mx)
89 98 return out
90 99
91 100 def read_experts(self, layer: int, experts) -> Dict[int, Dict[str, mx.array]]:
@@ -94,7 +103,8 @@ class ExpertStore:
94 103 if self._pool is None or len(experts) == 1:
95 104 return {e: self.read_expert(layer, e) for e in experts}
96 105 t = time.perf_counter()
97 − res = dict(zip(experts, self._pool.map(lambda e: self.read_expert(layer, e), experts)))
106 + raw = dict(zip(experts, self._pool.map(lambda e: self.read_expert(layer, e, as_mx=False), experts)))
107 + res = {e: {n: self.to_mx(v) for n, v in d.items()} for e, d in raw.items()} # MLX arrays created here, in the caller's thread
98 108 self.read_time += 0 # per-read times already accumulated inside read_expert (sum of thread times, not wall)
99 109 self.wall_batch_time = getattr(self, "wall_batch_time", 0.0) + (time.perf_counter() - t)
100 110 return res
101 111
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere