Fix: MLX arrays created on the consuming thread only (worker/stacked-layout stream error)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 2 days ago (Nov 1, 2026) parent e0179f6 Browse files

18 changed files +889 −11

modified benchmarks/raw/gateG/energy_laptop/bench.jsonl +18 −0

+ {"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:25:58", "prompt_tokens": 18, "gen_tokens": 256, "ttft_ms": 473.99025002960116, "prompt_tps": 38.02749282251309, "gen_tps": 30.149703436329805, "decode_tps_wall": 30.03199408681774, "itl_ms_p50": 32.685291953384876, "itl_ms_p95": 43.34554204251617, "itl_ms_p99": 49.13512500934303, "itl_ms_mean": 33.29782221950225, "total_s": 8.964934916002676, "peak_mem_gb": 1.779299288, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "t_start_epoch": 1793514349.968512, "t_end_epoch": 1793514358.9347358, "t_first_epoch": 1793514350.4425023, "cache_hit_rate": 0.5059072218683037, "cache_misses": 17063, "cache_hits": 17471, "stall_s": 4.963466331246309, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.586632123333402, "prefetch": 0.0, "ensure": 5.089648996130563, "build": 0.053854094934649765}, "ssd_bytes": 60385001472, "ssd_read_s": 12.836250820546411}
+ {"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:26:07", "prompt_tokens": 14, "gen_tokens": 256, "ttft_ms": 449.9966250732541, "prompt_tps": 31.15616929910054, "gen_tps": 32.46190672778869, "decode_tps_wall": 32.33514887274489, "itl_ms_p50": 31.00487496703863, "itl_ms_p95": 41.70525004155934, "itl_ms_p99": 44.96754193678498, "itl_ms_mean": 30.926098529358995, "total_s": 8.336151750059798, "peak_mem_gb": 1.74502108, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "t_start_epoch": 1793514358.934883, "t_end_epoch": 1793514367.2724721, "t_first_epoch": 1793514359.3848798, "cache_hit_rate": 0.49322548618816797, "cache_misses": 17355, "cache_hits": 16891, "stall_s": 4.965477578341961, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.010114970151335, "prefetch": 0.0, "ensure": 5.079476987943053, "build": 0.04450405645184219}, "ssd_bytes": 61418373120, "ssd_read_s": 13.378405682509765}
+ {"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:26:14", "prompt_tokens": 19, "gen_tokens": 256, "ttft_ms": 430.6972920894623, "prompt_tps": 44.19175033284113, "gen_tps": 35.12275901609846, "decode_tps_wall": 34.985638138366845, "itl_ms_p50": 27.491708053275943, "itl_ms_p95": 44.86395907588303, "itl_ms_p99": 47.699542017653584, "itl_ms_mean": 28.583157352884026, "total_s": 7.719402417074889, "peak_mem_gb": 1.775334664, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array in descending ord", "t_start_epoch": 1793514367.272643, "t_end_epoch": 1793514374.9930902, "t_first_epoch": 1793514367.7033403, "cache_hit_rate": 0.6598117811084001, "cache_misses": 11712, "cache_hits": 22716, "stall_s": 4.332302771508694, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.0599256032146513, "prefetch": 0.0, "ensure": 4.420602378551848, "build": 0.04536858550272882}, "ssd_bytes": 41448112128, "ssd_read_s": 10.390336600947194}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:26:26", "prompt_tokens": 18, "gen_tokens": 256, "ttft_ms": 639.3235409632325, "prompt_tps": 28.174881999836646, "gen_tps": 23.48477154172949, "decode_tps_wall": 23.3930878035731, "itl_ms_p50": 42.089167051017284, "itl_ms_p95": 57.280458975583315, "itl_ms_p99": 63.42966703232378, "itl_ms_mean": 42.74767009797049, "total_s": 11.539979415945709, "peak_mem_gb": 1.816294424, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "t_start_epoch": 1793514374.993252, "t_end_epoch": 1793514386.534442, "t_first_epoch": 1793514375.6325755, "cache_hit_rate": 0.5050095558000811, "cache_misses": 17094, "cache_hits": 17440, "stall_s": 6.836809101747349, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 4.1504173673456535, "prefetch": 0.0, "ensure": 6.992573518422432, "build": 0.08305123588070273}, "ssd_bytes": 60494708736, "ssd_read_s": 18.493526841746643}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:26:37", "prompt_tokens": 14, "gen_tokens": 256, "ttft_ms": 531.4388750120997, "prompt_tps": 26.44934124112812, "gen_tps": 24.711482283116222, "decode_tps_wall": 24.61498107386689, "itl_ms_p50": 39.87624996807426, "itl_ms_p95": 57.44204192887992, "itl_ms_p99": 77.63004198204726, "itl_ms_mean": 40.62566601205617, "total_s": 10.890983708086424, "peak_mem_gb": 1.74502108, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "t_start_epoch": 1793514386.535053, "t_end_epoch": 1793514397.427111, "t_first_epoch": 1793514387.0664918, "cache_hit_rate": 0.49322548618816797, "cache_misses": 17355, "cache_hits": 16891, "stall_s": 6.635692013427615, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.8223506945651025, "prefetch": 0.0, "ensure": 6.766858409275301, "build": 0.056402937159873545}, "ssd_bytes": 61418373120, "ssd_read_s": 18.161584803368896}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:26:45", "prompt_tokens": 19, "gen_tokens": 256, "ttft_ms": 458.79750000312924, "prompt_tps": 41.46472689933416, "gen_tps": 35.435561883947024, "decode_tps_wall": 35.29719546445759, "itl_ms_p50": 27.789458981715143, "itl_ms_p95": 39.23400002531707, "itl_ms_p99": 43.578875018283725, "itl_ms_mean": 28.330862745368744, "total_s": 7.683167500072159, "peak_mem_gb": 1.775334664, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array in descending ord", "t_start_epoch": 1793514397.427271, "t_end_epoch": 1793514405.111447, "t_first_epoch": 1793514397.8860683, "cache_hit_rate": 0.6598117811084001, "cache_misses": 11712, "cache_hits": 22716, "stall_s": 4.178970686276443, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.186878505628556, "prefetch": 0.0, "ensure": 4.265643556253053, "build": 0.042900146334432065}, "ssd_bytes": 41448112128, "ssd_read_s": 10.073308067861944}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 0, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:26:53", "prompt_tokens": 24, "gen_tokens": 256, "ttft_ms": 623.6247919732705, "prompt_tps": 38.612191977894824, "gen_tps": 33.16532823969986, "decode_tps_wall": 33.03582378968533, "itl_ms_p50": 29.62212497368455, "itl_ms_p95": 43.53091702796519, "itl_ms_p99": 48.3604590408504, "itl_ms_mean": 30.27016993329002, "total_s": 8.342518124962226, "peak_mem_gb": 1.88188166, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n    // TODO: Implement the dot product computation using NEON intrinsics\n    // Start by loading the arrays int", "t_start_epoch": 1793514405.111955, "t_end_epoch": 1793514413.4554849, "t_first_epoch": 1793514405.7355797, "cache_hit_rate": 0.6277690213497635, "cache_misses": 12989, "cache_hits": 21906, "stall_s": 4.633197326445952, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.355849905521609, "prefetch": 0.0, "ensure": 4.731186131131835, "build": 0.047959326999261975}, "ssd_bytes": 45967343616, "ssd_read_s": 11.483838297543116}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:27:04", "prompt_tokens": 18, "gen_tokens": 256, "ttft_ms": 682.5923330616206, "prompt_tps": 26.453586285389044, "gen_tps": 25.752552973495998, "decode_tps_wall": 25.651980179231074, "itl_ms_p50": 37.54041704814881, "itl_ms_p95": 61.23966700397432, "itl_ms_p99": 66.31250004284084, "itl_ms_mean": 38.9833452627428, "total_s": 10.623345375061035, "peak_mem_gb": 1.823634456, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "t_start_epoch": 1793514413.4557571, "t_end_epoch": 1793514424.080147, "t_first_epoch": 1793514414.1383495, "cache_hit_rate": 0.5059072218683037, "cache_misses": 17063, "cache_hits": 17471, "stall_s": 6.51750842097681, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.7390375823015347, "prefetch": 0.0, "ensure": 6.631261183996685, "build": 0.04644273582380265}, "ssd_bytes": 60385001472, "ssd_read_s": 18.00621229421813}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:27:12", "prompt_tokens": 24, "gen_tokens": 256, "ttft_ms": 609.8394999280572, "prompt_tps": 39.39834350012532, "gen_tps": 33.499659360229174, "decode_tps_wall": 33.36883843156555, "itl_ms_p50": 29.542250093072653, "itl_ms_p95": 43.06070797611028, "itl_ms_p99": 46.94349993951619, "itl_ms_mean": 29.968079411899488, "total_s": 8.251699749962427, "peak_mem_gb": 1.917582228, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n    // TODO: Implement the dot product computation using NEON intrinsics\n    // Start by loading the arrays int", "t_start_epoch": 1793514424.0803132, "t_end_epoch": 1793514432.333015, "t_first_epoch": 1793514424.6901526, "cache_hit_rate": 0.6274537899412523, "cache_misses": 13000, "cache_hits": 21895, "stall_s": 4.565376511425711, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.3453963060164824, "prefetch": 0.0, "ensure": 4.660446377354674, "build": 0.04569679545238614}, "ssd_bytes": 46006272000, "ssd_read_s": 11.44613619998563}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:27:22", "prompt_tokens": 14, "gen_tokens": 256, "ttft_ms": 501.20487494859844, "prompt_tps": 27.96523686232182, "gen_tps": 27.75032254346846, "decode_tps_wall": 27.641952809705582, "itl_ms_p50": 36.01766692008823, "itl_ms_p95": 47.68933297600597, "itl_ms_p99": 52.69720801152289, "itl_ms_mean": 36.176894117585, "total_s": 9.726312874932773, "peak_mem_gb": 1.730718152, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "t_start_epoch": 1793514432.33316, "t_end_epoch": 1793514442.060885, "t_first_epoch": 1793514432.834365, "cache_hit_rate": 0.49538632249021786, "cache_misses": 17281, "cache_hits": 16965, "stall_s": 5.6758743927348405, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.65540738531854, "prefetch": 0.0, "ensure": 5.796999507001601, "build": 0.051514498074539006}, "ssd_bytes": 61156491264, "ssd_read_s": 15.6160844755359}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:27:29", "prompt_tokens": 19, "gen_tokens": 256, "ttft_ms": 502.0306670339778, "prompt_tps": 37.89478830273463, "gen_tps": 34.92529584914807, "decode_tps_wall": 34.78891696746336, "itl_ms_p50": 28.797582956030965, "itl_ms_p95": 38.754915934987366, "itl_ms_p99": 43.934584013186395, "itl_ms_mean": 28.74478676456812, "total_s": 7.831951291998848, "peak_mem_gb": 1.766782292, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array in descending ord", "t_start_epoch": 1793514442.0610409, "t_end_epoch": 1793514449.894342, "t_first_epoch": 1793514442.5630715, "cache_hit_rate": 0.6598117811084001, "cache_misses": 11712, "cache_hits": 22716, "stall_s": 4.281426511122845, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.2393267332809046, "prefetch": 0.0, "ensure": 4.365906663937494, "build": 0.04135329520795494}, "ssd_bytes": 41448112128, "ssd_read_s": 10.341405491228215}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 2, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:27:39", "prompt_tokens": 14, "gen_tokens": 256, "ttft_ms": 482.9200420062989, "prompt_tps": 29.02662330914426, "gen_tps": 29.608985443323636, "decode_tps_wall": 29.49336102167945, "itl_ms_p50": 33.19170803297311, "itl_ms_p95": 47.35679202713072, "itl_ms_p99": 53.26833401340991, "itl_ms_mean": 33.90593561937339, "total_s": 9.128933624946512, "peak_mem_gb": 1.73301178, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "t_start_epoch": 1793514449.894492, "t_end_epoch": 1793514459.024576, "t_first_epoch": 1793514450.3774118, "cache_hit_rate": 0.49383869649010104, "cache_misses": 17334, "cache_hits": 16912, "stall_s": 5.228863131022081, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.534531174809672, "prefetch": 0.0, "ensure": 5.34318076109048, "build": 0.045298933633603156}, "ssd_bytes": 61344055296, "ssd_read_s": 14.219157688086852}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 2, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:27:47", "prompt_tokens": 18, "gen_tokens": 256, "ttft_ms": 502.13170901406556, "prompt_tps": 35.90810218990763, "gen_tps": 30.72349003220354, "decode_tps_wall": 30.603541595917655, "itl_ms_p50": 31.901582959108055, "itl_ms_p95": 46.237667091190815, "itl_ms_p99": 53.17033396568149, "itl_ms_mean": 32.675956698207585, "total_s": 8.834500667057, "peak_mem_gb": 1.816294424, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "t_start_epoch": 1793514459.024839, "t_end_epoch": 1793514467.862941, "t_first_epoch": 1793514459.5269706, "cache_hit_rate": 0.5046910291307117, "cache_misses": 17105, "cache_hits": 17429, "stall_s": 4.990589527995326, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.485891460906714, "prefetch": 0.0, "ensure": 5.102543488028459, "build": 0.04451872012577951}, "ssd_bytes": 60533637120, "ssd_read_s": 13.334814880625345}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 2, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:27:55", "prompt_tokens": 19, "gen_tokens": 256, "ttft_ms": 493.39433398563415, "prompt_tps": 38.57391560044966, "gen_tps": 37.846937735738834, "decode_tps_wall": 37.6991578148187, "itl_ms_p50": 26.25908306799829, "itl_ms_p95": 37.12208301294595, "itl_ms_p99": 40.99449992645532, "itl_ms_mean": 26.525791502082903, "total_s": 7.257471167016774, "peak_mem_gb": 1.773794588, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array in descending ord", "t_start_epoch": 1793514467.863209, "t_end_epoch": 1793514475.1218529, "t_first_epoch": 1793514468.3566034, "cache_hit_rate": 0.659840827233647, "cache_misses": 11711, "cache_hits": 22717, "stall_s": 3.7971609133528545, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.158985341200605, "prefetch": 0.0, "ensure": 3.880476161488332, "build": 0.03890139679424465}, "ssd_bytes": 41444573184, "ssd_read_s": 8.934044936555438}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "none", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "f5650ca"}, "seed": 0, "warmup": false, "repeat": 2, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:28:02", "prompt_tokens": 24, "gen_tokens": 256, "ttft_ms": 660.2665830869228, "prompt_tps": 36.401945768739516, "gen_tps": 36.0720181380442, "decode_tps_wall": 35.93119493066561, "itl_ms_p50": 27.531874948181212, "itl_ms_p95": 41.34691599756479, "itl_ms_p99": 46.20891599915922, "itl_ms_mean": 27.830969772356397, "total_s": 7.757163875037804, "peak_mem_gb": 1.924660096, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n    // TODO: Implement the dot product computation using NEON intrinsics\n    // Start by loading the arrays int", "t_start_epoch": 1793514475.1230562, "t_end_epoch": 1793514482.88139, "t_first_epoch": 1793514475.7833228, "cache_hit_rate": 0.6277690213497635, "cache_misses": 12989, "cache_hits": 21906, "stall_s": 4.170646916842088, "hits_device": 0, "layers_complete": 0, "prefetch": null, "host_time_s": {"sync": 3.277813452645205, "prefetch": 0.0, "ensure": 4.25988938997034, "build": 0.03849247260950506}, "ssd_bytes": 45967343616, "ssd_read_s": 10.19479018333368}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --predictor probe --m 8 --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "e0179f6"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a C function that computes the dot product of two floa", "ts": "2026-11-01T01:30:09", "prompt_tokens": 24, "gen_tokens": 256, "ttft_ms": 2217.1235830755904, "prompt_tps": 10.9491894139912, "gen_tps": 45.25487565647324, "decode_tps_wall": 45.08706810413277, "itl_ms_p50": 22.148250020109117, "itl_ms_p95": 27.636207989417017, "itl_ms_p99": 29.460499994456768, "itl_ms_mean": 22.179308658757922, "total_s": 7.872847291058861, "peak_mem_gb": 1.9434972, "text": "#include <arm_neon.h>\n\nfloat dot_product_neon(const float *a, const float *b, int len) {\n    // TODO: Implement the dot product computation using NEON intrinsics\n    // Start by loading the arrays int", "t_start_epoch": 1793514601.242044, "t_end_epoch": 1793514609.116669, "t_first_epoch": 1793514603.4591675, "cache_hit_rate": 0.837569852414386, "cache_misses": 5668, "cache_hits": 29227, "stall_s": 1.7648288635537028, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 10497, "installed": 10497, "late": 231, "dup": 20463, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 5.376388295902871, "prefetch": 0.38092932023573667, "ensure": 1.7893478660844266, "build": 0.05343569233082235}, "ssd_bytes": 57207029760, "ssd_read_s": 6.779934052960016}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --predictor probe --m 8 --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "e0179f6"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:30:15", "prompt_tokens": 18, "gen_tokens": 256, "ttft_ms": 298.4869999345392, "prompt_tps": 60.80249156492609, "gen_tps": 39.53517597853951, "decode_tps_wall": 39.380788319710305, "itl_ms_p50": 25.2036249730736, "itl_ms_p95": 30.1873330026865, "itl_ms_p99": 32.05770906060934, "itl_ms_mean": 25.393092486660418, "total_s": 6.773725584032945, "peak_mem_gb": 1.838679744, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "t_start_epoch": 1793514609.117152, "t_end_epoch": 1793514615.891936, "t_first_epoch": 1793514609.415639, "cache_hit_rate": 0.8176869172409799, "cache_misses": 6296, "cache_hits": 28238, "stall_s": 1.6834739480400458, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 15936, "installed": 15936, "late": 107, "dup": 15816, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 4.424398881732486, "prefetch": 0.3463717408012599, "ensure": 1.7266047825105488, "build": 0.05043836368713528}, "ssd_bytes": 78677803008, "ssd_read_s": 7.841062449966557}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --predictor probe --m 8 --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "e0179f6"}, "seed": 0, "warmup": true, "repeat": -1, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:30:22", "prompt_tokens": 14, "gen_tokens": 256, "ttft_ms": 297.41445800755173, "prompt_tps": 47.21940919709833, "gen_tps": 38.782262704355546, "decode_tps_wall": 38.63083508335882, "itl_ms_p50": 26.01779205724597, "itl_ms_p95": 30.615541036240757, "itl_ms_p99": 32.95816702302545, "itl_ms_mean": 25.88605702781648, "total_s": 6.898359000100754, "peak_mem_gb": 1.751351044, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "t_start_epoch": 1793514615.892053, "t_end_epoch": 1793514622.791529, "t_first_epoch": 1793514616.1894674, "cache_hit_rate": 0.8065175494948316, "cache_misses": 6626, "cache_hits": 27620, "stall_s": 1.7857510162284598, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 16251, "installed": 16251, "late": 103, "dup": 15341, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 4.414945824421011, "prefetch": 0.3581021197605878, "ensure": 1.8337656208314002, "build": 0.05403699423186481}, "ssd_bytes": 80960421888, "ssd_read_s": 8.22376422479283}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --predictor probe --m 8 --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "e0179f6"}, "seed": 0, "warmup": true, "repeat": -1, "family": "code", "prompt": "Write a bash script that lists the ten largest files under a", "ts": "2026-11-01T01:30:29", "prompt_tokens": 19, "gen_tokens": 256, "ttft_ms": 390.1689579943195, "prompt_tps": 48.74155925723839, "gen_tps": 37.77430336988103, "decode_tps_wall": 37.626785667837176, "itl_ms_p50": 24.885875056497753, "itl_ms_p95": 40.63329100608826, "itl_ms_p99": 71.47024990990758, "itl_ms_mean": 26.57681176457189, "total_s": 7.167255957960151, "peak_mem_gb": 1.784851132, "text": "#!/bin/bash\n\n# Specify the directory\ndir=\"/path/to/directory\"\n\n# Get the size of each file and store it in an array\nfiles=( $(ls -l \"$dir\" | awk '{print $5, $9}') )\n\n# Sort the array in descending ord", "t_start_epoch": 1793514622.7916439, "t_end_epoch": 1793514629.960074, "t_first_epoch": 1793514623.1818128, "cache_hit_rate": 0.848495410712211, "cache_misses": 5216, "cache_hits": 29212, "stall_s": 2.3131685683038086, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 9759, "installed": 9758, "late": 341, "dup": 21945, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 4.139780909870751, "prefetch": 0.5597494770772755, "ensure": 2.1483880928717554, "build": 0.059471825836226344}, "ssd_bytes": 52995686400, "ssd_read_s": 7.924561083083972}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --predictor probe --m 8 --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "e0179f6"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Describe the main differences between a Mac Studio and a Mac", "ts": "2026-11-01T01:30:38", "prompt_tokens": 18, "gen_tokens": 256, "ttft_ms": 838.4084589779377, "prompt_tps": 21.496361739417928, "gen_tps": 32.63569056171431, "decode_tps_wall": 32.50862027189841, "itl_ms_p50": 27.306250063702464, "itl_ms_p95": 54.80616702698171, "itl_ms_p99": 118.00695804413408, "itl_ms_mean": 30.76107172916333, "total_s": 8.682481749914587, "peak_mem_gb": 1.792476808, "text": "\n\nMac Studio:\n- More powerful hardware options, including multiple GPUs and more RAM.\n- Higher-end CPU options, often with more cores and higher clock speeds.\n- Larger display with a higher resolution", "t_start_epoch": 1793514629.960502, "t_end_epoch": 1793514638.643897, "t_first_epoch": 1793514630.7989104, "cache_hit_rate": 0.8172815196617826, "cache_misses": 6310, "cache_hits": 28224, "stall_s": 3.7183112679049373, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 15840, "installed": 15836, "late": 1217, "dup": 15912, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 4.124537539551966, "prefetch": 1.1234240679768845, "ensure": 3.083217271254398, "build": 0.06652727141045034}, "ssd_bytes": 78387609600, "ssd_read_s": 13.75253405154217}
++{"backend": "titan", "capacity": 16, "policy": "lru", "predictor": "probe", "m": 8, "prefetch_workers": 2, "fused_upgate": false, "nocache": true, "max_tokens": 256, "split": "heldout", "tag": "energy", "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b", "cmd": "benchmarks/scripts/bench_decode.py --model /Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b --backend titan --capacity 16 --nocache --predictor probe --m 8 --traces benchmarks/raw/gateC/traces_olmoe_v2_laptop --families general,code --split heldout --repeats 3 --max-tokens 256 --tag energy --out benchmarks/raw/gateG/energy_laptop/bench.jsonl", "env": {"host": "MacBookPro-de-simon-pierre-2.local", "chip": "Apple M5 Max", "ram_gb": 48, "macos": "27.0", "python": "3.12.13", "mlx": "0.32.4", "mlx_lm": "0.32.2", "numpy": "2.5.4", "git": "e0179f6"}, "seed": 0, "warmup": false, "repeat": 0, "family": "general", "prompt": "Summarize the history of the Rideau Canal in one paragraph.", "ts": "2026-11-01T01:30:45", "prompt_tokens": 14, "gen_tokens": 256, "ttft_ms": 307.55045800469816, "prompt_tps": 45.69629934715006, "gen_tps": 36.686457870160076, "decode_tps_wall": 36.54326812941098, "itl_ms_p50": 27.61320793069899, "itl_ms_p95": 35.31170799396932, "itl_ms_p99": 41.34791693650186, "itl_ms_mean": 27.36482124309987, "total_s": 7.285579874995165, "peak_mem_gb": 1.753644552, "text": "\n\nThe Rideau Canal, officially opened in 1832, was constructed by the British to connect Ottawa to Montreal via the Ottawa River. It was designed by Colonel John By and took approximately three years ", "t_start_epoch": 1793514638.644236, "t_end_epoch": 1793514645.9310138, "t_first_epoch": 1793514638.9517865, "cache_hit_rate": 0.8069263563627869, "cache_misses": 6612, "cache_hits": 27634, "stall_s": 2.490456422092393, "hits_device": 0, "layers_complete": 0, "prefetch": {"issued": 16192, "installed": 16197, "late": 740, "dup": 15400, "predictor": "probe", "m": 8}, "host_time_s": {"sync": 3.88708005938679, "prefetch": 0.6125109859276563, "ensure": 2.402447963017039, "build": 0.06990473286714405}, "ssd_bytes": 80702078976, "ssd_read_s": 10.351799859432504}

modified benchmarks/raw/gateG/energy_laptop/power.jsonl +257 −0

+ {"t": 1793514382.3597271, "elapsed_ns": 1020930916, "cpu_mW": 33065.9, "gpu_mW": 921.708, "ane_mW": 0.0, "combined_mW": 33987.6, "thermal_pressure": "Nominal"}
+ {"t": 1793514383.381633, "elapsed_ns": 1022480000, "cpu_mW": 33172.3, "gpu_mW": 973.124, "ane_mW": 0.0, "combined_mW": 34145.4, "thermal_pressure": "Nominal"}
+ {"t": 1793514384.4036071, "elapsed_ns": 1021607958, "cpu_mW": 31626.6, "gpu_mW": 989.616, "ane_mW": 0.0, "combined_mW": 32616.2, "thermal_pressure": "Nominal"}
++{"t": 1793514385.4265668, "elapsed_ns": 1021478833, "cpu_mW": 32083.9, "gpu_mW": 1013.24, "ane_mW": 0.0, "combined_mW": 33097.1, "thermal_pressure": "Nominal"}
++{"t": 1793514386.4477959, "elapsed_ns": 1022218583, "cpu_mW": 29769.6, "gpu_mW": 969.46, "ane_mW": 0.0, "combined_mW": 30739.0, "thermal_pressure": "Nominal"}
++{"t": 1793514388.4907782, "elapsed_ns": 1022043125, "cpu_mW": 24989.2, "gpu_mW": 1.95686, "ane_mW": 0.0, "combined_mW": 24991.1, "thermal_pressure": "Nominal"}
++{"t": 1793514388.4927402, "elapsed_ns": 1022142625, "cpu_mW": 22316.8, "gpu_mW": 1948.85, "ane_mW": 0.0, "combined_mW": 24265.7, "thermal_pressure": "Nominal"}
++{"t": 1793514389.5131981, "elapsed_ns": 1021520708, "cpu_mW": 22422.5, "gpu_mW": 1148.29, "ane_mW": 0.0, "combined_mW": 23570.7, "thermal_pressure": "Nominal"}
++{"t": 1793514390.537643, "elapsed_ns": 1021666750, "cpu_mW": 23418.6, "gpu_mW": 1075.69, "ane_mW": 0.0, "combined_mW": 24494.3, "thermal_pressure": "Nominal"}
++{"t": 1793514391.564992, "elapsed_ns": 1022320541, "cpu_mW": 26876.1, "gpu_mW": 1052.51, "ane_mW": 0.0, "combined_mW": 27928.6, "thermal_pressure": "Nominal"}
++{"t": 1793514392.58288, "elapsed_ns": 1023332375, "cpu_mW": 20500.7, "gpu_mW": 1114.01, "ane_mW": 10.7492, "combined_mW": 21625.4, "thermal_pressure": "Nominal"}
++{"t": 1793514393.605533, "elapsed_ns": 1024985291, "cpu_mW": 17832.5, "gpu_mW": 937.574, "ane_mW": 0.0, "combined_mW": 18770.0, "thermal_pressure": "Nominal"}
++{"t": 1793514394.627129, "elapsed_ns": 1021616375, "cpu_mW": 16964.3, "gpu_mW": 1036.59, "ane_mW": 0.978841, "combined_mW": 18001.9, "thermal_pressure": "Nominal"}
++{"t": 1793514395.64869, "elapsed_ns": 1022121958, "cpu_mW": 20260.8, "gpu_mW": 1011.62, "ane_mW": 942.158, "combined_mW": 22214.6, "thermal_pressure": "Nominal"}
++{"t": 1793514397.691945, "elapsed_ns": 1021705166, "cpu_mW": 20638.0, "gpu_mW": 983.65, "ane_mW": 326.904, "combined_mW": 21948.6, "thermal_pressure": "Nominal"}
++{"t": 1793514397.693907, "elapsed_ns": 1021223958, "cpu_mW": 18150.8, "gpu_mW": 865.628, "ane_mW": 74.4205, "combined_mW": 19090.8, "thermal_pressure": "Nominal"}
++{"t": 1793514398.7125518, "elapsed_ns": 1021437250, "cpu_mW": 18890.0, "gpu_mW": 1095.52, "ane_mW": 60.6988, "combined_mW": 20046.3, "thermal_pressure": "Nominal"}
++{"t": 1793514399.733614, "elapsed_ns": 1021598916, "cpu_mW": 14279.6, "gpu_mW": 1336.14, "ane_mW": 0.978858, "combined_mW": 15616.7, "thermal_pressure": "Nominal"}
++{"t": 1793514400.755388, "elapsed_ns": 1021273958, "cpu_mW": 11530.7, "gpu_mW": 1264.11, "ane_mW": 0.979169, "combined_mW": 12795.8, "thermal_pressure": "Nominal"}
++{"t": 1793514401.77793, "elapsed_ns": 1021503583, "cpu_mW": 13151.2, "gpu_mW": 1193.34, "ane_mW": 0.0, "combined_mW": 14344.5, "thermal_pressure": "Nominal"}
++{"t": 1793514402.7992268, "elapsed_ns": 1021795875, "cpu_mW": 14334.6, "gpu_mW": 1358.39, "ane_mW": 0.978669, "combined_mW": 15693.9, "thermal_pressure": "Nominal"}
++{"t": 1793514403.821048, "elapsed_ns": 1021719499, "cpu_mW": 15825.3, "gpu_mW": 1177.43, "ane_mW": 0.978742, "combined_mW": 17003.7, "thermal_pressure": "Nominal"}
++{"t": 1793514405.862755, "elapsed_ns": 1021391750, "cpu_mW": 15437.8, "gpu_mW": 1190.53, "ane_mW": 0.0, "combined_mW": 16628.3, "thermal_pressure": "Nominal"}
++{"t": 1793514405.864445, "elapsed_ns": 1021392541, "cpu_mW": 17193.2, "gpu_mW": 1105.35, "ane_mW": 0.979056, "combined_mW": 18299.5, "thermal_pressure": "Nominal"}
++{"t": 1793514406.884599, "elapsed_ns": 1021353083, "cpu_mW": 17691.2, "gpu_mW": 1013.36, "ane_mW": 0.979093, "combined_mW": 18705.6, "thermal_pressure": "Nominal"}
++{"t": 1793514407.907895, "elapsed_ns": 1021093416, "cpu_mW": 12793.1, "gpu_mW": 1262.37, "ane_mW": 0.0, "combined_mW": 14055.5, "thermal_pressure": "Nominal"}
++{"t": 1793514408.928425, "elapsed_ns": 1022192500, "cpu_mW": 15325.9, "gpu_mW": 1120.14, "ane_mW": 0.978289, "combined_mW": 16447.0, "thermal_pressure": "Nominal"}
++{"t": 1793514409.950219, "elapsed_ns": 1022046958, "cpu_mW": 16094.2, "gpu_mW": 1130.09, "ane_mW": 0.0, "combined_mW": 17224.3, "thermal_pressure": "Nominal"}
++{"t": 1793514410.9723232, "elapsed_ns": 1021888666, "cpu_mW": 14387.1, "gpu_mW": 1100.9, "ane_mW": 0.0, "combined_mW": 15488.0, "thermal_pressure": "Nominal"}
++{"t": 1793514411.995709, "elapsed_ns": 1021708958, "cpu_mW": 16325.6, "gpu_mW": 1093.27, "ane_mW": 0.0, "combined_mW": 17418.9, "thermal_pressure": "Nominal"}
++{"t": 1793514413.016373, "elapsed_ns": 1022038208, "cpu_mW": 21400.4, "gpu_mW": 5.87062, "ane_mW": 0.0, "combined_mW": 21406.2, "thermal_pressure": "Nominal"}
++{"t": 1793514415.059744, "elapsed_ns": 1022024208, "cpu_mW": 16490.8, "gpu_mW": 2297.4, "ane_mW": 0.0, "combined_mW": 18788.2, "thermal_pressure": "Nominal"}
++{"t": 1793514415.061377, "elapsed_ns": 1021963916, "cpu_mW": 15053.4, "gpu_mW": 835.646, "ane_mW": 0.0, "combined_mW": 15889.0, "thermal_pressure": "Nominal"}
++{"t": 1793514416.081883, "elapsed_ns": 1021660875, "cpu_mW": 10158.9, "gpu_mW": 911.261, "ane_mW": 0.0, "combined_mW": 11070.2, "thermal_pressure": "Nominal"}
++{"t": 1793514417.102023, "elapsed_ns": 1021549083, "cpu_mW": 13667.5, "gpu_mW": 738.095, "ane_mW": 0.0, "combined_mW": 14405.6, "thermal_pressure": "Nominal"}
++{"t": 1793514418.123428, "elapsed_ns": 1021050916, "cpu_mW": 13254.0, "gpu_mW": 760.001, "ane_mW": 0.0, "combined_mW": 14014.0, "thermal_pressure": "Nominal"}
++{"t": 1793514419.145301, "elapsed_ns": 1021496166, "cpu_mW": 14147.9, "gpu_mW": 754.775, "ane_mW": 0.0, "combined_mW": 14902.7, "thermal_pressure": "Nominal"}
++{"t": 1793514420.166976, "elapsed_ns": 1021468083, "cpu_mW": 15206.5, "gpu_mW": 1065.13, "ane_mW": 0.0, "combined_mW": 16271.7, "thermal_pressure": "Nominal"}
++{"t": 1793514422.2113018, "elapsed_ns": 1021702500, "cpu_mW": 15594.6, "gpu_mW": 1156.89, "ane_mW": 0.0, "combined_mW": 16751.5, "thermal_pressure": "Nominal"}
++{"t": 1793514422.2133288, "elapsed_ns": 1021771000, "cpu_mW": 16176.8, "gpu_mW": 1088.31, "ane_mW": 0.0, "combined_mW": 17265.1, "thermal_pressure": "Nominal"}
++{"t": 1793514423.232272, "elapsed_ns": 1021953249, "cpu_mW": 17117.2, "gpu_mW": 1056.8, "ane_mW": 0.0, "combined_mW": 18174.0, "thermal_pressure": "Nominal"}
++{"t": 1793514424.25409, "elapsed_ns": 1021522708, "cpu_mW": 16538.1, "gpu_mW": 1007.32, "ane_mW": 0.0, "combined_mW": 17545.4, "thermal_pressure": "Nominal"}
++{"t": 1793514425.275063, "elapsed_ns": 1021337083, "cpu_mW": 15063.6, "gpu_mW": 949.735, "ane_mW": 0.0, "combined_mW": 16013.3, "thermal_pressure": "Nominal"}
++{"t": 1793514426.2970428, "elapsed_ns": 1021887916, "cpu_mW": 11892.7, "gpu_mW": 1264.33, "ane_mW": 0.0, "combined_mW": 13157.0, "thermal_pressure": "Nominal"}
++{"t": 1793514427.3191779, "elapsed_ns": 1021075500, "cpu_mW": 11608.3, "gpu_mW": 1129.2, "ane_mW": 0.0, "combined_mW": 12737.5, "thermal_pressure": "Nominal"}
++{"t": 1793514428.341182, "elapsed_ns": 1022061000, "cpu_mW": 15656.6, "gpu_mW": 1096.8, "ane_mW": 0.0, "combined_mW": 16753.4, "thermal_pressure": "Nominal"}
++{"t": 1793514430.3845818, "elapsed_ns": 1022117250, "cpu_mW": 15581.4, "gpu_mW": 1116.31, "ane_mW": 0.0, "combined_mW": 16697.7, "thermal_pressure": "Nominal"}
++{"t": 1793514430.386417, "elapsed_ns": 1021989500, "cpu_mW": 15313.3, "gpu_mW": 1125.26, "ane_mW": 0.0, "combined_mW": 16438.5, "thermal_pressure": "Nominal"}
++{"t": 1793514431.406562, "elapsed_ns": 1021725541, "cpu_mW": 16490.7, "gpu_mW": 1092.27, "ane_mW": 0.0, "combined_mW": 17583.0, "thermal_pressure": "Nominal"}
++{"t": 1793514432.4275572, "elapsed_ns": 1021943125, "cpu_mW": 17140.9, "gpu_mW": 1175.21, "ane_mW": 0.0, "combined_mW": 18316.1, "thermal_pressure": "Nominal"}
++{"t": 1793514433.44819, "elapsed_ns": 1020863916, "cpu_mW": 18794.9, "gpu_mW": 1031.48, "ane_mW": 0.0, "combined_mW": 19826.3, "thermal_pressure": "Nominal"}
++{"t": 1793514434.470166, "elapsed_ns": 1021013916, "cpu_mW": 17423.9, "gpu_mW": 916.736, "ane_mW": 0.0, "combined_mW": 18340.6, "thermal_pressure": "Nominal"}
++{"t": 1793514435.492258, "elapsed_ns": 1021749208, "cpu_mW": 17873.3, "gpu_mW": 998.288, "ane_mW": 0.0, "combined_mW": 18871.6, "thermal_pressure": "Nominal"}
++{"t": 1793514436.5140748, "elapsed_ns": 1021631791, "cpu_mW": 20340.0, "gpu_mW": 1009.17, "ane_mW": 0.0, "combined_mW": 21349.2, "thermal_pressure": "Nominal"}
++{"t": 1793514437.535533, "elapsed_ns": 1022006916, "cpu_mW": 19875.6, "gpu_mW": 1043.05, "ane_mW": 0.0, "combined_mW": 20918.6, "thermal_pressure": "Nominal"}
++{"t": 1793514439.579565, "elapsed_ns": 1021884250, "cpu_mW": 16247.4, "gpu_mW": 1093.08, "ane_mW": 0.0, "combined_mW": 17340.5, "thermal_pressure": "Nominal"}
++{"t": 1793514439.581311, "elapsed_ns": 1021793583, "cpu_mW": 17301.9, "gpu_mW": 1060.88, "ane_mW": 0.0, "combined_mW": 18362.8, "thermal_pressure": "Nominal"}
++{"t": 1793514440.6016781, "elapsed_ns": 1021849083, "cpu_mW": 16753.9, "gpu_mW": 1061.8, "ane_mW": 0.0, "combined_mW": 17815.7, "thermal_pressure": "Nominal"}
++{"t": 1793514441.6226392, "elapsed_ns": 1021883249, "cpu_mW": 15540.9, "gpu_mW": 1078.4, "ane_mW": 0.0, "combined_mW": 16619.3, "thermal_pressure": "Nominal"}
++{"t": 1793514442.644178, "elapsed_ns": 1021513249, "cpu_mW": 14733.0, "gpu_mW": 1062.15, "ane_mW": 0.0, "combined_mW": 15795.2, "thermal_pressure": "Nominal"}
++{"t": 1793514443.665327, "elapsed_ns": 1021821499, "cpu_mW": 15329.5, "gpu_mW": 984.516, "ane_mW": 0.0, "combined_mW": 16314.0, "thermal_pressure": "Nominal"}
++{"t": 1793514444.68732, "elapsed_ns": 1020946833, "cpu_mW": 14259.3, "gpu_mW": 1212.6, "ane_mW": 0.0, "combined_mW": 15471.9, "thermal_pressure": "Nominal"}
++{"t": 1793514445.709019, "elapsed_ns": 1021898750, "cpu_mW": 14109.0, "gpu_mW": 1228.11, "ane_mW": 0.0, "combined_mW": 15337.1, "thermal_pressure": "Nominal"}
++{"t": 1793514447.7523189, "elapsed_ns": 1021779916, "cpu_mW": 14862.3, "gpu_mW": 1190.08, "ane_mW": 0.0, "combined_mW": 16052.4, "thermal_pressure": "Nominal"}
++{"t": 1793514447.754268, "elapsed_ns": 1021237291, "cpu_mW": 13904.7, "gpu_mW": 1178.96, "ane_mW": 0.0, "combined_mW": 15083.7, "thermal_pressure": "Nominal"}
++{"t": 1793514448.7734919, "elapsed_ns": 1021718458, "cpu_mW": 13841.4, "gpu_mW": 1134.36, "ane_mW": 0.0, "combined_mW": 14975.7, "thermal_pressure": "Nominal"}
++{"t": 1793514449.79485, "elapsed_ns": 1021626958, "cpu_mW": 16962.2, "gpu_mW": 1123.7, "ane_mW": 0.0, "combined_mW": 18085.9, "thermal_pressure": "Nominal"}
++{"t": 1793514450.817858, "elapsed_ns": 1021582166, "cpu_mW": 16915.9, "gpu_mW": 1081.66, "ane_mW": 0.0, "combined_mW": 17997.6, "thermal_pressure": "Nominal"}
++{"t": 1793514451.8394592, "elapsed_ns": 1021603541, "cpu_mW": 14946.1, "gpu_mW": 932.847, "ane_mW": 0.0, "combined_mW": 15879.0, "thermal_pressure": "Nominal"}
++{"t": 1793514452.861504, "elapsed_ns": 1022583500, "cpu_mW": 12901.6, "gpu_mW": 1180.34, "ane_mW": 0.0, "combined_mW": 14082.0, "thermal_pressure": "Nominal"}
++{"t": 1793514453.8831072, "elapsed_ns": 1022009000, "cpu_mW": 14385.4, "gpu_mW": 1157.52, "ane_mW": 0.0, "combined_mW": 15542.9, "thermal_pressure": "Nominal"}
++{"t": 1793514455.927351, "elapsed_ns": 1021453708, "cpu_mW": 16403.1, "gpu_mW": 1107.25, "ane_mW": 0.0, "combined_mW": 17510.3, "thermal_pressure": "Nominal"}
++{"t": 1793514455.929131, "elapsed_ns": 1021713708, "cpu_mW": 22752.9, "gpu_mW": 1042.37, "ane_mW": 0.0, "combined_mW": 23795.3, "thermal_pressure": "Nominal"}
++{"t": 1793514456.950163, "elapsed_ns": 1022238875, "cpu_mW": 20376.8, "gpu_mW": 1113.24, "ane_mW": 0.0, "combined_mW": 21490.1, "thermal_pressure": "Nominal"}
++{"t": 1793514457.971405, "elapsed_ns": 1022243583, "cpu_mW": 13198.4, "gpu_mW": 1191.5, "ane_mW": 0.0, "combined_mW": 14389.9, "thermal_pressure": "Nominal"}
++{"t": 1793514458.993207, "elapsed_ns": 1022400000, "cpu_mW": 12345.5, "gpu_mW": 1033.84, "ane_mW": 0.0, "combined_mW": 13379.3, "thermal_pressure": "Nominal"}
++{"t": 1793514460.0145972, "elapsed_ns": 1021578166, "cpu_mW": 15596.5, "gpu_mW": 1049.36, "ane_mW": 0.0, "combined_mW": 16645.8, "thermal_pressure": "Nominal"}
++{"t": 1793514461.038023, "elapsed_ns": 1021601208, "cpu_mW": 15055.8, "gpu_mW": 1111.0, "ane_mW": 0.0, "combined_mW": 16166.8, "thermal_pressure": "Nominal"}
++{"t": 1793514462.059385, "elapsed_ns": 1021782875, "cpu_mW": 13829.7, "gpu_mW": 1145.06, "ane_mW": 0.0, "combined_mW": 14974.8, "thermal_pressure": "Nominal"}
++{"t": 1793514464.1031199, "elapsed_ns": 1022869250, "cpu_mW": 17346.3, "gpu_mW": 1100.82, "ane_mW": 0.0, "combined_mW": 18447.1, "thermal_pressure": "Nominal"}
++{"t": 1793514464.104785, "elapsed_ns": 1021913666, "cpu_mW": 16754.8, "gpu_mW": 1151.76, "ane_mW": 0.0, "combined_mW": 17906.6, "thermal_pressure": "Nominal"}
++{"t": 1793514465.1260831, "elapsed_ns": 1021606291, "cpu_mW": 12847.4, "gpu_mW": 1182.45, "ane_mW": 0.0, "combined_mW": 14029.9, "thermal_pressure": "Nominal"}
++{"t": 1793514466.1469, "elapsed_ns": 1022077458, "cpu_mW": 13046.0, "gpu_mW": 1218.11, "ane_mW": 0.0, "combined_mW": 14264.1, "thermal_pressure": "Nominal"}
++{"t": 1793514467.170579, "elapsed_ns": 1022134000, "cpu_mW": 16310.0, "gpu_mW": 1088.9, "ane_mW": 0.0, "combined_mW": 17398.9, "thermal_pressure": "Nominal"}
++{"t": 1793514468.19178, "elapsed_ns": 1021845625, "cpu_mW": 16206.0, "gpu_mW": 1111.71, "ane_mW": 0.0, "combined_mW": 17317.7, "thermal_pressure": "Nominal"}
++{"t": 1793514469.213527, "elapsed_ns": 1022946916, "cpu_mW": 16437.8, "gpu_mW": 1133.98, "ane_mW": 0.0, "combined_mW": 17571.8, "thermal_pressure": "Nominal"}
++{"t": 1793514470.2353349, "elapsed_ns": 1021579958, "cpu_mW": 13671.0, "gpu_mW": 1229.47, "ane_mW": 0.0, "combined_mW": 14900.4, "thermal_pressure": "Nominal"}
++{"t": 1793514472.277964, "elapsed_ns": 1021693624, "cpu_mW": 14884.1, "gpu_mW": 1229.33, "ane_mW": 0.0, "combined_mW": 16113.4, "thermal_pressure": "Nominal"}
++{"t": 1793514472.279568, "elapsed_ns": 1021562708, "cpu_mW": 14678.5, "gpu_mW": 1228.51, "ane_mW": 0.0, "combined_mW": 15907.0, "thermal_pressure": "Nominal"}
++{"t": 1793514473.2998478, "elapsed_ns": 1021710333, "cpu_mW": 13312.0, "gpu_mW": 1293.91, "ane_mW": 0.0, "combined_mW": 14605.9, "thermal_pressure": "Nominal"}
++{"t": 1793514474.321727, "elapsed_ns": 1021494416, "cpu_mW": 11549.7, "gpu_mW": 1273.62, "ane_mW": 0.0, "combined_mW": 12823.4, "thermal_pressure": "Nominal"}
++{"t": 1793514475.344286, "elapsed_ns": 1021627666, "cpu_mW": 16016.6, "gpu_mW": 1178.51, "ane_mW": 0.0, "combined_mW": 17195.1, "thermal_pressure": "Nominal"}
++{"t": 1793514476.3647418, "elapsed_ns": 1022014208, "cpu_mW": 13622.1, "gpu_mW": 1021.51, "ane_mW": 0.0, "combined_mW": 14643.6, "thermal_pressure": "Nominal"}
++{"t": 1793514477.387168, "elapsed_ns": 1021841708, "cpu_mW": 12423.6, "gpu_mW": 1263.41, "ane_mW": 0.0, "combined_mW": 13687.1, "thermal_pressure": "Nominal"}
++{"t": 1793514478.408648, "elapsed_ns": 1021330000, "cpu_mW": 13194.6, "gpu_mW": 1225.85, "ane_mW": 0.0, "combined_mW": 14420.4, "thermal_pressure": "Nominal"}
++{"t": 1793514480.4519072, "elapsed_ns": 1021926666, "cpu_mW": 17705.8, "gpu_mW": 1127.28, "ane_mW": 0.0, "combined_mW": 18833.1, "thermal_pressure": "Nominal"}
++{"t": 1793514480.453556, "elapsed_ns": 1021187875, "cpu_mW": 14033.7, "gpu_mW": 1217.21, "ane_mW": 0.0, "combined_mW": 15250.9, "thermal_pressure": "Nominal"}
++{"t": 1793514481.4732, "elapsed_ns": 1021968833, "cpu_mW": 11436.7, "gpu_mW": 1264.23, "ane_mW": 0.0, "combined_mW": 12701.0, "thermal_pressure": "Nominal"}
++{"t": 1793514482.494974, "elapsed_ns": 1021597291, "cpu_mW": 12738.9, "gpu_mW": 1126.67, "ane_mW": 0.0, "combined_mW": 13865.5, "thermal_pressure": "Nominal"}
++{"t": 1793514483.515433, "elapsed_ns": 1021494083, "cpu_mW": 11437.2, "gpu_mW": 1239.36, "ane_mW": 0.0, "combined_mW": 12676.5, "thermal_pressure": "Nominal"}
++{"t": 1793514484.536913, "elapsed_ns": 1020892958, "cpu_mW": 7864.68, "gpu_mW": 594.578, "ane_mW": 0.0, "combined_mW": 8459.26, "thermal_pressure": "Nominal"}
++{"t": 1793514485.557188, "elapsed_ns": 1021537041, "cpu_mW": 5633.67, "gpu_mW": 57.7561, "ane_mW": 0.0, "combined_mW": 5691.42, "thermal_pressure": "Nominal"}
++{"t": 1793514487.6002078, "elapsed_ns": 1020647625, "cpu_mW": 4620.6, "gpu_mW": 51.9278, "ane_mW": 0.0, "combined_mW": 4672.52, "thermal_pressure": "Nominal"}
++{"t": 1793514487.601894, "elapsed_ns": 1021108541, "cpu_mW": 8648.44, "gpu_mW": 47.9871, "ane_mW": 0.0, "combined_mW": 8696.43, "thermal_pressure": "Nominal"}
++{"t": 1793514488.628555, "elapsed_ns": 1021436208, "cpu_mW": 5524.57, "gpu_mW": 42.0976, "ane_mW": 0.0, "combined_mW": 5566.67, "thermal_pressure": "Nominal"}
++{"t": 1793514489.644935, "elapsed_ns": 1022073458, "cpu_mW": 8911.3, "gpu_mW": 45.985, "ane_mW": 0.0, "combined_mW": 8957.28, "thermal_pressure": "Nominal"}
++{"t": 1793514490.6656032, "elapsed_ns": 1021647666, "cpu_mW": 5522.45, "gpu_mW": 51.877, "ane_mW": 0.0, "combined_mW": 5574.33, "thermal_pressure": "Nominal"}
++{"t": 1793514491.687382, "elapsed_ns": 1021523958, "cpu_mW": 5109.03, "gpu_mW": 48.9465, "ane_mW": 0.0, "combined_mW": 5157.98, "thermal_pressure": "Nominal"}
++{"t": 1793514492.7088158, "elapsed_ns": 1021930875, "cpu_mW": 6269.5, "gpu_mW": 41.0987, "ane_mW": 0.0, "combined_mW": 6310.6, "thermal_pressure": "Nominal"}
++{"t": 1793514493.7309928, "elapsed_ns": 1021636666, "cpu_mW": 9869.46, "gpu_mW": 56.7717, "ane_mW": 0.0, "combined_mW": 9926.23, "thermal_pressure": "Nominal"}
++{"t": 1793514495.773961, "elapsed_ns": 1021476875, "cpu_mW": 13784.9, "gpu_mW": 49.9277, "ane_mW": 0.0, "combined_mW": 13834.9, "thermal_pressure": "Nominal"}
++{"t": 1793514495.7756412, "elapsed_ns": 1022070541, "cpu_mW": 14115.5, "gpu_mW": 64.5748, "ane_mW": 0.0, "combined_mW": 14180.0, "thermal_pressure": "Nominal"}
++{"t": 1793514496.795663, "elapsed_ns": 1021417375, "cpu_mW": 14237.1, "gpu_mW": 58.7419, "ane_mW": 0.0, "combined_mW": 14295.8, "thermal_pressure": "Nominal"}
++{"t": 1793514497.817284, "elapsed_ns": 1021693500, "cpu_mW": 13863.3, "gpu_mW": 56.7685, "ane_mW": 0.0, "combined_mW": 13920.0, "thermal_pressure": "Nominal"}
++{"t": 1793514498.8393018, "elapsed_ns": 1021698541, "cpu_mW": 13827.0, "gpu_mW": 43.0655, "ane_mW": 0.0, "combined_mW": 13870.0, "thermal_pressure": "Nominal"}
++{"t": 1793514499.861644, "elapsed_ns": 1021565458, "cpu_mW": 19521.0, "gpu_mW": 56.7756, "ane_mW": 0.0, "combined_mW": 19577.8, "thermal_pressure": "Nominal"}
++{"t": 1793514500.88336, "elapsed_ns": 1022008291, "cpu_mW": 19885.4, "gpu_mW": 69.4711, "ane_mW": 0.0, "combined_mW": 19954.8, "thermal_pressure": "Nominal"}
++{"t": 1793514501.9054248, "elapsed_ns": 1022264291, "cpu_mW": 19776.7, "gpu_mW": 58.6932, "ane_mW": 0.0, "combined_mW": 19835.4, "thermal_pressure": "Nominal"}
++{"t": 1793514503.94885, "elapsed_ns": 1021672541, "cpu_mW": 19442.6, "gpu_mW": 68.5151, "ane_mW": 0.0, "combined_mW": 19511.1, "thermal_pressure": "Nominal"}
++{"t": 1793514503.9505222, "elapsed_ns": 1022086083, "cpu_mW": 18984.7, "gpu_mW": 62.617, "ane_mW": 0.0, "combined_mW": 19047.3, "thermal_pressure": "Nominal"}
++{"t": 1793514504.970506, "elapsed_ns": 1021910041, "cpu_mW": 18686.6, "gpu_mW": 54.7993, "ane_mW": 0.0, "combined_mW": 18741.4, "thermal_pressure": "Nominal"}
++{"t": 1793514505.992467, "elapsed_ns": 1021381416, "cpu_mW": 18623.8, "gpu_mW": 62.6602, "ane_mW": 0.0, "combined_mW": 18686.5, "thermal_pressure": "Nominal"}
++{"t": 1793514507.014581, "elapsed_ns": 1021875249, "cpu_mW": 19227.4, "gpu_mW": 60.6728, "ane_mW": 0.0, "combined_mW": 19288.1, "thermal_pressure": "Nominal"}
++{"t": 1793514508.03702, "elapsed_ns": 1021972249, "cpu_mW": 18336.1, "gpu_mW": 62.624, "ane_mW": 0.0, "combined_mW": 18398.7, "thermal_pressure": "Nominal"}
++{"t": 1793514509.058808, "elapsed_ns": 1022147291, "cpu_mW": 18492.4, "gpu_mW": 52.83, "ane_mW": 0.0, "combined_mW": 18545.3, "thermal_pressure": "Nominal"}
++{"t": 1793514510.0810182, "elapsed_ns": 1022380666, "cpu_mW": 18438.3, "gpu_mW": 68.4676, "ane_mW": 0.0, "combined_mW": 18506.8, "thermal_pressure": "Nominal"}
++{"t": 1793514512.123785, "elapsed_ns": 1021721874, "cpu_mW": 19911.5, "gpu_mW": 66.5543, "ane_mW": 0.0, "combined_mW": 19978.0, "thermal_pressure": "Nominal"}
++{"t": 1793514512.1259232, "elapsed_ns": 1022148541, "cpu_mW": 21309.0, "gpu_mW": 76.3098, "ane_mW": 0.0, "combined_mW": 21385.3, "thermal_pressure": "Nominal"}
++{"t": 1793514513.14463, "elapsed_ns": 1020759374, "cpu_mW": 22892.8, "gpu_mW": 70.5357, "ane_mW": 0.0, "combined_mW": 22963.3, "thermal_pressure": "Nominal"}
++{"t": 1793514514.166074, "elapsed_ns": 1020783583, "cpu_mW": 25540.2, "gpu_mW": 96.9843, "ane_mW": 0.0, "combined_mW": 25637.2, "thermal_pressure": "Nominal"}
++{"t": 1793514515.187936, "elapsed_ns": 1021559666, "cpu_mW": 26020.0, "gpu_mW": 75.3749, "ane_mW": 0.0, "combined_mW": 26095.4, "thermal_pressure": "Nominal"}
++{"t": 1793514516.20992, "elapsed_ns": 1021531708, "cpu_mW": 25808.3, "gpu_mW": 89.0819, "ane_mW": 0.0, "combined_mW": 25897.4, "thermal_pressure": "Nominal"}
++{"t": 1793514517.232925, "elapsed_ns": 1022469333, "cpu_mW": 26328.4, "gpu_mW": 89.9782, "ane_mW": 0.0, "combined_mW": 26418.4, "thermal_pressure": "Nominal"}
++{"t": 1793514518.254474, "elapsed_ns": 1021788375, "cpu_mW": 26758.0, "gpu_mW": 82.2088, "ane_mW": 0.0, "combined_mW": 26840.2, "thermal_pressure": "Nominal"}
++{"t": 1793514520.297753, "elapsed_ns": 1022129750, "cpu_mW": 27162.9, "gpu_mW": 90.0081, "ane_mW": 0.0, "combined_mW": 27252.9, "thermal_pressure": "Nominal"}
++{"t": 1793514520.299909, "elapsed_ns": 1021922000, "cpu_mW": 25562.6, "gpu_mW": 83.1766, "ane_mW": 0.0, "combined_mW": 25645.8, "thermal_pressure": "Nominal"}
++{"t": 1793514521.318976, "elapsed_ns": 1020875500, "cpu_mW": 25476.2, "gpu_mW": 82.2823, "ane_mW": 0.0, "combined_mW": 25558.5, "thermal_pressure": "Nominal"}
++{"t": 1793514522.340845, "elapsed_ns": 1021650583, "cpu_mW": 25642.8, "gpu_mW": 82.2199, "ane_mW": 0.0, "combined_mW": 25725.0, "thermal_pressure": "Nominal"}
++{"t": 1793514523.362674, "elapsed_ns": 1021897833, "cpu_mW": 25603.3, "gpu_mW": 85.1357, "ane_mW": 0.0, "combined_mW": 25688.5, "thermal_pressure": "Nominal"}
++{"t": 1793514524.3848588, "elapsed_ns": 1022047666, "cpu_mW": 26711.1, "gpu_mW": 76.3174, "ane_mW": 0.0, "combined_mW": 26787.4, "thermal_pressure": "Nominal"}
++{"t": 1793514525.4065568, "elapsed_ns": 1022037625, "cpu_mW": 26660.5, "gpu_mW": 76.3181, "ane_mW": 0.0, "combined_mW": 26736.8, "thermal_pressure": "Nominal"}
++{"t": 1793514526.429485, "elapsed_ns": 1021562041, "cpu_mW": 28996.8, "gpu_mW": 124.319, "ane_mW": 0.0, "combined_mW": 29121.1, "thermal_pressure": "Nominal"}
++{"t": 1793514527.450005, "elapsed_ns": 1022163208, "cpu_mW": 27119.9, "gpu_mW": 216.208, "ane_mW": 0.0, "combined_mW": 27336.1, "thermal_pressure": "Nominal"}
++{"t": 1793514529.493999, "elapsed_ns": 1021304166, "cpu_mW": 25635.8, "gpu_mW": 96.9349, "ane_mW": 0.0, "combined_mW": 25732.8, "thermal_pressure": "Nominal"}
++{"t": 1793514529.495698, "elapsed_ns": 1022108083, "cpu_mW": 25182.3, "gpu_mW": 96.8586, "ane_mW": 0.0, "combined_mW": 25279.1, "thermal_pressure": "Nominal"}
++{"t": 1793514530.516442, "elapsed_ns": 1022056333, "cpu_mW": 24693.4, "gpu_mW": 81.2088, "ane_mW": 0.0, "combined_mW": 24774.6, "thermal_pressure": "Nominal"}
++{"t": 1793514531.5377269, "elapsed_ns": 1022262875, "cpu_mW": 25314.4, "gpu_mW": 85.1053, "ane_mW": 0.0, "combined_mW": 25399.5, "thermal_pressure": "Nominal"}
++{"t": 1793514532.559656, "elapsed_ns": 1021201666, "cpu_mW": 25962.6, "gpu_mW": 78.3391, "ane_mW": 0.0, "combined_mW": 26040.9, "thermal_pressure": "Nominal"}
++{"t": 1793514533.581193, "elapsed_ns": 1021733333, "cpu_mW": 26784.9, "gpu_mW": 81.2345, "ane_mW": 0.0, "combined_mW": 26866.1, "thermal_pressure": "Nominal"}
++{"t": 1793514534.604172, "elapsed_ns": 1021031625, "cpu_mW": 26311.6, "gpu_mW": 96.9608, "ane_mW": 0.0, "combined_mW": 26408.6, "thermal_pressure": "Nominal"}
++{"t": 1793514535.625111, "elapsed_ns": 1022633458, "cpu_mW": 25383.5, "gpu_mW": 94.8531, "ane_mW": 0.0, "combined_mW": 25478.3, "thermal_pressure": "Nominal"}
++{"t": 1793514537.667336, "elapsed_ns": 1022133458, "cpu_mW": 25844.0, "gpu_mW": 89.0295, "ane_mW": 0.0, "combined_mW": 25933.0, "thermal_pressure": "Nominal"}
++{"t": 1793514537.669191, "elapsed_ns": 1020990416, "cpu_mW": 25004.2, "gpu_mW": 95.9852, "ane_mW": 0.0, "combined_mW": 25100.1, "thermal_pressure": "Nominal"}
++{"t": 1793514538.69014, "elapsed_ns": 1021334083, "cpu_mW": 21614.9, "gpu_mW": 60.7049, "ane_mW": 0.0, "combined_mW": 21675.6, "thermal_pressure": "Nominal"}
++{"t": 1793514539.711041, "elapsed_ns": 1022098916, "cpu_mW": 22790.4, "gpu_mW": 78.2703, "ane_mW": 0.0, "combined_mW": 22868.6, "thermal_pressure": "Nominal"}
++{"t": 1793514540.734891, "elapsed_ns": 1021523374, "cpu_mW": 22407.7, "gpu_mW": 93.9773, "ane_mW": 0.0, "combined_mW": 22501.7, "thermal_pressure": "Nominal"}
++{"t": 1793514541.75415, "elapsed_ns": 1022498833, "cpu_mW": 22451.9, "gpu_mW": 79.2177, "ane_mW": 0.0, "combined_mW": 22531.1, "thermal_pressure": "Nominal"}
++{"t": 1793514542.7761018, "elapsed_ns": 1021429083, "cpu_mW": 22847.4, "gpu_mW": 93.986, "ane_mW": 0.0, "combined_mW": 22941.4, "thermal_pressure": "Nominal"}
++{"t": 1793514543.798282, "elapsed_ns": 1021647958, "cpu_mW": 22011.5, "gpu_mW": 82.2201, "ane_mW": 0.0, "combined_mW": 22093.7, "thermal_pressure": "Nominal"}
++{"t": 1793514544.820381, "elapsed_ns": 1022020166, "cpu_mW": 23311.7, "gpu_mW": 96.867, "ane_mW": 0.0, "combined_mW": 23408.5, "thermal_pressure": "Nominal"}
++{"t": 1793514546.863437, "elapsed_ns": 1022159541, "cpu_mW": 21955.5, "gpu_mW": 85.1139, "ane_mW": 0.0, "combined_mW": 22040.6, "thermal_pressure": "Nominal"}
++{"t": 1793514546.865845, "elapsed_ns": 1021968875, "cpu_mW": 22660.2, "gpu_mW": 80.2373, "ane_mW": 0.0, "combined_mW": 22740.4, "thermal_pressure": "Nominal"}
++{"t": 1793514547.8850272, "elapsed_ns": 1021076041, "cpu_mW": 22223.6, "gpu_mW": 84.2249, "ane_mW": 0.0, "combined_mW": 22307.8, "thermal_pressure": "Nominal"}
++{"t": 1793514548.9071202, "elapsed_ns": 1021508083, "cpu_mW": 23728.6, "gpu_mW": 78.3156, "ane_mW": 0.0, "combined_mW": 23807.0, "thermal_pressure": "Nominal"}
++{"t": 1793514549.9292119, "elapsed_ns": 1022199083, "cpu_mW": 22779.3, "gpu_mW": 81.1975, "ane_mW": 0.0, "combined_mW": 22860.5, "thermal_pressure": "Nominal"}
++{"t": 1793514550.950944, "elapsed_ns": 1021774541, "cpu_mW": 21533.1, "gpu_mW": 73.4017, "ane_mW": 0.0, "combined_mW": 21606.5, "thermal_pressure": "Nominal"}
++{"t": 1793514551.971725, "elapsed_ns": 1021894958, "cpu_mW": 23978.0, "gpu_mW": 75.3502, "ane_mW": 0.0, "combined_mW": 24053.4, "thermal_pressure": "Nominal"}
++{"t": 1793514552.993418, "elapsed_ns": 1021023375, "cpu_mW": 21493.1, "gpu_mW": 88.1469, "ane_mW": 0.0, "combined_mW": 21581.3, "thermal_pressure": "Nominal"}
++{"t": 1793514555.037471, "elapsed_ns": 1020970624, "cpu_mW": 22527.6, "gpu_mW": 69.5417, "ane_mW": 0.0, "combined_mW": 22597.1, "thermal_pressure": "Nominal"}
++{"t": 1793514555.03964, "elapsed_ns": 1022603208, "cpu_mW": 22440.8, "gpu_mW": 65.5191, "ane_mW": 0.0, "combined_mW": 22506.3, "thermal_pressure": "Nominal"}
++{"t": 1793514556.058659, "elapsed_ns": 1021719166, "cpu_mW": 23632.7, "gpu_mW": 74.3844, "ane_mW": 0.0, "combined_mW": 23707.1, "thermal_pressure": "Nominal"}
++{"t": 1793514557.0797818, "elapsed_ns": 1021654958, "cpu_mW": 22088.7, "gpu_mW": 69.4951, "ane_mW": 0.0, "combined_mW": 22158.2, "thermal_pressure": "Nominal"}
++{"t": 1793514558.10219, "elapsed_ns": 1021193541, "cpu_mW": 23452.0, "gpu_mW": 83.2359, "ane_mW": 0.0, "combined_mW": 23535.2, "thermal_pressure": "Nominal"}
++{"t": 1793514559.122941, "elapsed_ns": 1021772874, "cpu_mW": 22866.1, "gpu_mW": 79.274, "ane_mW": 0.0, "combined_mW": 22945.4, "thermal_pressure": "Nominal"}
++{"t": 1793514560.144566, "elapsed_ns": 1020782083, "cpu_mW": 22559.2, "gpu_mW": 85.2288, "ane_mW": 0.0, "combined_mW": 22644.4, "thermal_pressure": "Nominal"}
++{"t": 1793514561.166204, "elapsed_ns": 1021794249, "cpu_mW": 23635.9, "gpu_mW": 90.0377, "ane_mW": 0.0, "combined_mW": 23725.9, "thermal_pressure": "Nominal"}
++{"t": 1793514563.209589, "elapsed_ns": 1022047541, "cpu_mW": 25811.9, "gpu_mW": 94.9075, "ane_mW": 0.0, "combined_mW": 25906.8, "thermal_pressure": "Nominal"}
++{"t": 1793514563.2113411, "elapsed_ns": 1021205916, "cpu_mW": 22840.6, "gpu_mW": 78.3388, "ane_mW": 0.0, "combined_mW": 22919.0, "thermal_pressure": "Nominal"}
++{"t": 1793514564.230853, "elapsed_ns": 1021970541, "cpu_mW": 23941.0, "gpu_mW": 91.9792, "ane_mW": 0.0, "combined_mW": 24033.0, "thermal_pressure": "Nominal"}
++{"t": 1793514565.252658, "elapsed_ns": 1020941416, "cpu_mW": 22888.7, "gpu_mW": 76.4001, "ane_mW": 0.0, "combined_mW": 22965.1, "thermal_pressure": "Nominal"}
++{"t": 1793514566.273998, "elapsed_ns": 1021302541, "cpu_mW": 22260.8, "gpu_mW": 95.9559, "ane_mW": 0.0, "combined_mW": 22356.7, "thermal_pressure": "Nominal"}
++{"t": 1793514567.295889, "elapsed_ns": 1022173416, "cpu_mW": 23539.1, "gpu_mW": 98.8091, "ane_mW": 0.0, "combined_mW": 23637.9, "thermal_pressure": "Nominal"}
++{"t": 1793514568.319764, "elapsed_ns": 1022106166, "cpu_mW": 21662.1, "gpu_mW": 92.9453, "ane_mW": 0.0, "combined_mW": 21755.1, "thermal_pressure": "Nominal"}
++{"t": 1793514569.3413289, "elapsed_ns": 1021949708, "cpu_mW": 23942.5, "gpu_mW": 78.2817, "ane_mW": 0.0, "combined_mW": 24020.8, "thermal_pressure": "Nominal"}
++{"t": 1793514570.363188, "elapsed_ns": 1023308958, "cpu_mW": 19676.4, "gpu_mW": 88.9272, "ane_mW": 0.0, "combined_mW": 19765.3, "thermal_pressure": "Nominal"}
++{"t": 1793514572.405995, "elapsed_ns": 1021976416, "cpu_mW": 22405.6, "gpu_mW": 91.9786, "ane_mW": 0.0, "combined_mW": 22497.6, "thermal_pressure": "Nominal"}
++{"t": 1793514572.4076219, "elapsed_ns": 1020872708, "cpu_mW": 22462.2, "gpu_mW": 80.3234, "ane_mW": 0.0, "combined_mW": 22542.5, "thermal_pressure": "Nominal"}
++{"t": 1793514573.4276311, "elapsed_ns": 1022055208, "cpu_mW": 23252.2, "gpu_mW": 89.0363, "ane_mW": 0.0, "combined_mW": 23341.2, "thermal_pressure": "Nominal"}
++{"t": 1793514574.449006, "elapsed_ns": 1021025458, "cpu_mW": 23627.2, "gpu_mW": 84.229, "ane_mW": 0.0, "combined_mW": 23711.5, "thermal_pressure": "Nominal"}
++{"t": 1793514575.471272, "elapsed_ns": 1021720250, "cpu_mW": 21855.3, "gpu_mW": 92.0017, "ane_mW": 0.0, "combined_mW": 21947.3, "thermal_pressure": "Nominal"}
++{"t": 1793514576.493875, "elapsed_ns": 1022116208, "cpu_mW": 22630.5, "gpu_mW": 73.3772, "ane_mW": 0.0, "combined_mW": 22703.9, "thermal_pressure": "Nominal"}
++{"t": 1793514577.514996, "elapsed_ns": 1022019250, "cpu_mW": 24012.3, "gpu_mW": 87.0825, "ane_mW": 0.0, "combined_mW": 24099.4, "thermal_pressure": "Nominal"}
++{"t": 1793514578.5362408, "elapsed_ns": 1021976124, "cpu_mW": 28597.5, "gpu_mW": 82.1937, "ane_mW": 0.0, "combined_mW": 28679.7, "thermal_pressure": "Nominal"}
++{"t": 1793514579.557139, "elapsed_ns": 1021521458, "cpu_mW": 27223.1, "gpu_mW": 96.9143, "ane_mW": 0.0, "combined_mW": 27320.0, "thermal_pressure": "Nominal"}
++{"t": 1793514581.5996008, "elapsed_ns": 1021027625, "cpu_mW": 29088.3, "gpu_mW": 256.604, "ane_mW": 0.0, "combined_mW": 29344.9, "thermal_pressure": "Nominal"}
++{"t": 1793514581.603251, "elapsed_ns": 1020728166, "cpu_mW": 26106.9, "gpu_mW": 471.232, "ane_mW": 0.0, "combined_mW": 26578.1, "thermal_pressure": "Nominal"}
++{"t": 1793514582.622281, "elapsed_ns": 1021810750, "cpu_mW": 26676.2, "gpu_mW": 311.212, "ane_mW": 0.0, "combined_mW": 26987.4, "thermal_pressure": "Nominal"}
++{"t": 1793514583.6432698, "elapsed_ns": 1020786958, "cpu_mW": 24951.3, "gpu_mW": 154.783, "ane_mW": 0.0, "combined_mW": 25106.1, "thermal_pressure": "Nominal"}
++{"t": 1793514584.665556, "elapsed_ns": 1021888458, "cpu_mW": 29507.1, "gpu_mW": 654.67, "ane_mW": 0.0, "combined_mW": 30161.8, "thermal_pressure": "Nominal"}
++{"t": 1793514585.688908, "elapsed_ns": 1021924916, "cpu_mW": 24335.4, "gpu_mW": 7978.08, "ane_mW": 0.0, "combined_mW": 32313.5, "thermal_pressure": "Nominal"}
++{"t": 1793514586.709344, "elapsed_ns": 1021219041, "cpu_mW": 25011.3, "gpu_mW": 394.626, "ane_mW": 0.0, "combined_mW": 25405.9, "thermal_pressure": "Nominal"}
++{"t": 1793514587.731756, "elapsed_ns": 1023015250, "cpu_mW": 24907.7, "gpu_mW": 101.66, "ane_mW": 0.0, "combined_mW": 25009.4, "thermal_pressure": "Nominal"}
++{"t": 1793514588.753751, "elapsed_ns": 1021932499, "cpu_mW": 29129.1, "gpu_mW": 2399.38, "ane_mW": 0.0, "combined_mW": 31528.5, "thermal_pressure": "Nominal"}
++{"t": 1793514590.7995431, "elapsed_ns": 1022791000, "cpu_mW": 28231.6, "gpu_mW": 3005.5, "ane_mW": 0.0, "combined_mW": 31237.1, "thermal_pressure": "Nominal"}
++{"t": 1793514590.801867, "elapsed_ns": 1021153583, "cpu_mW": 27854.8, "gpu_mW": 697.251, "ane_mW": 0.0, "combined_mW": 28552.0, "thermal_pressure": "Nominal"}
++{"t": 1793514591.819339, "elapsed_ns": 1022362333, "cpu_mW": 26885.8, "gpu_mW": 1284.28, "ane_mW": 0.0, "combined_mW": 28170.1, "thermal_pressure": "Nominal"}
++{"t": 1793514592.841732, "elapsed_ns": 1021977666, "cpu_mW": 25647.3, "gpu_mW": 591.011, "ane_mW": 0.0, "combined_mW": 26238.3, "thermal_pressure": "Nominal"}
++{"t": 1793514593.864361, "elapsed_ns": 1021584708, "cpu_mW": 26843.6, "gpu_mW": 805.611, "ane_mW": 0.0, "combined_mW": 27649.2, "thermal_pressure": "Nominal"}
++{"t": 1793514594.885059, "elapsed_ns": 1022235833, "cpu_mW": 28981.6, "gpu_mW": 402.06, "ane_mW": 0.0, "combined_mW": 29383.6, "thermal_pressure": "Nominal"}
++{"t": 1793514595.908468, "elapsed_ns": 1022254791, "cpu_mW": 25501.5, "gpu_mW": 466.616, "ane_mW": 0.0, "combined_mW": 25968.1, "thermal_pressure": "Nominal"}
++{"t": 1793514596.929143, "elapsed_ns": 1021760666, "cpu_mW": 25533.4, "gpu_mW": 1393.67, "ane_mW": 0.0, "combined_mW": 26927.0, "thermal_pressure": "Nominal"}
++{"t": 1793514598.973845, "elapsed_ns": 1022274375, "cpu_mW": 26681.7, "gpu_mW": 4379.45, "ane_mW": 0.0, "combined_mW": 31061.1, "thermal_pressure": "Nominal"}
++{"t": 1793514598.9761338, "elapsed_ns": 1021519666, "cpu_mW": 27560.9, "gpu_mW": 581.487, "ane_mW": 0.0, "combined_mW": 28142.4, "thermal_pressure": "Nominal"}
++{"t": 1793514599.996589, "elapsed_ns": 1022049916, "cpu_mW": 26450.8, "gpu_mW": 2766.01, "ane_mW": 0.0, "combined_mW": 29216.8, "thermal_pressure": "Nominal"}
++{"t": 1793514601.017697, "elapsed_ns": 1021806541, "cpu_mW": 23362.5, "gpu_mW": 2889.0, "ane_mW": 0.0, "combined_mW": 26251.5, "thermal_pressure": "Nominal"}
++{"t": 1793514602.038503, "elapsed_ns": 1021938250, "cpu_mW": 26732.5, "gpu_mW": 6007.21, "ane_mW": 0.0, "combined_mW": 32739.7, "thermal_pressure": "Nominal"}
++{"t": 1793514603.062836, "elapsed_ns": 1022131750, "cpu_mW": 21267.3, "gpu_mW": 4849.67, "ane_mW": 0.0, "combined_mW": 26117.0, "thermal_pressure": "Nominal"}
++{"t": 1793514604.084095, "elapsed_ns": 1020771708, "cpu_mW": 23382.3, "gpu_mW": 5966.07, "ane_mW": 0.0, "combined_mW": 29348.4, "thermal_pressure": "Nominal"}
++{"t": 1793514605.108299, "elapsed_ns": 1024470583, "cpu_mW": 19908.8, "gpu_mW": 1785.31, "ane_mW": 0.0, "combined_mW": 21694.1, "thermal_pressure": "Nominal"}
++{"t": 1793514606.130435, "elapsed_ns": 1022076166, "cpu_mW": 21146.2, "gpu_mW": 3732.6, "ane_mW": 0.0, "combined_mW": 24878.8, "thermal_pressure": "Nominal"}
++{"t": 1793514608.172689, "elapsed_ns": 1023282291, "cpu_mW": 23688.5, "gpu_mW": 9962.06, "ane_mW": 0.0, "combined_mW": 33650.5, "thermal_pressure": "Nominal"}
++{"t": 1793514608.174447, "elapsed_ns": 1022077375, "cpu_mW": 23678.2, "gpu_mW": 9913.14, "ane_mW": 0.0, "combined_mW": 33591.4, "thermal_pressure": "Nominal"}
++{"t": 1793514609.1942122, "elapsed_ns": 1022102583, "cpu_mW": 16287.0, "gpu_mW": 7.827, "ane_mW": 0.0, "combined_mW": 16294.8, "thermal_pressure": "Nominal"}
++{"t": 1793514610.2164102, "elapsed_ns": 1021012166, "cpu_mW": 14079.2, "gpu_mW": 3377.04, "ane_mW": 0.0, "combined_mW": 17456.2, "thermal_pressure": "Nominal"}
++{"t": 1793514611.238346, "elapsed_ns": 1021651958, "cpu_mW": 15655.0, "gpu_mW": 1541.62, "ane_mW": 0.0, "combined_mW": 17196.7, "thermal_pressure": "Nominal"}
++{"t": 1793514612.259666, "elapsed_ns": 1022117541, "cpu_mW": 15438.5, "gpu_mW": 1417.65, "ane_mW": 0.0, "combined_mW": 16856.2, "thermal_pressure": "Nominal"}
++{"t": 1793514613.281673, "elapsed_ns": 1021662666, "cpu_mW": 15223.2, "gpu_mW": 1414.36, "ane_mW": 0.0, "combined_mW": 16637.6, "thermal_pressure": "Nominal"}
++{"t": 1793514614.304944, "elapsed_ns": 1021745249, "cpu_mW": 15284.6, "gpu_mW": 1417.18, "ane_mW": 0.0, "combined_mW": 16701.8, "thermal_pressure": "Nominal"}
++{"t": 1793514616.347766, "elapsed_ns": 1022016083, "cpu_mW": 15099.6, "gpu_mW": 1411.92, "ane_mW": 0.0, "combined_mW": 16511.5, "thermal_pressure": "Nominal"}
++{"t": 1793514616.3495219, "elapsed_ns": 1022353708, "cpu_mW": 17499.8, "gpu_mW": 1395.8, "ane_mW": 0.0, "combined_mW": 18895.6, "thermal_pressure": "Nominal"}
++{"t": 1793514617.369737, "elapsed_ns": 1021157625, "cpu_mW": 18766.9, "gpu_mW": 1329.86, "ane_mW": 0.0, "combined_mW": 20096.8, "thermal_pressure": "Nominal"}
++{"t": 1793514618.391596, "elapsed_ns": 1022621416, "cpu_mW": 14450.1, "gpu_mW": 1391.52, "ane_mW": 0.0, "combined_mW": 15841.6, "thermal_pressure": "Nominal"}
++{"t": 1793514619.413666, "elapsed_ns": 1021993666, "cpu_mW": 15231.0, "gpu_mW": 1398.25, "ane_mW": 0.0, "combined_mW": 16629.3, "thermal_pressure": "Nominal"}
++{"t": 1793514620.435717, "elapsed_ns": 1021807500, "cpu_mW": 13641.5, "gpu_mW": 1400.46, "ane_mW": 0.0, "combined_mW": 15042.0, "thermal_pressure": "Nominal"}
++{"t": 1793514621.457668, "elapsed_ns": 1022103500, "cpu_mW": 17531.5, "gpu_mW": 1374.62, "ane_mW": 0.0, "combined_mW": 18906.1, "thermal_pressure": "Nominal"}
++{"t": 1793514622.479394, "elapsed_ns": 1021577833, "cpu_mW": 16720.2, "gpu_mW": 1716.95, "ane_mW": 0.0, "combined_mW": 18437.2, "thermal_pressure": "Nominal"}
++{"t": 1793514624.523325, "elapsed_ns": 1022127541, "cpu_mW": 19708.9, "gpu_mW": 1397.09, "ane_mW": 0.0, "combined_mW": 21106.0, "thermal_pressure": "Nominal"}
++{"t": 1793514624.5270631, "elapsed_ns": 1021298208, "cpu_mW": 20269.3, "gpu_mW": 1353.18, "ane_mW": 0.0, "combined_mW": 21622.5, "thermal_pressure": "Nominal"}
++{"t": 1793514625.544904, "elapsed_ns": 1021931583, "cpu_mW": 21425.1, "gpu_mW": 1422.8, "ane_mW": 0.0, "combined_mW": 22847.9, "thermal_pressure": "Nominal"}
++{"t": 1793514626.567263, "elapsed_ns": 1021341250, "cpu_mW": 20476.0, "gpu_mW": 1429.49, "ane_mW": 0.0, "combined_mW": 21905.5, "thermal_pressure": "Nominal"}
++{"t": 1793514627.589171, "elapsed_ns": 1022137583, "cpu_mW": 20095.1, "gpu_mW": 1182.82, "ane_mW": 0.0, "combined_mW": 21278.0, "thermal_pressure": "Nominal"}
++{"t": 1793514628.613048, "elapsed_ns": 1021251625, "cpu_mW": 19996.1, "gpu_mW": 1330.72, "ane_mW": 0.0, "combined_mW": 21326.8, "thermal_pressure": "Nominal"}
++{"t": 1793514629.633249, "elapsed_ns": 1022837041, "cpu_mW": 21384.6, "gpu_mW": 3196.99, "ane_mW": 0.0, "combined_mW": 24581.6, "thermal_pressure": "Nominal"}
++{"t": 1793514630.6989892, "elapsed_ns": 1023160541, "cpu_mW": 22916.2, "gpu_mW": 10281.9, "ane_mW": 0.0, "combined_mW": 33198.1, "thermal_pressure": "Nominal"}
++{"t": 1793514631.687009, "elapsed_ns": 1021780791, "cpu_mW": 22109.4, "gpu_mW": 863.199, "ane_mW": 0.0, "combined_mW": 22972.6, "thermal_pressure": "Nominal"}
++{"t": 1793514633.731264, "elapsed_ns": 1030373458, "cpu_mW": 23249.8, "gpu_mW": 1782.85, "ane_mW": 0.0, "combined_mW": 25032.7, "thermal_pressure": "Nominal"}
++{"t": 1793514633.7334049, "elapsed_ns": 1022465458, "cpu_mW": 22906.4, "gpu_mW": 6529.32, "ane_mW": 0.0, "combined_mW": 29435.7, "thermal_pressure": "Nominal"}
++{"t": 1793514634.753105, "elapsed_ns": 1022569666, "cpu_mW": 23174.9, "gpu_mW": 3640.83, "ane_mW": 0.0, "combined_mW": 26815.8, "thermal_pressure": "Nominal"}
++{"t": 1793514635.776225, "elapsed_ns": 1022206750, "cpu_mW": 20553.6, "gpu_mW": 3307.55, "ane_mW": 0.0, "combined_mW": 23861.1, "thermal_pressure": "Nominal"}
++{"t": 1793514636.801394, "elapsed_ns": 1021804333, "cpu_mW": 21215.4, "gpu_mW": 2.93598, "ane_mW": 0.0, "combined_mW": 21218.3, "thermal_pressure": "Nominal"}
++{"t": 1793514637.8185432, "elapsed_ns": 1022193791, "cpu_mW": 23387.0, "gpu_mW": 9600.92, "ane_mW": 0.0, "combined_mW": 32987.9, "thermal_pressure": "Nominal"}
++{"t": 1793514638.842475, "elapsed_ns": 1021944916, "cpu_mW": 21267.3, "gpu_mW": 8376.19, "ane_mW": 0.0, "combined_mW": 29643.5, "thermal_pressure": "Nominal"}
++{"t": 1793514639.864485, "elapsed_ns": 1020976916, "cpu_mW": 23274.8, "gpu_mW": 5490.82, "ane_mW": 0.0, "combined_mW": 28765.6, "thermal_pressure": "Nominal"}
++{"t": 1793514640.8875241, "elapsed_ns": 1022163250, "cpu_mW": 22014.1, "gpu_mW": 10267.4, "ane_mW": 0.0, "combined_mW": 32281.5, "thermal_pressure": "Nominal"}
++{"t": 1793514642.934209, "elapsed_ns": 1023654875, "cpu_mW": 27086.3, "gpu_mW": 9947.69, "ane_mW": 0.0, "combined_mW": 37034.0, "thermal_pressure": "Nominal"}
++{"t": 1793514642.9371352, "elapsed_ns": 1023348750, "cpu_mW": 25185.9, "gpu_mW": 9436.67, "ane_mW": 0.0, "combined_mW": 34622.6, "thermal_pressure": "Nominal"}
++{"t": 1793514643.956979, "elapsed_ns": 1023807208, "cpu_mW": 25876.9, "gpu_mW": 8864.95, "ane_mW": 0.0, "combined_mW": 34741.9, "thermal_pressure": "Nominal"}
++{"t": 1793514644.979489, "elapsed_ns": 1023149708, "cpu_mW": 24534.0, "gpu_mW": 5829.06, "ane_mW": 0.0, "combined_mW": 30363.1, "thermal_pressure": "Nominal"}
++{"t": 1793514646.002538, "elapsed_ns": 1022825125, "cpu_mW": 22665.7, "gpu_mW": 1668.91, "ane_mW": 0.0, "combined_mW": 24334.6, "thermal_pressure": "Nominal"}
++{"t": 1793514647.0245628, "elapsed_ns": 1021656541, "cpu_mW": 26014.6, "gpu_mW": 9672.53, "ane_mW": 0.0, "combined_mW": 35687.1, "thermal_pressure": "Nominal"}

added benchmarks/raw/tests/distributed_localhost_olmoe_20261101T012944.json +56 −0

++{
++  "rtt_ms_localhost": [
++    0.08566700853407383,
++    0.05250005051493645,
++    0.05170807708054781,
++    0.04841701593250036,
++    0.038207974284887314
++  ],
++  "prompts": [
++    {
++      "prompt": "The capital of Canada is",
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "remote": {
++        "remote_pairs": 312,
++        "local_pairs": 328,
++        "layers_remote": 16,
++        "remote_wait_s": 0.04496862518135458,
++        "failures": 0,
++        "workers": {
++          "w0": {
++            "worker": "w0",
++            "exec_calls": 16,
++            "exec_wall_s": 0.20909150002989918,
++            "bytes_sent": 333317,
++            "bytes_recv": 1281716
++          }
++        }
++      }
++    },
++    {
++      "prompt": "def fibonacci(n):\n    \"\"\"Retur",
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "remote": {
++        "remote_pairs": 1660,
++        "local_pairs": 1668,
++        "layers_remote": 56,
++        "remote_wait_s": 0.05559412541333586,
++        "failures": 0,
++        "workers": {
++          "w0": {
++            "worker": "w0",
++            "exec_calls": 56,
++            "exec_wall_s": 0.30413495854008943,
++            "bytes_sent": 1722764,
++            "bytes_recv": 6807094
++          }
++        }
++      }
++    }
++  ],
++  "greedy_equal": true,
++  "greedy_ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is",
++  "greedy_titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is"
++}
+ \ No newline at end of file

added benchmarks/raw/tests/distributed_localhost_olmoe_20261101T013029.json +56 −0

++{
++  "rtt_ms_localhost": [
++    0.08600007276982069,
++    0.0826249597594142,
++    0.06912497337907553,
++    0.08879194501787424,
++    0.06387499161064625
++  ],
++  "prompts": [
++    {
++      "prompt": "The capital of Canada is",
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "remote": {
++        "remote_pairs": 312,
++        "local_pairs": 328,
++        "layers_remote": 16,
++        "remote_wait_s": 0.3690654179081321,
++        "failures": 0,
++        "workers": {
++          "w0": {
++            "worker": "w0",
++            "exec_calls": 16,
++            "exec_wall_s": 0.5654866262339056,
++            "bytes_sent": 333317,
++            "bytes_recv": 1281719
++          }
++        }
++      }
++    },
++    {
++      "prompt": "def fibonacci(n):\n    \"\"\"Retur",
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "remote": {
++        "remote_pairs": 1660,
++        "local_pairs": 1668,
++        "layers_remote": 56,
++        "remote_wait_s": 0.3791747959330678,
++        "failures": 0,
++        "workers": {
++          "w0": {
++            "worker": "w0",
++            "exec_calls": 56,
++            "exec_wall_s": 0.6461280437652022,
++            "bytes_sent": 1722764,
++            "bytes_recv": 6807116
++          }
++        }
++      }
++    }
++  ],
++  "greedy_equal": true,
++  "greedy_ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is",
++  "greedy_titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is"
++}
+ \ No newline at end of file

added benchmarks/raw/tests/fused_upgate_olmoe_2026-11-01T012945.json +26 −0

++{
++  "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
++  "ts": "2026-11-01T01:29:45",
++  "cases": [
++    {
++      "capacity": 64,
++      "prompt": "The capital of Canada is",
++      "max_abs_dlogit": 0.0
++    },
++    {
++      "capacity": 64,
++      "prompt": "def fibonacci(n):\n    \"\"\"Retur",
++      "max_abs_dlogit": 0.0
++    },
++    {
++      "capacity": 16,
++      "prompt": "The capital of Canada is",
++      "max_abs_dlogit": 0.0
++    },
++    {
++      "capacity": 16,
++      "prompt": "def fibonacci(n):\n    \"\"\"Retur",
++      "max_abs_dlogit": 0.0
++    }
++  ]
++}
+ \ No newline at end of file

added benchmarks/raw/tests/fused_upgate_olmoe_2026-11-01T013029.json +26 −0

++{
++  "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
++  "ts": "2026-11-01T01:30:29",
++  "cases": [
++    {
++      "capacity": 64,
++      "prompt": "The capital of Canada is",
++      "max_abs_dlogit": 0.0
++    },
++    {
++      "capacity": 64,
++      "prompt": "def fibonacci(n):\n    \"\"\"Retur",
++      "max_abs_dlogit": 0.0
++    },
++    {
++      "capacity": 16,
++      "prompt": "The capital of Canada is",
++      "max_abs_dlogit": 0.0
++    },
++    {
++      "capacity": 16,
++      "prompt": "def fibonacci(n):\n    \"\"\"Retur",
++      "max_abs_dlogit": 0.0
++    }
++  ]
++}
+ \ No newline at end of file

added benchmarks/raw/tests/greedy_olmoe_cap16_20261101T013007.json +14 −0

++[
++  {
++    "prompt": "The capital of Canada is",
++    "ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
++    "titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
++    "equal": true
++  },
++  {
++    "prompt": "def fibonacci(n):\n    \"\"\"Return the n-th",
++    "ref": "   if n <= 0:\n        return \"Invalid input\"\n    elif n == 1:\n        return 0\n    elif n == 2:\n",
++    "titan": "   if n <= 0:\n        return \"Invalid input\"\n    elif n == 1:\n        return 0\n    elif n == 2:\n",
++    "equal": true
++  }
++]
+ \ No newline at end of file

added benchmarks/raw/tests/greedy_olmoe_cap16_20261101T013047.json +14 −0

++[
++  {
++    "prompt": "The capital of Canada is",
++    "ref": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
++    "titan": "Ottawa.\n\nOttawa is the capital of Canada, and it is located in the province of Ontario. It is situated on the south bank of the Ottawa",
++    "equal": true
++  },
++  {
++    "prompt": "def fibonacci(n):\n    \"\"\"Return the n-th",
++    "ref": "   if n <= 0:\n        return \"Invalid input\"\n    elif n == 1:\n        return 0\n    elif n == 2:\n",
++    "titan": "   if n <= 0:\n        return \"Invalid input\"\n    elif n == 1:\n        return 0\n    elif n == 2:\n",
++    "equal": true
++  }
++]
+ \ No newline at end of file

added benchmarks/raw/tests/numerical_olmoe_cap16_2026-11-01T012959.json +50 −0

++{
++  "capacity": 16,
++  "prompts": [
++    {
++      "prompt": "The capital of Canada is",
++      "tokens": 5,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.2580101180438449,
++      "misses": 440,
++      "stall_s": 0.09684595465660095,
++      "layers_routed": 46
++    },
++    {
++      "prompt": "def fibonacci(n):\n    \"\"\"Return the n-th",
++      "tokens": 21,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.42512908777969016,
++      "misses": 1002,
++      "stall_s": 0.20027187676168978,
++      "layers_routed": 127
++    },
++    {
++      "prompt": "Q: If a train travels 60 km in 45 minute",
++      "tokens": 25,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.41904344010530936,
++      "misses": 1324,
++      "stall_s": 0.2887474615126848,
++      "layers_routed": 161
++    },
++    {
++      "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
++      "tokens": 26,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.45278969957081544,
++      "misses": 1275,
++      "stall_s": 0.2674082911107689,
++      "layers_routed": 164
++    }
++  ],
++  "tol": 0.001,
++  "ts": "2026-11-01T01:29:59",
++  "max_abs_dlogit_worst": 0.0,
++  "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
++  "model_type": "olmoe"
++}
+ \ No newline at end of file

added benchmarks/raw/tests/numerical_olmoe_cap16_2026-11-01T013039.json +50 −0

++{
++  "capacity": 16,
++  "prompts": [
++    {
++      "prompt": "The capital of Canada is",
++      "tokens": 5,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.2580101180438449,
++      "misses": 440,
++      "stall_s": 0.09993178956210613,
++      "layers_routed": 46
++    },
++    {
++      "prompt": "def fibonacci(n):\n    \"\"\"Return the n-th",
++      "tokens": 21,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.42512908777969016,
++      "misses": 1002,
++      "stall_s": 0.2170187613228336,
++      "layers_routed": 127
++    },
++    {
++      "prompt": "Q: If a train travels 60 km in 45 minute",
++      "tokens": 25,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.41904344010530936,
++      "misses": 1324,
++      "stall_s": 0.3052907877136022,
++      "layers_routed": 161
++    },
++    {
++      "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
++      "tokens": 26,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.45278969957081544,
++      "misses": 1275,
++      "stall_s": 0.2862527520628646,
++      "layers_routed": 164
++    }
++  ],
++  "tol": 0.001,
++  "ts": "2026-11-01T01:30:39",
++  "max_abs_dlogit_worst": 0.0,
++  "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
++  "model_type": "olmoe"
++}
+ \ No newline at end of file

added benchmarks/raw/tests/numerical_olmoe_cap32_2026-11-01T012957.json +50 −0

++{
++  "capacity": 32,
++  "prompts": [
++    {
++      "prompt": "The capital of Canada is",
++      "tokens": 5,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.0,
++      "misses": 430,
++      "stall_s": 0.42347062879707664,
++      "layers_routed": 16
++    },
++    {
++      "prompt": "def fibonacci(n):\n    \"\"\"Return the n-th",
++      "tokens": 21,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.6060606060606061,
++      "misses": 429,
++      "stall_s": 0.46165179007221013,
++      "layers_routed": 40
++    },
++    {
++      "prompt": "Q: If a train travels 60 km in 45 minute",
++      "tokens": 25,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.6090458488228006,
++      "misses": 631,
++      "stall_s": 0.32894921104889363,
++      "layers_routed": 58
++    },
++    {
++      "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
++      "tokens": 26,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.6224188790560472,
++      "misses": 512,
++      "stall_s": 0.18187279207631946,
++      "layers_routed": 49
++    }
++  ],
++  "tol": 0.001,
++  "ts": "2026-11-01T01:29:57",
++  "max_abs_dlogit_worst": 0.0,
++  "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
++  "model_type": "olmoe"
++}
+ \ No newline at end of file

added benchmarks/raw/tests/numerical_olmoe_cap32_2026-11-01T013038.json +50 −0

++{
++  "capacity": 32,
++  "prompts": [
++    {
++      "prompt": "The capital of Canada is",
++      "tokens": 5,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.0,
++      "misses": 430,
++      "stall_s": 0.0899640420684591,
++      "layers_routed": 16
++    },
++    {
++      "prompt": "def fibonacci(n):\n    \"\"\"Return the n-th",
++      "tokens": 21,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.6060606060606061,
++      "misses": 429,
++      "stall_s": 0.28829907963518053,
++      "layers_routed": 40
++    },
++    {
++      "prompt": "Q: If a train travels 60 km in 45 minute",
++      "tokens": 25,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.6090458488228006,
++      "misses": 631,
++      "stall_s": 0.14359562494792044,
++      "layers_routed": 58
++    },
++    {
++      "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
++      "tokens": 26,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.6224188790560472,
++      "misses": 512,
++      "stall_s": 0.1363065792247653,
++      "layers_routed": 49
++    }
++  ],
++  "tol": 0.001,
++  "ts": "2026-11-01T01:30:38",
++  "max_abs_dlogit_worst": 0.0,
++  "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
++  "model_type": "olmoe"
++}
+ \ No newline at end of file

added benchmarks/raw/tests/numerical_olmoe_cap64_2026-11-01T012954.json +50 −0

++{
++  "capacity": 64,
++  "prompts": [
++    {
++      "prompt": "The capital of Canada is",
++      "tokens": 5,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.0,
++      "misses": 430,
++      "stall_s": 0.4835599169600755,
++      "layers_routed": 16
++    },
++    {
++      "prompt": "def fibonacci(n):\n    \"\"\"Return the n-th",
++      "tokens": 21,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.4637904468412943,
++      "misses": 348,
++      "stall_s": 0.42762762669008225,
++      "layers_routed": 16
++    },
++    {
++      "prompt": "Q: If a train travels 60 km in 45 minute",
++      "tokens": 25,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.823076923076923,
++      "misses": 138,
++      "stall_s": 0.1677537460345775,
++      "layers_routed": 16
++    },
++    {
++      "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
++      "tokens": 26,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.9234828496042217,
++      "misses": 58,
++      "stall_s": 0.0731603738386184,
++      "layers_routed": 16
++    }
++  ],
++  "tol": 0.001,
++  "ts": "2026-11-01T01:29:54",
++  "max_abs_dlogit_worst": 0.0,
++  "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
++  "model_type": "olmoe"
++}
+ \ No newline at end of file

added benchmarks/raw/tests/numerical_olmoe_cap64_2026-11-01T013035.json +50 −0

++{
++  "capacity": 64,
++  "prompts": [
++    {
++      "prompt": "The capital of Canada is",
++      "tokens": 5,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.0,
++      "misses": 430,
++      "stall_s": 0.3988114590756595,
++      "layers_routed": 16
++    },
++    {
++      "prompt": "def fibonacci(n):\n    \"\"\"Return the n-th",
++      "tokens": 21,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.4637904468412943,
++      "misses": 348,
++      "stall_s": 0.3186756670475006,
++      "layers_routed": 16
++    },
++    {
++      "prompt": "Q: If a train travels 60 km in 45 minute",
++      "tokens": 25,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.823076923076923,
++      "misses": 138,
++      "stall_s": 0.09854987578000873,
++      "layers_routed": 16
++    },
++    {
++      "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
++      "tokens": 26,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.9234828496042217,
++      "misses": 58,
++      "stall_s": 0.0453380401013419,
++      "layers_routed": 16
++    }
++  ],
++  "tol": 0.001,
++  "ts": "2026-11-01T01:30:35",
++  "max_abs_dlogit_worst": 0.0,
++  "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
++  "model_type": "olmoe"
++}
+ \ No newline at end of file

added benchmarks/raw/tests/numerical_olmoe_cap8_2026-11-01T013001.json +50 −0

++{
++  "capacity": 8,
++  "prompts": [
++    {
++      "prompt": "The capital of Canada is",
++      "tokens": 5,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.1625,
++      "misses": 536,
++      "stall_s": 0.09062562719918787,
++      "layers_routed": 80
++    },
++    {
++      "prompt": "def fibonacci(n):\n    \"\"\"Return the n-th",
++      "tokens": 21,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.4425981873111782,
++      "misses": 1476,
++      "stall_s": 0.24081315705552697,
++      "layers_routed": 331
++    },
++    {
++      "prompt": "Q: If a train travels 60 km in 45 minute",
++      "tokens": 25,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.375,
++      "misses": 1995,
++      "stall_s": 0.3570274900412187,
++      "layers_routed": 399
++    },
++    {
++      "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
++      "tokens": 26,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.3878048780487805,
++      "misses": 2008,
++      "stall_s": 0.39880511723458767,
++      "layers_routed": 410
++    }
++  ],
++  "tol": 0.001,
++  "ts": "2026-11-01T01:30:01",
++  "max_abs_dlogit_worst": 0.0,
++  "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
++  "model_type": "olmoe"
++}
+ \ No newline at end of file

added benchmarks/raw/tests/numerical_olmoe_cap8_2026-11-01T013041.json +50 −0

++{
++  "capacity": 8,
++  "prompts": [
++    {
++      "prompt": "The capital of Canada is",
++      "tokens": 5,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.1625,
++      "misses": 536,
++      "stall_s": 0.11774066393263638,
++      "layers_routed": 80
++    },
++    {
++      "prompt": "def fibonacci(n):\n    \"\"\"Return the n-th",
++      "tokens": 21,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.4425981873111782,
++      "misses": 1476,
++      "stall_s": 0.34902799827978015,
++      "layers_routed": 331
++    },
++    {
++      "prompt": "Q: If a train travels 60 km in 45 minute",
++      "tokens": 25,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.375,
++      "misses": 1995,
++      "stall_s": 0.4580805116565898,
++      "layers_routed": 399
++    },
++    {
++      "prompt": "Les mod\u00e8les \u00e0 m\u00e9lange d'experts activent",
++      "tokens": 26,
++      "max_abs_dlogit": 0.0,
++      "argmax_equal": true,
++      "hit_rate": 0.3878048780487805,
++      "misses": 2008,
++      "stall_s": 0.4504705760627985,
++      "layers_routed": 410
++    }
++  ],
++  "tol": 0.001,
++  "ts": "2026-11-01T01:30:41",
++  "max_abs_dlogit_worst": 0.0,
++  "model": "/Users/simon-pierreboucher/.cache/huggingface/hub/models--mlx-community--OLMoE-1B-7B-0125-Instruct-4bit/snapshots/e23844197887b031e7ddddbb0b8959c5a6853a7b",
++  "model_type": "olmoe"
++}
+ \ No newline at end of file

modified titan/prefetch.py +2 −1

				`@@ -38,7 +38,7 @@ class Prefetcher:`
38	# ---- reading ---------------------------------------------------------------	38	# ---- reading ---------------------------------------------------------------
39	def _fetch(self, layer: int, expert: int):	39	def _fetch(self, layer: int, expert: int):
40	t = time.perf_counter()	40	t = time.perf_counter()
41	− tensors = self.store.read_expert(layer, expert) # pread + np -> mx (CPU side, no GPU work)	41	+ tensors = self.store.read_expert(layer, expert, as_mx=False) # pread + numpy only; MLX arrays are created in drain()
42	return tensors, time.perf_counter() - t	42	return tensors, time.perf_counter() - t
43		43
44	def issue(self, layer: int, experts: List[int]):	44	def issue(self, layer: int, experts: List[int]):
				`@@ -63,6 +63,7 @@ class Prefetcher:`
63	self.late[layer] += 1	63	self.late[layer] += 1
64	if fut.done():	64	if fut.done():
65	tensors, _ = fut.result()	65	tensors, _ = fut.result()
		66	+ tensors = {n: self.store.to_mx(v) for n, v in tensors.items()}
66	del inf[e]	67	del inf[e]
67	if tab.insert_prefetched(e, tensors, protected) is not None:	68	if tab.insert_prefetched(e, tensors, protected) is not None:
68	self.installed[layer] += 1	69	self.installed[layer] += 1

modified titan/store.py +20 −10

				`@@ -49,7 +49,7 @@ class ExpertStore:`
49	self._fds[path] = fd	49	self._fds[path] = fd
50	return fd	50	return fd
51		51
52	− def read_tensor(self, loc: TensorLoc) -> mx.array:	52	+ def read_tensor(self, loc: TensorLoc, as_mx: bool = True):
53	fd = self._fd(loc.file)	53	fd = self._fd(loc.file)
54	t = time.perf_counter()	54	t = time.perf_counter()
55	buf = os.pread(fd, loc.nbytes, loc.offset)	55	buf = os.pread(fd, loc.nbytes, loc.offset)
				`@@ -58,22 +58,31 @@ class ExpertStore:`
58	self.reads += 1	58	self.reads += 1
59	if len(buf) != loc.nbytes:	59	if len(buf) != loc.nbytes:
60	raise IOError(f"short read for {loc.key}: {len(buf)} != {loc.nbytes}")	60	raise IOError(f"short read for {loc.key}: {len(buf)} != {loc.nbytes}")
61	− arr = np.frombuffer(buf, dtype=NP_DTYPE[loc.dtype]).reshape(loc.shape)	61	+ return self._wrap(loc, buf, as_mx)
62	− if loc.dtype == "BF16":
63	− return mx.array(arr).view(mx.bfloat16)
64	− return mx.array(arr)
65		62
66	− def _wrap(self, loc: TensorLoc, buf) -> mx.array:	63	+ def _wrap(self, loc: TensorLoc, buf, as_mx: bool = True):
67	arr = np.frombuffer(buf, dtype=NP_DTYPE[loc.dtype]).reshape(loc.shape)	64	arr = np.frombuffer(buf, dtype=NP_DTYPE[loc.dtype]).reshape(loc.shape)
		65	+ if not as_mx:
		66	+ return (arr, loc.dtype)
68	if loc.dtype == "BF16":	67	if loc.dtype == "BF16":
69	return mx.array(arr).view(mx.bfloat16)	68	return mx.array(arr).view(mx.bfloat16)
70	return mx.array(arr)	69	return mx.array(arr)
71		70
72	− def read_expert(self, layer: int, expert: int) -> Dict[str, mx.array]:	71	+ @staticmethod
		72	+ def to_mx(t):
		73	+ """Convert a numpy (array, dtype) pair produced by a reader thread into an MLX array ON THE CALLING THREAD.
		74	+ MLX arrays must be created in the thread that evaluates them (streams are thread-local): creating them in reader
		75	+ threads raised 'There is no Stream(gpu, N) in current thread' on the worker (2026-11-01)."""
		76	+ if isinstance(t, mx.array):
		77	+ return t
		78	+ arr, dt = t
		79	+ return mx.array(arr).view(mx.bfloat16) if dt == "BF16" else mx.array(arr)
		80	+
		81	+ def read_expert(self, layer: int, expert: int, as_mx: bool = True):
73	c = self._contig.get((layer, expert))	82	c = self._contig.get((layer, expert))
74	if c is None:	83	if c is None:
75	es = self.spec.experts[(layer, expert)]	84	es = self.spec.experts[(layer, expert)]
76	− return {name: self.read_tensor(loc) for name, loc in es.tensors.items()}	85	+ return {name: self.read_tensor(loc, as_mx) for name, loc in es.tensors.items()}
77	path, off, n, locs = c	86	path, off, n, locs = c
78	fd = self._fd(path)	87	fd = self._fd(path)
79	t = time.perf_counter()	88	t = time.perf_counter()
				`@@ -85,7 +94,7 @@ class ExpertStore:`
85	out = {}	94	out = {}
86	for name, loc in locs:	95	for name, loc in locs:
87	a = loc.offset - off	96	a = loc.offset - off
88	− out[name] = self._wrap(loc, memoryview(buf)[a:a + loc.nbytes])	97	+ out[name] = self._wrap(loc, memoryview(buf)[a:a + loc.nbytes], as_mx)
89	return out	98	return out
90		99
91	def read_experts(self, layer: int, experts) -> Dict[int, Dict[str, mx.array]]:	100	def read_experts(self, layer: int, experts) -> Dict[int, Dict[str, mx.array]]:
				`@@ -94,7 +103,8 @@ class ExpertStore:`
94	if self._pool is None or len(experts) == 1:	103	if self._pool is None or len(experts) == 1:
95	return {e: self.read_expert(layer, e) for e in experts}	104	return {e: self.read_expert(layer, e) for e in experts}
96	t = time.perf_counter()	105	t = time.perf_counter()
97	− res = dict(zip(experts, self._pool.map(lambda e: self.read_expert(layer, e), experts)))	106	+ raw = dict(zip(experts, self._pool.map(lambda e: self.read_expert(layer, e, as_mx=False), experts)))
		107	+ res = {e: {n: self.to_mx(v) for n, v in d.items()} for e, d in raw.items()} # MLX arrays created here, in the caller's thread
98	self.read_time += 0 # per-read times already accumulated inside read_expert (sum of thread times, not wall)	108	self.read_time += 0 # per-read times already accumulated inside read_expert (sum of thread times, not wall)
99	self.wall_batch_time = getattr(self, "wall_batch_time", 0.0) + (time.perf_counter() - t)	109	self.wall_batch_time = getattr(self, "wall_batch_time", 0.0) + (time.perf_counter() - t)
100	return res	110	return res
101		111

spb/wp33_uqo

Fix: MLX arrays created on the consuming thread only (worker/stacked-layout stream error)

Theme