Gate F script (Qwen3 two-node, cold local vs odd/popular placement), nocache flag, LOG (probe depths, host-key fix)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
6 changed files +93 −15
modified
benchmarks/scripts/bench_distributed.py
+3 −2
@@ -48,12 +48,13 @@ def main(): | |||||
| 48 | ap.add_argument("--placement", default="none") | 48 | ap.add_argument("--placement", default="none") | ||
| 49 | ap.add_argument("--capacity", type=int, required=True) | 49 | ap.add_argument("--capacity", type=int, required=True) | ||
| 50 | ap.add_argument("--predictor", default="none"); ap.add_argument("--m", type=int, default=8); ap.add_argument("--traces", default=None) | 50 | ap.add_argument("--predictor", default="none"); ap.add_argument("--m", type=int, default=8); ap.add_argument("--traces", default=None) | ||
| 51 | + ap.add_argument("--nocache", action="store_true", help="F_NOCACHE on the local checkpoint (cold-NVMe local tier)") | ||||
| 51 | ap.add_argument("--workloads", default=str(Path(__file__).resolve().parents[1] / "configs" / "workloads_v2.yaml")) | 52 | ap.add_argument("--workloads", default=str(Path(__file__).resolve().parents[1] / "configs" / "workloads_v2.yaml")) | ||
| 52 | ap.add_argument("--families", default="general,code"); ap.add_argument("--split", default="heldout") | 53 | ap.add_argument("--families", default="general,code"); ap.add_argument("--split", default="heldout") | ||
| 53 | ap.add_argument("--max-tokens", type=int, default=64); ap.add_argument("--repeats", type=int, default=2); ap.add_argument("--warmup", type=int, default=1) | 54 | ap.add_argument("--max-tokens", type=int, default=64); ap.add_argument("--repeats", type=int, default=2); ap.add_argument("--warmup", type=int, default=1) | ||
| 54 | ap.add_argument("--seed", type=int, default=0); ap.add_argument("--tag", default=""); ap.add_argument("--out", required=True) | 55 | ap.add_argument("--seed", type=int, default=0); ap.add_argument("--tag", default=""); ap.add_argument("--out", required=True) | ||
| 55 | a = ap.parse_args() | 56 | a = ap.parse_args() | ||
| 56 | − tm = TitanModel(a.model, capacity=a.capacity, predictor=a.predictor, m=a.m, traces_dir=a.traces) | 57 | + tm = TitanModel(a.model, capacity=a.capacity, predictor=a.predictor, m=a.m, traces_dir=a.traces, nocache=a.nocache) | ||
| 57 | token = open(os.path.expanduser(a.token_file), "rb").read().strip() | 58 | token = open(os.path.expanduser(a.token_file), "rb").read().strip() | ||
| 58 | workers = {} | 59 | workers = {} | ||
| 59 | for spec in [w for w in a.workers.split(",") if w]: | 60 | for spec in [w for w in a.workers.split(",") if w]: | ||
@@ -81,7 +82,7 @@ def main(): | |||||
| 81 | for rep in range(a.warmup + a.repeats): | 82 | for rep in range(a.warmup + a.repeats): | ||
| 82 | ps = prompts[:]; rng.shuffle(ps); order += [(rep, f, p) for f, p in ps] | 83 | ps = prompts[:]; rng.shuffle(ps); order += [(rep, f, p) for f, p in ps] | ||
| 83 | base = {"backend": "titan-distributed", "capacity": a.capacity, "placement": a.placement, "workers": list(workers), "rtt_ms": rtts, | 84 | base = {"backend": "titan-distributed", "capacity": a.capacity, "placement": a.placement, "workers": list(workers), "rtt_ms": rtts, | ||
| 84 | − "n_remote_experts": len(placement), "predictor": a.predictor, "m": a.m, "max_tokens": a.max_tokens, "split": a.split, "tag": a.tag, | 85 | + "n_remote_experts": len(placement), "predictor": a.predictor, "m": a.m, "max_tokens": a.max_tokens, "split": a.split, "tag": a.tag, "nocache": a.nocache, | ||
| 85 | "model": os.path.realpath(a.model), "cmd": " ".join(sys.argv), "env": env_record(), "seed": a.seed, "dlogit_vs_mlx_lm": dlogit} | 86 | "model": os.path.realpath(a.model), "cmd": " ".join(sys.argv), "env": env_record(), "seed": a.seed, "dlogit_vs_mlx_lm": dlogit} | ||
| 86 | os.makedirs(os.path.dirname(a.out), exist_ok=True) | 87 | os.makedirs(os.path.dirname(a.out), exist_ok=True) | ||
| 87 | with open(a.out, "a") as f: | 88 | with open(a.out, "a") as f: | ||
added
benchmarks/scripts/run_gateF_qwen3.sh
+15 −0
@@ -0,0 +1,15 @@ | |||||
| 1 | +#!/bin/bash | ||||
| 2 | +# Gate F on the medium model: coordinator M3U96a (bounded local residency, cold NVMe via purge + F_NOCACHE) vs remote execution | ||||
| 3 | +# on M3U96b with static (odd) and frequency-based (popular: least popular experts remote) placements. Run ON M3U96a. | ||||
| 4 | +cd ~/wp33/titan-moe | ||||
| 5 | +M=$(ls -d ~/.cache/huggingface/hub/models--mlx-community--Qwen3-30B-A3B-Instruct-2507-4bit/snapshots/*) | ||||
| 6 | +OUT=benchmarks/raw/gateF/two_node_qwen3_m3u96a_m3u96b.jsonl | ||||
| 7 | +mkdir -p benchmarks/raw/gateF | ||||
| 8 | +for cap in 32 64; do | ||||
| 9 | + for pl in none odd popular:benchmarks/raw/gateF/traces_qwen3_m3u96a; do | ||||
| 10 | + echo "$SUDO_PW" | sudo -S purge 2>/dev/null; sleep 2 | ||||
| 11 | + .venv/bin/python benchmarks/scripts/bench_distributed.py --model $M --workers w0=192.168.2.85:9412 --placement $pl --capacity $cap \ | ||||
| 12 | + --nocache --families general,code --split heldout --repeats 2 --max-tokens 64 --out $OUT 2>&1 | grep -v Fetching | tail -2 | ||||
| 13 | + done | ||||
| 14 | +done | ||||
| 15 | +echo GATEF-DONE | ||||
modified
paper/main.pdf
+0 −0
Binary file not shown.
modified
paper/sections/ablations.tex
+7 −2
@@ -7,6 +7,11 @@ transport $\times$ kernels $\times$ batch) is executed gate by gate. In this ver | |||||
| 7 | (Section~\ref{sec:implementation}). | 7 | (Section~\ref{sec:implementation}). | ||
| 8 | \item \textbf{Fast path} (device-side slot map when a layer is complete): 203 $\to$ 489~\tps{} at 64 slots. | 8 | \item \textbf{Fast path} (device-side slot map when a layer is complete): 203 $\to$ 489~\tps{} at 64 slots. | ||
| 9 | \item \textbf{Eviction policy} (LRU vs LFU): within 0.01 hit rate at every capacity in simulation. | 9 | \item \textbf{Eviction policy} (LRU vs LFU): within 0.01 hit rate at every capacity in simulation. | ||
| 10 | − \item \textbf{Predictor} (none / prev-rank / transition$_d$ / probe / oracle) and \textbf{depth}: Table~\ref{tab:sim}, \textsc{pending} until the v2 | 10 | + \item \textbf{Storage layout} (scattered safetensors vs expert-contiguous records) and \textbf{reader parallelism}: 8--10 $\to$ 18 $\to$ 36~\tps{} | ||
| 11 | − simulation and its online confirmation complete. | 11 | + on the cold tier at 16 slots. | ||
| 12 | + \item \textbf{Predictor} (none / prev-rank / transition$_d$ / probe$_d$ / oracle), \textbf{depth} and \textbf{prefetch width} $m$: Table~\ref{tab:sim} | ||||
| 13 | + (simulation) and Tables~\ref{tab:cold}--\ref{tab:attr} (online); the best online configuration ties with no prefetch. | ||||
| 14 | + \item \textbf{Kernel fusion} (up+gate): no measurable effect (Table~\ref{tab:fused}). | ||||
| 15 | + \item \textbf{Placement} (local vs odd experts remote over 1~GbE): $-80\,\%$ throughput (Figure~\ref{fig:twonode}); frequency- and | ||||
| 16 | + cost-aware placement are exercised in Gate~F. | ||||
| 12 | \end{itemize} | 17 | \end{itemize} | ||
modified
paper/sections/results.tex
+61 −11
@@ -47,8 +47,13 @@ model does 53 misses per token (Prop.~\ref{prop:capacity}), i.e.\ about 11~ms of | |||||
| 47 | \label{sec:results-locality} | 47 | \label{sec:results-locality} | ||
| 48 | From the v1 traces (18 prompts, 3\,686 tokens), per-layer LRU hit rates are 0.57 at 16 slots, 0.80 at 32 and 0.93 at 48 (LFU | 48 | From the v1 traces (18 prompts, 3\,686 tokens), per-layer LRU hit rates are 0.57 at 16 slots, 0.80 at 32 and 0.93 at 48 (LFU | ||
| 49 | within 0.01), nearly identical on fit and held-out prompts (\code{benchmarks/processed/gateC/sim\_olmoe\_laptop\_ssd\_v1.json}); | 49 | within 0.01), nearly identical on fit and held-out prompts (\code{benchmarks/processed/gateC/sim\_olmoe\_laptop\_ssd\_v1.json}); | ||
| 50 | −a one-layer-ahead transition predictor has recall@8 of 0.50 on held-out prompts; a linear probe on the previous layer's | 50 | +a one-layer-ahead transition predictor has recall@8 of 0.50 on held-out prompts. On the v2 traces (65 prompts, 6 families, | ||
| 51 | −log-probabilities only 0.16 with these few traces. Table~\ref{tab:sim} is generated from the simulation JSON (v2 traces when available, else v1). | 51 | +\code{benchmarks/processed/gateC/}) the transition predictor keeps almost all of its accuracy when it looks further ahead | ||
| 52 | +(recall@8 0.534 / 0.525 / 0.517 at depth 1 / 2 / 3), and a per-layer logistic probe on the previous layer's router | ||||
| 53 | +log-probabilities reaches recall@8 of \textbf{0.761} held-out (0.779 fit) with prefetch precision 0.74--0.83, lifting the simulated | ||||
| 54 | +hit rate at 16 slots from 0.58 (LRU) to 0.81 (oracle 0.94). A first version of the probe scored 0.16 because of a label-alignment | ||||
| 55 | +bug across traces; the fix is recorded in the log and the buggy rows are excluded from Table~\ref{tab:sim} and Figure~\ref{fig:sim}. | ||||
| 56 | +The rank-based previous-token predictor hurts at every capacity (precision 0.18--0.47, it evicts useful experts). | ||||
| 52 | 57 | ||||
| 53 | \input{tables/sim} | 58 | \input{tables/sim} | ||
| 54 | 59 | ||||
@@ -62,12 +67,39 @@ by slot capacity and predictor. Generated by \code{scripts/make\_figures.py} fro | |||||
| 62 | 67 | ||||
| 63 | \subsection{Prefetching (H1)} | 68 | \subsection{Prefetching (H1)} | ||
| 64 | \label{sec:results-prefetch} | 69 | \label{sec:results-prefetch} | ||
| 65 | −With the one-layer horizon the online prefetcher raised the hit rate from 0.48 to 0.63 (general) and from 0.67 to 0.73 | 70 | +\paragraph{Page-cache tier, one-layer horizon.} The online prefetcher raised the hit rate from 0.48 to 0.63 (general) and from | ||
| 66 | −(code) but reduced throughput from 51 to 35 and from 73 to 56~\tps{} (Table~\ref{tab:capacity}): the lead time | 71 | +0.67 to 0.73 (code) but reduced throughput from 51 to 35 and from 73 to 56~\tps{} (Table~\ref{tab:capacity}): the lead time | ||
| 67 | −$d\,\tau_\ell = 0.13$~ms is shorter than the 0.21~ms fetch, so predicted experts arrive late and are waited for, and the | 72 | +$d\,\tau_\ell = 0.13$~ms is shorter than the 0.21~ms fetch, so predicted experts arrive late, and the 50\,\% of wrong predictions | ||
| 68 | −50\,\% of wrong predictions add traffic. This is exactly the regime excluded by condition~\eqref{eq:condition}. | 73 | +add traffic. This is the regime excluded by condition~\eqref{eq:condition}. | ||
| 69 | −Depth-$d$ and token-horizon prefetching on the cold-NVMe tier are reported in Table~\ref{tab:cold} (generated from | 74 | + | ||
| 70 | −\code{benchmarks/raw/gateC/cold\_nvme\_laptop/sweep.jsonl}; rows present only for configurations that have run). | 75 | +\paragraph{Cold-NVMe tier.} The SSD tier itself had to be engineered before the question could be asked: in the published | ||
| 76 | +OLMoE checkpoint the nine tensors of one expert are scattered over a 1.5~GB span of the file (hash order of keys), so each miss | ||||
| 77 | +cost nine random reads and 16-slot decoding ran at 8--10~\tps{}; parallel demand reads raised it to 18 and the expert-contiguous | ||||
| 78 | +layout of Section~\ref{sec:architecture} (one 4~KiB-aligned record per expert, one \code{pread}) to 36~\tps{} | ||||
| 79 | +(\code{benchmarks/raw/gateC/cold\_nvme\_laptop/}). Table~\ref{tab:cold} then compares predictors on that tier with the page cache | ||||
| 80 | +purged before every configuration: at 16 slots the depth-2 transition predictor cuts the stall by a third (0.77 vs 1.15~s per 64 | ||||
| 81 | +tokens) and ties with demand loading (35.0 vs 36.4~\tps{}); every other predictor, including depth 1 and 3 and the rank-based one, | ||||
| 82 | +loses; at 32 slots all of them lose to demand loading (41--42 vs 50.6~\tps{}). | ||||
| 83 | + | ||||
| 84 | +\paragraph{Attribution.} Figure~\ref{fig:attribution} and Table~\ref{tab:attr} split the host time per token into the wait for | ||||
| 85 | +the GPU graph (which includes the slice writes that install experts), prefetch bookkeeping and late waits, and demand reads. | ||||
| 86 | +The probe raises the hit rate to 0.78 and cuts demand reads from 19.9 to 12.8~ms per token, but the GPU wait grows from 14.7 | ||||
| 87 | +to 19.7~ms and late waits add 3~ms: each installed expert costs nine lazy slice writes (about 0.13~ms of graph work on the | ||||
| 88 | +critical path), and the prefetcher installs more experts than it saves reads for. Confidence thresholds and deeper horizons | ||||
| 89 | +change the mix, not the sum. \textbf{H1 is therefore not supported on this model and tier}: the best predictor ties with no | ||||
| 90 | +prefetch. The attribution is positive knowledge: the bottleneck is the install path, not prediction accuracy (the probe is | ||||
| 91 | +already within 0.16 of the oracle's hit rate at 16 slots), so a single-copy install (one record per expert written in one | ||||
| 92 | +operation) is the change that would make the probe's 7~ms of saved reads count, and it is the first item of version~0.2. | ||||
| 93 | + | ||||
| 94 | +\input{tables/attr} | ||||
| 95 | + | ||||
| 96 | +\begin{figure}[t] | ||||
| 97 | +\centering | ||||
| 98 | +\includegraphics[width=\textwidth]{attribution.pdf} | ||||
| 99 | +\caption{Host time per token on the cold-NVMe tier (OLMoE, 16 slots, held-out general prompts): GPU wait including installs, | ||||
| 100 | +prefetch bookkeeping and late waits, demand reads. Labels give the median decode throughput and hit rate.} | ||||
| 101 | +\label{fig:attribution} | ||||
| 102 | +\end{figure} | ||||
| 71 | 103 | ||||
| 72 | \input{tables/cold} | 104 | \input{tables/cold} | ||
| 73 | 105 | ||||
@@ -106,6 +138,24 @@ anticipates; H2 can only be tested in the regime where a node cannot hold the ex | |||||
| 106 | \label{fig:twonode} | 138 | \label{fig:twonode} | ||
| 107 | \end{figure} | 139 | \end{figure} | ||
| 108 | 140 | ||||
| 109 | −\subsection{Metal kernels (H4), energy frontier (H5)} | 141 | +The round trip was measured directly (\code{benchmarks/raw/gateE/remote\_cost\_breakdown\_m3u96a\_m3u96b.txt}): application-level | ||
| 110 | −\textsc{pending}: Gates D and the energy study have protocols (Section~\ref{sec:algorithms}, \code{research/protocol.md}) and no | 142 | +ping 0.74--1.17~ms; one layer's request (one row, eight pairs) 2.59~ms; eight rows with 64 pairs 7.3~ms; 32 rows with 256 pairs | ||
| 111 | −logs in this version. | 143 | +19.1~ms, of which about 9~ms is the 1~MB reply on the wire. The worker therefore adds 1.2--1.7~ms per call to the network | ||
| 144 | +latency, which bounds what a faster transport could recover without also changing the worker's execution path. | ||||
| 145 | + | ||||
| 146 | +\subsection{Metal kernels (H4)} | ||||
| 147 | +\label{sec:results-metal} | ||||
| 148 | +Profiling on M4M64a (\code{benchmarks/raw/gateD/}) shows that a decode step with all experts resident takes 2.99~ms | ||||
| 149 | +(334~\tps{} synchronous) and moves 452~MB of expert weights, i.e.\ 151~GB/s or about 28\,\% of the chip's nominal bandwidth; | ||||
| 150 | +a \gatherqmm{} over 1 to 16 experts takes the same 0.17~ms (latency-bound; 250~GB/s only at 64 experts), and in-graph | ||||
| 151 | +elementwise ops cost 5--7~$\mu$s each against a 0.16~ms synchronisation floor. The step is launch-bound, so the most a kernel | ||||
| 152 | +fusion can save is a few dispatches per layer. We tested the cheapest one, a single \gatherqmm{} over a stacked up+gate buffer | ||||
| 153 | +(bit-identical logits, \code{tests/test\_fused\_upgate.py}): $-0.2\,\%$ with all experts resident and $+2.6\,\%$ on the cold tier, | ||||
| 154 | +both within the run-to-run range (Table~\ref{tab:fused}). \textbf{H4 is not supported} at batch size one on this model; the profile | ||||
| 155 | +says that batching tokens, not kernels, is what would change the regime. | ||||
| 156 | + | ||||
| 157 | +\input{tables/fused} | ||||
| 158 | + | ||||
| 159 | +\subsection{Energy frontier (H5)} | ||||
| 160 | +\textsc{pending}: the \code{powermetrics} sampler (\code{benchmarks/scripts/power\_sampler.py}) is written; no energy run has been | ||||
| 161 | +logged in this version. | ||||
modified
research/LOG.md
+7 −0
@@ -144,3 +144,10 @@ Chronological, append-only. Times are local (America/Toronto). Every entry names | |||||
| 144 | prefetches/token and 25 late waits → 25.3; probe2 31.3 (hit 0.74), probe3 30.7 (hit 0.72). Every predictor converts 7–11 ms of saved | 144 | prefetches/token and 25 late waits → 25.3; probe2 31.3 (hit 0.74), probe3 30.7 (hit 0.72). Every predictor converts 7–11 ms of saved | ||
| 145 | reads into ≈ 5 ms of extra GPU-side install work plus 1–3 ms of late waits: on this model and tier the install path (9 lazy slice | 145 | reads into ≈ 5 ms of extra GPU-side install work plus 1–3 ms of late waits: on this model and tier the install path (9 lazy slice | ||
| 146 | writes ≈ 0.13 ms per expert) is the bottleneck, not prediction accuracy. Identified fix for v0.2: single-copy record install. | 146 | writes ≈ 0.13 ms per expert) is the bottleneck, not prediction accuracy. Identified fix for v0.2: single-copy record install. | ||
| 147 | +- Probe depth simulation (`sim_olmoe_v2_ssd043_m8_probe_depths.json`, held-out, C = 16): recall@8 0.761 / 0.743 / 0.731 at depth 1 / 2 / 3, | ||||
| 148 | + hit 0.81 / 0.78 / 0.76 → the probe also keeps its accuracy with a longer lead. | ||||
| 149 | +- Gate F preparation: Qwen3-30B routing traces collected on M3U96a (`benchmarks/raw/gateF/traces_qwen3_m3u96a`, 65 prompts, 128 tokens); | ||||
| 150 | + contiguous layout built on M3U96a (6 144 records, 16.3 GB). The LAN copy M3U96a → M3U96b first failed on a **stale SSH host key** for | ||||
| 151 | + 192.168.2.85 in M3U96a's `known_hosts` (the LAN was renumbered by DHCP on 2026-09-21). Verified that the key presented on the LAN | ||||
| 152 | + (`SHA256:zcdc123MTDYreNavJ9lUAS1OWSFSzxvxipqaBSygHsA`) is M3U96b's actual `/etc/ssh/ssh_host_ed25519_key.pub` fingerprint before | ||||
| 153 | + replacing the entry (backup `~/.ssh/known_hosts.bak-20261101-wp33` on M3U96a); copy and layout on M3U96b restarted. | ||||
| 147 | 154 | ||||