Gate F figure/table generators; implementation section: threads and dtypes lessons
3 changed files +38 −2
modified
paper/sections/implementation.tex
+6 −0
@@ -21,6 +21,12 @@ of \code{pread}, 0.06~ms of array creation and the remainder in the lazy install | |||||
| 21 | $\max|\Delta\mathrm{logit}| = 0.0195$ against mlx-lm; using mlx-lm's compiled SwiGLU gives $0.0$. Numerical parity in float16 is | 21 | $\max|\Delta\mathrm{logit}| = 0.0195$ against mlx-lm; using mlx-lm's compiled SwiGLU gives $0.0$. Numerical parity in float16 is | ||
| 22 | a property of the exact kernel sequence, not only of the weights, and the test suite pins it. | 22 | a property of the exact kernel sequence, not only of the weights, and the test suite pins it. | ||
| 23 | 23 | ||||
| 24 | +\paragraph{Threads and dtypes.} Two defects were found by the exactness tests rather than by inspection. MLX arrays created in reader | ||||
| 25 | +threads cannot be evaluated in another thread (streams are thread-local), so reader threads now return NumPy views and the consuming | ||||
| 26 | +thread creates the MLX arrays. And the first transport cast activations to float16 on the wire, which is exact for OLMoE (float16) but | ||||
| 27 | +not for Qwen3 (bfloat16): the first two-node sweep on Qwen3 showed $\max|\Delta\mathrm{logit}| = 0.65$ and was discarded; activations and | ||||
| 28 | +results now travel in the model's dtype and the Qwen3 two-node path is bit-identical again. | ||||
| 29 | + | ||||
| 24 | \paragraph{Reproducibility.} Every benchmark writes JSON lines with the command, git commit, host, chip, memory, macOS, | 30 | \paragraph{Reproducibility.} Every benchmark writes JSON lines with the command, git commit, host, chip, memory, macOS, | ||
| 25 | library versions, seed and timestamps; raw logs are append-only under \code{benchmarks/raw/} and tables and figures are | 31 | library versions, seed and timestamps; raw logs are append-only under \code{benchmarks/raw/} and tables and figures are | ||
| 26 | generated from them by scripts. Model revisions are pinned by commit hash in \code{benchmarks/configs/models.yaml}; | 32 | generated from them by scripts. Model revisions are pinned by commit hash in \code{benchmarks/configs/models.yaml}; | ||
modified
scripts/make_figures.py
+24 −1
@@ -100,5 +100,28 @@ def fig_attribution(): | |||||
| 100 | ax.set_title("OLMoE, 16 slots, cold NVMe (M5 Max): where the decode step waits", fontsize=9) | 100 | ax.set_title("OLMoE, 16 slots, cold NVMe (M5 Max): where the decode step waits", fontsize=9) | ||
| 101 | save(fig, "attribution") | 101 | save(fig, "attribution") | ||
| 102 | 102 | ||||
| 103 | +def fig_gateF(): | ||||
| 104 | + p = os.path.join(ROOT, "benchmarks/raw/gateF/two_node_qwen3_m3u96a_m3u96b.jsonl") | ||||
| 105 | + if not os.path.exists(p): return | ||||
| 106 | + recs = [json.loads(l) for l in open(p) if l.strip()]; recs = [r for r in recs if not r["warmup"]] | ||||
| 107 | + groups = {} | ||||
| 108 | + for r in recs: groups.setdefault((r["capacity"], r["placement"].split(":")[0]), []).append(r) | ||||
| 109 | + caps = sorted({k[0] for k in groups}); pls = ["none", "odd", "popular"] | ||||
| 110 | + fig, axes = plt.subplots(1, 2, figsize=(7.2, 2.8)) | ||||
| 111 | + w = 0.25 | ||||
| 112 | + for i, pl in enumerate(pls): | ||||
| 113 | + tps = [statistics.median(x["gen_tps"] for x in groups.get((c, pl), [])) if (c, pl) in groups else 0 for c in caps] | ||||
| 114 | + axes[0].bar([j + (i - 1) * w for j in range(len(caps))], tps, width=w, color=PAL[i], label={"none": "local (cold NVMe)", "odd": "odd experts remote", "popular": "least popular remote"}[pl]) | ||||
| 115 | + st = [statistics.median(x["stall_s"] / x["gen_tokens"] * 1e3 for x in groups.get((c, pl), [])) if (c, pl) in groups else 0 for c in caps] | ||||
| 116 | + rw = [statistics.median((x["remote"]["remote_wait_s"] if x.get("remote") else 0) / x["gen_tokens"] * 1e3 for x in groups.get((c, pl), [])) if (c, pl) in groups else 0 for c in caps] | ||||
| 117 | + axes[1].bar([j + (i - 1) * w for j in range(len(caps))], st, width=w, color=PAL[i], alpha=0.5) | ||||
| 118 | + axes[1].bar([j + (i - 1) * w for j in range(len(caps))], rw, bottom=st, width=w, color=PAL[i]) | ||||
| 119 | + for ax in axes: | ||||
| 120 | + ax.set_xticks(range(len(caps))); ax.set_xticklabels([f"C={c} of 128" for c in caps]) | ||||
| 121 | + axes[0].set_ylabel("decode tok/s (median)"); axes[0].legend(frameon=False, fontsize=7) | ||||
| 122 | + axes[1].set_ylabel("ms per token: local stall (light) + remote wait (dark)") | ||||
| 123 | + fig.suptitle("Qwen3-30B-A3B, M3U96a coordinator (cold NVMe) + M3U96b worker over 1 GbE", fontsize=9) | ||||
| 124 | + save(fig, "gateF_qwen3") | ||||
| 125 | + | ||||
| 103 | if __name__ == "__main__": | 126 | if __name__ == "__main__": | ||
| 104 | − fig_sim_hit_vs_capacity(); fig_capacity_sweep(); fig_two_node(); fig_attribution() | 127 | + fig_sim_hit_vs_capacity(); fig_capacity_sweep(); fig_two_node(); fig_attribution(); fig_gateF() | ||
modified
scripts/make_tables.py
+8 −1
@@ -43,7 +43,7 @@ def bench_summary(paths, label, caption, keys=("backend", "capacity", "predictor | |||||
| 43 | for k in sorted(groups, key=lambda x: tuple(str(v) for v in x)): | 43 | for k in sorted(groups, key=lambda x: tuple(str(v) for v in x)): | ||
| 44 | g = groups[k] | 44 | g = groups[k] | ||
| 45 | med = lambda f: statistics.median([r[f] for r in g if r.get(f) is not None]) if any(r.get(f) is not None for r in g) else None | 45 | med = lambda f: statistics.median([r[f] for r in g if r.get(f) is not None]) if any(r.get(f) is not None for r in g) else None | ||
| 46 | − cfg = (f"C={k[1]}, {k[2]}, m={k[3]}" if keys[2] == "predictor" else f"C={k[1]}, fused={k[2]}") + (", cold" if k[4] else "") if k[0] == "titan" else "--" | 46 | + cfg = (f"C={k[1]}, {k[2]}, m={k[3]}" if keys[2] == "predictor" else (f"C={k[1]}, placement={k[2]}" if keys[2] == "placement" else f"C={k[1]}, fused={k[2]}")) + (", cold" if k[4] else "") if k[0].startswith("titan") else "--" | ||
| 47 | f = lambda v, fmt: (fmt % v) if v is not None else "--" | 47 | f = lambda v, fmt: (fmt % v) if v is not None else "--" | ||
| 48 | lines.append(f"{k[0].replace('_', '\\_')} & {cfg.replace('_', '\\_')} & {len(g)} & {f(med('gen_tps'), '%.1f')} & {f(med('itl_ms_p95'), '%.1f')} & {f(med('cache_hit_rate'), '%.3f')} & {f(med('stall_s'), '%.2f')} & {f(med('ttft_ms'), '%.0f')} \\\\") | 48 | lines.append(f"{k[0].replace('_', '\\_')} & {cfg.replace('_', '\\_')} & {len(g)} & {f(med('gen_tps'), '%.1f')} & {f(med('itl_ms_p95'), '%.1f')} & {f(med('cache_hit_rate'), '%.3f')} & {f(med('stall_s'), '%.2f')} & {f(med('ttft_ms'), '%.0f')} \\\\") | ||
| 49 | lines += ["\\bottomrule", "\\end{tabular}", "\\begin{tablenotes}\\footnotesize", "\\item Sources: " + ", ".join(f"\\code{{{os.path.relpath(p, ROOT).replace('_', '\\_')}}}" for p in paths) + ".", "\\end{tablenotes}", "\\end{table}"] | 49 | lines += ["\\bottomrule", "\\end{tabular}", "\\begin{tablenotes}\\footnotesize", "\\item Sources: " + ", ".join(f"\\code{{{os.path.relpath(p, ROOT).replace('_', '\\_')}}}" for p in paths) + ".", "\\end{tablenotes}", "\\end{table}"] | ||
@@ -63,6 +63,13 @@ if __name__ == "__main__": | |||||
| 63 | fused = os.path.join(ROOT, "benchmarks/raw/gateD/fused_upgate_laptop.jsonl") | 63 | fused = os.path.join(ROOT, "benchmarks/raw/gateD/fused_upgate_laptop.jsonl") | ||
| 64 | if os.path.exists(fused): | 64 | if os.path.exists(fused): | ||
| 65 | bench_summary([fused], "tab:fused", "H4: fused up+gate gather\\_qmm vs two kernels (laptop; 64 slots resident, 128 tokens, 12 runs; 16 slots cold, 64 tokens, 4 runs).", keys=("backend", "capacity", "fused_upgate", "m", "nocache", "split")) | 65 | bench_summary([fused], "tab:fused", "H4: fused up+gate gather\\_qmm vs two kernels (laptop; 64 slots resident, 128 tokens, 12 runs; 16 slots cold, 64 tokens, 4 runs).", keys=("backend", "capacity", "fused_upgate", "m", "nocache", "split")) | ||
| 66 | + gf = os.path.join(ROOT, "benchmarks/raw/gateF/two_node_qwen3_m3u96a_m3u96b.jsonl") | ||||
| 67 | + if os.path.exists(gf): | ||||
| 68 | + recs = [json.loads(l) for l in open(gf) if l.strip()] | ||||
| 69 | + for r in recs: r["placement"] = r["placement"].split(":")[0] | ||||
| 70 | + tmp = gf + ".tmp"; open(tmp, "w").write("\n".join(json.dumps(r) for r in recs) + "\n") | ||||
| 71 | + bench_summary([tmp], "tab:gateF", "Gate F: Qwen3-30B-A3B on M3U96a (bounded residency, cold NVMe) with M3U96b as expert worker over 1 GbE; placements: none (all local), odd (half remote, static), popular (least popular half remote, frequency-based); 64 tokens, held-out general/code, 8 runs per row.", keys=("backend", "capacity", "placement", "m", "nocache", "split")) | ||||
| 72 | + os.remove(tmp) | ||||
| 66 | q = os.path.join(ROOT, "benchmarks/raw/gateC/qwen3_m3u96a/baseline.jsonl") | 73 | q = os.path.join(ROOT, "benchmarks/raw/gateC/qwen3_m3u96a/baseline.jsonl") | ||
| 67 | if os.path.exists(q): | 74 | if os.path.exists(q): | ||
| 68 | bench_summary([q], "tab:qwen3", "Qwen3-30B-A3B-Instruct-2507 4-bit on M3U96a (M3 Ultra, 96 GB): mlx-lm vs TITAN all-resident (128 slots, preloaded), 64 tokens, 3 repeats.") | 75 | bench_summary([q], "tab:qwen3", "Qwen3-30B-A3B-Instruct-2507 4-bit on M3U96a (M3 Ultra, 96 GB): mlx-lm vs TITAN all-resident (128 slots, preloaded), 64 tokens, 3 repeats.") | ||
| 69 | 76 | ||||