1#!/usr/bin/env python32"""Generate LaTeX table fragments under paper/tables/ from processed/raw logs. Every number in the paper's tables comes from here."""3import json, os, sys, glob, statistics4ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))5OUT = os.path.join(ROOT, "paper", "tables"); os.makedirs(OUT, exist_ok=True)67def sim_table(path, label, caption):8 d = json.load(open(path)); rows = [r for r in d["results"] if r["predictor"] != "probe"] # buggy probe rows replaced below9 for extra in ("sim_olmoe_v2_ssd043_m8_probe_fixed.json", "sim_olmoe_v2_ssd043_m8_probe_depths.json"):10 q = os.path.join(os.path.dirname(path), extra)11 if os.path.exists(q):12 have = {r["predictor"] for r in rows}13 rows += [r for r in json.load(open(q))["results"] if r["predictor"] not in have]14 caps = sorted({r["capacity"] for r in rows}); preds = []15 for r in rows:16 if r["predictor"] not in preds: preds.append(r["predictor"])17 lines = ["\\begin{table}[t]", "\\centering\\small", f"\\caption{{{caption}}}", f"\\label{{{label}}}",18 "\\begin{tabular}{ll" + "rr" * len(caps) + "}", "\\toprule",19 "Predictor & split & " + " & ".join(f"\\multicolumn{{2}}{{c}}{{$C={c}$}}" for c in caps) + " \\\\",20 " & & " + " & ".join("hit & stall" for _ in caps) + " \\\\", "\\midrule"]21 for p in preds:22 for split in ("fit", "heldout"):23 cells = []24 for c in caps:25 m = [r for r in rows if r["predictor"] == p and r["split"] == split and r["capacity"] == c and r["policy"] == "lru"]26 cells.append(f"{m[0]['hit_rate']:.3f} & {m[0]['stall_ms_per_token']:.1f}" if m else "-- & --")27 lines.append(f"{p.replace('_', '\\_')} & {split} & " + " & ".join(cells) + " \\\\")28 lines += ["\\bottomrule", "\\end{tabular}", "\\begin{tablenotes}\\footnotesize", f"\\item Source: \\code{{{os.path.relpath(path, ROOT).replace('_', '\\_')}}}; t\\_read {d['t_read_ms']} ms, t\\_layer {d['t_layer_ms']} ms, {d['loaders']} loader(s); hit = per-layer hit rate, stall in ms per token (modelled).", "\\end{tablenotes}", "\\end{table}"]29 open(os.path.join(OUT, f"{label.split(':')[-1]}.tex"), "w").write("\n".join(lines) + "\n")30 print("wrote", label)3132def bench_summary(paths, label, caption, keys=("backend", "capacity", "predictor", "m", "nocache", "split")):33 """Median decode tok/s, hit rate and stall per configuration from bench_decode JSONL files (warm-up excluded)."""34 recs = []35 for p in paths:36 recs += [json.loads(l) for l in open(p) if l.strip()]37 recs = [r for r in recs if not r.get("warmup")]38 groups = {}39 for r in recs:40 k = tuple(r.get(x) for x in keys); groups.setdefault(k, []).append(r)41 lines = ["\\begin{table}[t]", "\\centering\\small", f"\\caption{{{caption}}}", f"\\label{{{label}}}",42 "\\begin{tabular}{llrrrrrr}", "\\toprule", "Backend & cfg & $n$ & decode tok/s (median) & p95 ITL ms & hit & stall s & TTFT ms \\\\", "\\midrule"]43 for k in sorted(groups, key=lambda x: tuple(str(v) for v in x)):44 g = groups[k]45 med = lambda f: statistics.median([r[f] for r in g if r.get(f) is not None]) if any(r.get(f) is not None for r in g) else None46 cfg = (f"C={k[1]}, {k[2]}, m={k[3]}" if keys[2] == "predictor" else (f"C={k[1]}, placement={k[2]}" if keys[2] == "placement" else f"C={k[1]}, fused={k[2]}")) + (", cold" if k[4] else "") if k[0].startswith("titan") else "--"47 f = lambda v, fmt: (fmt % v) if v is not None else "--"48 lines.append(f"{k[0].replace('_', '\\_')} & {cfg.replace('_', '\\_')} & {len(g)} & {f(med('gen_tps'), '%.1f')} & {f(med('itl_ms_p95'), '%.1f')} & {f(med('cache_hit_rate'), '%.3f')} & {f(med('stall_s'), '%.2f')} & {f(med('ttft_ms'), '%.0f')} \\\\")49 lines += ["\\bottomrule", "\\end{tabular}", "\\begin{tablenotes}\\footnotesize", "\\item Sources: " + ", ".join(f"\\code{{{os.path.relpath(p, ROOT).replace('_', '\\_')}}}" for p in paths) + ".", "\\end{tablenotes}", "\\end{table}"]50 open(os.path.join(OUT, f"{label.split(':')[-1]}.tex"), "w").write("\n".join(lines) + "\n")51 print("wrote", label)5253if __name__ == "__main__":54 v2 = os.path.join(ROOT, "benchmarks/processed/gateC/sim_olmoe_v2_ssd043_m8.json")55 v1 = os.path.join(ROOT, "benchmarks/processed/gateC/sim_olmoe_laptop_ssd_v1.json")56 sim_table(v2 if os.path.exists(v2) else v1, "tab:sim", "Trace-driven simulation (OLMoE, per-layer LRU tables, cold per prompt): hit rate and modelled stall per token by predictor, split and capacity $C$.")57 cold = os.path.join(ROOT, "benchmarks/raw/gateC/cold_nvme_laptop/sweep.jsonl")58 if os.path.exists(cold):59 bench_summary([cold], "tab:cold", "Cold-NVMe tier on the M5 Max laptop (purge + F\\_NOCACHE), OLMoE 4-bit, held-out general/code/reasoning prompts, 64 tokens, 12 runs per row.")60 attr = os.path.join(ROOT, "benchmarks/raw/gateC/cold_nvme_laptop/attribution.jsonl")61 if os.path.exists(attr):62 bench_summary([attr], "tab:attr", "Predictor comparison with host-time attribution, cold NVMe, 16 slots, held-out general prompts (4 runs per row); see Figure~\\ref{fig:attribution} for the breakdown.")63 fused = os.path.join(ROOT, "benchmarks/raw/gateD/fused_upgate_laptop.jsonl")64 if os.path.exists(fused):65 bench_summary([fused], "tab:fused", "H4: fused up+gate gather\\_qmm vs two kernels (laptop; 64 slots resident, 128 tokens, 12 runs; 16 slots cold, 64 tokens, 4 runs).", keys=("backend", "capacity", "fused_upgate", "m", "nocache", "split"))66 gf = os.path.join(ROOT, "benchmarks/raw/gateF/two_node_qwen3_m3u96a_m3u96b.jsonl")67 if os.path.exists(gf):68 recs = [json.loads(l) for l in open(gf) if l.strip()]69 for r in recs: r["placement"] = r["placement"].split(":")[0]70 tmp = gf + ".tmp"; open(tmp, "w").write("\n".join(json.dumps(r) for r in recs) + "\n")71 bench_summary([tmp], "tab:gateF", "Gate F: Qwen3-30B-A3B on M3U96a (bounded residency, cold NVMe) with M3U96b as expert worker over 1 GbE; placements: none (all local), odd (half remote, static), popular (least popular half remote, frequency-based); 64 tokens, held-out general/code, 8 runs per row.", keys=("backend", "capacity", "placement", "m", "nocache", "split"))72 os.remove(tmp)73 q = os.path.join(ROOT, "benchmarks/raw/gateC/qwen3_m3u96a/baseline.jsonl")74 if os.path.exists(q):75 bench_summary([q], "tab:qwen3", "Qwen3-30B-A3B-Instruct-2507 4-bit on M3U96a (M3 Ultra, 96 GB): mlx-lm vs TITAN all-resident (128 slots, preloaded), 64 tokens, 3 repeats.")76