1"""Results-dependent LaTeX tables + macros (called by scripts/make_tables.py). Reads run manifests/metrics under2experiments/runs (local copies), results/raw/*.json and results/analyzed/*.json. Every number is traceable to a run id."""3from __future__ import annotations45import glob, json, math, re6from collections import defaultdict7from pathlib import Path89import numpy as np1011OUT = Path("paper/tables")12RUNS = "experiments/runs"131415def _runs(prefix):16 return sorted(d for d in glob.glob(f"{RUNS}/{prefix}_seed*") if Path(d, "manifest.json").exists())171819def _final_val(run):20 rows = [json.loads(l) for l in open(Path(run) / "metrics.jsonl")]21 last = [r for r in rows if any(k.startswith("val_loss_b") for k in r)]22 return last[-1] if last else {}232425def table_seed_matrix(macro):26 """Phase 4 decisive matrix: PRISME vs global recurrence vs dense, 5 seeds, val loss by budget and exact match."""27 systems = [("p4v2_prisme", "PRISME sparse editor"), ("p4v2_gr", "Global recurrence (inspired)"), ("p4v2_dense", "Dense 16-slot writes")]28 rows, raw = [], {}29 for pre, lab in systems:30 runs = [r for r in _runs(pre) if json.load(open(Path(r) / "manifest.json")).get("state") == "completed"]31 if not runs:32 continue33 vals = defaultdict(list); ems = defaultdict(list)34 for r in runs:35 fv = _final_val(r)36 for k, v in fv.items():37 if k.startswith("val_loss_b"):38 vals[int(k[10:])].append(v)39 fem = json.load(open(Path(r) / "manifest.json")).get("final_exact_match", {})40 for split, d in fem.items():41 for b, e in d.items():42 ems[(split, int(b))].append(e["accuracy"])43 raw[pre] = {"n_seeds": len(runs), "val": {b: v for b, v in vals.items()}, "em": {f"{s}_b{b}": v for (s, b), v in ems.items()}, "run_ids": [Path(r).name for r in runs]}44 rows.append((lab, len(runs), vals, ems))45 if not rows:46 return47 Path("results/analyzed").mkdir(parents=True, exist_ok=True)48 Path("results/analyzed/seed_matrix_v2.json").write_text(json.dumps(raw, indent=1))49 def ms(v):50 v = np.asarray(v, float)51 return f"{v.mean():.4f} $\\pm$ {v.std(ddof=1):.4f}" if len(v) > 1 else f"{v.mean():.4f}"52 def pct(v):53 v = 100 * np.asarray(v, float)54 return f"{v.mean():.1f} $\\pm$ {v.std(ddof=1):.1f}" if len(v) > 1 else f"{v.mean():.1f}"55 head = r"""\begin{table}[t]\centering\small56\caption{Phase 4 decisive comparison on the frozen 5.6M-parameter synthetic backbone (char-level): held-out57answer-span loss on dev-iid under random forced actions at fixed budgets, and free-generation exact match58(dev-iid, 1,000 stratified items). Mean $\pm$ SD over independent training seeds of the adapter; the backbone,59data, recipe and symmetry-breaking device are identical. Budget 0 is the frozen backbone for every system.}60\label{tab:seedmatrix}61\begin{tabular}{llrrrrr}\toprule62System & seeds & loss b=0 & loss b=1 & loss b=2 & loss b=4 & loss b=8\\\midrule63"""64 body = ""65 for lab, n, vals, ems in rows:66 body += f"{lab} & {n} & " + " & ".join(ms(vals.get(b, [float('nan')])) for b in [0, 1, 2, 4, 8]) + r"\\" + "\n"67 body += r"\midrule" + "\n" + r"\multicolumn{7}{l}{\emph{Exact match (\%), dev-iid}: b=0 / b=1 / b=2 / b=8}\\" + "\n"68 for lab, n, vals, ems in rows:69 cells = [pct(ems.get(("dev_iid", b), [float("nan")])) for b in [0, 1, 2, 8]]70 body += f"{lab} & {n} & \\multicolumn{{5}}{{l}}{{{' / '.join(cells)}}}\\\\\n"71 body += r"\multicolumn{7}{l}{\emph{Exact match (\%), held-out template / depth / compose at b=2}}\\" + "\n"72 for lab, n, vals, ems in rows:73 cells = [pct(ems.get((s, 2), [float("nan")])) for s in ["dev_template", "dev_depth", "dev_compose"]]74 body += f"{lab} & {n} & \\multicolumn{{5}}{{l}}{{{' / '.join(cells)}}}\\\\\n"75 (OUT / "seed_matrix.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")76 for pre, lab in systems:77 tag = pre.replace("p4v2_", "vtwo")78 if pre not in raw: # system not (yet) completed: emit explicit placeholders so the manuscript compiles and the audit flags them79 macro(f"{tag}seeds", "0")80 for bn in ("one", "two", "four", "eight"):81 macro(f"{tag}gainb{bn}", "n/a")82 for e in ("embzero", "embtwo", "embeight"):83 macro(f"{tag}{e}", "n/a")84 if pre in raw:85 v = raw[pre]["val"]86 for b, bn in [(1, "one"), (2, "two"), (4, "four"), (8, "eight")]:87 if 0 in v and b in v:88 macro(f"{tag}gainb{bn}", f"{100 * (1 - np.mean(v[b]) / np.mean(v[0])):.1f}")89 macro(f"{tag}seeds", str(raw[pre]["n_seeds"]))90 em = raw[pre]["em"]91 if "dev_iid_b0" in em and "dev_iid_b2" in em:92 macro(f"{tag}embzero", f"{100 * np.mean(em['dev_iid_b0']):.1f}"); macro(f"{tag}embtwo", f"{100 * np.mean(em['dev_iid_b2']):.1f}")93 macro(f"{tag}embeight", f"{100 * np.mean(em.get('dev_iid_b8', em['dev_iid_b2'])):.1f}")94 macro("vtwobzeroloss", f"{np.mean(raw[systems[0][0]]['val'][0]):.4f}" if systems[0][0] in raw else "n/a")959697def table_routing(macro):98 files = sorted(glob.glob("results/raw/routing_v1ctrl.json")) or sorted(glob.glob("results/raw/routing_*.json"))99 if not files:100 return101 d = json.load(open(files[-1]))102 head = r"""\begin{table}[t]\centering\small103\caption{Routing comparison on dev-iid (tiny scale, editor checkpoint of the locked run): held-out answer-span loss,104executed edits per token and analytic extra operations per token (relative to the backbone). Oracles read the105target and are diagnostic ceilings, not deployable systems.}106\label{tab:routing}107\begin{tabular}{llrrr}\toprule108Policy & setting & loss & edits/token & extra ops\\\midrule109"""110 body = ""111 for r in d["records"]:112 if r["policy"] == "oracle_two_step":113 continue114 setting = ", ".join(f"{k}={v}" for k, v in r.items() if k in ("budget", "threshold", "lam", "kap") and v is not None)115 body += f"{r['policy'].replace('_', ' ')}{' (diag.)' if r.get('diagnostic') else ''} & {setting} & {r['loss']:.4f} & {r['edits_per_token']:.2f} & {r['extra_ops_ratio']:.3f}\\\\\n"116 def pick(pol, **kw):117 for r in d["records"]:118 if r["policy"] == pol and all(r.get(k) == v for k, v in kw.items()):119 return r120 return None121 b0 = pick("budget0")122 if b0:123 macro("routingbzero", f"{b0['loss']:.4f}")124 for pol, tag in [("uniform", "uniform"), ("random_forced", "random"), ("fixed_pair", "pair")]:125 for b in (1, 2, 8):126 r = pick(pol, budget=b)127 if r:128 macro(f"routing{tag}b{'one' if b == 1 else 'two' if b == 2 else 'eight'}", f"{100 * (r['loss'] / b0['loss'] - 1):+.1f}")129 r = pick("controller_locked")130 if r:131 macro("routingctrllocked", f"{100 * (r['loss'] / b0['loss'] - 1):+.1f}"); macro("routingctrlledits", f"{r['edits_per_token']:.2f}"); macro("routingctrllambda", f"{r['lam']}")132 for b in (1, 8):133 r = pick("oracle1_greedy", budget=b)134 if r:135 macro(f"routingoracleb{'one' if b == 1 else 'eight'}", f"{100 * (r['loss'] / b0['loss'] - 1):+.1f}"); macro(f"routingoracleedits{'one' if b == 1 else 'eight'}", f"{r['edits_per_token']:.2f}")136 best_ent = min((r for r in d["records"] if r["policy"] == "entropy"), key=lambda r: r["loss"], default=None)137 best_norm = min((r for r in d["records"] if r["policy"] == "norm"), key=lambda r: r["loss"], default=None)138 if best_ent: macro("routingentropybest", f"{100 * (best_ent['loss'] / b0['loss'] - 1):+.1f}")139 if best_norm: macro("routingnormbest", f"{100 * (best_norm['loss'] / b0['loss'] - 1):+.1f}")140 two = [r for r in d["records"] if r["policy"] == "oracle_two_step"]141 if two:142 t = two[0]143 body += r"\midrule" + f"\\multicolumn{{5}}{{l}}{{Two-step oracle (diag.): one-step {t['one_step_gain']:.4f}, two-step {t['two_step_gain']:.4f}, greedy-then-best {t['greedy_then_best_gain']:.4f} nats/token}}\\\\\n"144 macro("oracleonestep", f"{t['one_step_gain']:.4f}"); macro("oracletwostep", f"{t['two_step_gain']:.4f}")145 (OUT / "routing.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")146147148def table_controller(macro):149 runs = sorted(glob.glob(f"{RUNS}/controller_p4*_seed*/manifest.json"))150 if not runs:151 return152 head = r"""\begin{table}[t]\centering\small153\caption{Value-controller fit on logged counterfactual labels (tiny scale). Calibration-partition diagnostics:154Huber loss of $\hat g$, sign accuracy, Brier score and ECE of the harm head, one-step oracle opportunity and the155regret of the controller's greedy choice on the exhaustive-audit states (captured fraction $=1-$ regret/opportunity).}156\label{tab:controller}157\begin{tabular}{lrrrrrrr}\toprule158Controller features & labels & Huber & sign acc. & Brier & ECE & opportunity & captured\\\midrule159"""160 body = ""161 for mp in runs:162 m = json.load(open(mp))163 if m.get("state") != "completed":164 continue165 ev = m.get("eval_calib", {})166 feats = "h + entropy" if "hent" in m["run_id"] else "z only"167 body += f"{feats} & {ev.get('n_labels', 0):,} & {ev.get('huber', float('nan')):.4f} & {ev.get('sign_acc', float('nan')):.3f} & {ev.get('harm_brier', float('nan')):.3f} & {ev.get('harm_ece', float('nan')):.3f} & {ev.get('oracle_opportunity_per_state', float('nan')):.4f} & {ev.get('captured_fraction', float('nan')):.3f}\\\\\n"168 macro("ctrl" + ("hent" if "hent" in m["run_id"] else "z") + "signacc", f"{100 * ev.get('sign_acc', float('nan')):.1f}")169 (OUT / "controller.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")170171172def table_labels(macro):173 files = sorted(glob.glob("results/analyzed/labels_labels_v1ctrl*.json")) or sorted(glob.glob("results/analyzed/labels_*.json"))174 if not files:175 return176 d = json.load(open(files[-1]))177 for part in ("train", "calib"):178 if part in d and "audit" in d[part]:179 a = d[part]["audit"]180 macro(f"labels{part}within", f"{100 * a['within_share']:.0f}")181 macro(f"labels{part}allpos", f"{100 * a['frac_states_all_positive']:.0f}")182 macro(f"labels{part}allneg", f"{100 * a['frac_states_all_negative']:.0f}")183 macro(f"labels{part}oracle", f"{a['oracle_one_step_mean']:.4f}")184 macro(f"labels{part}stoponly", f"{a['oracle_state_level_mean']:.4f}")185 macro(f"labels{part}random", f"{a['random_action_mean']:.4f}")186 macro(f"labels{part}nstates", f"{a['n_states']:,}")187 if "random_gain_high_vs_low_base_loss" in a:188 macro(f"labels{part}oraclehigh", f"{a['oracle_gain_high_vs_low_base_loss']['high']:.4f}")189 macro(f"labels{part}randomhigh", f"{a['random_gain_high_vs_low_base_loss']['high']:+.4f}")190 macro(f"labels{part}topdecoracle", f"{a['top_decile_base_loss']['oracle']:.3f}"); macro(f"labels{part}topdecrandom", f"{a['top_decile_base_loss']['random']:+.3f}")191 macro(f"labels{part}fracposhigh", f"{100 * a['frac_actions_positive_high_vs_low']['high']:.0f}"); macro(f"labels{part}fracposlow", f"{100 * a['frac_actions_positive_high_vs_low']['low']:.0f}")192193194def table_latency(macro):195 files = sorted(glob.glob("results/raw/latency_*.json"))196 if not files:197 return198 d = json.load(open(files[-1]))199 head = r"""\begin{table}[t]\centering\small200\caption{Measured decode latency per generated token (fp32, prompt %d tokens, %d generated, warm-up and201synchronisation), device %s. Median and p95 over generated tokens of three repetitions.}202\label{tab:latency}203\begin{tabular}{llrrrr}\toprule204Policy & budget & batch & median ms & p95 ms & cache bytes\\\midrule205""" % (d["prompt_len"], d["n_new"], d["device"].replace("_", " "))206 body = ""207 for r in d["records"]:208 body += f"{r['policy'].replace('_', ' ')} & {r['budget']} & {r['batch']} & {r['median_ms']:.2f} & {r['p95_ms']:.2f} & {r['cache_bytes']:,}\\\\\n"209 (OUT / "latency.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")210 b0 = [r for r in d["records"] if r["policy"] == "budget0" and r["batch"] == 1]211 if b0:212 macro("latencybzero", f"{b0[0]['median_ms']:.2f}")213214215def make_all(macro):216 table_seed_matrix(macro); table_routing(macro); table_controller(macro); table_labels(macro); table_latency(macro)217218219def table_routing_seeds(macro):220 """Per-seed routing summary for the v2 PRISME editors: budget 0, best fixed schedule, locked controller, oracle."""221 files = sorted(glob.glob("results/raw/routing_v2s*.json"))222 if not files:223 return224 rows = []225 for f in files:226 d = json.load(open(f)); recs = d["records"]227 def pick(pol, **kw):228 for r in recs:229 if r["policy"] == pol and all(r.get(k) == v for k, v in kw.items()):230 return r231 b0 = pick("budget0"); fixed = [r for r in recs if r["policy"] in ("random_forced", "uniform", "fixed_pair")]232 best_fixed = min(fixed, key=lambda r: r["loss"]); ctl = pick("controller_locked"); ora = pick("oracle1_greedy", budget=8)233 ent = min((r for r in recs if r["policy"] == "entropy"), key=lambda r: r["loss"]); nrm = min((r for r in recs if r["policy"] == "norm"), key=lambda r: r["loss"])234 seed = Path(f).stem.replace("routing_v2s", "")235 man = sorted(glob.glob(f"{RUNS}/controller_v2s{seed}_seed*/manifest.json"))236 ev = json.load(open(man[-1])).get("eval_calib", {}) if man else {}237 rows.append({"seed": seed, "b0": b0["loss"], "best_fixed": best_fixed["loss"], "best_fixed_name": f"{best_fixed['policy'].replace('_', ' ')} b={best_fixed['budget']}",238 "entropy": ent["loss"], "norm": nrm["loss"], "ctrl": ctl["loss"], "ctrl_edits": ctl["edits_per_token"], "ctrl_lam": ctl["lam"], "ctrl_kap": ctl["kap"],239 "oracle8": ora["loss"], "oracle8_edits": ora["edits_per_token"], "sign_acc": ev.get("sign_acc", float("nan")), "captured": ev.get("captured_fraction", float("nan"))})240 Path("results/analyzed/routing_seeds_v2.json").write_text(json.dumps(rows, indent=1))241 head = r"""\begin{table}[t]\centering\small242\caption{Learned routing per adapter seed (v2 backbone, dev-iid, 640 items): held-out answer-span loss of the243frozen backbone, of the best fixed schedule (named), of the best entropy and norm routers, of the locked244controller ($\lambda,\kappa$ selected on calibration rollouts; edits per token in parentheses), and of the245diagnostic greedy one-step oracle at budget 8. Calibration sign accuracy and captured fraction of the oracle246opportunity for each controller.}247\label{tab:routingseeds}248\begin{tabular}{lrlrrrrrr}\toprule249seed & budget 0 & best fixed & entropy & norm & controller (edits) & oracle b=8 (edits) & sign acc. & captured\\\midrule250"""251 body = ""252 for r in rows:253 body += (f"{r['seed']} & {r['b0']:.4f} & {r['best_fixed']:.4f} ({r['best_fixed_name']}) & {r['entropy']:.4f} & {r['norm']:.4f} & "254 f"{r['ctrl']:.4f} ({r['ctrl_edits']:.2f}) & {r['oracle8']:.4f} ({r['oracle8_edits']:.2f}) & {r['sign_acc']:.3f} & {r['captured']:+.3f}\\\\\n")255 (OUT / "routing_seeds.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")256 b0 = np.array([r["b0"] for r in rows]); ctl = np.array([r["ctrl"] for r in rows]); bf = np.array([r["best_fixed"] for r in rows]); ora = np.array([r["oracle8"] for r in rows])257 macro("vtwoctrlseeds", str(len(rows))); macro("vtwoctrlrel", f"{100 * (ctl / b0 - 1).mean():+.1f}"); macro("vtwoctrlrelsd", f"{100 * (ctl / b0 - 1).std(ddof=1) if len(rows) > 1 else 0:.1f}")258 macro("vtwobestfixedrel", f"{100 * (bf / b0 - 1).mean():+.1f}"); macro("vtwooraclerel", f"{100 * (ora / b0 - 1).mean():+.1f}")259 macro("vtwoctrlbetter", str(int((ctl < b0).sum())))260261262_old_make_all = make_all263def make_all(macro): # noqa: F811264 _old_make_all(macro); table_routing_seeds(macro)265266267def table_em_families(macro):268 """Per-family exact match (dev-iid) averaged over seeds, by system and budget, with correct<->incorrect transitions."""269 systems = [("p4v2_prisme", "PRISME"), ("p4v2_gr", "Global rec."), ("p4v2_dense", "Dense")]270 data = {}271 for pre, lab in systems:272 for run in _runs(pre):273 if json.load(open(Path(run) / "manifest.json")).get("state") != "completed":274 continue275 recs = {}276 for b in (0, 1, 2, 8):277 p = Path(run) / f"em_dev_iid_b{b}.json"278 if p.exists():279 recs[b] = {r["uid"]: r for r in json.load(open(p))}280 if 0 in recs:281 data.setdefault(pre, []).append(recs)282 if not data:283 return284 fams = ["arith", "bind", "fsm", "excl", "sched"]285 head = r"""\begin{table}[t]\centering\small286\caption{Per-family exact match on dev-iid (1,000 stratified items; mean over adapter seeds) at budgets 0, 2 and2878, and item transitions between budget 0 and budget 2 (wrong$\to$right / right$\to$wrong, per 1,000 items).}288\label{tab:emfamilies}289\begin{tabular}{llrrrrrr}\toprule290System & budget & """ + " & ".join(fams) + r""" & transitions ($+$/$-$)\\\midrule291"""292 body = ""293 for pre, lab in systems:294 if pre not in data:295 continue296 for b in (0, 2, 8):297 cells = []298 for fam in fams:299 vals = [np.mean([r["correct"] for r in recs[b].values() if r["family"] == fam]) for recs in data[pre] if b in recs]300 cells.append(f"{100 * np.mean(vals):.1f}" if vals else "n/a")301 tr = ""302 if b == 2:303 plus, minus = [], []304 for recs in data[pre]:305 if 2 in recs:306 common = set(recs[0]) & set(recs[2])307 plus.append(sum(1 for u in common if not recs[0][u]["correct"] and recs[2][u]["correct"]) / len(common) * 1000)308 minus.append(sum(1 for u in common if recs[0][u]["correct"] and not recs[2][u]["correct"]) / len(common) * 1000)309 if plus:310 tr = f"{np.mean(plus):.0f} / {np.mean(minus):.0f}"311 macro(f"{pre.replace('p4v2_', 'vtwo')}transplus", f"{np.mean(plus):.0f}"); macro(f"{pre.replace('p4v2_', 'vtwo')}transminus", f"{np.mean(minus):.0f}")312 body += f"{lab} & {b} & " + " & ".join(cells) + f" & {tr}\\\\\n"313 (OUT / "em_families.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")314315316_old_make_all2 = make_all317def make_all(macro): # noqa: F811318 _old_make_all2(macro); table_em_families(macro)319320321def table_pilot(macro):322 """103M pilot: Stage A/B manifests, held-out perplexity by policy (evaluation.run), routing and latency at 103M."""323 import math324 rows = []325 def man(prefix):326 ms = sorted(glob.glob(f"{RUNS}/{prefix}_seed*/manifest.json"))327 return json.load(open(ms[-1])) if ms else None328 pre = man("pretrain103m"); en = man("pretrain_enlarged102m"); ed = man("editor_pilot_103m_lr3e4"); gr = man("gr_pilot_103m")329 if pre:330 macro("pilotbbtokens", f"{pre['train_tokens']/1e6:.1f}"); macro("pilotbbhours", f"{pre['wall_seconds']/3600:.1f}"); macro("pilotbbppl", f"{math.exp(pre['final_val_loss']):.1f}")331 macro("pilotbbpeakgb", f"{pre['peak_memory_bytes']/1e9:.0f}"); macro("pilotbbflops", f"{pre['estimated_train_flops']/1e16:.1f}")332 if en and en.get("state") == "completed":333 macro("pilotenppl", f"{math.exp(en['final_val_loss']):.1f}"); macro("pilotenhours", f"{en['wall_seconds']/3600:.1f}")334 else:335 macro("pilotenppl", "n/a"); macro("pilotenhours", "n/a")336 for m, tag in [(ed, "piloted"), (gr, "pilotgr")]:337 if m:338 fv = _final_val(f"{RUNS}/{m['run_id']}")339 b0 = fv.get("val_loss_b0"); 340 for b in (1, 2, 4, 8):341 macro(f"{tag}gainb{ {1:'one',2:'two',4:'four',8:'eight'}[b] }", f"{100*(1 - fv.get(f'val_loss_b{b}', b0)/b0):.2f}" if b0 else "n/a")342 macro(f"{tag}minutes", f"{m['wall_seconds']/60:.0f}"); macro(f"{tag}peakgb", f"{m['peak_memory_bytes']/1e9:.1f}"); macro(f"{tag}params", f"{m['n_trainable']:,}")343 else:344 for bn in ("one", "two", "four", "eight"):345 macro(f"{tag}gainb{bn}", "n/a")346 macro(f"{tag}minutes", "n/a"); macro(f"{tag}peakgb", "n/a"); macro(f"{tag}params", "n/a")347 # perplexity table from evaluation.run outputs348 evs = sorted(glob.glob("results/raw/eval_*_val.json"))349 if evs:350 head = r"""\begin{table}[t]\centering\small351\caption{103M pilot: held-out cross-entropy (nats/token) and perplexity on 327,680 FineWeb-Edu validation tokens352(40 $\times$ 16 $\times$ 512) under forced random budgets and, for PRISME, under the locked controller.}353\label{tab:pilotppl}354\begin{tabular}{llrr}\toprule355System & policy & loss & ppl\\\midrule356"""357 body = ""358 for f in evs:359 d = json.load(open(f))360 for pol, r in d.get("perplexity", {}).items():361 body += f"{d['system'].replace('_', ' ')} & {pol.replace('_', ' ')} & {r['loss']:.4f} & {r['ppl']:.2f}\\\\\n"362 if pol == "forced_b0":363 macro("pilotppl" + re.sub(r"[^a-z]", "", d["system"]), f"{r['ppl']:.2f}")364 (OUT / "pilot_ppl.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")365 rt = Path("results/raw/routing_103m_val.json")366 if rt.exists():367 d = json.load(open(rt)); recs = d["records"]368 def pick(pol, **kw):369 for r in recs:370 if r["policy"] == pol and all(r.get(k) == v for k, v in kw.items()):371 return r372 b0 = pick("budget0"); ctl = pick("controller_locked"); ora = pick("oracle1_greedy", budget=8); uni1 = pick("uniform", budget=1)373 if b0 and ctl:374 macro("pilotroutingctrl", f"{100*(ctl['loss']/b0['loss']-1):+.2f}"); macro("pilotroutingctrledits", f"{ctl['edits_per_token']:.2f}")375 if b0 and ora:376 macro("pilotroutingoracle", f"{100*(ora['loss']/b0['loss']-1):+.1f}"); macro("pilotroutingoracleedits", f"{ora['edits_per_token']:.2f}")377 if b0 and uni1:378 macro("pilotroutinguniformone", f"{100*(uni1['loss']/b0['loss']-1):+.2f}")379 head = r"""\begin{table}[t]\centering\small380\caption{103M pilot: routing comparison on validation text (20 $\times$ 8 $\times$ 512 tokens). Oracles are diagnostic.}381\label{tab:pilotrouting}382\begin{tabular}{llrrr}\toprule383Policy & setting & loss & edits/token & extra ops\\\midrule384"""385 body = ""386 for r in recs:387 if r["policy"] == "oracle_two_step":388 continue389 setting = ", ".join(f"{k}={v}" for k, v in r.items() if k in ("budget", "threshold", "lam", "kap") and v is not None)390 body += f"{r['policy'].replace('_', ' ')}{' (diag.)' if r.get('diagnostic') else ''} & {setting} & {r['loss']:.4f} & {r['edits_per_token']:.2f} & {r['extra_ops_ratio']:.3f}\\\\\n"391 (OUT / "pilot_routing.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")392 lt = Path("results/raw/latency_103m_mps.json")393 if lt.exists():394 d = json.load(open(lt))395 head = r"""\begin{table}[t]\centering\small396\caption{103M pilot: measured decode latency per generated token on %s (fp32, prompt %d tokens, %d generated, three397repetitions, median and p95), batch 1 and 8.}398\label{tab:pilotlatency}399\begin{tabular}{llrrrr}\toprule400Policy & budget & batch & median ms & p95 ms & cache bytes\\\midrule401""" % (d["device"], d["prompt_len"], d["n_new"])402 body = ""403 for r in d["records"]:404 body += f"{r['policy'].replace('_', ' ')} & {r['budget']} & {r['batch']} & {r['median_ms']:.2f} & {r['p95_ms']:.2f} & {r['cache_bytes']:,}\\\\\n"405 (OUT / "pilot_latency.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")406 b0 = [r for r in d["records"] if r["policy"] == "budget0" and r["batch"] == 1]; s8 = [r for r in d["records"] if r["policy"] == "forced_single" and r["budget"] == 8 and r["batch"] == 1]407 d8 = [r for r in d["records"] if r["policy"] == "dense_forced" and r["budget"] == 8 and r["batch"] == 1]; c = [r for r in d["records"] if r["policy"] == "controller" and r["batch"] == 1]408 if b0: macro("pilotlatbzero", f"{b0[0]['median_ms']:.1f}")409 if s8: macro("pilotlatseight", f"{s8[0]['median_ms']:.1f}")410 if d8: macro("pilotlatdeight", f"{d8[0]['median_ms']:.1f}")411 if c: macro("pilotlatctrl", f"{c[0]['median_ms']:.1f}")412413414_old_make_all3 = make_all415def make_all(macro): # noqa: F811416 _old_make_all3(macro); table_pilot(macro)417