1"""Results-dependent LaTeX tables + macros (called by scripts/make_tables.py). Reads run manifests/metrics under2experiments/runs (local copies), results/raw/*.json and results/analyzed/*.json. Every number is traceable to a run id."""3from __future__ import annotations45import glob, json, math, re6from collections import defaultdict7from pathlib import Path89import numpy as np1011OUT = Path("paper/tables")12RUNS = "experiments/runs"131415def _runs(prefix):16 return sorted(d for d in glob.glob(f"{RUNS}/{prefix}_seed*") if Path(d, "manifest.json").exists())171819def _final_val(run):20 rows = [json.loads(l) for l in open(Path(run) / "metrics.jsonl")]21 last = [r for r in rows if any(k.startswith("val_loss_b") for k in r)]22 return last[-1] if last else {}232425def table_seed_matrix(macro):26 """Phase 4 decisive matrix: PRISME vs global recurrence vs dense, 5 seeds, val loss by budget and exact match."""27 systems = [("p4v2_prisme", "PRISME sparse editor"), ("p4v2_gr", "Global recurrence (inspired)"), ("p4v2_dense", "Dense 16-slot writes")]28 rows, raw = [], {}29 for pre, lab in systems:30 runs = [r for r in _runs(pre) if json.load(open(Path(r) / "manifest.json")).get("state") == "completed"]31 if not runs:32 continue33 vals = defaultdict(list); ems = defaultdict(list)34 for r in runs:35 fv = _final_val(r)36 for k, v in fv.items():37 if k.startswith("val_loss_b"):38 vals[int(k[10:])].append(v)39 fem = json.load(open(Path(r) / "manifest.json")).get("final_exact_match", {})40 for split, d in fem.items():41 for b, e in d.items():42 ems[(split, int(b))].append(e["accuracy"])43 raw[pre] = {"n_seeds": len(runs), "val": {b: v for b, v in vals.items()}, "em": {f"{s}_b{b}": v for (s, b), v in ems.items()}, "run_ids": [Path(r).name for r in runs]}44 rows.append((lab, len(runs), vals, ems))45 if not rows:46 return47 Path("results/analyzed").mkdir(parents=True, exist_ok=True)48 Path("results/analyzed/seed_matrix_v2.json").write_text(json.dumps(raw, indent=1))49 def ms(v):50 v = np.asarray(v, float)51 return f"{v.mean():.4f} $\\pm$ {v.std(ddof=1):.4f}" if len(v) > 1 else f"{v.mean():.4f}"52 def pct(v):53 v = 100 * np.asarray(v, float)54 return f"{v.mean():.1f} $\\pm$ {v.std(ddof=1):.1f}" if len(v) > 1 else f"{v.mean():.1f}"55 head = r"""\begin{table}[t]\centering\small56\caption{Phase 4 decisive comparison on the frozen 5.6M-parameter synthetic backbone (char-level): held-out57answer-span loss on dev-iid under random forced actions at fixed budgets, and free-generation exact match58(dev-iid, 1,000 stratified items). Mean $\pm$ SD over independent training seeds of the adapter; the backbone,59data, recipe and symmetry-breaking device are identical. Budget 0 is the frozen backbone for every system.}60\label{tab:seedmatrix}61\begin{tabular}{llrrrrr}\toprule62System & seeds & loss b=0 & loss b=1 & loss b=2 & loss b=4 & loss b=8\\\midrule63"""64 body = ""65 for lab, n, vals, ems in rows:66 body += f"{lab} & {n} & " + " & ".join(ms(vals.get(b, [float('nan')])) for b in [0, 1, 2, 4, 8]) + r"\\" + "\n"67 body += r"\midrule" + "\n" + r"\multicolumn{7}{l}{\emph{Exact match (\%), dev-iid}: b=0 / b=1 / b=2 / b=8}\\" + "\n"68 for lab, n, vals, ems in rows:69 cells = [pct(ems.get(("dev_iid", b), [float("nan")])) for b in [0, 1, 2, 8]]70 body += f"{lab} & {n} & \\multicolumn{{5}}{{l}}{{{' / '.join(cells)}}}\\\\\n"71 body += r"\multicolumn{7}{l}{\emph{Exact match (\%), held-out template / depth / compose at b=2}}\\" + "\n"72 for lab, n, vals, ems in rows:73 cells = [pct(ems.get((s, 2), [float("nan")])) for s in ["dev_template", "dev_depth", "dev_compose"]]74 body += f"{lab} & {n} & \\multicolumn{{5}}{{l}}{{{' / '.join(cells)}}}\\\\\n"75 (OUT / "seed_matrix.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")76 for pre, lab in systems:77 tag = pre.replace("p4v2_", "vtwo")78 if pre not in raw: # system not (yet) completed: emit explicit placeholders so the manuscript compiles and the audit flags them79 macro(f"{tag}seeds", "0")80 for bn in ("one", "two", "four", "eight"):81 macro(f"{tag}gainb{bn}", "n/a")82 for e in ("embzero", "embtwo", "embeight"):83 macro(f"{tag}{e}", "n/a")84 if pre in raw:85 v = raw[pre]["val"]86 for b, bn in [(1, "one"), (2, "two"), (4, "four"), (8, "eight")]:87 if 0 in v and b in v:88 macro(f"{tag}gainb{bn}", f"{100 * (1 - np.mean(v[b]) / np.mean(v[0])):.1f}")89 macro(f"{tag}seeds", str(raw[pre]["n_seeds"]))90 em = raw[pre]["em"]91 if "dev_iid_b0" in em and "dev_iid_b2" in em:92 macro(f"{tag}embzero", f"{100 * np.mean(em['dev_iid_b0']):.1f}"); macro(f"{tag}embtwo", f"{100 * np.mean(em['dev_iid_b2']):.1f}")93 macro(f"{tag}embeight", f"{100 * np.mean(em.get('dev_iid_b8', em['dev_iid_b2'])):.1f}")94 macro("vtwobzeroloss", f"{np.mean(raw[systems[0][0]]['val'][0]):.4f}" if systems[0][0] in raw else "n/a")959697def table_routing(macro):98 files = sorted(glob.glob("results/raw/routing_v1ctrl.json")) or sorted(glob.glob("results/raw/routing_*.json"))99 if not files:100 return101 d = json.load(open(files[-1]))102 head = r"""\begin{table}[t]\centering\small103\caption{Routing comparison on dev-iid (tiny scale, editor checkpoint of the locked run): held-out answer-span loss,104executed edits per token and analytic extra operations per token (relative to the backbone). Oracles read the105target and are diagnostic ceilings, not deployable systems.}106\label{tab:routing}107\begin{tabular}{llrrr}\toprule108Policy & setting & loss & edits/token & extra ops\\\midrule109"""110 body = ""111 for r in d["records"]:112 if r["policy"] == "oracle_two_step":113 continue114 setting = ", ".join(f"{k}={v}" for k, v in r.items() if k in ("budget", "threshold", "lam", "kap") and v is not None)115 body += f"{r['policy'].replace('_', ' ')}{' (diag.)' if r.get('diagnostic') else ''} & {setting} & {r['loss']:.4f} & {r['edits_per_token']:.2f} & {r['extra_ops_ratio']:.3f}\\\\\n"116 def pick(pol, **kw):117 for r in d["records"]:118 if r["policy"] == pol and all(r.get(k) == v for k, v in kw.items()):119 return r120 return None121 b0 = pick("budget0")122 if b0:123 macro("routingbzero", f"{b0['loss']:.4f}")124 for pol, tag in [("uniform", "uniform"), ("random_forced", "random"), ("fixed_pair", "pair")]:125 for b in (1, 2, 8):126 r = pick(pol, budget=b)127 if r:128 macro(f"routing{tag}b{'one' if b == 1 else 'two' if b == 2 else 'eight'}", f"{100 * (r['loss'] / b0['loss'] - 1):+.1f}")129 r = pick("controller_locked")130 if r:131 macro("routingctrllocked", f"{100 * (r['loss'] / b0['loss'] - 1):+.1f}"); macro("routingctrlledits", f"{r['edits_per_token']:.2f}"); macro("routingctrllambda", f"{r['lam']}")132 for b in (1, 8):133 r = pick("oracle1_greedy", budget=b)134 if r:135 macro(f"routingoracleb{'one' if b == 1 else 'eight'}", f"{100 * (r['loss'] / b0['loss'] - 1):+.1f}"); macro(f"routingoracleedits{'one' if b == 1 else 'eight'}", f"{r['edits_per_token']:.2f}")136 best_ent = min((r for r in d["records"] if r["policy"] == "entropy"), key=lambda r: r["loss"], default=None)137 best_norm = min((r for r in d["records"] if r["policy"] == "norm"), key=lambda r: r["loss"], default=None)138 if best_ent: macro("routingentropybest", f"{100 * (best_ent['loss'] / b0['loss'] - 1):+.1f}")139 if best_norm: macro("routingnormbest", f"{100 * (best_norm['loss'] / b0['loss'] - 1):+.1f}")140 two = [r for r in d["records"] if r["policy"] == "oracle_two_step"]141 if two:142 t = two[0]143 body += r"\midrule" + f"\\multicolumn{{5}}{{l}}{{Two-step oracle (diag.): one-step {t['one_step_gain']:.4f}, two-step {t['two_step_gain']:.4f}, greedy-then-best {t['greedy_then_best_gain']:.4f} nats/token}}\\\\\n"144 macro("oracleonestep", f"{t['one_step_gain']:.4f}"); macro("oracletwostep", f"{t['two_step_gain']:.4f}")145 (OUT / "routing.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")146147148def table_controller(macro):149 runs = sorted(glob.glob(f"{RUNS}/controller_p4*_seed*/manifest.json"))150 if not runs:151 return152 head = r"""\begin{table}[t]\centering\small153\caption{Value-controller fit on logged counterfactual labels (tiny scale). Calibration-partition diagnostics:154Huber loss of $\hat g$, sign accuracy, Brier score and ECE of the harm head, one-step oracle opportunity and the155regret of the controller's greedy choice on the exhaustive-audit states (captured fraction $=1-$ regret/opportunity).}156\label{tab:controller}157\begin{tabular}{lrrrrrrr}\toprule158Controller features & labels & Huber & sign acc. & Brier & ECE & opportunity & captured\\\midrule159"""160 body = ""161 for mp in runs:162 m = json.load(open(mp))163 if m.get("state") != "completed":164 continue165 ev = m.get("eval_calib", {})166 feats = "h + entropy" if "hent" in m["run_id"] else "z only"167 body += f"{feats} & {ev.get('n_labels', 0):,} & {ev.get('huber', float('nan')):.4f} & {ev.get('sign_acc', float('nan')):.3f} & {ev.get('harm_brier', float('nan')):.3f} & {ev.get('harm_ece', float('nan')):.3f} & {ev.get('oracle_opportunity_per_state', float('nan')):.4f} & {ev.get('captured_fraction', float('nan')):.3f}\\\\\n"168 macro("ctrl" + ("hent" if "hent" in m["run_id"] else "z") + "signacc", f"{100 * ev.get('sign_acc', float('nan')):.1f}")169 (OUT / "controller.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")170171172def table_labels(macro):173 files = sorted(glob.glob("results/analyzed/labels_labels_v1ctrl*.json")) or sorted(glob.glob("results/analyzed/labels_*.json"))174 if not files:175 return176 d = json.load(open(files[-1]))177 for part in ("train", "calib"):178 if part in d and "audit" in d[part]:179 a = d[part]["audit"]180 macro(f"labels{part}within", f"{100 * a['within_share']:.0f}")181 macro(f"labels{part}allpos", f"{100 * a['frac_states_all_positive']:.0f}")182 macro(f"labels{part}allneg", f"{100 * a['frac_states_all_negative']:.0f}")183 macro(f"labels{part}oracle", f"{a['oracle_one_step_mean']:.4f}")184 macro(f"labels{part}stoponly", f"{a['oracle_state_level_mean']:.4f}")185 macro(f"labels{part}random", f"{a['random_action_mean']:.4f}")186 macro(f"labels{part}nstates", f"{a['n_states']:,}")187 if "random_gain_high_vs_low_base_loss" in a:188 macro(f"labels{part}oraclehigh", f"{a['oracle_gain_high_vs_low_base_loss']['high']:.4f}")189 macro(f"labels{part}randomhigh", f"{a['random_gain_high_vs_low_base_loss']['high']:+.4f}")190 macro(f"labels{part}topdecoracle", f"{a['top_decile_base_loss']['oracle']:.3f}"); macro(f"labels{part}topdecrandom", f"{a['top_decile_base_loss']['random']:+.3f}")191 macro(f"labels{part}fracposhigh", f"{100 * a['frac_actions_positive_high_vs_low']['high']:.0f}"); macro(f"labels{part}fracposlow", f"{100 * a['frac_actions_positive_high_vs_low']['low']:.0f}")192193194def table_latency(macro):195 files = sorted(glob.glob("results/raw/latency_*.json"))196 if not files:197 return198 d = json.load(open(files[-1]))199 head = r"""\begin{table}[t]\centering\small200\caption{Measured decode latency per generated token (fp32, prompt %d tokens, %d generated, warm-up and201synchronisation), device %s. Median and p95 over generated tokens of three repetitions.}202\label{tab:latency}203\begin{tabular}{llrrrr}\toprule204Policy & budget & batch & median ms & p95 ms & cache bytes\\\midrule205""" % (d["prompt_len"], d["n_new"], d["device"].replace("_", " "))206 body = ""207 for r in d["records"]:208 body += f"{r['policy'].replace('_', ' ')} & {r['budget']} & {r['batch']} & {r['median_ms']:.2f} & {r['p95_ms']:.2f} & {r['cache_bytes']:,}\\\\\n"209 (OUT / "latency.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")210 b0 = [r for r in d["records"] if r["policy"] == "budget0" and r["batch"] == 1]211 if b0:212 macro("latencybzero", f"{b0[0]['median_ms']:.2f}")213214215def make_all(macro):216 table_seed_matrix(macro); table_routing(macro); table_controller(macro); table_labels(macro); table_latency(macro)217218219def table_routing_seeds(macro):220 """Per-seed routing summary for the v2 PRISME editors: budget 0, best fixed schedule, locked controller, oracle."""221 files = sorted(glob.glob("results/raw/routing_v2s*.json"))222 if not files:223 return224 rows = []225 for f in files:226 d = json.load(open(f)); recs = d["records"]227 def pick(pol, **kw):228 for r in recs:229 if r["policy"] == pol and all(r.get(k) == v for k, v in kw.items()):230 return r231 b0 = pick("budget0"); fixed = [r for r in recs if r["policy"] in ("random_forced", "uniform", "fixed_pair")]232 best_fixed = min(fixed, key=lambda r: r["loss"]); ctl = pick("controller_locked"); ora = pick("oracle1_greedy", budget=8)233 ent = min((r for r in recs if r["policy"] == "entropy"), key=lambda r: r["loss"]); nrm = min((r for r in recs if r["policy"] == "norm"), key=lambda r: r["loss"])234 seed = Path(f).stem.replace("routing_v2s", "")235 man = sorted(glob.glob(f"{RUNS}/controller_v2s{seed}_seed*/manifest.json"))236 ev = json.load(open(man[-1])).get("eval_calib", {}) if man else {}237 rows.append({"seed": seed, "b0": b0["loss"], "best_fixed": best_fixed["loss"], "best_fixed_name": f"{best_fixed['policy'].replace('_', ' ')} b={best_fixed['budget']}",238 "entropy": ent["loss"], "norm": nrm["loss"], "ctrl": ctl["loss"], "ctrl_edits": ctl["edits_per_token"], "ctrl_lam": ctl["lam"], "ctrl_kap": ctl["kap"],239 "oracle8": ora["loss"], "oracle8_edits": ora["edits_per_token"], "sign_acc": ev.get("sign_acc", float("nan")), "captured": ev.get("captured_fraction", float("nan"))})240 Path("results/analyzed/routing_seeds_v2.json").write_text(json.dumps(rows, indent=1))241 head = r"""\begin{table}[t]\centering\small242\caption{Learned routing per adapter seed (v2 backbone, dev-iid, 640 items): held-out answer-span loss of the243frozen backbone, of the best fixed schedule (named), of the best entropy and norm routers, of the locked244controller ($\lambda,\kappa$ selected on calibration rollouts; edits per token in parentheses), and of the245diagnostic greedy one-step oracle at budget 8. Calibration sign accuracy and captured fraction of the oracle246opportunity for each controller.}247\label{tab:routingseeds}248\begin{tabular}{lrlrrrrrr}\toprule249seed & budget 0 & best fixed & entropy & norm & controller (edits) & oracle b=8 (edits) & sign acc. & captured\\\midrule250"""251 body = ""252 for r in rows:253 body += (f"{r['seed']} & {r['b0']:.4f} & {r['best_fixed']:.4f} ({r['best_fixed_name']}) & {r['entropy']:.4f} & {r['norm']:.4f} & "254 f"{r['ctrl']:.4f} ({r['ctrl_edits']:.2f}) & {r['oracle8']:.4f} ({r['oracle8_edits']:.2f}) & {r['sign_acc']:.3f} & {r['captured']:+.3f}\\\\\n")255 (OUT / "routing_seeds.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")256 b0 = np.array([r["b0"] for r in rows]); ctl = np.array([r["ctrl"] for r in rows]); bf = np.array([r["best_fixed"] for r in rows]); ora = np.array([r["oracle8"] for r in rows])257 macro("vtwoctrlseeds", str(len(rows))); macro("vtwoctrlrel", f"{100 * (ctl / b0 - 1).mean():+.1f}"); macro("vtwoctrlrelsd", f"{100 * (ctl / b0 - 1).std(ddof=1) if len(rows) > 1 else 0:.1f}")258 macro("vtwobestfixedrel", f"{100 * (bf / b0 - 1).mean():+.1f}"); macro("vtwooraclerel", f"{100 * (ora / b0 - 1).mean():+.1f}")259 macro("vtwoctrlbetter", str(int((ctl < b0).sum())))260261262_old_make_all = make_all263def make_all(macro): # noqa: F811264 _old_make_all(macro); table_routing_seeds(macro)265266267def table_em_families(macro):268 """Per-family exact match (dev-iid) averaged over seeds, by system and budget, with correct<->incorrect transitions."""269 systems = [("p4v2_prisme", "PRISME"), ("p4v2_gr", "Global rec."), ("p4v2_dense", "Dense")]270 data = {}271 for pre, lab in systems:272 for run in _runs(pre):273 if json.load(open(Path(run) / "manifest.json")).get("state") != "completed":274 continue275 recs = {}276 for b in (0, 1, 2, 8):277 p = Path(run) / f"em_dev_iid_b{b}.json"278 if p.exists():279 recs[b] = {r["uid"]: r for r in json.load(open(p))}280 if 0 in recs:281 data.setdefault(pre, []).append(recs)282 if not data:283 return284 fams = ["arith", "bind", "fsm", "excl", "sched"]285 head = r"""\begin{table}[t]\centering\small286\caption{Per-family exact match on dev-iid (1,000 stratified items; mean over adapter seeds) at budgets 0, 2 and2878, and item transitions between budget 0 and budget 2 (wrong$\to$right / right$\to$wrong, per 1,000 items).}288\label{tab:emfamilies}289\begin{tabular}{llrrrrrr}\toprule290System & budget & """ + " & ".join(fams) + r""" & transitions ($+$/$-$)\\\midrule291"""292 body = ""293 for pre, lab in systems:294 if pre not in data:295 continue296 for b in (0, 2, 8):297 cells = []298 for fam in fams:299 vals = [np.mean([r["correct"] for r in recs[b].values() if r["family"] == fam]) for recs in data[pre] if b in recs]300 cells.append(f"{100 * np.mean(vals):.1f}" if vals else "n/a")301 tr = ""302 if b == 2:303 plus, minus = [], []304 for recs in data[pre]:305 if 2 in recs:306 common = set(recs[0]) & set(recs[2])307 plus.append(sum(1 for u in common if not recs[0][u]["correct"] and recs[2][u]["correct"]) / len(common) * 1000)308 minus.append(sum(1 for u in common if recs[0][u]["correct"] and not recs[2][u]["correct"]) / len(common) * 1000)309 if plus:310 tr = f"{np.mean(plus):.0f} / {np.mean(minus):.0f}"311 macro(f"{pre.replace('p4v2_', 'vtwo')}transplus", f"{np.mean(plus):.0f}"); macro(f"{pre.replace('p4v2_', 'vtwo')}transminus", f"{np.mean(minus):.0f}")312 body += f"{lab} & {b} & " + " & ".join(cells) + f" & {tr}\\\\\n"313 (OUT / "em_families.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")314315316_old_make_all2 = make_all317def make_all(macro): # noqa: F811318 _old_make_all2(macro); table_em_families(macro)319320321def table_pilot(macro):322 """103M pilot: Stage A/B manifests, held-out perplexity by policy (evaluation.run), routing and latency at 103M."""323 import math324 rows = []325 def man(prefix):326 ms = sorted(glob.glob(f"{RUNS}/{prefix}_seed*/manifest.json"))327 return json.load(open(ms[-1])) if ms else None328 pre = man("pretrain103m"); en = man("pretrain_enlarged102m"); ed = man("editor_pilot_103m_lr3e4"); gr = man("gr_pilot_103m")329 if pre:330 macro("pilotbbtokens", f"{pre['train_tokens']/1e6:.1f}"); macro("pilotbbhours", f"{pre['wall_seconds']/3600:.1f}"); macro("pilotbbppl", f"{math.exp(pre['final_val_loss']):.1f}")331 macro("pilotbbpeakgb", f"{pre['peak_memory_bytes']/1e9:.0f}"); macro("pilotbbflops", f"{pre['estimated_train_flops']/1e16:.1f}")332 if en and en.get("state") == "completed":333 macro("pilotenppl", f"{math.exp(en['final_val_loss']):.1f}"); macro("pilotenhours", f"{en['wall_seconds']/3600:.1f}")334 else:335 macro("pilotenppl", "n/a"); macro("pilotenhours", "n/a")336 for m, tag in [(ed, "piloted"), (gr, "pilotgr")]:337 if m:338 fv = _final_val(f"{RUNS}/{m['run_id']}")339 b0 = fv.get("val_loss_b0"); 340 for b in (1, 2, 4, 8):341 macro(f"{tag}gainb{ {1:'one',2:'two',4:'four',8:'eight'}[b] }", f"{100*(1 - fv.get(f'val_loss_b{b}', b0)/b0):.2f}" if b0 else "n/a")342 macro(f"{tag}minutes", f"{m['wall_seconds']/60:.0f}"); macro(f"{tag}peakgb", f"{m['peak_memory_bytes']/1e9:.1f}"); macro(f"{tag}params", f"{m['n_trainable']:,}")343 else:344 for bn in ("one", "two", "four", "eight"):345 macro(f"{tag}gainb{bn}", "n/a")346 macro(f"{tag}minutes", "n/a"); macro(f"{tag}peakgb", "n/a"); macro(f"{tag}params", "n/a")347 # perplexity table from evaluation.run outputs348 evs = sorted(glob.glob("results/raw/eval_*_val.json")) + sorted(glob.glob("results/raw/eval_*103m*_test.json")) + sorted(glob.glob("results/raw/eval_enlarged*_test.json"))349 evs = [f for f in evs if "p4v2" not in f]350 if evs:351 head = r"""\begin{table}[t]\centering\small352\caption{103M pilot: held-out cross-entropy (nats/token) and perplexity on 327,680 FineWeb-Edu validation tokens and353on the LOCKED test split (40 $\times$ 16 $\times$ 512 tokens each) under forced random budgets. Budget 0 of the adapters354equals the backbone bitwise.}355\label{tab:pilotppl}356\begin{tabular}{lllrr}\toprule357System & split & policy & loss & ppl\\\midrule358"""359 body = ""360 for f in evs:361 d = json.load(open(f))362 for pol, r in d.get("perplexity", {}).items():363 if pol == "controller":364 continue365 body += f"{d['system'].replace('_', ' ')} & {d['split']} & {pol.replace('_', ' ')} & {r['loss']:.4f} & {r['ppl']:.2f}\\\\\n"366 if pol == "forced_b0":367 macro("pilotppl" + re.sub(r"[^a-z]", "", d["system"]) + d["split"], f"{r['ppl']:.2f}")368 if d["split"] == "test" and pol in ("forced_b2", "forced_b4"):369 b0 = d["perplexity"]["forced_b0"]["loss"]370 macro("pilottest" + re.sub(r"[^a-z]", "", d["system"]) + {"forced_b2": "btwo", "forced_b4": "bfour"}[pol], f"{100*(r['loss']/b0-1):+.2f}")371 (OUT / "pilot_ppl.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")372 rt = Path("results/raw/routing_103m_val.json")373 if rt.exists():374 d = json.load(open(rt)); recs = d["records"]375 def pick(pol, **kw):376 for r in recs:377 if r["policy"] == pol and all(r.get(k) == v for k, v in kw.items()):378 return r379 b0 = pick("budget0"); ctl = pick("controller_locked"); ora = pick("oracle1_greedy", budget=8); uni1 = pick("uniform", budget=1)380 if b0 and ctl:381 macro("pilotroutingctrl", f"{100*(ctl['loss']/b0['loss']-1):+.2f}"); macro("pilotroutingctrledits", f"{ctl['edits_per_token']:.2f}")382 if b0 and ora:383 macro("pilotroutingoracle", f"{100*(ora['loss']/b0['loss']-1):+.1f}"); macro("pilotroutingoracleedits", f"{ora['edits_per_token']:.2f}")384 if b0 and uni1:385 macro("pilotroutinguniformone", f"{100*(uni1['loss']/b0['loss']-1):+.2f}")386 head = r"""\begin{table}[t]\centering\small387\caption{103M pilot: routing comparison on validation text (20 $\times$ 8 $\times$ 512 tokens). Oracles are diagnostic.}388\label{tab:pilotrouting}389\begin{tabular}{llrrr}\toprule390Policy & setting & loss & edits/token & extra ops\\\midrule391"""392 body = ""393 for r in recs:394 if r["policy"] == "oracle_two_step":395 continue396 setting = ", ".join(f"{k}={v}" for k, v in r.items() if k in ("budget", "threshold", "lam", "kap") and v is not None)397 body += f"{r['policy'].replace('_', ' ')}{' (diag.)' if r.get('diagnostic') else ''} & {setting} & {r['loss']:.4f} & {r['edits_per_token']:.2f} & {r['extra_ops_ratio']:.3f}\\\\\n"398 (OUT / "pilot_routing.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")399 lt = Path("results/raw/latency_103m_mps.json")400 if lt.exists():401 d = json.load(open(lt))402 head = r"""\begin{table}[t]\centering\small403\caption{103M pilot: measured decode latency per generated token on %s (fp32, prompt %d tokens, %d generated, three404repetitions, median and p95), batch 1 and 8.}405\label{tab:pilotlatency}406\begin{tabular}{llrrrr}\toprule407Policy & budget & batch & median ms & p95 ms & cache bytes\\\midrule408""" % (d["device"], d["prompt_len"], d["n_new"])409 body = ""410 for r in d["records"]:411 body += f"{r['policy'].replace('_', ' ')} & {r['budget']} & {r['batch']} & {r['median_ms']:.2f} & {r['p95_ms']:.2f} & {r['cache_bytes']:,}\\\\\n"412 (OUT / "pilot_latency.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")413 b0 = [r for r in d["records"] if r["policy"] == "budget0" and r["batch"] == 1]; s8 = [r for r in d["records"] if r["policy"] == "forced_single" and r["budget"] == 8 and r["batch"] == 1]414 d8 = [r for r in d["records"] if r["policy"] == "dense_forced" and r["budget"] == 8 and r["batch"] == 1]; c = [r for r in d["records"] if r["policy"] == "controller" and r["batch"] == 1]415 if b0: macro("pilotlatbzero", f"{b0[0]['median_ms']:.1f}")416 if s8: macro("pilotlatseight", f"{s8[0]['median_ms']:.1f}")417 if d8: macro("pilotlatdeight", f"{d8[0]['median_ms']:.1f}")418 if c: macro("pilotlatctrl", f"{c[0]['median_ms']:.1f}")419420421_old_make_all3 = make_all422def make_all(macro): # noqa: F811423 _old_make_all3(macro); table_pilot(macro)424425426def table_test(macro):427 """Locked test regimes (synthetic): exact match by system and budget, mean ± SD over seeds, plus paired statistics."""428 files = sorted(glob.glob("results/raw/eval_p4v2_*_test_*.json"))429 if not files:430 return431 from evaluation.statistical_analysis import paired_bootstrap, seed_interval432 acc = defaultdict(lambda: defaultdict(list)); items = defaultdict(dict)433 for f in files:434 d = json.load(open(f)); sysname = d["system"]; base = sysname.rsplit("_s", 1)[0]; seed = sysname.rsplit("_s", 1)[1]435 for pol, em in d["exact_match"].items():436 acc[(base, pol)][d["split"]].append(em["accuracy"])437 ip = Path(f).parent / f"items_{sysname}_{d['split']}_{pol}.json"438 if ip.exists():439 items[(base, pol, d["split"])][seed] = {r["uid"]: r["correct"] for r in json.load(open(ip))}440 splits = ["test_iid", "test_template", "test_depth", "test_compose"]441 labels = {"p4v2_prisme": "PRISME sparse editor", "p4v2_gr": "Global recurrence", "p4v2_dense": "Dense 16-slot"}442 head = r"""\begin{table}[t]\centering\small443\caption{LOCKED test regimes (synthetic suite, 1,000 stratified items per regime, unsealed 2026-10-30 after the444pre-registration was frozen): exact match (\%) at budget 0 (frozen backbone, identical for all systems) and at forced445random budget 2, mean $\pm$ SD over adapter seeds. ``pooled'' = the pre-registered primary outcome (iid+template+depth).}446\label{tab:test}447\begin{tabular}{llrrrrr}\toprule448System & budget & iid & template & depth & compose & pooled\\\midrule449"""450 body = ""; pooled = {}451 for base, lab in labels.items():452 for pol in ("budget0", "forced_b2"):453 k = (base, pol)454 if k not in acc:455 continue456 cells = []457 for sp in splits:458 v = np.array(acc[k].get(sp, [np.nan]))459 cells.append(f"{100*v.mean():.1f} $\\pm$ {100*v.std(ddof=1):.1f}" if len(v) > 1 else f"{100*v.mean():.1f}")460 # pooled per seed461 per_seed = []462 n_seeds = min(len(acc[k][sp]) for sp in splits[:3] if sp in acc[k])463 for i in range(n_seeds):464 per_seed.append(np.mean([acc[k][sp][i] for sp in splits[:3]]))465 pooled[k] = per_seed466 pv = np.array(per_seed)467 cells.append(f"{100*pv.mean():.1f} $\\pm$ {100*pv.std(ddof=1):.1f}" if len(pv) > 1 else f"{100*pv.mean():.1f}")468 body += f"{lab} & {0 if pol == 'budget0' else 2} & " + " & ".join(cells) + r"\\" + "\n"469 (OUT / "test.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")470 # pre-registered P1: pooled EM differences at b=2 (paired by item within seed, t over seeds)471 stats = {}472 def pooled_items(base, pol, seed):473 out = {}474 for sp in splits[:3]:475 out.update({f"{sp}:{u}": c for u, c in items.get((base, pol, sp), {}).get(seed, {}).items()})476 return out477 for a, b, name in [("p4v2_prisme", None, "prismebtwovsbzero"), ("p4v2_prisme", "p4v2_gr", "prismevsgr"), ("p4v2_prisme", "p4v2_dense", "prismevsdense")]:478 diffs = []479 seeds = sorted({s for (bb, pol, sp), d in items.items() if bb == a and pol == "forced_b2" for s in d})480 for s in seeds:481 x = pooled_items(a, "forced_b2", s)482 y = pooled_items(a, "budget0", s) if b is None else pooled_items(b, "forced_b2", s)483 common = sorted(set(x) & set(y))484 if common:485 diffs.append(paired_bootstrap(np.array([x[u] for u in common]), np.array([y[u] for u in common]))["diff"])486 if diffs:487 si = seed_interval(diffs); stats[name] = si488 macro(f"test{name}", f"{100*si['mean']:+.1f}"); macro(f"test{name}lo", f"{100*si['ci_low']:+.1f}"); macro(f"test{name}hi", f"{100*si['ci_high']:+.1f}"); macro(f"test{name}n", str(si["n"]))489 Path("results/analyzed/test_statistics.json").write_text(json.dumps({"pooled_per_seed": {f"{k[0]}|{k[1]}": v for k, v in pooled.items()}, "stats": stats}, indent=1))490491492_old_make_all4 = make_all493def make_all(macro): # noqa: F811494 _old_make_all4(macro); table_test(macro)495496497def table_controls(macro):498 import torch499 from training.train_editor import build_model500 from training.common import load_yaml501 rows = []502 def count(exp, label, data, compute, regime):503 cfg, m = build_model(exp)504 nb = sum(p.numel() for p in m.backbone.parameters()); tot = sum(p.numel() for p in m.parameters())505 rows.append((label, nb, tot - nb, data, compute, regime))506 with torch.device("meta"):507 count(load_yaml("configs/experiments/p4v2_prisme.yaml"), "Frozen backbone v2 (budget 0)", "synthetic train, 5k steps $\\times$ 64", "backbone only", "no workspace (bitwise bypass)")508 count(load_yaml("configs/experiments/p4v2_prisme.yaml"), "PRISME sparse editor (tiny)", "same + 3k adapter steps $\\times$ 32", "frozen bb fwd + editor", "1--2 slots/action, 0--8 actions")509 count(load_yaml("configs/experiments/p4v2_gr.yaml"), "Global-recurrence adapter (tiny)", "same", "frozen bb fwd + block", "whole latent/action, 0--8 actions")510 count(load_yaml("configs/experiments/p4v2_dense.yaml"), "Dense 16-slot editor (tiny)", "same", "frozen bb fwd + 16 lanes", "16 slots/action, 0--8 actions")511 e = load_yaml("configs/experiments/editor_pilot_103m_lr3e4.yaml"); e["init"] = {}512 count(e, "PRISME-103M sparse editor", "FineWeb-Edu 98.3M tok + 24.6M adapter tok", "$6NP$ + editor", "1--2 slots/action")513 g = load_yaml("configs/experiments/gr_pilot_103m.yaml"); g["init"] = {}514 count(g, "Global-recurrence adapter (103M)", "same", "$6NP$ + block", "whole latent/action")515 count({"model": "configs/baselines/transformer_enlarged_102m.yaml", "mode": "backbone_only"}, "Plain decoder, parameter-matched (102.2M)", "FineWeb-Edu 98.3M tok (same recipe)", "$6NP$", "no workspace")516 with open(OUT / "controls.tex", "w") as f:517 f.write(r"""\begin{table}[t]\centering\small518\caption{Systems and controls. ``bb'' = backbone, ``$6NP$'' = the usual training-FLOP estimate for $N$ tokens and $P$519parameters; adapter training additionally executes up to 8 editor applications per position. Adapter parameters are520unique parameters outside the backbone (counted by instantiation; the 103M editor includes the residual-plus-entropy521controller features).}522\label{tab:controls}523\begin{tabular}{lrrlll}\toprule524System & backbone & adapter & training exposure & training compute & inference regime\\\midrule525""")526 for lab, nb, ad, data, comp, reg in rows:527 f.write(f"{lab} & {nb:,} & {ad:,} & {data} & {comp} & {reg}\\\\\n")528 f.write(r"\bottomrule\end{tabular}\end{table}" + "\n")529530531_old_make_all5 = make_all532def make_all(macro): # noqa: F811533 _old_make_all5(macro); table_controls(macro)534535536def table_ablations(macro):537 """Tiny-scale ablations of the sparse editor (v2 backbone, 3 seeds): slots M, editor depth, pair scheme."""538 systems = [("p4v2_prisme", "Reference: $M=16$, $L_w=2$, adjacent pairs"), ("p4v2_abl_M4", "$M = 4$ slots"), ("p4v2_abl_M8", "$M = 8$ slots"),539 ("p4v2_abl_M32", "$M = 32$ slots"), ("p4v2_abl_L1", "$L_w = 1$ block"), ("p4v2_abl_L4", "$L_w = 4$ blocks"), ("p4v2_abl_pairoffset", "offset pairs (i, i+M/2)")]540 rows = []541 for pre, lab in systems:542 runs = [r for r in _runs(pre) if json.load(open(Path(r) / "manifest.json")).get("state") == "completed"]543 if not runs:544 continue545 vals = defaultdict(list); ems = defaultdict(list); params = []546 for r in runs:547 fv = _final_val(r)548 for k, v in fv.items():549 if k.startswith("val_loss_b"):550 vals[int(k[10:])].append(v)551 m = json.load(open(Path(r) / "manifest.json")); params.append(m.get("n_trainable", 0))552 for b, e in m.get("final_exact_match", {}).get("dev_iid", {}).items():553 ems[int(b)].append(e["accuracy"])554 rows.append((lab, len(runs), int(np.mean(params)), vals, ems))555 if len(rows) < 2:556 return557 def ms(v, f=4):558 v = np.asarray(v, float); return (f"{v.mean():.{f}f} $\\pm$ {v.std(ddof=1):.{f}f}" if len(v) > 1 else f"{v.mean():.{f}f}")559 def pct(v):560 v = 100 * np.asarray(v, float); return (f"{v.mean():.1f} $\\pm$ {v.std(ddof=1):.1f}" if len(v) > 1 else f"{v.mean():.1f}")561 head = r"""\begin{table}[t]\centering\small562\caption{Ablations of the sparse editor on the frozen v2 synthetic backbone (dev-iid; random forced budgets; mean $\pm$ SD563over adapter seeds; the reference row is the 5-seed system of Table~\ref{tab:seedmatrix}). ``params'' = trainable adapter564parameters excluding the controller.}565\label{tab:ablations}566\begin{tabular}{lrrrrrr}\toprule567Variant & seeds & params & loss b=1 & loss b=2 & loss b=8 & EM b=2 (\%)\\\midrule568"""569 body = ""570 for lab, n, pr, vals, ems in rows:571 body += f"{lab} & {n} & {pr:,} & {ms(vals.get(1, [np.nan]))} & {ms(vals.get(2, [np.nan]))} & {ms(vals.get(8, [np.nan]))} & {pct(ems.get(2, [np.nan]))}\\\\\n"572 (OUT / "ablations.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")573 for pre, lab in systems[1:]:574 for r in rows:575 pass576 ref = next((r for r in rows if r[0].startswith("Reference")), None)577 keys = {"Reference": "ref", "$M = 4$": "mfour", "$M = 8$": "meight", "$M = 32$": "mthirtytwo", "$L_w = 1$": "lone", "$L_w = 4$": "lfour", "offset": "offset"}578 for lab, n, pr, vals, ems in rows:579 key = next(v for k, v in keys.items() if lab.startswith(k))580 if 2 in vals and 0 in vals:581 macro("abl" + key + "gainbtwo", f"{100*(1-np.mean(vals[2])/np.mean(vals[0])):.1f}")582 if 8 in vals and 0 in vals:583 macro("abl" + key + "gainbeight", f"{100*(1-np.mean(vals[8])/np.mean(vals[0])):.1f}")584 if 2 in ems:585 macro("abl" + key + "embtwo", f"{100*np.mean(ems[2]):.1f}")586 macro("abl" + key + "seeds", str(n))587588589_old_make_all6 = make_all590def make_all(macro): # noqa: F811591 _old_make_all6(macro); table_ablations(macro)592