1"""Results-dependent LaTeX tables + macros (called by scripts/make_tables.py). Reads run manifests/metrics under2experiments/runs (local copies), results/raw/*.json and results/analyzed/*.json. Every number is traceable to a run id."""3from __future__ import annotations45import glob, json, math6from collections import defaultdict7from pathlib import Path89import numpy as np1011OUT = Path("paper/tables")12RUNS = "experiments/runs"131415def _runs(prefix):16 return sorted(d for d in glob.glob(f"{RUNS}/{prefix}_seed*") if Path(d, "manifest.json").exists())171819def _final_val(run):20 rows = [json.loads(l) for l in open(Path(run) / "metrics.jsonl")]21 last = [r for r in rows if any(k.startswith("val_loss_b") for k in r)]22 return last[-1] if last else {}232425def table_seed_matrix(macro):26 """Phase 4 decisive matrix: PRISME vs global recurrence vs dense, 5 seeds, val loss by budget and exact match."""27 systems = [("p4v2_prisme", "PRISME sparse editor"), ("p4v2_gr", "Global recurrence (inspired)"), ("p4v2_dense", "Dense 16-slot writes")]28 rows, raw = [], {}29 for pre, lab in systems:30 runs = [r for r in _runs(pre) if json.load(open(Path(r) / "manifest.json")).get("state") == "completed"]31 if not runs:32 continue33 vals = defaultdict(list); ems = defaultdict(list)34 for r in runs:35 fv = _final_val(r)36 for k, v in fv.items():37 if k.startswith("val_loss_b"):38 vals[int(k[10:])].append(v)39 fem = json.load(open(Path(r) / "manifest.json")).get("final_exact_match", {})40 for split, d in fem.items():41 for b, e in d.items():42 ems[(split, int(b))].append(e["accuracy"])43 raw[pre] = {"n_seeds": len(runs), "val": {b: v for b, v in vals.items()}, "em": {f"{s}_b{b}": v for (s, b), v in ems.items()}, "run_ids": [Path(r).name for r in runs]}44 rows.append((lab, len(runs), vals, ems))45 if not rows:46 return47 Path("results/analyzed").mkdir(parents=True, exist_ok=True)48 Path("results/analyzed/seed_matrix_v2.json").write_text(json.dumps(raw, indent=1))49 def ms(v):50 v = np.asarray(v, float)51 return f"{v.mean():.4f} $\\pm$ {v.std(ddof=1):.4f}" if len(v) > 1 else f"{v.mean():.4f}"52 def pct(v):53 v = 100 * np.asarray(v, float)54 return f"{v.mean():.1f} $\\pm$ {v.std(ddof=1):.1f}" if len(v) > 1 else f"{v.mean():.1f}"55 head = r"""\begin{table}[t]\centering\small56\caption{Phase 4 decisive comparison on the frozen 5.6M-parameter synthetic backbone (char-level): held-out57answer-span loss on dev-iid under random forced actions at fixed budgets, and free-generation exact match58(dev-iid, 1,000 stratified items). Mean $\pm$ SD over independent training seeds of the adapter; the backbone,59data, recipe and symmetry-breaking device are identical. Budget 0 is the frozen backbone for every system.}60\label{tab:seedmatrix}61\begin{tabular}{llrrrrr}\toprule62System & seeds & loss b=0 & loss b=1 & loss b=2 & loss b=4 & loss b=8\\\midrule63"""64 body = ""65 for lab, n, vals, ems in rows:66 body += f"{lab} & {n} & " + " & ".join(ms(vals.get(b, [float('nan')])) for b in [0, 1, 2, 4, 8]) + r"\\" + "\n"67 body += r"\midrule" + "\n" + r"\multicolumn{7}{l}{\emph{Exact match (\%), dev-iid}: b=0 / b=1 / b=2 / b=8}\\" + "\n"68 for lab, n, vals, ems in rows:69 cells = [pct(ems.get(("dev_iid", b), [float("nan")])) for b in [0, 1, 2, 8]]70 body += f"{lab} & {n} & \\multicolumn{{5}}{{l}}{{{' / '.join(cells)}}}\\\\\n"71 body += r"\multicolumn{7}{l}{\emph{Exact match (\%), held-out template / depth / compose at b=2}}\\" + "\n"72 for lab, n, vals, ems in rows:73 cells = [pct(ems.get((s, 2), [float("nan")])) for s in ["dev_template", "dev_depth", "dev_compose"]]74 body += f"{lab} & {n} & \\multicolumn{{5}}{{l}}{{{' / '.join(cells)}}}\\\\\n"75 (OUT / "seed_matrix.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")76 for pre, lab in systems:77 tag = pre.replace("p4v2_", "vtwo")78 if pre not in raw: # system not (yet) completed: emit explicit placeholders so the manuscript compiles and the audit flags them79 macro(f"{tag}seeds", "0")80 for bn in ("one", "two", "four", "eight"):81 macro(f"{tag}gainb{bn}", "n/a")82 for e in ("embzero", "embtwo", "embeight"):83 macro(f"{tag}{e}", "n/a")84 if pre in raw:85 v = raw[pre]["val"]86 for b, bn in [(1, "one"), (2, "two"), (4, "four"), (8, "eight")]:87 if 0 in v and b in v:88 macro(f"{tag}gainb{bn}", f"{100 * (1 - np.mean(v[b]) / np.mean(v[0])):.1f}")89 macro(f"{tag}seeds", str(raw[pre]["n_seeds"]))90 em = raw[pre]["em"]91 if "dev_iid_b0" in em and "dev_iid_b2" in em:92 macro(f"{tag}embzero", f"{100 * np.mean(em['dev_iid_b0']):.1f}"); macro(f"{tag}embtwo", f"{100 * np.mean(em['dev_iid_b2']):.1f}")93 macro(f"{tag}embeight", f"{100 * np.mean(em.get('dev_iid_b8', em['dev_iid_b2'])):.1f}")94 macro("vtwobzeroloss", f"{np.mean(raw[systems[0][0]]['val'][0]):.4f}" if systems[0][0] in raw else "n/a")959697def table_routing(macro):98 files = sorted(glob.glob("results/raw/routing_v1ctrl.json")) or sorted(glob.glob("results/raw/routing_*.json"))99 if not files:100 return101 d = json.load(open(files[-1]))102 head = r"""\begin{table}[t]\centering\small103\caption{Routing comparison on dev-iid (tiny scale, editor checkpoint of the locked run): held-out answer-span loss,104executed edits per token and analytic extra operations per token (relative to the backbone). Oracles read the105target and are diagnostic ceilings, not deployable systems.}106\label{tab:routing}107\begin{tabular}{llrrr}\toprule108Policy & setting & loss & edits/token & extra ops\\\midrule109"""110 body = ""111 for r in d["records"]:112 if r["policy"] == "oracle_two_step":113 continue114 setting = ", ".join(f"{k}={v}" for k, v in r.items() if k in ("budget", "threshold", "lam", "kap") and v is not None)115 body += f"{r['policy'].replace('_', ' ')}{' (diag.)' if r.get('diagnostic') else ''} & {setting} & {r['loss']:.4f} & {r['edits_per_token']:.2f} & {r['extra_ops_ratio']:.3f}\\\\\n"116 def pick(pol, **kw):117 for r in d["records"]:118 if r["policy"] == pol and all(r.get(k) == v for k, v in kw.items()):119 return r120 return None121 b0 = pick("budget0")122 if b0:123 macro("routingbzero", f"{b0['loss']:.4f}")124 for pol, tag in [("uniform", "uniform"), ("random_forced", "random"), ("fixed_pair", "pair")]:125 for b in (1, 2, 8):126 r = pick(pol, budget=b)127 if r:128 macro(f"routing{tag}b{'one' if b == 1 else 'two' if b == 2 else 'eight'}", f"{100 * (r['loss'] / b0['loss'] - 1):+.1f}")129 r = pick("controller_locked")130 if r:131 macro("routingctrllocked", f"{100 * (r['loss'] / b0['loss'] - 1):+.1f}"); macro("routingctrlledits", f"{r['edits_per_token']:.2f}"); macro("routingctrllambda", f"{r['lam']}")132 for b in (1, 8):133 r = pick("oracle1_greedy", budget=b)134 if r:135 macro(f"routingoracleb{'one' if b == 1 else 'eight'}", f"{100 * (r['loss'] / b0['loss'] - 1):+.1f}"); macro(f"routingoracleedits{'one' if b == 1 else 'eight'}", f"{r['edits_per_token']:.2f}")136 best_ent = min((r for r in d["records"] if r["policy"] == "entropy"), key=lambda r: r["loss"], default=None)137 best_norm = min((r for r in d["records"] if r["policy"] == "norm"), key=lambda r: r["loss"], default=None)138 if best_ent: macro("routingentropybest", f"{100 * (best_ent['loss'] / b0['loss'] - 1):+.1f}")139 if best_norm: macro("routingnormbest", f"{100 * (best_norm['loss'] / b0['loss'] - 1):+.1f}")140 two = [r for r in d["records"] if r["policy"] == "oracle_two_step"]141 if two:142 t = two[0]143 body += r"\midrule" + f"\\multicolumn{{5}}{{l}}{{Two-step oracle (diag.): one-step {t['one_step_gain']:.4f}, two-step {t['two_step_gain']:.4f}, greedy-then-best {t['greedy_then_best_gain']:.4f} nats/token}}\\\\\n"144 macro("oracleonestep", f"{t['one_step_gain']:.4f}"); macro("oracletwostep", f"{t['two_step_gain']:.4f}")145 (OUT / "routing.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")146147148def table_controller(macro):149 runs = sorted(glob.glob(f"{RUNS}/controller_p4*_seed*/manifest.json"))150 if not runs:151 return152 head = r"""\begin{table}[t]\centering\small153\caption{Value-controller fit on logged counterfactual labels (tiny scale). Calibration-partition diagnostics:154Huber loss of $\hat g$, sign accuracy, Brier score and ECE of the harm head, one-step oracle opportunity and the155regret of the controller's greedy choice on the exhaustive-audit states (captured fraction $=1-$ regret/opportunity).}156\label{tab:controller}157\begin{tabular}{lrrrrrrr}\toprule158Controller features & labels & Huber & sign acc. & Brier & ECE & opportunity & captured\\\midrule159"""160 body = ""161 for mp in runs:162 m = json.load(open(mp))163 if m.get("state") != "completed":164 continue165 ev = m.get("eval_calib", {})166 feats = "h + entropy" if "hent" in m["run_id"] else "z only"167 body += f"{feats} & {ev.get('n_labels', 0):,} & {ev.get('huber', float('nan')):.4f} & {ev.get('sign_acc', float('nan')):.3f} & {ev.get('harm_brier', float('nan')):.3f} & {ev.get('harm_ece', float('nan')):.3f} & {ev.get('oracle_opportunity_per_state', float('nan')):.4f} & {ev.get('captured_fraction', float('nan')):.3f}\\\\\n"168 macro("ctrl" + ("hent" if "hent" in m["run_id"] else "z") + "signacc", f"{100 * ev.get('sign_acc', float('nan')):.1f}")169 (OUT / "controller.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")170171172def table_labels(macro):173 files = sorted(glob.glob("results/analyzed/labels_labels_v1ctrl*.json")) or sorted(glob.glob("results/analyzed/labels_*.json"))174 if not files:175 return176 d = json.load(open(files[-1]))177 for part in ("train", "calib"):178 if part in d and "audit" in d[part]:179 a = d[part]["audit"]180 macro(f"labels{part}within", f"{100 * a['within_share']:.0f}")181 macro(f"labels{part}allpos", f"{100 * a['frac_states_all_positive']:.0f}")182 macro(f"labels{part}allneg", f"{100 * a['frac_states_all_negative']:.0f}")183 macro(f"labels{part}oracle", f"{a['oracle_one_step_mean']:.4f}")184 macro(f"labels{part}stoponly", f"{a['oracle_state_level_mean']:.4f}")185 macro(f"labels{part}random", f"{a['random_action_mean']:.4f}")186 macro(f"labels{part}nstates", f"{a['n_states']:,}")187 if "random_gain_high_vs_low_base_loss" in a:188 macro(f"labels{part}oraclehigh", f"{a['oracle_gain_high_vs_low_base_loss']['high']:.4f}")189 macro(f"labels{part}randomhigh", f"{a['random_gain_high_vs_low_base_loss']['high']:+.4f}")190 macro(f"labels{part}topdecoracle", f"{a['top_decile_base_loss']['oracle']:.3f}"); macro(f"labels{part}topdecrandom", f"{a['top_decile_base_loss']['random']:+.3f}")191 macro(f"labels{part}fracposhigh", f"{100 * a['frac_actions_positive_high_vs_low']['high']:.0f}"); macro(f"labels{part}fracposlow", f"{100 * a['frac_actions_positive_high_vs_low']['low']:.0f}")192193194def table_latency(macro):195 files = sorted(glob.glob("results/raw/latency_*.json"))196 if not files:197 return198 d = json.load(open(files[-1]))199 head = r"""\begin{table}[t]\centering\small200\caption{Measured decode latency per generated token (fp32, prompt %d tokens, %d generated, warm-up and201synchronisation), device %s. Median and p95 over generated tokens of three repetitions.}202\label{tab:latency}203\begin{tabular}{llrrrr}\toprule204Policy & budget & batch & median ms & p95 ms & cache bytes\\\midrule205""" % (d["prompt_len"], d["n_new"], d["device"].replace("_", " "))206 body = ""207 for r in d["records"]:208 body += f"{r['policy'].replace('_', ' ')} & {r['budget']} & {r['batch']} & {r['median_ms']:.2f} & {r['p95_ms']:.2f} & {r['cache_bytes']:,}\\\\\n"209 (OUT / "latency.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")210 b0 = [r for r in d["records"] if r["policy"] == "budget0" and r["batch"] == 1]211 if b0:212 macro("latencybzero", f"{b0[0]['median_ms']:.2f}")213214215def make_all(macro):216 table_seed_matrix(macro); table_routing(macro); table_controller(macro); table_labels(macro); table_latency(macro)217218219def table_routing_seeds(macro):220 """Per-seed routing summary for the v2 PRISME editors: budget 0, best fixed schedule, locked controller, oracle."""221 files = sorted(glob.glob("results/raw/routing_v2s*.json"))222 if not files:223 return224 rows = []225 for f in files:226 d = json.load(open(f)); recs = d["records"]227 def pick(pol, **kw):228 for r in recs:229 if r["policy"] == pol and all(r.get(k) == v for k, v in kw.items()):230 return r231 b0 = pick("budget0"); fixed = [r for r in recs if r["policy"] in ("random_forced", "uniform", "fixed_pair")]232 best_fixed = min(fixed, key=lambda r: r["loss"]); ctl = pick("controller_locked"); ora = pick("oracle1_greedy", budget=8)233 ent = min((r for r in recs if r["policy"] == "entropy"), key=lambda r: r["loss"]); nrm = min((r for r in recs if r["policy"] == "norm"), key=lambda r: r["loss"])234 seed = Path(f).stem.replace("routing_v2s", "")235 man = sorted(glob.glob(f"{RUNS}/controller_v2s{seed}_seed*/manifest.json"))236 ev = json.load(open(man[-1])).get("eval_calib", {}) if man else {}237 rows.append({"seed": seed, "b0": b0["loss"], "best_fixed": best_fixed["loss"], "best_fixed_name": f"{best_fixed['policy'].replace('_', ' ')} b={best_fixed['budget']}",238 "entropy": ent["loss"], "norm": nrm["loss"], "ctrl": ctl["loss"], "ctrl_edits": ctl["edits_per_token"], "ctrl_lam": ctl["lam"], "ctrl_kap": ctl["kap"],239 "oracle8": ora["loss"], "oracle8_edits": ora["edits_per_token"], "sign_acc": ev.get("sign_acc", float("nan")), "captured": ev.get("captured_fraction", float("nan"))})240 Path("results/analyzed/routing_seeds_v2.json").write_text(json.dumps(rows, indent=1))241 head = r"""\begin{table}[t]\centering\small242\caption{Learned routing per adapter seed (v2 backbone, dev-iid, 640 items): held-out answer-span loss of the243frozen backbone, of the best fixed schedule (named), of the best entropy and norm routers, of the locked244controller ($\lambda,\kappa$ selected on calibration rollouts; edits per token in parentheses), and of the245diagnostic greedy one-step oracle at budget 8. Calibration sign accuracy and captured fraction of the oracle246opportunity for each controller.}247\label{tab:routingseeds}248\begin{tabular}{lrlrrrrrr}\toprule249seed & budget 0 & best fixed & entropy & norm & controller (edits) & oracle b=8 (edits) & sign acc. & captured\\\midrule250"""251 body = ""252 for r in rows:253 body += (f"{r['seed']} & {r['b0']:.4f} & {r['best_fixed']:.4f} ({r['best_fixed_name']}) & {r['entropy']:.4f} & {r['norm']:.4f} & "254 f"{r['ctrl']:.4f} ({r['ctrl_edits']:.2f}) & {r['oracle8']:.4f} ({r['oracle8_edits']:.2f}) & {r['sign_acc']:.3f} & {r['captured']:+.3f}\\\\\n")255 (OUT / "routing_seeds.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")256 b0 = np.array([r["b0"] for r in rows]); ctl = np.array([r["ctrl"] for r in rows]); bf = np.array([r["best_fixed"] for r in rows]); ora = np.array([r["oracle8"] for r in rows])257 macro("vtwoctrlseeds", str(len(rows))); macro("vtwoctrlrel", f"{100 * (ctl / b0 - 1).mean():+.1f}"); macro("vtwoctrlrelsd", f"{100 * (ctl / b0 - 1).std(ddof=1) if len(rows) > 1 else 0:.1f}")258 macro("vtwobestfixedrel", f"{100 * (bf / b0 - 1).mean():+.1f}"); macro("vtwooraclerel", f"{100 * (ora / b0 - 1).mean():+.1f}")259 macro("vtwoctrlbetter", str(int((ctl < b0).sum())))260261262_old_make_all = make_all263def make_all(macro): # noqa: F811264 _old_make_all(macro); table_routing_seeds(macro)265