1"""Results-dependent LaTeX tables + macros (called by scripts/make_tables.py). Reads run manifests/metrics under2experiments/runs (local copies), results/raw/*.json and results/analyzed/*.json. Every number is traceable to a run id."""3from __future__ import annotations45import glob, json, math6from collections import defaultdict7from pathlib import Path89import numpy as np1011OUT = Path("paper/tables")12RUNS = "experiments/runs"131415def _runs(prefix):16 return sorted(d for d in glob.glob(f"{RUNS}/{prefix}_seed*") if Path(d, "manifest.json").exists())171819def _final_val(run):20 rows = [json.loads(l) for l in open(Path(run) / "metrics.jsonl")]21 last = [r for r in rows if any(k.startswith("val_loss_b") for k in r)]22 return last[-1] if last else {}232425def table_seed_matrix(macro):26 """Phase 4 decisive matrix: PRISME vs global recurrence vs dense, 5 seeds, val loss by budget and exact match."""27 systems = [("p4v2_prisme", "PRISME sparse editor"), ("p4v2_gr", "Global recurrence (inspired)"), ("p4v2_dense", "Dense 16-slot writes")]28 rows, raw = [], {}29 for pre, lab in systems:30 runs = [r for r in _runs(pre) if json.load(open(Path(r) / "manifest.json")).get("state") == "completed"]31 if not runs:32 continue33 vals = defaultdict(list); ems = defaultdict(list)34 for r in runs:35 fv = _final_val(r)36 for k, v in fv.items():37 if k.startswith("val_loss_b"):38 vals[int(k[10:])].append(v)39 fem = json.load(open(Path(r) / "manifest.json")).get("final_exact_match", {})40 for split, d in fem.items():41 for b, e in d.items():42 ems[(split, int(b))].append(e["accuracy"])43 raw[pre] = {"n_seeds": len(runs), "val": {b: v for b, v in vals.items()}, "em": {f"{s}_b{b}": v for (s, b), v in ems.items()}, "run_ids": [Path(r).name for r in runs]}44 rows.append((lab, len(runs), vals, ems))45 if not rows:46 return47 Path("results/analyzed").mkdir(parents=True, exist_ok=True)48 Path("results/analyzed/seed_matrix_v2.json").write_text(json.dumps(raw, indent=1))49 def ms(v):50 v = np.asarray(v, float)51 return f"{v.mean():.4f} $\\pm$ {v.std(ddof=1):.4f}" if len(v) > 1 else f"{v.mean():.4f}"52 def pct(v):53 v = 100 * np.asarray(v, float)54 return f"{v.mean():.1f} $\\pm$ {v.std(ddof=1):.1f}" if len(v) > 1 else f"{v.mean():.1f}"55 head = r"""\begin{table}[t]\centering\small56\caption{Phase 4 decisive comparison on the frozen 5.6M-parameter synthetic backbone (char-level): held-out57answer-span loss on dev-iid under random forced actions at fixed budgets, and free-generation exact match58(dev-iid, 1,000 stratified items). Mean $\pm$ SD over independent training seeds of the adapter; the backbone,59data, recipe and symmetry-breaking device are identical. Budget 0 is the frozen backbone for every system.}60\label{tab:seedmatrix}61\begin{tabular}{llrrrrr}\toprule62System & seeds & loss b=0 & loss b=1 & loss b=2 & loss b=4 & loss b=8\\\midrule63"""64 body = ""65 for lab, n, vals, ems in rows:66 body += f"{lab} & {n} & " + " & ".join(ms(vals.get(b, [float('nan')])) for b in [0, 1, 2, 4, 8]) + r"\\" + "\n"67 body += r"\midrule" + "\n" + r"\multicolumn{7}{l}{\emph{Exact match (\%), dev-iid}: b=0 / b=1 / b=2 / b=8}\\" + "\n"68 for lab, n, vals, ems in rows:69 cells = [pct(ems.get(("dev_iid", b), [float("nan")])) for b in [0, 1, 2, 8]]70 body += f"{lab} & {n} & \\multicolumn{{5}}{{l}}{{{' / '.join(cells)}}}\\\\\n"71 body += r"\multicolumn{7}{l}{\emph{Exact match (\%), held-out template / depth / compose at b=2}}\\" + "\n"72 for lab, n, vals, ems in rows:73 cells = [pct(ems.get((s, 2), [float("nan")])) for s in ["dev_template", "dev_depth", "dev_compose"]]74 body += f"{lab} & {n} & \\multicolumn{{5}}{{l}}{{{' / '.join(cells)}}}\\\\\n"75 (OUT / "seed_matrix.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")76 for pre, lab in systems:77 if pre in raw:78 v = raw[pre]["val"]79 if 0 in v and 2 in v:80 macro(f"{pre.replace('p4v2_', 'vtwo')}gainbtwo", f"{100 * (1 - np.mean(v[2]) / np.mean(v[0])):.2f}")81 macro(f"{pre.replace('p4v2_', 'vtwo')}seeds", str(raw[pre]["n_seeds"]))828384def table_routing(macro):85 files = sorted(glob.glob("results/raw/routing_*.json"))86 if not files:87 return88 d = json.load(open(files[-1]))89 head = r"""\begin{table}[t]\centering\small90\caption{Routing comparison on dev-iid (tiny scale, editor checkpoint of the locked run): held-out answer-span loss,91executed edits per token and analytic extra operations per token (relative to the backbone). Oracles read the92target and are diagnostic ceilings, not deployable systems.}93\label{tab:routing}94\begin{tabular}{llrrr}\toprule95Policy & setting & loss & edits/token & extra ops\\\midrule96"""97 body = ""98 for r in d["records"]:99 if r["policy"] == "oracle_two_step":100 continue101 setting = ", ".join(f"{k}={v}" for k, v in r.items() if k in ("budget", "threshold", "lam", "kap") and v is not None)102 body += f"{r['policy'].replace('_', ' ')}{' (diag.)' if r.get('diagnostic') else ''} & {setting} & {r['loss']:.4f} & {r['edits_per_token']:.2f} & {r['extra_ops_ratio']:.3f}\\\\\n"103 two = [r for r in d["records"] if r["policy"] == "oracle_two_step"]104 if two:105 t = two[0]106 body += r"\midrule" + f"\\multicolumn{{5}}{{l}}{{Two-step oracle (diag.): one-step {t['one_step_gain']:.4f}, two-step {t['two_step_gain']:.4f}, greedy-then-best {t['greedy_then_best_gain']:.4f} nats/token}}\\\\\n"107 macro("oracleonestep", f"{t['one_step_gain']:.4f}"); macro("oracletwostep", f"{t['two_step_gain']:.4f}")108 (OUT / "routing.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")109110111def table_controller(macro):112 runs = sorted(glob.glob(f"{RUNS}/controller_p4*_seed*/manifest.json"))113 if not runs:114 return115 head = r"""\begin{table}[t]\centering\small116\caption{Value-controller fit on logged counterfactual labels (tiny scale). Calibration-partition diagnostics:117Huber loss of $\hat g$, sign accuracy, Brier score and ECE of the harm head, one-step oracle opportunity and the118regret of the controller's greedy choice on the exhaustive-audit states (captured fraction $=1-$ regret/opportunity).}119\label{tab:controller}120\begin{tabular}{lrrrrrrr}\toprule121Controller features & labels & Huber & sign acc. & Brier & ECE & opportunity & captured\\\midrule122"""123 body = ""124 for mp in runs:125 m = json.load(open(mp))126 if m.get("state") != "completed":127 continue128 ev = m.get("eval_calib", {})129 feats = "h + entropy" if "hent" in m["run_id"] else "z only"130 body += f"{feats} & {ev.get('n_labels', 0):,} & {ev.get('huber', float('nan')):.4f} & {ev.get('sign_acc', float('nan')):.3f} & {ev.get('harm_brier', float('nan')):.3f} & {ev.get('harm_ece', float('nan')):.3f} & {ev.get('oracle_opportunity_per_state', float('nan')):.4f} & {ev.get('captured_fraction', float('nan')):.3f}\\\\\n"131 macro("ctrl" + ("hent" if "hent" in m["run_id"] else "z") + "signacc", f"{100 * ev.get('sign_acc', float('nan')):.1f}")132 (OUT / "controller.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")133134135def table_labels(macro):136 files = sorted(glob.glob("results/analyzed/labels_*.json"))137 if not files:138 return139 d = json.load(open(files[-1]))140 for part in ("train", "calib"):141 if part in d and "audit" in d[part]:142 a = d[part]["audit"]143 macro(f"labels{part}within", f"{100 * a['within_share']:.0f}")144 macro(f"labels{part}allpos", f"{100 * a['frac_states_all_positive']:.0f}")145 macro(f"labels{part}allneg", f"{100 * a['frac_states_all_negative']:.0f}")146 macro(f"labels{part}oracle", f"{a['oracle_one_step_mean']:.4f}")147 macro(f"labels{part}stoponly", f"{a['oracle_state_level_mean']:.4f}")148 macro(f"labels{part}random", f"{a['random_action_mean']:.4f}")149 macro(f"labels{part}nstates", f"{a['n_states']:,}")150151152def table_latency(macro):153 files = sorted(glob.glob("results/raw/latency_*.json"))154 if not files:155 return156 d = json.load(open(files[-1]))157 head = r"""\begin{table}[t]\centering\small158\caption{Measured decode latency per generated token (fp32, prompt %d tokens, %d generated, warm-up and159synchronisation), device %s. Median and p95 over generated tokens of three repetitions.}160\label{tab:latency}161\begin{tabular}{llrrrr}\toprule162Policy & budget & batch & median ms & p95 ms & cache bytes\\\midrule163""" % (d["prompt_len"], d["n_new"], d["device"].replace("_", " "))164 body = ""165 for r in d["records"]:166 body += f"{r['policy'].replace('_', ' ')} & {r['budget']} & {r['batch']} & {r['median_ms']:.2f} & {r['p95_ms']:.2f} & {r['cache_bytes']:,}\\\\\n"167 (OUT / "latency.tex").write_text(head + body + r"\bottomrule\end{tabular}\end{table}" + "\n")168 b0 = [r for r in d["records"] if r["policy"] == "budget0" and r["batch"] == 1]169 if b0:170 macro("latencybzero", f"{b0[0]['median_ms']:.2f}")171172173def make_all(macro):174 table_seed_matrix(macro); table_routing(macro); table_controller(macro); table_labels(macro); table_latency(macro)175