SPB Git
spboucher.ai

spb/wp31_uqo

Public

WP31: frontier figure, GR cost model

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Oct 30, 2026) parent 42249a4 Browse files

24 changed files +134 −30

24 changed files
Unified Split
added configs/experiments/locked_evaluation.yaml +31 −0
@@ -0,0 +1,31 @@
1 +# Locked evaluation of the 103M pilot: held-out token cross-entropy at forced budgets and under the controller.
2 +# Splits: val (development) now; test (LOCKED) only after protocol/preregistration.md is frozen and the manifest unsealed.
3 +out_dir: results/raw
4 +data: {kind: tokens, dir: data/processed/fineweb_edu_300m}
5 +splits: [val]
6 +seq_len: 512
7 +batch_size: 16
8 +n_batches: 40 # 40 x 16 x 512 = 327,680 held-out tokens
9 +budgets: [0, 1, 2, 4, 8]
10 +systems:
11 + backbone_103m:
12 + mode: backbone_only
13 + model: from_checkpoint
14 + checkpoint: PRETRAIN103M_CKPT
15 + controller: false
16 + enlarged_102m:
17 + mode: backbone_only
18 + model: from_checkpoint
19 + checkpoint: ENLARGED_CKPT
20 + controller: false
21 + prisme_103m_editor:
22 + mode: prisme
23 + model: from_checkpoint
24 + checkpoint: EDITOR103M_CKPT
25 + controller: false
26 + gr_103m:
27 + mode: global_recurrence
28 + baseline_kwargs: {ffn_w: 2560}
29 + model: from_checkpoint
30 + checkpoint: GR103M_CKPT
31 + controller: false
modified paper/main.log +44 −29
@@ -1,4 +1,4 @@
1 −This is pdfTeX, Version 3.141592653-2.6-1.40.29 (TeX Live 2026) (preloaded format=pdflatex 2026.3.22) 30 OCT 2026 00:58
1 +This is pdfTeX, Version 3.141592653-2.6-1.40.29 (TeX Live 2026) (preloaded format=pdflatex 2026.3.22) 30 OCT 2026 01:04
2 2 entering extended mode
3 3 restricted \write18 enabled.
4 4 %&-line parsing enabled.
@@ -1318,23 +1318,31 @@ Overfull \hbox (133.05286pt too wide) in paragraph at lines 7--18
1318 1318 [][]
1319 1319 []
1320 1320
1321 −) [16] [17 <../results/figures/learnability_bootstrap.pdf>]
1322 −<../results/figures/labels_analysis.pdf, id=518, 548.77519pt x 192.17598pt>
1321 +)
1322 +<../results/figures/frontier_v2.pdf, id=486, 305.58919pt x 208.87035pt>
1323 +File: ../results/figures/frontier_v2.pdf Graphic file (type pdf)
1324 +<use ../results/figures/frontier_v2.pdf>
1325 +Package pdftex.def Info: ../results/figures/frontier_v2.pdf used on input line
1326 + 53.
1327 +(pdftex.def) Requested size: 338.22418pt x 231.19547pt.
1328 + [16] [17 <../results/figures/learnability_bootstrap.pdf>]
1329 +<../results/figures/labels_analysis.pdf, id=519, 548.77519pt x 192.17598pt>
1323 1330 File: ../results/figures/labels_analysis.pdf Graphic file (type pdf)
1324 1331 <use ../results/figures/labels_analysis.pdf>
1325 1332 Package pdftex.def Info: ../results/figures/labels_analysis.pdf used on input
1326 −line 72.
1333 +line 83.
1327 1334 (pdftex.def) Requested size: 469.75502pt x 164.50862pt.
1328 1335
1329 1336 (./tables/routing.tex) (./tables/controller.tex) (./tables/routing_seeds.tex
1330 −Overfull \hbox (109.74734pt too wide) in paragraph at lines 8--14
1337 +Overfull \hbox (109.74734pt too wide) in paragraph at lines 8--15
1331 1338 [][]
1332 1339 []
1333 1340
1334 −) [18] (./tables/stats.tex) [19 <../results/figures/labels_analysis.pdf>]
1335 −(./tables/latency.tex) (./sections/results_pilot.tex))
1336 −(./sections/mechanistic.tex [20]) (./sections/limitations.tex [21])
1337 −(./sections/conclusion.tex) (./sections/reproducibility.tex [22])
1341 +) [18] [19 <../results/figures/frontier_v2.pdf>] (./tables/stats.tex)
1342 +(./tables/latency.tex) [20 <../results/figures/labels_analysis.pdf>]
1343 +(./sections/results_pilot.tex)) (./sections/mechanistic.tex [21])
1344 +(./sections/limitations.tex [22]) (./sections/conclusion.tex)
1345 +(./sections/reproducibility.tex)
1338 1346 Overfull \hbox (51.18217pt too wide) in paragraph at lines 2--101
1339 1347 \T1/ptm/m/n/12 (-20) num-ber in the manuscript is pro-duced by \T1/cmtt/m/n/12
1340 1348 scripts/make_tables.py \T1/ptm/m/n/12 (-20) and \T1/cmtt/m/n/12 scripts/reprodu
@@ -1348,18 +1356,18 @@ cript_claims.py \T1/ptm/m/n/12 (-20) checks the chain. \T1/cmtt/m/n/12 REPRODUC
1348 1356 E.md
1349 1357 []
1350 1358
1351 −[23] (./main.bbl [24] [25]
1359 +[23] [24] (./main.bbl [25] [26]
1352 1360 Underfull \hbox (badness 1297) in paragraph at lines 208--211
1353 1361 []\T1/ptm/b/n/12 (+20) You, Yichen, Tianyu Fu, Aosong Feng, Xing-tai Lv, Xue-fe
1354 1362 i Ning, Ning Ding, and
1355 1363 []
1356 1364
1357 −[26]) (./appendix/appendix.tex (./appendix/notation.tex
1365 +[27]) (./appendix/appendix.tex (./appendix/notation.tex
1358 1366 Overfull \hbox (12.19864pt too wide) in paragraph at lines 2--16
1359 1367 []
1360 1368 []
1361 1369
1362 −) (./appendix/proofs.tex [27]) (./appendix/params.tex [28]
1370 +) (./appendix/proofs.tex [28]) (./appendix/params.tex [29]
1363 1371 Overfull \hbox (92.32437pt too wide) detected at line 12
1364 1372 \OML/ztmcm/m/it/12 N[] \OT1/ztmcm/m/n/12 = \OML/ztmcm/m/it/12 d \OT1/ztmcm/m/n/
1365 1373 12 + 3\OML/ztmcm/m/it/12 dw \OT1/ztmcm/m/n/12 + \OML/ztmcm/m/it/12 Mw \OT1/ztmc
@@ -1468,7 +1476,7 @@ LaTeX Warning: `h' float specifier changed to `ht'.
1468 1476 ) (./appendix/prereg.tex
1469 1477 LaTeX Font Info: Font shape `T1/cmtt/bx/n' in size <12> not available
1470 1478 (Font) Font shape `T1/cmtt/m/n' tried instead on input line 1.
1471 − [29]
1479 + [30]
1472 1480 LaTeX Font Info: Trying to load font information for TS1+ptm on input line 8
1473 1481 .
1474 1482
@@ -1481,7 +1489,7 @@ Overfull \hbox (80.09856pt too wide) in paragraph at lines 10--12
1481 1489 T1/cmtt/m/n/12 data/generators/synthetic.py\T1/ptm/m/n/12 (-20) ,
1482 1490 []
1483 1491
1484 −[30]) (./appendix/runs.tex
1492 +[31]) (./appendix/runs.tex
1485 1493 Overfull \hbox (121.91187pt too wide) in paragraph at lines 1--14
1486 1494 \T1/cmtt/m/n/12 training.pretrain\T1/ptm/m/n/12 (-20) , \T1/cmtt/m/n/12 trainin
1487 1495 g.train_editor\T1/ptm/m/n/12 (-20) , \T1/cmtt/m/n/12 training.collect_intervent
@@ -1496,14 +1504,14 @@ tion.latency \T1/ptm/m/n/12 (-20) or \T1/cmtt/m/n/12 evaluation.run\T1/ptm/m/n/
1496 1504 12 (-20) ; launch scripts un-der \T1/cmtt/m/n/12 scripts/
1497 1505 []
1498 1506
1499 −[31] (./appendix/runs_index.tex [32]
1507 +[32] (./appendix/runs_index.tex [33]
1500 1508 Overfull \hbox (25.12868pt too wide) in paragraph at lines 52--54
1501 1509 []\T1/ptm/b/n/10 (-20) run_id\T1/ptm/m/n/10 (-20) : p4_tiny_backbone_v2_seed01_
1502 1510 20261029233624; \T1/ptm/b/n/10 (-20) state\T1/ptm/m/n/10 (-20) : stopped_overfi
1503 1511 tting_step9300_checkpoint_step5000_used;
1504 1512 []
1505 1513
1506 −[33])) (./appendix/results.tex (./tables/em_families.tex))
1514 +[34])) (./appendix/results.tex (./tables/em_families.tex))
1507 1515 Overfull \hbox (168.4564pt too wide) in paragraph at lines 2--16
1508 1516 \T1/ptm/m/n/12 (-20) records (\T1/cmtt/m/n/12 experiments/runs/*/em_*.json\T1/p
1509 1517 tm/m/n/12 (-20) , \T1/cmtt/m/n/12 results/raw/items_*.json\T1/ptm/m/n/12 (-20)
@@ -1541,7 +1549,7 @@ Overfull \hbox (47.44559pt too wide) in paragraph at lines 12--14
1541 1549 ;
1542 1550 []
1543 1551
1544 −[34]
1552 +[35]
1545 1553 Package microtype Info: Loading generic protrusion settings for font family
1546 1554 (microtype) `cmtt' (encoding: TS1).
1547 1555 (microtype) For optimal results, create family-specific settings.
@@ -1560,8 +1568,15 @@ Overfull \hbox (12.79033pt too wide) in paragraph at lines 18--20
1560 1568 run \T1/cmtt/m/n/12 p4_tiny_backbone_v2_seed01_* \T1/ptm/m/n/12 (-20) (state:
1561 1569 []
1562 1570
1563 −)) [35{/usr/local/texlive/2026/texmf-dist/fonts/enc/dvips/cm-super/cm-super-ts1
1564 −.enc}] [36] [37] [38] [39] (./main.aux)
1571 +
1572 +Overfull \hbox (12.01894pt too wide) in paragraph at lines 22--23
1573 +\T1/ptm/m/n/12 (-20) 0.4334); $\OML/ztmcm/m/it/12 ^^U$\T1/ptm/m/n/12 (-20) =0 i
1574 +s worse (+1.3 %).; \T1/ptm/b/n/12 (-20) ev-i-dence\T1/ptm/m/n/12 (-20) : \T1/cm
1575 +tt/m/n/12 experiments/runs/controller_v1rank_seed01_*\T1/ptm/m/n/12 (-20) ,
1576 + []
1577 +
1578 +)) [36{/usr/local/texlive/2026/texmf-dist/fonts/enc/dvips/cm-super/cm-super-ts1
1579 +.enc}] [37] [38] [39] [40] (./main.aux)
1565 1580 ***********
1566 1581 LaTeX2e <2025-11-01>
1567 1582 L3 programming layer <2026-01-19>
@@ -1571,13 +1586,13 @@ L3 programming layer <2026-01-19>
1571 1586 LaTeX Warning: There were undefined references.
1572 1587
1573 1588 Package rerunfilecheck Info: File `main.out' has not changed.
1574 −(rerunfilecheck) Checksum: A0F1ADB953B7C9C76AC4BF5EDF00287D;9020.
1589 +(rerunfilecheck) Checksum: 445BE7B843FC16B6CF1F9C8E67681D49;9020.
1575 1590 )
1576 1591 Here is how much of TeX's memory you used:
1577 − 36063 strings out of 467525
1578 − 701177 string characters out of 5418982
1579 − 1129055 words of memory out of 5000000
1580 − 63386 multiletter control sequences out of 15000+600000
1592 + 36073 strings out of 467525
1593 + 701525 string characters out of 5418982
1594 + 1129340 words of memory out of 5000000
1595 + 63394 multiletter control sequences out of 15000+600000
1581 1596 793198 words of font info for 663 fonts, out of 8000000 for 9000
1582 1597 1141 hyphenation exceptions out of 8191
1583 1598 84i,13n,104p,1001b,1501s stack positions out of 10000i,1000n,20000p,200000b,200000s
@@ -1596,10 +1611,10 @@ exlive/2026/texmf-dist/fonts/type1/urw/symbol/usyr.pfb></usr/local/texlive/2026
1596 1611 e1/urw/times/utmbi8a.pfb></usr/local/texlive/2026/texmf-dist/fonts/type1/urw/ti
1597 1612 mes/utmr8a.pfb></usr/local/texlive/2026/texmf-dist/fonts/type1/urw/times/utmri8
1598 1613 a.pfb>
1599 −Output written on main.pdf (39 pages, 416998 bytes).
1614 +Output written on main.pdf (40 pages, 433081 bytes).
1600 1615 PDF statistics:
1601 − 764 PDF objects out of 1000 (max. 8388607)
1602 − 663 compressed objects within 7 object streams
1603 − 155 named destinations out of 1000 (max. 500000)
1604 − 143772 words of extra memory for PDF output out of 154059 (max. 10000000)
1616 + 783 PDF objects out of 1000 (max. 8388607)
1617 + 675 compressed objects within 7 object streams
1618 + 157 named destinations out of 1000 (max. 500000)
1619 + 143777 words of extra memory for PDF output out of 154059 (max. 10000000)
1605 1620
modified paper/main.pdf +0 −0

Binary file not shown.

modified paper/sections/results.tex +11 −0
@@ -44,7 +44,18 @@ latent, parameter-matched, same bootstrap---reduces the loss by \vtwogrgainbone{
44 44 good at every budget and better at budget 8, where the sparse editor's gain shrinks to \vtwoprismegainbeight{}\,\%.
45 45 Dense writes (Table~\ref{tab:seedmatrix}, third row) answer the second half of H2 directly. None of the
46 46 adapters improves the held-out template, depth or compose regimes by more than one point (H6).
47 +Figure~\ref{fig:frontier} places the same runs on the quality-versus-analytic-cost plane: the global adapter is
48 +both cheaper per step (one block at width $w$ over the whole latent costs less than two shared blocks with slot
49 +re-keying for two lanes) and better at every budget, so it dominates the sparse editor along the entire frontier;
50 +H4 is refuted at this scale for the comparison the proposal considered most important.
47 51 \input{tables/seed_matrix.tex}
52 +\begin{figure}[t]\centering
53 +\includegraphics[width=0.72\textwidth]{../results/figures/frontier_v2.pdf}
54 +\caption{Quality versus analytic extra operations per token at the tiny scale (five adapter seeds, mean $\pm$ SD),
55 +random forced budgets $b\in\{0,1,2,4,8\}$, context 160. Costs from \texttt{prisme/flops.py}; the sparse editor's
56 +cost is the average of single and pair actions.}
57 +\label{fig:frontier}
58 +\end{figure}
48 59
49 60 \subsection{Oracle opportunity and the structure of intervention gains (Phase 5)}\label{sec:results-oracle}
50 61 Figure~\ref{fig:labels} and Table~\ref{tab:controller} summarise the counterfactual labels collected with the
modified prisme/flops.py +12 −0
@@ -112,3 +112,15 @@ def action_cost_proxy(c: PrismeConfig, T: int) -> list[float]:
112 112 e = c.n_editor_blocks * sum(blk.values()) + 2 * k * 2 * c.w + controller_step_ops(c)
113 113 costs.append(e / base)
114 114 return costs
115 +
116 +
117 +def global_recurrence_step_ops(c: PrismeConfig, T: int, ffn_w: int | None = None) -> float:
118 + """One refinement step of the global-recurrence adapter (baselines/global_recurrence.py) for one position at
119 + context T: q/k/v/o projections (8w^2), causal attention over T positions (4Tw), SwiGLU (6 w f), plus the
120 + down/up projections charged once per token (4dw)."""
121 + w, f = c.w, (ffn_w or c.ffn_w)
122 + return 8 * w * w + 4 * T * w + 6 * w * f
123 +
124 +
125 +def global_recurrence_token_ops(c: PrismeConfig) -> float:
126 + return 2 * 2 * c.d_model * c.w
added results/figures/frontier_v2.pdf +0 −0

Binary file not shown.

added results/figures/frontier_v2.png +0 −0

Binary file not shown.

modified results/figures/labels_analysis.pdf +0 −0

Binary file not shown.

modified results/figures/latency_p4_editor_v1_Ustd02_seed01_20261029235050_mps.pdf +0 −0

Binary file not shown.

modified results/figures/learnability_bootstrap.pdf +0 −0

Binary file not shown.

modified results/figures/learnability_v2.pdf +0 −0

Binary file not shown.

modified results/figures/learnability_v2.png +0 −0

Binary file not shown.

modified results/figures/routing_controller_p4_hent_seed01_20261030002015_dev_iid.pdf +0 −0

Binary file not shown.

modified results/figures/routing_mini.pdf +0 −0

Binary file not shown.

modified results/figures/routing_v1ctrl.pdf +0 −0

Binary file not shown.

added results/figures/routing_v1rank.pdf +0 −0

Binary file not shown.

added results/figures/routing_v1rank.png +0 −0

Binary file not shown.

added results/figures/routing_v2s1.pdf +0 −0

Binary file not shown.

added results/figures/routing_v2s1.png +0 −0

Binary file not shown.

modified results/figures/routing_v2s2.pdf +0 −0

Binary file not shown.

modified results/figures/routing_v2s3.pdf +0 −0

Binary file not shown.

modified results/figures/routing_v2s4.pdf +0 −0

Binary file not shown.

modified results/figures/routing_v2s5.pdf +0 −0

Binary file not shown.

modified scripts/reproduce_figures.py +36 −1
@@ -178,9 +178,44 @@ def main(argv=None):
178 178 fig_learnability(args.runs, ["p4_editor_v1_scaled", "p4_editor_v1_Ustd02", "p4_editor_v1_Ustd05"], "learnability_bootstrap", "Stage B bootstrap on the frozen tiny backbone (dev-iid, random forced actions)")
179 179 fig_learnability(args.runs, ["p4v2_prisme", "p4v2_gr", "p4v2_dense"], "learnability_v2", "Stage B on the v2 tiny backbone: sparse editor vs global recurrence vs dense writes")
180 180 fig_labels(f"{args.analyzed}/labels_*.json")
181 − fig_routing(args.results); fig_latency(args.results); fig_exact_match(args.results); fig_pretrain(args.runs)
181 + fig_routing(args.results); fig_latency(args.results); fig_exact_match(args.results); fig_pretrain(args.runs); fig_frontier_v2()
182 182 print("made:", MADE); print("skipped (inputs absent):", SKIPPED)
183 183
184 184
185 +
186 +def fig_frontier_v2(analyzed="results/analyzed/seed_matrix_v2.json"):
187 + """Quality vs analytic extra ops (tiny scale): PRISME sparse editor vs global recurrence vs dense, by budget."""
188 + p = Path(analyzed)
189 + if not p.exists():
190 + SKIPPED.append("frontier_v2"); return
191 + import yaml
192 + from prisme.config import PrismeConfig
193 + from prisme.flops import ops_per_token, global_recurrence_step_ops, global_recurrence_token_ops
194 + d = json.loads(p.read_text())
195 + cfg = PrismeConfig.from_dict(yaml.safe_load(open("configs/experiments/p4v2_prisme.yaml"))["model"])
196 + T = 160 # typical synthetic sequence length
197 + base = ops_per_token(cfg, T, 0, 1).backbone
198 + fig, ax = plt.subplots(figsize=(4.4, 2.8))
199 + labels = {"p4v2_prisme": "sparse editor (1.5 slots avg.)", "p4v2_gr": "global recurrence", "p4v2_dense": "dense 16-slot writes"}
200 + for i, (key, lab) in enumerate(labels.items()):
201 + if key not in d:
202 + continue
203 + v = d[key]["val"]; xs, ys, es = [], [], []
204 + for b in sorted(int(k) for k in v):
205 + if key == "p4v2_gr":
206 + x = (b * global_recurrence_step_ops(cfg, T, 640) + (global_recurrence_token_ops(cfg) if b else 0)) / base
207 + elif key == "p4v2_dense":
208 + x = ops_per_token(cfg, T, b, cfg.n_slots, use_controller=False).workspace / base if b else 0.0
209 + else:
210 + x = 0.5 * (ops_per_token(cfg, T, b, 1, use_controller=False).workspace + ops_per_token(cfg, T, b, 2, use_controller=False).workspace) / base if b else 0.0
211 + xs.append(x); ys.append(np.mean(v[str(b)])); es.append(np.std(v[str(b)], ddof=1) if len(v[str(b)]) > 1 else 0)
212 + ax.errorbar(xs, ys, yerr=es, marker="o", color=CAT[i], label=lab, capsize=2)
213 + for x, y, b in zip(xs, ys, sorted(int(k) for k in v)):
214 + ax.annotate(f"b={b}", (x, y), textcoords="offset points", xytext=(4, 3), fontsize=6, color="#52514e")
215 + ax.set_xlabel("analytic extra operations per token (× backbone)"); ax.set_ylabel("dev-iid answer-span loss (nats)")
216 + ax.set_title("Quality vs cost under random forced budgets (5 seeds)", loc="left"); ax.legend()
217 + MADE.append(save(fig, "frontier_v2"))
218 +
219 +
185 220 if __name__ == "__main__":
186 221 main()
187 222
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere