1% ============================================================================2% papers.bib -- UQO Working Paper 31 (PRISME), literature review3% Compiled 2026-10-29 by verifying every entry against primary sources:4% * arXiv Atom API (https://export.arxiv.org/api/query?id_list=...) for5% title, author list, first-submission date, latest version, comments/DOI;6% * GitHub REST API (api.github.com/repos/<owner>/<repo>) for code + licence;7% * JMLR / PMLR / project pages where cited.8% Convention: `year` = year of first arXiv submission unless a venue is given.9% Venues are stated ONLY when verified from arXiv metadata (comment/DOI field),10% a PMLR/JMLR page, or an official repository description. Where a venue is11% widely known but was NOT verified here, it is flagged in `note`.12% `note` also records code availability and licence (SPDX id from GitHub API).13% ============================================================================1415% ---------------------------------------------------------------------------16% 1. Core references requested (12)17% ---------------------------------------------------------------------------1819@inproceedings{dehghani2019universal,20 title = {Universal Transformers},21 author = {Dehghani, Mostafa and Gouws, Stephan and Vinyals, Oriol and Uszkoreit, Jakob and Kaiser, {\L}ukasz},22 booktitle = {International Conference on Learning Representations (ICLR)},23 year = {2019},24 eprint = {1807.03819},25 archivePrefix = {arXiv},26 url = {https://arxiv.org/abs/1807.03819},27 note = {arXiv v1 2018-07-10, v3 2019-03-05; ICLR 2019 per arXiv comment. Mechanism: depth-shared transformer block applied recurrently with ACT-style per-position dynamic halting. Code: tensorflow/tensor2tensor (Apache-2.0).}28}2930@inproceedings{banino2021pondernet,31 title = {PonderNet: Learning to Ponder},32 author = {Banino, Andrea and Balaguer, Jan and Blundell, Charles},33 booktitle = {8th ICML Workshop on Automated Machine Learning (AutoML)},34 year = {2021},35 eprint = {2107.05407},36 archivePrefix = {arXiv},37 url = {https://arxiv.org/abs/2107.05407},38 note = {arXiv v1 2021-07-12, v2 2021-09-02; venue per arXiv comment. Mechanism: probabilistic halting (geometric prior, KL-regularised) over recurrent steps; expected-loss training. No official code release verified.}39}4041@inproceedings{hawthorne2022perceiverar,42 title = {General-purpose, long-context autoregressive modeling with {Perceiver AR}},43 author = {Hawthorne, Curtis and Jaegle, Andrew and Cangea, C{\u{a}}t{\u{a}}lina and Borgeaud, Sebastian and Nash, Charlie and Malinowski, Mateusz and Dieleman, Sander and Vinyals, Oriol and Botvinick, Matthew and Simon, Ian and Sheahan, Hannah and Zeghidour, Neil and Alayrac, Jean-Baptiste and Carreira, Jo{\~a}o and Engel, Jesse},44 booktitle = {International Conference on Machine Learning (ICML)},45 year = {2022},46 eprint = {2202.07765},47 archivePrefix = {arXiv},48 url = {https://arxiv.org/abs/2202.07765},49 note = {arXiv v1 2022-02-15, v2 2022-06-14; ICML 2022 per arXiv comment. Mechanism: small set of latents (last positions) cross-attend causally to a long input; latents re-created per forward pass. Code: google-research/perceiver-ar (Apache-2.0).}50}5152@article{goyal2019rims,53 title = {Recurrent Independent Mechanisms},54 author = {Goyal, Anirudh and Lamb, Alex and Hoffmann, Jordan and Sodhani, Shagun and Levine, Sergey and Bengio, Yoshua and Sch{\"o}lkopf, Bernhard},55 journal = {arXiv preprint arXiv:1909.10893},56 year = {2019},57 url = {https://arxiv.org/abs/1909.10893},58 note = {arXiv v1 2019-09-24, v6 2020-11-17. Venue (ICLR 2021) NOT verified from arXiv metadata. Mechanism: K recurrent modules, top-k activated per step by input attention, sparse communication; only selected modules update. Code: anirudh9119/RIMs (no licence file in repo).}59}6061@article{raposo2024mixtureofdepths,62 title = {Mixture-of-Depths: Dynamically allocating compute in transformer-based language models},63 author = {Raposo, David and Ritter, Sam and Richards, Blake and Lillicrap, Timothy and Humphreys, Peter Conway and Santoro, Adam},64 journal = {arXiv preprint arXiv:2404.02258},65 year = {2024},66 url = {https://arxiv.org/abs/2404.02258},67 note = {arXiv v1 2024-04-02 (single version). Mechanism: per-layer top-k token routing with fixed capacity; non-routed tokens bypass the block through the residual (exact skip). No official code release found.}68}6970@article{geiping2025recurrentdepth,71 title = {Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach},72 author = {Geiping, Jonas and McLeish, Sean and Jain, Neel and Kirchenbauer, John and Singh, Siddharth and Bartoldson, Brian R. and Kailkhura, Bhavya and Bhatele, Abhinav and Goldstein, Tom},73 journal = {arXiv preprint arXiv:2502.05171},74 year = {2025},75 url = {https://arxiv.org/abs/2502.05171},76 note = {Huginn. arXiv v1 2025-02-07, v2 2025-02-17. Mechanism: prelude / recurrent core / coda; core iterated r times with random r at training; zero-shot per-token exit via KL threshold between iterations. Code: seal-rg/recurrent-pretraining (Apache-2.0); model huggingface.co/tomg-group-umd/huginn-0125.}77}7879@inproceedings{bae2025mixtureofrecursions,80 title = {Mixture-of-Recursions: Learning Dynamic Recursive Depths for Adaptive Token-Level Computation},81 author = {Bae, Sangmin and Kim, Yujin and Bayat, Reza and Kim, Sungnyun and Ha, Jiyoun and Schuster, Tal and Fisch, Adam and Harutyunyan, Hrayr and Ji, Ziwei and Courville, Aaron and Yun, Se-Young},82 booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},83 year = {2025},84 eprint = {2507.10524},85 archivePrefix = {arXiv},86 url = {https://arxiv.org/abs/2507.10524},87 note = {arXiv v1 2025-07-14, v3 2025-10-25. NeurIPS 2025 per official repository description. Mechanism: shared recursive block + lightweight routers (expert-choice / token-choice) assigning a recursion depth per token; recursion-wise KV caching. Code: raymin0223/mixture\_of\_recursions (Apache-2.0).}88}8990@article{zhu2025ouro,91 title = {Scaling Latent Reasoning via Looped Language Models},92 author = {Zhu, Rui-Jie and Wang, Zixuan and Hua, Kai and Zhang, Tianyu and Li, Ziniu and Que, Haoran and Wei, Boyi and Wen, Zixin and Yin, Fan and Xing, He and Li, Lu and Shi, Jiajun and Ma, Kaijing and Li, Shanda and Kergan, Taylor and Smith, Andrew and Qu, Xingwei and Hui, Mude and Wu, Bohong and Min, Qiyang and Huang, Hongzhi and Zhou, Xun and Ye, Wei and Liu, Jiaheng and Yang, Jian and Shi, Yunfeng and Lin, Chenghua and Zhao, Enduo and Cai, Tianle and Zhang, Ge and Huang, Wenhao and Bengio, Yoshua and Eshraghian, Jason},93 journal = {arXiv preprint arXiv:2510.25741},94 year = {2025},95 url = {https://arxiv.org/abs/2510.25741},96 note = {Ouro / LoopLM. arXiv v1 2025-10-29, v6 2026-10-11. Mechanism: weight-tied stack looped up to 4 times; per-step exit gate lambda\_t = sigma(Linear(h\_t)) trained (Stage I) with entropy-regularised expected loss and (Stage II, LM frozen) with BCE against labels w = sigma(k(max(0, L\_{t-1}-L\_t) - gamma)) derived from observed per-token loss improvements. Weights: huggingface.co/ByteDance/Ouro-1.4B, Ouro-2.6B (+ -Thinking) under Apache-2.0 (research-use notice). No official training code found; community reimplementation rkstgr/LoopLM (no licence). No venue listed.}97}9899@article{wang2026recurtrace,100 title = {RecurTrace: Adaptive Latent Reasoning with Loop-Time Memory},101 author = {Wang, Yuxiang and Feng, Kunyu and Shen, Yingda and Xu, Haoning and Wang, Junyu and Wu, Zhizheng},102 journal = {arXiv preprint arXiv:2609.03379},103 year = {2026},104 url = {https://arxiv.org/abs/2609.03379},105 note = {arXiv v1 2026-09-03, v3 2026-10-19. Mechanism: Loop Memory Attention (each looped layer attends over a window W of its own outputs from earlier loops at the same position; empty at loop 1 so the term is zero) + sequence-level halting head p\_t = sigma(w^T z\_t + b) trained with BCE against oracle labels 1[min\_{t'>t} L\_{t'} < L\_t - delta]. Code: JensenYX/RecurTrace (Apache-2.0, repo created 2026-10-17). No venue listed.}106}107108@article{ladhak2026elastic,109 title = {One Model to Loop Them All: Elastic Latent Recurrence for Language Models},110 author = {Ladhak, Faisal and Acharya, Shantanu and Akle Serano, Santiago and Sun, Simeng and Ginsburg, Boris and Puvvada, Krishna C.},111 journal = {arXiv preprint arXiv:2610.12582},112 year = {2026},113 url = {https://arxiv.org/abs/2610.12582},114 note = {ELR. arXiv v1 2026-10-08. Mechanism: training recipe removing the "one-loop tax": gated state update with learned step size, mixed loop counts per optimizer update, loop-index/budget conditioning (zero-initialised when added late), last-loop-only KV cache. Reports Koi-4B checkpoint. No code or checkpoint link verified (paper HTML links only to the nvidia nemotron-post-training-v3 HF collection); third-party claims of a public "Dory1.0-4B" release were NOT verified.}115}116117@article{behrouz2024titans,118 title = {Titans: Learning to Memorize at Test Time},119 author = {Behrouz, Ali and Zhong, Peilin and Mirrokni, Vahab},120 journal = {arXiv preprint arXiv:2501.00663},121 year = {2024},122 url = {https://arxiv.org/abs/2501.00663},123 note = {arXiv v1 2024-12-31 (single version). Mechanism: neural long-term memory module updated at test time by gradient (surprise) with forgetting; persistent memory tokens; three integration variants (MAC/MAG/MAL). No official code; unofficial lucidrains/titans-pytorch (MIT).}124}125126@inproceedings{elhoushi2024layerskip,127 title = {LayerSkip: Enabling Early Exit Inference and Self-Speculative Decoding},128 author = {Elhoushi, Mostafa and Shrivastava, Akshat and Liskovich, Diana and Hosmer, Basil and Wasti, Bram and Lai, Liangzhen and Mahmoud, Anas and Acun, Bilge and Agarwal, Saurabh and Roman, Ahmed and Aly, Ahmed A. and Chen, Beidi and Wu, Carole-Jean},129 booktitle = {Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (ACL)},130 year = {2024},131 doi = {10.18653/v1/2024.acl-long.681},132 eprint = {2404.16710},133 archivePrefix = {arXiv},134 url = {https://arxiv.org/abs/2404.16710},135 note = {arXiv v1 2024-04-25, v4 2024-10-18. Mechanism: layer dropout + shared early-exit loss; self-speculative decoding (draft with early layers, verify with the rest). Code: facebookresearch/LayerSkip (CC BY-NC 4.0).}136}137138% ---------------------------------------------------------------------------139% 2. Adaptive computation, looped / recurrent-depth, latent reasoning140% ---------------------------------------------------------------------------141142@article{graves2016act,143 title = {Adaptive Computation Time for Recurrent Neural Networks},144 author = {Graves, Alex},145 journal = {arXiv preprint arXiv:1603.08983},146 year = {2016},147 url = {https://arxiv.org/abs/1603.08983},148 note = {arXiv v1 2016-03-29, v6 2017-02-21. Mechanism: sigmoidal halting unit accumulated to a threshold; ponder cost penalty. No official code.}149}150151@inproceedings{hao2024coconut,152 title = {Training Large Language Models to Reason in a Continuous Latent Space},153 author = {Hao, Shibo and Sukhbaatar, Sainbayar and Su, DiJia and Li, Xian and Hu, Zhiting and Weston, Jason and Tian, Yuandong},154 booktitle = {Conference on Language Modeling (COLM)},155 year = {2025},156 eprint = {2412.06769},157 archivePrefix = {arXiv},158 url = {https://arxiv.org/abs/2412.06769},159 note = {Coconut. arXiv v1 2024-12-09, v4 2026-08-23; COLM 2025 per arXiv comment. Mechanism: last hidden state fed back as next input embedding ("continuous thought") for a fixed number of latent steps, curriculum from CoT. Code: facebookresearch/coconut (MIT).}160}161162@article{giannou2023looped,163 title = {Looped Transformers as Programmable Computers},164 author = {Giannou, Angeliki and Rajput, Shashank and Sohn, Jy-yong and Lee, Kangwook and Lee, Jason D. and Papailiopoulos, Dimitris},165 journal = {arXiv preprint arXiv:2301.13196},166 year = {2023},167 url = {https://arxiv.org/abs/2301.13196},168 note = {arXiv v1 2023-01-30. Venue (ICML 2023) NOT verified from arXiv metadata. Constructive result: a looped transformer with a scratchpad can emulate a programmable computer.}169}170171@inproceedings{saunshi2025reasoning,172 title = {Reasoning with Latent Thoughts: On the Power of Looped Transformers},173 author = {Saunshi, Nikunj and Dikkala, Nishanth and Li, Zhiyuan and Kumar, Sanjiv and Reddi, Sashank J.},174 booktitle = {International Conference on Learning Representations (ICLR)},175 year = {2025},176 eprint = {2502.17416},177 archivePrefix = {arXiv},178 url = {https://arxiv.org/abs/2502.17416},179 note = {arXiv v1 2025-02-24; ICLR 2025 per arXiv comment. Theory + experiments: k-layer block looped L times matches kL-layer models on reasoning; looping-inspired regularisation.}180}181182@inproceedings{fan2025loopedlength,183 title = {Looped Transformers for Length Generalization},184 author = {Fan, Ying and Du, Yilun and Ramchandran, Kannan and Lee, Kangwook},185 booktitle = {International Conference on Learning Representations (ICLR)},186 year = {2025},187 eprint = {2409.15647},188 archivePrefix = {arXiv},189 url = {https://arxiv.org/abs/2409.15647},190 note = {arXiv v1 2024-09-24, v5 2025-05-12; ICLR 2025 per arXiv comment. Adaptive number of loops tied to problem length for algorithmic tasks.}191}192193@inproceedings{bae2024relaxed,194 title = {Relaxed Recursive Transformers: Effective Parameter Sharing with Layer-wise LoRA},195 author = {Bae, Sangmin and Fisch, Adam and Harutyunyan, Hrayr and Ji, Ziwei and Kim, Seungyeon and Schuster, Tal},196 booktitle = {International Conference on Learning Representations (ICLR)},197 year = {2025},198 eprint = {2410.20672},199 archivePrefix = {arXiv},200 url = {https://arxiv.org/abs/2410.20672},201 note = {arXiv v1 2024-10-28, v3 2025-02-28; ICLR 2025 per arXiv comment. Recursive (looped) transformers with per-loop LoRA; continuous depth-wise batching for early exit. Predecessor of Mixture-of-Recursions.}202}203204@inproceedings{goyal2023pause,205 title = {Think before you speak: Training Language Models With Pause Tokens},206 author = {Goyal, Sachin and Ji, Ziwei and Rawat, Ankit Singh and Menon, Aditya Krishna and Kumar, Sanjiv and Nagarajan, Vaishnavh},207 booktitle = {International Conference on Learning Representations (ICLR)},208 year = {2024},209 eprint = {2310.02226},210 archivePrefix = {arXiv},211 url = {https://arxiv.org/abs/2310.02226},212 note = {arXiv v1 2023-10-03, v3 2024-04-21; ICLR 2024 per arXiv comment. Mechanism: learnable dummy tokens appended before the answer to buy extra (dense, uniform) computation whose outputs are discarded.}213}214215@article{sun2025latenttokens,216 title = {Enhancing Latent Computation in Transformers with Latent Tokens},217 author = {Sun, Yuchang and Chen, Yanxi and Li, Yaliang and Ding, Bolin},218 journal = {arXiv preprint arXiv:2505.12629},219 year = {2025},220 url = {https://arxiv.org/abs/2505.12629},221 note = {arXiv v1 2025-05-19. Mechanism: dummy non-linguistic tokens inserted before verbal tokens, parameter-efficient training, loss terms on them dropped.}222}223224@inproceedings{zeng2025ponderlm,225 title = {PonderLM: Pretraining Language Models to Ponder in Continuous Space},226 author = {Zeng, Boyi and Song, Shixiang and Huang, Siyuan and Wang, Yixuan and Li, He and He, Ziwei and Wang, Xinbing and Li, Zhiyu and Lin, Zhouhan},227 booktitle = {International Conference on Learning Representations (ICLR)},228 year = {2026},229 eprint = {2505.20674},230 archivePrefix = {arXiv},231 url = {https://arxiv.org/abs/2505.20674},232 note = {arXiv v1 2025-05-27, v3 2026-02-20; ICLR 2026 per arXiv comment. Mechanism: within one token step, feed back the probability-weighted sum of token embeddings as the next input for additional forward passes (self-supervised). Code: LUMIA-Group/PonderingLM (Apache-2.0).}233}234235@article{chen2025innerthinking,236 title = {Inner Thinking Transformer: Leveraging Dynamic Depth Scaling to Foster Adaptive Internal Thinking},237 author = {Chen, Yilong and Shang, Junyuan and Zhang, Zhenyu and Xie, Yanxi and Sheng, Jiawei and Liu, Tingwen and Wang, Shuohuan and Sun, Yu and Wu, Hua and Wang, Haifeng},238 journal = {arXiv preprint arXiv:2502.13842},239 year = {2025},240 url = {https://arxiv.org/abs/2502.13842},241 note = {arXiv v1 2025-02-19, v2 2025-02-23. Mechanism: Adaptive Token Routing selects critical tokens for extra inner iterations of a layer; residual thinking connections; thinking-step encoding.}242}243244@article{koishekenov2025etd,245 title = {Encode, Think, Decode: Scaling test-time reasoning with recursive latent thoughts},246 author = {Koishekenov, Yeskendir and Lipani, Aldo and Cancedda, Nicola},247 journal = {arXiv preprint arXiv:2510.07358},248 year = {2025},249 url = {https://arxiv.org/abs/2510.07358},250 note = {arXiv v1 2025-10-08. Mechanism: mid-training to iterate a small subset of reasoning-relevant layers; explores adaptive per-token depth.}251}252253@article{wu2025parallelloop,254 title = {Parallel Loop Transformer for Efficient Test-Time Computation Scaling},255 author = {Wu, Bohong and Chen, Mengzhao and Luo, Xiang and Yan, Shen and Yu, Qifan and Xia, Fan and Zhang, Tianqi and Zhan, Hongrui and Zhong, Zheng and Zhou, Xun and Qiao, Siyuan and Bin, Xingyan},256 journal = {arXiv preprint arXiv:2510.24824},257 year = {2025},258 url = {https://arxiv.org/abs/2510.24824},259 note = {arXiv v1 2025-10-28. Mechanism: cross-loop parallelism (different loops for different tokens in one pass), first-loop KV sharing, gated sliding-window attention.}260}261262@inproceedings{fu2026thinkathard,263 title = {Think-at-Hard: Dynamic Looped Transformers for Improved Reasoning},264 author = {Fu, Tianyu and You, Yichen and Chen, Zekai and Dai, Guohao and Yang, Huazhong and Wang, Yu},265 booktitle = {Proceedings of the 43rd International Conference on Machine Learning (ICML), PMLR 306},266 pages = {32129--32154},267 year = {2026},268 eprint = {2511.08577},269 archivePrefix = {arXiv},270 url = {https://proceedings.mlr.press/v306/fu26t.html},271 note = {TaH. arXiv v1 2025-11-11 (earlier title "Selective Latent Iterations to Improve Reasoning Language Models"), v4 2026-08-28; ICML 2026 verified on PMLR. Mechanism: lightweight neural decider triggers a second latent iteration only at tokens likely wrong after the first pass (oracle = token mismatch); depth-aware LoRA; duo-causal attention across iteration depth. Code: thu-nics/TaH (Apache-2.0).}272}273274@article{you2026tah2,275 title = {Improving Test-Time Scaling with Adaptive Looped Transformers},276 author = {You, Yichen and Fu, Tianyu and Feng, Aosong and Lv, Xingtai and Ning, Xuefei and Ding, Ning and Wang, Yu},277 journal = {arXiv preprint arXiv:2609.35748},278 year = {2026},279 url = {https://arxiv.org/abs/2609.35748},280 note = {TaH2. arXiv v1 2026-09-28. Mechanism: joint post-training of backbone and per-token iteration decider with "lookahead depth supervision": online per-token labels from the loss gain L\_m - L\_{m+1} (no-gradient lookahead branch for tokens that already stopped), coverage filtering (rho=0.99), cost-sensitive BCE; binary continue/stop, absorbing stop; no propensity logging. Code stated as github.com/thu-nics/TaH (shared with TaH).}281}282283@inproceedings{jeddi2026loopformer,284 title = {LoopFormer: Elastic-Depth Looped Transformers for Latent Reasoning via Shortcut Modulation},285 author = {Jeddi, Ahmadreza and Ciccone, Marco and Taati, Babak},286 booktitle = {International Conference on Learning Representations (ICLR)},287 year = {2026},288 eprint = {2602.11451},289 archivePrefix = {arXiv},290 url = {https://arxiv.org/abs/2602.11451},291 note = {arXiv v1 2026-02-11; ICLR 2026 per arXiv comment. Mechanism: shortcut-consistency training on variable-length loop trajectories conditioned on time and step size (budget-conditioned depth).}292}293294@article{song2026adaponderlm,295 title = {AdaPonderLM: Gated Pondering Language Models with Token-Wise Adaptive Depth},296 author = {Song, Shixiang and Li, He and Wang, Zitong and Zeng, Boyi and Song, Feichen and Wang, Yixuan and Xu, Zhiqin John and He, Ziwei and Lin, Zhouhan},297 journal = {arXiv preprint arXiv:2603.01914},298 year = {2026},299 url = {https://arxiv.org/abs/2603.01914},300 note = {arXiv v1 2026-03-02, v2 2026-03-11. Mechanism: iteration-specific MLP gates with monotonic halting mask learned self-supervised during pretraining; KV reuse for halted tokens.}301}302303@article{noci2026lookahead,304 title = {Thinking into the Future: Latent Lookahead Training for Transformers},305 author = {Noci, Lorenzo and Bachmann, Gregor and Moosavi-Dezfooli, Seyed-Mohsen and Nabi, Moin},306 journal = {arXiv preprint arXiv:2603.20219},307 year = {2026},308 url = {https://arxiv.org/abs/2603.20219},309 note = {arXiv v1 2026-03-03. Mechanism: at selected positions, recursively feed hidden states back for tau latent steps supervised against the next tau tokens.}310}311312@inproceedings{frey2026adaptiveloops,313 title = {Adaptive Loops and Memory in Transformers: Think Harder or Know More?},314 author = {Frey, Markus and Shomali, Behzad and Bashir, Ali Hamza and Berghaus, David and Koehler, Joachim and Ali, Mehdi},315 booktitle = {Latent \& Implicit Thinking Workshop @ ICLR 2026},316 year = {2026},317 eprint = {2603.08391},318 archivePrefix = {arXiv},319 url = {https://arxiv.org/abs/2603.08391},320 note = {arXiv v1 2026-03-09, v3 2026-03-11; workshop per arXiv comment. Mechanism: per-layer adaptive looping with learned halting + gated memory banks.}321}322323@article{park2026loopus,324 title = {LoopUS: Recasting Pretrained LLMs into Looped Latent Refinement Models},325 author = {Park, Taekhyun and Lee, Yongjae and Kim, Dohee and Bae, Hyerim},326 journal = {arXiv preprint arXiv:2605.11011},327 year = {2026},328 url = {https://arxiv.org/abs/2605.11011},329 note = {arXiv v1 2026-05-10. Mechanism: encoder / looped block / decoder recast of a pretrained LLM; input-dependent selective gate; random deep supervision; confidence head for early exit.}330}331332@article{huang2026lrt,333 title = {Latent Recurrent Transformer: Architecture Exploration, Training Strategies, and Scaling Behavior},334 author = {Huang, Zeyi and He, Xuehai and Ren, LiLiang and Wang, Yiping and Peng, Baolin and Cheng, Hao and Wang, Shuohang and He, Pengcheng and Gao, Jianfeng and Lee, Yong Jae and Shen, Yelong},335 journal = {arXiv preprint arXiv:2605.26797},336 year = {2026},337 url = {https://arxiv.org/abs/2605.26797},338 note = {arXiv v1 2026-05-26, v2 2026-09-01. Mechanism: reuse a high-level source-layer hidden state of the previous token as recurrent memory for the next token; interleaved parallel training; one forward per token.}339}340341@article{li2026later,342 title = {LaTER: Efficient Test-Time Reasoning via Latent Exploration and Explicit Verification},343 author = {Li, Xuan and Wang, Yining and Liu, Yuchen and Liu, Guanjun and Qiu, Delai and Liu, Shengping and Liang, Jiaen and Huang, Wei and Yu, Jun and Zhu, Junnan},344 journal = {arXiv preprint arXiv:2605.07315},345 year = {2026},346 url = {https://arxiv.org/abs/2605.07315},347 note = {arXiv v1 2026-05-08. Mechanism: bounded latent exploration then explicit CoT; halting supervision from a constructed corpus. Code: TioeAre/LaTER (Apache-2.0).}348}349350@article{sharma2026readout,351 title = {Dense Supervision Is Not Enough: The Readout Blind Spot in Looped Language Models},352 author = {Sharma, Rituraj and Vu, Tu},353 journal = {arXiv preprint arXiv:2606.24898},354 year = {2026},355 url = {https://arxiv.org/abs/2606.24898},356 note = {arXiv v1 2026-06-12. Finding: per-loop cross-entropy through scale-invariant readouts does not control recurrent hidden-state scale; relevant to readout design in looped/edited states.}357}358359@article{zeng2026depthattention,360 title = {Depth-Attention: Cross-Layer Value Mixing for Language Models},361 author = {Zeng, Boyi and Hao, Yiqin and Wang, Zitong and Song, Shixiang and Li, He and Song, Feichen and Liu, Yifan and He, Ziwei and Wang, Xinbing and Lin, Zhouhan},362 journal = {arXiv preprint arXiv:2606.05014},363 year = {2026},364 url = {https://arxiv.org/abs/2606.05014},365 note = {arXiv v1 2026-06-03, v2 2026-06-13. Mechanism: query attends over keys of earlier layers at the same position inside attention; no extra persistent state.}366}367368@article{fu2026discoloop,369 title = {DiscoLoop: Looping Discrete Embeddings and Continuous Hidden States for Multi-hop Reasoning},370 author = {Fu, Hengyu and Guo, Tianyu and Wang, Zixuan and Zhu, Hanlin and Lee, Jason D. and Jiao, Jiantao and Russell, Stuart and Mei, Song},371 journal = {arXiv preprint arXiv:2607.00341},372 year = {2026},373 url = {https://arxiv.org/abs/2607.00341},374 note = {arXiv v1 2026-07-01, v2 2026-07-27.}375}376377@article{popescu2026adaptivedepth,378 title = {Adaptive Depth in Looped Transformers: Diagnosing Learned Halting Gates and Trajectory Readouts},379 author = {Popescu, Andrei Cristian and S{\'a}ez de Oc{\'a}riz Borde, Haitz and Li{\`o}, Pietro},380 journal = {arXiv preprint arXiv:2607.20519},381 year = {2026},382 url = {https://arxiv.org/abs/2607.20519},383 note = {arXiv v1 2026-07-08. Finding: jointly trained halting gates entangle exit selection and trajectory formation; post-hoc confidence readouts on fixed-prior-supervised trajectories often match learned gates (incl. on Ouro checkpoints).}384}385386@article{logan2026fixedpoint,387 title = {Per-Token Fixed-Point Convergence in Depth-Recurrent Transformers},388 author = {Logan, Joe},389 journal = {arXiv preprint arXiv:2607.14427},390 year = {2026},391 url = {https://arxiv.org/abs/2607.14427},392 note = {arXiv v1 2026-07-15. Finding: training-free convergence-based exit allocates depth better than a trained router. Code: jlognn/depth-recurrent-convergence (MIT).}393}394395@article{dau2026beyonddepth,396 title = {Beyond Depth Truncation: Controlled Evaluation of Depth Utilization in Recursive Language Models},397 author = {Dau, Ha Van and Khuat, Thanh Tung and Dung, Nguyen Thanh},398 journal = {arXiv preprint arXiv:2609.19934},399 year = {2026},400 url = {https://arxiv.org/abs/2609.19934},401 note = {arXiv v1 2026-09-17.}402}403404@article{huang2026trading,405 title = {Trading Depth for Time in Recurrent Transformers},406 author = {Huang, Zeyi and He, Xuehai and Shen, Yelong and Lee, Yong Jae},407 journal = {arXiv preprint arXiv:2609.21605},408 year = {2026},409 url = {https://arxiv.org/abs/2609.21605},410 note = {arXiv v1 2026-09-18, v2 2026-10-13.}411}412413@article{ye2026reflux,414 title = {Write Back the $\Delta$: Revisiting the Same Tokens with Fresh Representations},415 author = {Ye, Wencheng and Hu, Anning and Zhang, Xiangdong and Wang, Tianyi and Li, Yikang and Jin, Hengyu and Li, Bing and Yan, Junchi},416 journal = {arXiv preprint arXiv:2609.32457},417 year = {2026},418 url = {https://arxiv.org/abs/2609.32457},419 note = {ReFlux. arXiv v1 2026-09-26. Mechanism: learnable feedback graph over a frozen backbone; router (per block, shared across tokens) selects a sparse set of deep-to-shallow edges carrying the depth increment Delta = z\_s - z\_d, written additively; trained with next-token loss + L1/L2 route regularisation (Gumbel-sigmoid); no counterfactual labels. With no active edges the backbone is unchanged. Code: gooogleshanghai/reflux (MIT).}420}421422@article{olszowiec2026monitoring,423 title = {Monitoring latent reasoning in looped language models},424 author = {Olszowiec, Pawel and Tomaszewska, Paulina and {\.Z}elaszczyk, Maciej and Byra, Michal and Chung, Neo Christopher},425 journal = {arXiv preprint arXiv:2610.18127},426 year = {2026},427 url = {https://arxiv.org/abs/2610.18127},428 note = {arXiv v1 2026-10-13.}429}430431% ---------------------------------------------------------------------------432% 3. Slots, workspaces, memory-augmented networks433% ---------------------------------------------------------------------------434435@inproceedings{locatello2020slot,436 title = {Object-Centric Learning with Slot Attention},437 author = {Locatello, Francesco and Weissenborn, Dirk and Unterthiner, Thomas and Mahendran, Aravindh and Heigold, Georg and Uszkoreit, Jakob and Dosovitskiy, Alexey and Kipf, Thomas},438 booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},439 year = {2020},440 eprint = {2006.15055},441 archivePrefix = {arXiv},442 url = {https://arxiv.org/abs/2006.15055},443 note = {arXiv v1 2020-06-26; NeurIPS 2020 per arXiv comment. Mechanism: K slots sampled per input, iteratively refined by competitive (softmax over slots) cross-attention + GRU; all slots updated every iteration. Code: google-research/google-research/slot\_attention (Apache-2.0).}444}445446@inproceedings{goyal2022workspace,447 title = {Coordination Among Neural Modules Through a Shared Global Workspace},448 author = {Goyal, Anirudh and Didolkar, Aniket and Lamb, Alex and Badola, Kartikeya and Ke, Nan Rosemary and Rahaman, Nasim and Binas, Jonathan and Blundell, Charles and Mozer, Michael and Bengio, Yoshua},449 booktitle = {International Conference on Learning Representations (ICLR)},450 year = {2022},451 eprint = {2103.01197},452 archivePrefix = {arXiv},453 url = {https://arxiv.org/abs/2103.01197},454 note = {arXiv v1 2021-03-01, v2 2022-03-22; ICLR 2022 per arXiv comment. Mechanism: bandwidth-limited shared workspace (few slots); specialist modules compete (top-k) to write; broadcast back. Workspace persists across time steps.}455}456457@inproceedings{jaegle2021perceiver,458 title = {Perceiver: General Perception with Iterative Attention},459 author = {Jaegle, Andrew and Gimeno, Felix and Brock, Andrew and Zisserman, Andrew and Vinyals, Oriol and Carreira, Joao},460 booktitle = {International Conference on Machine Learning (ICML)},461 year = {2021},462 eprint = {2103.03206},463 archivePrefix = {arXiv},464 url = {https://arxiv.org/abs/2103.03206},465 note = {arXiv v1 2021-03-04; ICML 2021 per arXiv comment. Mechanism: learned latent array cross-attends to inputs, then latent self-attention; latents re-initialised per forward pass; weight sharing across iterations.}466}467468@article{graves2014ntm,469 title = {Neural Turing Machines},470 author = {Graves, Alex and Wayne, Greg and Danihelka, Ivo},471 journal = {arXiv preprint arXiv:1410.5401},472 year = {2014},473 url = {https://arxiv.org/abs/1410.5401},474 note = {arXiv v1 2014-10-20, v2 2014-12-10. Mechanism: differentiable content/location addressing, soft (dense) erase/add writes to persistent memory.}475}476477@article{weston2014memory,478 title = {Memory Networks},479 author = {Weston, Jason and Chopra, Sumit and Bordes, Antoine},480 journal = {arXiv preprint arXiv:1410.3916},481 year = {2014},482 url = {https://arxiv.org/abs/1410.3916},483 note = {arXiv v1 2014-10-15, v11 2015-11-29. Venue (ICLR 2015) NOT verified from arXiv metadata.}484}485486@inproceedings{rae2016sparse,487 title = {Scaling Memory-Augmented Neural Networks with Sparse Reads and Writes},488 author = {Rae, Jack W. and Hunt, Jonathan J. and Harley, Tim and Danihelka, Ivo and Senior, Andrew and Wayne, Greg and Graves, Alex and Lillicrap, Timothy P.},489 booktitle = {Advances in Neural Information Processing Systems (NIPS)},490 year = {2016},491 eprint = {1610.09027},492 archivePrefix = {arXiv},493 url = {https://arxiv.org/abs/1610.09027},494 note = {Sparse Access Memory (SAM). arXiv v1 2016-10-27; NIPS 2016 per arXiv comment. Mechanism: sparse (top-k) differentiable reads and writes to a large persistent memory for efficiency.}495}496497@article{wu2020memformer,498 title = {Memformer: A Memory-Augmented Transformer for Sequence Modeling},499 author = {Wu, Qingyang and Lan, Zhenzhong and Qian, Kun and Gu, Jing and Geramifard, Alborz and Yu, Zhou},500 journal = {arXiv preprint arXiv:2010.06891},501 year = {2020},502 url = {https://arxiv.org/abs/2010.06891},503 note = {arXiv v1 2020-10-14, v2 2022-04-12. Mechanism: fixed-size external memory slots read by cross-attention and written each segment (memory writer with slot-wise attention); persistent across segments. Official code not verified.}504}505506@inproceedings{bulatov2022rmt,507 title = {Recurrent Memory Transformer},508 author = {Bulatov, Aydar and Kuratov, Yuri and Burtsev, Mikhail S.},509 booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},510 year = {2022},511 eprint = {2207.06881},512 archivePrefix = {arXiv},513 url = {https://arxiv.org/abs/2207.06881},514 note = {arXiv v1 2022-07-14; NeurIPS 2022 per arXiv comment. Mechanism: memory tokens prepended/appended to each segment, carried across segments. Code: booydar/recurrent-memory-transformer (no licence file).}515}516517@inproceedings{ryoo2023ttm,518 title = {Token Turing Machines},519 author = {Ryoo, Michael S. and Gopalakrishnan, Keerthana and Kahatapitiya, Kumara and Xiao, Ted and Rao, Kanishka and Stone, Austin and Lu, Yao and Ibarz, Julian and Arnab, Anurag},520 booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},521 year = {2023},522 eprint = {2211.09119},523 archivePrefix = {arXiv},524 url = {https://arxiv.org/abs/2211.09119},525 note = {arXiv v1 2022-11-16; CVPR 2023 per arXiv journal-ref. Mechanism: token-set memory read/written by a transformer controller via token summarisation; persistent across steps. Code: google-research/scenic (Apache-2.0).}526}527528@article{berges2024memorylayers,529 title = {Memory Layers at Scale},530 author = {Berges, Vincent-Pierre and O{\u{g}}uz, Barlas and Haziza, Daniel and Yih, Wen-tau and Zettlemoyer, Luke and Ghosh, Gargi},531 journal = {arXiv preprint arXiv:2412.09764},532 year = {2024},533 url = {https://arxiv.org/abs/2412.09764},534 note = {arXiv v1 2024-12-12, v2 2024-12-20. Mechanism: sparsely activated trainable key-value memory layers (product keys) replacing FFNs.}535}536537@article{darcet2023registers,538 title = {Vision Transformers Need Registers},539 author = {Darcet, Timoth{\'e}e and Oquab, Maxime and Mairal, Julien and Bojanowski, Piotr},540 journal = {arXiv preprint arXiv:2309.16588},541 year = {2023},542 url = {https://arxiv.org/abs/2309.16588},543 note = {arXiv v1 2023-09-28, v2 2024-04-12. Venue (ICLR 2024) NOT verified from arXiv metadata. Mechanism: extra learnable register tokens providing scratch capacity, discarded at output.}544}545546% ---------------------------------------------------------------------------547% 4. Long-context / recurrence / linear-time baselines548% ---------------------------------------------------------------------------549550@inproceedings{dai2019transformerxl,551 title = {Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context},552 author = {Dai, Zihang and Yang, Zhilin and Yang, Yiming and Carbonell, Jaime and Le, Quoc V. and Salakhutdinov, Ruslan},553 booktitle = {Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics (ACL)},554 year = {2019},555 eprint = {1901.02860},556 archivePrefix = {arXiv},557 url = {https://arxiv.org/abs/1901.02860},558 note = {arXiv v1 2019-01-09; ACL 2019 per arXiv comment. Code: kimiyoung/transformer-xl (Apache-2.0).}559}560561@article{rae2019compressive,562 title = {Compressive Transformers for Long-Range Sequence Modelling},563 author = {Rae, Jack W. and Potapenko, Anna and Jayakumar, Siddhant M. and Lillicrap, Timothy P.},564 journal = {arXiv preprint arXiv:1911.05507},565 year = {2019},566 url = {https://arxiv.org/abs/1911.05507},567 note = {arXiv v1 2019-11-13. Venue (ICLR 2020) NOT verified from arXiv metadata. Mechanism: compress old memories into a secondary compressed memory.}568}569570@article{gu2023mamba,571 title = {Mamba: Linear-Time Sequence Modeling with Selective State Spaces},572 author = {Gu, Albert and Dao, Tri},573 journal = {arXiv preprint arXiv:2312.00752},574 year = {2023},575 url = {https://arxiv.org/abs/2312.00752},576 note = {arXiv v1 2023-12-01, v2 2024-05-31. Code: state-spaces/mamba (Apache-2.0). Included only as the canonical fixed-state recurrent baseline.}577}578579% ---------------------------------------------------------------------------580% 5. Sparse routing / mixture-of-experts581% ---------------------------------------------------------------------------582583@article{shazeer2017moe,584 title = {Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer},585 author = {Shazeer, Noam and Mirhoseini, Azalia and Maziarz, Krzysztof and Davis, Andy and Le, Quoc and Hinton, Geoffrey and Dean, Jeff},586 journal = {arXiv preprint arXiv:1701.06538},587 year = {2017},588 url = {https://arxiv.org/abs/1701.06538},589 note = {arXiv v1 2017-01-23. Venue (ICLR 2017) NOT verified from arXiv metadata. Mechanism: noisy top-k gating over experts; load-balancing losses.}590}591592@article{fedus2022switch,593 title = {Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity},594 author = {Fedus, William and Zoph, Barret and Shazeer, Noam},595 journal = {Journal of Machine Learning Research},596 year = {2022},597 eprint = {2101.03961},598 archivePrefix = {arXiv},599 url = {https://arxiv.org/abs/2101.03961},600 note = {arXiv v1 2021-01-11, v3 2022-06-16; JMLR per arXiv comment. Mechanism: top-1 expert routing per token.}601}602603% ---------------------------------------------------------------------------604% 6. Value learning, off-policy / counterfactual evaluation605% ---------------------------------------------------------------------------606607@article{ernst2005tree,608 title = {Tree-Based Batch Mode Reinforcement Learning},609 author = {Ernst, Damien and Geurts, Pierre and Wehenkel, Louis},610 journal = {Journal of Machine Learning Research},611 volume = {6},612 pages = {503--556},613 year = {2005},614 url = {https://www.jmlr.org/papers/v6/ernst05a.html},615 note = {Fitted Q iteration from logged transitions (verified on JMLR page).}616}617618@inproceedings{mnih2013atari,619 title = {Playing Atari with Deep Reinforcement Learning},620 author = {Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Graves, Alex and Antonoglou, Ioannis and Wierstra, Daan and Riedmiller, Martin},621 booktitle = {NIPS Deep Learning Workshop},622 year = {2013},623 eprint = {1312.5602},624 archivePrefix = {arXiv},625 url = {https://arxiv.org/abs/1312.5602},626 note = {arXiv v1 2013-12-19; workshop per arXiv comment. Deep Q-network for discrete action selection with greedy argmax over predicted action values.}627}628629@inproceedings{dudik2011doublyrobust,630 title = {Doubly Robust Policy Evaluation and Learning},631 author = {Dud{\'i}k, Miroslav and Langford, John and Li, Lihong},632 booktitle = {International Conference on Machine Learning (ICML)},633 year = {2011},634 eprint = {1103.4601},635 archivePrefix = {arXiv},636 url = {https://arxiv.org/abs/1103.4601},637 note = {arXiv v1 2011-03-23; ICML 2011 per arXiv comment. Off-policy evaluation of contextual-bandit policies from logged data with propensities (DR estimator).}638}639640@article{bottou2013counterfactual,641 title = {Counterfactual Reasoning and Learning Systems},642 author = {Bottou, L{\'e}on and Peters, Jonas and Qui{\~n}onero-Candela, Joaquin and Charles, Denis X. and Chickering, D. Max and Portugaly, Elon and Ray, Dipankar and Simard, Patrice and Snelson, Ed},643 journal = {arXiv preprint arXiv:1209.2355},644 year = {2012},645 url = {https://arxiv.org/abs/1209.2355},646 note = {arXiv v1 2012-09-11, v5 2013-07-27. Venue (JMLR 2013) NOT verified from arXiv metadata. Counterfactual estimation from logged randomised decisions.}647}648649@article{swaminathan2015crm,650 title = {Counterfactual Risk Minimization: Learning from Logged Bandit Feedback},651 author = {Swaminathan, Adith and Joachims, Thorsten},652 journal = {arXiv preprint arXiv:1502.02362},653 year = {2015},654 url = {https://arxiv.org/abs/1502.02362},655 note = {arXiv v1 2015-02-09, v2 2015-05-20. Venue (ICML 2015) NOT verified from arXiv metadata. Learning a policy from logged data with recorded propensities via variance-regularised IPS.}656}657658% ---------------------------------------------------------------------------659% 7. Early exit, speculative decoding, test-time compute660% ---------------------------------------------------------------------------661662@inproceedings{schuster2022calm,663 title = {Confident Adaptive Language Modeling},664 author = {Schuster, Tal and Fisch, Adam and Gupta, Jai and Dehghani, Mostafa and Bahri, Dara and Tran, Vinh Q. and Tay, Yi and Metzler, Donald},665 booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},666 year = {2022},667 eprint = {2207.07061},668 archivePrefix = {arXiv},669 url = {https://arxiv.org/abs/2207.07061},670 note = {CALM. arXiv v1 2022-07-14; NeurIPS 2022 (oral) per arXiv comment. Mechanism: per-token early exit with calibrated confidence thresholds (softmax diff / hidden-state saturation / learned classifier).}671}672673@inproceedings{leviathan2023speculative,674 title = {Fast Inference from Transformers via Speculative Decoding},675 author = {Leviathan, Yaniv and Kalman, Matan and Matias, Yossi},676 booktitle = {International Conference on Machine Learning (ICML)},677 year = {2023},678 eprint = {2211.17192},679 archivePrefix = {arXiv},680 url = {https://arxiv.org/abs/2211.17192},681 note = {arXiv v1 2022-11-30; ICML 2023 oral per arXiv comment.}682}683684@article{snell2024testtime,685 title = {Scaling LLM Test-Time Compute Optimally can be More Effective than Scaling Model Parameters},686 author = {Snell, Charlie and Lee, Jaehoon and Xu, Kelvin and Kumar, Aviral},687 journal = {arXiv preprint arXiv:2408.03314},688 year = {2024},689 url = {https://arxiv.org/abs/2408.03314},690 note = {arXiv v1 2024-08-06. Compute-optimal allocation of test-time compute (verifier search / revisions) as a function of prompt difficulty.}691}692693@article{ye2024physics,694 title = {Physics of Language Models: Part 2.1, Grade-School Math and the Hidden Reasoning Process},695 author = {Ye, Tian and Xu, Zicheng and Li, Yuanzhi and Allen-Zhu, Zeyuan},696 journal = {arXiv preprint arXiv:2407.20311},697 year = {2024},698 url = {https://arxiv.org/abs/2407.20311},699 note = {arXiv v1 2024-07-29. Controlled synthetic study of reasoning; depth matters for multi-step reasoning more than width (cited as the "depth vs. reasoning" evidence).}700}701702% ---------------------------------------------------------------------------703% 8. Zero-initialised gates / exact-identity insertion (for C3)704% ---------------------------------------------------------------------------705706@article{bachlechner2020rezero,707 title = {ReZero is All You Need: Fast Convergence at Large Depth},708 author = {Bachlechner, Thomas and Majumder, Bodhisattwa Prasad and Mao, Huanru Henry and Cottrell, Garrison W. and McAuley, Julian},709 journal = {arXiv preprint arXiv:2003.04887},710 year = {2020},711 url = {https://arxiv.org/abs/2003.04887},712 note = {arXiv v1 2020-03-10, v2 2020-06-25. Mechanism: residual branch scaled by a scalar initialised at zero, so each block starts as the identity.}713}714715@article{zhang2023controlnet,716 title = {Adding Conditional Control to Text-to-Image Diffusion Models},717 author = {Zhang, Lvmin and Rao, Anyi and Agrawala, Maneesh},718 journal = {arXiv preprint arXiv:2302.05543},719 year = {2023},720 url = {https://arxiv.org/abs/2302.05543},721 note = {ControlNet. arXiv v1 2023-02-10, v3 2023-11-26. Venue (ICCV 2023) NOT verified from arXiv metadata. Mechanism: "zero convolutions" so the added branch is exactly inactive at initialisation and the frozen backbone is reproduced. Code: lllyasviel/ControlNet.}722}723724@inproceedings{alayrac2022flamingo,725 title = {Flamingo: a Visual Language Model for Few-Shot Learning},726 author = {Alayrac, Jean-Baptiste and Donahue, Jeff and Luc, Pauline and Miech, Antoine and Barr, Iain and Hasson, Yana and Lenc, Karel and Mensch, Arthur and Millican, Katie and Reynolds, Malcolm and Ring, Roman and Rutherford, Eliza and Cabi, Serkan and Han, Tengda and Gong, Zhitao and Samangooei, Sina and Monteiro, Marianne and Menick, Jacob and Borgeaud, Sebastian and Brock, Andrew and Nematzadeh, Aida and Sharifzadeh, Sahand and Binkowski, Mikolaj and Barreira, Ricardo and Vinyals, Oriol and Zisserman, Andrew and Simonyan, Karen},727 booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},728 year = {2022},729 eprint = {2204.14198},730 archivePrefix = {arXiv},731 url = {https://arxiv.org/abs/2204.14198},732 note = {arXiv v1 2022-04-29; NeurIPS 2022 per arXiv comment. Mechanism: gated cross-attention layers with tanh gates initialised at zero so the frozen LM is reproduced exactly at initialisation.}733}734735% ---------------------------------------------------------------------------736% 9. Interpretability-style sparse interventions (for C1/C2 context)737% ---------------------------------------------------------------------------738739@article{ernst2026dsi,740 title = {Distributed Sparse Interventions in Language Models},741 author = {Ernst, Maximilian S. and Linhardt, Lorenz and Peikert, Aaron and Eberle, Oliver},742 journal = {arXiv preprint arXiv:2607.07128},743 year = {2026},744 url = {https://arxiv.org/abs/2607.07128},745 note = {arXiv v1 2026-07-08, v2 2026-10-22. Mechanism: steering via a sparse set (<0.1\%) of neurons chosen iteratively; interventions are static per task, not per-token learned actions.}746}747748% ---------------------------------------------------------------------------749% 10. Backbone building blocks and scaling laws750% ---------------------------------------------------------------------------751752@article{shazeer2020glu,753 title = {{GLU} Variants Improve Transformer},754 author = {Shazeer, Noam},755 journal = {arXiv preprint arXiv:2002.05202},756 year = {2020},757 url = {https://arxiv.org/abs/2002.05202},758 note = {arXiv v1 2020-02-12. SwiGLU feed-forward.}759}760761@article{su2021roformer,762 title = {RoFormer: Enhanced Transformer with Rotary Position Embedding},763 author = {Su, Jianlin and Lu, Yu and Pan, Shengfeng and Murtadha, Ahmed and Wen, Bo and Liu, Yunfeng},764 journal = {arXiv preprint arXiv:2104.09864},765 year = {2021},766 url = {https://arxiv.org/abs/2104.09864},767 note = {arXiv v1 2021-04-20, v5 2023-11-08. Journal version (Neurocomputing 2024) NOT verified here. RoPE.}768}769770@inproceedings{ainslie2023gqa,771 title = {{GQA}: Training Generalized Multi-Query Transformer Models from Multi-Head Checkpoints},772 author = {Ainslie, Joshua and Lee-Thorp, James and de Jong, Michiel and Zemlyanskiy, Yury and Lebr{\'o}n, Federico and Sanghai, Sumit},773 booktitle = {Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (EMNLP)},774 year = {2023},775 eprint = {2305.13245},776 archivePrefix = {arXiv},777 url = {https://arxiv.org/abs/2305.13245},778 note = {arXiv v1 2023-05-22, v3 2023-12-23; EMNLP 2023 per arXiv comment.}779}780781@inproceedings{zhang2019rmsnorm,782 title = {Root Mean Square Layer Normalization},783 author = {Zhang, Biao and Sennrich, Rico},784 booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},785 year = {2019},786 eprint = {1910.07467},787 archivePrefix = {arXiv},788 url = {https://arxiv.org/abs/1910.07467},789 note = {arXiv v1 2019-10-16; NeurIPS 2019 per arXiv comment.}790}791792@article{kaplan2020scaling,793 title = {Scaling Laws for Neural Language Models},794 author = {Kaplan, Jared and McCandlish, Sam and Henighan, Tom and Brown, Tom B. and Chess, Benjamin and Child, Rewon and Gray, Scott and Radford, Alec and Wu, Jeffrey and Amodei, Dario},795 journal = {arXiv preprint arXiv:2001.08361},796 year = {2020},797 url = {https://arxiv.org/abs/2001.08361},798 note = {arXiv v1 2020-01-23.}799}800801@article{hoffmann2022chinchilla,802 title = {Training Compute-Optimal Large Language Models},803 author = {Hoffmann, Jordan and Borgeaud, Sebastian and Mensch, Arthur and Buchatskaya, Elena and Cai, Trevor and Rutherford, Eliza and de Las Casas, Diego and Hendricks, Lisa Anne and Welbl, Johannes and Clark, Aidan and Hennigan, Tom and Noland, Eric and Millican, Katie and van den Driessche, George and Damoc, Bogdan and Guy, Aurelia and Osindero, Simon and Simonyan, Karen and Elsen, Erich and Rae, Jack W. and Vinyals, Oriol and Sifre, Laurent},804 journal = {arXiv preprint arXiv:2203.15556},805 year = {2022},806 url = {https://arxiv.org/abs/2203.15556},807 note = {Chinchilla. arXiv v1 2022-03-29. Venue (NeurIPS 2022) NOT verified from arXiv metadata.}808}809810@article{penedo2024fineweb,811 title = {The {FineWeb} Datasets: Decanting the Web for the Finest Text Data at Scale},812 author = {Penedo, Guilherme and Kydl{\'\i}{\v{c}}ek, Hynek and Ben allal, Loubna and Lozhkov, Anton and Mitchell, Margaret and Raffel, Colin and Von Werra, Leandro and Wolf, Thomas},813 year = {2024},814 journal = {arXiv preprint arXiv:2406.17557},815 url = {https://arxiv.org/abs/2406.17557},816 note = {Verified on the arXiv API 2026-10-30 (published 2024-06-25). FineWeb-Edu sample-10BT, ODC-By 1.0.}817}818