1% ============================================================================2% papers.bib -- UQO Working Paper 31 (PRISME), literature review3% Compiled 2026-10-29 by verifying every entry against primary sources:4% * arXiv Atom API (https://export.arxiv.org/api/query?id_list=...) for5% title, author list, first-submission date, latest version, comments/DOI;6% * GitHub REST API (api.github.com/repos/<owner>/<repo>) for code + licence;7% * JMLR / PMLR / project pages where cited.8% Convention: `year` = year of first arXiv submission unless a venue is given.9% Venues are stated ONLY when verified from arXiv metadata (comment/DOI field),10% a PMLR/JMLR page, or an official repository description. Where a venue is11% widely known but was NOT verified here, it is flagged in `note`.12% `note` also records code availability and licence (SPDX id from GitHub API).13% ============================================================================1415% ---------------------------------------------------------------------------16% 1. Core references requested (12)17% ---------------------------------------------------------------------------1819@inproceedings{dehghani2019universal,20 title = {Universal Transformers},21 author = {Dehghani, Mostafa and Gouws, Stephan and Vinyals, Oriol and Uszkoreit, Jakob and Kaiser, {\L}ukasz},22 booktitle = {International Conference on Learning Representations (ICLR)},23 year = {2019},24 eprint = {1807.03819},25 archivePrefix = {arXiv},26 url = {https://arxiv.org/abs/1807.03819},27}2829@inproceedings{banino2021pondernet,30 title = {PonderNet: Learning to Ponder},31 author = {Banino, Andrea and Balaguer, Jan and Blundell, Charles},32 booktitle = {8th ICML Workshop on Automated Machine Learning (AutoML)},33 year = {2021},34 eprint = {2107.05407},35 archivePrefix = {arXiv},36 url = {https://arxiv.org/abs/2107.05407},37}3839@inproceedings{hawthorne2022perceiverar,40 title = {General-purpose, long-context autoregressive modeling with {Perceiver AR}},41 author = {Hawthorne, Curtis and Jaegle, Andrew and Cangea, C{\u{a}}t{\u{a}}lina and Borgeaud, Sebastian and Nash, Charlie and Malinowski, Mateusz and Dieleman, Sander and Vinyals, Oriol and Botvinick, Matthew and Simon, Ian and Sheahan, Hannah and Zeghidour, Neil and Alayrac, Jean-Baptiste and Carreira, Jo{\~a}o and Engel, Jesse},42 booktitle = {International Conference on Machine Learning (ICML)},43 year = {2022},44 eprint = {2202.07765},45 archivePrefix = {arXiv},46 url = {https://arxiv.org/abs/2202.07765},47}4849@article{goyal2019rims,50 title = {Recurrent Independent Mechanisms},51 author = {Goyal, Anirudh and Lamb, Alex and Hoffmann, Jordan and Sodhani, Shagun and Levine, Sergey and Bengio, Yoshua and Sch{\"o}lkopf, Bernhard},52 journal = {arXiv preprint arXiv:1909.10893},53 year = {2019},54 url = {https://arxiv.org/abs/1909.10893},55}5657@article{raposo2024mixtureofdepths,58 title = {Mixture-of-Depths: Dynamically allocating compute in transformer-based language models},59 author = {Raposo, David and Ritter, Sam and Richards, Blake and Lillicrap, Timothy and Humphreys, Peter Conway and Santoro, Adam},60 journal = {arXiv preprint arXiv:2404.02258},61 year = {2024},62 url = {https://arxiv.org/abs/2404.02258},63}6465@article{geiping2025recurrentdepth,66 title = {Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach},67 author = {Geiping, Jonas and McLeish, Sean and Jain, Neel and Kirchenbauer, John and Singh, Siddharth and Bartoldson, Brian R. and Kailkhura, Bhavya and Bhatele, Abhinav and Goldstein, Tom},68 journal = {arXiv preprint arXiv:2502.05171},69 year = {2025},70 url = {https://arxiv.org/abs/2502.05171},71}7273@inproceedings{bae2025mixtureofrecursions,74 title = {Mixture-of-Recursions: Learning Dynamic Recursive Depths for Adaptive Token-Level Computation},75 author = {Bae, Sangmin and Kim, Yujin and Bayat, Reza and Kim, Sungnyun and Ha, Jiyoun and Schuster, Tal and Fisch, Adam and Harutyunyan, Hrayr and Ji, Ziwei and Courville, Aaron and Yun, Se-Young},76 booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},77 year = {2025},78 eprint = {2507.10524},79 archivePrefix = {arXiv},80 url = {https://arxiv.org/abs/2507.10524},81}8283@article{zhu2025ouro,84 title = {Scaling Latent Reasoning via Looped Language Models},85 author = {Zhu, Rui-Jie and Wang, Zixuan and Hua, Kai and Zhang, Tianyu and Li, Ziniu and Que, Haoran and Wei, Boyi and Wen, Zixin and Yin, Fan and Xing, He and Li, Lu and Shi, Jiajun and Ma, Kaijing and Li, Shanda and Kergan, Taylor and Smith, Andrew and Qu, Xingwei and Hui, Mude and Wu, Bohong and Min, Qiyang and Huang, Hongzhi and Zhou, Xun and Ye, Wei and Liu, Jiaheng and Yang, Jian and Shi, Yunfeng and Lin, Chenghua and Zhao, Enduo and Cai, Tianle and Zhang, Ge and Huang, Wenhao and Bengio, Yoshua and Eshraghian, Jason},86 journal = {arXiv preprint arXiv:2510.25741},87 year = {2025},88 url = {https://arxiv.org/abs/2510.25741},89}9091@article{wang2026recurtrace,92 title = {RecurTrace: Adaptive Latent Reasoning with Loop-Time Memory},93 author = {Wang, Yuxiang and Feng, Kunyu and Shen, Yingda and Xu, Haoning and Wang, Junyu and Wu, Zhizheng},94 journal = {arXiv preprint arXiv:2609.03379},95 year = {2026},96 url = {https://arxiv.org/abs/2609.03379},97}9899@article{ladhak2026elastic,100 title = {One Model to Loop Them All: Elastic Latent Recurrence for Language Models},101 author = {Ladhak, Faisal and Acharya, Shantanu and Akle Serano, Santiago and Sun, Simeng and Ginsburg, Boris and Puvvada, Krishna C.},102 journal = {arXiv preprint arXiv:2610.12582},103 year = {2026},104 url = {https://arxiv.org/abs/2610.12582},105}106107@article{behrouz2024titans,108 title = {Titans: Learning to Memorize at Test Time},109 author = {Behrouz, Ali and Zhong, Peilin and Mirrokni, Vahab},110 journal = {arXiv preprint arXiv:2501.00663},111 year = {2024},112 url = {https://arxiv.org/abs/2501.00663},113}114115@inproceedings{elhoushi2024layerskip,116 title = {LayerSkip: Enabling Early Exit Inference and Self-Speculative Decoding},117 author = {Elhoushi, Mostafa and Shrivastava, Akshat and Liskovich, Diana and Hosmer, Basil and Wasti, Bram and Lai, Liangzhen and Mahmoud, Anas and Acun, Bilge and Agarwal, Saurabh and Roman, Ahmed and Aly, Ahmed A. and Chen, Beidi and Wu, Carole-Jean},118 booktitle = {Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (ACL)},119 year = {2024},120 doi = {10.18653/v1/2024.acl-long.681},121 eprint = {2404.16710},122 archivePrefix = {arXiv},123 url = {https://arxiv.org/abs/2404.16710},124}125126% ---------------------------------------------------------------------------127% 2. Adaptive computation, looped / recurrent-depth, latent reasoning128% ---------------------------------------------------------------------------129130@article{graves2016act,131 title = {Adaptive Computation Time for Recurrent Neural Networks},132 author = {Graves, Alex},133 journal = {arXiv preprint arXiv:1603.08983},134 year = {2016},135 url = {https://arxiv.org/abs/1603.08983},136}137138@inproceedings{hao2024coconut,139 title = {Training Large Language Models to Reason in a Continuous Latent Space},140 author = {Hao, Shibo and Sukhbaatar, Sainbayar and Su, DiJia and Li, Xian and Hu, Zhiting and Weston, Jason and Tian, Yuandong},141 booktitle = {Conference on Language Modeling (COLM)},142 year = {2025},143 eprint = {2412.06769},144 archivePrefix = {arXiv},145 url = {https://arxiv.org/abs/2412.06769},146}147148@article{giannou2023looped,149 title = {Looped Transformers as Programmable Computers},150 author = {Giannou, Angeliki and Rajput, Shashank and Sohn, Jy-yong and Lee, Kangwook and Lee, Jason D. and Papailiopoulos, Dimitris},151 journal = {arXiv preprint arXiv:2301.13196},152 year = {2023},153 url = {https://arxiv.org/abs/2301.13196},154}155156@inproceedings{saunshi2025reasoning,157 title = {Reasoning with Latent Thoughts: On the Power of Looped Transformers},158 author = {Saunshi, Nikunj and Dikkala, Nishanth and Li, Zhiyuan and Kumar, Sanjiv and Reddi, Sashank J.},159 booktitle = {International Conference on Learning Representations (ICLR)},160 year = {2025},161 eprint = {2502.17416},162 archivePrefix = {arXiv},163 url = {https://arxiv.org/abs/2502.17416},164}165166@inproceedings{fan2025loopedlength,167 title = {Looped Transformers for Length Generalization},168 author = {Fan, Ying and Du, Yilun and Ramchandran, Kannan and Lee, Kangwook},169 booktitle = {International Conference on Learning Representations (ICLR)},170 year = {2025},171 eprint = {2409.15647},172 archivePrefix = {arXiv},173 url = {https://arxiv.org/abs/2409.15647},174}175176@inproceedings{bae2024relaxed,177 title = {Relaxed Recursive Transformers: Effective Parameter Sharing with Layer-wise LoRA},178 author = {Bae, Sangmin and Fisch, Adam and Harutyunyan, Hrayr and Ji, Ziwei and Kim, Seungyeon and Schuster, Tal},179 booktitle = {International Conference on Learning Representations (ICLR)},180 year = {2025},181 eprint = {2410.20672},182 archivePrefix = {arXiv},183 url = {https://arxiv.org/abs/2410.20672},184}185186@inproceedings{goyal2023pause,187 title = {Think before you speak: Training Language Models With Pause Tokens},188 author = {Goyal, Sachin and Ji, Ziwei and Rawat, Ankit Singh and Menon, Aditya Krishna and Kumar, Sanjiv and Nagarajan, Vaishnavh},189 booktitle = {International Conference on Learning Representations (ICLR)},190 year = {2024},191 eprint = {2310.02226},192 archivePrefix = {arXiv},193 url = {https://arxiv.org/abs/2310.02226},194}195196@article{sun2025latenttokens,197 title = {Enhancing Latent Computation in Transformers with Latent Tokens},198 author = {Sun, Yuchang and Chen, Yanxi and Li, Yaliang and Ding, Bolin},199 journal = {arXiv preprint arXiv:2505.12629},200 year = {2025},201 url = {https://arxiv.org/abs/2505.12629},202}203204@inproceedings{zeng2025ponderlm,205 title = {PonderLM: Pretraining Language Models to Ponder in Continuous Space},206 author = {Zeng, Boyi and Song, Shixiang and Huang, Siyuan and Wang, Yixuan and Li, He and He, Ziwei and Wang, Xinbing and Li, Zhiyu and Lin, Zhouhan},207 booktitle = {International Conference on Learning Representations (ICLR)},208 year = {2026},209 eprint = {2505.20674},210 archivePrefix = {arXiv},211 url = {https://arxiv.org/abs/2505.20674},212}213214@article{chen2025innerthinking,215 title = {Inner Thinking Transformer: Leveraging Dynamic Depth Scaling to Foster Adaptive Internal Thinking},216 author = {Chen, Yilong and Shang, Junyuan and Zhang, Zhenyu and Xie, Yanxi and Sheng, Jiawei and Liu, Tingwen and Wang, Shuohuan and Sun, Yu and Wu, Hua and Wang, Haifeng},217 journal = {arXiv preprint arXiv:2502.13842},218 year = {2025},219 url = {https://arxiv.org/abs/2502.13842},220}221222@article{koishekenov2025etd,223 title = {Encode, Think, Decode: Scaling test-time reasoning with recursive latent thoughts},224 author = {Koishekenov, Yeskendir and Lipani, Aldo and Cancedda, Nicola},225 journal = {arXiv preprint arXiv:2510.07358},226 year = {2025},227 url = {https://arxiv.org/abs/2510.07358},228}229230@article{wu2025parallelloop,231 title = {Parallel Loop Transformer for Efficient Test-Time Computation Scaling},232 author = {Wu, Bohong and Chen, Mengzhao and Luo, Xiang and Yan, Shen and Yu, Qifan and Xia, Fan and Zhang, Tianqi and Zhan, Hongrui and Zhong, Zheng and Zhou, Xun and Qiao, Siyuan and Bin, Xingyan},233 journal = {arXiv preprint arXiv:2510.24824},234 year = {2025},235 url = {https://arxiv.org/abs/2510.24824},236}237238@inproceedings{fu2026thinkathard,239 title = {Think-at-Hard: Dynamic Looped Transformers for Improved Reasoning},240 author = {Fu, Tianyu and You, Yichen and Chen, Zekai and Dai, Guohao and Yang, Huazhong and Wang, Yu},241 booktitle = {Proceedings of the 43rd International Conference on Machine Learning (ICML), PMLR 306},242 pages = {32129--32154},243 year = {2026},244 eprint = {2511.08577},245 archivePrefix = {arXiv},246 url = {https://proceedings.mlr.press/v306/fu26t.html},247}248249@article{you2026tah2,250 title = {Improving Test-Time Scaling with Adaptive Looped Transformers},251 author = {You, Yichen and Fu, Tianyu and Feng, Aosong and Lv, Xingtai and Ning, Xuefei and Ding, Ning and Wang, Yu},252 journal = {arXiv preprint arXiv:2609.35748},253 year = {2026},254 url = {https://arxiv.org/abs/2609.35748},255}256257@inproceedings{jeddi2026loopformer,258 title = {LoopFormer: Elastic-Depth Looped Transformers for Latent Reasoning via Shortcut Modulation},259 author = {Jeddi, Ahmadreza and Ciccone, Marco and Taati, Babak},260 booktitle = {International Conference on Learning Representations (ICLR)},261 year = {2026},262 eprint = {2602.11451},263 archivePrefix = {arXiv},264 url = {https://arxiv.org/abs/2602.11451},265}266267@article{song2026adaponderlm,268 title = {AdaPonderLM: Gated Pondering Language Models with Token-Wise Adaptive Depth},269 author = {Song, Shixiang and Li, He and Wang, Zitong and Zeng, Boyi and Song, Feichen and Wang, Yixuan and Xu, Zhiqin John and He, Ziwei and Lin, Zhouhan},270 journal = {arXiv preprint arXiv:2603.01914},271 year = {2026},272 url = {https://arxiv.org/abs/2603.01914},273}274275@article{noci2026lookahead,276 title = {Thinking into the Future: Latent Lookahead Training for Transformers},277 author = {Noci, Lorenzo and Bachmann, Gregor and Moosavi-Dezfooli, Seyed-Mohsen and Nabi, Moin},278 journal = {arXiv preprint arXiv:2603.20219},279 year = {2026},280 url = {https://arxiv.org/abs/2603.20219},281}282283@inproceedings{frey2026adaptiveloops,284 title = {Adaptive Loops and Memory in Transformers: Think Harder or Know More?},285 author = {Frey, Markus and Shomali, Behzad and Bashir, Ali Hamza and Berghaus, David and Koehler, Joachim and Ali, Mehdi},286 booktitle = {Latent \& Implicit Thinking Workshop @ ICLR 2026},287 year = {2026},288 eprint = {2603.08391},289 archivePrefix = {arXiv},290 url = {https://arxiv.org/abs/2603.08391},291}292293@article{park2026loopus,294 title = {LoopUS: Recasting Pretrained LLMs into Looped Latent Refinement Models},295 author = {Park, Taekhyun and Lee, Yongjae and Kim, Dohee and Bae, Hyerim},296 journal = {arXiv preprint arXiv:2605.11011},297 year = {2026},298 url = {https://arxiv.org/abs/2605.11011},299}300301@article{huang2026lrt,302 title = {Latent Recurrent Transformer: Architecture Exploration, Training Strategies, and Scaling Behavior},303 author = {Huang, Zeyi and He, Xuehai and Ren, LiLiang and Wang, Yiping and Peng, Baolin and Cheng, Hao and Wang, Shuohang and He, Pengcheng and Gao, Jianfeng and Lee, Yong Jae and Shen, Yelong},304 journal = {arXiv preprint arXiv:2605.26797},305 year = {2026},306 url = {https://arxiv.org/abs/2605.26797},307}308309@article{li2026later,310 title = {LaTER: Efficient Test-Time Reasoning via Latent Exploration and Explicit Verification},311 author = {Li, Xuan and Wang, Yining and Liu, Yuchen and Liu, Guanjun and Qiu, Delai and Liu, Shengping and Liang, Jiaen and Huang, Wei and Yu, Jun and Zhu, Junnan},312 journal = {arXiv preprint arXiv:2605.07315},313 year = {2026},314 url = {https://arxiv.org/abs/2605.07315},315}316317@article{sharma2026readout,318 title = {Dense Supervision Is Not Enough: The Readout Blind Spot in Looped Language Models},319 author = {Sharma, Rituraj and Vu, Tu},320 journal = {arXiv preprint arXiv:2606.24898},321 year = {2026},322 url = {https://arxiv.org/abs/2606.24898},323}324325@article{zeng2026depthattention,326 title = {Depth-Attention: Cross-Layer Value Mixing for Language Models},327 author = {Zeng, Boyi and Hao, Yiqin and Wang, Zitong and Song, Shixiang and Li, He and Song, Feichen and Liu, Yifan and He, Ziwei and Wang, Xinbing and Lin, Zhouhan},328 journal = {arXiv preprint arXiv:2606.05014},329 year = {2026},330 url = {https://arxiv.org/abs/2606.05014},331}332333@article{fu2026discoloop,334 title = {DiscoLoop: Looping Discrete Embeddings and Continuous Hidden States for Multi-hop Reasoning},335 author = {Fu, Hengyu and Guo, Tianyu and Wang, Zixuan and Zhu, Hanlin and Lee, Jason D. and Jiao, Jiantao and Russell, Stuart and Mei, Song},336 journal = {arXiv preprint arXiv:2607.00341},337 year = {2026},338 url = {https://arxiv.org/abs/2607.00341},339}340341@article{popescu2026adaptivedepth,342 title = {Adaptive Depth in Looped Transformers: Diagnosing Learned Halting Gates and Trajectory Readouts},343 author = {Popescu, Andrei Cristian and S{\'a}ez de Oc{\'a}riz Borde, Haitz and Li{\`o}, Pietro},344 journal = {arXiv preprint arXiv:2607.20519},345 year = {2026},346 url = {https://arxiv.org/abs/2607.20519},347}348349@article{logan2026fixedpoint,350 title = {Per-Token Fixed-Point Convergence in Depth-Recurrent Transformers},351 author = {Logan, Joe},352 journal = {arXiv preprint arXiv:2607.14427},353 year = {2026},354 url = {https://arxiv.org/abs/2607.14427},355}356357@article{dau2026beyonddepth,358 title = {Beyond Depth Truncation: Controlled Evaluation of Depth Utilization in Recursive Language Models},359 author = {Dau, Ha Van and Khuat, Thanh Tung and Dung, Nguyen Thanh},360 journal = {arXiv preprint arXiv:2609.19934},361 year = {2026},362 url = {https://arxiv.org/abs/2609.19934},363}364365@article{huang2026trading,366 title = {Trading Depth for Time in Recurrent Transformers},367 author = {Huang, Zeyi and He, Xuehai and Shen, Yelong and Lee, Yong Jae},368 journal = {arXiv preprint arXiv:2609.21605},369 year = {2026},370 url = {https://arxiv.org/abs/2609.21605},371}372373@article{ye2026reflux,374 title = {Write Back the $\Delta$: Revisiting the Same Tokens with Fresh Representations},375 author = {Ye, Wencheng and Hu, Anning and Zhang, Xiangdong and Wang, Tianyi and Li, Yikang and Jin, Hengyu and Li, Bing and Yan, Junchi},376 journal = {arXiv preprint arXiv:2609.32457},377 year = {2026},378 url = {https://arxiv.org/abs/2609.32457},379}380381@article{olszowiec2026monitoring,382 title = {Monitoring latent reasoning in looped language models},383 author = {Olszowiec, Pawel and Tomaszewska, Paulina and {\.Z}elaszczyk, Maciej and Byra, Michal and Chung, Neo Christopher},384 journal = {arXiv preprint arXiv:2610.18127},385 year = {2026},386 url = {https://arxiv.org/abs/2610.18127},387}388389% ---------------------------------------------------------------------------390% 3. Slots, workspaces, memory-augmented networks391% ---------------------------------------------------------------------------392393@inproceedings{locatello2020slot,394 title = {Object-Centric Learning with Slot Attention},395 author = {Locatello, Francesco and Weissenborn, Dirk and Unterthiner, Thomas and Mahendran, Aravindh and Heigold, Georg and Uszkoreit, Jakob and Dosovitskiy, Alexey and Kipf, Thomas},396 booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},397 year = {2020},398 eprint = {2006.15055},399 archivePrefix = {arXiv},400 url = {https://arxiv.org/abs/2006.15055},401}402403@inproceedings{goyal2022workspace,404 title = {Coordination Among Neural Modules Through a Shared Global Workspace},405 author = {Goyal, Anirudh and Didolkar, Aniket and Lamb, Alex and Badola, Kartikeya and Ke, Nan Rosemary and Rahaman, Nasim and Binas, Jonathan and Blundell, Charles and Mozer, Michael and Bengio, Yoshua},406 booktitle = {International Conference on Learning Representations (ICLR)},407 year = {2022},408 eprint = {2103.01197},409 archivePrefix = {arXiv},410 url = {https://arxiv.org/abs/2103.01197},411}412413@inproceedings{jaegle2021perceiver,414 title = {Perceiver: General Perception with Iterative Attention},415 author = {Jaegle, Andrew and Gimeno, Felix and Brock, Andrew and Zisserman, Andrew and Vinyals, Oriol and Carreira, Joao},416 booktitle = {International Conference on Machine Learning (ICML)},417 year = {2021},418 eprint = {2103.03206},419 archivePrefix = {arXiv},420 url = {https://arxiv.org/abs/2103.03206},421}422423@article{graves2014ntm,424 title = {Neural Turing Machines},425 author = {Graves, Alex and Wayne, Greg and Danihelka, Ivo},426 journal = {arXiv preprint arXiv:1410.5401},427 year = {2014},428 url = {https://arxiv.org/abs/1410.5401},429}430431@article{weston2014memory,432 title = {Memory Networks},433 author = {Weston, Jason and Chopra, Sumit and Bordes, Antoine},434 journal = {arXiv preprint arXiv:1410.3916},435 year = {2014},436 url = {https://arxiv.org/abs/1410.3916},437}438439@inproceedings{rae2016sparse,440 title = {Scaling Memory-Augmented Neural Networks with Sparse Reads and Writes},441 author = {Rae, Jack W. and Hunt, Jonathan J. and Harley, Tim and Danihelka, Ivo and Senior, Andrew and Wayne, Greg and Graves, Alex and Lillicrap, Timothy P.},442 booktitle = {Advances in Neural Information Processing Systems (NIPS)},443 year = {2016},444 eprint = {1610.09027},445 archivePrefix = {arXiv},446 url = {https://arxiv.org/abs/1610.09027},447}448449@article{wu2020memformer,450 title = {Memformer: A Memory-Augmented Transformer for Sequence Modeling},451 author = {Wu, Qingyang and Lan, Zhenzhong and Qian, Kun and Gu, Jing and Geramifard, Alborz and Yu, Zhou},452 journal = {arXiv preprint arXiv:2010.06891},453 year = {2020},454 url = {https://arxiv.org/abs/2010.06891},455}456457@inproceedings{bulatov2022rmt,458 title = {Recurrent Memory Transformer},459 author = {Bulatov, Aydar and Kuratov, Yuri and Burtsev, Mikhail S.},460 booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},461 year = {2022},462 eprint = {2207.06881},463 archivePrefix = {arXiv},464 url = {https://arxiv.org/abs/2207.06881},465}466467@inproceedings{ryoo2023ttm,468 title = {Token Turing Machines},469 author = {Ryoo, Michael S. and Gopalakrishnan, Keerthana and Kahatapitiya, Kumara and Xiao, Ted and Rao, Kanishka and Stone, Austin and Lu, Yao and Ibarz, Julian and Arnab, Anurag},470 booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},471 year = {2023},472 eprint = {2211.09119},473 archivePrefix = {arXiv},474 url = {https://arxiv.org/abs/2211.09119},475}476477@article{berges2024memorylayers,478 title = {Memory Layers at Scale},479 author = {Berges, Vincent-Pierre and O{\u{g}}uz, Barlas and Haziza, Daniel and Yih, Wen-tau and Zettlemoyer, Luke and Ghosh, Gargi},480 journal = {arXiv preprint arXiv:2412.09764},481 year = {2024},482 url = {https://arxiv.org/abs/2412.09764},483}484485@article{darcet2023registers,486 title = {Vision Transformers Need Registers},487 author = {Darcet, Timoth{\'e}e and Oquab, Maxime and Mairal, Julien and Bojanowski, Piotr},488 journal = {arXiv preprint arXiv:2309.16588},489 year = {2023},490 url = {https://arxiv.org/abs/2309.16588},491}492493% ---------------------------------------------------------------------------494% 4. Long-context / recurrence / linear-time baselines495% ---------------------------------------------------------------------------496497@inproceedings{dai2019transformerxl,498 title = {Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context},499 author = {Dai, Zihang and Yang, Zhilin and Yang, Yiming and Carbonell, Jaime and Le, Quoc V. and Salakhutdinov, Ruslan},500 booktitle = {Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics (ACL)},501 year = {2019},502 eprint = {1901.02860},503 archivePrefix = {arXiv},504 url = {https://arxiv.org/abs/1901.02860},505}506507@article{rae2019compressive,508 title = {Compressive Transformers for Long-Range Sequence Modelling},509 author = {Rae, Jack W. and Potapenko, Anna and Jayakumar, Siddhant M. and Lillicrap, Timothy P.},510 journal = {arXiv preprint arXiv:1911.05507},511 year = {2019},512 url = {https://arxiv.org/abs/1911.05507},513}514515@article{gu2023mamba,516 title = {Mamba: Linear-Time Sequence Modeling with Selective State Spaces},517 author = {Gu, Albert and Dao, Tri},518 journal = {arXiv preprint arXiv:2312.00752},519 year = {2023},520 url = {https://arxiv.org/abs/2312.00752},521}522523% ---------------------------------------------------------------------------524% 5. Sparse routing / mixture-of-experts525% ---------------------------------------------------------------------------526527@article{shazeer2017moe,528 title = {Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer},529 author = {Shazeer, Noam and Mirhoseini, Azalia and Maziarz, Krzysztof and Davis, Andy and Le, Quoc and Hinton, Geoffrey and Dean, Jeff},530 journal = {arXiv preprint arXiv:1701.06538},531 year = {2017},532 url = {https://arxiv.org/abs/1701.06538},533}534535@article{fedus2022switch,536 title = {Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity},537 author = {Fedus, William and Zoph, Barret and Shazeer, Noam},538 journal = {Journal of Machine Learning Research},539 year = {2022},540 eprint = {2101.03961},541 archivePrefix = {arXiv},542 url = {https://arxiv.org/abs/2101.03961},543}544545% ---------------------------------------------------------------------------546% 6. Value learning, off-policy / counterfactual evaluation547% ---------------------------------------------------------------------------548549@article{ernst2005tree,550 title = {Tree-Based Batch Mode Reinforcement Learning},551 author = {Ernst, Damien and Geurts, Pierre and Wehenkel, Louis},552 journal = {Journal of Machine Learning Research},553 volume = {6},554 pages = {503--556},555 year = {2005},556 url = {https://www.jmlr.org/papers/v6/ernst05a.html},557}558559@inproceedings{mnih2013atari,560 title = {Playing Atari with Deep Reinforcement Learning},561 author = {Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Graves, Alex and Antonoglou, Ioannis and Wierstra, Daan and Riedmiller, Martin},562 booktitle = {NIPS Deep Learning Workshop},563 year = {2013},564 eprint = {1312.5602},565 archivePrefix = {arXiv},566 url = {https://arxiv.org/abs/1312.5602},567}568569@inproceedings{dudik2011doublyrobust,570 title = {Doubly Robust Policy Evaluation and Learning},571 author = {Dud{\'i}k, Miroslav and Langford, John and Li, Lihong},572 booktitle = {International Conference on Machine Learning (ICML)},573 year = {2011},574 eprint = {1103.4601},575 archivePrefix = {arXiv},576 url = {https://arxiv.org/abs/1103.4601},577}578579@article{bottou2013counterfactual,580 title = {Counterfactual Reasoning and Learning Systems},581 author = {Bottou, L{\'e}on and Peters, Jonas and Qui{\~n}onero-Candela, Joaquin and Charles, Denis X. and Chickering, D. Max and Portugaly, Elon and Ray, Dipankar and Simard, Patrice and Snelson, Ed},582 journal = {arXiv preprint arXiv:1209.2355},583 year = {2012},584 url = {https://arxiv.org/abs/1209.2355},585}586587@article{swaminathan2015crm,588 title = {Counterfactual Risk Minimization: Learning from Logged Bandit Feedback},589 author = {Swaminathan, Adith and Joachims, Thorsten},590 journal = {arXiv preprint arXiv:1502.02362},591 year = {2015},592 url = {https://arxiv.org/abs/1502.02362},593}594595% ---------------------------------------------------------------------------596% 7. Early exit, speculative decoding, test-time compute597% ---------------------------------------------------------------------------598599@inproceedings{schuster2022calm,600 title = {Confident Adaptive Language Modeling},601 author = {Schuster, Tal and Fisch, Adam and Gupta, Jai and Dehghani, Mostafa and Bahri, Dara and Tran, Vinh Q. and Tay, Yi and Metzler, Donald},602 booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},603 year = {2022},604 eprint = {2207.07061},605 archivePrefix = {arXiv},606 url = {https://arxiv.org/abs/2207.07061},607}608609@inproceedings{leviathan2023speculative,610 title = {Fast Inference from Transformers via Speculative Decoding},611 author = {Leviathan, Yaniv and Kalman, Matan and Matias, Yossi},612 booktitle = {International Conference on Machine Learning (ICML)},613 year = {2023},614 eprint = {2211.17192},615 archivePrefix = {arXiv},616 url = {https://arxiv.org/abs/2211.17192},617}618619@article{snell2024testtime,620 title = {Scaling LLM Test-Time Compute Optimally can be More Effective than Scaling Model Parameters},621 author = {Snell, Charlie and Lee, Jaehoon and Xu, Kelvin and Kumar, Aviral},622 journal = {arXiv preprint arXiv:2408.03314},623 year = {2024},624 url = {https://arxiv.org/abs/2408.03314},625}626627@article{ye2024physics,628 title = {Physics of Language Models: Part 2.1, Grade-School Math and the Hidden Reasoning Process},629 author = {Ye, Tian and Xu, Zicheng and Li, Yuanzhi and Allen-Zhu, Zeyuan},630 journal = {arXiv preprint arXiv:2407.20311},631 year = {2024},632 url = {https://arxiv.org/abs/2407.20311},633}634635% ---------------------------------------------------------------------------636% 8. Zero-initialised gates / exact-identity insertion (for C3)637% ---------------------------------------------------------------------------638639@article{bachlechner2020rezero,640 title = {ReZero is All You Need: Fast Convergence at Large Depth},641 author = {Bachlechner, Thomas and Majumder, Bodhisattwa Prasad and Mao, Huanru Henry and Cottrell, Garrison W. and McAuley, Julian},642 journal = {arXiv preprint arXiv:2003.04887},643 year = {2020},644 url = {https://arxiv.org/abs/2003.04887},645}646647@article{zhang2023controlnet,648 title = {Adding Conditional Control to Text-to-Image Diffusion Models},649 author = {Zhang, Lvmin and Rao, Anyi and Agrawala, Maneesh},650 journal = {arXiv preprint arXiv:2302.05543},651 year = {2023},652 url = {https://arxiv.org/abs/2302.05543},653}654655@inproceedings{alayrac2022flamingo,656 title = {Flamingo: a Visual Language Model for Few-Shot Learning},657 author = {Alayrac, Jean-Baptiste and Donahue, Jeff and Luc, Pauline and Miech, Antoine and Barr, Iain and Hasson, Yana and Lenc, Karel and Mensch, Arthur and Millican, Katie and Reynolds, Malcolm and Ring, Roman and Rutherford, Eliza and Cabi, Serkan and Han, Tengda and Gong, Zhitao and Samangooei, Sina and Monteiro, Marianne and Menick, Jacob and Borgeaud, Sebastian and Brock, Andrew and Nematzadeh, Aida and Sharifzadeh, Sahand and Binkowski, Mikolaj and Barreira, Ricardo and Vinyals, Oriol and Zisserman, Andrew and Simonyan, Karen},658 booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},659 year = {2022},660 eprint = {2204.14198},661 archivePrefix = {arXiv},662 url = {https://arxiv.org/abs/2204.14198},663}664665% ---------------------------------------------------------------------------666% 9. Interpretability-style sparse interventions (for C1/C2 context)667% ---------------------------------------------------------------------------668669@article{ernst2026dsi,670 title = {Distributed Sparse Interventions in Language Models},671 author = {Ernst, Maximilian S. and Linhardt, Lorenz and Peikert, Aaron and Eberle, Oliver},672 journal = {arXiv preprint arXiv:2607.07128},673 year = {2026},674 url = {https://arxiv.org/abs/2607.07128},675}676677% ---------------------------------------------------------------------------678% 10. Backbone building blocks and scaling laws679% ---------------------------------------------------------------------------680681@article{shazeer2020glu,682 title = {{GLU} Variants Improve Transformer},683 author = {Shazeer, Noam},684 journal = {arXiv preprint arXiv:2002.05202},685 year = {2020},686 url = {https://arxiv.org/abs/2002.05202},687}688689@article{su2021roformer,690 title = {RoFormer: Enhanced Transformer with Rotary Position Embedding},691 author = {Su, Jianlin and Lu, Yu and Pan, Shengfeng and Murtadha, Ahmed and Wen, Bo and Liu, Yunfeng},692 journal = {arXiv preprint arXiv:2104.09864},693 year = {2021},694 url = {https://arxiv.org/abs/2104.09864},695}696697@inproceedings{ainslie2023gqa,698 title = {{GQA}: Training Generalized Multi-Query Transformer Models from Multi-Head Checkpoints},699 author = {Ainslie, Joshua and Lee-Thorp, James and de Jong, Michiel and Zemlyanskiy, Yury and Lebr{\'o}n, Federico and Sanghai, Sumit},700 booktitle = {Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (EMNLP)},701 year = {2023},702 eprint = {2305.13245},703 archivePrefix = {arXiv},704 url = {https://arxiv.org/abs/2305.13245},705}706707@inproceedings{zhang2019rmsnorm,708 title = {Root Mean Square Layer Normalization},709 author = {Zhang, Biao and Sennrich, Rico},710 booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},711 year = {2019},712 eprint = {1910.07467},713 archivePrefix = {arXiv},714 url = {https://arxiv.org/abs/1910.07467},715}716717@article{kaplan2020scaling,718 title = {Scaling Laws for Neural Language Models},719 author = {Kaplan, Jared and McCandlish, Sam and Henighan, Tom and Brown, Tom B. and Chess, Benjamin and Child, Rewon and Gray, Scott and Radford, Alec and Wu, Jeffrey and Amodei, Dario},720 journal = {arXiv preprint arXiv:2001.08361},721 year = {2020},722 url = {https://arxiv.org/abs/2001.08361},723}724725@article{hoffmann2022chinchilla,726 title = {Training Compute-Optimal Large Language Models},727 author = {Hoffmann, Jordan and Borgeaud, Sebastian and Mensch, Arthur and Buchatskaya, Elena and Cai, Trevor and Rutherford, Eliza and de Las Casas, Diego and Hendricks, Lisa Anne and Welbl, Johannes and Clark, Aidan and Hennigan, Tom and Noland, Eric and Millican, Katie and van den Driessche, George and Damoc, Bogdan and Guy, Aurelia and Osindero, Simon and Simonyan, Karen and Elsen, Erich and Rae, Jack W. and Vinyals, Oriol and Sifre, Laurent},728 journal = {arXiv preprint arXiv:2203.15556},729 year = {2022},730 url = {https://arxiv.org/abs/2203.15556},731}732733@article{penedo2024fineweb,734 title = {The {FineWeb} Datasets: Decanting the Web for the Finest Text Data at Scale},735 author = {Penedo, Guilherme and Kydl{\'\i}{\v{c}}ek, Hynek and Ben allal, Loubna and Lozhkov, Anton and Mitchell, Margaret and Raffel, Colin and Von Werra, Leandro and Wolf, Thomas},736 year = {2024},737 journal = {arXiv preprint arXiv:2406.17557},738 url = {https://arxiv.org/abs/2406.17557},739}740