% ============================================================================ % papers.bib -- UQO Working Paper 31 (PRISME), literature review % Compiled 2026-10-29 by verifying every entry against primary sources: % * arXiv Atom API (https://export.arxiv.org/api/query?id_list=...) for % title, author list, first-submission date, latest version, comments/DOI; % * GitHub REST API (api.github.com/repos//) for code + licence; % * JMLR / PMLR / project pages where cited. % Convention: `year` = year of first arXiv submission unless a venue is given. % Venues are stated ONLY when verified from arXiv metadata (comment/DOI field), % a PMLR/JMLR page, or an official repository description. Where a venue is % widely known but was NOT verified here, it is flagged in `note`. % `note` also records code availability and licence (SPDX id from GitHub API). % ============================================================================ % --------------------------------------------------------------------------- % 1. Core references requested (12) % --------------------------------------------------------------------------- @inproceedings{dehghani2019universal, title = {Universal Transformers}, author = {Dehghani, Mostafa and Gouws, Stephan and Vinyals, Oriol and Uszkoreit, Jakob and Kaiser, {\L}ukasz}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2019}, eprint = {1807.03819}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/1807.03819}, } @inproceedings{banino2021pondernet, title = {PonderNet: Learning to Ponder}, author = {Banino, Andrea and Balaguer, Jan and Blundell, Charles}, booktitle = {8th ICML Workshop on Automated Machine Learning (AutoML)}, year = {2021}, eprint = {2107.05407}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2107.05407}, } @inproceedings{hawthorne2022perceiverar, title = {General-purpose, long-context autoregressive modeling with {Perceiver AR}}, author = {Hawthorne, Curtis and Jaegle, Andrew and Cangea, C{\u{a}}t{\u{a}}lina and Borgeaud, Sebastian and Nash, Charlie and Malinowski, Mateusz and Dieleman, Sander and Vinyals, Oriol and Botvinick, Matthew and Simon, Ian and Sheahan, Hannah and Zeghidour, Neil and Alayrac, Jean-Baptiste and Carreira, Jo{\~a}o and Engel, Jesse}, booktitle = {International Conference on Machine Learning (ICML)}, year = {2022}, eprint = {2202.07765}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2202.07765}, } @article{goyal2019rims, title = {Recurrent Independent Mechanisms}, author = {Goyal, Anirudh and Lamb, Alex and Hoffmann, Jordan and Sodhani, Shagun and Levine, Sergey and Bengio, Yoshua and Sch{\"o}lkopf, Bernhard}, journal = {arXiv preprint arXiv:1909.10893}, year = {2019}, url = {https://arxiv.org/abs/1909.10893}, } @article{raposo2024mixtureofdepths, title = {Mixture-of-Depths: Dynamically allocating compute in transformer-based language models}, author = {Raposo, David and Ritter, Sam and Richards, Blake and Lillicrap, Timothy and Humphreys, Peter Conway and Santoro, Adam}, journal = {arXiv preprint arXiv:2404.02258}, year = {2024}, url = {https://arxiv.org/abs/2404.02258}, } @article{geiping2025recurrentdepth, title = {Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach}, author = {Geiping, Jonas and McLeish, Sean and Jain, Neel and Kirchenbauer, John and Singh, Siddharth and Bartoldson, Brian R. and Kailkhura, Bhavya and Bhatele, Abhinav and Goldstein, Tom}, journal = {arXiv preprint arXiv:2502.05171}, year = {2025}, url = {https://arxiv.org/abs/2502.05171}, } @inproceedings{bae2025mixtureofrecursions, title = {Mixture-of-Recursions: Learning Dynamic Recursive Depths for Adaptive Token-Level Computation}, author = {Bae, Sangmin and Kim, Yujin and Bayat, Reza and Kim, Sungnyun and Ha, Jiyoun and Schuster, Tal and Fisch, Adam and Harutyunyan, Hrayr and Ji, Ziwei and Courville, Aaron and Yun, Se-Young}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2025}, eprint = {2507.10524}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2507.10524}, } @article{zhu2025ouro, title = {Scaling Latent Reasoning via Looped Language Models}, author = {Zhu, Rui-Jie and Wang, Zixuan and Hua, Kai and Zhang, Tianyu and Li, Ziniu and Que, Haoran and Wei, Boyi and Wen, Zixin and Yin, Fan and Xing, He and Li, Lu and Shi, Jiajun and Ma, Kaijing and Li, Shanda and Kergan, Taylor and Smith, Andrew and Qu, Xingwei and Hui, Mude and Wu, Bohong and Min, Qiyang and Huang, Hongzhi and Zhou, Xun and Ye, Wei and Liu, Jiaheng and Yang, Jian and Shi, Yunfeng and Lin, Chenghua and Zhao, Enduo and Cai, Tianle and Zhang, Ge and Huang, Wenhao and Bengio, Yoshua and Eshraghian, Jason}, journal = {arXiv preprint arXiv:2510.25741}, year = {2025}, url = {https://arxiv.org/abs/2510.25741}, } @article{wang2026recurtrace, title = {RecurTrace: Adaptive Latent Reasoning with Loop-Time Memory}, author = {Wang, Yuxiang and Feng, Kunyu and Shen, Yingda and Xu, Haoning and Wang, Junyu and Wu, Zhizheng}, journal = {arXiv preprint arXiv:2609.03379}, year = {2026}, url = {https://arxiv.org/abs/2609.03379}, } @article{ladhak2026elastic, title = {One Model to Loop Them All: Elastic Latent Recurrence for Language Models}, author = {Ladhak, Faisal and Acharya, Shantanu and Akle Serano, Santiago and Sun, Simeng and Ginsburg, Boris and Puvvada, Krishna C.}, journal = {arXiv preprint arXiv:2610.12582}, year = {2026}, url = {https://arxiv.org/abs/2610.12582}, } @article{behrouz2024titans, title = {Titans: Learning to Memorize at Test Time}, author = {Behrouz, Ali and Zhong, Peilin and Mirrokni, Vahab}, journal = {arXiv preprint arXiv:2501.00663}, year = {2024}, url = {https://arxiv.org/abs/2501.00663}, } @inproceedings{elhoushi2024layerskip, title = {LayerSkip: Enabling Early Exit Inference and Self-Speculative Decoding}, author = {Elhoushi, Mostafa and Shrivastava, Akshat and Liskovich, Diana and Hosmer, Basil and Wasti, Bram and Lai, Liangzhen and Mahmoud, Anas and Acun, Bilge and Agarwal, Saurabh and Roman, Ahmed and Aly, Ahmed A. and Chen, Beidi and Wu, Carole-Jean}, booktitle = {Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (ACL)}, year = {2024}, doi = {10.18653/v1/2024.acl-long.681}, eprint = {2404.16710}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2404.16710}, } % --------------------------------------------------------------------------- % 2. Adaptive computation, looped / recurrent-depth, latent reasoning % --------------------------------------------------------------------------- @article{graves2016act, title = {Adaptive Computation Time for Recurrent Neural Networks}, author = {Graves, Alex}, journal = {arXiv preprint arXiv:1603.08983}, year = {2016}, url = {https://arxiv.org/abs/1603.08983}, } @inproceedings{hao2024coconut, title = {Training Large Language Models to Reason in a Continuous Latent Space}, author = {Hao, Shibo and Sukhbaatar, Sainbayar and Su, DiJia and Li, Xian and Hu, Zhiting and Weston, Jason and Tian, Yuandong}, booktitle = {Conference on Language Modeling (COLM)}, year = {2025}, eprint = {2412.06769}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2412.06769}, } @article{giannou2023looped, title = {Looped Transformers as Programmable Computers}, author = {Giannou, Angeliki and Rajput, Shashank and Sohn, Jy-yong and Lee, Kangwook and Lee, Jason D. and Papailiopoulos, Dimitris}, journal = {arXiv preprint arXiv:2301.13196}, year = {2023}, url = {https://arxiv.org/abs/2301.13196}, } @inproceedings{saunshi2025reasoning, title = {Reasoning with Latent Thoughts: On the Power of Looped Transformers}, author = {Saunshi, Nikunj and Dikkala, Nishanth and Li, Zhiyuan and Kumar, Sanjiv and Reddi, Sashank J.}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2025}, eprint = {2502.17416}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2502.17416}, } @inproceedings{fan2025loopedlength, title = {Looped Transformers for Length Generalization}, author = {Fan, Ying and Du, Yilun and Ramchandran, Kannan and Lee, Kangwook}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2025}, eprint = {2409.15647}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2409.15647}, } @inproceedings{bae2024relaxed, title = {Relaxed Recursive Transformers: Effective Parameter Sharing with Layer-wise LoRA}, author = {Bae, Sangmin and Fisch, Adam and Harutyunyan, Hrayr and Ji, Ziwei and Kim, Seungyeon and Schuster, Tal}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2025}, eprint = {2410.20672}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2410.20672}, } @inproceedings{goyal2023pause, title = {Think before you speak: Training Language Models With Pause Tokens}, author = {Goyal, Sachin and Ji, Ziwei and Rawat, Ankit Singh and Menon, Aditya Krishna and Kumar, Sanjiv and Nagarajan, Vaishnavh}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2024}, eprint = {2310.02226}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2310.02226}, } @article{sun2025latenttokens, title = {Enhancing Latent Computation in Transformers with Latent Tokens}, author = {Sun, Yuchang and Chen, Yanxi and Li, Yaliang and Ding, Bolin}, journal = {arXiv preprint arXiv:2505.12629}, year = {2025}, url = {https://arxiv.org/abs/2505.12629}, } @inproceedings{zeng2025ponderlm, title = {PonderLM: Pretraining Language Models to Ponder in Continuous Space}, author = {Zeng, Boyi and Song, Shixiang and Huang, Siyuan and Wang, Yixuan and Li, He and He, Ziwei and Wang, Xinbing and Li, Zhiyu and Lin, Zhouhan}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2026}, eprint = {2505.20674}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2505.20674}, } @article{chen2025innerthinking, title = {Inner Thinking Transformer: Leveraging Dynamic Depth Scaling to Foster Adaptive Internal Thinking}, author = {Chen, Yilong and Shang, Junyuan and Zhang, Zhenyu and Xie, Yanxi and Sheng, Jiawei and Liu, Tingwen and Wang, Shuohuan and Sun, Yu and Wu, Hua and Wang, Haifeng}, journal = {arXiv preprint arXiv:2502.13842}, year = {2025}, url = {https://arxiv.org/abs/2502.13842}, } @article{koishekenov2025etd, title = {Encode, Think, Decode: Scaling test-time reasoning with recursive latent thoughts}, author = {Koishekenov, Yeskendir and Lipani, Aldo and Cancedda, Nicola}, journal = {arXiv preprint arXiv:2510.07358}, year = {2025}, url = {https://arxiv.org/abs/2510.07358}, } @article{wu2025parallelloop, title = {Parallel Loop Transformer for Efficient Test-Time Computation Scaling}, author = {Wu, Bohong and Chen, Mengzhao and Luo, Xiang and Yan, Shen and Yu, Qifan and Xia, Fan and Zhang, Tianqi and Zhan, Hongrui and Zhong, Zheng and Zhou, Xun and Qiao, Siyuan and Bin, Xingyan}, journal = {arXiv preprint arXiv:2510.24824}, year = {2025}, url = {https://arxiv.org/abs/2510.24824}, } @inproceedings{fu2026thinkathard, title = {Think-at-Hard: Dynamic Looped Transformers for Improved Reasoning}, author = {Fu, Tianyu and You, Yichen and Chen, Zekai and Dai, Guohao and Yang, Huazhong and Wang, Yu}, booktitle = {Proceedings of the 43rd International Conference on Machine Learning (ICML), PMLR 306}, pages = {32129--32154}, year = {2026}, eprint = {2511.08577}, archivePrefix = {arXiv}, url = {https://proceedings.mlr.press/v306/fu26t.html}, } @article{you2026tah2, title = {Improving Test-Time Scaling with Adaptive Looped Transformers}, author = {You, Yichen and Fu, Tianyu and Feng, Aosong and Lv, Xingtai and Ning, Xuefei and Ding, Ning and Wang, Yu}, journal = {arXiv preprint arXiv:2609.35748}, year = {2026}, url = {https://arxiv.org/abs/2609.35748}, } @inproceedings{jeddi2026loopformer, title = {LoopFormer: Elastic-Depth Looped Transformers for Latent Reasoning via Shortcut Modulation}, author = {Jeddi, Ahmadreza and Ciccone, Marco and Taati, Babak}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2026}, eprint = {2602.11451}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2602.11451}, } @article{song2026adaponderlm, title = {AdaPonderLM: Gated Pondering Language Models with Token-Wise Adaptive Depth}, author = {Song, Shixiang and Li, He and Wang, Zitong and Zeng, Boyi and Song, Feichen and Wang, Yixuan and Xu, Zhiqin John and He, Ziwei and Lin, Zhouhan}, journal = {arXiv preprint arXiv:2603.01914}, year = {2026}, url = {https://arxiv.org/abs/2603.01914}, } @article{noci2026lookahead, title = {Thinking into the Future: Latent Lookahead Training for Transformers}, author = {Noci, Lorenzo and Bachmann, Gregor and Moosavi-Dezfooli, Seyed-Mohsen and Nabi, Moin}, journal = {arXiv preprint arXiv:2603.20219}, year = {2026}, url = {https://arxiv.org/abs/2603.20219}, } @inproceedings{frey2026adaptiveloops, title = {Adaptive Loops and Memory in Transformers: Think Harder or Know More?}, author = {Frey, Markus and Shomali, Behzad and Bashir, Ali Hamza and Berghaus, David and Koehler, Joachim and Ali, Mehdi}, booktitle = {Latent \& Implicit Thinking Workshop @ ICLR 2026}, year = {2026}, eprint = {2603.08391}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2603.08391}, } @article{park2026loopus, title = {LoopUS: Recasting Pretrained LLMs into Looped Latent Refinement Models}, author = {Park, Taekhyun and Lee, Yongjae and Kim, Dohee and Bae, Hyerim}, journal = {arXiv preprint arXiv:2605.11011}, year = {2026}, url = {https://arxiv.org/abs/2605.11011}, } @article{huang2026lrt, title = {Latent Recurrent Transformer: Architecture Exploration, Training Strategies, and Scaling Behavior}, author = {Huang, Zeyi and He, Xuehai and Ren, LiLiang and Wang, Yiping and Peng, Baolin and Cheng, Hao and Wang, Shuohang and He, Pengcheng and Gao, Jianfeng and Lee, Yong Jae and Shen, Yelong}, journal = {arXiv preprint arXiv:2605.26797}, year = {2026}, url = {https://arxiv.org/abs/2605.26797}, } @article{li2026later, title = {LaTER: Efficient Test-Time Reasoning via Latent Exploration and Explicit Verification}, author = {Li, Xuan and Wang, Yining and Liu, Yuchen and Liu, Guanjun and Qiu, Delai and Liu, Shengping and Liang, Jiaen and Huang, Wei and Yu, Jun and Zhu, Junnan}, journal = {arXiv preprint arXiv:2605.07315}, year = {2026}, url = {https://arxiv.org/abs/2605.07315}, } @article{sharma2026readout, title = {Dense Supervision Is Not Enough: The Readout Blind Spot in Looped Language Models}, author = {Sharma, Rituraj and Vu, Tu}, journal = {arXiv preprint arXiv:2606.24898}, year = {2026}, url = {https://arxiv.org/abs/2606.24898}, } @article{zeng2026depthattention, title = {Depth-Attention: Cross-Layer Value Mixing for Language Models}, author = {Zeng, Boyi and Hao, Yiqin and Wang, Zitong and Song, Shixiang and Li, He and Song, Feichen and Liu, Yifan and He, Ziwei and Wang, Xinbing and Lin, Zhouhan}, journal = {arXiv preprint arXiv:2606.05014}, year = {2026}, url = {https://arxiv.org/abs/2606.05014}, } @article{fu2026discoloop, title = {DiscoLoop: Looping Discrete Embeddings and Continuous Hidden States for Multi-hop Reasoning}, author = {Fu, Hengyu and Guo, Tianyu and Wang, Zixuan and Zhu, Hanlin and Lee, Jason D. and Jiao, Jiantao and Russell, Stuart and Mei, Song}, journal = {arXiv preprint arXiv:2607.00341}, year = {2026}, url = {https://arxiv.org/abs/2607.00341}, } @article{popescu2026adaptivedepth, title = {Adaptive Depth in Looped Transformers: Diagnosing Learned Halting Gates and Trajectory Readouts}, author = {Popescu, Andrei Cristian and S{\'a}ez de Oc{\'a}riz Borde, Haitz and Li{\`o}, Pietro}, journal = {arXiv preprint arXiv:2607.20519}, year = {2026}, url = {https://arxiv.org/abs/2607.20519}, } @article{logan2026fixedpoint, title = {Per-Token Fixed-Point Convergence in Depth-Recurrent Transformers}, author = {Logan, Joe}, journal = {arXiv preprint arXiv:2607.14427}, year = {2026}, url = {https://arxiv.org/abs/2607.14427}, } @article{dau2026beyonddepth, title = {Beyond Depth Truncation: Controlled Evaluation of Depth Utilization in Recursive Language Models}, author = {Dau, Ha Van and Khuat, Thanh Tung and Dung, Nguyen Thanh}, journal = {arXiv preprint arXiv:2609.19934}, year = {2026}, url = {https://arxiv.org/abs/2609.19934}, } @article{huang2026trading, title = {Trading Depth for Time in Recurrent Transformers}, author = {Huang, Zeyi and He, Xuehai and Shen, Yelong and Lee, Yong Jae}, journal = {arXiv preprint arXiv:2609.21605}, year = {2026}, url = {https://arxiv.org/abs/2609.21605}, } @article{ye2026reflux, title = {Write Back the $\Delta$: Revisiting the Same Tokens with Fresh Representations}, author = {Ye, Wencheng and Hu, Anning and Zhang, Xiangdong and Wang, Tianyi and Li, Yikang and Jin, Hengyu and Li, Bing and Yan, Junchi}, journal = {arXiv preprint arXiv:2609.32457}, year = {2026}, url = {https://arxiv.org/abs/2609.32457}, } @article{olszowiec2026monitoring, title = {Monitoring latent reasoning in looped language models}, author = {Olszowiec, Pawel and Tomaszewska, Paulina and {\.Z}elaszczyk, Maciej and Byra, Michal and Chung, Neo Christopher}, journal = {arXiv preprint arXiv:2610.18127}, year = {2026}, url = {https://arxiv.org/abs/2610.18127}, } % --------------------------------------------------------------------------- % 3. Slots, workspaces, memory-augmented networks % --------------------------------------------------------------------------- @inproceedings{locatello2020slot, title = {Object-Centric Learning with Slot Attention}, author = {Locatello, Francesco and Weissenborn, Dirk and Unterthiner, Thomas and Mahendran, Aravindh and Heigold, Georg and Uszkoreit, Jakob and Dosovitskiy, Alexey and Kipf, Thomas}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2020}, eprint = {2006.15055}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2006.15055}, } @inproceedings{goyal2022workspace, title = {Coordination Among Neural Modules Through a Shared Global Workspace}, author = {Goyal, Anirudh and Didolkar, Aniket and Lamb, Alex and Badola, Kartikeya and Ke, Nan Rosemary and Rahaman, Nasim and Binas, Jonathan and Blundell, Charles and Mozer, Michael and Bengio, Yoshua}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2022}, eprint = {2103.01197}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2103.01197}, } @inproceedings{jaegle2021perceiver, title = {Perceiver: General Perception with Iterative Attention}, author = {Jaegle, Andrew and Gimeno, Felix and Brock, Andrew and Zisserman, Andrew and Vinyals, Oriol and Carreira, Joao}, booktitle = {International Conference on Machine Learning (ICML)}, year = {2021}, eprint = {2103.03206}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2103.03206}, } @article{graves2014ntm, title = {Neural Turing Machines}, author = {Graves, Alex and Wayne, Greg and Danihelka, Ivo}, journal = {arXiv preprint arXiv:1410.5401}, year = {2014}, url = {https://arxiv.org/abs/1410.5401}, } @article{weston2014memory, title = {Memory Networks}, author = {Weston, Jason and Chopra, Sumit and Bordes, Antoine}, journal = {arXiv preprint arXiv:1410.3916}, year = {2014}, url = {https://arxiv.org/abs/1410.3916}, } @inproceedings{rae2016sparse, title = {Scaling Memory-Augmented Neural Networks with Sparse Reads and Writes}, author = {Rae, Jack W. and Hunt, Jonathan J. and Harley, Tim and Danihelka, Ivo and Senior, Andrew and Wayne, Greg and Graves, Alex and Lillicrap, Timothy P.}, booktitle = {Advances in Neural Information Processing Systems (NIPS)}, year = {2016}, eprint = {1610.09027}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/1610.09027}, } @article{wu2020memformer, title = {Memformer: A Memory-Augmented Transformer for Sequence Modeling}, author = {Wu, Qingyang and Lan, Zhenzhong and Qian, Kun and Gu, Jing and Geramifard, Alborz and Yu, Zhou}, journal = {arXiv preprint arXiv:2010.06891}, year = {2020}, url = {https://arxiv.org/abs/2010.06891}, } @inproceedings{bulatov2022rmt, title = {Recurrent Memory Transformer}, author = {Bulatov, Aydar and Kuratov, Yuri and Burtsev, Mikhail S.}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2022}, eprint = {2207.06881}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2207.06881}, } @inproceedings{ryoo2023ttm, title = {Token Turing Machines}, author = {Ryoo, Michael S. and Gopalakrishnan, Keerthana and Kahatapitiya, Kumara and Xiao, Ted and Rao, Kanishka and Stone, Austin and Lu, Yao and Ibarz, Julian and Arnab, Anurag}, booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, year = {2023}, eprint = {2211.09119}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2211.09119}, } @article{berges2024memorylayers, title = {Memory Layers at Scale}, author = {Berges, Vincent-Pierre and O{\u{g}}uz, Barlas and Haziza, Daniel and Yih, Wen-tau and Zettlemoyer, Luke and Ghosh, Gargi}, journal = {arXiv preprint arXiv:2412.09764}, year = {2024}, url = {https://arxiv.org/abs/2412.09764}, } @article{darcet2023registers, title = {Vision Transformers Need Registers}, author = {Darcet, Timoth{\'e}e and Oquab, Maxime and Mairal, Julien and Bojanowski, Piotr}, journal = {arXiv preprint arXiv:2309.16588}, year = {2023}, url = {https://arxiv.org/abs/2309.16588}, } % --------------------------------------------------------------------------- % 4. Long-context / recurrence / linear-time baselines % --------------------------------------------------------------------------- @inproceedings{dai2019transformerxl, title = {Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context}, author = {Dai, Zihang and Yang, Zhilin and Yang, Yiming and Carbonell, Jaime and Le, Quoc V. and Salakhutdinov, Ruslan}, booktitle = {Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics (ACL)}, year = {2019}, eprint = {1901.02860}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/1901.02860}, } @article{rae2019compressive, title = {Compressive Transformers for Long-Range Sequence Modelling}, author = {Rae, Jack W. and Potapenko, Anna and Jayakumar, Siddhant M. and Lillicrap, Timothy P.}, journal = {arXiv preprint arXiv:1911.05507}, year = {2019}, url = {https://arxiv.org/abs/1911.05507}, } @article{gu2023mamba, title = {Mamba: Linear-Time Sequence Modeling with Selective State Spaces}, author = {Gu, Albert and Dao, Tri}, journal = {arXiv preprint arXiv:2312.00752}, year = {2023}, url = {https://arxiv.org/abs/2312.00752}, } % --------------------------------------------------------------------------- % 5. Sparse routing / mixture-of-experts % --------------------------------------------------------------------------- @article{shazeer2017moe, title = {Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer}, author = {Shazeer, Noam and Mirhoseini, Azalia and Maziarz, Krzysztof and Davis, Andy and Le, Quoc and Hinton, Geoffrey and Dean, Jeff}, journal = {arXiv preprint arXiv:1701.06538}, year = {2017}, url = {https://arxiv.org/abs/1701.06538}, } @article{fedus2022switch, title = {Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity}, author = {Fedus, William and Zoph, Barret and Shazeer, Noam}, journal = {Journal of Machine Learning Research}, year = {2022}, eprint = {2101.03961}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2101.03961}, } % --------------------------------------------------------------------------- % 6. Value learning, off-policy / counterfactual evaluation % --------------------------------------------------------------------------- @article{ernst2005tree, title = {Tree-Based Batch Mode Reinforcement Learning}, author = {Ernst, Damien and Geurts, Pierre and Wehenkel, Louis}, journal = {Journal of Machine Learning Research}, volume = {6}, pages = {503--556}, year = {2005}, url = {https://www.jmlr.org/papers/v6/ernst05a.html}, } @inproceedings{mnih2013atari, title = {Playing Atari with Deep Reinforcement Learning}, author = {Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Graves, Alex and Antonoglou, Ioannis and Wierstra, Daan and Riedmiller, Martin}, booktitle = {NIPS Deep Learning Workshop}, year = {2013}, eprint = {1312.5602}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/1312.5602}, } @inproceedings{dudik2011doublyrobust, title = {Doubly Robust Policy Evaluation and Learning}, author = {Dud{\'i}k, Miroslav and Langford, John and Li, Lihong}, booktitle = {International Conference on Machine Learning (ICML)}, year = {2011}, eprint = {1103.4601}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/1103.4601}, } @article{bottou2013counterfactual, title = {Counterfactual Reasoning and Learning Systems}, author = {Bottou, L{\'e}on and Peters, Jonas and Qui{\~n}onero-Candela, Joaquin and Charles, Denis X. and Chickering, D. Max and Portugaly, Elon and Ray, Dipankar and Simard, Patrice and Snelson, Ed}, journal = {arXiv preprint arXiv:1209.2355}, year = {2012}, url = {https://arxiv.org/abs/1209.2355}, } @article{swaminathan2015crm, title = {Counterfactual Risk Minimization: Learning from Logged Bandit Feedback}, author = {Swaminathan, Adith and Joachims, Thorsten}, journal = {arXiv preprint arXiv:1502.02362}, year = {2015}, url = {https://arxiv.org/abs/1502.02362}, } % --------------------------------------------------------------------------- % 7. Early exit, speculative decoding, test-time compute % --------------------------------------------------------------------------- @inproceedings{schuster2022calm, title = {Confident Adaptive Language Modeling}, author = {Schuster, Tal and Fisch, Adam and Gupta, Jai and Dehghani, Mostafa and Bahri, Dara and Tran, Vinh Q. and Tay, Yi and Metzler, Donald}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2022}, eprint = {2207.07061}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2207.07061}, } @inproceedings{leviathan2023speculative, title = {Fast Inference from Transformers via Speculative Decoding}, author = {Leviathan, Yaniv and Kalman, Matan and Matias, Yossi}, booktitle = {International Conference on Machine Learning (ICML)}, year = {2023}, eprint = {2211.17192}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2211.17192}, } @article{snell2024testtime, title = {Scaling LLM Test-Time Compute Optimally can be More Effective than Scaling Model Parameters}, author = {Snell, Charlie and Lee, Jaehoon and Xu, Kelvin and Kumar, Aviral}, journal = {arXiv preprint arXiv:2408.03314}, year = {2024}, url = {https://arxiv.org/abs/2408.03314}, } @article{ye2024physics, title = {Physics of Language Models: Part 2.1, Grade-School Math and the Hidden Reasoning Process}, author = {Ye, Tian and Xu, Zicheng and Li, Yuanzhi and Allen-Zhu, Zeyuan}, journal = {arXiv preprint arXiv:2407.20311}, year = {2024}, url = {https://arxiv.org/abs/2407.20311}, } % --------------------------------------------------------------------------- % 8. Zero-initialised gates / exact-identity insertion (for C3) % --------------------------------------------------------------------------- @article{bachlechner2020rezero, title = {ReZero is All You Need: Fast Convergence at Large Depth}, author = {Bachlechner, Thomas and Majumder, Bodhisattwa Prasad and Mao, Huanru Henry and Cottrell, Garrison W. and McAuley, Julian}, journal = {arXiv preprint arXiv:2003.04887}, year = {2020}, url = {https://arxiv.org/abs/2003.04887}, } @article{zhang2023controlnet, title = {Adding Conditional Control to Text-to-Image Diffusion Models}, author = {Zhang, Lvmin and Rao, Anyi and Agrawala, Maneesh}, journal = {arXiv preprint arXiv:2302.05543}, year = {2023}, url = {https://arxiv.org/abs/2302.05543}, } @inproceedings{alayrac2022flamingo, title = {Flamingo: a Visual Language Model for Few-Shot Learning}, author = {Alayrac, Jean-Baptiste and Donahue, Jeff and Luc, Pauline and Miech, Antoine and Barr, Iain and Hasson, Yana and Lenc, Karel and Mensch, Arthur and Millican, Katie and Reynolds, Malcolm and Ring, Roman and Rutherford, Eliza and Cabi, Serkan and Han, Tengda and Gong, Zhitao and Samangooei, Sina and Monteiro, Marianne and Menick, Jacob and Borgeaud, Sebastian and Brock, Andrew and Nematzadeh, Aida and Sharifzadeh, Sahand and Binkowski, Mikolaj and Barreira, Ricardo and Vinyals, Oriol and Zisserman, Andrew and Simonyan, Karen}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2022}, eprint = {2204.14198}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2204.14198}, } % --------------------------------------------------------------------------- % 9. Interpretability-style sparse interventions (for C1/C2 context) % --------------------------------------------------------------------------- @article{ernst2026dsi, title = {Distributed Sparse Interventions in Language Models}, author = {Ernst, Maximilian S. and Linhardt, Lorenz and Peikert, Aaron and Eberle, Oliver}, journal = {arXiv preprint arXiv:2607.07128}, year = {2026}, url = {https://arxiv.org/abs/2607.07128}, } % --------------------------------------------------------------------------- % 10. Backbone building blocks and scaling laws % --------------------------------------------------------------------------- @article{shazeer2020glu, title = {{GLU} Variants Improve Transformer}, author = {Shazeer, Noam}, journal = {arXiv preprint arXiv:2002.05202}, year = {2020}, url = {https://arxiv.org/abs/2002.05202}, } @article{su2021roformer, title = {RoFormer: Enhanced Transformer with Rotary Position Embedding}, author = {Su, Jianlin and Lu, Yu and Pan, Shengfeng and Murtadha, Ahmed and Wen, Bo and Liu, Yunfeng}, journal = {arXiv preprint arXiv:2104.09864}, year = {2021}, url = {https://arxiv.org/abs/2104.09864}, } @inproceedings{ainslie2023gqa, title = {{GQA}: Training Generalized Multi-Query Transformer Models from Multi-Head Checkpoints}, author = {Ainslie, Joshua and Lee-Thorp, James and de Jong, Michiel and Zemlyanskiy, Yury and Lebr{\'o}n, Federico and Sanghai, Sumit}, booktitle = {Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (EMNLP)}, year = {2023}, eprint = {2305.13245}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/2305.13245}, } @inproceedings{zhang2019rmsnorm, title = {Root Mean Square Layer Normalization}, author = {Zhang, Biao and Sennrich, Rico}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2019}, eprint = {1910.07467}, archivePrefix = {arXiv}, url = {https://arxiv.org/abs/1910.07467}, } @article{kaplan2020scaling, title = {Scaling Laws for Neural Language Models}, author = {Kaplan, Jared and McCandlish, Sam and Henighan, Tom and Brown, Tom B. and Chess, Benjamin and Child, Rewon and Gray, Scott and Radford, Alec and Wu, Jeffrey and Amodei, Dario}, journal = {arXiv preprint arXiv:2001.08361}, year = {2020}, url = {https://arxiv.org/abs/2001.08361}, } @article{hoffmann2022chinchilla, title = {Training Compute-Optimal Large Language Models}, author = {Hoffmann, Jordan and Borgeaud, Sebastian and Mensch, Arthur and Buchatskaya, Elena and Cai, Trevor and Rutherford, Eliza and de Las Casas, Diego and Hendricks, Lisa Anne and Welbl, Johannes and Clark, Aidan and Hennigan, Tom and Noland, Eric and Millican, Katie and van den Driessche, George and Damoc, Bogdan and Guy, Aurelia and Osindero, Simon and Simonyan, Karen and Elsen, Erich and Rae, Jack W. and Vinyals, Oriol and Sifre, Laurent}, journal = {arXiv preprint arXiv:2203.15556}, year = {2022}, url = {https://arxiv.org/abs/2203.15556}, } @article{penedo2024fineweb, title = {The {FineWeb} Datasets: Decanting the Web for the Finest Text Data at Scale}, author = {Penedo, Guilherme and Kydl{\'\i}{\v{c}}ek, Hynek and Ben allal, Loubna and Lozhkov, Anton and Mitchell, Margaret and Raffel, Colin and Von Werra, Leandro and Wolf, Thomas}, year = {2024}, journal = {arXiv preprint arXiv:2406.17557}, url = {https://arxiv.org/abs/2406.17557}, }