WP44: pass-2 chain waits for a flag (pass 2 moved to M5M36 after two M3U96b freezes)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
4 changed files +86 −72
added
ingestion/run_pass2_chain.sh
+14 −0
@@ -0,0 +1,14 @@ | |||||
| 1 | +#!/bin/sh | ||||
| 2 | +# wait for the second snapshot pass, then normalise, compact, re-link, re-define the corpus and regenerate volume tables | ||||
| 3 | +cd ~/wp44_uqo; . .venv/bin/activate; ulimit -n 65536 | ||||
| 4 | +while [ ! -f logs/pass2_done.flag ]; do sleep 60; done # flag written after the pass-2 outputs (computed on M5M36) are copied back | ||||
| 5 | +python ingestion/build_corpus.py --cs-dir data/raw/openalex/works_pass2/cs --out data/processed_p2 --workers 12 > logs/build_corpus_p2.log 2>&1 | ||||
| 6 | +python - <<EOF3 | ||||
| 7 | +import os; os.makedirs("data/processed/tables_p2", exist_ok=True) | ||||
| 8 | +EOF3 | ||||
| 9 | +rm -rf data/processed/tables_p2; mv data/processed_p2/tables data/processed/tables_p2 | ||||
| 10 | +python ingestion/compact_tables.py ~/wp44_uqo _p2 > logs/compact_p2.log 2>&1 | ||||
| 11 | +python ingestion/link_sources.py --root ~/wp44_uqo > logs/link_sources_v2.log 2>&1 | ||||
| 12 | +python systematic_review/define_corpus.py --root ~/wp44_uqo > logs/define_corpus_v2.log 2>&1 | ||||
| 13 | +python bibliometrics/01_volume_series.py --root ~/wp44_uqo > logs/volume_series_v2.log 2>&1 | ||||
| 14 | +echo CHAIN_DONE >> logs/define_corpus_v2.log | ||||
modified
manuscript/references_part02.bib
+23 −23
@@ -20,7 +20,7 @@ | |||||
| 20 | note = {verified: arxiv API 2026-11-03} | 20 | note = {verified: arxiv API 2026-11-03} | ||
| 21 | } | 21 | } | ||
| 22 | 22 | ||||
| 23 | −@misc{culbert2025openalex, | 23 | +@misc{culbert2024openalex, | ||
| 24 | title = {Reference {Coverage} {Analysis} of {OpenAlex} compared to {Web} of {Science} and {Scopus}}, | 24 | title = {Reference {Coverage} {Analysis} of {OpenAlex} compared to {Web} of {Science} and {Scopus}}, | ||
| 25 | author = {Jack Culbert and Anne Hobert and Najko Jahn and Nick Haupka and Marion Schmidt and Paul Donner and Philipp Mayr}, | 25 | author = {Jack Culbert and Anne Hobert and Najko Jahn and Nick Haupka and Marion Schmidt and Paul Donner and Philipp Mayr}, | ||
| 26 | year = {2024}, | 26 | year = {2024}, | ||
@@ -40,7 +40,7 @@ | |||||
| 40 | note = {verified: arxiv API 2026-11-03} | 40 | note = {verified: arxiv API 2026-11-03} | ||
| 41 | } | 41 | } | ||
| 42 | 42 | ||||
| 43 | −@misc{bahdanau2015attention, | 43 | +@misc{bahdanau2014attention, | ||
| 44 | title = {Neural {Machine} {Translation} by {Jointly} {Learning} to {Align} and {Translate}}, | 44 | title = {Neural {Machine} {Translation} by {Jointly} {Learning} to {Align} and {Translate}}, | ||
| 45 | author = {Dzmitry Bahdanau and Kyunghyun Cho and Yoshua Bengio}, | 45 | author = {Dzmitry Bahdanau and Kyunghyun Cho and Yoshua Bengio}, | ||
| 46 | year = {2014}, | 46 | year = {2014}, | ||
@@ -70,7 +70,7 @@ | |||||
| 70 | note = {verified: arxiv API 2026-11-03} | 70 | note = {verified: arxiv API 2026-11-03} | ||
| 71 | } | 71 | } | ||
| 72 | 72 | ||||
| 73 | −@misc{kingma2014vae, | 73 | +@misc{kingma2013vae, | ||
| 74 | title = {{Auto-Encoding} {Variational} {Bayes}}, | 74 | title = {{Auto-Encoding} {Variational} {Bayes}}, | ||
| 75 | author = {Diederik P Kingma and Max Welling}, | 75 | author = {Diederik P Kingma and Max Welling}, | ||
| 76 | year = {2013}, | 76 | year = {2013}, | ||
@@ -150,7 +150,7 @@ | |||||
| 150 | note = {verified: arxiv API 2026-11-03} | 150 | note = {verified: arxiv API 2026-11-03} | ||
| 151 | } | 151 | } | ||
| 152 | 152 | ||||
| 153 | −@misc{raffel2020t5, | 153 | +@misc{raffel2019t5, | ||
| 154 | title = {Exploring the {Limits} of {Transfer} {Learning} with a {Unified} {Text-to-Text} {Transformer}}, | 154 | title = {Exploring the {Limits} of {Transfer} {Learning} with a {Unified} {Text-to-Text} {Transformer}}, | ||
| 155 | author = {Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu}, | 155 | author = {Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu}, | ||
| 156 | year = {2019}, | 156 | year = {2019}, | ||
@@ -190,7 +190,7 @@ | |||||
| 190 | note = {verified: arxiv API 2026-11-03} | 190 | note = {verified: arxiv API 2026-11-03} | ||
| 191 | } | 191 | } | ||
| 192 | 192 | ||||
| 193 | −@misc{fedus2022switch, | 193 | +@misc{fedus2021switch, | ||
| 194 | title = {Switch {Transformers:} {Scaling} to {Trillion} {Parameter} {Models} with {Simple} and {Efficient} {Sparsity}}, | 194 | title = {Switch {Transformers:} {Scaling} to {Trillion} {Parameter} {Models} with {Simple} and {Efficient} {Sparsity}}, | ||
| 195 | author = {William Fedus and Barret Zoph and Noam Shazeer}, | 195 | author = {William Fedus and Barret Zoph and Noam Shazeer}, | ||
| 196 | year = {2021}, | 196 | year = {2021}, | ||
@@ -200,7 +200,7 @@ | |||||
| 200 | note = {verified: arxiv API 2026-11-03} | 200 | note = {verified: arxiv API 2026-11-03} | ||
| 201 | } | 201 | } | ||
| 202 | 202 | ||||
| 203 | −@misc{dosovitskiy2021vit, | 203 | +@misc{dosovitskiy2020vit, | ||
| 204 | title = {An {Image} is {Worth} 16x16 {Words:} {Transformers} for {Image} {Recognition} at {Scale}}, | 204 | title = {An {Image} is {Worth} 16x16 {Words:} {Transformers} for {Image} {Recognition} at {Scale}}, | ||
| 205 | author = {Alexey Dosovitskiy and Lucas Beyer and Alexander Kolesnikov and Dirk Weissenborn and Xiaohua Zhai and Thomas Unterthiner and Mostafa Dehghani and Matthias Minderer and Georg Heigold and Sylvain Gelly and Jakob Uszkoreit and Neil Houlsby}, | 205 | author = {Alexey Dosovitskiy and Lucas Beyer and Alexander Kolesnikov and Dirk Weissenborn and Xiaohua Zhai and Thomas Unterthiner and Mostafa Dehghani and Matthias Minderer and Georg Heigold and Sylvain Gelly and Jakob Uszkoreit and Neil Houlsby}, | ||
| 206 | year = {2020}, | 206 | year = {2020}, | ||
@@ -250,7 +250,7 @@ | |||||
| 250 | note = {verified: arxiv API 2026-11-03} | 250 | note = {verified: arxiv API 2026-11-03} | ||
| 251 | } | 251 | } | ||
| 252 | 252 | ||||
| 253 | −@misc{song2021score, | 253 | +@misc{song2020score, | ||
| 254 | title = {{Score-Based} {Generative} {Modeling} through {Stochastic} {Differential} {Equations}}, | 254 | title = {{Score-Based} {Generative} {Modeling} through {Stochastic} {Differential} {Equations}}, | ||
| 255 | author = {Yang Song and Jascha Sohl-Dickstein and Diederik P. Kingma and Abhishek Kumar and Stefano Ermon and Ben Poole}, | 255 | author = {Yang Song and Jascha Sohl-Dickstein and Diederik P. Kingma and Abhishek Kumar and Stefano Ermon and Ben Poole}, | ||
| 256 | year = {2020}, | 256 | year = {2020}, | ||
@@ -360,7 +360,7 @@ | |||||
| 360 | note = {verified: arxiv API 2026-11-03} | 360 | note = {verified: arxiv API 2026-11-03} | ||
| 361 | } | 361 | } | ||
| 362 | 362 | ||||
| 363 | −@misc{hendrycks2021mmlu, | 363 | +@misc{hendrycks2020mmlu, | ||
| 364 | title = {Measuring {Massive} {Multitask} {Language} {Understanding}}, | 364 | title = {Measuring {Massive} {Multitask} {Language} {Understanding}}, | ||
| 365 | author = {Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt}, | 365 | author = {Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt}, | ||
| 366 | year = {2020}, | 366 | year = {2020}, | ||
@@ -380,7 +380,7 @@ | |||||
| 380 | note = {verified: arxiv API 2026-11-03} | 380 | note = {verified: arxiv API 2026-11-03} | ||
| 381 | } | 381 | } | ||
| 382 | 382 | ||||
| 383 | −@misc{srivastava2023bigbench, | 383 | +@misc{srivastava2022bigbench, | ||
| 384 | title = {Beyond the {Imitation} {Game:} {Quantifying} and extrapolating the capabilities of language models}, | 384 | title = {Beyond the {Imitation} {Game:} {Quantifying} and extrapolating the capabilities of language models}, | ||
| 385 | author = {Aarohi Srivastava and Abhinav Rastogi and Abhishek Rao and Abu Awal Md Shoeb and Abubakar Abid and Adam Fisch and Adam R. Brown and Adam Santoro and Aditya Gupta and Adrià Garriga-Alonso and others}, | 385 | author = {Aarohi Srivastava and Abhinav Rastogi and Abhishek Rao and Abu Awal Md Shoeb and Abubakar Abid and Adam Fisch and Adam R. Brown and Adam Santoro and Aditya Gupta and Adrià Garriga-Alonso and others}, | ||
| 386 | year = {2022}, | 386 | year = {2022}, | ||
@@ -410,7 +410,7 @@ | |||||
| 410 | note = {verified: arxiv API 2026-11-03} | 410 | note = {verified: arxiv API 2026-11-03} | ||
| 411 | } | 411 | } | ||
| 412 | 412 | ||||
| 413 | −@misc{rajbhandari2020zero, | 413 | +@misc{rajbhandari2019zero, | ||
| 414 | title = {{ZeRO:} {Memory} {Optimizations} {Toward} {Training} {Trillion} {Parameter} {Models}}, | 414 | title = {{ZeRO:} {Memory} {Optimizations} {Toward} {Training} {Trillion} {Parameter} {Models}}, | ||
| 415 | author = {Samyam Rajbhandari and Jeff Rasley and Olatunji Ruwase and Yuxiong He}, | 415 | author = {Samyam Rajbhandari and Jeff Rasley and Olatunji Ruwase and Yuxiong He}, | ||
| 416 | year = {2019}, | 416 | year = {2019}, | ||
@@ -540,7 +540,7 @@ | |||||
| 540 | note = {verified: arxiv API 2026-11-03} | 540 | note = {verified: arxiv API 2026-11-03} | ||
| 541 | } | 541 | } | ||
| 542 | 542 | ||||
| 543 | −@misc{hellerstein2019serverless, | 543 | +@misc{hellerstein2018serverless, | ||
| 544 | title = {Serverless {Computing:} {One} {Step} {Forward,} {Two} {Steps} {Back}}, | 544 | title = {Serverless {Computing:} {One} {Step} {Forward,} {Two} {Steps} {Back}}, | ||
| 545 | author = {Joseph M. Hellerstein and Jose Faleiro and Joseph E. Gonzalez and Johann Schleier-Smith and Vikram Sreekanti and Alexey Tumanov and Chenggang Wu}, | 545 | author = {Joseph M. Hellerstein and Jose Faleiro and Joseph E. Gonzalez and Johann Schleier-Smith and Vikram Sreekanti and Alexey Tumanov and Chenggang Wu}, | ||
| 546 | year = {2018}, | 546 | year = {2018}, | ||
@@ -580,7 +580,7 @@ | |||||
| 580 | note = {verified: arxiv API 2026-11-03} | 580 | note = {verified: arxiv API 2026-11-03} | ||
| 581 | } | 581 | } | ||
| 582 | 582 | ||||
| 583 | −@misc{szegedy2014intriguing, | 583 | +@misc{szegedy2013intriguing, | ||
| 584 | title = {Intriguing properties of neural networks}, | 584 | title = {Intriguing properties of neural networks}, | ||
| 585 | author = {Christian Szegedy and Wojciech Zaremba and Ilya Sutskever and Joan Bruna and Dumitru Erhan and Ian Goodfellow and Rob Fergus}, | 585 | author = {Christian Szegedy and Wojciech Zaremba and Ilya Sutskever and Joan Bruna and Dumitru Erhan and Ian Goodfellow and Rob Fergus}, | ||
| 586 | year = {2013}, | 586 | year = {2013}, | ||
@@ -590,7 +590,7 @@ | |||||
| 590 | note = {verified: arxiv API 2026-11-03} | 590 | note = {verified: arxiv API 2026-11-03} | ||
| 591 | } | 591 | } | ||
| 592 | 592 | ||||
| 593 | −@misc{mcmahan2017fl, | 593 | +@misc{mcmahan2016fl, | ||
| 594 | title = {{Communication-Efficient} {Learning} of {Deep} {Networks} from {Decentralized} {Data}}, | 594 | title = {{Communication-Efficient} {Learning} of {Deep} {Networks} from {Decentralized} {Data}}, | ||
| 595 | author = {H. Brendan McMahan and Eider Moore and Daniel Ramage and Seth Hampson and Blaise Agüera y Arcas}, | 595 | author = {H. Brendan McMahan and Eider Moore and Daniel Ramage and Seth Hampson and Blaise Agüera y Arcas}, | ||
| 596 | year = {2016}, | 596 | year = {2016}, | ||
@@ -600,7 +600,7 @@ | |||||
| 600 | note = {verified: arxiv API 2026-11-03} | 600 | note = {verified: arxiv API 2026-11-03} | ||
| 601 | } | 601 | } | ||
| 602 | 602 | ||||
| 603 | −@misc{li2021fno, | 603 | +@misc{li2020fno, | ||
| 604 | title = {Fourier {Neural} {Operator} for {Parametric} {Partial} {Differential} {Equations}}, | 604 | title = {Fourier {Neural} {Operator} for {Parametric} {Partial} {Differential} {Equations}}, | ||
| 605 | author = {Zongyi Li and Nikola Kovachki and Kamyar Azizzadenesheli and Burigede Liu and Kaushik Bhattacharya and Andrew Stuart and Anima Anandkumar}, | 605 | author = {Zongyi Li and Nikola Kovachki and Kamyar Azizzadenesheli and Burigede Liu and Kaushik Bhattacharya and Andrew Stuart and Anima Anandkumar}, | ||
| 606 | year = {2020}, | 606 | year = {2020}, | ||
@@ -610,7 +610,7 @@ | |||||
| 610 | note = {verified: arxiv API 2026-11-03} | 610 | note = {verified: arxiv API 2026-11-03} | ||
| 611 | } | 611 | } | ||
| 612 | 612 | ||||
| 613 | −@misc{luccioni2023bloom, | 613 | +@misc{luccioni2022bloom, | ||
| 614 | title = {Estimating the {Carbon} {Footprint} of {BLOOM,} a {176B} {Parameter} {Language} {Model}}, | 614 | title = {Estimating the {Carbon} {Footprint} of {BLOOM,} a {176B} {Parameter} {Language} {Model}}, | ||
| 615 | author = {Alexandra Sasha Luccioni and Sylvain Viguier and Anne-Laure Ligozat}, | 615 | author = {Alexandra Sasha Luccioni and Sylvain Viguier and Anne-Laure Ligozat}, | ||
| 616 | year = {2022}, | 616 | year = {2022}, | ||
@@ -620,7 +620,7 @@ | |||||
| 620 | note = {verified: arxiv API 2026-11-03} | 620 | note = {verified: arxiv API 2026-11-03} | ||
| 621 | } | 621 | } | ||
| 622 | 622 | ||||
| 623 | −@misc{levine2016visuomotor, | 623 | +@misc{levine2015visuomotor, | ||
| 624 | title = {{End-to-End} {Training} of {Deep} {Visuomotor} {Policies}}, | 624 | title = {{End-to-End} {Training} of {Deep} {Visuomotor} {Policies}}, | ||
| 625 | author = {Sergey Levine and Chelsea Finn and Trevor Darrell and Pieter Abbeel}, | 625 | author = {Sergey Levine and Chelsea Finn and Trevor Darrell and Pieter Abbeel}, | ||
| 626 | year = {2015}, | 626 | year = {2015}, | ||
@@ -710,7 +710,7 @@ | |||||
| 710 | note = {verified: arxiv API 2026-11-03} | 710 | note = {verified: arxiv API 2026-11-03} | ||
| 711 | } | 711 | } | ||
| 712 | 712 | ||||
| 713 | −@misc{yao2023react, | 713 | +@misc{yao2022react, | ||
| 714 | title = {{ReAct:} {Synergizing} {Reasoning} and {Acting} in {Language} {Models}}, | 714 | title = {{ReAct:} {Synergizing} {Reasoning} and {Acting} in {Language} {Models}}, | ||
| 715 | author = {Shunyu Yao and Jeffrey Zhao and Dian Yu and Nan Du and Izhak Shafran and Karthik Narasimhan and Yuan Cao}, | 715 | author = {Shunyu Yao and Jeffrey Zhao and Dian Yu and Nan Du and Izhak Shafran and Karthik Narasimhan and Yuan Cao}, | ||
| 716 | year = {2022}, | 716 | year = {2022}, | ||
@@ -740,7 +740,7 @@ | |||||
| 740 | note = {verified: arxiv API 2026-11-03} | 740 | note = {verified: arxiv API 2026-11-03} | ||
| 741 | } | 741 | } | ||
| 742 | 742 | ||||
| 743 | −@misc{radford2023whisper, | 743 | +@misc{radford2022whisper, | ||
| 744 | title = {Robust {Speech} {Recognition} via {Large-Scale} {Weak} {Supervision}}, | 744 | title = {Robust {Speech} {Recognition} via {Large-Scale} {Weak} {Supervision}}, | ||
| 745 | author = {Alec Radford and Jong Wook Kim and Tao Xu and Greg Brockman and Christine McLeavey and Ilya Sutskever}, | 745 | author = {Alec Radford and Jong Wook Kim and Tao Xu and Greg Brockman and Christine McLeavey and Ilya Sutskever}, | ||
| 746 | year = {2022}, | 746 | year = {2022}, | ||
@@ -850,7 +850,7 @@ | |||||
| 850 | note = {verified: arxiv API 2026-11-03} | 850 | note = {verified: arxiv API 2026-11-03} | ||
| 851 | } | 851 | } | ||
| 852 | 852 | ||||
| 853 | −@misc{pineau2021reproducibility, | 853 | +@misc{pineau2020reproducibility, | ||
| 854 | title = {Improving {Reproducibility} in {Machine} {Learning} {Research} {(A} {Report} from the {NeurIPS} 2019 {Reproducibility} {Program)}}, | 854 | title = {Improving {Reproducibility} in {Machine} {Learning} {Research} {(A} {Report} from the {NeurIPS} 2019 {Reproducibility} {Program)}}, | ||
| 855 | author = {Joelle Pineau and Philippe Vincent-Lamarre and Koustuv Sinha and Vincent Larivière and Alina Beygelzimer and Florence d'Alché-Buc and Emily Fox and Hugo Larochelle}, | 855 | author = {Joelle Pineau and Philippe Vincent-Lamarre and Koustuv Sinha and Vincent Larivière and Alina Beygelzimer and Florence d'Alché-Buc and Emily Fox and Hugo Larochelle}, | ||
| 856 | year = {2020}, | 856 | year = {2020}, | ||
@@ -1559,7 +1559,7 @@ | |||||
| 1559 | note = {verified: crossref 2026-11-03} | 1559 | note = {verified: crossref 2026-11-03} | ||
| 1560 | } | 1560 | } | ||
| 1561 | 1561 | ||||
| 1562 | −@inproceedings{williams2018finegrained, | 1562 | +@inproceedings{williams2019finegrained, | ||
| 1563 | title = {{ON} {SOME} {FINE-GRAINED} {QUESTIONS} {IN} {ALGORITHMS} {AND} {COMPLEXITY}}, | 1563 | title = {{ON} {SOME} {FINE-GRAINED} {QUESTIONS} {IN} {ALGORITHMS} {AND} {COMPLEXITY}}, | ||
| 1564 | author = {WILLIAMS, VIRGINIA VASSILEVSKA}, | 1564 | author = {WILLIAMS, VIRGINIA VASSILEVSKA}, | ||
| 1565 | year = {2019}, | 1565 | year = {2019}, | ||
@@ -1663,7 +1663,7 @@ | |||||
| 1663 | note = {verified: crossref 2026-11-03} | 1663 | note = {verified: crossref 2026-11-03} | ||
| 1664 | } | 1664 | } | ||
| 1665 | 1665 | ||||
| 1666 | −@article{jung2017rustbelt, | 1666 | +@article{jung2018rustbelt, | ||
| 1667 | title = {{RustBelt:} securing the foundations of the {Rust} programming language}, | 1667 | title = {{RustBelt:} securing the foundations of the {Rust} programming language}, | ||
| 1668 | author = {Jung, Ralf and Jourdan, Jacques-Henri and Krebbers, Robbert and Dreyer, Derek}, | 1668 | author = {Jung, Ralf and Jourdan, Jacques-Henri and Krebbers, Robbert and Dreyer, Derek}, | ||
| 1669 | year = {2018}, | 1669 | year = {2018}, | ||
@@ -1755,7 +1755,7 @@ | |||||
| 1755 | note = {verified: crossref 2026-11-03} | 1755 | note = {verified: crossref 2026-11-03} | ||
| 1756 | } | 1756 | } | ||
| 1757 | 1757 | ||||
| 1758 | −@article{allamanis2018survey, | 1758 | +@article{allamanis2019survey, | ||
| 1759 | title = {{A} {Survey} of {Machine} {Learning} for {Big} {Code} and {Naturalness}}, | 1759 | title = {{A} {Survey} of {Machine} {Learning} for {Big} {Code} and {Naturalness}}, | ||
| 1760 | author = {Allamanis, Miltiadis and Barr, Earl T. and Devanbu, Premkumar and Sutton, Charles}, | 1760 | author = {Allamanis, Miltiadis and Barr, Earl T. and Devanbu, Premkumar and Sutton, Charles}, | ||
| 1761 | year = {2019}, | 1761 | year = {2019}, | ||
@@ -2214,7 +2214,7 @@ | |||||
| 2214 | note = {verified: crossref 2026-11-03} | 2214 | note = {verified: crossref 2026-11-03} | ||
| 2215 | } | 2215 | } | ||
| 2216 | 2216 | ||||
| 2217 | −@article{bernal2017its, | 2217 | +@article{bernal2016its, | ||
| 2218 | title = {Interrupted time series regression for the evaluation of public health interventions: a tutorial}, | 2218 | title = {Interrupted time series regression for the evaluation of public health interventions: a tutorial}, | ||
| 2219 | author = {Lopez Bernal, James and Cummins, Steven and Gasparrini, Antonio}, | 2219 | author = {Lopez Bernal, James and Cummins, Steven and Gasparrini, Antonio}, | ||
| 2220 | year = {2016}, | 2220 | year = {2016}, | ||
modified
manuscript/references_verification_part02.csv
+23 −23
@@ -1,12 +1,12 @@ | |||||
| 1 | key,doi_or_arxiv,verified_via,verified_title,match,note | 1 | key,doi_or_arxiv,verified_via,verified_title,match,note | ||
| 2 | priem2022openalex,arXiv:2205.01833,arxiv,"OpenAlex: A fully-open index of scholarly works, authors, venues, institutions, and concepts",exact, | 2 | priem2022openalex,arXiv:2205.01833,arxiv,"OpenAlex: A fully-open index of scholarly works, authors, venues, institutions, and concepts",exact, | ||
| 3 | kinney2023s2,arXiv:2301.10140,arxiv,The Semantic Scholar Open Data Platform,exact, | 3 | kinney2023s2,arXiv:2301.10140,arxiv,The Semantic Scholar Open Data Platform,exact, | ||
| 4 | −culbert2025openalex,arXiv:2401.16359,arxiv,Reference Coverage Analysis of OpenAlex compared to Web of Science and Scopus,exact, | 4 | +culbert2024openalex,arXiv:2401.16359,arxiv,Reference Coverage Analysis of OpenAlex compared to Web of Science and Scopus,exact,key renamed from culbert2025openalex to match year of record | ||
| 5 | sutskever2014seq2seq,arXiv:1409.3215,arxiv,Sequence to Sequence Learning with Neural Networks,exact, | 5 | sutskever2014seq2seq,arXiv:1409.3215,arxiv,Sequence to Sequence Learning with Neural Networks,exact, | ||
| 6 | −bahdanau2015attention,arXiv:1409.0473,arxiv,Neural Machine Translation by Jointly Learning to Align and Translate,exact, | 6 | +bahdanau2014attention,arXiv:1409.0473,arxiv,Neural Machine Translation by Jointly Learning to Align and Translate,exact,key renamed from bahdanau2015attention to match year of record | ||
| 7 | goodfellow2014gan,arXiv:1406.2661,arxiv,Generative Adversarial Networks,exact, | 7 | goodfellow2014gan,arXiv:1406.2661,arxiv,Generative Adversarial Networks,exact, | ||
| 8 | mikolov2013word2vec,arXiv:1301.3781,arxiv,Efficient Estimation of Word Representations in Vector Space,exact, | 8 | mikolov2013word2vec,arXiv:1301.3781,arxiv,Efficient Estimation of Word Representations in Vector Space,exact, | ||
| 9 | −kingma2014vae,arXiv:1312.6114,arxiv,Auto-Encoding Variational Bayes,exact, | 9 | +kingma2013vae,arXiv:1312.6114,arxiv,Auto-Encoding Variational Bayes,exact,key renamed from kingma2014vae to match year of record | ||
| 10 | sevilla2022compute,arXiv:2202.05924,arxiv,Compute Trends Across Three Eras of Machine Learning,exact, | 10 | sevilla2022compute,arXiv:2202.05924,arxiv,Compute Trends Across Three Eras of Machine Learning,exact, | ||
| 11 | thompson2020limits,arXiv:2007.05558,arxiv,The Computational Limits of Deep Learning,exact, | 11 | thompson2020limits,arXiv:2007.05558,arxiv,The Computational Limits of Deep Learning,exact, | ||
| 12 | ahmed2020dedemocratization,arXiv:2010.15581,arxiv,The De-democratization of AI: Deep Learning and the Compute Divide in Artificial Intelligence Research,exact, | 12 | ahmed2020dedemocratization,arXiv:2010.15581,arxiv,The De-democratization of AI: Deep Learning and the Compute Divide in Artificial Intelligence Research,exact, | ||
@@ -14,17 +14,17 @@ abadi2016tensorflow,arXiv:1605.08695,arxiv,TensorFlow: A system for large-scale | |||||
| 14 | paszke2019pytorch,arXiv:1912.01703,arxiv,"PyTorch: An Imperative Style, High-Performance Deep Learning Library",exact, | 14 | paszke2019pytorch,arXiv:1912.01703,arxiv,"PyTorch: An Imperative Style, High-Performance Deep Learning Library",exact, | ||
| 15 | vaswani2017attention,arXiv:1706.03762,arxiv,Attention Is All You Need,exact, | 15 | vaswani2017attention,arXiv:1706.03762,arxiv,Attention Is All You Need,exact, | ||
| 16 | brown2020gpt3,arXiv:2005.14165,arxiv,Language Models are Few-Shot Learners,exact, | 16 | brown2020gpt3,arXiv:2005.14165,arxiv,Language Models are Few-Shot Learners,exact, | ||
| 17 | −raffel2020t5,arXiv:1910.10683,arxiv,Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer,exact, | 17 | +raffel2019t5,arXiv:1910.10683,arxiv,Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer,exact,key renamed from raffel2020t5 to match year of record | ||
| 18 | kaplan2020scaling,arXiv:2001.08361,arxiv,Scaling Laws for Neural Language Models,exact, | 18 | kaplan2020scaling,arXiv:2001.08361,arxiv,Scaling Laws for Neural Language Models,exact, | ||
| 19 | hoffmann2022chinchilla,arXiv:2203.15556,arxiv,Training Compute-Optimal Large Language Models,exact, | 19 | hoffmann2022chinchilla,arXiv:2203.15556,arxiv,Training Compute-Optimal Large Language Models,exact, | ||
| 20 | chowdhery2022palm,arXiv:2204.02311,arxiv,PaLM: Scaling Language Modeling with Pathways,exact, | 20 | chowdhery2022palm,arXiv:2204.02311,arxiv,PaLM: Scaling Language Modeling with Pathways,exact, | ||
| 21 | −fedus2022switch,arXiv:2101.03961,arxiv,Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity,exact, | 21 | +fedus2021switch,arXiv:2101.03961,arxiv,Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity,exact,key renamed from fedus2022switch to match year of record | ||
| 22 | −dosovitskiy2021vit,arXiv:2010.11929,arxiv,An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale,exact, | 22 | +dosovitskiy2020vit,arXiv:2010.11929,arxiv,An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale,exact,key renamed from dosovitskiy2021vit to match year of record | ||
| 23 | radford2021clip,arXiv:2103.00020,arxiv,Learning Transferable Visual Models From Natural Language Supervision,exact, | 23 | radford2021clip,arXiv:2103.00020,arxiv,Learning Transferable Visual Models From Natural Language Supervision,exact, | ||
| 24 | ramesh2021dalle,arXiv:2102.12092,arxiv,Zero-Shot Text-to-Image Generation,exact, | 24 | ramesh2021dalle,arXiv:2102.12092,arxiv,Zero-Shot Text-to-Image Generation,exact, | ||
| 25 | ho2020ddpm,arXiv:2006.11239,arxiv,Denoising Diffusion Probabilistic Models,exact, | 25 | ho2020ddpm,arXiv:2006.11239,arxiv,Denoising Diffusion Probabilistic Models,exact, | ||
| 26 | sohl2015diffusion,arXiv:1503.03585,arxiv,Deep Unsupervised Learning using Nonequilibrium Thermodynamics,exact, | 26 | sohl2015diffusion,arXiv:1503.03585,arxiv,Deep Unsupervised Learning using Nonequilibrium Thermodynamics,exact, | ||
| 27 | −song2021score,arXiv:2011.13456,arxiv,Score-Based Generative Modeling through Stochastic Differential Equations,exact, | 27 | +song2020score,arXiv:2011.13456,arxiv,Score-Based Generative Modeling through Stochastic Differential Equations,exact,key renamed from song2021score to match year of record | ||
| 28 | dhariwal2021beats,arXiv:2105.05233,arxiv,Diffusion Models Beat GANs on Image Synthesis,exact, | 28 | dhariwal2021beats,arXiv:2105.05233,arxiv,Diffusion Models Beat GANs on Image Synthesis,exact, | ||
| 29 | bommasani2021foundation,arXiv:2108.07258,arxiv,On the Opportunities and Risks of Foundation Models,exact, | 29 | bommasani2021foundation,arXiv:2108.07258,arxiv,On the Opportunities and Risks of Foundation Models,exact, | ||
| 30 | wei2022cot,arXiv:2201.11903,arxiv,Chain-of-Thought Prompting Elicits Reasoning in Large Language Models,exact, | 30 | wei2022cot,arXiv:2201.11903,arxiv,Chain-of-Thought Prompting Elicits Reasoning in Large Language Models,exact, | ||
@@ -35,12 +35,12 @@ christiano2017rlhf,arXiv:1706.03741,arxiv,Deep reinforcement learning from human | |||||
| 35 | stiennon2020summarize,arXiv:2009.01325,arxiv,Learning to summarize from human feedback,exact, | 35 | stiennon2020summarize,arXiv:2009.01325,arxiv,Learning to summarize from human feedback,exact, | ||
| 36 | bai2022hh,arXiv:2204.05862,arxiv,Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback,exact, | 36 | bai2022hh,arXiv:2204.05862,arxiv,Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback,exact, | ||
| 37 | lewis2020rag,arXiv:2005.11401,arxiv,Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks,exact, | 37 | lewis2020rag,arXiv:2005.11401,arxiv,Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks,exact, | ||
| 38 | −hendrycks2021mmlu,arXiv:2009.03300,arxiv,Measuring Massive Multitask Language Understanding,exact, | 38 | +hendrycks2020mmlu,arXiv:2009.03300,arxiv,Measuring Massive Multitask Language Understanding,exact,key renamed from hendrycks2021mmlu to match year of record | ||
| 39 | wang2019superglue,arXiv:1905.00537,arxiv,SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems,exact, | 39 | wang2019superglue,arXiv:1905.00537,arxiv,SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems,exact, | ||
| 40 | −srivastava2023bigbench,arXiv:2206.04615,arxiv,Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models,exact, | 40 | +srivastava2022bigbench,arXiv:2206.04615,arxiv,Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models,exact,key renamed from srivastava2023bigbench to match year of record | ||
| 41 | patterson2021carbon,arXiv:2104.10350,arxiv,Carbon Emissions and Large Neural Network Training,exact, | 41 | patterson2021carbon,arXiv:2104.10350,arxiv,Carbon Emissions and Large Neural Network Training,exact, | ||
| 42 | shoeybi2019megatron,arXiv:1909.08053,arxiv,Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism,exact, | 42 | shoeybi2019megatron,arXiv:1909.08053,arxiv,Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism,exact, | ||
| 43 | −rajbhandari2020zero,arXiv:1910.02054,arxiv,ZeRO: Memory Optimizations Toward Training Trillion Parameter Models,exact, | 43 | +rajbhandari2019zero,arXiv:1910.02054,arxiv,ZeRO: Memory Optimizations Toward Training Trillion Parameter Models,exact,key renamed from rajbhandari2020zero to match year of record | ||
| 44 | dao2022flashattention,arXiv:2205.14135,arxiv,FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness,exact, | 44 | dao2022flashattention,arXiv:2205.14135,arxiv,FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness,exact, | ||
| 45 | rae2021gopher,arXiv:2112.11446,arxiv,"Scaling Language Models: Methods, Analysis & Insights from Training Gopher",exact, | 45 | rae2021gopher,arXiv:2112.11446,arxiv,"Scaling Language Models: Methods, Analysis & Insights from Training Gopher",exact, | ||
| 46 | bigscience2022bloom,arXiv:2211.05100,arxiv,BLOOM: A 176B-Parameter Open-Access Multilingual Language Model,exact,key scao2022bloom renamed bigscience2022bloom (first author / publication year) | 46 | bigscience2022bloom,arXiv:2211.05100,arxiv,BLOOM: A 176B-Parameter Open-Access Multilingual Language Model,exact,key scao2022bloom renamed bigscience2022bloom (first author / publication year) | ||
@@ -53,15 +53,15 @@ reed2022gato,arXiv:2205.06175,arxiv,A Generalist Agent,exact, | |||||
| 53 | saharia2022imagen,arXiv:2205.11487,arxiv,Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding,exact, | 53 | saharia2022imagen,arXiv:2205.11487,arxiv,Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding,exact, | ||
| 54 | ramesh2022dalle2,arXiv:2204.06125,arxiv,Hierarchical Text-Conditional Image Generation with CLIP Latents,exact, | 54 | ramesh2022dalle2,arXiv:2204.06125,arxiv,Hierarchical Text-Conditional Image Generation with CLIP Latents,exact, | ||
| 55 | jonas2019serverless,arXiv:1902.03383,arxiv,Cloud Programming Simplified: A Berkeley View on Serverless Computing,exact, | 55 | jonas2019serverless,arXiv:1902.03383,arxiv,Cloud Programming Simplified: A Berkeley View on Serverless Computing,exact, | ||
| 56 | −hellerstein2019serverless,arXiv:1812.03651,arxiv,"Serverless Computing: One Step Forward, Two Steps Back",exact, | 56 | +hellerstein2018serverless,arXiv:1812.03651,arxiv,"Serverless Computing: One Step Forward, Two Steps Back",exact,key renamed from hellerstein2019serverless to match year of record | ||
| 57 | chen2018tvm,arXiv:1802.04799,arxiv,TVM: An Automated End-to-End Optimizing Compiler for Deep Learning,exact, | 57 | chen2018tvm,arXiv:1802.04799,arxiv,TVM: An Automated End-to-End Optimizing Compiler for Deep Learning,exact, | ||
| 58 | jacot2018ntk,arXiv:1806.07572,arxiv,Neural Tangent Kernel: Convergence and Generalization in Neural Networks,exact, | 58 | jacot2018ntk,arXiv:1806.07572,arxiv,Neural Tangent Kernel: Convergence and Generalization in Neural Networks,exact, | ||
| 59 | lu2021codexglue,arXiv:2102.04664,arxiv,CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation,exact, | 59 | lu2021codexglue,arXiv:2102.04664,arxiv,CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation,exact, | ||
| 60 | −szegedy2014intriguing,arXiv:1312.6199,arxiv,Intriguing properties of neural networks,exact, | 60 | +szegedy2013intriguing,arXiv:1312.6199,arxiv,Intriguing properties of neural networks,exact,key renamed from szegedy2014intriguing to match year of record | ||
| 61 | −mcmahan2017fl,arXiv:1602.05629,arxiv,Communication-Efficient Learning of Deep Networks from Decentralized Data,exact, | 61 | +mcmahan2016fl,arXiv:1602.05629,arxiv,Communication-Efficient Learning of Deep Networks from Decentralized Data,exact,key renamed from mcmahan2017fl to match year of record | ||
| 62 | −li2021fno,arXiv:2010.08895,arxiv,Fourier Neural Operator for Parametric Partial Differential Equations,exact, | 62 | +li2020fno,arXiv:2010.08895,arxiv,Fourier Neural Operator for Parametric Partial Differential Equations,exact,key renamed from li2021fno to match year of record | ||
| 63 | −luccioni2023bloom,arXiv:2211.02001,arxiv,"Estimating the Carbon Footprint of BLOOM, a 176B Parameter Language Model",exact, | 63 | +luccioni2022bloom,arXiv:2211.02001,arxiv,"Estimating the Carbon Footprint of BLOOM, a 176B Parameter Language Model",exact,key renamed from luccioni2023bloom to match year of record | ||
| 64 | −levine2016visuomotor,arXiv:1504.00702,arxiv,End-to-End Training of Deep Visuomotor Policies,exact, | 64 | +levine2015visuomotor,arXiv:1504.00702,arxiv,End-to-End Training of Deep Visuomotor Policies,exact,key renamed from levine2016visuomotor to match year of record | ||
| 65 | openai2023gpt4,arXiv:2303.08774,arxiv,GPT-4 Technical Report,exact, | 65 | openai2023gpt4,arXiv:2303.08774,arxiv,GPT-4 Technical Report,exact, | ||
| 66 | touvron2023llama,arXiv:2302.13971,arxiv,LLaMA: Open and Efficient Foundation Language Models,exact, | 66 | touvron2023llama,arXiv:2302.13971,arxiv,LLaMA: Open and Efficient Foundation Language Models,exact, | ||
| 67 | touvron2023llama2,arXiv:2307.09288,arxiv,Llama 2: Open Foundation and Fine-Tuned Chat Models,exact, | 67 | touvron2023llama2,arXiv:2307.09288,arxiv,Llama 2: Open Foundation and Fine-Tuned Chat Models,exact, | ||
@@ -70,10 +70,10 @@ bubeck2023sparks,arXiv:2303.12712,arxiv,Sparks of Artificial General Intelligenc | |||||
| 70 | openai2024o1,arXiv:2412.16720,arxiv,OpenAI o1 System Card,exact, | 70 | openai2024o1,arXiv:2412.16720,arxiv,OpenAI o1 System Card,exact, | ||
| 71 | deepseek2025r1,arXiv:2501.12948,arxiv,DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning,exact, | 71 | deepseek2025r1,arXiv:2501.12948,arxiv,DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning,exact, | ||
| 72 | gu2023mamba,arXiv:2312.00752,arxiv,Mamba: Linear-Time Sequence Modeling with Selective State Spaces,exact, | 72 | gu2023mamba,arXiv:2312.00752,arxiv,Mamba: Linear-Time Sequence Modeling with Selective State Spaces,exact, | ||
| 73 | −yao2023react,arXiv:2210.03629,arxiv,ReAct: Synergizing Reasoning and Acting in Language Models,exact, | 73 | +yao2022react,arXiv:2210.03629,arxiv,ReAct: Synergizing Reasoning and Acting in Language Models,exact,key renamed from yao2023react to match year of record | ||
| 74 | schick2023toolformer,arXiv:2302.04761,arxiv,Toolformer: Language Models Can Teach Themselves to Use Tools,exact, | 74 | schick2023toolformer,arXiv:2302.04761,arxiv,Toolformer: Language Models Can Teach Themselves to Use Tools,exact, | ||
| 75 | gemini2023,arXiv:2312.11805,arxiv,Gemini: A Family of Highly Capable Multimodal Models,exact, | 75 | gemini2023,arXiv:2312.11805,arxiv,Gemini: A Family of Highly Capable Multimodal Models,exact, | ||
| 76 | −radford2023whisper,arXiv:2212.04356,arxiv,Robust Speech Recognition via Large-Scale Weak Supervision,exact, | 76 | +radford2022whisper,arXiv:2212.04356,arxiv,Robust Speech Recognition via Large-Scale Weak Supervision,exact,key renamed from radford2023whisper to match year of record | ||
| 77 | brohan2023rt2,arXiv:2307.15818,arxiv,RT-2: Vision-Language-Action Models Transfer Web Knowledge to Robotic Control,exact, | 77 | brohan2023rt2,arXiv:2307.15818,arxiv,RT-2: Vision-Language-Action Models Transfer Web Knowledge to Robotic Control,exact, | ||
| 78 | liang2024monitoring,arXiv:2403.07183,arxiv,Monitoring AI-Modified Content at Scale: A Case Study on the Impact of ChatGPT on AI Conference Peer Reviews,exact, | 78 | liang2024monitoring,arXiv:2403.07183,arxiv,Monitoring AI-Modified Content at Scale: A Case Study on the Impact of ChatGPT on AI Conference Peer Reviews,exact, | ||
| 79 | liang2024mapping,arXiv:2404.01268,arxiv,Mapping the Increasing Use of LLMs in Scientific Papers,exact, | 79 | liang2024mapping,arXiv:2404.01268,arxiv,Mapping the Increasing Use of LLMs in Scientific Papers,exact, | ||
@@ -84,7 +84,7 @@ dehghani2021lottery,arXiv:2107.07002,arxiv,The Benchmark Lottery,exact, | |||||
| 84 | raji2021benchmark,arXiv:2111.15366,arxiv,AI and the Everything in the Whole Wide World Benchmark,exact, | 84 | raji2021benchmark,arXiv:2111.15366,arxiv,AI and the Everything in the Whole Wide World Benchmark,exact, | ||
| 85 | maslej2024aiindex,arXiv:2405.19522,arxiv,Artificial Intelligence Index Report 2024,exact, | 85 | maslej2024aiindex,arXiv:2405.19522,arxiv,Artificial Intelligence Index Report 2024,exact, | ||
| 86 | maslej2025aiindex,arXiv:2504.07139,arxiv,Artificial Intelligence Index Report 2025,exact, | 86 | maslej2025aiindex,arXiv:2504.07139,arxiv,Artificial Intelligence Index Report 2025,exact, | ||
| 87 | −pineau2021reproducibility,arXiv:2003.12206,arxiv,Improving Reproducibility in Machine Learning Research (A Report from the NeurIPS 2019 Reproducibility Program),exact, | 87 | +pineau2020reproducibility,arXiv:2003.12206,arxiv,Improving Reproducibility in Machine Learning Research (A Report from the NeurIPS 2019 Reproducibility Program),exact,key renamed from pineau2021reproducibility to match year of record | ||
| 88 | hendrycks2021unsolved,arXiv:2109.13916,arxiv,Unsolved Problems in ML Safety,exact, | 88 | hendrycks2021unsolved,arXiv:2109.13916,arxiv,Unsolved Problems in ML Safety,exact, | ||
| 89 | liang2025widespread,arXiv:2502.09747,arxiv,The Widespread Adoption of Large Language Model-Assisted Writing Across Society,exact,"key renamed: first author is Liang, not Kobak" | 89 | liang2025widespread,arXiv:2502.09747,arxiv,The Widespread Adoption of Large Language Model-Assisted Writing Across Society,exact,"key renamed: first author is Liang, not Kobak" | ||
| 90 | ley2002dblp,10.1007/3-540-45735-6_1,crossref,"The DBLP Computer Science Bibliography: Evolution, Research Issues, Perspectives",exact, | 90 | ley2002dblp,10.1007/3-540-45735-6_1,crossref,"The DBLP Computer Science Bibliography: Evolution, Research Issues, Perspectives",exact, | ||
@@ -146,7 +146,7 @@ jouppi2017tpu,10.1145/3079856.3080246,crossref,In-Datacenter Performance Analysi | |||||
| 146 | leiserson2020room,10.1126/science.aam9744,crossref,There’s plenty of room at the Top: What will drive computer performance after Moore’s law?,exact, | 146 | leiserson2020room,10.1126/science.aam9744,crossref,There’s plenty of room at the Top: What will drive computer performance after Moore’s law?,exact, | ||
| 147 | dennard1974scaling,10.1109/JSSC.1974.1050511,crossref,Design of ion-implanted MOSFET's with very small physical dimensions,exact, | 147 | dennard1974scaling,10.1109/JSSC.1974.1050511,crossref,Design of ion-implanted MOSFET's with very small physical dimensions,exact, | ||
| 148 | lattner2021mlir,10.1109/CGO51591.2021.9370308,crossref,MLIR: Scaling Compiler Infrastructure for Domain Specific Computation,exact, | 148 | lattner2021mlir,10.1109/CGO51591.2021.9370308,crossref,MLIR: Scaling Compiler Infrastructure for Domain Specific Computation,exact, | ||
| 149 | −williams2018finegrained,10.1142/9789813272880_0188,crossref,ON SOME FINE-GRAINED QUESTIONS IN ALGORITHMS AND COMPLEXITY,exact, | 149 | +williams2019finegrained,10.1142/9789813272880_0188,crossref,ON SOME FINE-GRAINED QUESTIONS IN ALGORITHMS AND COMPLEXITY,exact,key renamed from williams2018finegrained to match year of record | ||
| 150 | vassilevska2015hardness,10.4230/LIPIcs.IPEC.2015.17,openalex,Hardness of Easy Problems: Basing Hardness on Popular Conjectures such as the Strong Exponential Time Hypothesis (Invited Talk),minor,OpenAlex title adds '(Invited Talk)' | 150 | vassilevska2015hardness,10.4230/LIPIcs.IPEC.2015.17,openalex,Hardness of Easy Problems: Basing Hardness on Popular Conjectures such as the Strong Exponential Time Hypothesis (Invited Talk),minor,OpenAlex title adds '(Invited Talk)' | ||
| 151 | zhang2021rethinking,10.1145/3446776,crossref,Understanding deep learning (still) requires rethinking generalization,exact, | 151 | zhang2021rethinking,10.1145/3446776,crossref,Understanding deep learning (still) requires rethinking generalization,exact, | ||
| 152 | belkin2019reconciling,10.1073/pnas.1903070116,crossref,Reconciling modern machine-learning practice and the classical bias–variance trade-off,exact, | 152 | belkin2019reconciling,10.1073/pnas.1903070116,crossref,Reconciling modern machine-learning practice and the classical bias–variance trade-off,exact, | ||
@@ -155,7 +155,7 @@ arute2019supremacy,10.1038/s41586-019-1666-5,crossref,Quantum supremacy using a | |||||
| 155 | bharti2022nisq,10.1103/RevModPhys.94.015004,crossref,Noisy intermediate-scale quantum algorithms,exact, | 155 | bharti2022nisq,10.1103/RevModPhys.94.015004,crossref,Noisy intermediate-scale quantum algorithms,exact, | ||
| 156 | google2023surface,10.1038/s41586-022-05434-1,crossref,Suppressing quantum errors by scaling a surface code logical qubit,exact, | 156 | google2023surface,10.1038/s41586-022-05434-1,crossref,Suppressing quantum errors by scaling a surface code logical qubit,exact, | ||
| 157 | google2025threshold,10.1038/s41586-024-08449-y,crossref,Quantum error correction below the surface code threshold,exact,key google2024threshold renamed google2025threshold (first author / publication year) | 157 | google2025threshold,10.1038/s41586-024-08449-y,crossref,Quantum error correction below the surface code threshold,exact,key google2024threshold renamed google2025threshold (first author / publication year) | ||
| 158 | −jung2017rustbelt,10.1145/3158154,crossref,RustBelt: securing the foundations of the Rust programming language,exact, | 158 | +jung2018rustbelt,10.1145/3158154,crossref,RustBelt: securing the foundations of the Rust programming language,exact,key renamed from jung2017rustbelt to match year of record | ||
| 159 | matsakis2014rust,10.1145/2692956.2663188,crossref,The rust language,exact, | 159 | matsakis2014rust,10.1145/2692956.2663188,crossref,The rust language,exact, | ||
| 160 | leroy2009compcert,10.1145/1538788.1538814,crossref,Formal verification of a realistic compiler,exact, | 160 | leroy2009compcert,10.1145/1538788.1538814,crossref,Formal verification of a realistic compiler,exact, | ||
| 161 | klein2009sel4,10.1145/1629575.1629596,crossref,seL4: formal verification of an OS kernel,exact,title+subtitle fields | 161 | klein2009sel4,10.1145/1629575.1629596,crossref,seL4: formal verification of an OS kernel,exact,title+subtitle fields | ||
@@ -163,7 +163,7 @@ demoura2015lean,10.1007/978-3-319-21401-6_26,crossref,The Lean Theorem Prover (S | |||||
| 163 | demoura2021lean4,10.1007/978-3-030-79876-5_37,crossref,The Lean 4 Theorem Prover and Programming Language,exact, | 163 | demoura2021lean4,10.1007/978-3-030-79876-5_37,crossref,The Lean 4 Theorem Prover and Programming Language,exact, | ||
| 164 | mathlib2020,10.1145/3372885.3373824,crossref,The lean mathematical library,exact, | 164 | mathlib2020,10.1145/3372885.3373824,crossref,The lean mathematical library,exact, | ||
| 165 | hales2017kepler,10.1017/fmp.2017.1,crossref,A FORMAL PROOF OF THE KEPLER CONJECTURE,exact, | 165 | hales2017kepler,10.1017/fmp.2017.1,crossref,A FORMAL PROOF OF THE KEPLER CONJECTURE,exact, | ||
| 166 | −allamanis2018survey,10.1145/3212695,crossref,A Survey of Machine Learning for Big Code and Naturalness,exact, | 166 | +allamanis2019survey,10.1145/3212695,crossref,A Survey of Machine Learning for Big Code and Naturalness,exact,key renamed from allamanis2018survey to match year of record | ||
| 167 | bernstein2017pqc,10.1038/nature23461,crossref,Post-quantum cryptography,exact, | 167 | bernstein2017pqc,10.1038/nature23461,crossref,Post-quantum cryptography,exact, | ||
| 168 | shor1997algorithms,10.1137/S0097539795293172,crossref,Polynomial-Time Algorithms for Prime Factorization and Discrete Logarithms on a Quantum Computer,exact, | 168 | shor1997algorithms,10.1137/S0097539795293172,crossref,Polynomial-Time Algorithms for Prime Factorization and Discrete Logarithms on a Quantum Computer,exact, | ||
| 169 | nist2024fips203,10.6028/NIST.FIPS.203,crossref,Module-lattice-based key-encapsulation mechanism standard,exact, | 169 | nist2024fips203,10.6028/NIST.FIPS.203,crossref,Module-lattice-based key-encapsulation mechanism standard,exact, | ||
@@ -203,7 +203,7 @@ wang2023aiscience,10.1038/s41586-023-06221-2,crossref,Scientific discovery in th | |||||
| 203 | masanet2020datacenter,10.1126/science.aba3758,crossref,Recalibrating global data center energy-use estimates,exact, | 203 | masanet2020datacenter,10.1126/science.aba3758,crossref,Recalibrating global data center energy-use estimates,exact, | ||
| 204 | bai1998breaks,10.2307/2998540,crossref,Estimating and Testing Linear Models with Multiple Structural Changes,exact, | 204 | bai1998breaks,10.2307/2998540,crossref,Estimating and Testing Linear Models with Multiple Structural Changes,exact, | ||
| 205 | bai2003computation,10.1002/jae.659,crossref,Computation and analysis of multiple structural change models,exact, | 205 | bai2003computation,10.1002/jae.659,crossref,Computation and analysis of multiple structural change models,exact, | ||
| 206 | −bernal2017its,10.1093/ije/dyw098,crossref,Interrupted time series regression for the evaluation of public health interventions: a tutorial,exact, | 206 | +bernal2016its,10.1093/ije/dyw098,crossref,Interrupted time series regression for the evaluation of public health interventions: a tutorial,exact,key renamed from bernal2017its to match year of record | ||
| 207 | andrews1993tests,10.2307/2951764,crossref,Tests for Parameter Instability and Structural Change With Unknown Change Point,exact, | 207 | andrews1993tests,10.2307/2951764,crossref,Tests for Parameter Instability and Structural Change With Unknown Change Point,exact, | ||
| 208 | killick2012pelt,10.1080/01621459.2012.737745,crossref,Optimal Detection of Changepoints With a Linear Computational Cost,exact, | 208 | killick2012pelt,10.1080/01621459.2012.737745,crossref,Optimal Detection of Changepoints With a Linear Computational Cost,exact, | ||
| 209 | kobak2025delving,10.1126/sciadv.adt3813,crossref,Delving into LLM-assisted writing in biomedical publications through excess vocabulary,exact, | 209 | kobak2025delving,10.1126/sciadv.adt3813,crossref,Delving into LLM-assisted writing in biomedical publications through excess vocabulary,exact, | ||
modified
manuscript/sections/part02_evolution.tex
+26 −26
@@ -15,7 +15,7 @@ Any claim that computer science has been transformed since late 2022 presupposes | |||||
| 15 | 15 | ||||
| 16 | Four classification systems are relevant to a corpus of the kind assembled here, and they encode the structure of the discipline in different ways. | 16 | Four classification systems are relevant to a corpus of the kind assembled here, and they encode the structure of the discipline in different ways. | ||
| 17 | 17 | ||||
| 18 | −The \emph{ACM Computing Classification System} (CCS) is the only one produced by a professional society as an explicit ontology. Its 2012 revision replaced the 1998 four-level scheme with a poly-hierarchy of roughly two thousand concepts organised under thirteen top-level nodes, among them ``Theory of computation'', ``Computing methodologies'' (which houses artificial intelligence, machine learning, computer graphics and simulation side by side), ``Computer systems organization'', ``Software and its engineering'', ``Security and privacy'', ``Human-centered computing'' and ``Networks'' \citep{rous2012ccs}. The CCS is a statement of how the ACM understood the field in 2012. It is applied by authors at submission time, is used almost exclusively in ACM venues, and is therefore sparse in any corpus that spans IEEE, Springer, Elsevier, open-access publishers and preprint servers. Its value for this study is conceptual rather than operational: it provides an independent, expert-designed reference against which the forty categories of CLAUDE.md~\S4 can be positioned, and it documents a pre-transformer view in which large language models, foundation models, AI agents and AI safety do not exist as concepts at all. | 18 | +The \emph{ACM Computing Classification System} (CCS) is the only one produced by a professional society as an explicit ontology. Its 2012 revision replaced the 1998 four-level scheme with a poly-hierarchy of roughly two thousand concepts organised under about a dozen top-level nodes, among them ``Theory of computation'', ``Computing methodologies'' (which houses artificial intelligence, machine learning, computer graphics and simulation side by side), ``Computer systems organization'', ``Software and its engineering'', ``Security and privacy'', ``Human-centered computing'' and ``Networks'' \citep{rous2012ccs}. The CCS is a statement of how the ACM understood the field in 2012. It is applied by authors at submission time, is used almost exclusively in ACM venues, and is therefore sparse in any corpus that spans IEEE, Springer, Elsevier, open-access publishers and preprint servers. Its value for this study is conceptual rather than operational: it provides an independent, expert-designed reference against which the forty categories of CLAUDE.md~\S4 can be positioned, and it documents a pre-transformer view in which large language models, foundation models, AI agents and AI safety do not exist as concepts at all. | ||
| 19 | 19 | ||||
| 20 | The \emph{arXiv subject classes} are the second system. arXiv was created in 1991 for high-energy physics and its taxonomy reflects that origin; the computer science archive (\texttt{cs}) is organised into forty subject classes largely inherited from the ACM scheme of the 1990s \citep{ginsparg2011arxiv}. Authors choose a primary class and may add cross-lists, so the classification is self-reported, multi-label, and coarse. Its distinctive property is temporal: a preprint is classified at the moment of submission, before peer review, so the arXiv class records how authors positioned their work at the time, not how a later indexer reconstructed it. For the questions of this paper that property matters: when a paper on, say, program verification with language models is submitted to \texttt{cs.PL} with a cross-list to \texttt{cs.LG}, the pair of classes is a small but genuine piece of evidence about disciplinary boundaries as practitioners perceive them. | 20 | The \emph{arXiv subject classes} are the second system. arXiv was created in 1991 for high-energy physics and its taxonomy reflects that origin; the computer science archive (\texttt{cs}) is organised into forty subject classes largely inherited from the ACM scheme of the 1990s \citep{ginsparg2011arxiv}. Authors choose a primary class and may add cross-lists, so the classification is self-reported, multi-label, and coarse. Its distinctive property is temporal: a preprint is classified at the moment of submission, before peer review, so the arXiv class records how authors positioned their work at the time, not how a later indexer reconstructed it. For the questions of this paper that property matters: when a paper on, say, program verification with language models is submitted to \texttt{cs.PL} with a cross-list to \texttt{cs.LG}, the pair of classes is a small but genuine piece of evidence about disciplinary boundaries as practitioners perceive them. | ||
| 21 | 21 | ||||
@@ -46,7 +46,7 @@ The ontology's limitations are stated in Appendix~A and are not repeated here, e | |||||
| 46 | 46 | ||||
| 47 | The disagreements among the four systems are systematic, and they fall into three kinds. | 47 | The disagreements among the four systems are systematic, and they fall into three kinds. | ||
| 48 | 48 | ||||
| 49 | −The first is \emph{coverage}. Comparisons of bibliographic databases consistently show that no source indexes the whole of a discipline and that the gaps are not random: conference proceedings, preprints, and non-English venues are the most unevenly covered \citep{visser2021comparison,martinmartin2021coverage}. OpenAlex, the backbone of this corpus, inherits the broad but uneven coverage of Microsoft Academic and Crossref; its reference lists are less complete than those of the commercial indexes for part of the period, which affects citation-based measures more than counts \citep{culbert2025openalex}. The linking phase of this study illustrates the scale of the problem at the level of record identity. The OpenAlex snapshot contains \LinkOaCsRecords{} works classified in computer science between 2018 and 2026; DBLP contributes \LinkDblpTwoZeroOneEightTwoZeroTwoSix{} records over the same years, of which \LinkDblpLinkedDistinct{} could be matched to an OpenAlex work by DOI or by title and year in the first linking pass and \LinkDblpUnlinked{} could not; the arXiv \texttt{cs} harvest contributes \LinkArxivCsRecords{} preprints, of which \LinkArxivLinkedDistinct{} were matched, overwhelmingly through the arXiv identifier that OpenAlex stores. These are linkage statistics, not findings about research, but they establish that a classification applied to one source cannot simply be transferred to another: a non-trivial fraction of the venue-defined (DBLP) universe has no citation-defined (OpenAlex) counterpart, and vice versa. Part~III describes the second linking pass and the treatment of unmatched records. | 49 | +The first is \emph{coverage}. Comparisons of bibliographic databases consistently show that no source indexes the whole of a discipline and that the gaps are not random: conference proceedings, preprints, and non-English venues are the most unevenly covered \citep{visser2021comparison,martinmartin2021coverage}. OpenAlex, the backbone of this corpus, inherits the broad but uneven coverage of Microsoft Academic and Crossref; its reference lists are less complete than those of the commercial indexes for part of the period, which affects citation-based measures more than counts (a preprint finding; \citealp{culbert2024openalex}. The linking phase of this study illustrates the scale of the problem at the level of record identity. The OpenAlex snapshot contains \LinkOaCsRecords{} works classified in computer science between 2018 and 2026; DBLP contributes \LinkDblpTwoZeroOneEightTwoZeroTwoSix{} records over the same years, of which \LinkDblpLinkedDistinct{} could be matched to an OpenAlex work by DOI or by title and year in the first linking pass and \LinkDblpUnlinked{} could not; the arXiv \texttt{cs} harvest contributes \LinkArxivCsRecords{} preprints, of which \LinkArxivLinkedDistinct{} were matched, overwhelmingly through the arXiv identifier that OpenAlex stores. These are linkage statistics, not findings about research, but they establish that a classification applied to one source cannot simply be transferred to another: a non-trivial fraction of the venue-defined (DBLP) universe has no citation-defined (OpenAlex) counterpart, and vice versa. Part~III describes the second linking pass and the treatment of unmatched records. | ||
| 50 | 50 | ||||
| 51 | The second kind of disagreement is \emph{granularity and boundary}. The arXiv class \texttt{cs.LG} is one node among forty; in OpenAlex, machine learning spreads across dozens of topics in several fields, including statistics, engineering and medicine, because the citation network does not respect the boundary of the computer-science field. Conversely, OpenAlex's field~17 (``Computer Science'') contains clusters on educational technology, media studies and physics that no computer scientist would claim, and the ontology of this study deliberately leaves most of them unmapped (Appendix~A). The practical consequence is that the apparent size of ``AI'' depends on which boundary is used: a venue-based definition (DBLP streams of NeurIPS, ICML, ICLR, AAAI and the vision and language conferences) gives a different set from a topic-based definition, which gives a different set again from an arXiv-class definition, and the three sets diverge most in the applied sciences, where machine-learning methods are used without the paper belonging to any machine-learning community. The multi-label design of the ontology, and the use of all three signal types with capped weights in equation~\eqref{eq:p2-score}, is the response to this divergence: a work that is an AI paper by every criterion scores high on all of them, while a work that is an AI paper by vocabulary alone is held to a higher evidentiary standard. | 51 | The second kind of disagreement is \emph{granularity and boundary}. The arXiv class \texttt{cs.LG} is one node among forty; in OpenAlex, machine learning spreads across dozens of topics in several fields, including statistics, engineering and medicine, because the citation network does not respect the boundary of the computer-science field. Conversely, OpenAlex's field~17 (``Computer Science'') contains clusters on educational technology, media studies and physics that no computer scientist would claim, and the ontology of this study deliberately leaves most of them unmapped (Appendix~A). The practical consequence is that the apparent size of ``AI'' depends on which boundary is used: a venue-based definition (DBLP streams of NeurIPS, ICML, ICLR, AAAI and the vision and language conferences) gives a different set from a topic-based definition, which gives a different set again from an arXiv-class definition, and the three sets diverge most in the applied sciences, where machine-learning methods are used without the paper belonging to any machine-learning community. The multi-label design of the ontology, and the use of all three signal types with capped weights in equation~\eqref{eq:p2-score}, is the response to this divergence: a work that is an AI paper by every criterion scores high on all of them, while a work that is an AI paper by vocabulary alone is held to a higher evidentiary standard. | ||
| 52 | 52 | ||||
@@ -84,32 +84,32 @@ The propagation pattern was \emph{infrastructural}. The web changed the objects | |||||
| 84 | 84 | ||||
| 85 | The second transformation is usually dated to 2012, when a convolutional network trained on two GPUs reduced the top-5 error on the ImageNet Large Scale Visual Recognition Challenge by a margin that no incremental improvement in hand-designed features could match \citep{krizhevsky2017imagenet,deng2009imagenet,russakovsky2015imagenet}, and when deep networks replaced Gaussian mixtures in the acoustic models of the major speech laboratories at roughly the same time \citep{hinton2012speech}. The methods were not new: the long short-term memory cell dates from 1997 \citep{hochreiter1997lstm}, convolutional networks from the 1980s, and the retrospective of \citet{lecun2015deep} presents the decade's results as the delayed payoff of a thirty-year programme. What was new was the conjunction of labelled data at the scale of ImageNet, general-purpose graphics processors, and a benchmark with a public leaderboard. | 85 | The second transformation is usually dated to 2012, when a convolutional network trained on two GPUs reduced the top-5 error on the ImageNet Large Scale Visual Recognition Challenge by a margin that no incremental improvement in hand-designed features could match \citep{krizhevsky2017imagenet,deng2009imagenet,russakovsky2015imagenet}, and when deep networks replaced Gaussian mixtures in the acoustic models of the major speech laboratories at roughly the same time \citep{hinton2012speech}. The methods were not new: the long short-term memory cell dates from 1997 \citep{hochreiter1997lstm}, convolutional networks from the 1980s, and the retrospective of \citet{lecun2015deep} presents the decade's results as the delayed payoff of a thirty-year programme. What was new was the conjunction of labelled data at the scale of ImageNet, general-purpose graphics processors, and a benchmark with a public leaderboard. | ||
| 86 | 86 | ||||
| 87 | −The subsequent five years produced the architectural vocabulary that the transformer era would inherit. Distributed word representations trained by simple predictive objectives \citep{mikolov2013word2vec} made it natural to treat language as a sequence of dense vectors; the variational autoencoder \citep{kingma2014vae} and the generative adversarial network \citep{goodfellow2014gan} established deep generative modelling as a research programme; sequence-to-sequence learning with recurrent encoders and decoders \citep{sutskever2014seq2seq} and, crucially, the attention mechanism that let a decoder consult every encoder state \citep{bahdanau2015attention} reframed machine translation and then every conditional generation task; residual connections \citep{he2016resnet} removed the practical ceiling on depth. In reinforcement learning, deep Q-networks learned Atari games from pixels \citep{mnih2015dqn}, and the AlphaGo lineage combined deep networks with tree search to defeat professional Go players, first with human games as training data and then without them \citep{silver2016alphago,silver2017alphagozero}, generalising to chess and shogi with a single algorithm \citep{silver2018alphazero} and, in a different formulation, to real-time strategy games \citep{vinyals2019alphastar}. | 87 | +The subsequent five years produced the architectural vocabulary that the transformer era would inherit. Distributed word representations trained by simple predictive objectives \citep{mikolov2013word2vec} made it natural to treat language as a sequence of dense vectors; the variational autoencoder \citep{kingma2013vae} and the generative adversarial network \citep{goodfellow2014gan} established deep generative modelling as a research programme; sequence-to-sequence learning with recurrent encoders and decoders \citep{sutskever2014seq2seq} and, crucially, the attention mechanism that let a decoder consult every encoder state \citep{bahdanau2014attention} reframed machine translation and then every conditional generation task; residual connections \citep{he2016resnet} removed the practical ceiling on depth. In reinforcement learning, deep Q-networks learned Atari games from pixels \citep{mnih2015dqn}, and the AlphaGo lineage combined deep networks with tree search to defeat professional Go players, first with human games as training data and then without them \citep{silver2016alphago,silver2017alphagozero}, generalising to chess and shogi with a single algorithm \citep{silver2018alphazero} and, in a different formulation, to real-time strategy games \citep{vinyals2019alphastar}. | ||
| 88 | 88 | ||||
| 89 | The changes in publication practice were as consequential as the methods. Four deserve to be stated because the post-2022 data will be read against them. | 89 | The changes in publication practice were as consequential as the methods. Four deserve to be stated because the post-2022 data will be read against them. | ||
| 90 | 90 | ||||
| 91 | The first is the \emph{benchmark as the unit of progress}. ImageNet established the pattern in which a public dataset, a fixed metric and a leaderboard define a subfield's agenda; the pattern was reproduced in language understanding, question answering and reinforcement learning, and it made progress legible to outsiders at the cost of concentrating effort on whatever the benchmark measured. The critiques of that culture, which argue that benchmarks are chosen for convenience rather than validity and that leaderboard competition rewards narrow optimisation \citep{dehghani2021lottery,raji2021benchmark}, were written in the transformer era but describe a practice born in this one. | 91 | The first is the \emph{benchmark as the unit of progress}. ImageNet established the pattern in which a public dataset, a fixed metric and a leaderboard define a subfield's agenda; the pattern was reproduced in language understanding, question answering and reinforcement learning, and it made progress legible to outsiders at the cost of concentrating effort on whatever the benchmark measured. The critiques of that culture, which argue that benchmarks are chosen for convenience rather than validity and that leaderboard competition rewards narrow optimisation \citep{dehghani2021lottery,raji2021benchmark}, were written in the transformer era but describe a practice born in this one. | ||
| 92 | 92 | ||||
| 93 | −The second is the \emph{preprint-first, conference-second} publication cycle. Machine-learning results began to appear on arXiv months before, or instead of, their conference versions; the practice spread from a minority to a norm within the decade, and it is the reason the arXiv \texttt{cs} harvest is indispensable to this corpus. The accompanying costs were catalogued at the end of the period: speculation presented as explanation, mathematics as decoration, suggestive language and the conflation of explanation with description \citep{lipton2019troubling}. Concerns about reproducibility were raised at the same time, both in audits of what published papers actually report \citep{gundersen2018reproducibility} and in the organised reproducibility programmes that the major conferences introduced in response \citep{pineau2021reproducibility}. | 93 | +The second is the \emph{preprint-first, conference-second} publication cycle. Machine-learning results began to appear on arXiv months before, or instead of, their conference versions; the practice spread from a minority to a norm within the decade, and it is the reason the arXiv \texttt{cs} harvest is indispensable to this corpus. The accompanying costs were catalogued at the end of the period: speculation presented as explanation, mathematics as decoration, suggestive language and the conflation of explanation with description \citep{lipton2019troubling}. Concerns about reproducibility were raised at the same time, both in audits of what published papers actually report \citep{gundersen2018reproducibility} and in the organised reproducibility programmes that the major conferences introduced in response \citep{pineau2020reproducibility}. | ||
| 94 | 94 | ||||
| 95 | The third is the \emph{open-source framework as a shared instrument}. TensorFlow and PyTorch \citep{abadi2016tensorflow,paszke2019pytorch}, released by Google and Facebook respectively, lowered the cost of entry to deep learning to the point where a method published on arXiv could be reproduced and extended within days; they also standardised the computational substrate on NVIDIA GPUs and, later, on tensor processing units \citep{jouppi2017tpu}, which made compute a measurable input to research. The retrospective analysis of \citet{sevilla2022compute} identifies this period as the start of a ``deep learning era'' in which the training compute of notable systems grew with a doubling time of about six months, far faster than before 2010; \citet{thompson2020limits} argued that such growth is economically and environmentally unsustainable, a theme to which the sustainable-computing category of this study returns. | 95 | The third is the \emph{open-source framework as a shared instrument}. TensorFlow and PyTorch \citep{abadi2016tensorflow,paszke2019pytorch}, released by Google and Facebook respectively, lowered the cost of entry to deep learning to the point where a method published on arXiv could be reproduced and extended within days; they also standardised the computational substrate on NVIDIA GPUs and, later, on tensor processing units \citep{jouppi2017tpu}, which made compute a measurable input to research. The retrospective analysis of \citet{sevilla2022compute} identifies this period as the start of a ``deep learning era'' in which the training compute of notable systems grew with a doubling time of about six months, far faster than before 2010; \citet{thompson2020limits} argued that such growth is economically and environmentally unsustainable, a theme to which the sustainable-computing category of this study returns. | ||
| 96 | 96 | ||||
| 97 | The fourth is the \emph{entry of industry as a producer rather than a consumer of research}. The laboratories of Google, DeepMind, Facebook, Microsoft and Baidu published the decisive results of the period in the academic venues and on arXiv, and their access to data and compute began to separate them from universities. \citet{ahmed2020dedemocratization} documented the growing presence of large technology firms and elite universities at the major AI conferences and the relative decline of other institutions; the later and broader analysis of \citet{ahmed2023industry} found that industry produced an increasing share of the largest models, the most-cited papers and the leading benchmarks. Whether that concentration intensified after 2022 is hypothesis H6 of the protocol. | 97 | The fourth is the \emph{entry of industry as a producer rather than a consumer of research}. The laboratories of Google, DeepMind, Facebook, Microsoft and Baidu published the decisive results of the period in the academic venues and on arXiv, and their access to data and compute began to separate them from universities. \citet{ahmed2020dedemocratization} documented the growing presence of large technology firms and elite universities at the major AI conferences and the relative decline of other institutions; the later and broader analysis of \citet{ahmed2023industry} found that industry produced an increasing share of the largest models, the most-cited papers and the leading benchmarks. Whether that concentration intensified after 2022 is hypothesis H6 of the protocol. | ||
| 98 | 98 | ||||
| 99 | −The propagation pattern of this wave was \emph{methodological} and sequential. Deep learning entered computer vision and speech first, natural language processing second (roughly 2014--2016, with the sequence-to-sequence results), reinforcement learning and robotics third, and then, more slowly and with more resistance, the subfields whose objects are not perceptual: program analysis \citep{allamanis2018survey}, systems (learned index structures and learned scheduling), databases, and scientific computing \citep{raissi2019pinn}. In each case the receiving subfield first treated learning as a tool for a pre-existing problem, and only later, if at all, reorganised its problems around it. That two-stage pattern, tool adoption followed by problem redefinition, is the template against which Part~IX reads the post-2022 movement of methods across the forty categories. | 99 | +The propagation pattern of this wave was \emph{methodological} and sequential. Deep learning entered computer vision and speech first, natural language processing second (roughly 2014--2016, with the sequence-to-sequence results), reinforcement learning and robotics third, and then, more slowly and with more resistance, the subfields whose objects are not perceptual: program analysis \citep{allamanis2019survey}, systems (learned index structures and learned scheduling), databases, and scientific computing \citep{raissi2019pinn}. In each case the receiving subfield first treated learning as a tool for a pre-existing problem, and only later, if at all, reorganised its problems around it. That two-stage pattern, tool adoption followed by problem redefinition, is the template against which Part~IX reads the post-2022 movement of methods across the forty categories. | ||
| 100 | 100 | ||||
| 101 | \subsection{The transformer and pretraining era, 2017--2022} | 101 | \subsection{The transformer and pretraining era, 2017--2022} | ||
| 102 | \label{sec:p2-transformer} | 102 | \label{sec:p2-transformer} | ||
| 103 | 103 | ||||
| 104 | −The third transformation began with an architecture and ended with a change in what a model is for. The Transformer of \citet{vaswani2017attention} replaced recurrence with self-attention in the encoder--decoder framework of neural machine translation; its immediate merit was parallelism in training, and its lasting one was that the same block could be stacked to any depth and applied to any sequence of tokens. Within eighteen months it had been combined with the idea, already present in ELMo's contextual word vectors \citep{peters2018elmo}, that a language model trained on unlabelled text could be \emph{pretrained} once and \emph{fine-tuned} on many tasks: BERT \citep{devlin2019bert}, with its masked-language objective, set new results on every task of the GLUE benchmark \citep{wang2018glue} and on SQuAD \citep{rajpurkar2016squad}, and its successors showed that most of its design choices were less important than the amount of data and training \citep{liu2019roberta,raffel2020t5}. The generative branch of the family, the GPT series of decoder-only models, reached its third member in 2020 with 175 billion parameters and the demonstration that a sufficiently large language model could perform new tasks from a few examples placed in its prompt, without any gradient update \citep{brown2020gpt3}. In-context learning, more than any architectural novelty, is what separates the transformer era from the deep-learning wave: the unit of research shifted from the task-specific model to the general model and the prompt. | 104 | +The third transformation began with an architecture and ended with a change in what a model is for. The Transformer of \citet{vaswani2017attention} replaced recurrence with self-attention in the encoder--decoder framework of neural machine translation; its immediate merit was parallelism in training, and its lasting one was that the same block could be stacked to any depth and applied to any sequence of tokens. Within eighteen months it had been combined with the idea, already present in ELMo's contextual word vectors \citep{peters2018elmo}, that a language model trained on unlabelled text could be \emph{pretrained} once and \emph{fine-tuned} on many tasks: BERT \citep{devlin2019bert}, with its masked-language objective, set new results on every task of the GLUE benchmark \citep{wang2018glue} and on SQuAD \citep{rajpurkar2016squad}, and its successors showed that most of its design choices were less important than the amount of data and training \citep{liu2019roberta,raffel2019t5}. The generative branch of the family, the GPT series of decoder-only models, reached its third member in 2020 with 175 billion parameters and the demonstration that a sufficiently large language model could perform new tasks from a few examples placed in its prompt, without any gradient update \citep{brown2020gpt3}. In-context learning, more than any architectural novelty, is what separates the transformer era from the deep-learning wave: the unit of research shifted from the task-specific model to the general model and the prompt. | ||
| 105 | 105 | ||||
| 106 | −The era had a quantitative theory of its own growth. \citet{kaplan2020scaling} reported that the test loss of autoregressive language models falls as a power law in parameters, data and compute over several orders of magnitude, with the implication that larger models are more sample-efficient; \citet{hoffmann2022chinchilla} corrected the allocation, showing that models of the GPT-3 generation were undertrained relative to their size and that compute-optimal training requires data and parameters to scale together. Between the two papers, laboratories built models of increasing size on the strength of the first law (Gopher, PaLM, OPT and the open multilingual BLOOM \citep{rae2021gopher,chowdhery2022palm,zhang2022opt,bigscience2022bloom}), and the systems research needed to train them, model parallelism, memory-partitioned optimisers and IO-aware attention kernels \citep{shoeybi2019megatron,rajbhandari2020zero,dao2022flashattention}, became a research area in its own right, as did sparse mixtures of experts that decouple parameter count from per-token compute \citep{fedus2022switch}. The claim that certain abilities appear discontinuously with scale \citep{wei2022emergent} became one of the era's most cited and most contested propositions; the companion observation that prompting a model to write out intermediate reasoning improves its answers on arithmetic and symbolic tasks \citep{wei2022cot} established prompting as a method. New benchmarks were built at the scale of the models: a fifty-seven-subject examination of knowledge \citep{hendrycks2021mmlu}, a harder successor to GLUE \citep{wang2019superglue}, and a collaborative suite of more than two hundred tasks explicitly designed to resist saturation \citep{srivastava2023bigbench}. | 106 | +The era had a quantitative theory of its own growth. \citet{kaplan2020scaling} reported that the test loss of autoregressive language models falls as a power law in parameters, data and compute over several orders of magnitude, with the implication that larger models are more sample-efficient; \citet{hoffmann2022chinchilla} corrected the allocation, showing that models of the GPT-3 generation were undertrained relative to their size and that compute-optimal training requires data and parameters to scale together. Between the two papers, laboratories built models of increasing size on the strength of the first law (Gopher, PaLM, OPT and the open multilingual BLOOM \citep{rae2021gopher,chowdhery2022palm,zhang2022opt,bigscience2022bloom}), and the systems research needed to train them, model parallelism, memory-partitioned optimisers and IO-aware attention kernels \citep{shoeybi2019megatron,rajbhandari2019zero,dao2022flashattention}, became a research area in its own right, as did sparse mixtures of experts that decouple parameter count from per-token compute \citep{fedus2021switch}. The claim that certain abilities appear discontinuously with scale \citep{wei2022emergent} became one of the era's most cited and most contested propositions; the companion observation that prompting a model to write out intermediate reasoning improves its answers on arithmetic and symbolic tasks \citep{wei2022cot} established prompting as a method. New benchmarks were built at the scale of the models: a fifty-seven-subject examination of knowledge \citep{hendrycks2020mmlu}, a harder successor to GLUE \citep{wang2019superglue}, and a collaborative suite of more than two hundred tasks explicitly designed to resist saturation \citep{srivastava2022bigbench}. | ||
| 107 | 107 | ||||
| 108 | −The transformer did not remain a language architecture. Vision transformers showed that image patches could be treated as tokens and that, with enough data, the inductive bias of convolution was dispensable \citep{dosovitskiy2021vit}; self-supervised objectives, contrastive \citep{chen2020simclr}, self-distilled \citep{caron2021dino} and masked \citep{he2022mae}, removed the dependence on labels. Contrastive language--image pretraining aligned the two modalities in a shared embedding space and produced zero-shot classifiers from natural-language prompts \citep{radford2021clip}; in the same months, autoregressive text-to-image generation \citep{ramesh2021dalle} and then diffusion models, a decade-old idea \citep{sohl2015diffusion} made practical by denoising objectives \citep{ho2020ddpm}, score-based formulations \citep{song2021score} and architectural refinements that overtook adversarial models \citep{dhariwal2021beats}, converged on text-conditioned image synthesis at photographic quality \citep{ramesh2022dalle2,saharia2022imagen}. The latent-diffusion formulation \citep{rombach2022ldm}, released with open weights in the summer of 2022 under the name Stable Diffusion, is the generative-AI counterpart of ChatGPT: a system whose public availability, more than its technical novelty, changed who could use it. Speech followed the self-supervised path \citep{baevski2020wav2vec}; few-shot visual language models \citep{alayrac2022flamingo} and a single network trained across text, images, games and robot control \citep{reed2022gato} prefigured the multimodal and agentic systems of the following period. In structural biology, a transformer-based system solved, for most single-chain proteins, a problem that had resisted fifty years of effort \citep{jumper2021alphafold}, and was quickly joined by an independent three-track network \citep{baek2021rosettafold}. | 108 | +The transformer did not remain a language architecture. Vision transformers showed that image patches could be treated as tokens and that, with enough data, the inductive bias of convolution was dispensable \citep{dosovitskiy2020vit}; self-supervised objectives, contrastive \citep{chen2020simclr}, self-distilled \citep{caron2021dino} and masked \citep{he2022mae}, removed the dependence on labels. Contrastive language--image pretraining aligned the two modalities in a shared embedding space and produced zero-shot classifiers from natural-language prompts \citep{radford2021clip}; in the same months, autoregressive text-to-image generation \citep{ramesh2021dalle} and then diffusion models, a decade-old idea \citep{sohl2015diffusion} made practical by denoising objectives \citep{ho2020ddpm}, score-based formulations \citep{song2020score} and architectural refinements that overtook adversarial models \citep{dhariwal2021beats}, converged on text-conditioned image synthesis at photographic quality \citep{ramesh2022dalle2,saharia2022imagen}. The latent-diffusion formulation \citep{rombach2022ldm}, released with open weights in the summer of 2022 under the name Stable Diffusion, is the generative-AI counterpart of ChatGPT: a system whose public availability, more than its technical novelty, changed who could use it. Speech followed the self-supervised path \citep{baevski2020wav2vec}; few-shot visual language models \citep{alayrac2022flamingo} and a single network trained across text, images, games and robot control \citep{reed2022gato} prefigured the multimodal and agentic systems of the following period. In structural biology, a transformer-based system solved, for most single-chain proteins, a problem that had resisted fifty years of effort \citep{jumper2021alphafold}, and was quickly joined by an independent three-track network \citep{baek2021rosettafold}. | ||
| 109 | 109 | ||||
| 110 | Two further lines of work prepared the specific object that appeared in November 2022. Models trained on public code could synthesise functions from docstrings \citep{chen2021codex}, which turned software engineering into an early adopter of large models and created the first commercial product built on one. And the problem of making a language model do what a user asks, rather than what the pretraining distribution predicts, was addressed by reinforcement learning from human preferences, a method developed for control tasks \citep{christiano2017rlhf}, applied to summarisation \citep{stiennon2020summarize}, and then used to fine-tune GPT-3 into InstructGPT \citep{ouyang2022instructgpt}, whose 1.3-billion-parameter version was preferred by raters to the 175-billion-parameter base model. The same method, framed around helpfulness and harmlessness, was being developed in parallel elsewhere \citep{bai2022hh}. Retrieval-augmented generation, which conditions a generator on documents retrieved from an external index, was proposed in the same period \citep{lewis2020rag} and would become the standard answer to the models' factual unreliability. | 110 | Two further lines of work prepared the specific object that appeared in November 2022. Models trained on public code could synthesise functions from docstrings \citep{chen2021codex}, which turned software engineering into an early adopter of large models and created the first commercial product built on one. And the problem of making a language model do what a user asks, rather than what the pretraining distribution predicts, was addressed by reinforcement learning from human preferences, a method developed for control tasks \citep{christiano2017rlhf}, applied to summarisation \citep{stiennon2020summarize}, and then used to fine-tune GPT-3 into InstructGPT \citep{ouyang2022instructgpt}, whose 1.3-billion-parameter version was preferred by raters to the 175-billion-parameter base model. The same method, framed around helpfulness and harmlessness, was being developed in parallel elsewhere \citep{bai2022hh}. Retrieval-augmented generation, which conditions a generator on documents retrieved from an external index, was proposed in the same period \citep{lewis2020rag} and would become the standard answer to the models' factual unreliability. | ||
| 111 | 111 | ||||
| 112 | −The era also produced its own critique. The report that coined the term ``foundation model'' argued that the homogenisation of research around a few pretrained models was a change in the structure of the field, with risks that were social and institutional as much as technical \citep{bommasani2021foundation}; the ``stochastic parrots'' paper questioned whether ever-larger language models were a sound research direction at all, on environmental, documentary and epistemic grounds \citep{bender2021parrots}. The environmental argument had been quantified for NLP models by \citet{strubell2019energy}, contested and refined for the largest models by \citet{patterson2021carbon}, and turned into a research agenda by \citet{schwartz2020green}; the carbon footprint of a 176-billion-parameter model was measured end to end by \citet{luccioni2023bloom}. | 112 | +The era also produced its own critique. The report that coined the term ``foundation model'' argued that the homogenisation of research around a few pretrained models was a change in the structure of the field, with risks that were social and institutional as much as technical \citep{bommasani2021foundation}; the ``stochastic parrots'' paper questioned whether ever-larger language models were a sound research direction at all, on environmental, documentary and epistemic grounds \citep{bender2021parrots}. The environmental argument had been quantified for NLP models by \citet{strubell2019energy}, contested and refined for the largest models by \citet{patterson2021carbon}, and turned into a research agenda by \citet{schwartz2020green}; the carbon footprint of a 176-billion-parameter model was measured end to end by \citet{luccioni2022bloom}. | ||
| 113 | 113 | ||||
| 114 | What changed in publication practice during this era can be summarised in three observations. The leading models were described in \emph{technical reports} rather than in peer-reviewed papers, often with hundreds of authors, sometimes without the training data, the code, or the weights; the preprint became the version of record for the most consequential work in the discipline. The infrastructure for sharing models became an institution in its own right: the Transformers library \citep{wolf2020transformers} and the model hub built around it made the pretrained model, not the paper, the thing that propagated. And the compute required to participate at the frontier grew beyond the reach of universities, so that the concentration documented by \citet{ahmed2023industry} is, in this era, a concentration of capability rather than merely of attention. | 114 | What changed in publication practice during this era can be summarised in three observations. The leading models were described in \emph{technical reports} rather than in peer-reviewed papers, often with hundreds of authors, sometimes without the training data, the code, or the weights; the preprint became the version of record for the most consequential work in the discipline. The infrastructure for sharing models became an institution in its own right: the Transformers library \citep{wolf2020transformers} and the model hub built around it made the pretrained model, not the paper, the thing that propagated. And the compute required to participate at the frontier grew beyond the reach of universities, so that the concentration documented by \citet{ahmed2023industry} is, in this era, a concentration of capability rather than merely of attention. | ||
| 115 | 115 | ||||
@@ -152,7 +152,7 @@ The first is the dominance of the preprint. By 2022 the arXiv was the first and | |||||
| 152 | 152 | ||||
| 153 | The second is the conference calendar. The major venues of every area have fixed annual deadlines, and preprint postings cluster in the weeks before and after them; monthly series of computer-science output are therefore strongly seasonal, with a pattern that differs by area because the calendars differ. Seasonality is not a nuisance to be removed once and forgotten; a change in the \emph{amplitude} of seasonality, for instance because a venue grew or because preprints began to be posted without regard to deadlines, is itself a structural change, and the seasonal component $\mathbf{s}_c(t)$ of equation~\eqref{eq:p2-its} is modelled separately for the baseline and the research period for that reason. | 153 | The second is the conference calendar. The major venues of every area have fixed annual deadlines, and preprint postings cluster in the weeks before and after them; monthly series of computer-science output are therefore strongly seasonal, with a pattern that differs by area because the calendars differ. Seasonality is not a nuisance to be removed once and forgotten; a change in the \emph{amplitude} of seasonality, for instance because a venue grew or because preprints began to be posted without regard to deadlines, is itself a structural change, and the seasonal component $\mathbf{s}_c(t)$ of equation~\eqref{eq:p2-its} is modelled separately for the baseline and the research period for that reason. | ||
| 154 | 154 | ||||
| 155 | −The third is the state of peer review. The volume of submissions to the leading machine-learning conferences had grown by an order of magnitude in a decade; reviewer pools had not. The debate about whether conferences remained an adequate venue of record \citep{vardi2009conferences,fortnow2009growup} was, by 2022, less about prestige than about capacity, and the reproducibility programmes of \citet{pineau2021reproducibility} were one institutional response. The relevance to this study is twofold. Reviewing capacity constrains the number of works that can appear in venue-defined categories, so that growth under capacity constraints shows up as preprint growth and as the founding of new venues rather than as growth in existing streams; and the reviews themselves became, after 2022, one of the first places where the use of language models was measured \citep{liang2024monitoring}. | 155 | +The third is the state of peer review. The volume of submissions to the leading machine-learning conferences had grown by an order of magnitude in a decade; reviewer pools had not. The debate about whether conferences remained an adequate venue of record \citep{vardi2009conferences,fortnow2009growup} was, by 2022, less about prestige than about capacity, and the reproducibility programmes of \citet{pineau2020reproducibility} were one institutional response. The relevance to this study is twofold. Reviewing capacity constrains the number of works that can appear in venue-defined categories, so that growth under capacity constraints shows up as preprint growth and as the founding of new venues rather than as growth in existing streams; and the reviews themselves became, after 2022, one of the first places where the use of language models was measured \citep{liang2024monitoring}. | ||
| 156 | 156 | ||||
| 157 | The fourth is the asymmetry between what is published and what is released. The model reports of Section~\ref{sec:p2-transformer} describe systems whose weights, data and code are variably available; the artefact-evaluation and badging practices adopted by systems and software-engineering venues during the period had no counterpart at the frontier of AI, where the most consequential systems were the least documented. The reproducibility indicators of RQ9, which the corpus measures through linked code repositories and data availability statements, are therefore expected to differ sharply by category, and a decline in the AI categories would be consistent with the literature rather than anomalous. | 157 | The fourth is the asymmetry between what is published and what is released. The model reports of Section~\ref{sec:p2-transformer} describe systems whose weights, data and code are variably available; the artefact-evaluation and badging practices adopted by systems and software-engineering venues during the period had no counterpart at the frontier of AI, where the most consequential systems were the least documented. The reproducibility indicators of RQ9, which the corpus measures through linked code repositories and data availability statements, are therefore expected to differ sharply by category, and a decline in the AI categories would be consistent with the literature rather than anomalous. | ||
| 158 | 158 | ||||
@@ -164,7 +164,7 @@ The hypotheses of this study concern not only the growth of AI but the condition | |||||
| 164 | \subsection{Systems, cloud and distributed computing} | 164 | \subsection{Systems, cloud and distributed computing} | ||
| 165 | \label{sec:p2-systems} | 165 | \label{sec:p2-systems} | ||
| 166 | 166 | ||||
| 167 | −The systems literature of the baseline period was organised around the consequences of the warehouse-scale computer \citep{barroso2018datacenter}. Three agendas dominated. The first was the \emph{serverless} abstraction, in which the unit of deployment is a stateless function and the provider manages everything else; the Berkeley view presented it as the next step in the simplification of cloud programming \citep{jonas2019serverless}, while a contemporaneous critique from the database community argued that its restrictions on state, communication and hardware access made it a step backward for data-intensive work \citep{hellerstein2019serverless}. The disagreement was productive: much of the 2019--2022 systems literature consists of attempts to give serverless platforms the state and the locality they lacked. The second agenda was \emph{resource disaggregation}, the separation of compute, memory and storage into independently scaled pools connected by fast networks, an idea with roots in blade-server memory expansion \citep{lim2009disaggregated} that became a central topic of operating-systems research once network latency fell within an order of magnitude of memory latency. The third was \emph{tail latency}: in a service composed of thousands of components, rare slow responses in any of them dominate the user-visible distribution, and the techniques of hedged requests, replication and tied requests described by \citet{dean2013tail} became the standard vocabulary of large-scale service design. | 167 | +The systems literature of the baseline period was organised around the consequences of the warehouse-scale computer \citep{barroso2018datacenter}. Three agendas dominated. The first was the \emph{serverless} abstraction, in which the unit of deployment is a stateless function and the provider manages everything else; the Berkeley view presented it as the next step in the simplification of cloud programming \citep{jonas2019serverless}, while a contemporaneous critique from the database community argued that its restrictions on state, communication and hardware access made it a step backward for data-intensive work \citep{hellerstein2018serverless}. The disagreement was productive: much of the 2019--2022 systems literature consists of attempts to give serverless platforms the state and the locality they lacked. The second agenda was \emph{resource disaggregation}, the separation of compute, memory and storage into independently scaled pools connected by fast networks, an idea with roots in blade-server memory expansion \citep{lim2009disaggregated} that became a central topic of operating-systems research once network latency fell within an order of magnitude of memory latency. The third was \emph{tail latency}: in a service composed of thousands of components, rare slow responses in any of them dominate the user-visible distribution, and the techniques of hedged requests, replication and tied requests described by \citet{dean2013tail} became the standard vocabulary of large-scale service design. | ||
| 168 | 168 | ||||
| 169 | Operating systems research in the period was marked by the consequences of the Spectre and Meltdown disclosures, discussed under security below, which forced kernel designers to reconsider the cost of isolation, and by the growth of in-kernel programmability. Distributed computing as a theory remained focused on consensus, fault tolerance and the coordination of replicated state, with blockchain systems supplying new adversarial models (Section~\ref{sec:p2-security}). The contact with machine learning before 2022 was of two kinds: ``systems for ML'', the training and serving infrastructure of Section~\ref{sec:p2-transformer}, which systems venues absorbed readily, and ``ML for systems'', the use of learned models for scheduling, configuration and resource management, which was an active but minority topic. The ontology treats cloud and edge computing as children of distributed computing precisely because the venue structure does: OSDI, SOSP, EuroSys and NSDI publish all three. | 169 | Operating systems research in the period was marked by the consequences of the Spectre and Meltdown disclosures, discussed under security below, which forced kernel designers to reconsider the cost of isolation, and by the growth of in-kernel programmability. Distributed computing as a theory remained focused on consensus, fault tolerance and the coordination of replicated state, with blockchain systems supplying new adversarial models (Section~\ref{sec:p2-security}). The contact with machine learning before 2022 was of two kinds: ``systems for ML'', the training and serving infrastructure of Section~\ref{sec:p2-transformer}, which systems venues absorbed readily, and ``ML for systems'', the use of learned models for scheduling, configuration and resource management, which was an active but minority topic. The ontology treats cloud and edge computing as children of distributed computing precisely because the venue structure does: OSDI, SOSP, EuroSys and NSDI publish all three. | ||
| 170 | 170 | ||||
@@ -180,18 +180,18 @@ Compilers underwent a parallel reorganisation around machine-learning workloads. | |||||
| 180 | 180 | ||||
| 181 | Two developments characterise theoretical computer science at the baseline, one internal and one at the boundary with machine learning. | 181 | Two developments characterise theoretical computer science at the baseline, one internal and one at the boundary with machine learning. | ||
| 182 | 182 | ||||
| 183 | −The internal development is \emph{fine-grained complexity}: the programme of explaining why polynomial-time problems such as edit distance, all-pairs shortest paths and orthogonal vectors have resisted improvement for decades, by proving that faster algorithms for one would contradict a small number of hardness conjectures, above all the strong exponential-time hypothesis \citep{vassilevska2015hardness,williams2018finegrained}. The programme reorganised algorithms research around conditional lower bounds in the same way that NP-completeness had organised it around unconditional intractability, and its vocabulary is a reliable marker of the complexity category in this corpus. | 183 | +The internal development is \emph{fine-grained complexity}: the programme of explaining why polynomial-time problems such as edit distance, all-pairs shortest paths and orthogonal vectors have resisted improvement for decades, by proving that faster algorithms for one would contradict a small number of hardness conjectures, above all the strong exponential-time hypothesis \citep{vassilevska2015hardness,williams2019finegrained}. The programme reorganised algorithms research around conditional lower bounds in the same way that NP-completeness had organised it around unconditional intractability, and its vocabulary is a reliable marker of the complexity category in this corpus. | ||
| 184 | 184 | ||||
| 185 | The boundary development is the \emph{theory of deep learning}. The empirical observation that networks with enough parameters to memorise random labels nevertheless generalise \citep{zhang2021rethinking} undermined the classical uniform-convergence account of generalisation and prompted new theory: the double-descent curve, which reconciles the classical bias--variance trade-off with the behaviour of overparameterised models \citep{belkin2019reconciling}, and the neural tangent kernel, which characterises the training dynamics of infinitely wide networks \citep{jacot2018ntk}. The machine-learning-theory leaf of the ontology houses this literature, which belongs sociologically to the learning conferences and intellectually to the theory community. Its presence is a reminder that the theory area was in contact with AI long before 2022, through learning theory, optimisation and the complexity of sampling and inference. | 185 | The boundary development is the \emph{theory of deep learning}. The empirical observation that networks with enough parameters to memorise random labels nevertheless generalise \citep{zhang2021rethinking} undermined the classical uniform-convergence account of generalisation and prompted new theory: the double-descent curve, which reconciles the classical bias--variance trade-off with the behaviour of overparameterised models \citep{belkin2019reconciling}, and the neural tangent kernel, which characterises the training dynamics of infinitely wide networks \citep{jacot2018ntk}. The machine-learning-theory leaf of the ontology houses this literature, which belongs sociologically to the learning conferences and intellectually to the theory community. Its presence is a reminder that the theory area was in contact with AI long before 2022, through learning theory, optimisation and the complexity of sampling and inference. | ||
| 186 | 186 | ||||
| 187 | \subsection{Programming languages, compilers and formal verification} | 187 | \subsection{Programming languages, compilers and formal verification} | ||
| 188 | \label{sec:p2-pl} | 188 | \label{sec:p2-pl} | ||
| 189 | 189 | ||||
| 190 | −The programming-languages literature of the baseline is marked by the success of a language. Rust, whose ownership type system gives memory safety without garbage collection \citep{matsakis2014rust}, became the first systems language to be adopted at scale on the strength of a safety guarantee; the RustBelt project supplied the semantic foundation by proving, in a separation-logic framework mechanised in Coq, that the safe fragment is sound and that the unsafe library code on which it rests respects the invariants \citep{jung2017rustbelt}. The result is characteristic of the period's formal-methods culture: a verification effort that follows an engineering artefact rather than preceding it, and that uses a proof assistant as its medium. | 190 | +The programming-languages literature of the baseline is marked by the success of a language. Rust, whose ownership type system gives memory safety without garbage collection \citep{matsakis2014rust}, became the first systems language to be adopted at scale on the strength of a safety guarantee; the RustBelt project supplied the semantic foundation by proving, in a separation-logic framework mechanised in Coq, that the safe fragment is sound and that the unsafe library code on which it rests respects the invariants \citep{jung2018rustbelt}. The result is characteristic of the period's formal-methods culture: a verification effort that follows an engineering artefact rather than preceding it, and that uses a proof assistant as its medium. | ||
| 191 | 191 | ||||
| 192 | Proof assistants themselves reached a new stage. The verified C compiler CompCert \citep{leroy2009compcert} and the verified microkernel seL4 \citep{klein2009sel4} had shown a decade earlier that production software could be proved correct; the Kepler conjecture was formally proved in 2017 after a twelve-year effort \citep{hales2017kepler}; and the Lean theorem prover \citep{demoura2015lean}, re-implemented as a programming language in its fourth version \citep{demoura2021lean4}, acquired a community-maintained mathematical library whose scale and organisation were new in formalised mathematics \citep{mathlib2020}. The relevance to the post-2022 period is direct: Lean and its library became the environment in which language models were later asked to prove theorems, so that formal verification, one of the categories predicted \emph{not} to break in H3, is also one where the contact with AI is most visible in the recent literature. The hypothesis is therefore a sharp one. | 192 | Proof assistants themselves reached a new stage. The verified C compiler CompCert \citep{leroy2009compcert} and the verified microkernel seL4 \citep{klein2009sel4} had shown a decade earlier that production software could be proved correct; the Kepler conjecture was formally proved in 2017 after a twelve-year effort \citep{hales2017kepler}; and the Lean theorem prover \citep{demoura2015lean}, re-implemented as a programming language in its fourth version \citep{demoura2021lean4}, acquired a community-maintained mathematical library whose scale and organisation were new in formalised mathematics \citep{mathlib2020}. The relevance to the post-2022 period is direct: Lean and its library became the environment in which language models were later asked to prove theorems, so that formal verification, one of the categories predicted \emph{not} to break in H3, is also one where the contact with AI is most visible in the recent literature. The hypothesis is therefore a sharp one. | ||
| 193 | 193 | ||||
| 194 | −Software engineering had already begun to absorb learned models before 2022. The survey of \citet{allamanis2018survey} organised a decade of ``big code'' research around the naturalness hypothesis, that source code is as statistically regular as natural language; the CodeXGLUE benchmark collection \citep{lu2021codexglue} standardised evaluation; and the Codex models \citep{chen2021codex} made program synthesis from natural language a product. The software-engineering category is thus expected to show an early and strong AI signal, and its baseline is not an AI-free one. | 194 | +Software engineering had already begun to absorb learned models before 2022. The survey of \citet{allamanis2019survey} organised a decade of ``big code'' research around the naturalness hypothesis, that source code is as statistically regular as natural language; the CodeXGLUE benchmark collection \citep{lu2021codexglue} standardised evaluation; and the Codex models \citep{chen2021codex} made program synthesis from natural language a product. The software-engineering category is thus expected to show an early and strong AI signal, and its baseline is not an AI-free one. | ||
| 195 | 195 | ||||
| 196 | \subsection{Security and cryptography} | 196 | \subsection{Security and cryptography} | ||
| 197 | \label{sec:p2-security} | 197 | \label{sec:p2-security} | ||
@@ -200,12 +200,12 @@ Three events organise the security and cryptography literature of the baseline. | |||||
| 200 | 200 | ||||
| 201 | The second is the standardisation of \emph{post-quantum cryptography}. Shor's algorithm \citep{shor1997algorithms} had made the eventual obsolescence of RSA and elliptic-curve schemes a matter of engineering timelines, and the case for migrating before large quantum computers exist was made to a general scientific audience by \citet{bernstein2017pqc}. The process run by the United States National Institute of Standards and Technology from 2016 selected lattice-based and hash-based schemes, published as federal standards in August 2024 \citep{nist2024fips203,nist2024fips204,nist2024fips205}. The applied-cryptography category of this corpus therefore contains a large and growing body of work whose dynamics are driven by a standards calendar entirely independent of language models, which makes it a useful placebo field. Fully homomorphic encryption, shown to be possible by \citet{gentry2009fhe}, remained the subject of a steady literature on efficiency. | 201 | The second is the standardisation of \emph{post-quantum cryptography}. Shor's algorithm \citep{shor1997algorithms} had made the eventual obsolescence of RSA and elliptic-curve schemes a matter of engineering timelines, and the case for migrating before large quantum computers exist was made to a general scientific audience by \citet{bernstein2017pqc}. The process run by the United States National Institute of Standards and Technology from 2016 selected lattice-based and hash-based schemes, published as federal standards in August 2024 \citep{nist2024fips203,nist2024fips204,nist2024fips205}. The applied-cryptography category of this corpus therefore contains a large and growing body of work whose dynamics are driven by a standards calendar entirely independent of language models, which makes it a useful placebo field. Fully homomorphic encryption, shown to be possible by \citet{gentry2009fhe}, remained the subject of a steady literature on efficiency. | ||
| 202 | 202 | ||||
| 203 | −The third is the maturation of \emph{adversarial machine learning}. The observation that imperceptible perturbations can change a network's output \citep{szegedy2014intriguing} and the attacks and evaluation standards that followed \citep{carlini2017robustness} created, by 2018, a field with ten years of history and a recognised research agenda \citep{biggio2018wild}. The ontology scores adversarial work in both cybersecurity and AI safety, which reflects the baseline state: the problem belongs to both communities and neither owns it. Blockchain systems, finally, entered the period with a systematised research agenda \citep{bonneau2015sok} and left it as a mature if contested area whose publication dynamics follow cryptocurrency markets more than academic calendars. | 203 | +The third is the maturation of \emph{adversarial machine learning}. The observation that imperceptible perturbations can change a network's output \citep{szegedy2013intriguing} and the attacks and evaluation standards that followed \citep{carlini2017robustness} created, by 2018, a field with ten years of history and a recognised research agenda \citep{biggio2018wild}. The ontology scores adversarial work in both cybersecurity and AI safety, which reflects the baseline state: the problem belongs to both communities and neither owns it. Blockchain systems, finally, entered the period with a systematised research agenda \citep{bonneau2015sok} and left it as a mature if contested area whose publication dynamics follow cryptocurrency markets more than academic calendars. | ||
| 204 | 204 | ||||
| 205 | \subsection{Privacy-preserving computation and federated learning} | 205 | \subsection{Privacy-preserving computation and federated learning} | ||
| 206 | \label{sec:p2-privacy} | 206 | \label{sec:p2-privacy} | ||
| 207 | 207 | ||||
| 208 | −Differential privacy, defined by \citet{dwork2006calibrating} and systematised in the monograph of \citet{dwork2014algorithmic}, was by 2018 the accepted formal standard for statistical disclosure control, and its combination with stochastic gradient descent \citep{abadi2016dpsgd} had made private training of deep networks practical at modest cost in accuracy. Federated learning, proposed as a way to train models on data that never leaves the devices that generated it \citep{mcmahan2017fl}, grew in five years into a field with its own open problems in optimisation, systems, privacy and incentives, catalogued in a survey of unusual breadth \citep{kairouz2021fl}. Both categories are keyword-driven in the ontology because neither has a venue of its own; both were already AI-adjacent at the baseline, so that any post-2022 growth they show cannot be read as a new contact with AI. | 208 | +Differential privacy, defined by \citet{dwork2006calibrating} and systematised in the monograph of \citet{dwork2014algorithmic}, was by 2018 the accepted formal standard for statistical disclosure control, and its combination with stochastic gradient descent \citep{abadi2016dpsgd} had made private training of deep networks practical at modest cost in accuracy. Federated learning, proposed as a way to train models on data that never leaves the devices that generated it \citep{mcmahan2016fl}, grew in five years into a field with its own open problems in optimisation, systems, privacy and incentives, catalogued in a survey of unusual breadth \citep{kairouz2021fl}. Both categories are keyword-driven in the ontology because neither has a venue of its own; both were already AI-adjacent at the baseline, so that any post-2022 growth they show cannot be read as a new contact with AI. | ||
| 209 | 209 | ||||
| 210 | \subsection{Data systems, information retrieval and networks} | 210 | \subsection{Data systems, information retrieval and networks} | ||
| 211 | \label{sec:p2-data} | 211 | \label{sec:p2-data} | ||
@@ -226,7 +226,7 @@ Computer graphics underwent a transformation of its own in 2020, when neural rad | |||||
| 226 | 226 | ||||
| 227 | Four leaves of the perception area lie outside the AI area in the ontology while being the historical home of deep learning, and their baseline is therefore the baseline of the deep-learning wave itself. Computer vision had, by 2022, completed the transition from convolutional to attention-based architectures and from supervised to self-supervised pretraining described in Section~\ref{sec:p2-transformer}; its venues (CVPR, ICCV, ECCV) were among the largest in the discipline, and its vocabulary was already that of foundation models. Natural language processing is the subfield in which the transformer era began, and its baseline is paradoxical: the methods that defined the field in 2018, task-specific architectures, feature engineering, and the pipeline of tagging, parsing and semantic analysis, had been displaced by pretrained models before 2022, so that the post-2022 question for NLP is whether the field continued to exist as a distinct community once its central method became everyone's method. The ontology keeps NLP as a task-framed leaf distinct from large language models for that reason, and the ratio of the two is one of the quantities Part~IX reports. | 227 | Four leaves of the perception area lie outside the AI area in the ontology while being the historical home of deep learning, and their baseline is therefore the baseline of the deep-learning wave itself. Computer vision had, by 2022, completed the transition from convolutional to attention-based architectures and from supervised to self-supervised pretraining described in Section~\ref{sec:p2-transformer}; its venues (CVPR, ICCV, ECCV) were among the largest in the discipline, and its vocabulary was already that of foundation models. Natural language processing is the subfield in which the transformer era began, and its baseline is paradoxical: the methods that defined the field in 2018, task-specific architectures, feature engineering, and the pipeline of tagging, parsing and semantic analysis, had been displaced by pretrained models before 2022, so that the post-2022 question for NLP is whether the field continued to exist as a distinct community once its central method became everyone's method. The ontology keeps NLP as a task-framed leaf distinct from large language models for that reason, and the ratio of the two is one of the quantities Part~IX reports. | ||
| 228 | 228 | ||||
| 229 | −Speech and audio computing had followed the same trajectory a few years behind: hybrid acoustic models \citep{hinton2012speech} gave way to end-to-end sequence models and then to self-supervised pretraining on unlabelled audio \citep{baevski2020wav2vec}, and the weakly supervised Whisper models trained on hundreds of thousands of hours of transcribed speech \citep{radford2023whisper} arrived in the same months as ChatGPT. Robotics is the slowest of the four, because physical data are expensive and simulation transfers imperfectly. End-to-end learned visuomotor policies had been demonstrated in 2016 \citep{levine2016visuomotor}, and reinforcement learning in simulation was an established research line, but most of the field's work in the baseline period remained in planning, control and perception for specific platforms. The vision-language-action models of 2023 \citep{brohan2023rt2}, which transfer web-scale pretraining to manipulation, are the point at which the foundation-model recipe reached robotics, and they fall inside the research period; robotics is therefore one of the categories where a late break, rather than one near the reference date, would be the expected signature of the transformation. | 229 | +Speech and audio computing had followed the same trajectory a few years behind: hybrid acoustic models \citep{hinton2012speech} gave way to end-to-end sequence models and then to self-supervised pretraining on unlabelled audio \citep{baevski2020wav2vec}, and the weakly supervised Whisper models trained on hundreds of thousands of hours of transcribed speech \citep{radford2022whisper} arrived in the same months as ChatGPT. Robotics is the slowest of the four, because physical data are expensive and simulation transfers imperfectly. End-to-end learned visuomotor policies had been demonstrated in 2016 \citep{levine2015visuomotor}, and reinforcement learning in simulation was an established research line, but most of the field's work in the baseline period remained in planning, control and perception for specific platforms. The vision-language-action models of 2023 \citep{brohan2023rt2}, which transfer web-scale pretraining to manipulation, are the point at which the foundation-model recipe reached robotics, and they fall inside the research period; robotics is therefore one of the categories where a late break, rather than one near the reference date, would be the expected signature of the transformation. | ||
| 230 | 230 | ||||
| 231 | \subsection{Quantum computing} | 231 | \subsection{Quantum computing} | ||
| 232 | \label{sec:p2-quantum} | 232 | \label{sec:p2-quantum} | ||
@@ -236,9 +236,9 @@ Quantum computing at the baseline is defined by a term coined to describe it. \c | |||||
| 236 | \subsection{Scientific computing, computational biology and sustainable computing} | 236 | \subsection{Scientific computing, computational biology and sustainable computing} | ||
| 237 | \label{sec:p2-science} | 237 | \label{sec:p2-science} | ||
| 238 | 238 | ||||
| 239 | −Scientific computing entered the period through the door of physics-informed learning. Physics-informed neural networks, which impose a partial differential equation as a loss term \citep{raissi2019pinn}, and neural operators, which learn mappings between function spaces rather than between vectors \citep{li2021fno}, created a literature that the review of \citet{karniadakis2021piml} already presented as a field. Computational biology had its own transformation in the baseline window and does not need the post-2022 one to explain its trajectory: the first AlphaFold had won the 2018 structure-prediction assessment \citep{senior2020alphafold}, the second effectively settled the single-chain problem in 2020--2021 \citep{jumper2021alphafold}, protein language models predicted structure from sequence alone at scale \citep{lin2023esm}, and the third AlphaFold extended prediction to complexes with nucleic acids and ligands \citep{abramson2024af3}. The review of \citet{wang2023aiscience} in 2023 took stock of ``AI for science'' across disciplines; the autonomous-scientific-discovery leaf of the ontology, which requires an AI or automation term, is the place where the corpus records the next step, in which models participate in the design of experiments rather than in the analysis of their results. | 239 | +Scientific computing entered the period through the door of physics-informed learning. Physics-informed neural networks, which impose a partial differential equation as a loss term \citep{raissi2019pinn}, and neural operators, which learn mappings between function spaces rather than between vectors \citep{li2020fno}, created a literature that the review of \citet{karniadakis2021piml} already presented as a field. Computational biology had its own transformation in the baseline window and does not need the post-2022 one to explain its trajectory: the first AlphaFold had won the 2018 structure-prediction assessment \citep{senior2020alphafold}, the second effectively settled the single-chain problem in 2020--2021 \citep{jumper2021alphafold}, protein language models predicted structure from sequence alone at scale \citep{lin2023esm}, and the third AlphaFold extended prediction to complexes with nucleic acids and ligands \citep{abramson2024af3}. The review of \citet{wang2023aiscience} in 2023 took stock of ``AI for science'' across disciplines; the autonomous-scientific-discovery leaf of the ontology, which requires an AI or automation term, is the place where the corpus records the next step, in which models participate in the design of experiments rather than in the analysis of their results. | ||
| 240 | 240 | ||||
| 241 | −Sustainable computing, finally, was an established concern before 2022 but a small category. The careful recalibration of global data-centre energy use \citep{masanet2020datacenter} found that efficiency gains had largely offset growth in demand through 2018, a conclusion that the training and serving of large models would later put under pressure; the energy and carbon studies cited in Section~\ref{sec:p2-transformer} \citep{strubell2019energy,schwartz2020green,patterson2021carbon,luccioni2023bloom} connected the concern directly to AI. The category is keyword-driven and is expected to grow; whether its growth is driven by AI workloads or by the broader energy agenda is a question for Part~IX. | 241 | +Sustainable computing, finally, was an established concern before 2022 but a small category. The careful recalibration of global data-centre energy use \citep{masanet2020datacenter} found that efficiency gains had largely offset growth in demand through 2018, a conclusion that the training and serving of large models would later put under pressure; the energy and carbon studies cited in Section~\ref{sec:p2-transformer} \citep{strubell2019energy,schwartz2020green,patterson2021carbon,luccioni2022bloom} connected the concern directly to AI. The category is keyword-driven and is expected to grow; whether its growth is driven by AI workloads or by the broader energy agenda is a question for Part~IX. | ||
| 242 | 242 | ||||
| 243 | Taken together, the baseline descriptions support a conclusion that the empirical parts must not forget: there was no AI-free area in 2022. Every non-AI category had a point of contact with machine learning before ChatGPT, ranging from marginal (complexity theory, post-quantum cryptography) to pervasive (information retrieval, software engineering, computational biology). The post-2022 question is therefore not whether AI \emph{entered} these areas but whether the rate and the kind of contact changed, and the baseline window exists so that the rate can be estimated before the reference date. | 243 | Taken together, the baseline descriptions support a conclusion that the empirical parts must not forget: there was no AI-free area in 2022. Every non-AI category had a point of contact with machine learning before ChatGPT, ranging from marginal (complexity theory, post-quantum cryptography) to pervasive (information retrieval, software engineering, computational biology). The post-2022 question is therefore not whether AI \emph{entered} these areas but whether the rate and the kind of contact changed, and the baseline window exists so that the rate can be estimated before the reference date. | ||
| 244 | 244 | ||||
@@ -255,7 +255,7 @@ For a study of the \emph{research} literature, this distinction has a consequenc | |||||
| 255 | \subsection{The releases before and after} | 255 | \subsection{The releases before and after} | ||
| 256 | \label{sec:p2-timeline} | 256 | \label{sec:p2-timeline} | ||
| 257 | 257 | ||||
| 258 | −ChatGPT was one release in a dense sequence, and the sequence matters because any of its members is a candidate reference date. Table~\ref{tab:p2-timeline} lists the ones whose primary documents could be verified. Within the twelve months before November 2022 came the compute-optimal scaling result \citep{hoffmann2022chinchilla}, the PaLM, OPT and BLOOM models \citep{chowdhery2022palm,zhang2022opt,bigscience2022bloom}, the chain-of-thought and emergent-abilities papers \citep{wei2022cot,wei2022emergent}, the second DALL-E and Imagen \citep{ramesh2022dalle2,saharia2022imagen}, the open release of Stable Diffusion \citep{rombach2022ldm}, the Flamingo and Gato systems \citep{alayrac2022flamingo,reed2022gato}, the Whisper speech model \citep{radford2023whisper}, and the ReAct formulation of language models that interleave reasoning with actions \citep{yao2023react}, posted in October 2022 and the direct ancestor of the agent literature. Within the twelve months after came LLaMA, the first competitive open-weight model family \citep{touvron2023llama}, Toolformer \citep{schick2023toolformer}, the GPT-4 technical report \citep{openai2023gpt4}, an early and much-debated qualitative evaluation of GPT-4 \citep{bubeck2023sparks}, Llama~2 with a permissive licence \citep{touvron2023llama2}, the RT-2 vision-language-action model that transferred web-scale pretraining to robot control \citep{brohan2023rt2}, Gaussian splatting in graphics \citep{kerbl2023gaussian}, the Mamba state-space architecture as the first serious challenger to attention \citep{gu2023mamba}, and the Gemini models \citep{gemini2023}. Later milestones, Llama~3 \citep{grattafiori2024llama3}, AlphaFold~3 \citep{abramson2024af3}, the o1 reasoning models \citep{openai2024o1} and DeepSeek-R1, which reported that reasoning behaviour could be elicited by reinforcement learning with verifiable rewards and released its weights \citep{deepseek2025r1}, fall inside the observation window and are candidate dates for the \emph{later} breaks that the changepoint methods of Part~IV may find. | 258 | +ChatGPT was one release in a dense sequence, and the sequence matters because any of its members is a candidate reference date. Table~\ref{tab:p2-timeline} lists the ones whose primary documents could be verified. Within the twelve months before November 2022 came the compute-optimal scaling result \citep{hoffmann2022chinchilla}, the PaLM, OPT and BLOOM models \citep{chowdhery2022palm,zhang2022opt,bigscience2022bloom}, the chain-of-thought and emergent-abilities papers \citep{wei2022cot,wei2022emergent}, the second DALL-E and Imagen \citep{ramesh2022dalle2,saharia2022imagen}, the open release of Stable Diffusion \citep{rombach2022ldm}, the Flamingo and Gato systems \citep{alayrac2022flamingo,reed2022gato}, the Whisper speech model \citep{radford2022whisper}, and the ReAct formulation of language models that interleave reasoning with actions \citep{yao2022react}, posted in October 2022 and the direct ancestor of the agent literature. Within the twelve months after came LLaMA, the first competitive open-weight model family \citep{touvron2023llama}, Toolformer \citep{schick2023toolformer}, the GPT-4 technical report \citep{openai2023gpt4}, an early and much-debated qualitative evaluation of GPT-4, circulated as a preprint \citep{bubeck2023sparks}, Llama~2 with a permissive licence \citep{touvron2023llama2}, the RT-2 vision-language-action model that transferred web-scale pretraining to robot control \citep{brohan2023rt2}, Gaussian splatting in graphics \citep{kerbl2023gaussian}, the Mamba state-space architecture as the first serious challenger to attention \citep{gu2023mamba}, and the Gemini models \citep{gemini2023}. Later milestones, Llama~3 \citep{grattafiori2024llama3}, AlphaFold~3 \citep{abramson2024af3}, the o1 reasoning models \citep{openai2024o1} and DeepSeek-R1, which reported that reasoning behaviour could be elicited by reinforcement learning with verifiable rewards and released its weights \citep{deepseek2025r1}, fall inside the observation window and are candidate dates for the \emph{later} breaks that the changepoint methods of Part~IV may find. | ||
| 259 | 259 | ||||
| 260 | \begin{table}[htbp] | 260 | \begin{table}[htbp] | ||
| 261 | \centering | 261 | \centering | ||
@@ -270,9 +270,9 @@ Date & Release or result & Document \\ | |||||
| 270 | 2022-04 & PaLM; DALL-E 2; Flamingo & \citet{chowdhery2022palm,ramesh2022dalle2,alayrac2022flamingo}, preprints \\ | 270 | 2022-04 & PaLM; DALL-E 2; Flamingo & \citet{chowdhery2022palm,ramesh2022dalle2,alayrac2022flamingo}, preprints \\ | ||
| 271 | 2022-05 & OPT; Gato; Imagen & \citet{zhang2022opt,reed2022gato,saharia2022imagen}, preprints \\ | 271 | 2022-05 & OPT; Gato; Imagen & \citet{zhang2022opt,reed2022gato,saharia2022imagen}, preprints \\ | ||
| 272 | 2022-06 & Latent diffusion (CVPR); open release as Stable Diffusion in August & \citet{rombach2022ldm} \\ | 272 | 2022-06 & Latent diffusion (CVPR); open release as Stable Diffusion in August & \citet{rombach2022ldm} \\ | ||
| 273 | −2022-10 & ReAct & \citet{yao2023react}, preprint \\ | 273 | +2022-10 & ReAct & \citet{yao2022react}, preprint \\ | ||
| 274 | 2022-11 & BLOOM; ChatGPT (30 November) & \citet{bigscience2022bloom}, preprint; \citet{openai2022chatgpt}, web page \\ | 274 | 2022-11 & BLOOM; ChatGPT (30 November) & \citet{bigscience2022bloom}, preprint; \citet{openai2022chatgpt}, web page \\ | ||
| 275 | −2022-12 & Whisper & \citet{radford2023whisper}, preprint \\ | 275 | +2022-12 & Whisper & \citet{radford2022whisper}, preprint \\ | ||
| 276 | 2023-02 & LLaMA; Toolformer & \citet{touvron2023llama,schick2023toolformer}, preprints \\ | 276 | 2023-02 & LLaMA; Toolformer & \citet{touvron2023llama,schick2023toolformer}, preprints \\ | ||
| 277 | 2023-03 & GPT-4 & \citet{openai2023gpt4}, technical report (preprint) \\ | 277 | 2023-03 & GPT-4 & \citet{openai2023gpt4}, technical report (preprint) \\ | ||
| 278 | 2023-07 & Llama 2; RT-2 & \citet{touvron2023llama2,brohan2023rt2}, preprints \\ | 278 | 2023-07 & Llama 2; RT-2 & \citet{touvron2023llama2,brohan2023rt2}, preprints \\ | ||
@@ -293,12 +293,12 @@ Two features of the table are relevant to identification. The first is that the | |||||
| 293 | 293 | ||||
| 294 | The mandate of this study (CLAUDE.md~\S6.5) states that the November 2022 boundary is a historical reference point and not proof of a causal effect of ChatGPT. The statement is a constraint on interpretation, but it is also a methodological position that can be made precise. | 294 | The mandate of this study (CLAUDE.md~\S6.5) states that the November 2022 boundary is a historical reference point and not proof of a causal effect of ChatGPT. The statement is a constraint on interpretation, but it is also a methodological position that can be made precise. | ||
| 295 | 295 | ||||
| 296 | −Let $N_{c,t}$ denote the number of works assigned to category $c$ in month $t$, and consider the simplest model that the ``post-ChatGPT'' framing suggests, an interrupted time series of the form used in programme evaluation \citep{bernal2017its}: | 296 | +Let $N_{c,t}$ denote the number of works assigned to category $c$ in month $t$, and consider the simplest model that the ``post-ChatGPT'' framing suggests, an interrupted time series of the form used in programme evaluation \citep{bernal2016its}: | ||
| 297 | \begin{equation} | 297 | \begin{equation} | ||
| 298 | \log N_{c,t} \;=\; \alpha_c + \beta_c\, t + \delta_c\, \mathbf{1}[t \geq \tau] + \gamma_c\,(t-\tau)\,\mathbf{1}[t \geq \tau] + \mathbf{s}_c(t) + \varepsilon_{c,t}, | 298 | \log N_{c,t} \;=\; \alpha_c + \beta_c\, t + \delta_c\, \mathbf{1}[t \geq \tau] + \gamma_c\,(t-\tau)\,\mathbf{1}[t \geq \tau] + \mathbf{s}_c(t) + \varepsilon_{c,t}, | ||
| 299 | \label{eq:p2-its} | 299 | \label{eq:p2-its} | ||
| 300 | \end{equation} | 300 | \end{equation} | ||
| 301 | −with $\tau$ fixed at December 2022, $\mathbf{s}_c(t)$ a seasonal component tied to the conference calendar, $\delta_c$ a level shift and $\gamma_c$ a change in trend. Three conditions must hold for $\delta_c$ or $\gamma_c$ to be read as the effect of the event at $\tau$: the pre-period trend must be well specified, so that an acceleration already under way is not attributed to the break; nothing else that affects $N_{c,t}$ may change at $\tau$; and the date $\tau$ must have been chosen independently of the data. The first condition is testable and is tested in Part~IX with pre-period trend diagnostics. The second fails on its face: Table~\ref{tab:p2-timeline} lists a dozen co-occurring events, the compute trends of \citet{sevilla2022compute} were already exponential, and the growth of the arXiv itself, documented month by month on its statistics page \citep{arxiv2026stats}, is a confounder that applies to every category. The third condition is the subtle one. The date was chosen because of what happened afterwards, which is the definition of selecting a break on the outcome; a test of a known break at $\tau$ \citep{bernal2017its} has the wrong size when $\tau$ was in fact selected from a set of candidates, and the appropriate tools are those for a break at an \emph{unknown} date, the supremum-type tests of \citet{andrews1993tests}, the multiple-break estimation of \citet{bai1998breaks,bai2003computation}, and penalised changepoint search \citep{killick2012pelt}, which let the data locate the break and report whether its location is consistent with the reference date. Equation~\eqref{eq:p2-its} is therefore the \emph{descriptive} object of the study, a convenient summary of before and after; the inferential objects are the unknown-break estimates, their confidence sets, and the placebo breaks at other dates in the baseline, as specified in Part~IV. | 301 | +with $\tau$ fixed at December 2022, $\mathbf{s}_c(t)$ a seasonal component tied to the conference calendar, $\delta_c$ a level shift and $\gamma_c$ a change in trend. Three conditions must hold for $\delta_c$ or $\gamma_c$ to be read as the effect of the event at $\tau$: the pre-period trend must be well specified, so that an acceleration already under way is not attributed to the break; nothing else that affects $N_{c,t}$ may change at $\tau$; and the date $\tau$ must have been chosen independently of the data. The first condition is testable and is tested in Part~IX with pre-period trend diagnostics. The second fails on its face: Table~\ref{tab:p2-timeline} lists a dozen co-occurring events, the compute trends of \citet{sevilla2022compute} were already exponential, and the growth of the arXiv itself, documented month by month on its statistics page \citep{arxiv2026stats}, is a confounder that applies to every category. The third condition is the subtle one. The date was chosen because of what happened afterwards, which is the definition of selecting a break on the outcome; a test of a known break at $\tau$ \citep{bernal2016its} has the wrong size when $\tau$ was in fact selected from a set of candidates, and the appropriate tools are those for a break at an \emph{unknown} date, the supremum-type tests of \citet{andrews1993tests}, the multiple-break estimation of \citet{bai1998breaks,bai2003computation}, and penalised changepoint search \citep{killick2012pelt}, which let the data locate the break and report whether its location is consistent with the reference date. Equation~\eqref{eq:p2-its} is therefore the \emph{descriptive} object of the study, a convenient summary of before and after; the inferential objects are the unknown-break estimates, their confidence sets, and the placebo breaks at other dates in the baseline, as specified in Part~IV. | ||
| 302 | 302 | ||||
| 303 | The descriptive use of the date has a positive justification that goes beyond the avoidance of a fallacy. Dividing the window at a date that the whole community recognises makes the comparison between periods legible, and legibility is a scientific virtue when the quantities compared are themselves complicated. The baseline of 2018--2022 is long enough to contain the whole transformer era and the second half of the deep-learning wave, so that the ``before'' is not an AI-free past but a period of fast change in its own right; the research period of 2022--2026 is long enough for publication lags, which in computer science run from a few weeks for a preprint to more than a year for a journal, to have worked through for all but the last months. Any claim of the form ``after November 2022, $X$ increased'' in this manuscript is to be read as a comparison of two periods so defined, with the structural-break analysis as a separate and more demanding question. | 303 | The descriptive use of the date has a positive justification that goes beyond the avoidance of a fallacy. Dividing the window at a date that the whole community recognises makes the comparison between periods legible, and legibility is a scientific virtue when the quantities compared are themselves complicated. The baseline of 2018--2022 is long enough to contain the whole transformer era and the second half of the deep-learning wave, so that the ``before'' is not an AI-free past but a period of fast change in its own right; the research period of 2022--2026 is long enough for publication lags, which in computer science run from a few weeks for a preprint to more than a year for a journal, to have worked through for all but the last months. Any claim of the form ``after November 2022, $X$ increased'' in this manuscript is to be read as a comparison of two periods so defined, with the structural-break analysis as a separate and more demanding question. | ||
| 304 | 304 | ||||
@@ -319,7 +319,7 @@ What this literature does not establish is whether the growth after 2022 differs | |||||
| 319 | \subsection{LLM-assisted writing} | 319 | \subsection{LLM-assisted writing} | ||
| 320 | \label{sec:p2-claim-writing} | 320 | \label{sec:p2-claim-writing} | ||
| 321 | 321 | ||||
| 322 | −The most direct evidence of a change in the literature after 2022 is a change in its language. \citet{liang2024mapping} estimated, with a distributional method that measures the frequency of words whose use the models favour, that the fraction of abstracts in computer science with substantial LLM-modified content rose rapidly from the end of 2022, with computer science leading other fields; the same group found a comparable rise in the reviews of major machine-learning conferences \citep{liang2024monitoring}, and later extended the method to writing across society \citep{liang2025widespread}. \citet{kobak2025delving} obtained the same conclusion for biomedical abstracts by a different route, measuring the excess frequency of words such as ``delves'' and ``underscores'' relative to a pre-2023 baseline, in a study designed to need no training data from the models themselves; earlier and smaller analyses of the same marker words had reached the same result \citep{geng2024writing,gray2024contamination}. Surveys of researchers confirm the adoption that the text analyses infer: a Nature survey of 1{,}600 scientists found wide use of generative tools for writing and coding, with strong concerns about reliability and misuse \citep{vannoorden2023survey}, and the multidisciplinary collection assembled by \citet{dwivedi2023sowhat} and the integrity analysis of \citet{binnashwan2023chatgpt} document the reactions of the research and education communities in the first months. | 322 | +The most direct evidence of a change in the literature after 2022 is a change in its language. In a preprint, \citet{liang2024mapping} estimated, with a distributional method that measures the frequency of words whose use the models favour, that the fraction of abstracts in computer science with substantial LLM-modified content rose rapidly from the end of 2022, with computer science leading other fields; the same group found a comparable rise in the reviews of major machine-learning conferences \citep{liang2024monitoring}, and later extended the method to writing across society \citep{liang2025widespread}; both are cited here in their preprint versions. \citet{kobak2025delving} obtained the same conclusion for biomedical abstracts by a different route, measuring the excess frequency of words such as ``delves'' and ``underscores'' relative to a pre-2023 baseline, in a study designed to need no training data from the models themselves; earlier and smaller preprint analyses of the same marker words had reached the same result \citep{geng2024writing,gray2024contamination}. Surveys of researchers confirm the adoption that the text analyses infer: a Nature survey of 1{,}600 scientists found wide use of generative tools for writing and coding, with strong concerns about reliability and misuse \citep{vannoorden2023survey}, and the multidisciplinary collection assembled by \citet{dwivedi2023sowhat} and the integrity analysis of \citet{binnashwan2023chatgpt} document the reactions of the research and education communities in the first months. | ||
| 323 | 323 | ||||
| 324 | For this study the finding is a measurement problem before it is a result. The classification of Section~\ref{sec:p2-ontology} and the embedding methods of Part~IV both read text, and text that is partly produced by a model trained on the pre-2023 literature is, in a precise sense, less informative about what is new in a paper. Two responses are built into the design. The venue and arXiv signals of equation~\eqref{eq:p2-score} are immune to prose; and the robustness checks of Appendix~D re-estimate the main breaks with classifications that use no keyword signals, so that a break present in the keyword-based series and absent in the venue-based one is flagged as possibly linguistic. | 324 | For this study the finding is a measurement problem before it is a result. The classification of Section~\ref{sec:p2-ontology} and the embedding methods of Part~IV both read text, and text that is partly produced by a model trained on the pre-2023 literature is, in a precise sense, less informative about what is new in a paper. Two responses are built into the design. The venue and arXiv signals of equation~\eqref{eq:p2-score} are immune to prose; and the robustness checks of Appendix~D re-estimate the main breaks with classifications that use no keyword signals, so that a break present in the keyword-based series and absent in the venue-based one is flagged as possibly linguistic. | ||
| 325 | 325 | ||||
@@ -333,7 +333,7 @@ A second body of work concerns not the quantity of research but its quality. \ci | |||||
| 333 | \subsection{Benchmark saturation and the reproducibility of evaluation} | 333 | \subsection{Benchmark saturation and the reproducibility of evaluation} | ||
| 334 | \label{sec:p2-claim-benchmarks} | 334 | \label{sec:p2-claim-benchmarks} | ||
| 335 | 335 | ||||
| 336 | −The benchmark culture of Section~\ref{sec:p2-dl} produced a literature on its own dynamics. \citet{ott2022benchmarks} mapped the creation and saturation of benchmarks across AI tasks and found that many become saturated within a few years of their creation while others are abandoned before saturation; the critiques of \citet{dehghani2021lottery} and \citet{raji2021benchmark} argued that the choice of benchmark is itself an under-examined research decision and that no benchmark can measure general capability. The post-2022 literature extended the concern to \emph{contamination}, the presence of test data in the training corpora of models trained on the web, which makes a benchmark result uninterpretable without knowledge of the training data that the model reports of Table~\ref{tab:p2-timeline} typically withhold. \citet{kapoor2023leakage} documented data leakage as a general cause of irreproducible results in machine-learning-based science, and the agenda of \citet{hendrycks2021unsolved} placed robustness, monitoring and alignment among the open problems of the field. RQ7, which asks whether the period's advances are breakthroughs or incremental benchmark gains, and RQ9, on reproducibility and evaluation standards, are the corpus-level counterparts of these claims. | 336 | +The benchmark culture of Section~\ref{sec:p2-dl} produced a literature on its own dynamics. In a preprint, \citet{ott2022benchmarks} mapped the creation and saturation of benchmarks across AI tasks and found that many become saturated within a few years of their creation while others are abandoned before saturation; the critiques of \citet{dehghani2021lottery} and \citet{raji2021benchmark} argued that the choice of benchmark is itself an under-examined research decision and that no benchmark can measure general capability. The post-2022 literature extended the concern to \emph{contamination}, the presence of test data in the training corpora of models trained on the web, which makes a benchmark result uninterpretable without knowledge of the training data that the model reports of Table~\ref{tab:p2-timeline} typically withhold. \citet{kapoor2023leakage} documented data leakage as a general cause of irreproducible results in machine-learning-based science, and the agenda of \citet{hendrycks2021unsolved} placed robustness, monitoring and alignment among the open problems of the field. RQ7, which asks whether the period's advances are breakthroughs or incremental benchmark gains, and RQ9, on reproducibility and evaluation standards, are the corpus-level counterparts of these claims. | ||
| 337 | 337 | ||||
| 338 | \subsection{Concentration, disruption and the pace of science} | 338 | \subsection{Concentration, disruption and the pace of science} | ||
| 339 | \label{sec:p2-claim-pace} | 339 | \label{sec:p2-claim-pace} | ||
| 340 | 340 | ||||