1@misc{lu2024aiscientist,2 author = {Chris Lu and Cong Lu and Robert Tjarko Lange and Jakob Foerster and Jeff Clune and David Ha},3 title = {{The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery}},4 year = {2024},5 howpublished = {arXiv preprint arXiv:2408.06292},6 eprint = {2408.06292},7 archivePrefix = {arXiv}8}9% verified via https://export.arxiv.org/api/query?id_list=2408.06292 on 2026-10-30 (comment placed after the first entry so that no % line precedes it)1011% verified via https://api.crossref.org/works/10.1038/s41586-026-10265-5 on 2026-10-3012@article{lu2026aiscientist,13 author = {Lu, Chris and Lu, Cong and Lange, Robert Tjarko and Yamada, Yutaro and Hu, Shengran and Foerster, Jakob and Ha, David and Clune, Jeff},14 title = {{Towards end-to-end automation of AI research}},15 year = {2026},16 journal = {Nature},17 volume = {651},18 number = {8107},19 pages = {914--919},20 doi = {10.1038/s41586-026-10265-5}21}2223% verified via https://export.arxiv.org/api/query?id_list=2504.08066 on 2026-10-3024@misc{yamada2025aiscientistv2,25 author = {Yutaro Yamada and Robert Tjarko Lange and Cong Lu and Shengran Hu and Chris Lu and Jakob Foerster and Jeff Clune and David Ha},26 title = {{The AI Scientist-v2: Workshop-Level Automated Scientific Discovery via Agentic Tree Search}},27 year = {2025},28 howpublished = {arXiv preprint arXiv:2504.08066},29 eprint = {2504.08066},30 archivePrefix = {arXiv}31}3233% verified via https://api.crossref.org/works/10.18653/v1/2025.findings-emnlp.320 on 2026-10-3034@inproceedings{schmidgall2025agentlab,35 author = {Schmidgall, Samuel and Su, Yusheng and Wang, Ze and Sun, Ximeng and Wu, Jialian and Yu, Xiaodong and Liu, Jiang and Moor, Michael and Liu, Zicheng and Barsoum, Emad},36 title = {{Agent Laboratory: Using LLM Agents as Research Assistants}},37 year = {2025},38 booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2025},39 pages = {5977--6043},40 doi = {10.18653/v1/2025.findings-emnlp.320},41 eprint = {2501.04227},42 archivePrefix = {arXiv}43}4445% verified via https://api.crossref.org/works/10.1038/s41586-023-06792-0 on 2026-10-3046@article{boiko2023coscientist,47 author = {Boiko, Daniil A. and MacKnight, Robert and Kline, Ben and Gomes, Gabe},48 title = {{Autonomous chemical research with large language models}},49 year = {2023},50 journal = {Nature},51 volume = {624},52 number = {7992},53 pages = {570--578},54 doi = {10.1038/s41586-023-06792-0}55}5657% verified via https://api.crossref.org/works/10.1038/s42256-024-00832-8 on 2026-10-3058@article{bran2024chemcrow,59 author = {M. Bran, Andres and Cox, Sam and Schilter, Oliver and Baldassari, Carlo and White, Andrew D. and Schwaller, Philippe},60 title = {{Augmenting large language models with chemistry tools}},61 year = {2024},62 journal = {Nature Machine Intelligence},63 volume = {6},64 number = {5},65 pages = {525--535},66 doi = {10.1038/s42256-024-00832-8}67}6869% verified via https://api.crossref.org/works/10.1056/AIoa2400555 on 2026-10-3070@article{ifargan2025datatopaper,71 author = {Ifargan, Tal and Hafner, Lukas and Kern, Maor and Alcalay, Ori and Kishony, Roy},72 title = {{Autonomous LLM-Driven Research — from Data to Human-Verifiable Research Papers}},73 year = {2025},74 journal = {NEJM AI},75 volume = {2},76 number = {1},77 doi = {10.1056/AIoa2400555},78 eprint = {2404.17605},79 archivePrefix = {arXiv}80}8182% verified via https://api.crossref.org/works/10.1038/s41586-026-10644-y on 2026-10-3083@article{gottweis2026coscientist,84 author = {Gottweis, Juraj and Weng, Wei-Hung and Daryin, Alexander and Tu, Tao and Sirkovic, Petar and Myaskovsky, Artiom and Glowaty, Grzegorz and Weissenberger, Felix and Orlandi, Alessio and Popovici, Dan and Palepu, Anil and Rong, Keran and Tanno, Ryutaro and Saab, Khaled and Zhang, Fan and Blum, Jacob and Carroll, Andrew and Kulkarni, Kavita and Tomašev, Nenad and Zverinski, Dina and Rendulic, Ivor and Vedadi, Elahe and Hasler, Florian and Rimanic, Luka and Boia, Marina and Budiselic, Ivan and Feinstein, Ben and Bellaiche, Mathias and Sheffer, Tom and Freyberg, Jan and Ratcliff, Jeremy and Bertolli, Ottavia and Chou, Katherine and Hassidim, Avinatan and Gokturk, Burak and Vahdat, Amin and Guan, Yuan and Dhillon, Vikram and Vaishnav, Eeshit Dhaval and Lee, Byron and Costa, Tiago R. D. and Penadés, José R. and Peltz, Gary and Matias, Yossi and Manyika, James and Hassabis, Demis and Xu, Yunhan and Kohli, Pushmeet and Pawlosky, Annalisa and Karthikesalingam, Alan and Natarajan, Vivek},85 title = {{Accelerating scientific discovery with Co-Scientist}},86 year = {2026},87 journal = {Nature},88 volume = {655},89 number = {8122},90 pages = {487--496},91 doi = {10.1038/s41586-026-10644-y},92 eprint = {2502.18864},93 archivePrefix = {arXiv}94}9596% verified via https://api.crossref.org/works/10.1038/s41586-025-09442-9 on 2026-10-3097@article{swanson2025virtuallab,98 author = {Swanson, Kyle and Wu, Wesley and Bulaong, Nash L. and Pak, John E. and Zou, James},99 title = {{The Virtual Lab of AI agents designs new SARS-CoV-2 nanobodies}},100 year = {2025},101 journal = {Nature},102 volume = {646},103 number = {8085},104 pages = {716--723},105 doi = {10.1038/s41586-025-09442-9}106}107108% verified via https://export.arxiv.org/api/query?id_list=2511.02824 on 2026-10-30109@misc{mitchener2025kosmos,110 author = {Ludovico Mitchener and Angela Yiu and Benjamin Chang and Mathieu Bourdenx and Tyler Nadolski and Arvis Sulovari and Eric C. Landsness and Daniel L. Barabasi and Siddharth Narayanan and Nicky Evans and Shriya Reddy and Martha Foiani and Aizad Kamal and Leah P. Shriver and Fang Cao and Asmamaw T. Wassie and Jon M. Laurent and Edwin Melville-Green and Mayk Caldas and Albert Bou and Kaleigh F. Roberts and Sladjana Zagorac and Timothy C. Orr and Miranda E. Orr and Kevin J. Zwezdaryk and Ali E. Ghareeb and Laurie McCoy and Bruna Gomes and Euan A. Ashley and Karen E. Duff and Tonio Buonassisi and Tom Rainforth and Randall J. Bateman and Michael Skarlinski and Samuel G. Rodriques and Michaela M. Hinks and Andrew D. White},111 title = {{Kosmos: An AI Scientist for Autonomous Discovery}},112 year = {2025},113 howpublished = {arXiv preprint arXiv:2511.02824},114 eprint = {2511.02824},115 archivePrefix = {arXiv}116}117118% verified via https://export.arxiv.org/api/query?id_list=2509.08713 on 2026-10-30119@misc{luo2025pitfalls,120 author = {Ziming Luo and Atoosa Kasirzadeh and Nihar B. Shah},121 title = {{The More You Automate, the Less You See: Hidden Pitfalls of AI Scientist Systems}},122 year = {2025},123 howpublished = {arXiv preprint arXiv:2509.08713},124 eprint = {2509.08713},125 archivePrefix = {arXiv}126}127128% verified via https://api.crossref.org/works/10.1145/3769733.3769747 on 2026-10-30129@article{beel2025sakana,130 author = {Beel, Joeran and Kan, Min-Yen and Baumgart, Moritz},131 title = {{Evaluating Sakana's AI Scientist: Bold Claims, Mixed Results, and a Promising Future?}},132 year = {2025},133 journal = {ACM SIGIR Forum},134 volume = {59},135 number = {1},136 pages = {1--20},137 doi = {10.1145/3769733.3769747},138 eprint = {2502.14297},139 archivePrefix = {arXiv}140}141142% verified via https://export.arxiv.org/api/query?id_list=2607.12301 on 2026-10-30143@misc{luo2026xscientist,144 author = {Jixiang Luo},145 title = {{XScientist: A Git-Like Research Protocol for Long-Running Autonomous Scientific Discovery}},146 year = {2026},147 howpublished = {arXiv preprint arXiv:2607.12301},148 eprint = {2607.12301},149 archivePrefix = {arXiv}150}151152% verified via https://export.arxiv.org/api/query?id_list=2608.05179 on 2026-10-30153@misc{ding2026verificationgap,154 author = {Tianyu Ding and Aditya Nannapaneni and Bingfan Liu and Ling Zhang},155 title = {{Autonomous Research Agents: A Survey of AI Scientists and the Verification Gap}},156 year = {2026},157 howpublished = {arXiv preprint arXiv:2608.05179},158 eprint = {2608.05179},159 archivePrefix = {arXiv}160}161162% verified via https://export.arxiv.org/api/query?id_list=2410.05080 on 2026-10-30163@inproceedings{chen2025scienceagentbench,164 author = {Ziru Chen and Shijie Chen and Yuting Ning and Qianheng Zhang and Boshi Wang and Botao Yu and Yifei Li and Zeyi Liao and Chen Wei and Zitong Lu and Vishal Dey and Mingyi Xue and Frazier N. Baker and Benjamin Burns and Daniel Adu-Ampratwum and Xuhui Huang and Xia Ning and Song Gao and Yu Su and Huan Sun},165 title = {{ScienceAgentBench: Toward Rigorous Assessment of Language Agents for Data-Driven Scientific Discovery}},166 year = {2024},167 booktitle = {ICLR 2025},168 eprint = {2410.05080},169 archivePrefix = {arXiv}170}171172% verified via https://export.arxiv.org/api/query?id_list=2407.01725 on 2026-10-30173@inproceedings{majumder2025discoverybench,174 author = {Bodhisattwa Prasad Majumder and Harshit Surana and Dhruv Agarwal and Bhavana Dalvi Mishra and Abhijeetsingh Meena and Aryan Prakhar and Tirth Vora and Tushar Khot and Ashish Sabharwal and Peter Clark},175 title = {{DiscoveryBench: Towards Data-Driven Discovery with Large Language Models}},176 year = {2024},177 booktitle = {ICLR 2025},178 eprint = {2407.01725},179 archivePrefix = {arXiv}180}181182% verified via https://export.arxiv.org/api/query?id_list=2409.07703 on 2026-10-30183@inproceedings{jing2025dsbench,184 author = {Liqiang Jing and Zhehui Huang and Xiaoyang Wang and Wenlin Yao and Wenhao Yu and Kaixin Ma and Hongming Zhang and Xinya Du and Dong Yu},185 title = {{DSBench: How Far Are Data Science Agents from Becoming Data Science Experts?}},186 year = {2024},187 booktitle = {ICLR 2025},188 eprint = {2409.07703},189 archivePrefix = {arXiv}190}191192% verified via https://export.arxiv.org/api/query?id_list=2310.03302 on 2026-10-30193@inproceedings{huang2024mlagentbench,194 author = {Qian Huang and Jian Vora and Percy Liang and Jure Leskovec},195 title = {{MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation}},196 year = {2023},197 booktitle = {ICML 2024},198 eprint = {2310.03302},199 archivePrefix = {arXiv}200}201202% verified via https://export.arxiv.org/api/query?id_list=2408.09667 on 2026-10-30203@inproceedings{gu2024blade,204 author = {Ken Gu and Ruoxi Shang and Ruien Jiang and Keying Kuang and Richard-John Lin and Donghe Lyu and Yue Mao and Youran Pan and Teng Wu and Jiaqian Yu and Yikun Zhang and Tianmai M. Zhang and Lanyi Zhu and Mike A. Merrill and Jeffrey Heer and Tim Althoff},205 title = {{BLADE: Benchmarking Language Model Agents for Data-Driven Science}},206 year = {2024},207 booktitle = {EMNLP 2024},208 eprint = {2408.09667},209 archivePrefix = {arXiv}210}211212% verified via https://export.arxiv.org/api/query?id_list=2310.06770 on 2026-10-30213@inproceedings{jimenez2024swebench,214 author = {Carlos E. Jimenez and John Yang and Alexander Wettig and Shunyu Yao and Kexin Pei and Ofir Press and Karthik Narasimhan},215 title = {{SWE-bench: Can Language Models Resolve Real-World GitHub Issues?}},216 year = {2023},217 booktitle = {ICLR 2024},218 eprint = {2310.06770},219 archivePrefix = {arXiv}220}221222% verified via https://export.arxiv.org/api/query?id_list=2410.07095 on 2026-10-30223@inproceedings{chan2025mlebench,224 author = {Jun Shern Chan and Neil Chowdhury and Oliver Jaffe and James Aung and Dane Sherburn and Evan Mays and Giulio Starace and Kevin Liu and Leon Maksin and Tejal Patwardhan and Lilian Weng and Aleksander Mądry},225 title = {{MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering}},226 year = {2024},227 booktitle = {ICLR 2025},228 eprint = {2410.07095},229 archivePrefix = {arXiv}230}231232% verified via https://export.arxiv.org/api/query?id_list=2409.11363 on 2026-10-30233@article{siegel2024corebench,234 author = {Zachary S. Siegel and Sayash Kapoor and Nitya Nadgir and Benedikt Stroebl and Arvind Narayanan},235 title = {{CORE-Bench: Fostering the Credibility of Published Research Through a Computational Reproducibility Agent Benchmark}},236 year = {2024},237 journal = {Transactions on Machine Learning Research},238 eprint = {2409.11363},239 archivePrefix = {arXiv}240}241242% verified via https://export.arxiv.org/api/query?id_list=2504.01848 on 2026-10-30243@inproceedings{starace2025paperbench,244 author = {Giulio Starace and Oliver Jaffe and Dane Sherburn and James Aung and Jun Shern Chan and Leon Maksin and Rachel Dias and Evan Mays and Benjamin Kinsella and Wyatt Thompson and Johannes Heidecke and Amelia Glaese and Tejal Patwardhan},245 title = {{PaperBench: Evaluating AI's Ability to Replicate AI Research}},246 year = {2025},247 booktitle = {ICML 2025},248 eprint = {2504.01848},249 archivePrefix = {arXiv}250}251252% verified via https://api.crossref.org/works/10.1093/bioinformatics/bts480 on 2026-10-30253@article{koster2012snakemake,254 author = {Köster, Johannes and Rahmann, Sven},255 title = {{Snakemake—a scalable bioinformatics workflow engine}},256 year = {2012},257 journal = {Bioinformatics},258 volume = {28},259 number = {19},260 pages = {2520--2522},261 doi = {10.1093/bioinformatics/bts480}262}263264% verified via https://api.crossref.org/works/10.1038/nbt.3820 on 2026-10-30265@article{ditommaso2017nextflow,266 author = {Di Tommaso, Paolo and Chatzou, Maria and Floden, Evan W and Barja, Pablo Prieto and Palumbo, Emilio and Notredame, Cedric},267 title = {{Nextflow enables reproducible computational workflows}},268 year = {2017},269 journal = {Nature Biotechnology},270 volume = {35},271 number = {4},272 pages = {316--319},273 doi = {10.1038/nbt.3820}274}275276% verified via https://api.crossref.org/works/10.1145/2452376.2452478 on 2026-10-30277@inproceedings{missier2013prov,278 author = {Missier, Paolo and Belhajjame, Khalid and Cheney, James},279 title = {{The W3C PROV family of specifications for modelling provenance metadata}},280 year = {2013},281 booktitle = {Proceedings of the 16th International Conference on Extending Database Technology},282 pages = {773--776},283 doi = {10.1145/2452376.2452478}284}285286% verified via https://api.crossref.org/works/10.1145/2882903.2899401 on 2026-10-30287@inproceedings{chirigati2016reprozip,288 author = {Chirigati, Fernando and Rampin, Rémi and Shasha, Dennis and Freire, Juliana},289 title = {{ReproZip}},290 year = {2016},291 booktitle = {Proceedings of the 2016 International Conference on Management of Data},292 pages = {2085--2088},293 doi = {10.1145/2882903.2899401}294}295296% verified via https://api.crossref.org/works/10.1371/journal.pcbi.1003285 on 2026-10-30297@article{sandve2013tensimplerules,298 author = {Sandve, Geir Kjetil and Nekrutenko, Anton and Taylor, James and Hovig, Eivind},299 title = {{Ten Simple Rules for Reproducible Computational Research}},300 year = {2013},301 journal = {PLoS Computational Biology},302 volume = {9},303 number = {10},304 pages = {e1003285},305 doi = {10.1371/journal.pcbi.1003285}306}307308% verified via https://api.crossref.org/works/10.1073/pnas.1708290115 on 2026-10-30309@article{stodden2018journalpolicy,310 author = {Stodden, Victoria and Seiler, Jennifer and Ma, Zhaokun},311 title = {{An empirical analysis of journal policy effectiveness for computational reproducibility}},312 year = {2018},313 journal = {Proceedings of the National Academy of Sciences},314 volume = {115},315 number = {11},316 pages = {2584--2589},317 doi = {10.1073/pnas.1708290115}318}319320% verified via https://api.crossref.org/works/10.1126/science.1213847 on 2026-10-30321@article{peng2011reproducible,322 author = {Peng, Roger D.},323 title = {{Reproducible Research in Computational Science}},324 year = {2011},325 journal = {Science},326 volume = {334},327 number = {6060},328 pages = {1226--1227},329 doi = {10.1126/science.1213847}330}331332% verified via https://api.crossref.org/works/10.17226/25303 on 2026-10-30333@book{nasem2019reproducibility,334 author = {{National Academies of Sciences, Engineering, and Medicine}},335 title = {{Reproducibility and Replicability in Science}},336 year = {2019},337 publisher = {The National Academies Press},338 address = {Washington, DC},339 doi = {10.17226/25303}340}341342% verified via https://api.crossref.org/works/10.1098/rsta.2020.0069 on 2026-10-30343@article{krafczyk2021reproduction,344 author = {Krafczyk, M. S. and Shi, A. and Bhaskar, A. and Marinov, D. and Stodden, V.},345 title = {{Learning from reproducing computational results: introducing three principles and the <i>Reproduction Package</i>}},346 year = {2021},347 journal = {Philosophical Transactions of the Royal Society A: Mathematical, Physical and Engineering Sciences},348 volume = {379},349 number = {2197},350 doi = {10.1098/rsta.2020.0069}351}352353% verified via https://api.crossref.org/works/10.1038/s41597-022-01143-6 on 2026-10-30354@article{trisovic2022codequality,355 author = {Trisovic, Ana and Lau, Matthew K. and Pasquier, Thomas and Crosas, Mercè},356 title = {{A large-scale study on research code quality and execution}},357 year = {2022},358 journal = {Scientific Data},359 volume = {9},360 number = {1},361 doi = {10.1038/s41597-022-01143-6}362}363364% verified via https://api.crossref.org/works/10.1073/pnas.1708274114 on 2026-10-30365@article{nosek2018preregistration,366 author = {Nosek, Brian A. and Ebersole, Charles R. and DeHaven, Alexander C. and Mellor, David T.},367 title = {{The preregistration revolution}},368 year = {2018},369 journal = {Proceedings of the National Academy of Sciences},370 volume = {115},371 number = {11},372 pages = {2600--2606},373 doi = {10.1073/pnas.1708274114}374}375376% verified via https://api.crossref.org/works/10.1016/j.cortex.2012.12.016 on 2026-10-30377@article{chambers2013registeredreports,378 author = {Chambers, Christopher D.},379 title = {{Registered Reports: A new publishing initiative at Cortex}},380 year = {2013},381 journal = {Cortex},382 volume = {49},383 number = {3},384 pages = {609--610},385 doi = {10.1016/j.cortex.2012.12.016}386}387388% verified via https://api.crossref.org/works/10.1038/s41562-021-01193-7 on 2026-10-30389@article{chambers2022registeredreports,390 author = {Chambers, Christopher D. and Tzavella, Loukia},391 title = {{The past, present and future of Registered Reports}},392 year = {2021},393 journal = {Nature Human Behaviour},394 volume = {6},395 number = {1},396 pages = {29--42},397 doi = {10.1038/s41562-021-01193-7}398}399400% verified via https://api.crossref.org/works/10.1177/0956797611417632 on 2026-10-30401@article{simmons2011falsepositive,402 author = {Simmons, Joseph P. and Nelson, Leif D. and Simonsohn, Uri},403 title = {{False-Positive Psychology}},404 year = {2011},405 journal = {Psychological Science},406 volume = {22},407 number = {11},408 pages = {1359--1366},409 doi = {10.1177/0956797611417632}410}411412% verified via https://api.crossref.org/works/10.1038/s41562-016-0021 on 2026-10-30413@article{munafo2017manifesto,414 author = {Munafò, Marcus R. and Nosek, Brian A. and Bishop, Dorothy V. M. and Button, Katherine S. and Chambers, Christopher D. and Percie du Sert, Nathalie and Simonsohn, Uri and Wagenmakers, Eric-Jan and Ware, Jennifer J. and Ioannidis, John P. A.},415 title = {{A manifesto for reproducible science}},416 year = {2017},417 journal = {Nature Human Behaviour},418 volume = {1},419 number = {1},420 doi = {10.1038/s41562-016-0021}421}422423% verified via https://api.crossref.org/works/10.1511/2014.111.460 on 2026-10-30424@article{gelman2014statisticalcrisis,425 author = {Gelman, Andrew and Loken, Eric},426 title = {{The Statistical Crisis in Science}},427 year = {2014},428 journal = {American Scientist},429 volume = {102},430 number = {6},431 pages = {460},432 doi = {10.1511/2014.111.460}433}434435% verified via https://api.crossref.org/works/10.1126/science.aab2374 on 2026-10-30436@article{nosek2015top,437 author = {Nosek, B. A. and Alter, G. and Banks, G. C. and Borsboom, D. and Bowman, S. D. and Breckler, S. J. and Buck, S. and Chambers, C. D. and Chin, G. and Christensen, G. and Contestabile, M. and Dafoe, A. and Eich, E. and Freese, J. and Glennerster, R. and Goroff, D. and Green, D. P. and Hesse, B. and Humphreys, M. and Ishiyama, J. and Karlan, D. and Kraut, A. and Lupia, A. and Mabry, P. and Madon, T. and Malhotra, N. and Mayo-Wilson, E. and McNutt, M. and Miguel, E. and Paluck, E. Levy and Simonsohn, U. and Soderberg, C. and Spellman, B. A. and Turitto, J. and VandenBos, G. and Vazire, S. and Wagenmakers, E. J. and Wilson, R. and Yarkoni, T.},438 title = {{Promoting an open research culture}},439 year = {2015},440 journal = {Science},441 volume = {348},442 number = {6242},443 pages = {1422--1425},444 doi = {10.1126/science.aab2374}445}446447% verified via https://api.crossref.org/works/10.1177/25152459211007467 on 2026-10-30448@article{scheel2021excess,449 author = {Scheel, Anne M. and Schijen, Mitchell R. M. J. and Lakens, Daniël},450 title = {{An Excess of Positive Results: Comparing the Standard Psychology Literature With Registered Reports}},451 year = {2021},452 journal = {Advances in Methods and Practices in Psychological Science},453 volume = {4},454 number = {2},455 doi = {10.1177/25152459211007467}456}457458% verified via https://api.crossref.org/works/10.1126/science.aac4716 on 2026-10-30459@article{osc2015reproducibility,460 author = {{Open Science Collaboration}},461 title = {{Estimating the reproducibility of psychological science}},462 year = {2015},463 journal = {Science},464 volume = {349},465 number = {6251},466 doi = {10.1126/science.aac4716}467}468469% verified via https://api.crossref.org/works/10.1207/s15327957pspr0203_4 on 2026-10-30470@article{kerr1998harking,471 author = {Kerr, Norbert L.},472 title = {{HARKing: Hypothesizing After the Results are Known}},473 year = {1998},474 journal = {Personality and Social Psychology Review},475 volume = {2},476 number = {3},477 pages = {196--217},478 doi = {10.1207/s15327957pspr0203_4}479}480481% verified via https://export.arxiv.org/api/query?id_list=2308.03688 on 2026-10-30482@inproceedings{liu2024agentbench,483 author = {Xiao Liu and Hao Yu and Hanchen Zhang and Yifan Xu and Xuanyu Lei and Hanyu Lai and Yu Gu and Hangliang Ding and Kaiwen Men and Kejuan Yang and Shudan Zhang and Xiang Deng and Aohan Zeng and Zhengxiao Du and Chenhui Zhang and Sheng Shen and Tianjun Zhang and Yu Su and Huan Sun and Minlie Huang and Yuxiao Dong and Jie Tang},484 title = {{AgentBench: Evaluating LLMs as Agents}},485 year = {2023},486 booktitle = {ICLR 2024},487 eprint = {2308.03688},488 archivePrefix = {arXiv}489}490491% verified via https://export.arxiv.org/api/query?id_list=2406.12045 on 2026-10-30492@misc{yao2024taubench,493 author = {Shunyu Yao and Noah Shinn and Pedram Razavi and Karthik Narasimhan},494 title = {{$$\tau$$-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains}},495 year = {2024},496 howpublished = {arXiv preprint arXiv:2406.12045},497 eprint = {2406.12045},498 archivePrefix = {arXiv}499}500501% verified via https://export.arxiv.org/api/query?id_list=2407.01502 on 2026-10-30502@article{kapoor2024agentsthatmatter,503 author = {Sayash Kapoor and Benedikt Stroebl and Zachary S. Siegel and Nitya Nadgir and Arvind Narayanan},504 title = {{AI Agents That Matter}},505 year = {2024},506 journal = {Transactions on Machine Learning Research},507 eprint = {2407.01502},508 archivePrefix = {arXiv}509}510511% verified via https://api.crossref.org/works/10.18653/v1/2023.findings-emnlp.722 on 2026-10-30512@inproceedings{sainz2023contamination,513 author = {Sainz, Oscar and Campos, Jon and García-Ferrero, Iker and Etxaniz, Julen and de Lacalle, Oier Lopez and Agirre, Eneko},514 title = {{NLP Evaluation in trouble: On the Need to Measure LLM Data Contamination for each Benchmark}},515 year = {2023},516 booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2023},517 pages = {10776--10787},518 doi = {10.18653/v1/2023.findings-emnlp.722},519 eprint = {2310.18018},520 archivePrefix = {arXiv}521}522523% verified via https://export.arxiv.org/api/query?id_list=2306.05685 on 2026-10-30524@inproceedings{zheng2023mtbench,525 author = {Lianmin Zheng and Wei-Lin Chiang and Ying Sheng and Siyuan Zhuang and Zhanghao Wu and Yonghao Zhuang and Zi Lin and Zhuohan Li and Dacheng Li and Eric P. Xing and Hao Zhang and Joseph E. Gonzalez and Ion Stoica},526 title = {{Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena}},527 year = {2023},528 booktitle = {NeurIPS 2023 Datasets and Benchmarks},529 eprint = {2306.05685},530 archivePrefix = {arXiv}531}532533% verified via https://export.arxiv.org/api/query?id_list=2406.12624 on 2026-10-30534@inproceedings{thakur2025judgingjudges,535 author = {Aman Singh Thakur and Kartik Choudhary and Venkat Srinik Ramayapally and Sankaran Vaidyanathan and Dieuwke Hupkes},536 title = {{Judging the Judges: Evaluating Alignment and Vulnerabilities in LLMs-as-Judges}},537 year = {2024},538 booktitle = {GEM2 Workshop, ACL 2025},539 eprint = {2406.12624},540 archivePrefix = {arXiv}541}542543% verified via https://export.arxiv.org/api/query?id_list=2404.13076 on 2026-10-30544@inproceedings{panickssery2024selfpreference,545 author = {Arjun Panickssery and Samuel R. Bowman and Shi Feng},546 title = {{LLM Evaluators Recognize and Favor Their Own Generations}},547 year = {2024},548 booktitle = {NeurIPS 2024},549 eprint = {2404.13076},550 archivePrefix = {arXiv}551}552553% verified via https://export.arxiv.org/api/query?id_list=2507.02825 on 2026-10-30554@misc{zhu2025abc,555 author = {Yuxuan Zhu and Tengjun Jin and Yada Pruksachatkun and Andy Zhang and Shu Liu and Sasha Cui and Sayash Kapoor and Shayne Longpre and Kevin Meng and Rebecca Weiss and Fazl Barez and Rahul Gupta and Jwala Dhamala and Jacob Merizian and Mario Giulianelli and Harry Coppock and Cozmin Ududec and Jasjeet Sekhon and Jacob Steinhardt and Antony Kellermann and Sarah Schwettmann and Matei Zaharia and Ion Stoica and Percy Liang and Daniel Kang},556 title = {{Establishing Best Practices for Building Rigorous Agentic Benchmarks}},557 year = {2025},558 howpublished = {arXiv preprint arXiv:2507.02825},559 eprint = {2507.02825},560 archivePrefix = {arXiv}561}562563% verified via https://export.arxiv.org/api/query?id_list=2510.11977 on 2026-10-30564@misc{kapoor2025hal,565 author = {Sayash Kapoor and Benedikt Stroebl and Peter Kirgis and Nitya Nadgir and Zachary S Siegel and Boyi Wei and Tianci Xue and Ziru Chen and Felix Chen and Saiteja Utpala and Franck Ndzomga and Dheeraj Oruganty and Sophie Luskin and Kangheng Liu and Botao Yu and Amit Arora and Dongyoon Hahm and Harsh Trivedi and Huan Sun and Juyong Lee and Tengjun Jin and Yifan Mai and Yifei Zhou and Yuxuan Zhu and Rishi Bommasani and Daniel Kang and Dawn Song and Peter Henderson and Yu Su and Percy Liang and Arvind Narayanan},566 title = {{Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation}},567 year = {2025},568 howpublished = {arXiv preprint arXiv:2510.11977},569 eprint = {2510.11977},570 archivePrefix = {arXiv}571}572573% verified via https://export.arxiv.org/api/query?id_list=2510.12367 on 2026-10-30574@misc{li2025llmreval,575 author = {Rui Li and Jia-Chen Gu and Po-Nien Kung and Heming Xia and Junfeng liu and Xiangwen Kong and Zhifang Sui and Nanyun Peng},576 title = {{LLM-REVal: Can We Trust LLM Reviewers Yet?}},577 year = {2025},578 howpublished = {arXiv preprint arXiv:2510.12367},579 eprint = {2510.12367},580 archivePrefix = {arXiv}581}582583% verified via https://api.crossref.org/works/10.1518/001872097778543886 on 2026-10-30584@article{parasuraman1997automation,585 author = {Parasuraman, Raja and Riley, Victor},586 title = {{Humans and Automation: Use, Misuse, Disuse, Abuse}},587 year = {1997},588 journal = {Human Factors: The Journal of the Human Factors and Ergonomics Society},589 volume = {39},590 number = {2},591 pages = {230--253},592 doi = {10.1518/001872097778543886}593}594595% verified via https://api.crossref.org/works/10.1109/3468.844354 on 2026-10-30596@article{parasuraman2000levels,597 author = {Parasuraman, R. and Sheridan, T.B. and Wickens, C.D.},598 title = {{A model for types and levels of human interaction with automation}},599 year = {2000},600 journal = {IEEE Transactions on Systems, Man, and Cybernetics - Part A: Systems and Humans},601 volume = {30},602 number = {3},603 pages = {286--297},604 doi = {10.1109/3468.844354}605}606607% verified via https://api.crossref.org/works/10.1136/amiajnl-2011-000089 on 2026-10-30608@article{goddard2012automationbias,609 author = {Goddard, Kate and Roudsari, Abdul and Wyatt, Jeremy C},610 title = {{Automation bias: a systematic review of frequency, effect mediators, and mitigators}},611 year = {2012},612 journal = {Journal of the American Medical Informatics Association},613 volume = {19},614 number = {1},615 pages = {121--127},616 doi = {10.1136/amiajnl-2011-000089}617}618619% verified via https://api.crossref.org/works/10.1145/3593013.3594087 on 2026-10-30620@inproceedings{lai2023humanai,621 author = {Lai, Vivian and Chen, Chacha and Smith-Renner, Alison and Liao, Q. Vera and Tan, Chenhao},622 title = {{Towards a Science of Human-AI Decision Making: An Overview of Design Space in Empirical Human-Subject Studies}},623 year = {2023},624 booktitle = {2023 ACM Conference on Fairness Accountability and Transparency},625 pages = {1369--1385},626 doi = {10.1145/3593013.3594087},627 eprint = {2112.11471},628 archivePrefix = {arXiv}629}630631% verified via https://api.crossref.org/works/10.1145/3411764.3445717 on 2026-10-30632@inproceedings{bansal2021whole,633 author = {Bansal, Gagan and Wu, Tongshuang and Zhou, Joyce and Fok, Raymond and Nushi, Besmira and Kamar, Ece and Ribeiro, Marco Tulio and Weld, Daniel},634 title = {{Does the Whole Exceed its Parts? The Effect of AI Explanations on Complementary Team Performance}},635 year = {2021},636 booktitle = {Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems},637 pages = {1--16},638 doi = {10.1145/3411764.3445717}639}640641% verified via https://export.arxiv.org/api/query?id_list=1902.03245 on 2026-10-30642@inproceedings{lubars2019delegability,643 author = {Brian Lubars and Chenhao Tan},644 title = {{Ask Not What AI Can Do, But What AI Should Do: Towards a Framework of Task Delegability}},645 year = {2019},646 booktitle = {NeurIPS 2019},647 eprint = {1902.03245},648 archivePrefix = {arXiv}649}650651% verified via https://api.crossref.org/works/10.1038/s41562-024-02024-1 on 2026-10-30652@article{vaccaro2024combinations,653 author = {Vaccaro, Michelle and Almaatouq, Abdullah and Malone, Thomas},654 title = {{When combinations of humans and AI are useful: A systematic review and meta-analysis}},655 year = {2024},656 journal = {Nature Human Behaviour},657 volume = {8},658 number = {12},659 pages = {2293--2303},660 doi = {10.1038/s41562-024-02024-1}661}662663% verified via https://api.crossref.org/works/10.1016/0005-1098(83)90046-8 on 2026-10-30664@article{bainbridge1983ironies,665 author = {Bainbridge, Lisanne},666 title = {{Ironies of automation}},667 year = {1983},668 journal = {Automatica},669 volume = {19},670 number = {6},671 pages = {775--779},672 doi = {10.1016/0005-1098(83)90046-8}673}674675% verified via https://api.crossref.org/works/10.1145/3449287 on 2026-10-30676@article{bucinca2021trust,677 author = {Buçinca, Zana and Malaya, Maja Barbara and Gajos, Krzysztof Z.},678 title = {{To Trust or to Think}},679 year = {2021},680 journal = {Proceedings of the ACM on Human-Computer Interaction},681 volume = {5},682 number = {CSCW1},683 pages = {1--21},684 doi = {10.1145/3449287}685}686687% verified via https://export.arxiv.org/api/query?id_list=2507.09089 on 2026-10-30688@misc{becker2025metr,689 author = {Joel Becker and Nate Rush and Elizabeth Barnes and David Rein},690 title = {{Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity}},691 year = {2025},692 howpublished = {arXiv preprint arXiv:2507.09089},693 eprint = {2507.09089},694 archivePrefix = {arXiv}695}696697% verified via https://api.crossref.org/works/10.1145/2408776.2408794 on 2026-10-30698@article{dean2013tail,699 author = {Dean, Jeffrey and Barroso, Luiz André},700 title = {{The tail at scale}},701 year = {2013},702 journal = {Communications of the ACM},703 volume = {56},704 number = {2},705 pages = {74--80},706 doi = {10.1145/2408776.2408794}707}708709% verified via https://api.semanticscholar.org/graph/v1/paper/87c9e34c39dd1fbd470010dc1ea98ba3517f5a2a on 2026-10-30710@inproceedings{gray1986whystop,711 author = {Jim Gray},712 title = {{Why Do Computers Stop and What Can Be Done About It?}},713 year = {1986},714 booktitle = {Symposium on Reliability in Distributed Software and Database Systems}715}716717% verified via https://api.crossref.org/works/10.1109/MS.2016.60 on 2026-10-30718@article{basiri2016chaos,719 author = {Basiri, Ali and Behnam, Niosha and de Rooij, Ruud and Hochstein, Lorin and Kosewski, Luke and Reynolds, Justin and Rosenthal, Casey},720 title = {{Chaos Engineering}},721 year = {2016},722 journal = {IEEE Software},723 volume = {33},724 number = {3},725 pages = {35--41},726 doi = {10.1109/MS.2016.60}727}728729% verified via https://api.crossref.org/works/10.1145/2160718.2160734 on 2026-10-30730@article{helland2012idempotence,731 author = {Helland, Pat},732 title = {{Idempotence is not a medical condition}},733 year = {2012},734 journal = {Communications of the ACM},735 volume = {55},736 number = {5},737 pages = {56--65},738 doi = {10.1145/2160718.2160734}739}740741% verified via https://api.crossref.org/works/10.1145/3485510 on 2026-10-30742@article{burckhardt2021durablefunctions,743 author = {Burckhardt, Sebastian and Gillum, Chris and Justo, David and Kallas, Konstantinos and McMahon, Connor and Meiklejohn, Christopher S.},744 title = {{Durable functions: semantics for stateful serverless}},745 year = {2021},746 journal = {Proceedings of the ACM on Programming Languages},747 volume = {5},748 number = {OOPSLA},749 pages = {1--27},750 doi = {10.1145/3485510}751}752753% verified via https://export.arxiv.org/api/query?id_list=2504.13359 on 2026-10-30754@misc{erol2025costofpass,755 author = {Mehmet Hamza Erol and Batu El and Mirac Suzgun and Mert Yuksekgonul and James Zou},756 title = {{Cost-of-Pass: An Economic Framework for Evaluating Language Models}},757 year = {2025},758 howpublished = {arXiv preprint arXiv:2504.13359},759 eprint = {2504.13359},760 archivePrefix = {arXiv}761}762763% verified via https://export.arxiv.org/api/query?id_list=2411.05285 on 2026-10-30764@misc{dong2024agentops,765 author = {Liming Dong and Qinghua Lu and Liming Zhu},766 title = {{AgentOps: Enabling Observability of LLM Agents}},767 year = {2024},768 howpublished = {arXiv preprint arXiv:2411.05285},769 eprint = {2411.05285},770 archivePrefix = {arXiv}771}772773% verified via https://export.arxiv.org/api/query?id_list=2603.29231 on 2026-10-30774@misc{khanal2026reliability,775 author = {Aaditya Khanal and Yangyang Tao and Junxiu Zhou},776 title = {{Beyond pass@1: A Reliability Science Framework for Long-Horizon LLM Agents}},777 year = {2026},778 howpublished = {arXiv preprint arXiv:2603.29231},779 eprint = {2603.29231},780 archivePrefix = {arXiv}781}782783% verified via https://export.arxiv.org/api/query?id_list=2608.14380 on 2026-10-30784@misc{zhuang2026agentrewind,785 author = {Yu Zhuang and Kefei Chen and Yitong Duan and Shuxin Zheng and Jian Li and Xu-Yao Zhang},786 title = {{AgentRewind: Recoverable Execution for Long-Horizon LLM Agents}},787 year = {2026},788 howpublished = {arXiv preprint arXiv:2608.14380},789 eprint = {2608.14380},790 archivePrefix = {arXiv}791}792793% verified via https://api.semanticscholar.org/graph/v1/paper/003d5a65de0ac72daaf105ded903cb3eb88585b3 on 2026-10-30794@techreport{sigelman2010dapper,795 author = {B. Sigelman and L. Barroso and M. Burrows and Patrick Stephenson and M. Plakal and Donald Beaver and Saul Jaspan and C. Shanbhag},796 title = {{Dapper, a Large-Scale Distributed Systems Tracing Infrastructure}},797 year = {2010},798 institution = {Google},799 number = {dapper-2010-1}800}801802% verified via https://api.crossref.org/works/10.18653/v1/2020.emnlp-main.609 on 2026-10-30803@inproceedings{wadden2020scifact,804 author = {Wadden, David and Lin, Shanchuan and Lo, Kyle and Wang, Lucy Lu and van Zuylen, Madeleine and Cohan, Arman and Hajishirzi, Hannaneh},805 title = {{Fact or Fiction: Verifying Scientific Claims}},806 year = {2020},807 booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},808 pages = {7534--7550},809 doi = {10.18653/v1/2020.emnlp-main.609}810}811812% verified via https://api.crossref.org/works/10.18653/v1/N18-1074 on 2026-10-30813@inproceedings{thorne2018fever,814 author = {Thorne, James and Vlachos, Andreas and Christodoulopoulos, Christos and Mittal, Arpit},815 title = {{FEVER: a Large-scale Dataset for Fact Extraction and VERification}},816 year = {2018},817 booktitle = {Proceedings of the 2018 Conference of the North American Chapter of818 the Association for Computational Linguistics: Human Language819 Technologies, Volume 1 (Long Papers)},820 pages = {809--819},821 doi = {10.18653/v1/N18-1074}822}823824% verified via https://export.arxiv.org/api/query?id_list=2212.08037 on 2026-10-30825@misc{bohnet2022attributedqa,826 author = {Bernd Bohnet and Vinh Q. Tran and Pat Verga and Roee Aharoni and Daniel Andor and Livio Baldini Soares and Massimiliano Ciaramita and Jacob Eisenstein and Kuzman Ganchev and Jonathan Herzig and Kai Hui and Tom Kwiatkowski and Ji Ma and Jianmo Ni and Lierni Sestorain Saralegui and Tal Schuster and William W. Cohen and Michael Collins and Dipanjan Das and Donald Metzler and Slav Petrov and Kellie Webster},827 title = {{Attributed Question Answering: Evaluation and Modeling for Attributed Large Language Models}},828 year = {2022},829 howpublished = {arXiv preprint arXiv:2212.08037},830 eprint = {2212.08037},831 archivePrefix = {arXiv}832}833834% verified via https://api.crossref.org/works/10.1038/s41598-023-41032-5 on 2026-10-30835@article{walters2023fabrication,836 author = {Walters, William H. and Wilder, Esther Isabelle},837 title = {{Fabrication and errors in the bibliographic citations generated by ChatGPT}},838 year = {2023},839 journal = {Scientific Reports},840 volume = {13},841 number = {1},842 doi = {10.1038/s41598-023-41032-5}843}844845% verified via https://api.crossref.org/works/10.7759/cureus.35179 on 2026-10-30846@article{alkaissi2023hallucinations,847 author = {Alkaissi, Hussam and McFarlane, Samy I},848 title = {{Artificial Hallucinations in ChatGPT: Implications in Scientific Writing}},849 year = {2023},850 journal = {Cureus},851 doi = {10.7759/cureus.35179}852}853854% verified via https://api.crossref.org/works/10.18653/v1/2023.acl-long.910 on 2026-10-30855@inproceedings{gao2023rarr,856 author = {Gao, Luyu and Dai, Zhuyun and Pasupat, Panupong and Chen, Anthony and Chaganty, Arun Tejasvi and Fan, Yicheng and Zhao, Vincent and Lao, Ni and Lee, Hongrae and Juan, Da-Cheng and Guu, Kelvin},857 title = {{RARR: Researching and Revising What Language Models Say, Using Language Models}},858 year = {2023},859 booktitle = {Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},860 pages = {16477--16508},861 doi = {10.18653/v1/2023.acl-long.910}862}863864% verified via https://export.arxiv.org/api/query?id_list=2203.11171 on 2026-10-30865@inproceedings{wang2023selfconsistency,866 author = {Xuezhi Wang and Jason Wei and Dale Schuurmans and Quoc Le and Ed Chi and Sharan Narang and Aakanksha Chowdhery and Denny Zhou},867 title = {{Self-Consistency Improves Chain of Thought Reasoning in Language Models}},868 year = {2022},869 booktitle = {ICLR 2023},870 eprint = {2203.11171},871 archivePrefix = {arXiv}872}873874% verified via https://api.crossref.org/works/10.18653/v1/2023.emnlp-main.741 on 2026-10-30875@inproceedings{min2023factscore,876 author = {Min, Sewon and Krishna, Kalpesh and Lyu, Xinxi and Lewis, Mike and Yih, Wen-tau and Koh, Pang and Iyyer, Mohit and Zettlemoyer, Luke and Hajishirzi, Hannaneh},877 title = {{FActScore: Fine-grained Atomic Evaluation of Factual Precision in Long Form Text Generation}},878 year = {2023},879 booktitle = {Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},880 pages = {12076--12100},881 doi = {10.18653/v1/2023.emnlp-main.741},882 eprint = {2305.14251},883 archivePrefix = {arXiv}884}885886% verified via https://export.arxiv.org/api/query?id_list=2310.01798 on 2026-10-30887@inproceedings{huang2024selfcorrect,888 author = {Jie Huang and Xinyun Chen and Swaroop Mishra and Huaixiu Steven Zheng and Adams Wei Yu and Xinying Song and Denny Zhou},889 title = {{Large Language Models Cannot Self-Correct Reasoning Yet}},890 year = {2023},891 booktitle = {ICLR 2024},892 eprint = {2310.01798},893 archivePrefix = {arXiv}894}895896% verified via https://export.arxiv.org/api/query?id_list=2601.18724 on 2026-10-30897@inproceedings{sakai2026hallucitation,898 author = {Yusuke Sakai and Hidetaka Kamigaito and Taro Watanabe},899 title = {{HalluCitation Matters: Revealing the Impact of Hallucinated References with 300 Hallucinated Papers in ACL Conferences}},900 year = {2026},901 booktitle = {ACL 2026 (per Semantic Scholar venue record)},902 eprint = {2601.18724},903 archivePrefix = {arXiv}904}905906% verified via https://export.arxiv.org/api/query?id_list=2605.07723 on 2026-10-30907@misc{zhao2026hallucinationswild,908 author = {Zhenyue Zhao and Yihe Wang and Toby Stuart and Mathijs De Vaan and Paul Ginsparg and Yian Yin},909 title = {{LLM hallucinations in the wild: Large-scale evidence from non-existent citations}},910 year = {2026},911 howpublished = {arXiv preprint arXiv:2605.07723},912 eprint = {2605.07723},913 archivePrefix = {arXiv}914}915916