@misc{lu2024aiscientist,
  author = {Chris Lu and Cong Lu and Robert Tjarko Lange and Jakob Foerster and Jeff Clune and David Ha},
  title = {{The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery}},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2408.06292},
  eprint = {2408.06292},
  archivePrefix = {arXiv}
}
% verified via https://export.arxiv.org/api/query?id_list=2408.06292 on 2026-10-30  (comment placed after the first entry so that no % line precedes it)

% verified via https://api.crossref.org/works/10.1038/s41586-026-10265-5 on 2026-10-30
@article{lu2026aiscientist,
  author = {Lu, Chris and Lu, Cong and Lange, Robert Tjarko and Yamada, Yutaro and Hu, Shengran and Foerster, Jakob and Ha, David and Clune, Jeff},
  title = {{Towards end-to-end automation of AI research}},
  year = {2026},
  journal = {Nature},
  volume = {651},
  number = {8107},
  pages = {914--919},
  doi = {10.1038/s41586-026-10265-5}
}

% verified via https://export.arxiv.org/api/query?id_list=2504.08066 on 2026-10-30
@misc{yamada2025aiscientistv2,
  author = {Yutaro Yamada and Robert Tjarko Lange and Cong Lu and Shengran Hu and Chris Lu and Jakob Foerster and Jeff Clune and David Ha},
  title = {{The AI Scientist-v2: Workshop-Level Automated Scientific Discovery via Agentic Tree Search}},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2504.08066},
  eprint = {2504.08066},
  archivePrefix = {arXiv}
}

% verified via https://api.crossref.org/works/10.18653/v1/2025.findings-emnlp.320 on 2026-10-30
@inproceedings{schmidgall2025agentlab,
  author = {Schmidgall, Samuel and Su, Yusheng and Wang, Ze and Sun, Ximeng and Wu, Jialian and Yu, Xiaodong and Liu, Jiang and Moor, Michael and Liu, Zicheng and Barsoum, Emad},
  title = {{Agent Laboratory: Using LLM Agents as Research Assistants}},
  year = {2025},
  booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2025},
  pages = {5977--6043},
  doi = {10.18653/v1/2025.findings-emnlp.320},
  eprint = {2501.04227},
  archivePrefix = {arXiv}
}

% verified via https://api.crossref.org/works/10.1038/s41586-023-06792-0 on 2026-10-30
@article{boiko2023coscientist,
  author = {Boiko, Daniil A. and MacKnight, Robert and Kline, Ben and Gomes, Gabe},
  title = {{Autonomous chemical research with large language models}},
  year = {2023},
  journal = {Nature},
  volume = {624},
  number = {7992},
  pages = {570--578},
  doi = {10.1038/s41586-023-06792-0}
}

% verified via https://api.crossref.org/works/10.1038/s42256-024-00832-8 on 2026-10-30
@article{bran2024chemcrow,
  author = {M. Bran, Andres and Cox, Sam and Schilter, Oliver and Baldassari, Carlo and White, Andrew D. and Schwaller, Philippe},
  title = {{Augmenting large language models with chemistry tools}},
  year = {2024},
  journal = {Nature Machine Intelligence},
  volume = {6},
  number = {5},
  pages = {525--535},
  doi = {10.1038/s42256-024-00832-8}
}

% verified via https://api.crossref.org/works/10.1056/AIoa2400555 on 2026-10-30
@article{ifargan2025datatopaper,
  author = {Ifargan, Tal and Hafner, Lukas and Kern, Maor and Alcalay, Ori and Kishony, Roy},
  title = {{Autonomous LLM-Driven Research — from Data to Human-Verifiable Research Papers}},
  year = {2025},
  journal = {NEJM AI},
  volume = {2},
  number = {1},
  doi = {10.1056/AIoa2400555},
  eprint = {2404.17605},
  archivePrefix = {arXiv}
}

% verified via https://api.crossref.org/works/10.1038/s41586-026-10644-y on 2026-10-30
@article{gottweis2026coscientist,
  author = {Gottweis, Juraj and Weng, Wei-Hung and Daryin, Alexander and Tu, Tao and Sirkovic, Petar and Myaskovsky, Artiom and Glowaty, Grzegorz and Weissenberger, Felix and Orlandi, Alessio and Popovici, Dan and Palepu, Anil and Rong, Keran and Tanno, Ryutaro and Saab, Khaled and Zhang, Fan and Blum, Jacob and Carroll, Andrew and Kulkarni, Kavita and Tomašev, Nenad and Zverinski, Dina and Rendulic, Ivor and Vedadi, Elahe and Hasler, Florian and Rimanic, Luka and Boia, Marina and Budiselic, Ivan and Feinstein, Ben and Bellaiche, Mathias and Sheffer, Tom and Freyberg, Jan and Ratcliff, Jeremy and Bertolli, Ottavia and Chou, Katherine and Hassidim, Avinatan and Gokturk, Burak and Vahdat, Amin and Guan, Yuan and Dhillon, Vikram and Vaishnav, Eeshit Dhaval and Lee, Byron and Costa, Tiago R. D. and Penadés, José R. and Peltz, Gary and Matias, Yossi and Manyika, James and Hassabis, Demis and Xu, Yunhan and Kohli, Pushmeet and Pawlosky, Annalisa and Karthikesalingam, Alan and Natarajan, Vivek},
  title = {{Accelerating scientific discovery with Co-Scientist}},
  year = {2026},
  journal = {Nature},
  volume = {655},
  number = {8122},
  pages = {487--496},
  doi = {10.1038/s41586-026-10644-y},
  eprint = {2502.18864},
  archivePrefix = {arXiv}
}

% verified via https://api.crossref.org/works/10.1038/s41586-025-09442-9 on 2026-10-30
@article{swanson2025virtuallab,
  author = {Swanson, Kyle and Wu, Wesley and Bulaong, Nash L. and Pak, John E. and Zou, James},
  title = {{The Virtual Lab of AI agents designs new SARS-CoV-2 nanobodies}},
  year = {2025},
  journal = {Nature},
  volume = {646},
  number = {8085},
  pages = {716--723},
  doi = {10.1038/s41586-025-09442-9}
}

% verified via https://export.arxiv.org/api/query?id_list=2511.02824 on 2026-10-30
@misc{mitchener2025kosmos,
  author = {Ludovico Mitchener and Angela Yiu and Benjamin Chang and Mathieu Bourdenx and Tyler Nadolski and Arvis Sulovari and Eric C. Landsness and Daniel L. Barabasi and Siddharth Narayanan and Nicky Evans and Shriya Reddy and Martha Foiani and Aizad Kamal and Leah P. Shriver and Fang Cao and Asmamaw T. Wassie and Jon M. Laurent and Edwin Melville-Green and Mayk Caldas and Albert Bou and Kaleigh F. Roberts and Sladjana Zagorac and Timothy C. Orr and Miranda E. Orr and Kevin J. Zwezdaryk and Ali E. Ghareeb and Laurie McCoy and Bruna Gomes and Euan A. Ashley and Karen E. Duff and Tonio Buonassisi and Tom Rainforth and Randall J. Bateman and Michael Skarlinski and Samuel G. Rodriques and Michaela M. Hinks and Andrew D. White},
  title = {{Kosmos: An AI Scientist for Autonomous Discovery}},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2511.02824},
  eprint = {2511.02824},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2509.08713 on 2026-10-30
@misc{luo2025pitfalls,
  author = {Ziming Luo and Atoosa Kasirzadeh and Nihar B. Shah},
  title = {{The More You Automate, the Less You See: Hidden Pitfalls of AI Scientist Systems}},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2509.08713},
  eprint = {2509.08713},
  archivePrefix = {arXiv}
}

% verified via https://api.crossref.org/works/10.1145/3769733.3769747 on 2026-10-30
@article{beel2025sakana,
  author = {Beel, Joeran and Kan, Min-Yen and Baumgart, Moritz},
  title = {{Evaluating Sakana's AI Scientist: Bold Claims, Mixed Results, and a Promising Future?}},
  year = {2025},
  journal = {ACM SIGIR Forum},
  volume = {59},
  number = {1},
  pages = {1--20},
  doi = {10.1145/3769733.3769747},
  eprint = {2502.14297},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2607.12301 on 2026-10-30
@misc{luo2026xscientist,
  author = {Jixiang Luo},
  title = {{XScientist: A Git-Like Research Protocol for Long-Running Autonomous Scientific Discovery}},
  year = {2026},
  howpublished = {arXiv preprint arXiv:2607.12301},
  eprint = {2607.12301},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2608.05179 on 2026-10-30
@misc{ding2026verificationgap,
  author = {Tianyu Ding and Aditya Nannapaneni and Bingfan Liu and Ling Zhang},
  title = {{Autonomous Research Agents: A Survey of AI Scientists and the Verification Gap}},
  year = {2026},
  howpublished = {arXiv preprint arXiv:2608.05179},
  eprint = {2608.05179},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2410.05080 on 2026-10-30
@inproceedings{chen2025scienceagentbench,
  author = {Ziru Chen and Shijie Chen and Yuting Ning and Qianheng Zhang and Boshi Wang and Botao Yu and Yifei Li and Zeyi Liao and Chen Wei and Zitong Lu and Vishal Dey and Mingyi Xue and Frazier N. Baker and Benjamin Burns and Daniel Adu-Ampratwum and Xuhui Huang and Xia Ning and Song Gao and Yu Su and Huan Sun},
  title = {{ScienceAgentBench: Toward Rigorous Assessment of Language Agents for Data-Driven Scientific Discovery}},
  year = {2024},
  booktitle = {ICLR 2025},
  eprint = {2410.05080},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2407.01725 on 2026-10-30
@inproceedings{majumder2025discoverybench,
  author = {Bodhisattwa Prasad Majumder and Harshit Surana and Dhruv Agarwal and Bhavana Dalvi Mishra and Abhijeetsingh Meena and Aryan Prakhar and Tirth Vora and Tushar Khot and Ashish Sabharwal and Peter Clark},
  title = {{DiscoveryBench: Towards Data-Driven Discovery with Large Language Models}},
  year = {2024},
  booktitle = {ICLR 2025},
  eprint = {2407.01725},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2409.07703 on 2026-10-30
@inproceedings{jing2025dsbench,
  author = {Liqiang Jing and Zhehui Huang and Xiaoyang Wang and Wenlin Yao and Wenhao Yu and Kaixin Ma and Hongming Zhang and Xinya Du and Dong Yu},
  title = {{DSBench: How Far Are Data Science Agents from Becoming Data Science Experts?}},
  year = {2024},
  booktitle = {ICLR 2025},
  eprint = {2409.07703},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2310.03302 on 2026-10-30
@inproceedings{huang2024mlagentbench,
  author = {Qian Huang and Jian Vora and Percy Liang and Jure Leskovec},
  title = {{MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation}},
  year = {2023},
  booktitle = {ICML 2024},
  eprint = {2310.03302},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2408.09667 on 2026-10-30
@inproceedings{gu2024blade,
  author = {Ken Gu and Ruoxi Shang and Ruien Jiang and Keying Kuang and Richard-John Lin and Donghe Lyu and Yue Mao and Youran Pan and Teng Wu and Jiaqian Yu and Yikun Zhang and Tianmai M. Zhang and Lanyi Zhu and Mike A. Merrill and Jeffrey Heer and Tim Althoff},
  title = {{BLADE: Benchmarking Language Model Agents for Data-Driven Science}},
  year = {2024},
  booktitle = {EMNLP 2024},
  eprint = {2408.09667},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2310.06770 on 2026-10-30
@inproceedings{jimenez2024swebench,
  author = {Carlos E. Jimenez and John Yang and Alexander Wettig and Shunyu Yao and Kexin Pei and Ofir Press and Karthik Narasimhan},
  title = {{SWE-bench: Can Language Models Resolve Real-World GitHub Issues?}},
  year = {2023},
  booktitle = {ICLR 2024},
  eprint = {2310.06770},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2410.07095 on 2026-10-30
@inproceedings{chan2025mlebench,
  author = {Jun Shern Chan and Neil Chowdhury and Oliver Jaffe and James Aung and Dane Sherburn and Evan Mays and Giulio Starace and Kevin Liu and Leon Maksin and Tejal Patwardhan and Lilian Weng and Aleksander Mądry},
  title = {{MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering}},
  year = {2024},
  booktitle = {ICLR 2025},
  eprint = {2410.07095},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2409.11363 on 2026-10-30
@article{siegel2024corebench,
  author = {Zachary S. Siegel and Sayash Kapoor and Nitya Nadgir and Benedikt Stroebl and Arvind Narayanan},
  title = {{CORE-Bench: Fostering the Credibility of Published Research Through a Computational Reproducibility Agent Benchmark}},
  year = {2024},
  journal = {Transactions on Machine Learning Research},
  eprint = {2409.11363},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2504.01848 on 2026-10-30
@inproceedings{starace2025paperbench,
  author = {Giulio Starace and Oliver Jaffe and Dane Sherburn and James Aung and Jun Shern Chan and Leon Maksin and Rachel Dias and Evan Mays and Benjamin Kinsella and Wyatt Thompson and Johannes Heidecke and Amelia Glaese and Tejal Patwardhan},
  title = {{PaperBench: Evaluating AI's Ability to Replicate AI Research}},
  year = {2025},
  booktitle = {ICML 2025},
  eprint = {2504.01848},
  archivePrefix = {arXiv}
}

% verified via https://api.crossref.org/works/10.1093/bioinformatics/bts480 on 2026-10-30
@article{koster2012snakemake,
  author = {Köster, Johannes and Rahmann, Sven},
  title = {{Snakemake—a scalable bioinformatics workflow engine}},
  year = {2012},
  journal = {Bioinformatics},
  volume = {28},
  number = {19},
  pages = {2520--2522},
  doi = {10.1093/bioinformatics/bts480}
}

% verified via https://api.crossref.org/works/10.1038/nbt.3820 on 2026-10-30
@article{ditommaso2017nextflow,
  author = {Di Tommaso, Paolo and Chatzou, Maria and Floden, Evan W and Barja, Pablo Prieto and Palumbo, Emilio and Notredame, Cedric},
  title = {{Nextflow enables reproducible computational workflows}},
  year = {2017},
  journal = {Nature Biotechnology},
  volume = {35},
  number = {4},
  pages = {316--319},
  doi = {10.1038/nbt.3820}
}

% verified via https://api.crossref.org/works/10.1145/2452376.2452478 on 2026-10-30
@inproceedings{missier2013prov,
  author = {Missier, Paolo and Belhajjame, Khalid and Cheney, James},
  title = {{The W3C PROV family of specifications for modelling provenance metadata}},
  year = {2013},
  booktitle = {Proceedings of the 16th International Conference on Extending Database Technology},
  pages = {773--776},
  doi = {10.1145/2452376.2452478}
}

% verified via https://api.crossref.org/works/10.1145/2882903.2899401 on 2026-10-30
@inproceedings{chirigati2016reprozip,
  author = {Chirigati, Fernando and Rampin, Rémi and Shasha, Dennis and Freire, Juliana},
  title = {{ReproZip}},
  year = {2016},
  booktitle = {Proceedings of the 2016 International Conference on Management of Data},
  pages = {2085--2088},
  doi = {10.1145/2882903.2899401}
}

% verified via https://api.crossref.org/works/10.1371/journal.pcbi.1003285 on 2026-10-30
@article{sandve2013tensimplerules,
  author = {Sandve, Geir Kjetil and Nekrutenko, Anton and Taylor, James and Hovig, Eivind},
  title = {{Ten Simple Rules for Reproducible Computational Research}},
  year = {2013},
  journal = {PLoS Computational Biology},
  volume = {9},
  number = {10},
  pages = {e1003285},
  doi = {10.1371/journal.pcbi.1003285}
}

% verified via https://api.crossref.org/works/10.1073/pnas.1708290115 on 2026-10-30
@article{stodden2018journalpolicy,
  author = {Stodden, Victoria and Seiler, Jennifer and Ma, Zhaokun},
  title = {{An empirical analysis of journal policy effectiveness for computational reproducibility}},
  year = {2018},
  journal = {Proceedings of the National Academy of Sciences},
  volume = {115},
  number = {11},
  pages = {2584--2589},
  doi = {10.1073/pnas.1708290115}
}

% verified via https://api.crossref.org/works/10.1126/science.1213847 on 2026-10-30
@article{peng2011reproducible,
  author = {Peng, Roger D.},
  title = {{Reproducible Research in Computational Science}},
  year = {2011},
  journal = {Science},
  volume = {334},
  number = {6060},
  pages = {1226--1227},
  doi = {10.1126/science.1213847}
}

% verified via https://api.crossref.org/works/10.17226/25303 on 2026-10-30
@book{nasem2019reproducibility,
  author = {{National Academies of Sciences, Engineering, and Medicine}},
  title = {{Reproducibility and Replicability in Science}},
  year = {2019},
  publisher = {The National Academies Press},
  address = {Washington, DC},
  doi = {10.17226/25303}
}

% verified via https://api.crossref.org/works/10.1098/rsta.2020.0069 on 2026-10-30
@article{krafczyk2021reproduction,
  author = {Krafczyk, M. S. and Shi, A. and Bhaskar, A. and Marinov, D. and Stodden, V.},
  title = {{Learning from reproducing computational results: introducing three principles and the <i>Reproduction Package</i>}},
  year = {2021},
  journal = {Philosophical Transactions of the Royal Society A: Mathematical, Physical and Engineering Sciences},
  volume = {379},
  number = {2197},
  doi = {10.1098/rsta.2020.0069}
}

% verified via https://api.crossref.org/works/10.1038/s41597-022-01143-6 on 2026-10-30
@article{trisovic2022codequality,
  author = {Trisovic, Ana and Lau, Matthew K. and Pasquier, Thomas and Crosas, Mercè},
  title = {{A large-scale study on research code quality and execution}},
  year = {2022},
  journal = {Scientific Data},
  volume = {9},
  number = {1},
  doi = {10.1038/s41597-022-01143-6}
}

% verified via https://api.crossref.org/works/10.1073/pnas.1708274114 on 2026-10-30
@article{nosek2018preregistration,
  author = {Nosek, Brian A. and Ebersole, Charles R. and DeHaven, Alexander C. and Mellor, David T.},
  title = {{The preregistration revolution}},
  year = {2018},
  journal = {Proceedings of the National Academy of Sciences},
  volume = {115},
  number = {11},
  pages = {2600--2606},
  doi = {10.1073/pnas.1708274114}
}

% verified via https://api.crossref.org/works/10.1016/j.cortex.2012.12.016 on 2026-10-30
@article{chambers2013registeredreports,
  author = {Chambers, Christopher D.},
  title = {{Registered Reports: A new publishing initiative at Cortex}},
  year = {2013},
  journal = {Cortex},
  volume = {49},
  number = {3},
  pages = {609--610},
  doi = {10.1016/j.cortex.2012.12.016}
}

% verified via https://api.crossref.org/works/10.1038/s41562-021-01193-7 on 2026-10-30
@article{chambers2022registeredreports,
  author = {Chambers, Christopher D. and Tzavella, Loukia},
  title = {{The past, present and future of Registered Reports}},
  year = {2021},
  journal = {Nature Human Behaviour},
  volume = {6},
  number = {1},
  pages = {29--42},
  doi = {10.1038/s41562-021-01193-7}
}

% verified via https://api.crossref.org/works/10.1177/0956797611417632 on 2026-10-30
@article{simmons2011falsepositive,
  author = {Simmons, Joseph P. and Nelson, Leif D. and Simonsohn, Uri},
  title = {{False-Positive Psychology}},
  year = {2011},
  journal = {Psychological Science},
  volume = {22},
  number = {11},
  pages = {1359--1366},
  doi = {10.1177/0956797611417632}
}

% verified via https://api.crossref.org/works/10.1038/s41562-016-0021 on 2026-10-30
@article{munafo2017manifesto,
  author = {Munafò, Marcus R. and Nosek, Brian A. and Bishop, Dorothy V. M. and Button, Katherine S. and Chambers, Christopher D. and Percie du Sert, Nathalie and Simonsohn, Uri and Wagenmakers, Eric-Jan and Ware, Jennifer J. and Ioannidis, John P. A.},
  title = {{A manifesto for reproducible science}},
  year = {2017},
  journal = {Nature Human Behaviour},
  volume = {1},
  number = {1},
  doi = {10.1038/s41562-016-0021}
}

% verified via https://api.crossref.org/works/10.1511/2014.111.460 on 2026-10-30
@article{gelman2014statisticalcrisis,
  author = {Gelman, Andrew and Loken, Eric},
  title = {{The Statistical Crisis in Science}},
  year = {2014},
  journal = {American Scientist},
  volume = {102},
  number = {6},
  pages = {460},
  doi = {10.1511/2014.111.460}
}

% verified via https://api.crossref.org/works/10.1126/science.aab2374 on 2026-10-30
@article{nosek2015top,
  author = {Nosek, B. A. and Alter, G. and Banks, G. C. and Borsboom, D. and Bowman, S. D. and Breckler, S. J. and Buck, S. and Chambers, C. D. and Chin, G. and Christensen, G. and Contestabile, M. and Dafoe, A. and Eich, E. and Freese, J. and Glennerster, R. and Goroff, D. and Green, D. P. and Hesse, B. and Humphreys, M. and Ishiyama, J. and Karlan, D. and Kraut, A. and Lupia, A. and Mabry, P. and Madon, T. and Malhotra, N. and Mayo-Wilson, E. and McNutt, M. and Miguel, E. and Paluck, E. Levy and Simonsohn, U. and Soderberg, C. and Spellman, B. A. and Turitto, J. and VandenBos, G. and Vazire, S. and Wagenmakers, E. J. and Wilson, R. and Yarkoni, T.},
  title = {{Promoting an open research culture}},
  year = {2015},
  journal = {Science},
  volume = {348},
  number = {6242},
  pages = {1422--1425},
  doi = {10.1126/science.aab2374}
}

% verified via https://api.crossref.org/works/10.1177/25152459211007467 on 2026-10-30
@article{scheel2021excess,
  author = {Scheel, Anne M. and Schijen, Mitchell R. M. J. and Lakens, Daniël},
  title = {{An Excess of Positive Results: Comparing the Standard Psychology Literature With Registered Reports}},
  year = {2021},
  journal = {Advances in Methods and Practices in Psychological Science},
  volume = {4},
  number = {2},
  doi = {10.1177/25152459211007467}
}

% verified via https://api.crossref.org/works/10.1126/science.aac4716 on 2026-10-30
@article{osc2015reproducibility,
  author = {{Open Science Collaboration}},
  title = {{Estimating the reproducibility of psychological science}},
  year = {2015},
  journal = {Science},
  volume = {349},
  number = {6251},
  doi = {10.1126/science.aac4716}
}

% verified via https://api.crossref.org/works/10.1207/s15327957pspr0203_4 on 2026-10-30
@article{kerr1998harking,
  author = {Kerr, Norbert L.},
  title = {{HARKing: Hypothesizing After the Results are Known}},
  year = {1998},
  journal = {Personality and Social Psychology Review},
  volume = {2},
  number = {3},
  pages = {196--217},
  doi = {10.1207/s15327957pspr0203_4}
}

% verified via https://export.arxiv.org/api/query?id_list=2308.03688 on 2026-10-30
@inproceedings{liu2024agentbench,
  author = {Xiao Liu and Hao Yu and Hanchen Zhang and Yifan Xu and Xuanyu Lei and Hanyu Lai and Yu Gu and Hangliang Ding and Kaiwen Men and Kejuan Yang and Shudan Zhang and Xiang Deng and Aohan Zeng and Zhengxiao Du and Chenhui Zhang and Sheng Shen and Tianjun Zhang and Yu Su and Huan Sun and Minlie Huang and Yuxiao Dong and Jie Tang},
  title = {{AgentBench: Evaluating LLMs as Agents}},
  year = {2023},
  booktitle = {ICLR 2024},
  eprint = {2308.03688},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2406.12045 on 2026-10-30
@misc{yao2024taubench,
  author = {Shunyu Yao and Noah Shinn and Pedram Razavi and Karthik Narasimhan},
  title = {{$$\tau$$-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains}},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2406.12045},
  eprint = {2406.12045},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2407.01502 on 2026-10-30
@article{kapoor2024agentsthatmatter,
  author = {Sayash Kapoor and Benedikt Stroebl and Zachary S. Siegel and Nitya Nadgir and Arvind Narayanan},
  title = {{AI Agents That Matter}},
  year = {2024},
  journal = {Transactions on Machine Learning Research},
  eprint = {2407.01502},
  archivePrefix = {arXiv}
}

% verified via https://api.crossref.org/works/10.18653/v1/2023.findings-emnlp.722 on 2026-10-30
@inproceedings{sainz2023contamination,
  author = {Sainz, Oscar and Campos, Jon and García-Ferrero, Iker and Etxaniz, Julen and de Lacalle, Oier Lopez and Agirre, Eneko},
  title = {{NLP Evaluation in trouble: On the Need to Measure LLM Data Contamination for each Benchmark}},
  year = {2023},
  booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2023},
  pages = {10776--10787},
  doi = {10.18653/v1/2023.findings-emnlp.722},
  eprint = {2310.18018},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2306.05685 on 2026-10-30
@inproceedings{zheng2023mtbench,
  author = {Lianmin Zheng and Wei-Lin Chiang and Ying Sheng and Siyuan Zhuang and Zhanghao Wu and Yonghao Zhuang and Zi Lin and Zhuohan Li and Dacheng Li and Eric P. Xing and Hao Zhang and Joseph E. Gonzalez and Ion Stoica},
  title = {{Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena}},
  year = {2023},
  booktitle = {NeurIPS 2023 Datasets and Benchmarks},
  eprint = {2306.05685},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2406.12624 on 2026-10-30
@inproceedings{thakur2025judgingjudges,
  author = {Aman Singh Thakur and Kartik Choudhary and Venkat Srinik Ramayapally and Sankaran Vaidyanathan and Dieuwke Hupkes},
  title = {{Judging the Judges: Evaluating Alignment and Vulnerabilities in LLMs-as-Judges}},
  year = {2024},
  booktitle = {GEM2 Workshop, ACL 2025},
  eprint = {2406.12624},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2404.13076 on 2026-10-30
@inproceedings{panickssery2024selfpreference,
  author = {Arjun Panickssery and Samuel R. Bowman and Shi Feng},
  title = {{LLM Evaluators Recognize and Favor Their Own Generations}},
  year = {2024},
  booktitle = {NeurIPS 2024},
  eprint = {2404.13076},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2507.02825 on 2026-10-30
@misc{zhu2025abc,
  author = {Yuxuan Zhu and Tengjun Jin and Yada Pruksachatkun and Andy Zhang and Shu Liu and Sasha Cui and Sayash Kapoor and Shayne Longpre and Kevin Meng and Rebecca Weiss and Fazl Barez and Rahul Gupta and Jwala Dhamala and Jacob Merizian and Mario Giulianelli and Harry Coppock and Cozmin Ududec and Jasjeet Sekhon and Jacob Steinhardt and Antony Kellermann and Sarah Schwettmann and Matei Zaharia and Ion Stoica and Percy Liang and Daniel Kang},
  title = {{Establishing Best Practices for Building Rigorous Agentic Benchmarks}},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2507.02825},
  eprint = {2507.02825},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2510.11977 on 2026-10-30
@misc{kapoor2025hal,
  author = {Sayash Kapoor and Benedikt Stroebl and Peter Kirgis and Nitya Nadgir and Zachary S Siegel and Boyi Wei and Tianci Xue and Ziru Chen and Felix Chen and Saiteja Utpala and Franck Ndzomga and Dheeraj Oruganty and Sophie Luskin and Kangheng Liu and Botao Yu and Amit Arora and Dongyoon Hahm and Harsh Trivedi and Huan Sun and Juyong Lee and Tengjun Jin and Yifan Mai and Yifei Zhou and Yuxuan Zhu and Rishi Bommasani and Daniel Kang and Dawn Song and Peter Henderson and Yu Su and Percy Liang and Arvind Narayanan},
  title = {{Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation}},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2510.11977},
  eprint = {2510.11977},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2510.12367 on 2026-10-30
@misc{li2025llmreval,
  author = {Rui Li and Jia-Chen Gu and Po-Nien Kung and Heming Xia and Junfeng liu and Xiangwen Kong and Zhifang Sui and Nanyun Peng},
  title = {{LLM-REVal: Can We Trust LLM Reviewers Yet?}},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2510.12367},
  eprint = {2510.12367},
  archivePrefix = {arXiv}
}

% verified via https://api.crossref.org/works/10.1518/001872097778543886 on 2026-10-30
@article{parasuraman1997automation,
  author = {Parasuraman, Raja and Riley, Victor},
  title = {{Humans and Automation: Use, Misuse, Disuse, Abuse}},
  year = {1997},
  journal = {Human Factors: The Journal of the Human Factors and Ergonomics Society},
  volume = {39},
  number = {2},
  pages = {230--253},
  doi = {10.1518/001872097778543886}
}

% verified via https://api.crossref.org/works/10.1109/3468.844354 on 2026-10-30
@article{parasuraman2000levels,
  author = {Parasuraman, R. and Sheridan, T.B. and Wickens, C.D.},
  title = {{A model for types and levels of human interaction with automation}},
  year = {2000},
  journal = {IEEE Transactions on Systems, Man, and Cybernetics - Part A: Systems and Humans},
  volume = {30},
  number = {3},
  pages = {286--297},
  doi = {10.1109/3468.844354}
}

% verified via https://api.crossref.org/works/10.1136/amiajnl-2011-000089 on 2026-10-30
@article{goddard2012automationbias,
  author = {Goddard, Kate and Roudsari, Abdul and Wyatt, Jeremy C},
  title = {{Automation bias: a systematic review of frequency, effect mediators, and mitigators}},
  year = {2012},
  journal = {Journal of the American Medical Informatics Association},
  volume = {19},
  number = {1},
  pages = {121--127},
  doi = {10.1136/amiajnl-2011-000089}
}

% verified via https://api.crossref.org/works/10.1145/3593013.3594087 on 2026-10-30
@inproceedings{lai2023humanai,
  author = {Lai, Vivian and Chen, Chacha and Smith-Renner, Alison and Liao, Q. Vera and Tan, Chenhao},
  title = {{Towards a Science of Human-AI Decision Making: An Overview of Design Space in Empirical Human-Subject Studies}},
  year = {2023},
  booktitle = {2023 ACM Conference on Fairness Accountability and Transparency},
  pages = {1369--1385},
  doi = {10.1145/3593013.3594087},
  eprint = {2112.11471},
  archivePrefix = {arXiv}
}

% verified via https://api.crossref.org/works/10.1145/3411764.3445717 on 2026-10-30
@inproceedings{bansal2021whole,
  author = {Bansal, Gagan and Wu, Tongshuang and Zhou, Joyce and Fok, Raymond and Nushi, Besmira and Kamar, Ece and Ribeiro, Marco Tulio and Weld, Daniel},
  title = {{Does the Whole Exceed its Parts? The Effect of AI Explanations on Complementary Team Performance}},
  year = {2021},
  booktitle = {Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems},
  pages = {1--16},
  doi = {10.1145/3411764.3445717}
}

% verified via https://export.arxiv.org/api/query?id_list=1902.03245 on 2026-10-30
@inproceedings{lubars2019delegability,
  author = {Brian Lubars and Chenhao Tan},
  title = {{Ask Not What AI Can Do, But What AI Should Do: Towards a Framework of Task Delegability}},
  year = {2019},
  booktitle = {NeurIPS 2019},
  eprint = {1902.03245},
  archivePrefix = {arXiv}
}

% verified via https://api.crossref.org/works/10.1038/s41562-024-02024-1 on 2026-10-30
@article{vaccaro2024combinations,
  author = {Vaccaro, Michelle and Almaatouq, Abdullah and Malone, Thomas},
  title = {{When combinations of humans and AI are useful: A systematic review and meta-analysis}},
  year = {2024},
  journal = {Nature Human Behaviour},
  volume = {8},
  number = {12},
  pages = {2293--2303},
  doi = {10.1038/s41562-024-02024-1}
}

% verified via https://api.crossref.org/works/10.1016/0005-1098(83)90046-8 on 2026-10-30
@article{bainbridge1983ironies,
  author = {Bainbridge, Lisanne},
  title = {{Ironies of automation}},
  year = {1983},
  journal = {Automatica},
  volume = {19},
  number = {6},
  pages = {775--779},
  doi = {10.1016/0005-1098(83)90046-8}
}

% verified via https://api.crossref.org/works/10.1145/3449287 on 2026-10-30
@article{bucinca2021trust,
  author = {Buçinca, Zana and Malaya, Maja Barbara and Gajos, Krzysztof Z.},
  title = {{To Trust or to Think}},
  year = {2021},
  journal = {Proceedings of the ACM on Human-Computer Interaction},
  volume = {5},
  number = {CSCW1},
  pages = {1--21},
  doi = {10.1145/3449287}
}

% verified via https://export.arxiv.org/api/query?id_list=2507.09089 on 2026-10-30
@misc{becker2025metr,
  author = {Joel Becker and Nate Rush and Elizabeth Barnes and David Rein},
  title = {{Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity}},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2507.09089},
  eprint = {2507.09089},
  archivePrefix = {arXiv}
}

% verified via https://api.crossref.org/works/10.1145/2408776.2408794 on 2026-10-30
@article{dean2013tail,
  author = {Dean, Jeffrey and Barroso, Luiz André},
  title = {{The tail at scale}},
  year = {2013},
  journal = {Communications of the ACM},
  volume = {56},
  number = {2},
  pages = {74--80},
  doi = {10.1145/2408776.2408794}
}

% verified via https://api.semanticscholar.org/graph/v1/paper/87c9e34c39dd1fbd470010dc1ea98ba3517f5a2a on 2026-10-30
@inproceedings{gray1986whystop,
  author = {Jim Gray},
  title = {{Why Do Computers Stop and What Can Be Done About It?}},
  year = {1986},
  booktitle = {Symposium on Reliability in Distributed Software and Database Systems}
}

% verified via https://api.crossref.org/works/10.1109/MS.2016.60 on 2026-10-30
@article{basiri2016chaos,
  author = {Basiri, Ali and Behnam, Niosha and de Rooij, Ruud and Hochstein, Lorin and Kosewski, Luke and Reynolds, Justin and Rosenthal, Casey},
  title = {{Chaos Engineering}},
  year = {2016},
  journal = {IEEE Software},
  volume = {33},
  number = {3},
  pages = {35--41},
  doi = {10.1109/MS.2016.60}
}

% verified via https://api.crossref.org/works/10.1145/2160718.2160734 on 2026-10-30
@article{helland2012idempotence,
  author = {Helland, Pat},
  title = {{Idempotence is not a medical condition}},
  year = {2012},
  journal = {Communications of the ACM},
  volume = {55},
  number = {5},
  pages = {56--65},
  doi = {10.1145/2160718.2160734}
}

% verified via https://api.crossref.org/works/10.1145/3485510 on 2026-10-30
@article{burckhardt2021durablefunctions,
  author = {Burckhardt, Sebastian and Gillum, Chris and Justo, David and Kallas, Konstantinos and McMahon, Connor and Meiklejohn, Christopher S.},
  title = {{Durable functions: semantics for stateful serverless}},
  year = {2021},
  journal = {Proceedings of the ACM on Programming Languages},
  volume = {5},
  number = {OOPSLA},
  pages = {1--27},
  doi = {10.1145/3485510}
}

% verified via https://export.arxiv.org/api/query?id_list=2504.13359 on 2026-10-30
@misc{erol2025costofpass,
  author = {Mehmet Hamza Erol and Batu El and Mirac Suzgun and Mert Yuksekgonul and James Zou},
  title = {{Cost-of-Pass: An Economic Framework for Evaluating Language Models}},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2504.13359},
  eprint = {2504.13359},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2411.05285 on 2026-10-30
@misc{dong2024agentops,
  author = {Liming Dong and Qinghua Lu and Liming Zhu},
  title = {{AgentOps: Enabling Observability of LLM Agents}},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2411.05285},
  eprint = {2411.05285},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2603.29231 on 2026-10-30
@misc{khanal2026reliability,
  author = {Aaditya Khanal and Yangyang Tao and Junxiu Zhou},
  title = {{Beyond pass@1: A Reliability Science Framework for Long-Horizon LLM Agents}},
  year = {2026},
  howpublished = {arXiv preprint arXiv:2603.29231},
  eprint = {2603.29231},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2608.14380 on 2026-10-30
@misc{zhuang2026agentrewind,
  author = {Yu Zhuang and Kefei Chen and Yitong Duan and Shuxin Zheng and Jian Li and Xu-Yao Zhang},
  title = {{AgentRewind: Recoverable Execution for Long-Horizon LLM Agents}},
  year = {2026},
  howpublished = {arXiv preprint arXiv:2608.14380},
  eprint = {2608.14380},
  archivePrefix = {arXiv}
}

% verified via https://api.semanticscholar.org/graph/v1/paper/003d5a65de0ac72daaf105ded903cb3eb88585b3 on 2026-10-30
@techreport{sigelman2010dapper,
  author = {B. Sigelman and L. Barroso and M. Burrows and Patrick Stephenson and M. Plakal and Donald Beaver and Saul Jaspan and C. Shanbhag},
  title = {{Dapper, a Large-Scale Distributed Systems Tracing Infrastructure}},
  year = {2010},
  institution = {Google},
  number = {dapper-2010-1}
}

% verified via https://api.crossref.org/works/10.18653/v1/2020.emnlp-main.609 on 2026-10-30
@inproceedings{wadden2020scifact,
  author = {Wadden, David and Lin, Shanchuan and Lo, Kyle and Wang, Lucy Lu and van Zuylen, Madeleine and Cohan, Arman and Hajishirzi, Hannaneh},
  title = {{Fact or Fiction: Verifying Scientific Claims}},
  year = {2020},
  booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {7534--7550},
  doi = {10.18653/v1/2020.emnlp-main.609}
}

% verified via https://api.crossref.org/works/10.18653/v1/N18-1074 on 2026-10-30
@inproceedings{thorne2018fever,
  author = {Thorne, James and Vlachos, Andreas and Christodoulopoulos, Christos and Mittal, Arpit},
  title = {{FEVER: a Large-scale Dataset for Fact Extraction and VERification}},
  year = {2018},
  booktitle = {Proceedings of the 2018 Conference of the North American Chapter of
          the Association for Computational Linguistics: Human Language
          Technologies, Volume 1 (Long Papers)},
  pages = {809--819},
  doi = {10.18653/v1/N18-1074}
}

% verified via https://export.arxiv.org/api/query?id_list=2212.08037 on 2026-10-30
@misc{bohnet2022attributedqa,
  author = {Bernd Bohnet and Vinh Q. Tran and Pat Verga and Roee Aharoni and Daniel Andor and Livio Baldini Soares and Massimiliano Ciaramita and Jacob Eisenstein and Kuzman Ganchev and Jonathan Herzig and Kai Hui and Tom Kwiatkowski and Ji Ma and Jianmo Ni and Lierni Sestorain Saralegui and Tal Schuster and William W. Cohen and Michael Collins and Dipanjan Das and Donald Metzler and Slav Petrov and Kellie Webster},
  title = {{Attributed Question Answering: Evaluation and Modeling for Attributed Large Language Models}},
  year = {2022},
  howpublished = {arXiv preprint arXiv:2212.08037},
  eprint = {2212.08037},
  archivePrefix = {arXiv}
}

% verified via https://api.crossref.org/works/10.1038/s41598-023-41032-5 on 2026-10-30
@article{walters2023fabrication,
  author = {Walters, William H. and Wilder, Esther Isabelle},
  title = {{Fabrication and errors in the bibliographic citations generated by ChatGPT}},
  year = {2023},
  journal = {Scientific Reports},
  volume = {13},
  number = {1},
  doi = {10.1038/s41598-023-41032-5}
}

% verified via https://api.crossref.org/works/10.7759/cureus.35179 on 2026-10-30
@article{alkaissi2023hallucinations,
  author = {Alkaissi, Hussam and McFarlane, Samy I},
  title = {{Artificial Hallucinations in ChatGPT: Implications in Scientific Writing}},
  year = {2023},
  journal = {Cureus},
  doi = {10.7759/cureus.35179}
}

% verified via https://api.crossref.org/works/10.18653/v1/2023.acl-long.910 on 2026-10-30
@inproceedings{gao2023rarr,
  author = {Gao, Luyu and Dai, Zhuyun and Pasupat, Panupong and Chen, Anthony and Chaganty, Arun Tejasvi and Fan, Yicheng and Zhao, Vincent and Lao, Ni and Lee, Hongrae and Juan, Da-Cheng and Guu, Kelvin},
  title = {{RARR: Researching and Revising What Language Models Say, Using Language Models}},
  year = {2023},
  booktitle = {Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  pages = {16477--16508},
  doi = {10.18653/v1/2023.acl-long.910}
}

% verified via https://export.arxiv.org/api/query?id_list=2203.11171 on 2026-10-30
@inproceedings{wang2023selfconsistency,
  author = {Xuezhi Wang and Jason Wei and Dale Schuurmans and Quoc Le and Ed Chi and Sharan Narang and Aakanksha Chowdhery and Denny Zhou},
  title = {{Self-Consistency Improves Chain of Thought Reasoning in Language Models}},
  year = {2022},
  booktitle = {ICLR 2023},
  eprint = {2203.11171},
  archivePrefix = {arXiv}
}

% verified via https://api.crossref.org/works/10.18653/v1/2023.emnlp-main.741 on 2026-10-30
@inproceedings{min2023factscore,
  author = {Min, Sewon and Krishna, Kalpesh and Lyu, Xinxi and Lewis, Mike and Yih, Wen-tau and Koh, Pang and Iyyer, Mohit and Zettlemoyer, Luke and Hajishirzi, Hannaneh},
  title = {{FActScore: Fine-grained Atomic Evaluation of Factual Precision in Long Form Text Generation}},
  year = {2023},
  booktitle = {Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},
  pages = {12076--12100},
  doi = {10.18653/v1/2023.emnlp-main.741},
  eprint = {2305.14251},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2310.01798 on 2026-10-30
@inproceedings{huang2024selfcorrect,
  author = {Jie Huang and Xinyun Chen and Swaroop Mishra and Huaixiu Steven Zheng and Adams Wei Yu and Xinying Song and Denny Zhou},
  title = {{Large Language Models Cannot Self-Correct Reasoning Yet}},
  year = {2023},
  booktitle = {ICLR 2024},
  eprint = {2310.01798},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2601.18724 on 2026-10-30
@inproceedings{sakai2026hallucitation,
  author = {Yusuke Sakai and Hidetaka Kamigaito and Taro Watanabe},
  title = {{HalluCitation Matters: Revealing the Impact of Hallucinated References with 300 Hallucinated Papers in ACL Conferences}},
  year = {2026},
  booktitle = {ACL 2026 (per Semantic Scholar venue record)},
  eprint = {2601.18724},
  archivePrefix = {arXiv}
}

% verified via https://export.arxiv.org/api/query?id_list=2605.07723 on 2026-10-30
@misc{zhao2026hallucinationswild,
  author = {Zhenyue Zhao and Yihe Wang and Toby Stuart and Mathijs De Vaan and Paul Ginsparg and Yian Yin},
  title = {{LLM hallucinations in the wild: Large-scale evidence from non-existent citations}},
  year = {2026},
  howpublished = {arXiv preprint arXiv:2605.07723},
  eprint = {2605.07723},
  archivePrefix = {arXiv}
}