% WP42 literature base -- every entry verified against Crossref, arXiv, or the official publisher page on 2026-11-02.
% See literature/VERIFICATION_LOG.csv for the exact URL fetched per key. Keys: authorYEARkeyword.

@article{abadie2023clustering,
  author = {Abadie, Alberto and Athey, Susan and Imbens, Guido W and Wooldridge, Jeffrey M},
  title = {When Should You Adjust Standard Errors for Clustering?},
  journal = {The Quarterly Journal of Economics},
  year = {2023},
  volume = {138},
  number = {1},
  pages = {1--35},
  doi = {10.1093/qje/qjac038},
  url = {https://doi.org/10.1093/qje/qjac038},
  note = {verified: crossref 2026-11-02. Crossref issued 2022-10-06 (online first); print volume 138(1), 2023.}
}

@incollection{acemoglu2011skills,
  author = {Acemoglu, Daron and Autor, David},
  title = {Skills, Tasks and Technologies: Implications for Employment and Earnings},
  booktitle = {Handbook of Labor Economics},
  year = {2011},
  publisher = {Elsevier},
  doi = {10.1016/S0169-7218(11)02410-5},
  url = {https://doi.org/10.1016/S0169-7218(11)02410-5},
  note = {verified: crossref 2026-11-02}
}

@article{acemoglu2018race,
  author = {Acemoglu, Daron and Restrepo, Pascual},
  title = {The Race between Man and Machine: Implications of Technology for Growth, Factor Shares, and Employment},
  journal = {American Economic Review},
  year = {2018},
  volume = {108},
  number = {6},
  pages = {1488--1542},
  doi = {10.1257/aer.20160696},
  url = {https://doi.org/10.1257/aer.20160696},
  note = {verified: crossref 2026-11-02}
}

@article{acemoglu2019automation,
  author = {Acemoglu, Daron and Restrepo, Pascual},
  title = {Automation and New Tasks: How Technology Displaces and Reinstates Labor},
  journal = {Journal of Economic Perspectives},
  year = {2019},
  volume = {33},
  number = {2},
  pages = {3--30},
  doi = {10.1257/jep.33.2.3},
  url = {https://doi.org/10.1257/jep.33.2.3},
  note = {verified: crossref 2026-11-02}
}

@article{acemoglu2020robots,
  author = {Acemoglu, Daron and Restrepo, Pascual},
  title = {Robots and Jobs: Evidence from US Labor Markets},
  journal = {Journal of Political Economy},
  year = {2020},
  volume = {128},
  number = {6},
  pages = {2188--2244},
  doi = {10.1086/705716},
  url = {https://doi.org/10.1086/705716},
  note = {verified: crossref 2026-11-02}
}

@article{acemoglu2025macroeconomics,
  author = {Acemoglu, Daron},
  title = {The simple macroeconomics of AI},
  journal = {Economic Policy},
  year = {2025},
  volume = {40},
  number = {121},
  pages = {13--58},
  doi = {10.1093/epolic/eiae042},
  url = {https://doi.org/10.1093/epolic/eiae042},
  note = {verified: crossref 2026-11-02}
}

@techreport{agarwal2023radiology,
  author = {Agarwal, Nikhil and Moehring, Alex and Rajpurkar, Pranav and Salz, Tobias},
  title = {Combining Human Expertise with Artificial Intelligence: Experimental Evidence from Radiology},
  year = {2023},
  number = {31422},
  institution = {National Bureau of Economic Research},
  type = {NBER Working Paper},
  doi = {10.3386/w31422},
  url = {https://doi.org/10.3386/w31422},
  note = {verified: crossref 2026-11-02}
}

@article{agrawal2019prediction,
  author = {Agrawal, Ajay and Gans, Joshua S. and Goldfarb, Avi},
  title = {Artificial Intelligence: The Ambiguous Labor Market Impact of Automating Prediction},
  journal = {Journal of Economic Perspectives},
  year = {2019},
  volume = {33},
  number = {2},
  pages = {31--50},
  doi = {10.1257/jep.33.2.31},
  url = {https://doi.org/10.1257/jep.33.2.31},
  note = {verified: crossref 2026-11-02}
}

@misc{anthropic2024computeruse,
  author = {{Anthropic}},
  title = {Introducing computer use, a new Claude 3.5 Sonnet, and Claude 3.5 Haiku},
  year = {2024},
  month = oct,
  howpublished = {Anthropic announcement, 22 October 2024},
  url = {https://www.anthropic.com/news/3-5-models-and-computer-use},
  note = {verified: web (official Anthropic page fetched 2026-11-02; H1 and datePublished 2024-10-22 match)}
}

@techreport{anthropic2025opus4card,
  author = {{Anthropic}},
  title = {System Card: Claude Opus 4 \& Claude Sonnet 4},
  year = {2025},
  month = may,
  institution = {Anthropic},
  url = {https://assets.claude.com/6d8a8055020700718b0c49369f60816ba2a7c285.pdf},
  note = {verified: web (PDF fetched 2026-11-02; title page reads "System Card: Claude Opus 4 \& Claude Sonnet 4, May 2025"; changelog to 2 Sept. 2025)}
}

@misc{appel2025uneven,
  author = {Appel, Ruth and McCrory, Peter and Tamkin, Alex and McCain, Miles and Neylon, Tyler and Stern, Michael},
  title = {Anthropic Economic Index report: Uneven geographic and enterprise AI adoption},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2511.15080},
  eprint = {2511.15080},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2511.15080},
  note = {verified: arxiv 2511.15080}
}

@article{arntz2017revisiting,
  author = {Arntz, Melanie and Gregory, Terry and Zierahn, Ulrich},
  title = {Revisiting the risk of automation},
  journal = {Economics Letters},
  year = {2017},
  volume = {159},
  pages = {157--160},
  doi = {10.1016/j.econlet.2017.07.001},
  url = {https://doi.org/10.1016/j.econlet.2017.07.001},
  note = {verified: crossref 2026-11-02}
}

@misc{athey2016econometrics,
  author = {Athey, Susan and Imbens, Guido},
  title = {The Econometrics of Randomized Experiments},
  year = {2016},
  howpublished = {arXiv preprint arXiv:1607.00698},
  eprint = {1607.00698},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/1607.00698},
  note = {verified: arxiv 1607.00698. Published as chapter 3 of the Handbook of Economic Field Experiments, vol. 1 (2017), doi:10.1016/bs.hefe.2016.10.003.}
}

@article{athey2020allocation,
  author = {Athey, Susan C. and Bryan, Kevin A. and Gans, Joshua S.},
  title = {The Allocation of Decision Authority to Human and Artificial Intelligence},
  journal = {AEA Papers and Proceedings},
  year = {2020},
  volume = {110},
  pages = {80--84},
  doi = {10.1257/pandp.20201034},
  url = {https://doi.org/10.1257/pandp.20201034},
  note = {verified: crossref 2026-11-02}
}

@article{autor2003skill,
  author = {Autor, D. H. and Levy, F. and Murnane, R. J.},
  title = {The Skill Content of Recent Technological Change: An Empirical Exploration},
  journal = {The Quarterly Journal of Economics},
  year = {2003},
  volume = {118},
  number = {4},
  pages = {1279--1333},
  doi = {10.1162/003355303322552801},
  url = {https://doi.org/10.1162/003355303322552801},
  note = {verified: crossref 2026-11-02}
}

@article{autor2015jobs,
  author = {Autor, David H.},
  title = {Why Are There Still So Many Jobs? The History and Future of Workplace Automation},
  journal = {Journal of Economic Perspectives},
  year = {2015},
  volume = {29},
  number = {3},
  pages = {3--30},
  doi = {10.1257/jep.29.3.3},
  url = {https://doi.org/10.1257/jep.29.3.3},
  note = {verified: crossref 2026-11-02}
}

@misc{backlund2025vendingbench,
  author = {Backlund, Axel and Petersson, Lukas},
  title = {Vending-Bench: A Benchmark for Long-Term Coherence of Autonomous Agents},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2502.15840},
  eprint = {2502.15840},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2502.15840},
  note = {verified: arxiv 2502.15840}
}

@article{bates2015lme4,
  author = {Bates, Douglas and Mächler, Martin and Bolker, Ben and Walker, Steve},
  title = {Fitting Linear Mixed-Effects Models Using lme4},
  journal = {Journal of Statistical Software},
  year = {2015},
  volume = {67},
  number = {1},
  doi = {10.18637/jss.v067.i01},
  url = {https://doi.org/10.18637/jss.v067.i01},
  note = {verified: crossref 2026-11-02}
}

@misc{becker2025metr,
  author = {Becker, Joel and Rush, Nate and Barnes, Elizabeth and Rein, David},
  title = {Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2507.09089},
  eprint = {2507.09089},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2507.09089},
  note = {verified: arxiv 2507.09089}
}

@misc{bowman2022oversight,
  author = {Bowman, Samuel R. and Hyun, Jeeyoon and Perez, Ethan and Chen, Edwin and Pettit, Craig and Heiner, Scott and Lukošiūtė, Kamilė and Askell, Amanda and Jones, Andy and Chen, Anna and Goldie, Anna and Mirhoseini, Azalia and McKinnon, Cameron and Olah, Christopher and Amodei, Daniela and Amodei, Dario and Drain, Dawn and Li, Dustin and Tran-Johnson, Eli and Kernion, Jackson and Kerr, Jamie and Mueller, Jared and Ladish, Jeffrey and Landau, Joshua and Ndousse, Kamal and Lovitt, Liane and Elhage, Nelson and Schiefer, Nicholas and Joseph, Nicholas and Mercado, Noemí and DasSarma, Nova and Larson, Robin and McCandlish, Sam and Kundu, Sandipan and Johnston, Scott and Kravec, Shauna and Showk, Sheer El and Fort, Stanislav and Telleen-Lawton, Timothy and Brown, Tom and Henighan, Tom and Hume, Tristan and Bai, Yuntao and Hatfield-Dodds, Zac and Mann, Ben and Kaplan, Jared},
  title = {Measuring Progress on Scalable Oversight for Large Language Models},
  year = {2022},
  howpublished = {arXiv preprint arXiv:2211.03540},
  eprint = {2211.03540},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2211.03540},
  note = {verified: arxiv 2211.03540}
}

@article{brynjolfsson2017workforce,
  author = {Brynjolfsson, Erik and Mitchell, Tom},
  title = {What can machine learning do? Workforce implications},
  journal = {Science},
  year = {2017},
  volume = {358},
  number = {6370},
  pages = {1530--1534},
  doi = {10.1126/science.aap8062},
  url = {https://doi.org/10.1126/science.aap8062},
  note = {verified: crossref 2026-11-02}
}

@article{brynjolfsson2018machines,
  author = {Brynjolfsson, Erik and Mitchell, Tom and Rock, Daniel},
  title = {What Can Machines Learn and What Does It Mean for Occupations and the Economy?},
  journal = {AEA Papers and Proceedings},
  year = {2018},
  volume = {108},
  pages = {43--47},
  doi = {10.1257/pandp.20181019},
  url = {https://doi.org/10.1257/pandp.20181019},
  note = {verified: crossref 2026-11-02}
}

@techreport{brynjolfsson2025canaries,
  author = {Brynjolfsson, Erik and Chandar, Bharat and Chen, Ruyu},
  title = {Canaries in the Coal Mine? Six Facts about the Recent Employment Effects of Artificial Intelligence},
  year = {2025},
  institution = {Stanford Digital Economy Lab},
  type = {Working Paper},
  url = {https://digitaleconomy.stanford.edu/publication/canaries-in-the-coal-mine-six-facts-about-the-recent-employment-effects-of-artificial-intelligence},
  note = {verified: web (official Stanford Digital Economy Lab page fetched 2026-11-02; authors and title match; first version August 2025, revised version dated 2026 on the page)}
}

@article{brynjolfsson2025generative,
  author = {Brynjolfsson, Erik and Li, Danielle and Raymond, Lindsey},
  title = {Generative AI at Work},
  journal = {The Quarterly Journal of Economics},
  year = {2025},
  volume = {140},
  number = {2},
  pages = {889--942},
  doi = {10.1093/qje/qjae044},
  url = {https://doi.org/10.1093/qje/qjae044},
  note = {verified: crossref 2026-11-02}
}

@article{cameron2008bootstrap,
  author = {Cameron, A. Colin and Gelbach, Jonah B. and Miller, Douglas L.},
  title = {Bootstrap-Based Improvements for Inference with Clustered Errors},
  journal = {Review of Economics and Statistics},
  year = {2008},
  volume = {90},
  number = {3},
  pages = {414--427},
  doi = {10.1162/rest.90.3.414},
  url = {https://doi.org/10.1162/rest.90.3.414},
  note = {verified: crossref 2026-11-02}
}

@article{cameron2015practitioner,
  author = {Colin Cameron, A. and Miller, Douglas L.},
  title = {A Practitioner’s Guide to Cluster-Robust Inference},
  journal = {Journal of Human Resources},
  year = {2015},
  volume = {50},
  number = {2},
  pages = {317--372},
  doi = {10.3368/jhr.50.2.317},
  url = {https://doi.org/10.3368/jhr.50.2.317},
  note = {verified: crossref 2026-11-02}
}

@misc{catalini2026agi,
  author = {Catalini, Christian and Hui, Xiang and Wu, Jane},
  title = {Some Simple Economics of AGI},
  year = {2026},
  howpublished = {arXiv preprint arXiv:2602.20946},
  eprint = {2602.20946},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2602.20946},
  note = {verified: arxiv 2602.20946}
}

@techreport{cazzaniga2024genai,
  author = {Cazzaniga, Mauro and Jaumotte, Florence and Li, Longji and Melina, Giovanni and Panton, Augustus and Pizzinelli, Carlo and Rockall, Emma and Mendes Tavares, Marina},
  title = {Gen-AI: Artificial Intelligence and the Future of Work},
  year = {2024},
  number = {SDN/2024/001},
  institution = {International Monetary Fund},
  type = {IMF Staff Discussion Note},
  doi = {10.5089/9798400262548.006},
  url = {https://doi.org/10.5089/9798400262548.006},
  note = {verified: crossref 2026-11-02}
}

@misc{cemri2025fail,
  author = {Cemri, Mert and Pan, Melissa Z. and Yang, Shuyi and Agrawal, Lakshya A. and Chopra, Bhavya and Tiwari, Rishabh and Keutzer, Kurt and Parameswaran, Aditya and Klein, Dan and Ramchandran, Kannan and Zaharia, Matei and Gonzalez, Joseph E. and Stoica, Ion},
  title = {Why Do Multi-Agent LLM Systems Fail?},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2503.13657},
  eprint = {2503.13657},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2503.13657},
  note = {verified: arxiv 2503.13657}
}

@misc{chan2024mlebench,
  author = {Chan, Jun Shern and Chowdhury, Neil and Jaffe, Oliver and Aung, James and Sherburn, Dane and Mays, Evan and Starace, Giulio and Liu, Kevin and Maksin, Leon and Patwardhan, Tejal and Weng, Lilian and Mądry, Aleksander},
  title = {MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2410.07095},
  eprint = {2410.07095},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2410.07095},
  note = {verified: arxiv 2410.07095}
}

@misc{chen2023selfdebug,
  author = {Chen, Xinyun and Lin, Maxwell and Schärli, Nathanael and Zhou, Denny},
  title = {Teaching Large Language Models to Self-Debug},
  year = {2023},
  howpublished = {arXiv preprint arXiv:2304.05128},
  eprint = {2304.05128},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2304.05128},
  note = {verified: arxiv 2304.05128}
}

@misc{choi2023lawyering,
  author = {Choi, Jonathan H. and Monahan, Amy and Schwarcz, Daniel},
  title = {Lawyering in the Age of Artificial Intelligence},
  year = {2023},
  howpublished = {SSRN working paper},
  doi = {10.2139/ssrn.4626276},
  url = {https://doi.org/10.2139/ssrn.4626276},
  note = {verified: crossref 2026-11-02}
}

@article{cox1972regression,
  author = {Cox, D. R.},
  title = {Regression Models and Life-Tables},
  journal = {Journal of the Royal Statistical Society Series B: Statistical Methodology},
  year = {1972},
  volume = {34},
  number = {2},
  pages = {187--202},
  doi = {10.1111/j.2517-6161.1972.tb00899.x},
  url = {https://doi.org/10.1111/j.2517-6161.1972.tb00899.x},
  note = {verified: crossref 2026-11-02}
}

@misc{cui2024developers,
  author = {Cui, Zheyuan and Demirer, Mert and Jaffe, Sonia and Musolff, Leon and Peng, Sida and Salz, Tobias},
  title = {The Effects of Generative AI on High Skilled Work: Evidence from Three Field Experiments with Software Developers},
  year = {2024},
  howpublished = {SSRN working paper},
  doi = {10.2139/ssrn.4945566},
  url = {https://doi.org/10.2139/ssrn.4945566},
  note = {verified: crossref 2026-11-02}
}

@techreport{dellacqua2023jagged,
  author = {Dell'Acqua, Fabrizio and McFowland III, Edward and Mollick, Ethan R. and Lifshitz-Assaf, Hila and Kellogg, Katherine and Rajendran, Saran and Krayer, Lisa and Candelon, François and Lakhani, Karim R.},
  title = {Navigating the Jagged Technological Frontier: Field Experimental Evidence of the Effects of Artificial Intelligence on Knowledge Worker Productivity and Quality},
  year = {2023},
  number = {24-013},
  institution = {Harvard Business School},
  type = {Working Paper},
  doi = {10.2139/ssrn.4573321},
  url = {https://doi.org/10.2139/ssrn.4573321},
  note = {verified: crossref 2026-11-02. SSRN posted content.}
}

@techreport{dellacqua2025cybernetic,
  author = {Dell'Acqua, Fabrizio and Ayoubi, Charles and Lifshitz-Assaf, Hila and Sadun, Raffaella and Mollick, Ethan R. and Mollick, Lilach and Han, Yi and Goldman, Jeff and Nair, Hari and Taub, Stew and Lakhani, Karim R.},
  title = {The Cybernetic Teammate: A Field Experiment on Generative AI Reshaping Teamwork and Expertise},
  year = {2025},
  number = {25-043},
  institution = {Harvard Business School},
  type = {Working Paper},
  doi = {10.2139/ssrn.5188231},
  url = {https://doi.org/10.2139/ssrn.5188231},
  note = {verified: crossref 2026-11-02. SSRN posted content.}
}

@article{dietvorst2015aversion,
  author = {Dietvorst, Berkeley J. and Simmons, Joseph P. and Massey, Cade},
  title = {Algorithm aversion: People erroneously avoid algorithms after seeing them err.},
  journal = {Journal of Experimental Psychology: General},
  year = {2015},
  volume = {144},
  number = {1},
  pages = {114--126},
  doi = {10.1037/xge0000033},
  url = {https://doi.org/10.1037/xge0000033},
  note = {verified: crossref 2026-11-02}
}

@misc{drouin2024workarena,
  author = {Drouin, Alexandre and Gasse, Maxime and Caccia, Massimo and Laradji, Issam H. and Del Verme, Manuel and Marty, Tom and Boisvert, Léo and Thakkar, Megh and Cappart, Quentin and Vazquez, David and Chapados, Nicolas and Lacoste, Alexandre},
  title = {WorkArena: How Capable Are Web Agents at Solving Common Knowledge Work Tasks?},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2403.07718},
  eprint = {2403.07718},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2403.07718},
  note = {verified: arxiv 2403.07718}
}

@misc{dziri2023faith,
  author = {Dziri, Nouha and Lu, Ximing and Sclar, Melanie and Li, Xiang Lorraine and Jiang, Liwei and Lin, Bill Yuchen and West, Peter and Bhagavatula, Chandra and Bras, Ronan Le and Hwang, Jena D. and Sanyal, Soumya and Welleck, Sean and Ren, Xiang and Ettinger, Allyson and Harchaoui, Zaid and Choi, Yejin},
  title = {Faith and Fate: Limits of Transformers on Compositionality},
  year = {2023},
  howpublished = {arXiv preprint arXiv:2305.18654},
  eprint = {2305.18654},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2305.18654},
  note = {verified: arxiv 2305.18654}
}

@article{eloundou2024gpts,
  author = {Eloundou, Tyna and Manning, Sam and Mishkin, Pamela and Rock, Daniel},
  title = {GPTs are GPTs: Labor market impact potential of LLMs},
  journal = {Science},
  year = {2024},
  volume = {384},
  number = {6702},
  pages = {1306--1308},
  doi = {10.1126/science.adj0998},
  url = {https://doi.org/10.1126/science.adj0998},
  note = {verified: crossref 2026-11-02}
}

@article{felten2021exposure,
  author = {Felten, Edward and Raj, Manav and Seamans, Robert},
  title = {Occupational, industry, and geographic exposure to artificial intelligence: A novel dataset and its potential uses},
  journal = {Strategic Management Journal},
  year = {2021},
  volume = {42},
  number = {12},
  pages = {2195--2217},
  doi = {10.1002/smj.3286},
  url = {https://doi.org/10.1002/smj.3286},
  note = {verified: crossref 2026-11-02}
}

@misc{felten2023language,
  author = {Felten, Ed and Raj, Manav and Seamans, Robert},
  title = {How will Language Modelers like ChatGPT Affect Occupations and Industries?},
  year = {2023},
  howpublished = {arXiv preprint arXiv:2303.01157},
  eprint = {2303.01157},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2303.01157},
  note = {verified: arxiv 2303.01157}
}

@article{fine1999competing,
  author = {Fine, Jason P. and Gray, Robert J.},
  title = {A Proportional Hazards Model for the Subdistribution of a Competing Risk},
  journal = {Journal of the American Statistical Association},
  year = {1999},
  volume = {94},
  number = {446},
  pages = {496--509},
  doi = {10.1080/01621459.1999.10474144},
  url = {https://doi.org/10.1080/01621459.1999.10474144},
  note = {verified: crossref 2026-11-02}
}

@article{frey2017future,
  author = {Frey, Carl Benedikt and Osborne, Michael A.},
  title = {The future of employment: How susceptible are jobs to computerisation?},
  journal = {Technological Forecasting and Social Change},
  year = {2017},
  volume = {114},
  pages = {254--280},
  doi = {10.1016/j.techfore.2016.08.019},
  url = {https://doi.org/10.1016/j.techfore.2016.08.019},
  note = {verified: crossref 2026-11-02}
}

@techreport{gmyrek2023generative,
  author = {Gmyrek, Pawel and Berg, Janine and Bescond, David},
  title = {Generative AI and jobs: A global analysis of potential effects on job quantity and quality},
  year = {2023},
  number = {96},
  institution = {International Labour Organization},
  type = {ILO Working Paper},
  address = {Geneva},
  doi = {10.54394/FHEM8239},
  url = {https://doi.org/10.54394/FHEM8239},
  note = {verified: crossref 2026-11-02}
}

@techreport{gmyrek2025refined,
  author = {Gmyrek, Pawel and Berg, Janine and Kamiński, Karol and Konopczyński, Filip and Ładna, Agnieszka and Nafradi, Balint and Rosłaniec, Konrad and Troszyński, Marek},
  title = {Generative AI and jobs: A refined global index of occupational exposure},
  year = {2025},
  number = {140},
  institution = {International Labour Organization},
  type = {ILO Working Paper},
  address = {Geneva},
  doi = {10.54394/HETP0387},
  url = {https://doi.org/10.54394/HETP0387},
  note = {verified: crossref 2026-11-02}
}

@misc{gu2024survey,
  author = {Gu, Jiawei and Jiang, Xuhui and Shi, Zhichao and Tan, Hexiang and Zhai, Xuehao and Xu, Chengjin and Li, Wei and Shen, Yinghan and Ma, Shengjie and Liu, Honghao and Wang, Saizhuo and Zhang, Kun and Wang, Yuanzhuo and Gao, Wen and Ni, Lionel and Guo, Jian},
  title = {A Survey on LLM-as-a-Judge},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2411.15594},
  eprint = {2411.15594},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2411.15594},
  note = {verified: arxiv 2411.15594}
}

@misc{guha2023legalbench,
  author = {Guha, Neel and Nyarko, Julian and Ho, Daniel E. and Ré, Christopher and Chilton, Adam and Narayana, Aditya and Chohlas-Wood, Alex and Peters, Austin and Waldon, Brandon and Rockmore, Daniel N. and Zambrano, Diego and Talisman, Dmitry and Hoque, Enam and Surani, Faiz and Fagan, Frank and Sarfaty, Galit and Dickinson, Gregory M. and Porat, Haggai and Hegland, Jason and Wu, Jessica and Nudell, Joe and Niklaus, Joel and Nay, John and Choi, Jonathan H. and Tobia, Kevin and Hagan, Margaret and Ma, Megan and Livermore, Michael and Rasumov-Rahe, Nikon and Holzenberger, Nils and Kolt, Noam and Henderson, Peter and Rehaag, Sean and Goel, Sharad and Gao, Shang and Williams, Spencer and Gandhi, Sunny and Zur, Tom and Iyer, Varun and Li, Zehua},
  title = {LegalBench: A Collaboratively Built Benchmark for Measuring Legal Reasoning in Large Language Models},
  year = {2023},
  howpublished = {arXiv preprint arXiv:2308.11462},
  eprint = {2308.11462},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2308.11462},
  note = {verified: arxiv 2308.11462}
}

@article{gwet2008interrater,
  author = {Gwet, Kilem Li},
  title = {Computing inter‐rater reliability and its variance in the presence of high agreement},
  journal = {British Journal of Mathematical and Statistical Psychology},
  year = {2008},
  volume = {61},
  number = {1},
  pages = {29--48},
  doi = {10.1348/000711006X126600},
  url = {https://doi.org/10.1348/000711006X126600},
  note = {verified: crossref 2026-11-02}
}

@misc{hadfield2025economy,
  author = {Hadfield, Gillian K. and Koh, Andrew},
  title = {An Economy of AI Agents},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2509.01063},
  eprint = {2509.01063},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2509.01063},
  note = {verified: arxiv 2509.01063}
}

@techreport{hampole2025labor,
  author = {Hampole, Menaka and Papanikolaou, Dimitris and Schmidt, Lawrence and Seegmiller, Bryan},
  title = {Artificial Intelligence and the Labor Market},
  year = {2025},
  number = {33509},
  institution = {National Bureau of Economic Research},
  type = {NBER Working Paper},
  doi = {10.3386/w33509},
  url = {https://doi.org/10.3386/w33509},
  note = {verified: crossref 2026-11-02}
}

@misc{handa2025economic,
  author = {Handa, Kunal and Tamkin, Alex and McCain, Miles and Huang, Saffron and Durmus, Esin and Heck, Sarah and Mueller, Jared and Hong, Jerry and Ritchie, Stuart and Belonax, Tim and Troy, Kevin K. and Amodei, Dario and Kaplan, Jared and Clark, Jack and Ganguli, Deep},
  title = {Which Economic Tasks are Performed with AI? Evidence from Millions of Claude Conversations},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2503.04761},
  eprint = {2503.04761},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2503.04761},
  note = {verified: arxiv 2503.04761}
}

@article{hayes2007reliability,
  author = {Hayes, Andrew F. and Krippendorff, Klaus},
  title = {Answering the Call for a Standard Reliability Measure for Coding Data},
  journal = {Communication Methods and Measures},
  year = {2007},
  volume = {1},
  number = {1},
  pages = {77--89},
  doi = {10.1080/19312450709336664},
  url = {https://doi.org/10.1080/19312450709336664},
  note = {verified: crossref 2026-11-02}
}

@article{holmstrom1979moral,
  author = {Holmstrom, Bengt},
  title = {Moral Hazard and Observability},
  journal = {The Bell Journal of Economics},
  year = {1979},
  volume = {10},
  number = {1},
  pages = {74},
  doi = {10.2307/3003320},
  url = {https://doi.org/10.2307/3003320},
  note = {verified: crossref 2026-11-02}
}

@article{holmstrom1991multitask,
  author = {Holmstrom, Bengt and Milgrom, Paul},
  title = {Multitask Principal–Agent Analyses: Incentive Contracts, Asset Ownership, and Job Design},
  journal = {The Journal of Law, Economics, and Organization},
  year = {1991},
  volume = {7},
  number = {special\_issue},
  pages = {24--52},
  doi = {10.1093/jleo/7.special_issue.24},
  url = {https://doi.org/10.1093/jleo/7.special_issue.24},
  note = {verified: crossref 2026-11-02}
}

@techreport{horton2023silicus,
  author = {Horton, John and Filippas, Apostolos and Manning, Benjamin},
  title = {Large Language Models as Simulated Economic Agents: What Can We Learn from Homo Silicus?},
  year = {2023},
  number = {31122},
  institution = {National Bureau of Economic Research},
  type = {NBER Working Paper},
  doi = {10.3386/w31122},
  url = {https://doi.org/10.3386/w31122},
  note = {verified: crossref 2026-11-02}
}

@misc{huang2023selfcorrect,
  author = {Huang, Jie and Chen, Xinyun and Mishra, Swaroop and Zheng, Huaixiu Steven and Yu, Adams Wei and Song, Xinying and Zhou, Denny},
  title = {Large Language Models Cannot Self-Correct Reasoning Yet},
  year = {2023},
  howpublished = {arXiv preprint arXiv:2310.01798},
  eprint = {2310.01798},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2310.01798},
  note = {verified: arxiv 2310.01798}
}

@techreport{humlum2025still,
  author = {Humlum, Anders and Vestergaard, Emilie},
  title = {Still Waters, Rapid Currents: Early Labor Market Transformation under Generative AI},
  year = {2025},
  number = {33777},
  institution = {National Bureau of Economic Research},
  type = {NBER Working Paper},
  doi = {10.3386/w33777},
  url = {https://doi.org/10.3386/w33777},
  note = {verified: crossref 2026-11-02. Previously circulated as "Large Language Models, Small Labor Market Effects" (May 2025).}
}

@article{ide2025knowledge,
  author = {Ide, Enrique and Talamàs, Eduard},
  title = {Artificial Intelligence in the Knowledge Economy},
  journal = {Journal of Political Economy},
  year = {2025},
  volume = {133},
  number = {12},
  pages = {3762--3800},
  doi = {10.1086/737233},
  url = {https://doi.org/10.1086/737233},
  note = {verified: crossref 2026-11-02}
}

@misc{immorlica2024agents,
  author = {Immorlica, Nicole and Lucier, Brendan and Slivkins, Aleksandrs},
  title = {Generative AI as Economic Agents},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2406.00477},
  eprint = {2406.00477},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2406.00477},
  note = {verified: arxiv 2406.00477}
}

@misc{jimenez2024swebench,
  author = {Jimenez, Carlos E. and Yang, John and Wettig, Alexander and Yao, Shunyu and Pei, Kexin and Press, Ofir and Narasimhan, Karthik},
  title = {SWE-bench: Can Language Models Resolve Real-World GitHub Issues?},
  year = {2023},
  howpublished = {arXiv preprint arXiv:2310.06770},
  eprint = {2310.06770},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2310.06770},
  note = {verified: arxiv 2310.06770}
}

@article{kaplan1958nonparametric,
  author = {Kaplan, E. L. and Meier, Paul},
  title = {Nonparametric Estimation from Incomplete Observations},
  journal = {Journal of the American Statistical Association},
  year = {1958},
  volume = {53},
  number = {282},
  pages = {457--481},
  doi = {10.1080/01621459.1958.10501452},
  url = {https://doi.org/10.1080/01621459.1958.10501452},
  note = {verified: crossref 2026-11-02}
}

@misc{kapoor2024matter,
  author = {Kapoor, Sayash and Stroebl, Benedikt and Siegel, Zachary S. and Nadgir, Nitya and Narayanan, Arvind},
  title = {AI Agents That Matter},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2407.01502},
  eprint = {2407.01502},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2407.01502},
  note = {verified: arxiv 2407.01502}
}

@misc{kolt2025governing,
  author = {Kolt, Noam},
  title = {Governing AI Agents},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2501.07913},
  eprint = {2501.07913},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2501.07913},
  note = {verified: arxiv 2501.07913}
}

@article{korinek2023generative,
  author = {Korinek, Anton},
  title = {Generative AI for Economic Research: Use Cases and Implications for Economists},
  journal = {Journal of Economic Literature},
  year = {2023},
  volume = {61},
  number = {4},
  pages = {1281--1317},
  doi = {10.1257/jel.20231736},
  url = {https://doi.org/10.1257/jel.20231736},
  note = {verified: crossref 2026-11-02}
}

@article{krippendorff2004reliability,
  author = {Krippendorff, Klaus},
  title = {Reliability in Content Analysis: Some Common Misconceptions and Recommendations},
  journal = {Human Communication Research},
  year = {2004},
  volume = {30},
  number = {3},
  pages = {411--433},
  doi = {10.1111/j.1468-2958.2004.tb00738.x},
  url = {https://doi.org/10.1111/j.1468-2958.2004.tb00738.x},
  note = {verified: crossref 2026-11-02}
}

@misc{kwa2025horizon,
  author = {Kwa, Thomas and West, Ben and Becker, Joel and Deng, Amy and Garcia, Katharyn and Hasin, Max and Jawhar, Sami and Kinniment, Megan and Rush, Nate and Von Arx, Sydney and Bloom, Ryan and Broadley, Thomas and Du, Haoxing and Goodrich, Brian and Jurkovic, Nikola and Miles, Luke Harold and Nix, Seraphina and Lin, Tao and Painter, Chris and Parikh, Neev and Rein, David and Sato, Lucas Jun Koba and Wijk, Hjalmar and Ziegler, Daniel M. and Barnes, Elizabeth and Chan, Lawrence},
  title = {Measuring AI Ability to Complete Long Software Tasks},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2503.14499},
  eprint = {2503.14499},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2503.14499},
  note = {verified: arxiv 2503.14499}
}

@article{list2019multiple,
  author = {List, John A. and Shaikh, Azeem M. and Xu, Yang},
  title = {Multiple hypothesis testing in experimental economics},
  journal = {Experimental Economics},
  year = {2019},
  volume = {22},
  number = {4},
  pages = {773--793},
  doi = {10.1007/s10683-018-09597-5},
  url = {https://doi.org/10.1007/s10683-018-09597-5},
  note = {verified: crossref 2026-11-02}
}

@misc{liu2024agentbench,
  author = {Liu, Xiao and Yu, Hao and Zhang, Hanchen and Xu, Yifan and Lei, Xuanyu and Lai, Hanyu and Gu, Yu and Ding, Hangliang and Men, Kaiwen and Yang, Kejuan and Zhang, Shudan and Deng, Xiang and Zeng, Aohan and Du, Zhengxiao and Zhang, Chenhui and Shen, Sheng and Zhang, Tianjun and Su, Yu and Sun, Huan and Huang, Minlie and Dong, Yuxiao and Tang, Jie},
  title = {AgentBench: Evaluating LLMs as Agents},
  year = {2023},
  howpublished = {arXiv preprint arXiv:2308.03688},
  eprint = {2308.03688},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2308.03688},
  note = {verified: arxiv 2308.03688}
}

@misc{liu2026claudecode,
  author = {Liu, Jiacheng and Zhao, Xiaohan and Shang, Xinyi and Shen, Zhiqiang},
  title = {Dive into Claude Code: The Design Space of Today's and Future AI Agent Systems},
  year = {2026},
  howpublished = {arXiv preprint arXiv:2604.14228},
  eprint = {2604.14228},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2604.14228},
  note = {verified: arxiv 2604.14228}
}

@misc{ma2024spreadsheetbench,
  author = {Ma, Zeyao and Zhang, Bohan and Zhang, Jing and Yu, Jifan and Zhang, Xiaokang and Zhang, Xiaohan and Luo, Sijia and Wang, Xi and Tang, Jie},
  title = {SpreadsheetBench: Towards Challenging Real World Spreadsheet Manipulation},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2406.14991},
  eprint = {2406.14991},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2406.14991},
  note = {verified: arxiv 2406.14991}
}

@misc{madaan2023selfrefine,
  author = {Madaan, Aman and Tandon, Niket and Gupta, Prakhar and Hallinan, Skyler and Gao, Luyu and Wiegreffe, Sarah and Alon, Uri and Dziri, Nouha and Prabhumoye, Shrimai and Yang, Yiming and Gupta, Shashank and Majumder, Bodhisattwa Prasad and Hermann, Katherine and Welleck, Sean and Yazdanbakhsh, Amir and Clark, Peter},
  title = {Self-Refine: Iterative Refinement with Self-Feedback},
  year = {2023},
  howpublished = {arXiv preprint arXiv:2303.17651},
  eprint = {2303.17651},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2303.17651},
  note = {verified: arxiv 2303.17651}
}

@misc{merrill2026terminalbench,
  author = {Merrill, Mike A. and Shaw, Alexander G. and Carlini, Nicholas and Li, Boxuan and Raj, Harsh and Bercovich, Ivan and Shi, Lin and Shin, Jeong Yeon and Walshe, Thomas and Buchanan, E. Kelly and Shen, Junhong and Ye, Guanghao and Lin, Haowei and Poulos, Jason and Wang, Maoyu and Nezhurina, Marianna and Jitsev, Jenia and Lu, Di and Mastromichalakis, Orfeas Menis and Xu, Zhiwei and Chen, Zizhao and Liu, Yue and Zhang, Robert and Chen, Leon Liangyu and Kashyap, Anurag and Uslu, Jan-Lucas and Li, Jeffrey and Wu, Jianbo and Yan, Minghao and Bian, Song and Sharma, Vedang and Sun, Ke and Dillmann, Steven and Anand, Akshay and Lanpouthakoun, Andrew and Koopah, Bardia and Hu, Changran and Guha, Etash and Dreiman, Gabriel H. S. and Zhu, Jiacheng and Krauth, Karl and Zhong, Li and Muennighoff, Niklas and Amanfu, Robert and Tan, Shangyin and Pimpalgaonkar, Shreyas and Aggarwal, Tushar and Lin, Xiangning and Lan, Xin and Zhao, Xuandong and Liang, Yiqing and Wang, Yuanli and Wang, Zilong and Zhou, Changzhi and Heineman, David and Liu, Hange and Trivedi, Harsh and Yang, John and Lin, Junhong and Shetty, Manish and Yang, Michael and Omi, Nabil and Raoof, Negin and Li, Shanda and Zhuo, Terry Yue and Lin, Wuwei and Dai, Yiwei and Wang, Yuxin and Chai, Wenhao and Zhou, Shang and Wahdany, Dariush and She, Ziyu and Hu, Jiaming and Dong, Zhikang and Zhu, Yuxuan and Cui, Sasha and Saiyed, Ahson and Kolbeinsson, Arinbjörn and Hu, Jesse and Rytting, Christopher Michael and Marten, Ryan and Wang, Yixin and Dimakis, Alex and Konwinski, Andy and Schmidt, Ludwig},
  title = {Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces},
  year = {2026},
  howpublished = {arXiv preprint arXiv:2601.11868},
  eprint = {2601.11868},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2601.11868},
  note = {verified: arxiv 2601.11868}
}

@misc{mialon2023gaia,
  author = {Mialon, Grégoire and Fourrier, Clémentine and Swift, Craig and Wolf, Thomas and LeCun, Yann and Scialom, Thomas},
  title = {GAIA: a benchmark for General AI Assistants},
  year = {2023},
  howpublished = {arXiv preprint arXiv:2311.12983},
  eprint = {2311.12983},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2311.12983},
  note = {verified: arxiv 2311.12983}
}

@misc{miller2024errorbars,
  author = {Miller, Evan},
  title = {Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2411.00640},
  eprint = {2411.00640},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2411.00640},
  note = {verified: arxiv 2411.00640}
}

@misc{miserendino2025swelancer,
  author = {Miserendino, Samuel and Wang, Michele and Patwardhan, Tejal and Heidecke, Johannes},
  title = {SWE-Lancer: Can Frontier LLMs Earn \$1 Million from Real-World Freelance Software Engineering?},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2502.12115},
  eprint = {2502.12115},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2502.12115},
  note = {verified: arxiv 2502.12115}
}

@article{nosek2018preregistration,
  author = {Nosek, Brian A. and Ebersole, Charles R. and DeHaven, Alexander C. and Mellor, David T.},
  title = {The preregistration revolution},
  journal = {Proceedings of the National Academy of Sciences},
  year = {2018},
  volume = {115},
  number = {11},
  pages = {2600--2606},
  doi = {10.1073/pnas.1708274114},
  url = {https://doi.org/10.1073/pnas.1708274114},
  note = {verified: crossref 2026-11-02}
}

@article{noy2023experimental,
  author = {Noy, Shakked and Zhang, Whitney},
  title = {Experimental evidence on the productivity effects of generative artificial intelligence},
  journal = {Science},
  year = {2023},
  volume = {381},
  number = {6654},
  pages = {187--192},
  doi = {10.1126/science.adh2586},
  url = {https://doi.org/10.1126/science.adh2586},
  note = {verified: crossref 2026-11-02}
}

@article{olken2015preanalysis,
  author = {Olken, Benjamin A.},
  title = {Promises and Perils of Pre-Analysis Plans},
  journal = {Journal of Economic Perspectives},
  year = {2015},
  volume = {29},
  number = {3},
  pages = {61--80},
  doi = {10.1257/jep.29.3.61},
  url = {https://doi.org/10.1257/jep.29.3.61},
  note = {verified: crossref 2026-11-02}
}

@misc{otis2024entrepreneurial,
  author = {Otis, Nicholas and Clarke, Rowan and Delecourt, Solène and Holtz, David and Koning, Rembrand},
  title = {The Uneven Impact of Generative AI on Entrepreneurial Performance},
  year = {2024},
  howpublished = {SSRN working paper},
  doi = {10.2139/ssrn.4671369},
  url = {https://doi.org/10.2139/ssrn.4671369},
  note = {verified: crossref 2026-11-02}
}

@misc{panickssery2024selfpreference,
  author = {Panickssery, Arjun and Bowman, Samuel R. and Feng, Shi},
  title = {LLM Evaluators Recognize and Favor Their Own Generations},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2404.13076},
  eprint = {2404.13076},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2404.13076},
  note = {verified: arxiv 2404.13076}
}

@misc{patwardhan2025gdpval,
  author = {Patwardhan, Tejal and Dias, Rachel and Proehl, Elizabeth and Kim, Grace and Wang, Michele and Watkins, Olivia and Fishman, Simón Posada and Aljubeh, Marwan and Thacker, Phoebe and Fauconnet, Laurance and Kim, Natalie S. and Chao, Patrick and Miserendino, Samuel and Chabot, Gildas and Li, David and Sharman, Michael and Barr, Alexandra and Glaese, Amelia and Tworek, Jerry},
  title = {GDPval: Evaluating AI Model Performance on Real-World Economically Valuable Tasks},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2510.04374},
  eprint = {2510.04374},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2510.04374},
  note = {verified: arxiv 2510.04374}
}

@misc{peng2023copilot,
  author = {Peng, Sida and Kalliamvakou, Eirini and Cihon, Peter and Demirer, Mert},
  title = {The Impact of AI on Developer Productivity: Evidence from GitHub Copilot},
  year = {2023},
  howpublished = {arXiv preprint arXiv:2302.06590},
  eprint = {2302.06590},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2302.06590},
  note = {verified: arxiv 2302.06590}
}

@techreport{pizzinelli2023exposure,
  author = {Pizzinelli, Carlo and Panton, Augustus and Mendes Tavares, Marina and Cazzaniga, Mauro and Li, Longji},
  title = {Labor Market Exposure to AI: Cross-country Differences and Distributional Implications},
  year = {2023},
  number = {WP/23/216},
  institution = {International Monetary Fund},
  type = {IMF Working Paper},
  doi = {10.5089/9798400254802.001},
  url = {https://doi.org/10.5089/9798400254802.001},
  note = {verified: crossref 2026-11-02}
}

@misc{polo2024tinybenchmarks,
  author = {Polo, Felipe Maia and Weber, Lucas and Choshen, Leshem and Sun, Yuekai and Xu, Gongjun and Yurochkin, Mikhail},
  title = {tinyBenchmarks: evaluating LLMs with fewer examples},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2402.14992},
  eprint = {2402.14992},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2402.14992},
  note = {verified: arxiv 2402.14992}
}

@misc{rein2025hcast,
  author = {Rein, David and Becker, Joel and Deng, Amy and Nix, Seraphina and Canal, Chris and O'Connel, Daniel and Arnott, Pip and Bloom, Ryan and Broadley, Thomas and Garcia, Katharyn and Goodrich, Brian and Hasin, Max and Jawhar, Sami and Kinniment, Megan and Kwa, Thomas and Lajko, Aron and Rush, Nate and Sato, Lucas Jun Koba and Von Arx, Sydney and West, Ben and Chan, Lawrence and Barnes, Elizabeth},
  title = {HCAST: Human-Calibrated Autonomy Software Tasks},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2503.17354},
  eprint = {2503.17354},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2503.17354},
  note = {verified: arxiv 2503.17354}
}

@techreport{restrepo2025missed,
  author = {Restrepo, Pascual},
  title = {We Won't be Missed: Work and Growth in the AGI World},
  year = {2025},
  number = {34423},
  institution = {National Bureau of Economic Research},
  type = {NBER Working Paper},
  doi = {10.3386/w34423},
  url = {https://doi.org/10.3386/w34423},
  note = {verified: crossref 2026-11-02}
}

@misc{shinn2023reflexion,
  author = {Shinn, Noah and Cassano, Federico and Berman, Edward and Gopinath, Ashwin and Narasimhan, Karthik and Yao, Shunyu},
  title = {Reflexion: Language Agents with Verbal Reinforcement Learning},
  year = {2023},
  howpublished = {arXiv preprint arXiv:2303.11366},
  eprint = {2303.11366},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2303.11366},
  note = {verified: arxiv 2303.11366}
}

@article{simmons2011falsepositive,
  author = {Simmons, Joseph P. and Nelson, Leif D. and Simonsohn, Uri},
  title = {False-Positive Psychology},
  journal = {Psychological Science},
  year = {2011},
  volume = {22},
  number = {11},
  pages = {1359--1366},
  doi = {10.1177/0956797611417632},
  url = {https://doi.org/10.1177/0956797611417632},
  note = {verified: crossref 2026-11-02}
}

@misc{sinha2025horizon,
  author = {Sinha, Akshit and Arun, Arvindh and Goel, Shashwat and Staab, Steffen and Geiping, Jonas},
  title = {The Illusion of Diminishing Returns: Measuring Long Horizon Execution in LLMs},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2509.09677},
  eprint = {2509.09677},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2509.09677},
  note = {verified: arxiv 2509.09677}
}

@misc{sreenivas2026contracting,
  author = {Sreenivas, Nanda Kishore and Larson, Kate},
  title = {Contracting for LLM Delegation: Moral Hazard in Technology and Effort Choice},
  year = {2026},
  howpublished = {arXiv preprint arXiv:2608.18232},
  eprint = {2608.18232},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2608.18232},
  note = {verified: arxiv 2608.18232}
}

@misc{stroebl2024flaws,
  author = {Stroebl, Benedikt and Kapoor, Sayash and Narayanan, Arvind},
  title = {The Limits of Inference Scaling Through Resampling},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2411.17501},
  eprint = {2411.17501},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2411.17501},
  note = {verified: arxiv 2411.17501}
}

@misc{svanberg2024costeffective,
  author = {Svanberg, Maja and Li, Wensu and Fleming, Martin and Goehring, Brian and Thompson, Neil},
  title = {Beyond AI Exposure: Which Tasks are Cost-Effective to Automate with Computer Vision?},
  year = {2024},
  howpublished = {SSRN working paper},
  doi = {10.2139/ssrn.4700751},
  url = {https://doi.org/10.2139/ssrn.4700751},
  note = {verified: crossref 2026-11-02}
}

@misc{tomlinson2025working,
  author = {Tomlinson, Kiran and Jaffe, Sonia and Wang, Will and Counts, Scott and Suri, Siddharth},
  title = {Working with AI: Measuring the Applicability of Generative AI to Occupations},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2507.07935},
  eprint = {2507.07935},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2507.07935},
  note = {verified: arxiv 2507.07935}
}

@inproceedings{trivedi2024appworld,
  author = {Trivedi, Harsh and Khot, Tushar and Hartmann, Mareike and Manku, Ruskin and Dong, Vinty and Li, Edward and Gupta, Shashank and Sabharwal, Ashish and Balasubramanian, Niranjan},
  title = {AppWorld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents},
  booktitle = {Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  year = {2024},
  doi = {10.18653/v1/2024.acl-long.850},
  url = {https://doi.org/10.18653/v1/2024.acl-long.850},
  note = {verified: crossref 2026-11-02}
}

@article{vaccaro2024combinations,
  author = {Vaccaro, Michelle and Almaatouq, Abdullah and Malone, Thomas},
  title = {When combinations of humans and AI are useful: A systematic review and meta-analysis},
  journal = {Nature Human Behaviour},
  year = {2024},
  volume = {8},
  number = {12},
  pages = {2293--2303},
  doi = {10.1038/s41562-024-02024-1},
  url = {https://doi.org/10.1038/s41562-024-02024-1},
  note = {verified: crossref 2026-11-02}
}

@inproceedings{vania2021irt,
  author = {Vania, Clara and Htut, Phu Mon and Huang, William and Mungra, Dhara and Pang, Richard Yuanzhe and Phang, Jason and Liu, Haokun and Cho, Kyunghyun and Bowman, Samuel R.},
  title = {Comparing Test Sets with Item Response Theory},
  booktitle = {Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},
  year = {2021},
  doi = {10.18653/v1/2021.acl-long.92},
  url = {https://doi.org/10.18653/v1/2021.acl-long.92},
  note = {verified: crossref 2026-11-02}
}

@inproceedings{wang2024fair,
  author = {Wang, Peiyi and Li, Lei and Chen, Liang and Cai, Zefan and Zhu, Dawei and Lin, Binghuai and Cao, Yunbo and Kong, Lingpeng and Liu, Qi and Liu, Tianyu and Sui, Zhifang},
  title = {Large Language Models are not Fair Evaluators},
  booktitle = {Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  year = {2024},
  doi = {10.18653/v1/2024.acl-long.511},
  url = {https://doi.org/10.18653/v1/2024.acl-long.511},
  note = {verified: crossref 2026-11-02}
}

@misc{wang2024officebench,
  author = {Wang, Zilong and Cui, Yuedong and Zhong, Li and Zhang, Zimin and Yin, Da and Lin, Bill Yuchen and Shang, Jingbo},
  title = {OfficeBench: Benchmarking Language Agents across Multiple Applications for Office Automation},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2407.19056},
  eprint = {2407.19056},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2407.19056},
  note = {verified: arxiv 2407.19056}
}

@misc{wang2024openhands,
  author = {Wang, Xingyao and Li, Boxuan and Song, Yufan and Xu, Frank F. and Tang, Xiangru and Zhuge, Mingchen and Pan, Jiayi and Song, Yueqi and Li, Bowen and Singh, Jaskirat and Tran, Hoang H. and Li, Fuqiang and Ma, Ren and Zheng, Mingzhang and Qian, Bill and Shao, Yanjun and Muennighoff, Niklas and Zhang, Yizhe and Hui, Binyuan and Lin, Junyang and Brennan, Robert and Peng, Hao and Ji, Heng and Neubig, Graham},
  title = {OpenHands: An Open Platform for AI Software Developers as Generalist Agents},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2407.16741},
  eprint = {2407.16741},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2407.16741},
  note = {verified: arxiv 2407.16741}
}

@misc{wijk2024rebench,
  author = {Wijk, Hjalmar and Lin, Tao and Becker, Joel and Jawhar, Sami and Parikh, Neev and Broadley, Thomas and Chan, Lawrence and Chen, Michael and Clymer, Josh and Dhyani, Jai and Ericheva, Elena and Garcia, Katharyn and Goodrich, Brian and Jurkovic, Nikola and Karnofsky, Holden and Kinniment, Megan and Lajko, Aron and Nix, Seraphina and Sato, Lucas and Saunders, William and Taran, Maksym and West, Ben and Barnes, Elizabeth},
  title = {RE-Bench: Evaluating frontier AI R\&D capabilities of language model agents against human experts},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2411.15114},
  eprint = {2411.15114},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2411.15114},
  note = {verified: arxiv 2411.15114}
}

@misc{xie2024osworld,
  author = {Xie, Tianbao and Zhang, Danyang and Chen, Jixuan and Li, Xiaochuan and Zhao, Siheng and Cao, Ruisheng and Hua, Toh Jing and Cheng, Zhoujun and Shin, Dongchan and Lei, Fangyu and Liu, Yitao and Xu, Yiheng and Zhou, Shuyan and Savarese, Silvio and Xiong, Caiming and Zhong, Victor and Yu, Tao},
  title = {OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2404.07972},
  eprint = {2404.07972},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2404.07972},
  note = {verified: arxiv 2404.07972}
}

@misc{xu2024agentcompany,
  author = {Xu, Frank F. and Song, Yufan and Li, Boxuan and Tang, Yuxuan and Jain, Kritanjali and Bao, Mengxue and Wang, Zora Z. and Zhou, Xuhui and Guo, Zhitong and Cao, Murong and Yang, Mingyang and Lu, Hao Yang and Martin, Amaad and Su, Zhe and Maben, Leander and Mehta, Raj and Chi, Wayne and Jang, Lawrence and Xie, Yiqing and Zhou, Shuyan and Neubig, Graham},
  title = {TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2412.14161},
  eprint = {2412.14161},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2412.14161},
  note = {verified: arxiv 2412.14161}
}

@misc{yang2024sweagent,
  author = {Yang, John and Jimenez, Carlos E. and Wettig, Alexander and Lieret, Kilian and Yao, Shunyu and Narasimhan, Karthik and Press, Ofir},
  title = {SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2405.15793},
  eprint = {2405.15793},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2405.15793},
  note = {verified: arxiv 2405.15793}
}

@misc{yao2024taubench,
  author = {Yao, Shunyu and Shinn, Noah and Razavi, Pedram and Narasimhan, Karthik},
  title = {$\tau$-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains},
  year = {2024},
  howpublished = {arXiv preprint arXiv:2406.12045},
  eprint = {2406.12045},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2406.12045},
  note = {verified: arxiv 2406.12045}
}

@misc{yuan2026osworld2,
  author = {Yuan, Mengqi and Zhou, Zilong and Xiong, Xinzhuang and Wu, Weiming and Sun, Jiayang and Song, Jiamin and Cui, Kaiqian and Wang, Bowen and Wu, Haoyuan and Li, Yitong and Lu, Dunjie and Lu, Haikong and Zhen, Qi and Wang, Xinyuan and Deng, Jiaqi and Yang, Yuhao and Chen, Cheng and Zheng, Boyuan and Su, Alex and Yu, Xiao and Zou, Hao and Agashe, Saaket and Lu, Xing Han and Kaur, Manpreet and Qi, Zhengyang and Chen, Vincent Sunn and Sala, Frederic and Liu, Dayiheng and Lin, Junyang and Yu, Zhou and Su, Yu and Reddy, Siva and Wang, Xin Eric and Qi, Peng and Xie, Tianbao and Yu, Tao},
  title = {OSWorld 2.0: Benchmarking Computer Use Agents on Long-Horizon Real-World Tasks},
  year = {2026},
  howpublished = {arXiv preprint arXiv:2606.29537},
  eprint = {2606.29537},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2606.29537},
  note = {verified: arxiv 2606.29537}
}

@misc{zheng2023judge,
  author = {Zheng, Lianmin and Chiang, Wei-Lin and Sheng, Ying and Zhuang, Siyuan and Wu, Zhanghao and Zhuang, Yonghao and Lin, Zi and Li, Zhuohan and Li, Dacheng and Xing, Eric P. and Zhang, Hao and Gonzalez, Joseph E. and Stoica, Ion},
  title = {Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena},
  year = {2023},
  howpublished = {arXiv preprint arXiv:2306.05685},
  eprint = {2306.05685},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2306.05685},
  note = {verified: arxiv 2306.05685}
}

@misc{zhou2024webarena,
  author = {Zhou, Shuyan and Xu, Frank F. and Zhu, Hao and Zhou, Xuhui and Lo, Robert and Sridhar, Abishek and Cheng, Xianyi and Ou, Tianyue and Bisk, Yonatan and Fried, Daniel and Alon, Uri and Neubig, Graham},
  title = {WebArena: A Realistic Web Environment for Building Autonomous Agents},
  year = {2023},
  howpublished = {arXiv preprint arXiv:2307.13854},
  eprint = {2307.13854},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2307.13854},
  note = {verified: arxiv 2307.13854}
}

@misc{zhou2025lost,
  author = {Zhou, Hongli and Huang, Hui and Zhao, Ziqing and Han, Lvyuan and Wang, Huicheng and Chen, Kehai and Yang, Muyun and Bao, Wei and Dong, Jian and Xu, Bing and Zhu, Conghui and Cao, Hailong and Zhao, Tiejun},
  title = {Lost in Benchmarks? Rethinking Large Language Model Benchmarking with Item Response Theory},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2505.15055},
  eprint = {2505.15055},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2505.15055},
  note = {verified: arxiv 2505.15055}
}

@misc{zhu2025rigorous,
  author = {Zhu, Yuxuan and Jin, Tengjun and Pruksachatkun, Yada and Zhang, Andy and Liu, Shu and Cui, Sasha and Kapoor, Sayash and Longpre, Shayne and Meng, Kevin and Weiss, Rebecca and Barez, Fazl and Gupta, Rahul and Dhamala, Jwala and Merizian, Jacob and Giulianelli, Mario and Coppock, Harry and Ududec, Cozmin and Sekhon, Jasjeet and Steinhardt, Jacob and Kellermann, Antony and Schwettmann, Sarah and Zaharia, Matei and Stoica, Ion and Liang, Percy and Kang, Daniel},
  title = {Establishing Best Practices for Building Rigorous Agentic Benchmarks},
  year = {2025},
  howpublished = {arXiv preprint arXiv:2507.02825},
  eprint = {2507.02825},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2507.02825},
  note = {verified: arxiv 2507.02825}
}