1{2 "arXiv:1902.03245": {3 "paperId": "67a0d44650f5b7944fc5bc069c87bffa44b6cba4",4 "externalIds": {5 "MAG": "2970030663",6 "ArXiv": "1902.03245",7 "DBLP": "journals/corr/abs-1902-03245",8 "CorpusId": 604403979 },10 "publicationVenue": {11 "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",12 "name": "Neural Information Processing Systems",13 "type": "conference",14 "alternate_names": [15 "Neural Inf Process Syst",16 "NeurIPS",17 "NIPS"18 ],19 "url": "http://neurips.cc/"20 },21 "title": "Ask Not What AI Can Do, But What AI Should Do: Towards a Framework of Task Delegability",22 "venue": "Neural Information Processing Systems",23 "year": 2019,24 "citationCount": 117,25 "openAccessPdf": {26 "url": "",27 "status": null,28 "license": null,29 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/1902.03245, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."30 },31 "publicationTypes": [32 "JournalArticle",33 "Review"34 ],35 "journal": {36 "name": "ArXiv",37 "volume": "abs/1902.03245"38 },39 "authors": [40 {41 "authorId": "70025737",42 "name": "Brian Lubars"43 },44 {45 "authorId": "40348583",46 "name": "Chenhao Tan"47 }48 ],49 "abstract": "While artificial intelligence (AI) holds promise for addressing societal challenges, issues of exactly which tasks to automate and to what extent to do so remain understudied. We approach this problem of task delegability from a human-centered perspective by developing a framework on human perception of task delegation to AI. We consider four high-level factors that can contribute to a delegation decision: motivation, difficulty, risk, and trust. To obtain an empirical understanding of human preferences in different tasks, we build a dataset of 100 tasks from academic papers, popular media portrayal of AI, and everyday life, and administer a survey based on our proposed framework. We find little preference for full AI control and a strong preference for machine-in-the-loop designs, in which humans play the leading role. Among the four factors, trust is the most correlated with human preferences of optimal human-machine delegation. This framework represents a first step towards characterizing human preferences of AI automation across tasks. We hope this work encourages future efforts towards understanding such individual attitudes; our goal is to inform the public and the AI research community rather than dictating any direction in technology development."50 },51 "arXiv:2005.11401": {52 "paperId": "659bf9ce7175e1ec266ff54359e2bd76e0b7ff31",53 "externalIds": {54 "DBLP": "conf/nips/LewisPPPKGKLYR020",55 "MAG": "3027879771",56 "ArXiv": "2005.11401",57 "CorpusId": 21886957558 },59 "publicationVenue": {60 "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",61 "name": "Neural Information Processing Systems",62 "type": "conference",63 "alternate_names": [64 "Neural Inf Process Syst",65 "NeurIPS",66 "NIPS"67 ],68 "url": "http://neurips.cc/"69 },70 "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",71 "venue": "Neural Information Processing Systems",72 "year": 2020,73 "citationCount": 20321,74 "openAccessPdf": {75 "url": "",76 "status": null,77 "license": null,78 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2005.11401, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."79 },80 "publicationTypes": [81 "JournalArticle"82 ],83 "journal": {84 "name": "ArXiv",85 "volume": "abs/2005.11401"86 },87 "authors": [88 {89 "authorId": "145222654",90 "name": "Patrick Lewis"91 },92 {93 "authorId": "3439053",94 "name": "Ethan Perez"95 },96 {97 "authorId": "120174856",98 "name": "Aleksandra Piktus"99 },100 {101 "authorId": "40052301",102 "name": "F. Petroni"103 },104 {105 "authorId": "2067091563",106 "name": "Vladimir Karpukhin"107 },108 {109 "authorId": "2519247661",110 "name": "Naman Goyal"111 },112 {113 "authorId": "103131985",114 "name": "Heinrich Kuttler"115 },116 {117 "authorId": "35084211",118 "name": "M. Lewis"119 },120 {121 "authorId": "144105277",122 "name": "Wen-tau Yih"123 },124 {125 "authorId": "2620211",126 "name": "Tim Rockt\u00e4schel"127 },128 {129 "authorId": "48662861",130 "name": "Sebastian Riedel"131 },132 {133 "authorId": "1743722",134 "name": "Douwe Kiela"135 }136 ],137 "abstract": "Large pre-trained language models have been shown to store factual knowledge in their parameters, and achieve state-of-the-art results when fine-tuned on downstream NLP tasks. However, their ability to access and precisely manipulate knowledge is still limited, and hence on knowledge-intensive tasks, their performance lags behind task-specific architectures. Additionally, providing provenance for their decisions and updating their world knowledge remain open research problems. Pre-trained models with a differentiable access mechanism to explicit non-parametric memory can overcome this issue, but have so far been only investigated for extractive downstream tasks. We explore a general-purpose fine-tuning recipe for retrieval-augmented generation (RAG) -- models which combine pre-trained parametric and non-parametric memory for language generation. We introduce RAG models where the parametric memory is a pre-trained seq2seq model and the non-parametric memory is a dense vector index of Wikipedia, accessed with a pre-trained neural retriever. We compare two RAG formulations, one which conditions on the same retrieved passages across the whole generated sequence, the other can use different passages per token. We fine-tune and evaluate our models on a wide range of knowledge-intensive NLP tasks and set the state-of-the-art on three open domain QA tasks, outperforming parametric seq2seq models and task-specific retrieve-and-extract architectures. For language generation tasks, we find that RAG models generate more specific, diverse and factual language than a state-of-the-art parametric-only seq2seq baseline."138 },139 "arXiv:2112.09332": {140 "paperId": "2f3efe44083af91cef562c1a3451eee2f8601d22",141 "externalIds": {142 "DBLP": "journals/corr/abs-2112-09332",143 "ArXiv": "2112.09332",144 "CorpusId": 245329531145 },146 "publicationVenue": {147 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",148 "name": "arXiv.org",149 "alternate_names": [150 "ArXiv"151 ],152 "issn": "2331-8422",153 "url": "https://arxiv.org"154 },155 "title": "WebGPT: Browser-assisted question-answering with human feedback",156 "venue": "arXiv.org",157 "year": 2021,158 "citationCount": 2173,159 "openAccessPdf": {160 "url": "",161 "status": null,162 "license": null,163 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2112.09332, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."164 },165 "publicationTypes": [166 "JournalArticle"167 ],168 "journal": {169 "name": "ArXiv",170 "volume": "abs/2112.09332"171 },172 "authors": [173 {174 "authorId": "7406311",175 "name": "Reiichiro Nakano"176 },177 {178 "authorId": "2052366271",179 "name": "Jacob Hilton"180 },181 {182 "authorId": "2054519183",183 "name": "S. Balaji"184 },185 {186 "authorId": "49387725",187 "name": "Jeff Wu"188 },189 {190 "authorId": "31793034",191 "name": "Long Ouyang"192 },193 {194 "authorId": "2149054292",195 "name": "Christina Kim"196 },197 {198 "authorId": "144239765",199 "name": "Christopher Hesse"200 },201 {202 "authorId": "150298413",203 "name": "Shantanu Jain"204 },205 {206 "authorId": "13622184",207 "name": "V. Kosaraju"208 },209 {210 "authorId": "2058848938",211 "name": "W. Saunders"212 },213 {214 "authorId": "2115903168",215 "name": "Xu Jiang"216 },217 {218 "authorId": "6062736",219 "name": "K. Cobbe"220 },221 {222 "authorId": "2146257131",223 "name": "Tyna Eloundou"224 },225 {226 "authorId": "2064404342",227 "name": "G. Krueger"228 },229 {230 "authorId": "2146257251",231 "name": "Kevin Button"232 },233 {234 "authorId": "2146257375",235 "name": "Matthew Knight"236 },237 {238 "authorId": "1490681878",239 "name": "Benjamin Chess"240 },241 {242 "authorId": "47971768",243 "name": "John D. Schulman"244 }245 ],246 "abstract": "We fine-tune GPT-3 to answer long-form questions using a text-based web-browsing environment, which allows the model to search and navigate the web. By setting up the task so that it can be performed by humans, we are able to train models on the task using imitation learning, and then optimize answer quality with human feedback. To make human evaluation of factual accuracy easier, models must collect references while browsing in support of their answers. We train and evaluate our models on ELI5, a dataset of questions asked by Reddit users. Our best model is obtained by fine-tuning GPT-3 using behavior cloning, and then performing rejection sampling against a reward model trained to predict human preferences. This model's answers are preferred by humans 56% of the time to those of our human demonstrators, and 69% of the time to the highest-voted answer from Reddit."247 },248 "arXiv:2112.11471": {249 "paperId": "449ff2404a7b11fa3c47cd228fa5469b00ffef20",250 "externalIds": {251 "DBLP": "journals/corr/abs-2112-11471",252 "ArXiv": "2112.11471",253 "CorpusId": 245385821254 },255 "publicationVenue": {256 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",257 "name": "arXiv.org",258 "alternate_names": [259 "ArXiv"260 ],261 "issn": "2331-8422",262 "url": "https://arxiv.org"263 },264 "title": "Towards a Science of Human-AI Decision Making: A Survey of Empirical Studies",265 "venue": "arXiv.org",266 "year": 2021,267 "citationCount": 258,268 "openAccessPdf": {269 "url": "",270 "status": null,271 "license": null,272 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2112.11471, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."273 },274 "publicationTypes": [275 "JournalArticle",276 "Review"277 ],278 "journal": {279 "name": "ArXiv",280 "volume": "abs/2112.11471"281 },282 "authors": [283 {284 "authorId": "120801533",285 "name": "V. Lai"286 },287 {288 "authorId": "2131059444",289 "name": "Cha-Cha Chen"290 },291 {292 "authorId": "144921048",293 "name": "Q. Liao"294 },295 {296 "authorId": "1405364873",297 "name": "Alison Smith-Renner"298 },299 {300 "authorId": "2111727675",301 "name": "Chenhao Tan"302 }303 ],304 "abstract": "As AI systems demonstrate increasingly strong predictive performance, their adoption has grown in numerous domains. However, in high-stakes domains such as criminal justice and healthcare, full automation is often not desirable due to safety, ethical, and legal concerns, yet fully manual approaches can be inaccurate and time consuming. As a result, there is growing interest in the research community to augment human decision making with AI assistance. Besides developing AI technologies for this purpose, the emerging field of human-AI decision making must embrace empirical approaches to form a foundational understanding of how humans interact and work with AI to make decisions. To invite and help structure research efforts towards a science of understanding and improving human-AI decision making, we survey recent literature of empirical human-subject studies on this topic. We summarize the study design choices made in over 100 papers in three important aspects: (1) decision tasks, (2) AI models and AI assistance elements, and (3) evaluation metrics. For each aspect, we summarize current trends, discuss gaps in current practices of the field, and make a list of recommendations for future research. Our survey highlights the need to develop common frameworks to account for the design and research spaces of human-AI decision making, so that researchers can make rigorous choices in study design, and the research community can build on each other's work and produce generalizable scientific knowledge. We also hope this survey will serve as a bridge for HCI and AI communities to work together to mutually shape the empirical science and computational technologies for human-AI decision making."305 },306 "arXiv:2203.11147": {307 "paperId": "8666f9f379389a5dff31e72fb0f992a37763ba41",308 "externalIds": {309 "DBLP": "journals/corr/abs-2203-11147",310 "ArXiv": "2203.11147",311 "DOI": "10.48550/arXiv.2203.11147",312 "CorpusId": 247594830313 },314 "publicationVenue": {315 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",316 "name": "arXiv.org",317 "alternate_names": [318 "ArXiv"319 ],320 "issn": "2331-8422",321 "url": "https://arxiv.org"322 },323 "title": "Teaching language models to support answers with verified quotes",324 "venue": "arXiv.org",325 "year": 2022,326 "citationCount": 391,327 "openAccessPdf": {328 "url": "http://arxiv.org/pdf/2203.11147",329 "status": "GREEN",330 "license": null,331 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2203.11147, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."332 },333 "publicationTypes": [334 "JournalArticle"335 ],336 "journal": {337 "name": "ArXiv",338 "volume": "abs/2203.11147"339 },340 "authors": [341 {342 "authorId": "10698483",343 "name": "Jacob Menick"344 },345 {346 "authorId": "1994939814",347 "name": "Maja Trebacz"348 },349 {350 "authorId": "148305440",351 "name": "Vladimir Mikulik"352 },353 {354 "authorId": "9958912",355 "name": "John Aslanides"356 },357 {358 "authorId": "2107148568",359 "name": "H. F. Song"360 },361 {362 "authorId": "2159545857",363 "name": "Martin J. Chadwick"364 },365 {366 "authorId": "2143471164",367 "name": "Mia Glaese"368 },369 {370 "authorId": "116494324",371 "name": "Susannah Young"372 },373 {374 "authorId": "2151248210",375 "name": "Lucy Campbell-Gillingham"376 },377 {378 "authorId": "2060655766",379 "name": "Geoffrey Irving"380 },381 {382 "authorId": "147687624",383 "name": "Nat McAleese"384 }385 ],386 "abstract": "Recent large language models often answer factual questions correctly. But users can't trust any given claim a model makes without fact-checking, because language models can hallucinate convincing nonsense. In this work we use reinforcement learning from human preferences (RLHP) to train\"open-book\"QA models that generate answers whilst also citing specific evidence for their claims, which aids in the appraisal of correctness. Supporting evidence is drawn from multiple documents found via a search engine, or from a single user-provided document. Our 280 billion parameter model, GopherCite, is able to produce answers with high quality supporting evidence and abstain from answering when unsure. We measure the performance of GopherCite by conducting human evaluation of answers to questions in a subset of the NaturalQuestions and ELI5 datasets. The model's response is found to be high-quality 80\\% of the time on this Natural Questions subset, and 67\\% of the time on the ELI5 subset. Abstaining from the third of questions for which it is most unsure improves performance to 90\\% and 80\\% respectively, approaching human baselines. However, analysis on the adversarial TruthfulQA dataset shows why citation is only one part of an overall strategy for safety and trustworthiness: not all claims supported by evidence are true."387 },388 "arXiv:2203.11171": {389 "paperId": "5f19ae1135a9500940978104ec15a5b8751bc7d2",390 "externalIds": {391 "DBLP": "conf/iclr/0002WSLCNCZ23",392 "ArXiv": "2203.11171",393 "CorpusId": 247595263394 },395 "publicationVenue": {396 "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",397 "name": "International Conference on Learning Representations",398 "type": "conference",399 "alternate_names": [400 "Int Conf Learn Represent",401 "ICLR"402 ],403 "url": "https://iclr.cc/"404 },405 "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models",406 "venue": "International Conference on Learning Representations",407 "year": 2022,408 "citationCount": 8129,409 "openAccessPdf": {410 "url": "",411 "status": null,412 "license": null,413 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2203.11171, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."414 },415 "publicationTypes": [416 "JournalArticle"417 ],418 "journal": {419 "name": "ArXiv",420 "volume": "abs/2203.11171"421 },422 "authors": [423 {424 "authorId": "2275277634",425 "name": "Xue-Zhi Wang"426 },427 {428 "authorId": "119640649",429 "name": "Jason Wei"430 },431 {432 "authorId": "1714772",433 "name": "Dale Schuurmans"434 },435 {436 "authorId": "1998340269",437 "name": "Quoc V. Le"438 },439 {440 "authorId": "2226805",441 "name": "Ed H. Chi"442 },443 {444 "authorId": "46617804",445 "name": "Sharan Narang"446 },447 {448 "authorId": "2841893",449 "name": "A. Chowdhery"450 },451 {452 "authorId": "65855107",453 "name": "Denny Zhou"454 }455 ],456 "abstract": "Chain-of-thought prompting combined with pre-trained large language models has achieved encouraging results on complex reasoning tasks. In this paper, we propose a new decoding strategy, self-consistency, to replace the naive greedy decoding used in chain-of-thought prompting. It first samples a diverse set of reasoning paths instead of only taking the greedy one, and then selects the most consistent answer by marginalizing out the sampled reasoning paths. Self-consistency leverages the intuition that a complex reasoning problem typically admits multiple different ways of thinking leading to its unique correct answer. Our extensive empirical evaluation shows that self-consistency boosts the performance of chain-of-thought prompting with a striking margin on a range of popular arithmetic and commonsense reasoning benchmarks, including GSM8K (+17.9%), SVAMP (+11.0%), AQuA (+12.2%), StrategyQA (+6.4%) and ARC-challenge (+3.9%)."457 },458 "arXiv:2210.13777": {459 "paperId": "f13b251c8346bc3be19b71b840449831e9716999",460 "externalIds": {461 "DBLP": "conf/emnlp/WaddenLKCBWH22",462 "ArXiv": "2210.13777",463 "DOI": "10.48550/arXiv.2210.13777",464 "CorpusId": 253107192465 },466 "publicationVenue": {467 "id": "41bf9ed3-85b3-4c90-b015-150e31690253",468 "name": "Conference on Empirical Methods in Natural Language Processing",469 "type": "conference",470 "alternate_names": [471 "Empir Method Nat Lang Process",472 "Empirical Methods in Natural Language Processing",473 "Conf Empir Method Nat Lang Process",474 "EMNLP"475 ],476 "url": "https://www.aclweb.org/portal/emnlp"477 },478 "title": "SciFact-Open: Towards open-domain scientific claim verification",479 "venue": "Conference on Empirical Methods in Natural Language Processing",480 "year": 2022,481 "citationCount": 136,482 "openAccessPdf": {483 "url": "http://arxiv.org/pdf/2210.13777",484 "status": "GREEN",485 "license": null,486 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2210.13777, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."487 },488 "publicationTypes": [489 "JournalArticle",490 "Conference"491 ],492 "journal": {493 "name": "ArXiv",494 "volume": "abs/2210.13777"495 },496 "authors": [497 {498 "authorId": "30051202",499 "name": "David Wadden"500 },501 {502 "authorId": "46258841",503 "name": "Kyle Lo"504 },505 {506 "authorId": "2003338023",507 "name": "Bailey Kuehl"508 },509 {510 "authorId": "2527954",511 "name": "Arman Cohan"512 },513 {514 "authorId": "46181066",515 "name": "Iz Beltagy"516 },517 {518 "authorId": "31860505",519 "name": "Lucy Lu Wang"520 },521 {522 "authorId": "2548384",523 "name": "Hannaneh Hajishirzi"524 }525 ],526 "abstract": "While research on scientific claim verification has led to the development of powerful systems that appear to approach human performance, these approaches have yet to be tested in a realistic setting against large corpora of scientific literature. Moving to this open-domain evaluation setting, however, poses unique challenges; in particular, it is infeasible to exhaustively annotate all evidence documents. In this work, we present SciFact-Open, a new test collection designed to evaluate the performance of scientific claim verification systems on a corpus of 500K research abstracts. Drawing upon pooling techniques from information retrieval, we collect evidence for scientific claims by pooling and annotating the top predictions of four state-of-the-art scientific claim verification models. We find that systems developed on smaller corpora struggle to generalize to SciFact-Open, exhibiting performance drops of at least 15 F1. In addition, analysis of the evidence in SciFact-Open reveals interesting phenomena likely to appear when claim verification systems are deployed in practice, e.g., cases where the evidence supports only a special case of the claim. Our dataset is available at https://github.com/dwadden/scifact-open."527 },528 "arXiv:2211.09110": {529 "paperId": "29abcf865613287c661385c39401424f709a3fda",530 "externalIds": {531 "DBLP": "journals/corr/abs-2211-09110",532 "ArXiv": "2211.09110",533 "DOI": "10.48550/arXiv.2211.09110",534 "CorpusId": 263423935535 },536 "publicationVenue": {537 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",538 "name": "arXiv.org",539 "alternate_names": [540 "ArXiv"541 ],542 "issn": "2331-8422",543 "url": "https://arxiv.org"544 },545 "title": "Holistic Evaluation of Language Models",546 "venue": "arXiv.org",547 "year": 2022,548 "citationCount": 12,549 "openAccessPdf": {550 "url": "https://arxiv.org/pdf/2211.09110",551 "status": "GREEN",552 "license": null,553 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2211.09110, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."554 },555 "publicationTypes": [556 "JournalArticle"557 ],558 "journal": {559 "name": "ArXiv",560 "volume": "abs/2211.09110"561 },562 "authors": [563 {564 "authorId": "2249641250",565 "name": "Percy Liang"566 },567 {568 "authorId": "2223138553",569 "name": "Rishi Bommasani"570 },571 {572 "authorId": "2110585783",573 "name": "Tony Lee"574 },575 {576 "authorId": "2754804",577 "name": "Dimitris Tsipras"578 },579 {580 "authorId": "1914569491",581 "name": "Dilara Soylu"582 },583 {584 "authorId": "19168196",585 "name": "Michihiro Yasunaga"586 },587 {588 "authorId": "9227100",589 "name": "Yian Zhang"590 },591 {592 "authorId": "2251007290",593 "name": "Deepak Narayanan"594 },595 {596 "authorId": "2250015839",597 "name": "Yuhuai Wu"598 },599 {600 "authorId": "32423266",601 "name": "Ananya Kumar"602 },603 {604 "authorId": "2250557275",605 "name": "Benjamin Newman"606 },607 {608 "authorId": "2250117174",609 "name": "Binhang Yuan"610 },611 {612 "authorId": "1748871792",613 "name": "Bobby Yan"614 },615 {616 "authorId": "2262511796",617 "name": "Ce Zhang"618 },619 {620 "authorId": "133749287",621 "name": "Christian Cosgrove"622 },623 {624 "authorId": "2250402802",625 "name": "Christopher D. Manning"626 },627 {628 "authorId": "2250761261",629 "name": "Christopher R\u00e9"630 },631 {632 "authorId": "1413421064",633 "name": "Diana Acosta-Navas"634 },635 {636 "authorId": "152951058",637 "name": "Drew A. Hudson"638 },639 {640 "authorId": "49456763",641 "name": "E. Zelikman"642 },643 {644 "authorId": "41152329",645 "name": "Esin Durmus"646 },647 {648 "authorId": "8759332",649 "name": "Faisal Ladhak"650 },651 {652 "authorId": "2047004093",653 "name": "Frieda Rong"654 },655 {656 "authorId": "40046694",657 "name": "Hongyu Ren"658 },659 {660 "authorId": "2250097082",661 "name": "Huaxiu Yao"662 },663 {664 "authorId": "2252087284",665 "name": "Jue Wang"666 },667 {668 "authorId": "50818255",669 "name": "Keshav Santhanam"670 },671 {672 "authorId": "2250741832",673 "name": "Laurel J. Orr"674 },675 {676 "authorId": "2118604716",677 "name": "Lucia Zheng"678 },679 {680 "authorId": "1387987315",681 "name": "Mert Y\u00fcksekg\u00f6n\u00fcl"682 },683 {684 "authorId": "51903517",685 "name": "M. Suzgun"686 },687 {688 "authorId": "2252992164",689 "name": "Nathan Kim"690 },691 {692 "authorId": "2820009",693 "name": "Neel Guha"694 },695 {696 "authorId": "22193324",697 "name": "Niladri S. Chatterji"698 },699 {700 "authorId": "144112155",701 "name": "O. Khattab"702 },703 {704 "authorId": "2250986700",705 "name": "Peter Henderson"706 },707 {708 "authorId": "144862341",709 "name": "Qian Huang"710 },711 {712 "authorId": "2121293578",713 "name": "Ryan A. Chi"714 },715 {716 "authorId": "2407771963",717 "name": "S. Xie"718 },719 {720 "authorId": "2852106",721 "name": "Shibani Santurkar"722 },723 {724 "authorId": "2242939863",725 "name": "Surya Ganguli"726 },727 {728 "authorId": "2117567142",729 "name": "Tatsunori B. Hashimoto"730 },731 {732 "authorId": "2251009433",733 "name": "Thomas Icard"734 },735 {736 "authorId": "2256233130",737 "name": "Tianyi Zhang"738 },739 {740 "authorId": "113810201",741 "name": "Vishrav Chaudhary"742 },743 {744 "authorId": "2127971344",745 "name": "William Wang"746 },747 {748 "authorId": "2250724754",749 "name": "Xuechen Li"750 },751 {752 "authorId": "2054708905",753 "name": "Yifan Mai"754 },755 {756 "authorId": "2273913293",757 "name": "Yu-Hui Zhang"758 },759 {760 "authorId": "2740047",761 "name": "Yuta Koreeda"762 }763 ],764 "abstract": "Language models (LMs) are becoming the foundation for almost all major language technologies, but their capabilities, limitations, and risks are not well understood. We present Holistic Evaluation of Language Models (HELM) to improve the transparency of language models. First, we taxonomize the vast space of potential scenarios (i.e. use cases) and metrics (i.e. desiderata) that are of interest for LMs. Then we select a broad subset based on coverage and feasibility, noting what's missing or underrepresented (e.g. question answering for neglected English dialects, metrics for trustworthiness). Second, we adopt a multi-metric approach: We measure 7 metrics (accuracy, calibration, robustness, fairness, bias, toxicity, and efficiency) for each of 16 core scenarios when possible (87.5% of the time). This ensures metrics beyond accuracy don't fall to the wayside, and that trade-offs are clearly exposed. We also perform 7 targeted evaluations, based on 26 targeted scenarios, to analyze specific aspects (e.g. reasoning, disinformation). Third, we conduct a large-scale evaluation of 30 prominent language models (spanning open, limited-access, and closed models) on all 42 scenarios, 21 of which were not previously used in mainstream LM evaluation. Prior to HELM, models on average were evaluated on just 17.9% of the core HELM scenarios, with some prominent models not sharing a single scenario in common. We improve this to 96.0%: now all 30 models have been densely benchmarked on the same core scenarios and metrics under standardized conditions. Our evaluation surfaces 25 top-level findings. For full transparency, we release all raw model prompts and completions publicly for further analysis, as well as a general modular toolkit. We intend for HELM to be a living benchmark for the community, continuously updated with new scenarios, metrics, and models."765 },766 "arXiv:2211.11501": {767 "paperId": "8a4fc5f00cd4aca61e148e46a2125c3a406719f1",768 "externalIds": {769 "DBLP": "journals/corr/abs-2211-11501",770 "ArXiv": "2211.11501",771 "DOI": "10.48550/arXiv.2211.11501",772 "CorpusId": 253734939773 },774 "publicationVenue": {775 "id": "fc0a208c-acb7-47dc-a0d4-af8190e21d29",776 "name": "International Conference on Machine Learning",777 "type": "conference",778 "alternate_names": [779 "ICML",780 "Int Conf Mach Learn"781 ],782 "url": "https://icml.cc/"783 },784 "title": "DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation",785 "venue": "International Conference on Machine Learning",786 "year": 2022,787 "citationCount": 659,788 "openAccessPdf": {789 "url": "https://arxiv.org/pdf/2211.11501",790 "status": "GREEN",791 "license": null,792 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2211.11501, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."793 },794 "publicationTypes": [795 "JournalArticle",796 "Conference"797 ],798 "journal": {799 "pages": "18319-18345"800 },801 "authors": [802 {803 "authorId": "2191638875",804 "name": "Yuhang Lai"805 },806 {807 "authorId": "2279871391",808 "name": "Cheng-Xi Li"809 },810 {811 "authorId": "2518870917",812 "name": "Yi-Ming Wang"813 },814 {815 "authorId": "2146331419",816 "name": "Tianyi Zhang"817 },818 {819 "authorId": "51011000",820 "name": "Ruiqi Zhong"821 },822 {823 "authorId": "1982950",824 "name": "Luke Zettlemoyer"825 },826 {827 "authorId": "3156075",828 "name": "S. Yih"829 },830 {831 "authorId": "47070750",832 "name": "Daniel Fried"833 },834 {835 "authorId": "2128046669",836 "name": "Si-yi Wang"837 },838 {839 "authorId": "48881008",840 "name": "Tao Yu"841 }842 ],843 "abstract": "We introduce DS-1000, a code generation benchmark with a thousand data science problems spanning seven Python libraries, such as NumPy and Pandas. Compared to prior works, DS-1000 incorporates three core features. First, our problems reflect diverse, realistic, and practical use cases since we collected them from StackOverflow. Second, our automatic evaluation is highly specific (reliable) -- across all Codex-002-predicted solutions that our evaluation accept, only 1.8% of them are incorrect; we achieve this with multi-criteria metrics, checking both functional correctness by running test cases and surface-form constraints by restricting API usages or keywords. Finally, we proactively defend against memorization by slightly modifying our problems to be different from the original StackOverflow source; consequently, models cannot answer them correctly by memorizing the solutions from pre-training. The current best public system (Codex-002) achieves 43.3% accuracy, leaving ample room for improvement. We release our benchmark at https://ds1000-code-gen.github.io."844 },845 "arXiv:2212.08037": {846 "paperId": "05d77715d49714506a920f26c5432b92078cd37c",847 "externalIds": {848 "DBLP": "journals/corr/abs-2212-08037",849 "ArXiv": "2212.08037",850 "DOI": "10.48550/arXiv.2212.08037",851 "CorpusId": 254685584852 },853 "publicationVenue": {854 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",855 "name": "arXiv.org",856 "alternate_names": [857 "ArXiv"858 ],859 "issn": "2331-8422",860 "url": "https://arxiv.org"861 },862 "title": "Attributed Question Answering: Evaluation and Modeling for Attributed Large Language Models",863 "venue": "arXiv.org",864 "year": 2022,865 "citationCount": 163,866 "openAccessPdf": {867 "url": "http://arxiv.org/pdf/2212.08037",868 "status": "GREEN",869 "license": null,870 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2212.08037, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."871 },872 "publicationTypes": [873 "JournalArticle"874 ],875 "journal": {876 "name": "ArXiv",877 "volume": "abs/2212.08037"878 },879 "authors": [880 {881 "authorId": "1678747",882 "name": "Bernd Bohnet"883 },884 {885 "authorId": "2057663102",886 "name": "Vinh Q. Tran"887 },888 {889 "authorId": "2986975",890 "name": "Pat Verga"891 },892 {893 "authorId": "2335771",894 "name": "Roee Aharoni"895 },896 {897 "authorId": "3365603",898 "name": "D. Andor"899 },900 {901 "authorId": "7353832",902 "name": "Livio Baldini Soares"903 },904 {905 "authorId": "2754495",906 "name": "M. Ciaramita"907 },908 {909 "authorId": "144154709",910 "name": "Jacob Eisenstein"911 },912 {913 "authorId": "144422385",914 "name": "K. Ganchev"915 },916 {917 "authorId": "47426264",918 "name": "Jonathan Herzig"919 },920 {921 "authorId": "47214884",922 "name": "Kai Hui"923 },924 {925 "authorId": "15652489",926 "name": "T. Kwiatkowski"927 },928 {929 "authorId": "2359098271",930 "name": "Ji Ma"931 },932 {933 "authorId": "2148023",934 "name": "Jianmo Ni"935 },936 {937 "authorId": "100831413",938 "name": "Lierni Sestorain Saralegui"939 },940 {941 "authorId": "32303439",942 "name": "Tal Schuster"943 },944 {945 "authorId": "50056360",946 "name": "William W. Cohen"947 },948 {949 "authorId": "123052390",950 "name": "Michael Collins"951 },952 {953 "authorId": "143790066",954 "name": "Dipanjan Das"955 },956 {957 "authorId": "1680617",958 "name": "Donald Metzler"959 },960 {961 "authorId": "1754497",962 "name": "Slav Petrov"963 },964 {965 "authorId": "20825661",966 "name": "Kellie Webster"967 }968 ],969 "abstract": "Large language models (LLMs) have shown impressive results while requiring little or no direct supervision. Further, there is mounting evidence that LLMs may have potential in information-seeking scenarios. We believe the ability of an LLM to attribute the text that it generates is likely to be crucial in this setting. We formulate and study Attributed QA as a key first step in the development of attributed LLMs. We propose a reproducible evaluation framework for the task and benchmark a broad set of architectures. We take human annotations as a gold standard and show that a correlated automatic metric is suitable for development. Our experimental work gives concrete answers to two key questions (How to measure attribution?, and How well do current state-of-the-art methods perform on attribution?), and give some hints as to how to address a third (How to build LLMs with attribution?)."970 },971 "arXiv:2302.06590": {972 "paperId": "15abedb29536d50afeeec739a25358255cbda3e8",973 "externalIds": {974 "DBLP": "journals/corr/abs-2302-06590",975 "ArXiv": "2302.06590",976 "CorpusId": 256826778977 },978 "publicationVenue": {979 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",980 "name": "arXiv.org",981 "alternate_names": [982 "ArXiv"983 ],984 "issn": "2331-8422",985 "url": "https://arxiv.org"986 },987 "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",988 "venue": "arXiv.org",989 "year": 2023,990 "citationCount": 878,991 "openAccessPdf": {992 "url": "",993 "status": null,994 "license": null,995 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2302.06590, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."996 },997 "publicationTypes": [998 "JournalArticle"999 ],1000 "journal": {1001 "name": "ArXiv",1002 "volume": "abs/2302.06590"1003 },1004 "authors": [1005 {1006 "authorId": "67042377",1007 "name": "Sida Peng"1008 },1009 {1010 "authorId": "1734481",1011 "name": "E. Kalliamvakou"1012 },1013 {1014 "authorId": "2205544450",1015 "name": "Peter Cihon"1016 },1017 {1018 "authorId": "88741890",1019 "name": "Mert Demirer"1020 }1021 ],1022 "abstract": "Generative AI tools hold promise to increase human productivity. This paper presents results from a controlled experiment with GitHub Copilot, an AI pair programmer. Recruited software developers were asked to implement an HTTP server in JavaScript as quickly as possible. The treatment group, with access to the AI pair programmer, completed the task 55.8% faster than the control group. Observed heterogenous effects show promise for AI pair programmers to help people transition into software development careers."1023 },1024 "arXiv:2305.02251": {1025 "paperId": "98f166771681acfe51c0f69558d8a977f0782739",1026 "externalIds": {1027 "DBLP": "journals/corr/abs-2305-02251",1028 "ArXiv": "2305.02251",1029 "DOI": "10.1007/s10994-025-06955-2",1030 "CorpusId": 2584616201031 },1032 "publicationVenue": {1033 "id": "22c9862f-a25e-40cd-9d31-d09e68a293e6",1034 "name": "Machine-mediated learning",1035 "type": "journal",1036 "alternate_names": [1037 "Mach learn",1038 "Machine Learning",1039 "Mach Learn"1040 ],1041 "issn": "0732-6718",1042 "alternate_issns": [1043 "0885-6125"1044 ],1045 "url": "http://www.springer.com/computer/artificial/journal/10994",1046 "alternate_urls": [1047 "https://link.springer.com/journal/10994",1048 "http://www.springer.com/west/home/computer/artificial?SGWID=4-147-70-35726603-0"1049 ]1050 },1051 "title": "Automated Scientific Discovery: From Equation Discovery to Autonomous Discovery Systems",1052 "venue": "Machine-mediated learning",1053 "year": 2023,1054 "citationCount": 30,1055 "openAccessPdf": {1056 "url": "https://link.springer.com/content/pdf/10.1007/s10994-025-06955-2.pdf",1057 "status": "HYBRID",1058 "license": "CCBY",1059 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2305.02251, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1060 },1061 "publicationTypes": [1062 "JournalArticle",1063 "Review"1064 ],1065 "journal": {1066 "name": "Machine Learning",1067 "volume": "115"1068 },1069 "authors": [1070 {1071 "authorId": "2058253559",1072 "name": "Stefan Kramer"1073 },1074 {1075 "authorId": "1382852308",1076 "name": "Mattia Cerrato"1077 },1078 {1079 "authorId": "2037478739",1080 "name": "Jannis Brugger"1081 },1082 {1083 "authorId": "1693549",1084 "name": "S. D\u017eeroski"1085 },1086 {1087 "authorId": "33654217",1088 "name": "R. King"1089 }1090 ],1091 "abstract": "The paper surveys automated scientific discovery, from equation discovery and symbolic regression to autonomous discovery systems and agents. It discusses the individual approaches from a \"big picture\" perspective and in context, but also discusses open issues and recent topics like the various roles of deep neural networks in this area, aiding in the discovery of human-interpretable knowledge. Further, we will present closed-loop scientific discovery systems, starting with the pioneering work on the Adam system up to current efforts in fields from material science to astronomy. Finally, we will elaborate on autonomy from a machine learning perspective, but also in analogy to the autonomy levels in autonomous driving. The maximal level, level five, is defined to require no human intervention at all in the production of scientific knowledge. Achieving this is one step towards solving the Nobel Turing Grand Challenge to develop AI Scientists: AI systems capable of making Nobel-quality scientific discoveries highly autonomously at a level comparable, and possibly superior, to the best human scientists by 2050."1092 },1093 "arXiv:2305.05176": {1094 "paperId": "585f8b9725f5f5e5495c3508d39f70d1c053e190",1095 "externalIds": {1096 "ArXiv": "2305.05176",1097 "DBLP": "journals/corr/abs-2305-05176",1098 "DOI": "10.48550/arXiv.2305.05176",1099 "CorpusId": 2585643491100 },1101 "publicationVenue": null,1102 "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",1103 "venue": "Trans. Mach. Learn. Res.",1104 "year": 2023,1105 "citationCount": 1227,1106 "openAccessPdf": {1107 "url": "http://arxiv.org/pdf/2305.05176",1108 "status": "GREEN",1109 "license": null,1110 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2305.05176, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1111 },1112 "publicationTypes": [1113 "JournalArticle",1114 "Review"1115 ],1116 "journal": {1117 "name": "ArXiv",1118 "volume": "abs/2305.05176"1119 },1120 "authors": [1121 {1122 "authorId": "2119322954",1123 "name": "Lingjiao Chen"1124 },1125 {1126 "authorId": "143834867",1127 "name": "M. Zaharia"1128 },1129 {1130 "authorId": "145085305",1131 "name": "James Y. Zou"1132 }1133 ],1134 "abstract": "There is a rapidly growing number of large language models (LLMs) that users can query for a fee. We review the cost associated with querying popular LLM APIs, e.g. GPT-4, ChatGPT, J1-Jumbo, and find that these models have heterogeneous pricing structures, with fees that can differ by two orders of magnitude. In particular, using LLMs on large collections of queries and text can be expensive. Motivated by this, we outline and discuss three types of strategies that users can exploit to reduce the inference cost associated with using LLMs: 1) prompt adaptation, 2) LLM approximation, and 3) LLM cascade. As an example, we propose FrugalGPT, a simple yet flexible instantiation of LLM cascade which learns which combinations of LLMs to use for different queries in order to reduce cost and improve accuracy. Our experiments show that FrugalGPT can match the performance of the best individual LLM (e.g. GPT-4) with up to 98% cost reduction or improve the accuracy over GPT-4 by 4% with the same cost. The ideas and findings presented here lay a foundation for using LLMs sustainably and efficiently."1135 },1136 "arXiv:2305.17926": {1137 "paperId": "38d64919ba526868a850a0e5f6239d4c474b7e7e",1138 "externalIds": {1139 "DBLP": "journals/corr/abs-2305-17926",1140 "ArXiv": "2305.17926",1141 "DOI": "10.48550/arXiv.2305.17926",1142 "CorpusId": 2589603391143 },1144 "publicationVenue": {1145 "id": "1e33b3be-b2ab-46e9-96e8-d4eb4bad6e44",1146 "name": "Annual Meeting of the Association for Computational Linguistics",1147 "type": "conference",1148 "alternate_names": [1149 "Annu Meet Assoc Comput Linguistics",1150 "Meeting of the Association for Computational Linguistics",1151 "ACL",1152 "Meet Assoc Comput Linguistics"1153 ],1154 "url": "https://www.aclweb.org/anthology/venues/acl/"1155 },1156 "title": "Large Language Models are not Fair Evaluators",1157 "venue": "Annual Meeting of the Association for Computational Linguistics",1158 "year": 2023,1159 "citationCount": 1454,1160 "openAccessPdf": {1161 "url": "https://arxiv.org/pdf/2305.17926",1162 "status": "GREEN",1163 "license": null,1164 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2305.17926, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1165 },1166 "publicationTypes": [1167 "JournalArticle",1168 "Conference"1169 ],1170 "journal": {1171 "name": "ArXiv",1172 "volume": "abs/2305.17926"1173 },1174 "authors": [1175 {1176 "authorId": "144202874",1177 "name": "Peiyi Wang"1178 },1179 {1180 "authorId": "49192881",1181 "name": "Lei Li"1182 },1183 {1184 "authorId": "2146034504",1185 "name": "Liang Chen"1186 },1187 {1188 "authorId": "2117632647",1189 "name": "Ze-Fan Cai"1190 },1191 {1192 "authorId": "2116276849",1193 "name": "Dawei Zhu"1194 },1195 {1196 "authorId": "3186130",1197 "name": "Bing-Huai Lin"1198 },1199 {1200 "authorId": "2154235",1201 "name": "Yun-Bo Cao"1202 },1203 {1204 "authorId": "2275651222",1205 "name": "Ling-Peng Kong"1206 },1207 {1208 "authorId": "2144831944",1209 "name": "Qi Liu"1210 },1211 {1212 "authorId": "1701889",1213 "name": "Tianyu Liu"1214 },1215 {1216 "authorId": "3335836",1217 "name": "Zhi-Fang Sui"1218 }1219 ],1220 "abstract": "In this paper, we uncover a systematic bias in the evaluation paradigm of adopting large language models~(LLMs), e.g., GPT-4, as a referee to score and compare the quality of responses generated by candidate models. We find that the quality ranking of candidate responses can be easily hacked by simply altering their order of appearance in the context. This manipulation allows us to skew the evaluation result, making one model appear considerably superior to the other, e.g., Vicuna-13B could beat ChatGPT on 66 over 80 tested queries with ChatGPT as an evaluator. To address this issue, we propose a calibration framework with three simple yet effective strategies: 1) Multiple Evidence Calibration, which requires the evaluator model to generate multiple evaluation evidence before assigning ratings; 2) Balanced Position Calibration, which aggregates results across various orders to determine the final score; 3) Human-in-the-Loop Calibration, which introduces a balanced position diversity entropy to measure the difficulty of each example and seeks human assistance when needed. We also manually annotate the\"win/tie/lose\"outcomes of responses from ChatGPT and Vicuna-13B in the Vicuna Benchmark's question prompt, and extensive experiments demonstrate that our approach successfully mitigates evaluation bias, resulting in closer alignment with human judgments. We release our code and human annotation at \\url{https://github.com/i-Eval/FairEval} to facilitate future research."1221 },1222 "arXiv:2305.18248": {1223 "paperId": "c18e13ba65c7247774301314d181c87ee5ebc847",1224 "externalIds": {1225 "DBLP": "journals/corr/abs-2305-18248",1226 "ArXiv": "2305.18248",1227 "ACL": "2024.findings-eacl.62",1228 "DOI": "10.48550/arXiv.2305.18248",1229 "CorpusId": 2589603461230 },1231 "publicationVenue": {1232 "id": "479d5605-51be-4346-b1d6-4334084504df",1233 "name": "Findings",1234 "type": "journal",1235 "issn": "2652-8800",1236 "url": "https://findingspress.org/"1237 },1238 "title": "Do Language Models Know When They\u2019re Hallucinating References?",1239 "venue": "Findings",1240 "year": 2023,1241 "citationCount": 177,1242 "openAccessPdf": {1243 "url": "https://arxiv.org/pdf/2305.18248",1244 "status": "GREEN",1245 "license": null,1246 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2305.18248, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1247 },1248 "publicationTypes": [1249 "JournalArticle"1250 ],1251 "journal": {1252 "pages": "912-928"1253 },1254 "authors": [1255 {1256 "authorId": "2075308090",1257 "name": "A. Agrawal"1258 },1259 {1260 "authorId": "51903517",1261 "name": "M. Suzgun"1262 },1263 {1264 "authorId": "143722101",1265 "name": "L. Mackey"1266 },1267 {1268 "authorId": "2186481",1269 "name": "A. Kalai"1270 }1271 ],1272 "abstract": "State-of-the-art language models (LMs) are notoriously susceptible to generating hallucinated information. Such inaccurate outputs not only undermine the reliability of these models but also limit their use and raise serious concerns about misinformation and propaganda. In this work, we focus on hallucinated book and article references and present them as the \u201cmodel organism\u201d of language model hallucination research, due to their frequent and easy-to-discern nature. We posit that if a language model cites a particular reference in its output, then it should ideally possess sufficient information about its authors and content, among other relevant details. Using this basic insight, we illustrate that one can identify hallucinated references without ever consulting any external resources, by asking a set of direct or indirect queries to the language model about the references. These queries can be considered as \u201cconsistency checks.\u201d Our findings highlight that while LMs, including GPT-4, often produce inconsistent author lists for hallucinated references, they also often accurately recall the authors of real references. In this sense, the LM can be said to \u201cknow\u201d when it is hallucinating references. Furthermore, these findings show how hallucinated references can be dissected to shed light on their nature."1273 },1274 "arXiv:2306.05685": {1275 "paperId": "a0a79dad89857a96f8f71b14238e5237cbfc4787",1276 "externalIds": {1277 "DBLP": "journals/corr/abs-2306-05685",1278 "ArXiv": "2306.05685",1279 "DOI": "10.52202/075280-2020",1280 "CorpusId": 2591293981281 },1282 "publicationVenue": {1283 "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",1284 "name": "Neural Information Processing Systems",1285 "type": "conference",1286 "alternate_names": [1287 "Neural Inf Process Syst",1288 "NeurIPS",1289 "NIPS"1290 ],1291 "url": "http://neurips.cc/"1292 },1293 "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",1294 "venue": "Neural Information Processing Systems",1295 "year": 2023,1296 "citationCount": 12661,1297 "openAccessPdf": {1298 "url": "https://arxiv.org/pdf/2306.05685",1299 "status": "GREEN",1300 "license": null,1301 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2306.05685, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1302 },1303 "publicationTypes": [1304 "JournalArticle"1305 ],1306 "journal": {1307 "name": "ArXiv",1308 "volume": "abs/2306.05685"1309 },1310 "authors": [1311 {1312 "authorId": "2149970173",1313 "name": "Lian-Min Zheng"1314 },1315 {1316 "authorId": "2537924",1317 "name": "Wei-Lin Chiang"1318 },1319 {1320 "authorId": "2209360681",1321 "name": "Ying Sheng"1322 },1323 {1324 "authorId": "92721493",1325 "name": "Si-Yuan Zhuang"1326 },1327 {1328 "authorId": "1390573666",1329 "name": "Zhanghao Wu"1330 },1331 {1332 "authorId": "2152482391",1333 "name": "Yong-Hao Zhuang"1334 },1335 {1336 "authorId": "143872641",1337 "name": "Zi Lin"1338 },1339 {1340 "authorId": "2141335450",1341 "name": "Zhuo-Han Li"1342 },1343 {1344 "authorId": "2265517766",1345 "name": "Da-Cheng Li"1346 },1347 {1348 "authorId": "143977260",1349 "name": "E. Xing"1350 },1351 {1352 "authorId": "145140331",1353 "name": "Haotong Zhang"1354 },1355 {1356 "authorId": "49988044",1357 "name": "Joseph E. Gonzalez"1358 },1359 {1360 "authorId": "2055174324",1361 "name": "Ion Stoica"1362 }1363 ],1364 "abstract": "Evaluating large language model (LLM) based chat assistants is challenging due to their broad capabilities and the inadequacy of existing benchmarks in measuring human preferences. To address this, we explore using strong LLMs as judges to evaluate these models on more open-ended questions. We examine the usage and limitations of LLM-as-a-judge, including position, verbosity, and self-enhancement biases, as well as limited reasoning ability, and propose solutions to mitigate some of them. We then verify the agreement between LLM judges and human preferences by introducing two benchmarks: MT-bench, a multi-turn question set; and Chatbot Arena, a crowdsourced battle platform. Our results reveal that strong LLM judges like GPT-4 can match both controlled and crowdsourced human preferences well, achieving over 80% agreement, the same level of agreement between humans. Hence, LLM-as-a-judge is a scalable and explainable way to approximate human preferences, which are otherwise very expensive to obtain. Additionally, we show our benchmark and traditional benchmarks complement each other by evaluating several variants of LLaMA and Vicuna. The MT-bench questions, 3K expert votes, and 30K conversations with human preferences are publicly available at https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge."1365 },1366 "arXiv:2308.03688": {1367 "paperId": "5dbf93a68b7fda600521f046dea35ea8ba9e884f",1368 "externalIds": {1369 "ArXiv": "2308.03688",1370 "DBLP": "journals/corr/abs-2308-03688",1371 "DOI": "10.48550/arXiv.2308.03688",1372 "CorpusId": 2606822491373 },1374 "publicationVenue": {1375 "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",1376 "name": "International Conference on Learning Representations",1377 "type": "conference",1378 "alternate_names": [1379 "Int Conf Learn Represent",1380 "ICLR"1381 ],1382 "url": "https://iclr.cc/"1383 },1384 "title": "AgentBench: Evaluating LLMs as Agents",1385 "venue": "International Conference on Learning Representations",1386 "year": 2023,1387 "citationCount": 1563,1388 "openAccessPdf": {1389 "url": "https://arxiv.org/pdf/2308.03688",1390 "status": "CLOSED",1391 "license": null,1392 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2308.03688, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1393 },1394 "publicationTypes": [1395 "JournalArticle"1396 ],1397 "journal": {1398 "name": "ArXiv",1399 "volume": "abs/2308.03688"1400 },1401 "authors": [1402 {1403 "authorId": "2111312892",1404 "name": "Xiao Liu"1405 },1406 {1407 "authorId": "2110750673",1408 "name": "Hao Yu"1409 },1410 {1411 "authorId": "2228752234",1412 "name": "Hanchen Zhang"1413 },1414 {1415 "authorId": "2138973537",1416 "name": "Yi-Fan Xu"1417 },1418 {1419 "authorId": "2181283109",1420 "name": "Xuanyu Lei"1421 },1422 {1423 "authorId": "2051311700",1424 "name": "Han-Yu Lai"1425 },1426 {1427 "authorId": "2022231256",1428 "name": "Yu Gu"1429 },1430 {1431 "authorId": "2116405624",1432 "name": "Yu-Xian Gu"1433 },1434 {1435 "authorId": "2364101023",1436 "name": "Han Ding"1437 },1438 {1439 "authorId": "2310631841",1440 "name": "Kai-Wen Men"1441 },1442 {1443 "authorId": "2218347104",1444 "name": "Kejuan Yang"1445 },1446 {1447 "authorId": "2300177449",1448 "name": "Shu-Dan Zhang"1449 },1450 {1451 "authorId": "145924070",1452 "name": "Xiang Deng"1453 },1454 {1455 "authorId": "2051712753",1456 "name": "Ao-Han Zeng"1457 },1458 {1459 "authorId": "66395694",1460 "name": "Zheng-Xiao Du"1461 },1462 {1463 "authorId": "2146063748",1464 "name": "Chenhui Zhang"1465 },1466 {1467 "authorId": "2152107339",1468 "name": "Sheng-Qi Shen"1469 },1470 {1471 "authorId": "1993655237",1472 "name": "Tianjun Zhang"1473 },1474 {1475 "authorId": "2191455",1476 "name": "Sheng Shen"1477 },1478 {1479 "authorId": "1758652",1480 "name": "Yu Su"1481 },1482 {1483 "authorId": "1515546612",1484 "name": "Huan Sun"1485 },1486 {1487 "authorId": "1730108",1488 "name": "Minlie Huang"1489 },1490 {1491 "authorId": "2047998",1492 "name": "Yu-Xiao Dong"1493 },1494 {1495 "authorId": "2148911956",1496 "name": "Jie Tang"1497 }1498 ],1499 "abstract": "The potential of Large Language Model (LLM) as agents has been widely acknowledged recently. Thus, there is an urgent need to quantitatively \\textit{evaluate LLMs as agents} on challenging tasks in interactive environments. We present AgentBench, a multi-dimensional benchmark that consists of 8 distinct environments to assess LLM-as-Agent's reasoning and decision-making abilities. Our extensive test over \\num API-based and open-sourced (OSS) LLMs shows that, while top commercial LLMs present a strong ability of acting as agents in complex environments, there is a significant disparity in performance between them and many OSS competitors that are no larger than 70B. We identify the typical reasons of failures in environments and LLMs, showing that poor long-term reasoning, decision-making, and instruction following abilities are the main obstacles for developing usable LLM agents. Improving instruction following and training on high quality multi-round alignment data could improve agent performance. And different from existing assumptions, training on code present ambivalent impacts on different agent tasks. Datasets, environments, and an integrated evaluation package for AgentBench are released at https://github.com/THUDM/AgentBench."1500 },1501 "arXiv:2308.08493": {1502 "paperId": "d6d0fd994f37b630f35945736b5e1bb198148404",1503 "externalIds": {1504 "ArXiv": "2308.08493",1505 "DBLP": "journals/corr/abs-2308-08493",1506 "DOI": "10.48550/arXiv.2308.08493",1507 "CorpusId": 2609255011508 },1509 "publicationVenue": {1510 "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",1511 "name": "International Conference on Learning Representations",1512 "type": "conference",1513 "alternate_names": [1514 "Int Conf Learn Represent",1515 "ICLR"1516 ],1517 "url": "https://iclr.cc/"1518 },1519 "title": "Time Travel in LLMs: Tracing Data Contamination in Large Language Models",1520 "venue": "International Conference on Learning Representations",1521 "year": 2023,1522 "citationCount": 235,1523 "openAccessPdf": {1524 "url": "https://arxiv.org/pdf/2308.08493",1525 "status": "GREEN",1526 "license": null,1527 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2308.08493, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1528 },1529 "publicationTypes": [1530 "JournalArticle"1531 ],1532 "journal": {1533 "name": "ArXiv",1534 "volume": "abs/2308.08493"1535 },1536 "authors": [1537 {1538 "authorId": "65754049",1539 "name": "Shahriar Golchin"1540 },1541 {1542 "authorId": "1760868",1543 "name": "M. Surdeanu"1544 }1545 ],1546 "abstract": "Data contamination, i.e., the presence of test data from downstream tasks in the training data of large language models (LLMs), is a potential major issue in measuring LLMs' real effectiveness on other tasks. We propose a straightforward yet effective method for identifying data contamination within LLMs. At its core, our approach starts by identifying potential contamination at the instance level; using this information, our approach then assesses wider contamination at the partition level. To estimate contamination of individual instances, we employ\"guided instruction:\"a prompt consisting of the dataset name, partition type, and the random-length initial segment of a reference instance, asking the LLM to complete it. An instance is flagged as contaminated if the LLM's output either exactly or nearly matches the latter segment of the reference. To understand if an entire partition is contaminated, we propose two ideas. The first idea marks a dataset partition as contaminated if the average overlap score with the reference instances (as measured by ROUGE-L or BLEURT) is statistically significantly better with the completions from guided instruction compared to a\"general instruction\"that does not include the dataset and partition name. The second idea marks a dataset partition as contaminated if a classifier based on GPT-4 with few-shot in-context learning prompt marks multiple generated completions as exact/near-exact matches of the corresponding reference instances. Our best method achieves an accuracy between 92% and 100% in detecting if an LLM is contaminated with seven datasets, containing train and test/validation partitions, when contrasted with manual evaluation by human experts. Further, our findings indicate that GPT-4 is contaminated with AG News, WNLI, and XSum datasets."1547 },1548 "arXiv:2309.11495": {1549 "paperId": "4b0b56be0ae9479d2bd5c2f0943db1906343c10f",1550 "externalIds": {1551 "DBLP": "journals/corr/abs-2309-11495",1552 "ArXiv": "2309.11495",1553 "DOI": "10.48550/arXiv.2309.11495",1554 "CorpusId": 2620625651555 },1556 "publicationVenue": {1557 "id": "1e33b3be-b2ab-46e9-96e8-d4eb4bad6e44",1558 "name": "Annual Meeting of the Association for Computational Linguistics",1559 "type": "conference",1560 "alternate_names": [1561 "Annu Meet Assoc Comput Linguistics",1562 "Meeting of the Association for Computational Linguistics",1563 "ACL",1564 "Meet Assoc Comput Linguistics"1565 ],1566 "url": "https://www.aclweb.org/anthology/venues/acl/"1567 },1568 "title": "Chain-of-Verification Reduces Hallucination in Large Language Models",1569 "venue": "Annual Meeting of the Association for Computational Linguistics",1570 "year": 2023,1571 "citationCount": 649,1572 "openAccessPdf": {1573 "url": "https://arxiv.org/pdf/2309.11495",1574 "status": "CLOSED",1575 "license": null,1576 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2309.11495, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1577 },1578 "publicationTypes": [1579 "JournalArticle",1580 "Conference"1581 ],1582 "journal": {1583 "pages": "3563-3578"1584 },1585 "authors": [1586 {1587 "authorId": "3448411",1588 "name": "S. Dhuliawala"1589 },1590 {1591 "authorId": "100653935",1592 "name": "M. Komeili"1593 },1594 {1595 "authorId": "2276766957",1596 "name": "Jing Xu"1597 },1598 {1599 "authorId": "48647153",1600 "name": "R. Raileanu"1601 },1602 {1603 "authorId": "2243015223",1604 "name": "Xian Li"1605 },1606 {1607 "authorId": "1709797",1608 "name": "Asli Celikyilmaz"1609 },1610 {1611 "authorId": "2243265350",1612 "name": "Jason E. Weston"1613 }1614 ],1615 "abstract": "Generation of plausible yet incorrect factual information, termed hallucination, is an unsolved issue in large language models. We study the ability of language models to deliberate on the responses they give in order to correct their mistakes. We develop the Chain-of-Verification (CoVe) method whereby the model first (i) drafts an initial response; then (ii) plans verification questions to fact-check its draft; (iii) answers those questions independently so the answers are not biased by other responses; and (iv) generates its final verified response. In experiments, we show CoVe decreases hallucinations across a variety of tasks, from list-based questions from Wikidata, closed book MultiSpanQA and longform text generation."1616 },1617 "arXiv:2310.01798": {1618 "paperId": "6d4bacb69923e1e94fb4de468b939ce6db32fb51",1619 "externalIds": {1620 "DBLP": "conf/iclr/0009CMZYSZ24",1621 "ArXiv": "2310.01798",1622 "DOI": "10.48550/arXiv.2310.01798",1623 "CorpusId": 2636091321624 },1625 "publicationVenue": {1626 "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",1627 "name": "International Conference on Learning Representations",1628 "type": "conference",1629 "alternate_names": [1630 "Int Conf Learn Represent",1631 "ICLR"1632 ],1633 "url": "https://iclr.cc/"1634 },1635 "title": "Large Language Models Cannot Self-Correct Reasoning Yet",1636 "venue": "International Conference on Learning Representations",1637 "year": 2023,1638 "citationCount": 1410,1639 "openAccessPdf": {1640 "url": "https://arxiv.org/pdf/2310.01798",1641 "status": "CLOSED",1642 "license": null,1643 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2310.01798, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1644 },1645 "publicationTypes": [1646 "JournalArticle"1647 ],1648 "journal": {1649 "name": "ArXiv",1650 "volume": "abs/2310.01798"1651 },1652 "authors": [1653 {1654 "authorId": "1490651934",1655 "name": "Jie Huang"1656 },1657 {1658 "authorId": "2238263119",1659 "name": "Xin-Yun Chen"1660 },1661 {1662 "authorId": "1817207",1663 "name": "Swaroop Mishra"1664 },1665 {1666 "authorId": "2253804927",1667 "name": "Huaixiu Steven Zheng"1668 },1669 {1670 "authorId": "40625240",1671 "name": "Adams Wei Yu"1672 },1673 {1674 "authorId": "2293658042",1675 "name": "Xinying Song"1676 },1677 {1678 "authorId": "2256313467",1679 "name": "Denny Zhou"1680 }1681 ],1682 "abstract": "Large Language Models (LLMs) have emerged as a groundbreaking technology with their unparalleled text generation capabilities across various applications. Nevertheless, concerns persist regarding the accuracy and appropriateness of their generated content. A contemporary methodology, self-correction, has been proposed as a remedy to these issues. Building upon this premise, this paper critically examines the role and efficacy of self-correction within LLMs, shedding light on its true potential and limitations. Central to our investigation is the notion of intrinsic self-correction, whereby an LLM attempts to correct its initial responses based solely on its inherent capabilities, without the crutch of external feedback. In the context of reasoning, our research indicates that LLMs struggle to self-correct their responses without external feedback, and at times, their performance even degrades after self-correction. Drawing from these insights, we offer suggestions for future research and practical applications in this field."1683 },1684 "arXiv:2310.03302": {1685 "paperId": "b8ee0b5322382807e687c95cc87b059d3f348495",1686 "externalIds": {1687 "DBLP": "conf/icml/HuangVLL24",1688 "ArXiv": "2310.03302",1689 "CorpusId": 2636715411690 },1691 "publicationVenue": {1692 "id": "fc0a208c-acb7-47dc-a0d4-af8190e21d29",1693 "name": "International Conference on Machine Learning",1694 "type": "conference",1695 "alternate_names": [1696 "ICML",1697 "Int Conf Mach Learn"1698 ],1699 "url": "https://icml.cc/"1700 },1701 "title": "MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation",1702 "venue": "International Conference on Machine Learning",1703 "year": 2023,1704 "citationCount": 438,1705 "openAccessPdf": {1706 "url": "",1707 "status": null,1708 "license": null,1709 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2310.03302, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1710 },1711 "publicationTypes": [1712 "JournalArticle",1713 "Conference"1714 ],1715 "journal": {1716 "pages": "20271-20309"1717 },1718 "authors": [1719 {1720 "authorId": "144862341",1721 "name": "Qian Huang"1722 },1723 {1724 "authorId": "66535001",1725 "name": "Jian Vora"1726 },1727 {1728 "authorId": "2249641250",1729 "name": "Percy Liang"1730 },1731 {1732 "authorId": "1702139",1733 "name": "J. Leskovec"1734 }1735 ],1736 "abstract": "A central aspect of machine learning research is experimentation, the process of designing and running experiments, analyzing the results, and iterating towards some positive outcome (e.g., improving accuracy). Could agents driven by powerful language models perform machine learning experimentation effectively? To answer this question, we introduce MLAgentBench, a suite of 13 tasks ranging from improving model performance on CIFAR-10 to recent research problems like BabyLM. For each task, an agent can perform actions like reading/writing files, executing code, and inspecting outputs. We then construct an agent that can perform ML experimentation based on ReAct framework. We benchmark agents based on Claude v1.0, Claude v2.1, Claude v3 Opus, GPT-4, GPT-4-turbo, Gemini-Pro, and Mixtral and find that a Claude v3 Opus agent is the best in terms of success rate. It can build compelling ML models over many tasks in MLAgentBench with 37.5% average success rate. Our agents also display highly interpretable plans and actions. However, the success rates vary considerably; they span from 100% on well-established older datasets to as low as 0% on recent Kaggle challenges created potentially after the underlying LM was trained. Finally, we identify several key challenges for LM-based agents such as long-term planning and reducing hallucination. Our code is released at https://github.com/snap-stanford/MLAgentBench."1737 },1738 "arXiv:2310.06770": {1739 "paperId": "94a5f96308729e31c1ffbc0f0618db87795092fe",1740 "externalIds": {1741 "DBLP": "journals/corr/abs-2310-06770",1742 "ArXiv": "2310.06770",1743 "DOI": "10.48550/arXiv.2310.06770",1744 "CorpusId": 2638296971745 },1746 "publicationVenue": {1747 "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",1748 "name": "International Conference on Learning Representations",1749 "type": "conference",1750 "alternate_names": [1751 "Int Conf Learn Represent",1752 "ICLR"1753 ],1754 "url": "https://iclr.cc/"1755 },1756 "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",1757 "venue": "International Conference on Learning Representations",1758 "year": 2023,1759 "citationCount": 4627,1760 "openAccessPdf": {1761 "url": "https://arxiv.org/pdf/2310.06770",1762 "status": "CLOSED",1763 "license": null,1764 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2310.06770, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1765 },1766 "publicationTypes": [1767 "JournalArticle"1768 ],1769 "journal": {1770 "name": "ArXiv",1771 "volume": "abs/2310.06770"1772 },1773 "authors": [1774 {1775 "authorId": "2155269683",1776 "name": "Carlos E. Jimenez"1777 },1778 {1779 "authorId": "2109727379",1780 "name": "John Yang"1781 },1782 {1783 "authorId": "2127066887",1784 "name": "Alexander Wettig"1785 },1786 {1787 "authorId": "2256984659",1788 "name": "Shun-Yu Yao"1789 },1790 {1791 "authorId": "40428350",1792 "name": "Ke-Xin Pei"1793 },1794 {1795 "authorId": "40170001",1796 "name": "Ofir Press"1797 },1798 {1799 "authorId": "2135381714",1800 "name": "Karthik R. Narasimhan"1801 }1802 ],1803 "abstract": "Language models have outpaced our ability to evaluate them effectively, but for their future development it is essential to study the frontier of their capabilities. We find real-world software engineering to be a rich, sustainable, and challenging testbed for evaluating the next generation of language models. To this end, we introduce SWE-bench, an evaluation framework consisting of $2,294$ software engineering problems drawn from real GitHub issues and corresponding pull requests across $12$ popular Python repositories. Given a codebase along with a description of an issue to be resolved, a language model is tasked with editing the codebase to address the issue. Resolving issues in SWE-bench frequently requires understanding and coordinating changes across multiple functions, classes, and even files simultaneously, calling for models to interact with execution environments, process extremely long contexts and perform complex reasoning that goes far beyond traditional code generation tasks. Our evaluations show that both state-of-the-art proprietary models and our fine-tuned model SWE-Llama can resolve only the simplest issues. The best-performing model, Claude 2, is able to solve a mere $1.96$% of the issues. Advances on SWE-bench represent steps towards LMs that are more practical, intelligent, and autonomous."1804 },1805 "arXiv:2311.01964": {1806 "paperId": "84725855d10b531eb8cbe54935dda0440c2fc750",1807 "externalIds": {1808 "ArXiv": "2311.01964",1809 "DBLP": "journals/corr/abs-2311-01964",1810 "DOI": "10.48550/arXiv.2311.01964",1811 "CorpusId": 2650190211812 },1813 "publicationVenue": {1814 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",1815 "name": "arXiv.org",1816 "alternate_names": [1817 "ArXiv"1818 ],1819 "issn": "2331-8422",1820 "url": "https://arxiv.org"1821 },1822 "title": "Don't Make Your LLM an Evaluation Benchmark Cheater",1823 "venue": "arXiv.org",1824 "year": 2023,1825 "citationCount": 263,1826 "openAccessPdf": {1827 "url": "",1828 "status": null,1829 "license": null,1830 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2311.01964, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1831 },1832 "publicationTypes": [1833 "JournalArticle"1834 ],1835 "journal": {1836 "name": "ArXiv",1837 "volume": "abs/2311.01964"1838 },1839 "authors": [1840 {1841 "authorId": "2265383494",1842 "name": "Kun Zhou"1843 },1844 {1845 "authorId": "1900406",1846 "name": "Yutao Zhu"1847 },1848 {1849 "authorId": "2256558402",1850 "name": "Zhi-Peng Chen"1851 },1852 {1853 "authorId": "2265461972",1854 "name": "Wentong Chen"1855 },1856 {1857 "authorId": "2257376413",1858 "name": "W. Zhao"1859 },1860 {1861 "authorId": "2265519430",1862 "name": "Xu Chen"1863 },1864 {1865 "authorId": "2257310922",1866 "name": "Yankai Lin"1867 },1868 {1869 "authorId": "2186578511",1870 "name": "Ji-Rong Wen"1871 },1872 {1873 "authorId": "2161986932",1874 "name": "Jia-Wei Han"1875 }1876 ],1877 "abstract": "Large language models~(LLMs) have greatly advanced the frontiers of artificial intelligence, attaining remarkable improvement in model capacity. To assess the model performance, a typical approach is to construct evaluation benchmarks for measuring the ability level of LLMs in different aspects. Despite that a number of high-quality benchmarks have been released, the concerns about the appropriate use of these benchmarks and the fair comparison of different models are increasingly growing. Considering these concerns, in this paper, we discuss the potential risk and impact of inappropriately using evaluation benchmarks and misleadingly interpreting the evaluation results. Specially, we focus on a special issue that would lead to inappropriate evaluation, \\ie \\emph{benchmark leakage}, referring that the data related to evaluation sets is occasionally used for model training. This phenomenon now becomes more common since pre-training data is often prepared ahead of model test. We conduct extensive experiments to study the effect of benchmark leverage, and find that it can dramatically boost the evaluation results, which would finally lead to an unreliable assessment of model performance. To improve the use of existing evaluation benchmarks, we finally present several guidelines for both LLM developers and benchmark maintainers. We hope this work can draw attention to appropriate training and evaluation of LLMs."1878 },1879 "arXiv:2311.09783": {1880 "paperId": "af565483dfbe3b0fa4fe9f715170666a06bce5ac",1881 "externalIds": {1882 "ACL": "2024.naacl-long.482",1883 "DBLP": "conf/naacl/Deng0TGC24",1884 "ArXiv": "2311.09783",1885 "DOI": "10.48550/arXiv.2311.09783",1886 "CorpusId": 2652206951887 },1888 "publicationVenue": {1889 "id": "01103732-3808-4930-b8e4-7e9e68d5c68d",1890 "name": "North American Chapter of the Association for Computational Linguistics",1891 "type": "conference",1892 "alternate_names": [1893 "North Am Chapter Assoc Comput Linguistics",1894 "NAACL"1895 ],1896 "url": "https://www.aclweb.org/portal/naacl"1897 },1898 "title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models",1899 "venue": "North American Chapter of the Association for Computational Linguistics",1900 "year": 2023,1901 "citationCount": 208,1902 "openAccessPdf": {1903 "url": "",1904 "status": null,1905 "license": null,1906 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2311.09783, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1907 },1908 "publicationTypes": [1909 "JournalArticle",1910 "Conference"1911 ],1912 "journal": {1913 "pages": "8706-8719"1914 },1915 "authors": [1916 {1917 "authorId": "2266840268",1918 "name": "Chunyuan Deng"1919 },1920 {1921 "authorId": "46316984",1922 "name": "Yilun Zhao"1923 },1924 {1925 "authorId": "47274259",1926 "name": "Xiangru Tang"1927 },1928 {1929 "authorId": "2201323142",1930 "name": "M. Gerstein"1931 },1932 {1933 "authorId": "2266838179",1934 "name": "Arman Cohan"1935 }1936 ],1937 "abstract": "Recent observations have underscored a disparity between the inflated benchmark scores and the actual performance of LLMs, raising concerns about potential contamination of evaluation benchmarks. This issue is especially critical for closed-source models and certain open-source models where training data transparency is lacking. In this paper we study data contamination by proposing two methods tailored for both open-source and proprietary LLMs. We first introduce a retrieval-based system to explore potential overlaps between evaluation benchmarks and pretraining corpora. We further present a novel investigation protocol named Testset Slot Guessing (TS-Guessing), applicable to both open and proprietary models. This approach entails masking a wrong answer in a multiple-choice question and prompting the model to fill in the gap. Additionally, it involves obscuring an unlikely word in an evaluation example and asking the model to produce it. We find that certain commercial LLMs could surprisingly guess the missing option in various test sets. Specifically, in the MMLU benchmark, ChatGPT and GPT-4 demonstrated an exact match rate of 52% and 57%, respectively, in guessing the missing options in benchmark test data. We hope these results underscore the need for more robust evaluation methodologies and benchmarks in the field."1938 },1939 "arXiv:2401.05507": {1940 "paperId": "ecd5d38b8590f6e6a504b29110d16ff717cd56ef",1941 "externalIds": {1942 "DBLP": "journals/corr/abs-2401-05507",1943 "ArXiv": "2401.05507",1944 "DOI": "10.48550/arXiv.2401.05507",1945 "CorpusId": 2669331851946 },1947 "publicationVenue": {1948 "id": "fc0a208c-acb7-47dc-a0d4-af8190e21d29",1949 "name": "International Conference on Machine Learning",1950 "type": "conference",1951 "alternate_names": [1952 "ICML",1953 "Int Conf Mach Learn"1954 ],1955 "url": "https://icml.cc/"1956 },1957 "title": "InfiAgent-DABench: Evaluating Agents on Data Analysis Tasks",1958 "venue": "International Conference on Machine Learning",1959 "year": 2024,1960 "citationCount": 173,1961 "openAccessPdf": {1962 "url": "",1963 "status": null,1964 "license": null,1965 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2401.05507, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."1966 },1967 "publicationTypes": [1968 "JournalArticle",1969 "Conference"1970 ],1971 "journal": {1972 "pages": "19544-19572"1973 },1974 "authors": [1975 {1976 "authorId": "2278917214",1977 "name": "Xueyu Hu"1978 },1979 {1980 "authorId": "2279090109",1981 "name": "Ziyu Zhao"1982 },1983 {1984 "authorId": "2279259092",1985 "name": "Shuang Wei"1986 },1987 {1988 "authorId": "2121930095",1989 "name": "Ziwei Chai"1990 },1991 {1992 "authorId": "2258790160",1993 "name": "Qian-Li Ma"1994 },1995 {1996 "authorId": "2279157833",1997 "name": "Guoyin Wang"1998 },1999 {2000 "authorId": "2279096273",2001 "name": "Xu-Wu Wang"2002 },2003 {2004 "authorId": "2279157215",2005 "name": "Jing Su"2006 },2007 {2008 "authorId": "2282345352",2009 "name": "Jingjing Xu"2010 },2011 {2012 "authorId": "2279262722",2013 "name": "Ming Zhu"2014 },2015 {2016 "authorId": "2117877910",2017 "name": "Yao Cheng"2018 },2019 {2020 "authorId": "2257015367",2021 "name": "Jianbo Yuan"2022 },2023 {2024 "authorId": "2372128161",2025 "name": "Ji-Wei Li"2026 },2027 {2028 "authorId": "33870528",2029 "name": "Kun Kuang"2030 },2031 {2032 "authorId": "2283100978",2033 "name": "Yang Yang"2034 },2035 {2036 "authorId": "2212243155",2037 "name": "Hongxia Yang"2038 },2039 {2040 "authorId": "2275491998",2041 "name": "Fei Wu"2042 }2043 ],2044 "abstract": "In this paper, we introduce InfiAgent-DABench, the first benchmark specifically designed to evaluate LLM-based agents on data analysis tasks. These tasks require agents to end-to-end solving complex tasks by interacting with an execution environment. This benchmark contains DAEval, a dataset consisting of 257 data analysis questions derived from 52 CSV files, and an agent framework which incorporates LLMs to serve as data analysis agents for both serving and evaluation. Since data analysis questions are often open-ended and hard to evaluate without human supervision, we adopt a format-prompting technique to convert each question into a closed-form format so that they can be automatically evaluated. Our extensive benchmarking of 34 LLMs uncovers the current challenges encountered in data analysis tasks. In addition, building on top of our agent framework, we develop a specialized agent, DAAgent, which surpasses GPT-3.5 by 3.9% on DABench. Evaluation datasets and toolkits for InfiAgent-DABench are released at https://github.com/InfiAgent/InfiAgent ."2045 },2046 "arXiv:2402.18679": {2047 "paperId": "01e863776846ebd1a9a7acc4a9ca24217f953aa2",2048 "externalIds": {2049 "ArXiv": "2402.18679",2050 "DBLP": "journals/corr/abs-2402-18679",2051 "DOI": "10.48550/arXiv.2402.18679",2052 "CorpusId": 2680632922053 },2054 "publicationVenue": {2055 "id": "1e33b3be-b2ab-46e9-96e8-d4eb4bad6e44",2056 "name": "Annual Meeting of the Association for Computational Linguistics",2057 "type": "conference",2058 "alternate_names": [2059 "Annu Meet Assoc Comput Linguistics",2060 "Meeting of the Association for Computational Linguistics",2061 "ACL",2062 "Meet Assoc Comput Linguistics"2063 ],2064 "url": "https://www.aclweb.org/anthology/venues/acl/"2065 },2066 "title": "Data Interpreter: An LLM Agent For Data Science",2067 "venue": "Annual Meeting of the Association for Computational Linguistics",2068 "year": 2024,2069 "citationCount": 309,2070 "openAccessPdf": {2071 "url": "",2072 "status": null,2073 "license": null,2074 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2402.18679, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."2075 },2076 "publicationTypes": [2077 "JournalArticle",2078 "Conference"2079 ],2080 "journal": {2081 "pages": "19796-19821"2082 },2083 "authors": [2084 {2085 "authorId": "2004160930",2086 "name": "Si-Rui Hong"2087 },2088 {2089 "authorId": "2289032497",2090 "name": "Yizhang Lin"2091 },2092 {2093 "authorId": "2288274546",2094 "name": "Bangbang Liu"2095 },2096 {2097 "authorId": "2253982381",2098 "name": "Bin-Hao Wu"2099 },2100 {2101 "authorId": "2390481614",2102 "name": "Danyang Li"2103 },2104 {2105 "authorId": "2288200541",2106 "name": "Jiaqi Chen"2107 },2108 {2109 "authorId": "2288348676",2110 "name": "Jiayi Zhang"2111 },2112 {2113 "authorId": "2267427792",2114 "name": "Jin-Lin Wang"2115 },2116 {2117 "authorId": "50081530",2118 "name": "Lingyao Zhang"2119 },2120 {2121 "authorId": "1722264111",2122 "name": "Ming-Chen Zhuge"2123 },2124 {2125 "authorId": "2179710545",2126 "name": "Taicheng Guo"2127 },2128 {2129 "authorId": "2278239110",2130 "name": "Tuo Zhou"2131 },2132 {2133 "authorId": "2112528845",2134 "name": "Wei Tao"2135 },2136 {2137 "authorId": "2287878279",2138 "name": "Wen-Yi Wang"2139 },2140 {2141 "authorId": "47274259",2142 "name": "Xiangru Tang"2143 },2144 {2145 "authorId": "2387162170",2146 "name": "Xiang Lu"2147 },2148 {2149 "authorId": "2288163160",2150 "name": "Xin-Bing Liang"2151 },2152 {2153 "authorId": "2288180253",2154 "name": "Yaying Fei"2155 },2156 {2157 "authorId": "2153513408",2158 "name": "Yu-Heng Cheng"2159 },2160 {2161 "authorId": "1797090",2162 "name": "Zhibin Gou"2163 },2164 {2165 "authorId": "2300805791",2166 "name": "Zong-Ze Xu"2167 },2168 {2169 "authorId": "2867893",2170 "name": "Chenglin Wu"2171 },2172 {2173 "authorId": "2266160753",2174 "name": "Li Zhang"2175 },2176 {2177 "authorId": "2324063491",2178 "name": "Min Yang"2179 },2180 {2181 "authorId": "51056401",2182 "name": "Xia-Wu Zheng"2183 }2184 ],2185 "abstract": "Large Language Model (LLM)-based agents have shown effectiveness across many applications. However, their use in data science scenarios requiring solving long-term interconnected tasks, dynamic data adjustments and domain expertise remains challenging. Previous approaches primarily focus on individual tasks, making it difficult to assess the complete data science workflow. Moreover, they struggle to handle real-time changes in intermediate data and fail to adapt dynamically to evolving task dependencies inherent to data science problems. In this paper, we present Data Interpreter, an LLM-based agent designed to automatically solve various data science problems end-to-end. Our Data Interpreter incorporates two key modules: 1) Hierarchical Graph Modeling, which breaks down complex problems into manageable subproblems, enabling dynamic node generation and graph optimization; and 2) Programmable Node Generation, a technique that refines and verifies each subproblem to iteratively improve code generation results and robustness. Extensive experiments consistently demonstrate the superiority of Data Interpreter. On InfiAgent-DABench, it achieves a 25% performance boost, raising accuracy from 75.9% to 94.9%. For machine learning and open-ended tasks, it improves performance from 88% to 95%, and from 60% to 97%, respectively. Moreover, on the MATH dataset, Data Interpreter achieves remarkable performance with a 26% improvement compared to state-of-the-art baselines. The code is available at https://github.com/geekan/MetaGPT."2186 },2187 "arXiv:2404.07738": {2188 "paperId": "51b7b3ad7645a69e3c1c80cae69473b8bd472f67",2189 "externalIds": {2190 "DBLP": "conf/naacl/BaekJCH25",2191 "ArXiv": "2404.07738",2192 "DOI": "10.48550/arXiv.2404.07738",2193 "CorpusId": 2690428442194 },2195 "publicationVenue": {2196 "id": "01103732-3808-4930-b8e4-7e9e68d5c68d",2197 "name": "North American Chapter of the Association for Computational Linguistics",2198 "type": "conference",2199 "alternate_names": [2200 "North Am Chapter Assoc Comput Linguistics",2201 "NAACL"2202 ],2203 "url": "https://www.aclweb.org/portal/naacl"2204 },2205 "title": "ResearchAgent: Iterative Research Idea Generation over Scientific Literature with Large Language Models",2206 "venue": "North American Chapter of the Association for Computational Linguistics",2207 "year": 2024,2208 "citationCount": 288,2209 "openAccessPdf": {2210 "url": "",2211 "status": null,2212 "license": null,2213 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2404.07738, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."2214 },2215 "publicationTypes": [2216 "JournalArticle",2217 "Conference",2218 "Review"2219 ],2220 "journal": {2221 "name": "ArXiv",2222 "volume": "abs/2404.07738"2223 },2224 "authors": [2225 {2226 "authorId": "90765684",2227 "name": "Jinheon Baek"2228 },2229 {2230 "authorId": "3001990",2231 "name": "S. Jauhar"2232 },2233 {2234 "authorId": "73040249",2235 "name": "Silviu Cucerzan"2236 },2237 {2238 "authorId": "2260611009",2239 "name": "Sung Ju Hwang"2240 }2241 ],2242 "abstract": "The pace of scientific research, vital for improving human life, is complex, slow, and needs specialized expertise. Meanwhile, novel, impactful research often stems from both a deep understanding of prior work, and a cross-pollination of ideas across domains and fields. To enhance the productivity of researchers, we propose ResearchAgent, which leverages the encyclopedic knowledge and linguistic reasoning capabilities of Large Language Models (LLMs) to assist them in their work. This system automatically defines novel problems, proposes methods and designs experiments, while iteratively refining them based on the feedback from collaborative LLM-powered reviewing agents. Specifically, starting with a core scientific paper, ResearchAgent is augmented not only with relevant publications by connecting information over an academic graph but also entities retrieved from a knowledge store derived from shared underlying concepts mined across numerous papers. Then, mimicking a scientific approach to improving ideas with peer discussions, we leverage multiple LLM-based ReviewingAgents that provide reviews and feedback via iterative revision processes. These reviewing agents are instantiated with human preference-aligned LLMs whose criteria for evaluation are elicited from actual human judgments via LLM prompting. We experimentally validate our ResearchAgent on scientific publications across multiple disciplines, showing its effectiveness in generating novel, clear, and valid ideas based on both human and model-based evaluation results. Our initial foray into AI-mediated scientific research has important implications for the development of future systems aimed at supporting researchers in their ideation and operationalization of novel work."2243 },2244 "arXiv:2404.13076": {2245 "paperId": "5c7f465d162aade4a4c0eefb02fd7aadeebdaf58",2246 "externalIds": {2247 "DBLP": "journals/corr/abs-2404-13076",2248 "ArXiv": "2404.13076",2249 "DOI": "10.48550/arXiv.2404.13076",2250 "CorpusId": 2692933112251 },2252 "publicationVenue": {2253 "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",2254 "name": "Neural Information Processing Systems",2255 "type": "conference",2256 "alternate_names": [2257 "Neural Inf Process Syst",2258 "NeurIPS",2259 "NIPS"2260 ],2261 "url": "http://neurips.cc/"2262 },2263 "title": "LLM Evaluators Recognize and Favor Their Own Generations",2264 "venue": "Neural Information Processing Systems",2265 "year": 2024,2266 "citationCount": 971,2267 "openAccessPdf": {2268 "url": "",2269 "status": null,2270 "license": null,2271 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2404.13076, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."2272 },2273 "publicationTypes": [2274 "JournalArticle"2275 ],2276 "journal": {2277 "name": "ArXiv",2278 "volume": "abs/2404.13076"2279 },2280 "authors": [2281 {2282 "authorId": "2279023095",2283 "name": "Arjun Panickssery"2284 },2285 {2286 "authorId": "2297768298",2287 "name": "Samuel R. Bowman"2288 },2289 {2290 "authorId": "2297816489",2291 "name": "Shi Feng"2292 }2293 ],2294 "abstract": "Self-evaluation using large language models (LLMs) has proven valuable not only in benchmarking but also methods like reward modeling, constitutional AI, and self-refinement. But new biases are introduced due to the same LLM acting as both the evaluator and the evaluatee. One such bias is self-preference, where an LLM evaluator scores its own outputs higher than others' while human annotators consider them of equal quality. But do LLMs actually recognize their own outputs when they give those texts higher scores, or is it just a coincidence? In this paper, we investigate if self-recognition capability contributes to self-preference. We discover that, out of the box, LLMs such as GPT-4 and Llama 2 have non-trivial accuracy at distinguishing themselves from other LLMs and humans. By fine-tuning LLMs, we discover a linear correlation between self-recognition capability and the strength of self-preference bias; using controlled experiments, we show that the causal explanation resists straightforward confounders. We discuss how self-recognition can interfere with unbiased evaluations and AI safety more generally."2295 },2296 "arXiv:2404.17605": {2297 "paperId": "e4eb81ad222ba047770d5a90bdd7406c138c6126",2298 "externalIds": {2299 "DBLP": "journals/corr/abs-2404-17605",2300 "ArXiv": "2404.17605",2301 "DOI": "10.48550/arXiv.2404.17605",2302 "CorpusId": 269448624,2303 "PubMed": "42932458"2304 },2305 "publicationVenue": null,2306 "title": "Autonomous LLM-driven research from data to human-verifiable research papers",2307 "venue": "NEJM AI",2308 "year": 2024,2309 "citationCount": 101,2310 "openAccessPdf": {2311 "url": "",2312 "status": null,2313 "license": null,2314 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2404.17605, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."2315 },2316 "publicationTypes": [2317 "JournalArticle",2318 "Review"2319 ],2320 "journal": {2321 "name": "NEJM AI",2322 "pages": "\n AIoa2400555\n ",2323 "volume": "2 1"2324 },2325 "authors": [2326 {2327 "authorId": "2298900687",2328 "name": "Tal Ifargan"2329 },2330 {2331 "authorId": "46959587",2332 "name": "L. Hafner"2333 },2334 {2335 "authorId": "49079077",2336 "name": "M. Kern"2337 },2338 {2339 "authorId": "2298896286",2340 "name": "Ori Alcalay"2341 },2342 {2343 "authorId": "2298897594",2344 "name": "Roy Kishony"2345 }2346 ],2347 "abstract": "BACKGROUND\nArtificial intelligence (AI) promises to accelerate scientific discovery, but it remains unclear whether AI systems can perform fully autonomous research, and whether they can do so while adhering to key scientific values, such as transparency, traceability, and verifiability. The aim of this study was to develop and evaluate an AI-automation platform that performs transparent, traceable, and human-verifiable scientific research.\n\n\nMETHODS\nTo mimic human scientific practices, we built \"data-to-paper,\" an automation platform that guides interacting large language model (LLM) agents through a complete stepwise research process that starts with annotated data and results in comprehensive research papers, while programmatically backtracing information flow and allowing human oversight and interactions. The platform can run fully autonomously (in autopilot mode) or with human intervention (in copilot mode).\n\n\nRESULTS\nIn autopilot mode, provided only with annotated data, data-to-paper raised hypotheses; designed research plans; wrote and debugged analysis codes; generated and interpreted results; and created complete, information-traceable research papers. Even though the research novelty of manuscripts created by data-to-paper was relatively limited, the process demonstrated the autonomous generation of de novo quantitative insights from data, such as unraveling associations between health indicators and clinical outcomes. For simple research goals and datasets, a fully autonomous cycle can create manuscripts that independently recapitulate the findings of peer-reviewed biomedical publications without major errors in about 80 to 90% of cases. Yet, as goal or data complexity increases, human copiloting becomes critical for ensuring accuracy and overall quality. By tracking information flow through the steps, the platform creates \"data-chained\" manuscripts, in which downstream results are programmatically linked to upstream code and data, thus setting a new standard for the verifiability of scientific outputs.\n\n\nCONCLUSIONS\nOur work demonstrates the potential for AI-driven acceleration of scientific discovery in data-driven biomedical research and beyond, while enhancing, rather than jeopardizing, traceability, transparency, and verifiability."2348 },2349 "arXiv:2405.15793": {2350 "paperId": "1c3c531fc0fbe79f97f367ed3648de8467caeeaa",2351 "externalIds": {2352 "DBLP": "journals/corr/abs-2405-15793",2353 "ArXiv": "2405.15793",2354 "DOI": "10.48550/arXiv.2405.15793",2355 "CorpusId": 2700636852356 },2357 "publicationVenue": {2358 "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",2359 "name": "Neural Information Processing Systems",2360 "type": "conference",2361 "alternate_names": [2362 "Neural Inf Process Syst",2363 "NeurIPS",2364 "NIPS"2365 ],2366 "url": "http://neurips.cc/"2367 },2368 "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",2369 "venue": "Neural Information Processing Systems",2370 "year": 2024,2371 "citationCount": 2323,2372 "openAccessPdf": {2373 "url": "",2374 "status": null,2375 "license": null,2376 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2405.15793, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."2377 },2378 "publicationTypes": [2379 "JournalArticle"2380 ],2381 "journal": {2382 "name": "ArXiv",2383 "volume": "abs/2405.15793"2384 },2385 "authors": [2386 {2387 "authorId": "2109727379",2388 "name": "John Yang"2389 },2390 {2391 "authorId": "2155269683",2392 "name": "Carlos E. Jimenez"2393 },2394 {2395 "authorId": "2127066887",2396 "name": "Alexander Wettig"2397 },2398 {2399 "authorId": "71565033",2400 "name": "Kilian Adriano Lieret"2401 },2402 {2403 "authorId": "2256984659",2404 "name": "Shun-Yu Yao"2405 },2406 {2407 "authorId": "2135381714",2408 "name": "Karthik R. Narasimhan"2409 },2410 {2411 "authorId": "40170001",2412 "name": "Ofir Press"2413 }2414 ],2415 "abstract": "Language model (LM) agents are increasingly being used to automate complicated tasks in digital environments. Just as humans benefit from powerful software applications, such as integrated development environments, for complex tasks like software engineering, we posit that LM agents represent a new category of end users with their own needs and abilities, and would benefit from specially-built interfaces to the software they use. We investigate how interface design affects the performance of language model agents. As a result of this exploration, we introduce SWE-agent: a system that facilitates LM agents to autonomously use computers to solve software engineering tasks. SWE-agent's custom agent-computer interface (ACI) significantly enhances an agent's ability to create and edit code files, navigate entire repositories, and execute tests and other programs. We evaluate SWE-agent on SWE-bench and HumanEvalFix, achieving state-of-the-art performance on both with a pass@1 rate of 12.5% and 87.7%, respectively, far exceeding the previous state-of-the-art achieved with non-interactive LMs. Finally, we provide insight on how the design of the ACI can impact agents' behavior and performance."2416 },2417 "arXiv:2406.04244": {2418 "paperId": "0fad9dd4f0ea41732594f90209907bfad1ba506e",2419 "externalIds": {2420 "DBLP": "journals/corr/abs-2406-04244",2421 "ArXiv": "2406.04244",2422 "DOI": "10.48550/arXiv.2406.04244",2423 "CorpusId": 2702857082424 },2425 "publicationVenue": {2426 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",2427 "name": "arXiv.org",2428 "alternate_names": [2429 "ArXiv"2430 ],2431 "issn": "2331-8422",2432 "url": "https://arxiv.org"2433 },2434 "title": "Benchmark Data Contamination of Large Language Models: A Survey",2435 "venue": "arXiv.org",2436 "year": 2024,2437 "citationCount": 169,2438 "openAccessPdf": {2439 "url": "",2440 "status": null,2441 "license": null,2442 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2406.04244, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."2443 },2444 "publicationTypes": [2445 "JournalArticle",2446 "Review"2447 ],2448 "journal": {2449 "name": "ArXiv",2450 "volume": "abs/2406.04244"2451 },2452 "authors": [2453 {2454 "authorId": "2153079650",2455 "name": "Cheng Xu"2456 },2457 {2458 "authorId": "2304954016",2459 "name": "Shu-Hao Guan"2460 },2461 {2462 "authorId": "2304952028",2463 "name": "Derek Greene"2464 },2465 {2466 "authorId": "2266906186",2467 "name": "Mohand-Tahar Kechadi"2468 }2469 ],2470 "abstract": "The rapid development of Large Language Models (LLMs) like GPT-4, Claude-3, and Gemini has transformed the field of natural language processing. However, it has also resulted in a significant issue known as Benchmark Data Contamination (BDC). This occurs when language models inadvertently incorporate evaluation benchmark information from their training data, leading to inaccurate or unreliable performance during the evaluation phase of the process. This paper reviews the complex challenge of BDC in LLM evaluation and explores alternative assessment methods to mitigate the risks associated with traditional benchmarks. The paper also examines challenges and future directions in mitigating BDC risks, highlighting the complexity of the issue and the need for innovative solutions to ensure the reliability of LLM evaluation in real-world applications."2471 },2472 "arXiv:2406.12045": {2473 "paperId": "70aa016c1f68fd5c0261f26ad20017b8307650af",2474 "externalIds": {2475 "DBLP": "journals/corr/abs-2406-12045",2476 "ArXiv": "2406.12045",2477 "DOI": "10.48550/arXiv.2406.12045",2478 "CorpusId": 2705625782479 },2480 "publicationVenue": {2481 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",2482 "name": "arXiv.org",2483 "alternate_names": [2484 "ArXiv"2485 ],2486 "issn": "2331-8422",2487 "url": "https://arxiv.org"2488 },2489 "title": "\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",2490 "venue": "arXiv.org",2491 "year": 2024,2492 "citationCount": 1635,2493 "openAccessPdf": {2494 "url": "",2495 "status": null,2496 "license": null,2497 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2406.12045, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."2498 },2499 "publicationTypes": [2500 "JournalArticle"2501 ],2502 "journal": {2503 "name": "ArXiv",2504 "volume": "abs/2406.12045"2505 },2506 "authors": [2507 {2508 "authorId": "2256984659",2509 "name": "Shun-Yu Yao"2510 },2511 {2512 "authorId": "2212367248",2513 "name": "Noah Shinn"2514 },2515 {2516 "authorId": "2395098792",2517 "name": "Pedram Razavi"2518 },2519 {2520 "authorId": "2135381714",2521 "name": "Karthik R. Narasimhan"2522 }2523 ],2524 "abstract": "Existing benchmarks do not test language agents on their interaction with human users or ability to follow domain-specific rules, both of which are vital for deploying them in real world applications. We propose $\\tau$-bench, a benchmark emulating dynamic conversations between a user (simulated by language models) and a language agent provided with domain-specific API tools and policy guidelines. We employ an efficient and faithful evaluation process that compares the database state at the end of a conversation with the annotated goal state. We also propose a new metric (pass^k) to evaluate the reliability of agent behavior over multiple trials. Our experiments show that even state-of-the-art function calling agents (like gpt-4o) succeed on<50% of the tasks, and are quite inconsistent (pass^8<25% in retail). Our findings point to the need for methods that can improve the ability of agents to act consistently and follow rules reliably."2525 },2526 "arXiv:2406.12624": {2527 "paperId": "7d784c87c5e46fedbe62006d7340baa7c7bec123",2528 "externalIds": {2529 "DBLP": "journals/corr/abs-2406-12624",2530 "ArXiv": "2406.12624",2531 "DOI": "10.48550/arXiv.2406.12624",2532 "CorpusId": 2705624252533 },2534 "publicationVenue": {2535 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",2536 "name": "arXiv.org",2537 "alternate_names": [2538 "ArXiv"2539 ],2540 "issn": "2331-8422",2541 "url": "https://arxiv.org"2542 },2543 "title": "Judging the Judges: Evaluating Alignment and Vulnerabilities in LLMs-as-Judges",2544 "venue": "arXiv.org",2545 "year": 2024,2546 "citationCount": 298,2547 "openAccessPdf": {2548 "url": "",2549 "status": null,2550 "license": null,2551 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2406.12624, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."2552 },2553 "publicationTypes": [2554 "JournalArticle"2555 ],2556 "journal": {2557 "name": "ArXiv",2558 "volume": "abs/2406.12624"2559 },2560 "authors": [2561 {2562 "authorId": "2307078343",2563 "name": "Aman Singh Thakur"2564 },2565 {2566 "authorId": "2310551241",2567 "name": "Kartik Choudhary"2568 },2569 {2570 "authorId": "2307074192",2571 "name": "Venkat Srinik Ramayapally"2572 },2573 {2574 "authorId": "65731722",2575 "name": "Sankaran Vaidyanathan"2576 },2577 {2578 "authorId": "3449411",2579 "name": "Dieuwke Hupkes"2580 }2581 ],2582 "abstract": "Offering a promising solution to the scalability challenges associated with human evaluation, the LLM-as-a-judge paradigm is rapidly gaining traction as an approach to evaluating large language models (LLMs). However, there are still many open questions about the strengths and weaknesses of this paradigm, and what potential biases it may hold. In this paper, we present a comprehensive study of the performance of various LLMs acting as judges, focusing on a clean scenario in which inter-human agreement is high. Investigating thirteen judge models of different model sizes and families, judging answers of nine different'examtaker models'- both base and instruction-tuned - we find that only the best (and largest) models achieve reasonable alignment with humans. However, they are still quite far behind inter-human agreement and their assigned scores may still differ with up to 5 points from human-assigned scores. In terms of their ranking of the nine exam-taker models, instead, also smaller models and even the lexical metric contains may provide a reasonable signal. Through error analysis and other studies, we identify vulnerabilities in judge models, such as their sensitivity to prompt complexity and length, and a tendency toward leniency. The fact that even the best judges differ from humans in this comparatively simple setup suggest that caution may be wise when using judges in more complex setups. Lastly, our research rediscovers the importance of using alignment metrics beyond simple percent alignment, showing that judges with high percent agreement can still assign vastly different scores."2583 },2584 "arXiv:2406.18403": {2585 "paperId": "b1e681dda709217a67316d2394bc992a26008ef7",2586 "externalIds": {2587 "DBLP": "conf/acl/BavarescoBBEFGG25",2588 "ArXiv": "2406.18403",2589 "DOI": "10.48550/arXiv.2406.18403",2590 "CorpusId": 2707380742591 },2592 "publicationVenue": {2593 "id": "1e33b3be-b2ab-46e9-96e8-d4eb4bad6e44",2594 "name": "Annual Meeting of the Association for Computational Linguistics",2595 "type": "conference",2596 "alternate_names": [2597 "Annu Meet Assoc Comput Linguistics",2598 "Meeting of the Association for Computational Linguistics",2599 "ACL",2600 "Meet Assoc Comput Linguistics"2601 ],2602 "url": "https://www.aclweb.org/anthology/venues/acl/"2603 },2604 "title": "LLMs instead of Human Judges? A Large Scale Empirical Study across 20 NLP Evaluation Tasks",2605 "venue": "Annual Meeting of the Association for Computational Linguistics",2606 "year": 2024,2607 "citationCount": 384,2608 "openAccessPdf": {2609 "url": "",2610 "status": null,2611 "license": null,2612 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2406.18403, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."2613 },2614 "publicationTypes": [2615 "JournalArticle",2616 "Conference"2617 ],2618 "journal": {2619 "pages": "238-255"2620 },2621 "authors": [2622 {2623 "authorId": "2308275559",2624 "name": "Anna Bavaresco"2625 },2626 {2627 "authorId": "2276333243",2628 "name": "Raffaella Bernardi"2629 },2630 {2631 "authorId": "2243233537",2632 "name": "Leonardo Bertolazzi"2633 },2634 {2635 "authorId": "2297018130",2636 "name": "Desmond Elliott"2637 },2638 {2639 "authorId": "2147411696",2640 "name": "Raquel Fern\u00e1ndez"2641 },2642 {2643 "authorId": "2307004114",2644 "name": "Albert Gatt"2645 },2646 {2647 "authorId": "2398366",2648 "name": "E. Ghaleb"2649 },2650 {2651 "authorId": "24068173",2652 "name": "Mario Giulianelli"2653 },2654 {2655 "authorId": "2140766524",2656 "name": "Michael Hanna"2657 },2658 {2659 "authorId": "2305819190",2660 "name": "Alexander Koller"2661 },2662 {2663 "authorId": "2348291963",2664 "name": "Andr\u00e9 F. T. Martins"2665 },2666 {2667 "authorId": "2286310256",2668 "name": "Philipp Mondorf"2669 },2670 {2671 "authorId": "2305680161",2672 "name": "Vera Neplenbroek"2673 },2674 {2675 "authorId": "2261362830",2676 "name": "S. Pezzelle"2677 },2678 {2679 "authorId": "2410144188",2680 "name": "Barbara Plank"2681 },2682 {2683 "authorId": "2257274000",2684 "name": "D. Schlangen"2685 },2686 {2687 "authorId": "3444866",2688 "name": "Alessandro Suglia"2689 },2690 {2691 "authorId": "1387995349",2692 "name": "Aditya K Surikuchi"2693 },2694 {2695 "authorId": "41071249",2696 "name": "Ece Takmaz"2697 },2698 {2699 "authorId": "50829868",2700 "name": "Alberto Testoni"2701 }2702 ],2703 "abstract": "There is an increasing trend towards evaluating NLP models with LLMs instead of human judgments, raising questions about the validity of these evaluations, as well as their reproducibility in the case of proprietary models. We provide JUDGE-BENCH, an extensible collection of 20 NLP datasets with human annotations covering a broad range of evaluated properties and types of data, and comprehensively evaluate 11 current LLMs, covering both open-weight and proprietary models, for their ability to replicate the annotations. Our evaluations show substantial variance across models and datasets. Models are reliable evaluators on some tasks, but overall display substantial variability depending on the property being evaluated, the expertise level of the human judges, and whether the language is human or model-generated. We conclude that LLMs should be carefully validated against human judgments before being used as evaluators."2704 },2705 "arXiv:2407.01502": {2706 "paperId": "edae954314571eb2913209a7e9825cdc14fd4c58",2707 "externalIds": {2708 "DBLP": "journals/corr/abs-2407-01502",2709 "ArXiv": "2407.01502",2710 "DOI": "10.48550/arXiv.2407.01502",2711 "CorpusId": 2708703602712 },2713 "publicationVenue": null,2714 "title": "AI Agents That Matter",2715 "venue": "Trans. Mach. Learn. Res.",2716 "year": 2024,2717 "citationCount": 287,2718 "openAccessPdf": {2719 "url": "",2720 "status": null,2721 "license": null,2722 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2407.01502, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."2723 },2724 "publicationTypes": [2725 "JournalArticle"2726 ],2727 "journal": {2728 "name": "Trans. Mach. Learn. Res.",2729 "volume": "2025"2730 },2731 "authors": [2732 {2733 "authorId": "39893263",2734 "name": "Sayash Kapoor"2735 },2736 {2737 "authorId": "2309175570",2738 "name": "Benedikt Stroebl"2739 },2740 {2741 "authorId": "2274102473",2742 "name": "Zachary S. Siegel"2743 },2744 {2745 "authorId": "2290073765",2746 "name": "Nitya Nadgir"2747 },2748 {2749 "authorId": "2064013115",2750 "name": "Arvind Narayanan"2751 }2752 ],2753 "abstract": "AI agents are an exciting new research direction, and agent development is driven by benchmarks. Our analysis of current agent benchmarks and evaluation practices reveals several shortcomings that hinder their usefulness in real-world applications. First, there is a narrow focus on accuracy without attention to other metrics. As a result, SOTA agents are needlessly complex and costly, and the community has reached mistaken conclusions about the sources of accuracy gains. Our focus on cost in addition to accuracy motivates the new goal of jointly optimizing the two metrics. We design and implement one such optimization, showing its potential to greatly reduce cost while maintaining accuracy. Second, the benchmarking needs of model and downstream developers have been conflated, making it hard to identify which agent would be best suited for a particular application. Third, many agent benchmarks have inadequate holdout sets, and sometimes none at all. This has led to agents that are fragile because they take shortcuts and overfit to the benchmark in various ways. We prescribe a principled framework for avoiding overfitting. Finally, there is a lack of standardization in evaluation practices, leading to a pervasive lack of reproducibility. We hope that the steps we introduce for addressing these shortcomings will spur the development of agents that are useful in the real world and not just accurate on benchmarks."2754 },2755 "arXiv:2407.01725": {2756 "paperId": "48c83799530dc523ee01e6c1c40ad577d5c10a16",2757 "externalIds": {2758 "DBLP": "conf/iclr/MajumderS0MMPVK25",2759 "ArXiv": "2407.01725",2760 "DOI": "10.48550/arXiv.2407.01725",2761 "CorpusId": 2708782322762 },2763 "publicationVenue": {2764 "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",2765 "name": "International Conference on Learning Representations",2766 "type": "conference",2767 "alternate_names": [2768 "Int Conf Learn Represent",2769 "ICLR"2770 ],2771 "url": "https://iclr.cc/"2772 },2773 "title": "DiscoveryBench: Towards Data-Driven Discovery with Large Language Models",2774 "venue": "International Conference on Learning Representations",2775 "year": 2024,2776 "citationCount": 118,2777 "openAccessPdf": {2778 "url": "",2779 "status": null,2780 "license": null,2781 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2407.01725, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."2782 },2783 "publicationTypes": [2784 "JournalArticle"2785 ],2786 "journal": {2787 "name": "ArXiv",2788 "volume": "abs/2407.01725"2789 },2790 "authors": [2791 {2792 "authorId": "3165738",2793 "name": "Bodhisattwa Prasad Majumder"2794 },2795 {2796 "authorId": "2762792",2797 "name": "Harshit Surana"2798 },2799 {2800 "authorId": "2057234493",2801 "name": "Dhruv Agarwal"2802 },2803 {2804 "authorId": "40135250",2805 "name": "Bhavana Dalvi"2806 },2807 {2808 "authorId": "2362917398",2809 "name": "A. Meena"2810 },2811 {2812 "authorId": "2309247064",2813 "name": "Aryan Prakhar"2814 },2815 {2816 "authorId": "2309246364",2817 "name": "Tirth Vora"2818 },2819 {2820 "authorId": "2236429",2821 "name": "Tushar Khot"2822 },2823 {2824 "authorId": "48229640",2825 "name": "Ashish Sabharwal"2826 },2827 {2828 "authorId": "2265493391",2829 "name": "Peter Clark"2830 }2831 ],2832 "abstract": "Can the rapid advances in code generation, function calling, and data analysis using large language models (LLMs) help automate the search and verification of hypotheses purely from a set of provided datasets? To evaluate this question, we present DiscoveryBench, the first comprehensive benchmark that formalizes the multi-step process of data-driven discovery. The benchmark is designed to systematically assess current model capabilities in discovery tasks and provide a useful resource for improving them. Our benchmark contains 264 tasks collected across 6 diverse domains, such as sociology and engineering, by manually deriving discovery workflows from published papers to approximate the real-world challenges faced by researchers, where each task is defined by a dataset, its metadata, and a discovery goal in natural language. We additionally provide 903 synthetic tasks to conduct controlled evaluations across task complexity. Furthermore, our structured formalism of data-driven discovery enables a facet-based evaluation that provides useful insights into different failure modes. We evaluate several popular LLM-based reasoning frameworks using both open and closed LLMs as baselines on DiscoveryBench and find that even the best system scores only 25%. Our benchmark, thus, illustrates the challenges in autonomous data-driven discovery and serves as a valuable resource for the community to make progress."2833 },2834 "arXiv:2407.13168": {2835 "paperId": "5bf3ea7b0825424c3c01e48e8fcd8215c950677e",2836 "externalIds": {2837 "ArXiv": "2407.13168",2838 "DBLP": "conf/nips/TianGZCFGHJKLLL24",2839 "DOI": "10.48550/arXiv.2407.13168",2840 "CorpusId": 2712700482841 },2842 "publicationVenue": {2843 "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",2844 "name": "Neural Information Processing Systems",2845 "type": "conference",2846 "alternate_names": [2847 "Neural Inf Process Syst",2848 "NeurIPS",2849 "NIPS"2850 ],2851 "url": "http://neurips.cc/"2852 },2853 "title": "SciCode: A Research Coding Benchmark Curated by Scientists",2854 "venue": "Neural Information Processing Systems",2855 "year": 2024,2856 "citationCount": 203,2857 "openAccessPdf": {2858 "url": "",2859 "status": null,2860 "license": null,2861 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2407.13168, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."2862 },2863 "publicationTypes": [2864 "JournalArticle"2865 ],2866 "journal": {2867 "name": "ArXiv",2868 "volume": "abs/2407.13168"2869 },2870 "authors": [2871 {2872 "authorId": "2311882756",2873 "name": "Min-Yang Tian"2874 },2875 {2876 "authorId": "49715441",2877 "name": "Luyu Gao"2878 },2879 {2880 "authorId": "2145401799",2881 "name": "S. Zhang"2882 },2883 {2884 "authorId": "2311990357",2885 "name": "Xinan Chen"2886 },2887 {2888 "authorId": "2931077",2889 "name": "Cun-Wei Fan"2890 },2891 {2892 "authorId": "2311997990",2893 "name": "Xuefei Guo"2894 },2895 {2896 "authorId": "2281073123",2897 "name": "R. Haas"2898 },2899 {2900 "authorId": "2311880186",2901 "name": "Pan Ji"2902 },2903 {2904 "authorId": "35701505",2905 "name": "Kittithat Krongchon"2906 },2907 {2908 "authorId": "2374330170",2909 "name": "Yao-Hui Li"2910 },2911 {2912 "authorId": "2152940692",2913 "name": "Shengyan Liu"2914 },2915 {2916 "authorId": "2311885961",2917 "name": "Di Luo"2918 },2919 {2920 "authorId": "2325196615",2921 "name": "Yu-Tao Ma"2922 },2923 {2924 "authorId": "2182144332",2925 "name": "Hao Tong"2926 },2927 {2928 "authorId": "2405014449",2929 "name": "Kha Trinh"2930 },2931 {2932 "authorId": "2312693814",2933 "name": "Chenyu Tian"2934 },2935 {2936 "authorId": "2311998534",2937 "name": "Zihan Wang"2938 },2939 {2940 "authorId": "2312000055",2941 "name": "Bohao Wu"2942 },2943 {2944 "authorId": "2088591255",2945 "name": "Yanyu Xiong"2946 },2947 {2948 "authorId": "2383429721",2949 "name": "Sheng Yin"2950 },2951 {2952 "authorId": "2268183167",2953 "name": "Min Zhu"2954 },2955 {2956 "authorId": "71565033",2957 "name": "Kilian Adriano Lieret"2958 },2959 {2960 "authorId": "7774667",2961 "name": "Yanxin Lu"2962 },2963 {2964 "authorId": "2217570953",2965 "name": "Genglin Liu"2966 },2967 {2968 "authorId": "2312034033",2969 "name": "Yu-Feng Du"2970 },2971 {2972 "authorId": "2242959702",2973 "name": "Tian-Hua Tao"2974 },2975 {2976 "authorId": "40170001",2977 "name": "Ofir Press"2978 },2979 {2980 "authorId": "2310820252",2981 "name": "James P. Callan"2982 },2983 {2984 "authorId": "2249761654",2985 "name": "Eliu A. Huerta"2986 },2987 {2988 "authorId": "2254026935",2989 "name": "Hao Peng"2990 }2991 ],2992 "abstract": "Since language models (LMs) now outperform average humans on many challenging tasks, it has become increasingly difficult to develop challenging, high-quality, and realistic evaluations. We address this issue by examining LMs' capabilities to generate code for solving real scientific research problems. Incorporating input from scientists and AI researchers in 16 diverse natural science sub-fields, including mathematics, physics, chemistry, biology, and materials science, we created a scientist-curated coding benchmark, SciCode. The problems in SciCode naturally factorize into multiple subproblems, each involving knowledge recall, reasoning, and code synthesis. In total, SciCode contains 338 subproblems decomposed from 80 challenging main problems. It offers optional descriptions specifying useful scientific background information and scientist-annotated gold-standard solutions and test cases for evaluation. Claude3.5-Sonnet, the best-performing model among those tested, can solve only 4.6% of the problems in the most realistic setting. We believe that SciCode demonstrates both contemporary LMs' progress towards becoming helpful scientific assistants and sheds light on the development and evaluation of scientific AI in the future."2993 },2994 "arXiv:2407.16741": {2995 "paperId": "1d07e5b6f978cf69c0186f3d5f434fa92d471e46",2996 "externalIds": {2997 "ArXiv": "2407.16741",2998 "DBLP": "conf/iclr/0001LSXTZPSLSTL25",2999 "CorpusId": 2714047733000 },3001 "publicationVenue": {3002 "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",3003 "name": "International Conference on Learning Representations",3004 "type": "conference",3005 "alternate_names": [3006 "Int Conf Learn Represent",3007 "ICLR"3008 ],3009 "url": "https://iclr.cc/"3010 },3011 "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",3012 "venue": "International Conference on Learning Representations",3013 "year": 2024,3014 "citationCount": 1291,3015 "openAccessPdf": {3016 "url": "",3017 "status": null,3018 "license": null,3019 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2407.16741, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."3020 },3021 "publicationTypes": [3022 "JournalArticle"3023 ],3024 "journal": null,3025 "authors": [3026 {3027 "authorId": "2144803999",3028 "name": "Xingyao Wang"3029 },3030 {3031 "authorId": "2313084397",3032 "name": "Bo-Xuan Li"3033 },3034 {3035 "authorId": "41077032",3036 "name": "Yufan Song"3037 },3038 {3039 "authorId": "40027632",3040 "name": "Frank F. Xu"3041 },3042 {3043 "authorId": "47274259",3044 "name": "Xiangru Tang"3045 },3046 {3047 "authorId": "1722264111",3048 "name": "Ming-Chen Zhuge"3049 },3050 {3051 "authorId": "2152946868",3052 "name": "Jiayi Pan"3053 },3054 {3055 "authorId": "2290031618",3056 "name": "Yue-Qi Song"3057 },3058 {3059 "authorId": "2132475886",3060 "name": "Bo-Wen Li"3061 },3062 {3063 "authorId": "2312910329",3064 "name": "Jaskirat Singh"3065 },3066 {3067 "authorId": "2313308109",3068 "name": "Hoang H. Tran"3069 },3070 {3071 "authorId": "2313768177",3072 "name": "Fuqiang Li"3073 },3074 {3075 "authorId": "2313325218",3076 "name": "Ren Ma"3077 },3078 {3079 "authorId": "2312719297",3080 "name": "Mingzhang Zheng"3081 },3082 {3083 "authorId": "2226120351",3084 "name": "Bill Qian"3085 },3086 {3087 "authorId": "2308941736",3088 "name": "Yanjun Shao"3089 },3090 {3091 "authorId": "2037383772",3092 "name": "Niklas Muennighoff"3093 },3094 {3095 "authorId": "48378494",3096 "name": "Yi-Zhe Zhang"3097 },3098 {3099 "authorId": "2321578848",3100 "name": "Bin-Yuan Hui"3101 },3102 {3103 "authorId": "2312866505",3104 "name": "Junyang Lin"3105 },3106 {3107 "authorId": "2312759262",3108 "name": "Robert Brennan"3109 },3110 {3111 "authorId": "2254026935",3112 "name": "Hao Peng"3113 },3114 {3115 "authorId": "2312388202",3116 "name": "H. Ji"3117 },3118 {3119 "authorId": "2285194103",3120 "name": "Graham Neubig"3121 }3122 ],3123 "abstract": "Software is one of the most powerful tools that we humans have at our disposal; it allows a skilled programmer to interact with the world in complex and profound ways. At the same time, thanks to improvements in large language models (LLMs), there has also been a rapid development in AI agents that interact with and affect change in their surrounding environments. In this paper, we introduce OpenHands (f.k.a. OpenDevin), a platform for the development of powerful and flexible AI agents that interact with the world in similar ways to those of a human developer: by writing code, interacting with a command line, and browsing the web. We describe how the platform allows for the implementation of new agents, safe interaction with sandboxed environments for code execution, coordination between multiple agents, and incorporation of evaluation benchmarks. Based on our currently incorporated benchmarks, we perform an evaluation of agents over 15 challenging tasks, including software engineering (e.g., SWE-BENCH) and web browsing (e.g., WEBARENA), among others. Released under the permissive MIT license, OpenHands is a community project spanning academia and industry with more than 2.1K contributions from over 188 contributors."3124 },3125 "arXiv:2408.06292": {3126 "paperId": "33161a5a9b5dcb635b5a97475e6a6209a69ada7d",3127 "externalIds": {3128 "DBLP": "journals/corr/abs-2408-06292",3129 "ArXiv": "2408.06292",3130 "CorpusId": 2718548873131 },3132 "publicationVenue": {3133 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",3134 "name": "arXiv.org",3135 "alternate_names": [3136 "ArXiv"3137 ],3138 "issn": "2331-8422",3139 "url": "https://arxiv.org"3140 },3141 "title": "The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery",3142 "venue": "arXiv.org",3143 "year": 2024,3144 "citationCount": 1342,3145 "openAccessPdf": {3146 "url": "",3147 "status": null,3148 "license": null,3149 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2408.06292, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."3150 },3151 "publicationTypes": [3152 "JournalArticle",3153 "Review"3154 ],3155 "journal": {3156 "name": "ArXiv",3157 "volume": "abs/2408.06292"3158 },3159 "authors": [3160 {3161 "authorId": "4420863",3162 "name": "Chris Lu"3163 },3164 {3165 "authorId": "2110752472",3166 "name": "Cong Lu"3167 },3168 {3169 "authorId": "2266838849",3170 "name": "R. Lange"3171 },3172 {3173 "authorId": "2275156099",3174 "name": "J. Foerster"3175 },3176 {3177 "authorId": "2303255568",3178 "name": "Jeff Clune"3179 },3180 {3181 "authorId": "2315921540",3182 "name": "David Ha"3183 }3184 ],3185 "abstract": "One of the grand challenges of artificial general intelligence is developing agents capable of conducting scientific research and discovering new knowledge. While frontier models have already been used as aides to human scientists, e.g. for brainstorming ideas, writing code, or prediction tasks, they still conduct only a small part of the scientific process. This paper presents the first comprehensive framework for fully automatic scientific discovery, enabling frontier large language models to perform research independently and communicate their findings. We introduce The AI Scientist, which generates novel research ideas, writes code, executes experiments, visualizes results, describes its findings by writing a full scientific paper, and then runs a simulated review process for evaluation. In principle, this process can be repeated to iteratively develop ideas in an open-ended fashion, acting like the human scientific community. We demonstrate its versatility by applying it to three distinct subfields of machine learning: diffusion modeling, transformer-based language modeling, and learning dynamics. Each idea is implemented and developed into a full paper at a cost of less than $15 per paper. To evaluate the generated papers, we design and validate an automated reviewer, which we show achieves near-human performance in evaluating paper scores. The AI Scientist can produce papers that exceed the acceptance threshold at a top machine learning conference as judged by our automated reviewer. This approach signifies the beginning of a new era in scientific discovery in machine learning: bringing the transformative benefits of AI agents to the entire research process of AI itself, and taking us closer to a world where endless affordable creativity and innovation can be unleashed on the world's most challenging problems. Our code is open-sourced at https://github.com/SakanaAI/AI-Scientist"3186 },3187 "arXiv:2408.09667": {3188 "paperId": "3cabe88c2ff476d35cd179525c7dfe474b4b75bb",3189 "externalIds": {3190 "DBLP": "journals/corr/abs-2408-09667",3191 "ArXiv": "2408.09667",3192 "DOI": "10.48550/arXiv.2408.09667",3193 "CorpusId": 2719034863194 },3195 "publicationVenue": {3196 "id": "41bf9ed3-85b3-4c90-b015-150e31690253",3197 "name": "Conference on Empirical Methods in Natural Language Processing",3198 "type": "conference",3199 "alternate_names": [3200 "Empir Method Nat Lang Process",3201 "Empirical Methods in Natural Language Processing",3202 "Conf Empir Method Nat Lang Process",3203 "EMNLP"3204 ],3205 "url": "https://www.aclweb.org/portal/emnlp"3206 },3207 "title": "BLADE: Benchmarking Language Model Agents for Data-Driven Science",3208 "venue": "Conference on Empirical Methods in Natural Language Processing",3209 "year": 2024,3210 "citationCount": 96,3211 "openAccessPdf": {3212 "url": "",3213 "status": null,3214 "license": null,3215 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2408.09667, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."3216 },3217 "publicationTypes": [3218 "JournalArticle",3219 "Conference"3220 ],3221 "journal": {3222 "name": "ArXiv",3223 "volume": "abs/2408.09667"3224 },3225 "authors": [3226 {3227 "authorId": "6639253",3228 "name": "Ken Gu"3229 },3230 {3231 "authorId": "2242899679",3232 "name": "Ruo-Xi Shang"3233 },3234 {3235 "authorId": "2316411779",3236 "name": "Ruien Jiang"3237 },3238 {3239 "authorId": "2316431927",3240 "name": "Keying Kuang"3241 },3242 {3243 "authorId": "2357993081",3244 "name": "Richard Lin"3245 },3246 {3247 "authorId": "2316432426",3248 "name": "Donghe Lyu"3249 },3250 {3251 "authorId": "2340586557",3252 "name": "Yue Mao"3253 },3254 {3255 "authorId": "2457265599",3256 "name": "Youbang Pan"3257 },3258 {3259 "authorId": "2365051240",3260 "name": "Teng Wu"3261 },3262 {3263 "authorId": "3276708",3264 "name": "Jia-Qian Yu"3265 },3266 {3267 "authorId": "2239169380",3268 "name": "Yikun Zhang"3269 },3270 {3271 "authorId": "2159536044",3272 "name": "Tian-Mai M. Zhang"3273 },3274 {3275 "authorId": "2404006955",3276 "name": "Lanyi Zhu"3277 },3278 {3279 "authorId": "2297185552",3280 "name": "Mike A. Merrill"3281 },3282 {3283 "authorId": "2275206828",3284 "name": "J. Heer"3285 },3286 {3287 "authorId": "2316432491",3288 "name": "Tim Althoff"3289 }3290 ],3291 "abstract": "Data-driven scientific discovery requires the iterative integration of scientific domain knowledge, statistical expertise, and an understanding of data semantics to make nuanced analytical decisions, e.g., about which variables, transformations, and statistical models to consider. LM-based agents equipped with planning, memory, and code execution capabilities have the potential to support data-driven science. However, evaluating agents on such open-ended tasks is challenging due to multiple valid approaches, partially correct steps, and different ways to express the same decisions. To address these challenges, we present BLADE, a benchmark to automatically evaluate agents'multifaceted approaches to open-ended research questions. BLADE consists of 12 datasets and research questions drawn from existing scientific literature, with ground truth collected from independent analyses by expert data scientists and researchers. To automatically evaluate agent responses, we developed corresponding computational methods to match different representations of analyses to this ground truth. Though language models possess considerable world knowledge, our evaluation shows that they are often limited to basic analyses. However, agents capable of interacting with the underlying data demonstrate improved, but still non-optimal, diversity in their analytical decision making. Our work enables the evaluation of agents for data-driven science and provides researchers deeper insights into agents'analysis approaches."3292 },3293 "arXiv:2409.05556": {3294 "paperId": "0f6439b95aed189908c628f1103dbcef8541cc2e",3295 "externalIds": {3296 "DBLP": "journals/corr/abs-2409-05556",3297 "ArXiv": "2409.05556",3298 "DOI": "10.48550/arXiv.2409.05556",3299 "CorpusId": 2725246553300 },3301 "publicationVenue": {3302 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",3303 "name": "arXiv.org",3304 "alternate_names": [3305 "ArXiv"3306 ],3307 "issn": "2331-8422",3308 "url": "https://arxiv.org"3309 },3310 "title": "SciAgents: Automating scientific discovery through multi-agent intelligent graph reasoning",3311 "venue": "arXiv.org",3312 "year": 2024,3313 "citationCount": 104,3314 "openAccessPdf": {3315 "url": "",3316 "status": null,3317 "license": null,3318 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2409.05556, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."3319 },3320 "publicationTypes": [3321 "JournalArticle"3322 ],3323 "journal": {3324 "name": "ArXiv",3325 "volume": "abs/2409.05556"3326 },3327 "authors": [3328 {3329 "authorId": "2310665423",3330 "name": "Alireza Ghafarollahi"3331 },3332 {3333 "authorId": "2269910426",3334 "name": "Markus J. Buehler"3335 }3336 ],3337 "abstract": "A key challenge in artificial intelligence is the creation of systems capable of autonomously advancing scientific understanding by exploring novel domains, identifying complex patterns, and uncovering previously unseen connections in vast scientific data. In this work, we present SciAgents, an approach that leverages three core concepts: (1) the use of large-scale ontological knowledge graphs to organize and interconnect diverse scientific concepts, (2) a suite of large language models (LLMs) and data retrieval tools, and (3) multi-agent systems with in-situ learning capabilities. Applied to biologically inspired materials, SciAgents reveals hidden interdisciplinary relationships that were previously considered unrelated, achieving a scale, precision, and exploratory power that surpasses traditional human-driven research methods. The framework autonomously generates and refines research hypotheses, elucidating underlying mechanisms, design principles, and unexpected material properties. By integrating these capabilities in a modular fashion, the intelligent system yields material discoveries, critique and improve existing hypotheses, retrieve up-to-date data about existing research, and highlights their strengths and limitations. Our case studies demonstrate scalable capabilities to combine generative AI, ontological representations, and multi-agent modeling, harnessing a `swarm of intelligence' similar to biological systems. This provides new avenues for materials discovery and accelerates the development of advanced materials by unlocking Nature's design principles."3338 },3339 "arXiv:2409.07703": {3340 "paperId": "395c978221a21ee47c84a40a2ef11fb4d012fca1",3341 "externalIds": {3342 "ArXiv": "2409.07703",3343 "CorpusId": 2725997603344 },3345 "publicationVenue": null,3346 "title": "DSBench: How Far Are Data Science Agents from Becoming Data Science Experts?",3347 "venue": "",3348 "year": 2024,3349 "citationCount": 141,3350 "openAccessPdf": {3351 "url": "",3352 "status": null,3353 "license": null,3354 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2409.07703, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."3355 },3356 "publicationTypes": null,3357 "journal": null,3358 "authors": [3359 {3360 "authorId": "2257043219",3361 "name": "Liqiang Jing"3362 },3363 {3364 "authorId": "2320838217",3365 "name": "Zhehui Huang"3366 },3367 {3368 "authorId": "2250363276",3369 "name": "Xiao-Yang Wang"3370 },3371 {3372 "authorId": "2087264100",3373 "name": "Wenlin Yao"3374 },3375 {3376 "authorId": "2265843326",3377 "name": "Wen-Hao Yu"3378 },3379 {3380 "authorId": "2311458574",3381 "name": "Kai-Xin Ma"3382 },3383 {3384 "authorId": "2254831297",3385 "name": "Hong-Ming Zhang"3386 },3387 {3388 "authorId": "2265463653",3389 "name": "Xinya Du"3390 },3391 {3392 "authorId": "2261392681",3393 "name": "Dong Yu"3394 }3395 ],3396 "abstract": "Large Language Models (LLMs) and Large Vision-Language Models (LVLMs) have demonstrated impressive language/vision reasoning abilities, igniting the recent trend of building agents for targeted applications such as shopping assistants or AI software engineers. Recently, many data science benchmarks have been proposed to investigate their performance in the data science domain. However, existing data science benchmarks still fall short when compared to real-world data science applications due to their simplified settings. To bridge this gap, we introduce DSBench, a comprehensive benchmark designed to evaluate data science agents with realistic tasks. This benchmark includes 466 data analysis tasks and 74 data modeling tasks, sourced from Eloquence and Kaggle competitions. DSBench offers a realistic setting by encompassing long contexts, multimodal task backgrounds, reasoning with large data files and multi-table structures, and performing end-to-end data modeling tasks. Our evaluation of state-of-the-art LLMs, LVLMs, and agents shows that they struggle with most tasks, with the best agent solving only 34.12% of data analysis tasks and achieving a 34.74% Relative Performance Gap (RPG). These findings underscore the need for further advancements in developing more practical, intelligent, and autonomous data science agents."3397 },3398 "arXiv:2409.11363": {3399 "paperId": "4c913d59d150fe7581386b87dfd9f90448a9adee",3400 "externalIds": {3401 "ArXiv": "2409.11363",3402 "DBLP": "journals/tmlr/SiegelKNSN24",3403 "DOI": "10.48550/arXiv.2409.11363",3404 "CorpusId": 2726944233405 },3406 "publicationVenue": null,3407 "title": "CORE-Bench: Fostering the Credibility of Published Research Through a Computational Reproducibility Agent Benchmark",3408 "venue": "Trans. Mach. Learn. Res.",3409 "year": 2024,3410 "citationCount": 147,3411 "openAccessPdf": {3412 "url": "",3413 "status": null,3414 "license": null,3415 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2409.11363, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."3416 },3417 "publicationTypes": [3418 "JournalArticle"3419 ],3420 "journal": {3421 "name": "ArXiv",3422 "volume": "abs/2409.11363"3423 },3424 "authors": [3425 {3426 "authorId": "2274102473",3427 "name": "Zachary S. Siegel"3428 },3429 {3430 "authorId": "39893263",3431 "name": "Sayash Kapoor"3432 },3433 {3434 "authorId": "2290073765",3435 "name": "Nitya Nadgir"3436 },3437 {3438 "authorId": "2309175570",3439 "name": "Benedikt Stroebl"3440 },3441 {3442 "authorId": "2064013115",3443 "name": "Arvind Narayanan"3444 }3445 ],3446 "abstract": "AI agents have the potential to aid users on a variety of consequential tasks, including conducting scientific research. To spur the development of useful agents, we need benchmarks that are challenging, but more crucially, directly correspond to real-world tasks of interest. This paper introduces such a benchmark, designed to measure the accuracy of AI agents in tackling a crucial yet surprisingly challenging aspect of scientific research: computational reproducibility. This task, fundamental to the scientific process, involves reproducing the results of a study using the provided code and data. We introduce CORE-Bench (Computational Reproducibility Agent Benchmark), a benchmark consisting of 270 tasks based on 90 scientific papers across three disciplines (computer science, social science, and medicine). Tasks in CORE-Bench consist of three difficulty levels and include both language-only and vision-language tasks. We provide an evaluation system to measure the accuracy of agents in a fast and parallelizable way, saving days of evaluation time for each run compared to a sequential implementation. We evaluated two baseline agents: the general-purpose AutoGPT and a task-specific agent called CORE-Agent. We tested both variants using two underlying language models: GPT-4o and GPT-4o-mini. The best agent achieved an accuracy of 21% on the hardest task, showing the vast scope for improvement in automating routine scientific tasks. Having agents that can reproduce existing work is a necessary step towards building agents that can conduct novel research and could verify and improve the performance of other research agents. We hope that CORE-Bench can improve the state of reproducibility and spur the development of future research agents."3447 },3448 "arXiv:2410.05080": {3449 "paperId": "45b7c7448768b51b6dbd9b76495c9cd9d110bd91",3450 "externalIds": {3451 "DBLP": "journals/corr/abs-2410-05080",3452 "ArXiv": "2410.05080",3453 "DOI": "10.48550/arXiv.2410.05080",3454 "CorpusId": 2731863113455 },3456 "publicationVenue": {3457 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",3458 "name": "arXiv.org",3459 "alternate_names": [3460 "ArXiv"3461 ],3462 "issn": "2331-8422",3463 "url": "https://arxiv.org"3464 },3465 "title": "ScienceAgentBench: Toward Rigorous Assessment of Language Agents for Data-Driven Scientific Discovery",3466 "venue": "arXiv.org",3467 "year": 2024,3468 "citationCount": 275,3469 "openAccessPdf": {3470 "url": "",3471 "status": null,3472 "license": null,3473 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2410.05080, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."3474 },3475 "publicationTypes": [3476 "JournalArticle",3477 "Review"3478 ],3479 "journal": {3480 "name": "ArXiv",3481 "volume": "abs/2410.05080"3482 },3483 "authors": [3484 {3485 "authorId": "11832104",3486 "name": "Ziru Chen"3487 },3488 {3489 "authorId": "2115939351",3490 "name": "Shijie Chen"3491 },3492 {3493 "authorId": "2266397759",3494 "name": "Yu-Ting Ning"3495 },3496 {3497 "authorId": "2261249755",3498 "name": "Qian-Heng Zhang"3499 },3500 {3501 "authorId": "7425689",3502 "name": "Boshi Wang"3503 },3504 {3505 "authorId": "2754371",3506 "name": "Botao Yu"3507 },3508 {3509 "authorId": "2324840035",3510 "name": "Yi-Fei Li"3511 },3512 {3513 "authorId": "2284224390",3514 "name": "Zeyi Liao"3515 },3516 {3517 "authorId": "2325857726",3518 "name": "Chen Wei"3519 },3520 {3521 "authorId": "1605906053",3522 "name": "Zi-Tong Lu"3523 },3524 {3525 "authorId": "2281742135",3526 "name": "Vishal Dey"3527 },3528 {3529 "authorId": "2125081864",3530 "name": "Ming-Yi Xue"3531 },3532 {3533 "authorId": "49780607",3534 "name": "Frazier N. Baker"3535 },3536 {3537 "authorId": "2302148833",3538 "name": "Benjamin Burns"3539 },3540 {3541 "authorId": "2348516502",3542 "name": "Daniel Adu-Ampratwum"3543 },3544 {3545 "authorId": "2325131030",3546 "name": "Xuhui Huang"3547 },3548 {3549 "authorId": "2281745183",3550 "name": "Xia Ning"3551 },3552 {3553 "authorId": "2265362739",3554 "name": "Song Gao"3555 },3556 {3557 "authorId": "1758652",3558 "name": "Yu Su"3559 },3560 {3561 "authorId": "2278123075",3562 "name": "Huan Sun"3563 }3564 ],3565 "abstract": "The advancements of large language models (LLMs) have piqued growing interest in developing LLM-based language agents to automate scientific discovery end-to-end, which has sparked both excitement and skepticism about their true capabilities. In this work, we call for rigorous assessment of agents on individual tasks in a scientific workflow before making bold claims on end-to-end automation. To this end, we present ScienceAgentBench, a new benchmark for evaluating language agents for data-driven scientific discovery. To ensure the scientific authenticity and real-world relevance of our benchmark, we extract 102 tasks from 44 peer-reviewed publications in four disciplines and engage nine subject matter experts to validate them. We unify the target output for every task to a self-contained Python program file and employ an array of evaluation metrics to examine the generated programs, execution results, and costs. Each task goes through multiple rounds of manual validation by annotators and subject matter experts to ensure its annotation quality and scientific plausibility. We also propose two effective strategies to mitigate data contamination concerns. Using ScienceAgentBench, we evaluate five open-weight and proprietary LLMs, each with three frameworks: direct prompting, OpenHands CodeAct, and self-debug. Given three attempts for each task, the best-performing agent can only solve 32.4% of the tasks independently and 34.3% with expert-provided knowledge. In addition, we evaluate OpenAI o1-preview with direct prompting and self-debug, which can boost the performance to 42.2%, demonstrating the effectiveness of increasing inference-time compute but with more than 10 times the cost of other LLMs. Still, our results underscore the limitations of current language agents in generating code for data-driven discovery, let alone end-to-end automation for scientific research."3566 },3567 "arXiv:2410.06992": {3568 "paperId": "05f4f076a8f277f46dff9b6abc3feff1fcf31ff5",3569 "externalIds": {3570 "DBLP": "journals/corr/abs-2410-06992",3571 "ArXiv": "2410.06992",3572 "DOI": "10.48550/arXiv.2410.06992",3573 "CorpusId": 2732286463574 },3575 "publicationVenue": {3576 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",3577 "name": "arXiv.org",3578 "alternate_names": [3579 "ArXiv"3580 ],3581 "issn": "2331-8422",3582 "url": "https://arxiv.org"3583 },3584 "title": "SWE-Bench+: Enhanced Coding Benchmark for LLMs",3585 "venue": "arXiv.org",3586 "year": 2024,3587 "citationCount": 134,3588 "openAccessPdf": {3589 "url": "",3590 "status": null,3591 "license": null,3592 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2410.06992, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."3593 },3594 "publicationTypes": [3595 "JournalArticle"3596 ],3597 "journal": {3598 "name": "ArXiv",3599 "volume": "abs/2410.06992"3600 },3601 "authors": [3602 {3603 "authorId": "2115012783",3604 "name": "Reem Aleithan"3605 },3606 {3607 "authorId": "2325736475",3608 "name": "Haoran Xue"3609 },3610 {3611 "authorId": "2224465665",3612 "name": "Mohammad Mahdi Mohajer"3613 },3614 {3615 "authorId": "2325097481",3616 "name": "Elijah Nnorom"3617 },3618 {3619 "authorId": "2325093115",3620 "name": "Gias Uddin"3621 },3622 {3623 "authorId": "2257127006",3624 "name": "Song Wang"3625 }3626 ],3627 "abstract": "Large Language Models (LLMs) in Software Engineering (SE) can offer assistance for coding. To facilitate a rigorous evaluation of LLMs in practical coding contexts, Carlos et al. introduced the SWE-bench dataset, which comprises 2,294 real-world GitHub issues and their corresponding pull requests, collected from 12 widely used Python repositories. Several impressive LLM-based toolkits recently are developed and evaluated on this dataset. However, a systematic evaluation of the quality of SWE-bench remains missing. In this paper, we addressed this gap by presenting an empirical analysis of the SWE-bench dataset. We conducted a manual screening of instances where SWEAgent + GPT-4 successfully resolved issues by comparing the model-generated patches with the actual pull requests. SWE-Agent+GPT-4 was at the top of SWE-bench leaderboard during the time of our study. Our analysis reveals some critical issues with the SWE-bench dataset: 1) 32.67% of the successful patches involve cheating as the solutions were directly provided in the issue report or the comments. We refer to as solution leakage problem. 2) 31.08% of the passed patches are suspicious patches due to weak test cases, i.e., the tests were not adequate to verify the correctness of a patch. When we filtered out these problematic issues, the resolution rate of SWE-Agent+GPT-4 dropped from 12.47% to 3.97%. We also observed that the same data quality issues also exist in the two variants of SWE-bench, i.e., SWE-bench Lite and SWE-Bench Verified. In addition, over 94% of the issues were created before LLM's knowledge cutoff dates, posing potential data leakage issues."3628 },3629 "arXiv:2410.07095": {3630 "paperId": "7c44b7fdcec2e517799f6c54f6ba42bf1a89d2e6",3631 "externalIds": {3632 "ArXiv": "2410.07095",3633 "DBLP": "journals/corr/abs-2410-07095",3634 "DOI": "10.48550/arXiv.2410.07095",3635 "CorpusId": 2732335503636 },3637 "publicationVenue": {3638 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",3639 "name": "arXiv.org",3640 "alternate_names": [3641 "ArXiv"3642 ],3643 "issn": "2331-8422",3644 "url": "https://arxiv.org"3645 },3646 "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering",3647 "venue": "arXiv.org",3648 "year": 2024,3649 "citationCount": 532,3650 "openAccessPdf": {3651 "url": "",3652 "status": null,3653 "license": null,3654 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2410.07095, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."3655 },3656 "publicationTypes": [3657 "JournalArticle",3658 "Review"3659 ],3660 "journal": {3661 "name": "ArXiv",3662 "volume": "abs/2410.07095"3663 },3664 "authors": [3665 {3666 "authorId": "2325723546",3667 "name": "Jun Shern Chan"3668 },3669 {3670 "authorId": "2325154459",3671 "name": "Neil Chowdhury"3672 },3673 {3674 "authorId": "2325151487",3675 "name": "Oliver Jaffe"3676 },3677 {3678 "authorId": "2325157923",3679 "name": "James Aung"3680 },3681 {3682 "authorId": "2231409794",3683 "name": "Dane Sherburn"3684 },3685 {3686 "authorId": "2325158061",3687 "name": "E. Mays"3688 },3689 {3690 "authorId": "2168285763",3691 "name": "Giulio Starace"3692 },3693 {3694 "authorId": "2325207065",3695 "name": "Kevin Liu"3696 },3697 {3698 "authorId": "2325157894",3699 "name": "Leon Maksin"3700 },3701 {3702 "authorId": "90169542",3703 "name": "Tejal Patwardhan"3704 },3705 {3706 "authorId": "2325162875",3707 "name": "Lilian Weng"3708 },3709 {3710 "authorId": "2258929982",3711 "name": "Aleksander Madry"3712 }3713 ],3714 "abstract": "We introduce MLE-bench, a benchmark for measuring how well AI agents perform at machine learning engineering. To this end, we curate 75 ML engineering-related competitions from Kaggle, creating a diverse set of challenging tasks that test real-world ML engineering skills such as training models, preparing datasets, and running experiments. We establish human baselines for each competition using Kaggle's publicly available leaderboards. We use open-source agent scaffolds to evaluate several frontier language models on our benchmark, finding that the best-performing setup--OpenAI's o1-preview with AIDE scaffolding--achieves at least the level of a Kaggle bronze medal in 16.9% of competitions. In addition to our main results, we investigate various forms of resource scaling for AI agents and the impact of contamination from pre-training. We open-source our benchmark code (github.com/openai/mle-bench/) to facilitate future research in understanding the ML engineering capabilities of AI agents."3715 },3716 "arXiv:2410.07331": {3717 "paperId": "0e28243cc36df0dfa857770e5a0a560275b225bf",3718 "externalIds": {3719 "DBLP": "journals/corr/abs-2410-07331",3720 "ArXiv": "2410.07331",3721 "ACL": "2024.emnlp-main.748",3722 "DOI": "10.48550/arXiv.2410.07331",3723 "CorpusId": 2732340393724 },3725 "publicationVenue": {3726 "id": "41bf9ed3-85b3-4c90-b015-150e31690253",3727 "name": "Conference on Empirical Methods in Natural Language Processing",3728 "type": "conference",3729 "alternate_names": [3730 "Empir Method Nat Lang Process",3731 "Empirical Methods in Natural Language Processing",3732 "Conf Empir Method Nat Lang Process",3733 "EMNLP"3734 ],3735 "url": "https://www.aclweb.org/portal/emnlp"3736 },3737 "title": "DA-Code: Agent Data Science Code Generation Benchmark for Large Language Models",3738 "venue": "Conference on Empirical Methods in Natural Language Processing",3739 "year": 2024,3740 "citationCount": 73,3741 "openAccessPdf": {3742 "url": "",3743 "status": null,3744 "license": null,3745 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2410.07331, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."3746 },3747 "publicationTypes": [3748 "JournalArticle",3749 "Conference"3750 ],3751 "journal": {3752 "pages": "13487-13521"3753 },3754 "authors": [3755 {3756 "authorId": "2261394934",3757 "name": "Yiming Huang"3758 },3759 {3760 "authorId": "2325193187",3761 "name": "Jianwen Luo"3762 },3763 {3764 "authorId": "2308879905",3765 "name": "Yang Yu"3766 },3767 {3768 "authorId": "2320286082",3769 "name": "Yi-Tong Zhang"3770 },3771 {3772 "authorId": "2257004176",3773 "name": "Fangyu Lei"3774 },3775 {3776 "authorId": "2216484616",3777 "name": "Yifan Wei"3778 },3779 {3780 "authorId": "1954845",3781 "name": "Shi-Zhu He"3782 },3783 {3784 "authorId": "2319173963",3785 "name": "Li-Fu Huang"3786 },3787 {3788 "authorId": "2325642391",3789 "name": "Xiao Liu"3790 },3791 {3792 "authorId": "2284076390",3793 "name": "Jun Zhao"3794 },3795 {3796 "authorId": "2283274776",3797 "name": "Kang Liu"3798 }3799 ],3800 "abstract": "We introduce DA-Code, a code generation benchmark specifically designed to assess LLMs on agent-based data science tasks. This benchmark features three core elements: First, the tasks within DA-Code are inherently challenging, setting them apart from traditional code generation tasks and demanding advanced coding skills in grounding and planning. Second, examples in DA-Code are all based on real and diverse data, covering a wide range of complex data wrangling and analytics tasks. Third, to solve the tasks, the models must utilize complex data science programming languages, including Python and SQL, to perform intricate data processing and derive the answers. We set up the benchmark in a controllable and executable environment that aligns with real-world data analysis scenarios and is scalable. The annotators meticulously designed the evaluation suite to ensure the accuracy and robustness of the evaluation. We developed the DA-Agent baseline. Experiments show that although the baseline performs better than other existing frameworks, using the current best LLMs achieves only 30.5% accuracy, leaving ample room for improvement. We release our benchmark at [link](https://github.com/yiyihum/dabench)"3801 },3802 "arXiv:2411.00816": {3803 "paperId": "92c82a51ad13c361d052987694cf93d6a72d5789",3804 "externalIds": {3805 "DBLP": "conf/iclr/WengZBZ00Y25",3806 "ArXiv": "2411.00816",3807 "DOI": "10.48550/arXiv.2411.00816",3808 "CorpusId": 2738119973809 },3810 "publicationVenue": {3811 "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",3812 "name": "International Conference on Learning Representations",3813 "type": "conference",3814 "alternate_names": [3815 "Int Conf Learn Represent",3816 "ICLR"3817 ],3818 "url": "https://iclr.cc/"3819 },3820 "title": "CycleResearcher: Improving Automated Research via Automated Review",3821 "venue": "International Conference on Learning Representations",3822 "year": 2024,3823 "citationCount": 180,3824 "openAccessPdf": {3825 "url": "",3826 "status": null,3827 "license": null,3828 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2411.00816, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."3829 },3830 "publicationTypes": [3831 "JournalArticle",3832 "Review"3833 ],3834 "journal": {3835 "name": "ArXiv",3836 "volume": "abs/2411.00816"3837 },3838 "authors": [3839 {3840 "authorId": "2142839441",3841 "name": "Yixuan Weng"3842 },3843 {3844 "authorId": "2316827669",3845 "name": "Min-Jun Zhu"3846 },3847 {3848 "authorId": "1993226927",3849 "name": "Guangsheng Bao"3850 },3851 {3852 "authorId": "2329135444",3853 "name": "Hong-Bo Zhang"3854 },3855 {3856 "authorId": "1519290245",3857 "name": "Jin-Dong Wang"3858 },3859 {3860 "authorId": "2325943212",3861 "name": "Yue Zhang"3862 },3863 {3864 "authorId": "2145500840",3865 "name": "Linyi Yang"3866 }3867 ],3868 "abstract": "The automation of scientific discovery has been a long-standing goal within the research community, driven by the potential to accelerate knowledge creation. While significant progress has been made using commercial large language models (LLMs) as research assistants or idea generators, the possibility of automating the entire research process with open-source LLMs remains largely unexplored. This paper explores the feasibility of using open-source post-trained LLMs as autonomous agents capable of performing the full cycle of automated research and review, from literature review and manuscript preparation to peer review and paper refinement. Our iterative preference training framework consists of CycleResearcher, which conducts research tasks, and CycleReviewer, which simulates the peer review process, providing iterative feedback via reinforcement learning. To train these models, we develop two new datasets, Review-5k and Research-14k, reflecting real-world machine learning research and peer review dynamics. Our results demonstrate that CycleReviewer achieves promising performance with a 26.89\\% reduction in mean absolute error (MAE) compared to individual human reviewers in predicting paper scores, indicating the potential of LLMs to effectively assist expert-level research evaluation. In research, the papers generated by the CycleResearcher model achieved a score of 5.36 in simulated peer reviews, showing some competitiveness in terms of simulated review scores compared to the preprint level of 5.24 from human experts, while still having room for improvement compared to the accepted paper level of 5.69. This work represents a significant step toward fully automated scientific inquiry, providing ethical safeguards and exploring AI-driven research capabilities. The code, dataset and model weight are released at https://wengsyx.github.io/Researcher/."3869 },3870 "arXiv:2411.05285": {3871 "paperId": "3d6b189c9f8bb2f956db3fb42032d60eab29139c",3872 "externalIds": {3873 "DBLP": "journals/corr/abs-2411-05285",3874 "ArXiv": "2411.05285",3875 "DOI": "10.48550/arXiv.2411.05285",3876 "CorpusId": 2739499193877 },3878 "publicationVenue": {3879 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",3880 "name": "arXiv.org",3881 "alternate_names": [3882 "ArXiv"3883 ],3884 "issn": "2331-8422",3885 "url": "https://arxiv.org"3886 },3887 "title": "AgentOps: Enabling Observability of LLM Agents",3888 "venue": "arXiv.org",3889 "year": 2024,3890 "citationCount": 63,3891 "openAccessPdf": {3892 "url": "",3893 "status": null,3894 "license": null,3895 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2411.05285, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."3896 },3897 "publicationTypes": [3898 "JournalArticle"3899 ],3900 "journal": {3901 "name": "ArXiv",3902 "volume": "abs/2411.05285"3903 },3904 "authors": [3905 {3906 "authorId": "2367200438",3907 "name": "Li-Ming Dong"3908 },3909 {3910 "authorId": "2151674574",3911 "name": "Qing-Hua Lu"3912 },3913 {3914 "authorId": "2125737414",3915 "name": "Li-Ming Zhu"3916 }3917 ],3918 "abstract": "Large language model (LLM) agents have demonstrated remarkable capabilities across various domains, gaining extensive attention from academia and industry. However, these agents raise significant concerns on AI safety due to their autonomous and non-deterministic behavior, as well as continuous evolving nature . From a DevOps perspective, enabling observability in agents is necessary to ensuring AI safety, as stakeholders can gain insights into the agents' inner workings, allowing them to proactively understand the agents, detect anomalies, and prevent potential failures. Therefore, in this paper, we present a comprehensive taxonomy of AgentOps, identifying the artifacts and associated data that should be traced throughout the entire lifecycle of agents to achieve effective observability. The taxonomy is developed based on a systematic mapping study of existing AgentOps tools. Our taxonomy serves as a reference template for developers to design and implement AgentOps infrastructure that supports monitoring, logging, and analytics. thereby ensuring AI safety."3919 },3920 "arXiv:2411.15114": {3921 "paperId": "5dbd9f9fd231863dda17d9e3caff2edb99a113e2",3922 "externalIds": {3923 "ArXiv": "2411.15114",3924 "DBLP": "journals/corr/abs-2411-15114",3925 "DOI": "10.48550/arXiv.2411.15114",3926 "CorpusId": 2741922623927 },3928 "publicationVenue": {3929 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",3930 "name": "arXiv.org",3931 "alternate_names": [3932 "ArXiv"3933 ],3934 "issn": "2331-8422",3935 "url": "https://arxiv.org"3936 },3937 "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts",3938 "venue": "arXiv.org",3939 "year": 2024,3940 "citationCount": 208,3941 "openAccessPdf": {3942 "url": "",3943 "status": null,3944 "license": null,3945 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2411.15114, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."3946 },3947 "publicationTypes": [3948 "JournalArticle"3949 ],3950 "journal": {3951 "name": "ArXiv",3952 "volume": "abs/2411.15114"3953 },3954 "authors": [3955 {3956 "authorId": "2331855692",3957 "name": "Hjalmar Wijk"3958 },3959 {3960 "authorId": "2228084357",3961 "name": "T. Lin"3962 },3963 {3964 "authorId": "2331857910",3965 "name": "Joel Becker"3966 },3967 {3968 "authorId": "2331855846",3969 "name": "Sami Jawhar"3970 },3971 {3972 "authorId": "1380182284",3973 "name": "Neev Parikh"3974 },3975 {3976 "authorId": "2331855786",3977 "name": "Thomas Broadley"3978 },3979 {3980 "authorId": "2164095723",3981 "name": "Lawrence Chan"3982 },3983 {3984 "authorId": "2332345490",3985 "name": "Michael Chen"3986 },3987 {3988 "authorId": "2291961652",3989 "name": "Joshua Clymer"3990 },3991 {3992 "authorId": "2331855463",3993 "name": "Jai Dhyani"3994 },3995 {3996 "authorId": "2331855896",3997 "name": "Elena Ericheva"3998 },3999 {4000 "authorId": "2331855858",4001 "name": "Katharyn Garcia"4002 },4003 {4004 "authorId": "2227287541",4005 "name": "Brian Goodrich"4006 },4007 {4008 "authorId": "2331855456",4009 "name": "Nikola Jurkovic"4010 },4011 {4012 "authorId": "1800253074",4013 "name": "Megan Kinniment"4014 },4015 {4016 "authorId": "2331855819",4017 "name": "Aron Lajko"4018 },4019 {4020 "authorId": "2164093876",4021 "name": "Seraphina Nix"4022 },4023 {4024 "authorId": "2275156833",4025 "name": "L. Sato"4026 },4027 {4028 "authorId": "2331855884",4029 "name": "William Saunders"4030 },4031 {4032 "authorId": "2331855792",4033 "name": "M. Taran"4034 },4035 {4036 "authorId": "2331856000",4037 "name": "Ben West"4038 },4039 {4040 "authorId": "2331855799",4041 "name": "Elizabeth Barnes"4042 }4043 ],4044 "abstract": "Frontier AI safety policies highlight automation of AI research and development (R&D) by AI agents as an important capability to anticipate. However, there exist few evaluations for AI R&D capabilities, and none that are highly realistic and have a direct comparison to human performance. We introduce RE-Bench (Research Engineering Benchmark, v1), which consists of 7 challenging, open-ended ML research engineering environments and data from 71 8-hour attempts by 61 distinct human experts. We confirm that our experts make progress in the environments given 8 hours, with 82% of expert attempts achieving a non-zero score and 24% matching or exceeding our strong reference solutions. We compare humans to several public frontier models through best-of-k with varying time budgets and agent designs, and find that the best AI agents achieve a score 4x higher than human experts when both are given a total time budget of 2 hours per environment. However, humans currently display better returns to increasing time budgets, narrowly exceeding the top AI agent scores given an 8-hour budget, and achieving 2x the score of the top AI agent when both are given 32 total hours (across different attempts). Qualitatively, we find that modern AI agents possess significant expertise in many ML topics -- e.g. an agent wrote a faster custom Triton kernel than any of our human experts' -- and can generate and test solutions over ten times faster than humans, at much lower cost. We open-source the evaluation environments, human expert data, analysis code and agent trajectories to facilitate future research."4045 },4046 "arXiv:2411.15594": {4047 "paperId": "e24424283c02fbe7f641e5b3490d7bb059f8355a",4048 "externalIds": {4049 "DBLP": "journals/corr/abs-2411-15594",4050 "ArXiv": "2411.15594",4051 "DOI": "10.48550/arXiv.2411.15594",4052 "CorpusId": 2742340144053 },4054 "publicationVenue": {4055 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",4056 "name": "arXiv.org",4057 "alternate_names": [4058 "ArXiv"4059 ],4060 "issn": "2331-8422",4061 "url": "https://arxiv.org"4062 },4063 "title": "A Survey on LLM-as-a-Judge",4064 "venue": "arXiv.org",4065 "year": 2024,4066 "citationCount": 1988,4067 "openAccessPdf": {4068 "url": "",4069 "status": null,4070 "license": null,4071 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2411.15594, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."4072 },4073 "publicationTypes": [4074 "JournalArticle",4075 "Review"4076 ],4077 "journal": {4078 "name": "ArXiv",4079 "volume": "abs/2411.15594"4080 },4081 "authors": [4082 {4083 "authorId": "2216587705",4084 "name": "Jiawei Gu"4085 },4086 {4087 "authorId": "144267788",4088 "name": "Xuhui Jiang"4089 },4090 {4091 "authorId": "2287881684",4092 "name": "Zhichao Shi"4093 },4094 {4095 "authorId": "2274159320",4096 "name": "He-Xiang Tan"4097 },4098 {4099 "authorId": "2332093190",4100 "name": "Xue-Hao Zhai"4101 },4102 {4103 "authorId": "2250617116",4104 "name": "Cheng-Jin Xu"4105 },4106 {4107 "authorId": "2330714501",4108 "name": "Wei Li"4109 },4110 {4111 "authorId": "1944248313",4112 "name": "Ying-Han Shen"4113 },4114 {4115 "authorId": "2311556497",4116 "name": "Sheng-Jie Ma"4117 },4118 {4119 "authorId": "2332306096",4120 "name": "Hong-Hao Liu"4121 },4122 {4123 "authorId": "2219600",4124 "name": "Yuan-Zhuo Wang"4125 },4126 {4127 "authorId": "2284217200",4128 "name": "Jian Guo"4129 }4130 ],4131 "abstract": "Accurate and consistent evaluation is crucial for decision-making across numerous fields, yet it remains a challenging task due to inherent subjectivity, variability, and scale. Large Language Models (LLMs) have achieved remarkable success across diverse domains, leading to the emergence of\"LLM-as-a-Judge,\"where LLMs are employed as evaluators for complex tasks. With their ability to process diverse data types and provide scalable, cost-effective, and consistent assessments, LLMs present a compelling alternative to traditional expert-driven evaluations. However, ensuring the reliability of LLM-as-a-Judge systems remains a significant challenge that requires careful design and standardization. This paper provides a comprehensive survey of LLM-as-a-Judge, addressing the core question: How can reliable LLM-as-a-Judge systems be built? We explore strategies to enhance reliability, including improving consistency, mitigating biases, and adapting to diverse assessment scenarios. Additionally, we propose methodologies for evaluating the reliability of LLM-as-a-Judge systems, supported by a novel benchmark designed for this purpose. To advance the development and real-world deployment of LLM-as-a-Judge systems, we also discussed practical applications, challenges, and future directions. This survey serves as a foundational reference for researchers and practitioners in this rapidly evolving field."4132 },4133 "arXiv:2501.04227": {4134 "paperId": "394924896e24c9b086d96d0958dae07f54ff9452",4135 "externalIds": {4136 "ArXiv": "2501.04227",4137 "DBLP": "conf/emnlp/SchmidgallSWSWYLMLB25",4138 "DOI": "10.18653/v1/2025.findings-emnlp.320",4139 "CorpusId": 2753580174140 },4141 "publicationVenue": {4142 "id": "41bf9ed3-85b3-4c90-b015-150e31690253",4143 "name": "Conference on Empirical Methods in Natural Language Processing",4144 "type": "conference",4145 "alternate_names": [4146 "Empir Method Nat Lang Process",4147 "Empirical Methods in Natural Language Processing",4148 "Conf Empir Method Nat Lang Process",4149 "EMNLP"4150 ],4151 "url": "https://www.aclweb.org/portal/emnlp"4152 },4153 "title": "Agent Laboratory: Using LLM Agents as Research Assistants",4154 "venue": "Conference on Empirical Methods in Natural Language Processing",4155 "year": 2025,4156 "citationCount": 573,4157 "openAccessPdf": {4158 "url": "https://aclanthology.org/2025.findings-emnlp.320.pdf",4159 "status": "HYBRID",4160 "license": "CCBY",4161 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2501.04227, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."4162 },4163 "publicationTypes": [4164 "JournalArticle",4165 "Conference",4166 "Review"4167 ],4168 "journal": {4169 "pages": "5977-6043"4170 },4171 "authors": [4172 {4173 "authorId": "1741847434",4174 "name": "Samuel Schmidgall"4175 },4176 {4177 "authorId": "2339527873",4178 "name": "Yu-Sheng Su"4179 },4180 {4181 "authorId": "2336047375",4182 "name": "Ze Wang"4183 },4184 {4185 "authorId": "2335618942",4186 "name": "Xi-Meng Sun"4187 },4188 {4189 "authorId": "2339249470",4190 "name": "Jia-Lian Wu"4191 },4192 {4193 "authorId": "2339225241",4194 "name": "Xiao-Dong Yu"4195 },4196 {4197 "authorId": "2335604069",4198 "name": "Jiang Liu"4199 },4200 {4201 "authorId": "2301157095",4202 "name": "Michael Moor"4203 },4204 {4205 "authorId": "2334872571",4206 "name": "Zi-Cheng Liu"4207 },4208 {4209 "authorId": "2271751612",4210 "name": "E. Barsoum"4211 }4212 ],4213 "abstract": "Historically, scientific discovery has been a lengthy and costly process, demanding substantial time and resources from initial conception to final results. To accelerate scientific discovery, reduce research costs, and improve research quality, we introduce Agent Laboratory, an autonomous LLM-based framework capable of completing the entire research process. This framework accepts a human-provided research idea and progresses through three stages--literature review, experimentation, and report writing to produce comprehensive research outputs, including a code repository and a research report, while enabling users to provide feedback and guidance at each stage. We deploy Agent Laboratory with various state-of-the-art LLMs and invite multiple researchers to assess its quality by participating in a survey, providing human feedback to guide the research process, and then evaluate the final paper. We found that: (1) Agent Laboratory driven by o1-preview generates the best research outcomes; (2) The generated machine learning code is able to achieve state-of-the-art performance compared to existing methods; (3) Human involvement, providing feedback at each stage, significantly improves the overall quality of research; (4) Agent Laboratory significantly reduces research expenses, achieving an 84% decrease compared to previous autonomous research methods. We hope Agent Laboratory enables researchers to allocate more effort toward creative ideation rather than low-level coding and writing, ultimately accelerating scientific discovery."4214 },4215 "arXiv:2502.14297": {4216 "paperId": "fd6d9fb41fc8d06747f36fe4dff391ac064605fb",4217 "externalIds": {4218 "DBLP": "journals/sigir/BeelKB25",4219 "ArXiv": "2502.14297",4220 "DOI": "10.1145/3769733.376974",4221 "CorpusId": 2764829654222 },4223 "publicationVenue": {4224 "id": "b2ff135b-16c2-40a9-a068-c27761bfda08",4225 "name": "SIGIR Forum",4226 "type": "journal",4227 "alternate_names": [4228 "Sigir Forum"4229 ],4230 "issn": "0163-5840",4231 "alternate_issns": [4232 "1558-0229"4233 ],4234 "url": "http://rave.ohiolink.edu/ejournals/issn/01635840/",4235 "alternate_urls": [4236 "http://portal.acm.org/sigir/newsletter",4237 "http://www.acm.org/sigs/sigir/forum/index.html"4238 ]4239 },4240 "title": "Evaluating Sakana's AI Scientist: Bold Claims, Mixed Results, and a Promising Future?",4241 "venue": "SIGIR Forum",4242 "year": 2025,4243 "citationCount": 38,4244 "openAccessPdf": {4245 "url": "",4246 "status": null,4247 "license": null,4248 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2502.14297, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."4249 },4250 "publicationTypes": [4251 "JournalArticle",4252 "Review"4253 ],4254 "journal": {4255 "name": "ACM SIGIR Forum",4256 "pages": "1 - 20",4257 "volume": "59"4258 },4259 "authors": [4260 {4261 "authorId": "2137867216",4262 "name": "Joeran Beel"4263 },4264 {4265 "authorId": "2270948852",4266 "name": "Min-Yen Kan"4267 },4268 {4269 "authorId": "2326137802",4270 "name": "Moritz Baumgart"4271 }4272 ],4273 "abstract": "Recently, Sakana.ai introduced the AI Scientist, a system claiming to automate the entire research lifecycle and conduct research autonomously, a concept we term Artificial Research Intelligence (ARI). Achieving ARI would be a major milestone toward Artificial General Intelligence (AGI) and a prerequisite to achieving Super Intelligence. The AI Scientist received much attention in the academic and broader AI community. A thorough evaluation of the AI Scientist, however, had not yet been conducted.1 We evaluated the AI Scientist and found several critical shortcomings. The system's literature review process is inadequate, relying on simplistic keyword searches rather than profound synthesis, which leads to poor novelty assessments. In our experiments, several generated research ideas were incorrectly classified as novel, including well-established concepts such as micro-batching for stochastic gradient descent (SGD). The AI Scientist also lacks robustness in experiment execution\u2014five out of twelve proposed experiments (42%) failed due to coding errors, and those that did run often produced logically flawed or misleading results. In one case, an experiment designed to optimize energy efficiency reported improvements in accuracy while consuming more computational resources, contradicting its stated goal. Furthermore, the system modifies experimental code minimally, with each iteration adding only 8% more characters on average, suggesting limited adaptability. The generated manuscripts were poorly substantiated, with a median of just five citations per paper\u2014most of which were outdated (only five out of 34 citations were from 2020 or later). Structural errors were frequent, including missing figures, repeated sections, and placeholder text such as \"Conclusions Here\". Hallucinated numerical results were contained in several manuscripts, undermining the reliability of its outputs. Despite its limitations, the AI Scientist represents a significant leap forward in research automation. It produces complete research manuscripts with minimal human intervention, challenging conventional expectations of AI-generated scientific work. Many reviewers or university instructors conducting only a superficial assessment may struggle to distinguish its output from that of human researchers, demonstrating how far AI has progressed in mimicking academic writing and structuring scientific arguments. While the quality of its manuscripts currently aligns with that of an unmotivated undergraduate student rushing to meet a deadline, this level of autonomy in research generation is remarkable. More strikingly, it achieves this at an unprecedented speed and cost efficiency\u2014our analysis indicates that generating a full research paper costs only $6\u2013$15, with just 3.5 hours of human involvement. This is significantly faster than traditional human researchers. Given that AI research automation was nearly nonexistent just a few years ago, the AI Scientist marks a substantial milestone toward Artificial Research Intelligence (ARI), signalling the acceleration of AI-driven scientific discovery. The AI Scientist also illustrates the urgent need for a discussion within the Information Retrieval (IR) and broader scientific communities. Whether and when ARI becomes a reality depends on how the academic and AI communities shape its development and governance. We propose concrete steps, including pilot projects and competitions, and standardized attribution frameworks such as research logs and markup languages."4274 },4275 "arXiv:2502.18864": {4276 "paperId": "390c607723e9767a9a873ded3bc1f66808215e8e",4277 "externalIds": {4278 "ArXiv": "2502.18864",4279 "PubMedCentral": "13345910",4280 "DOI": "10.1038/s41586-026-10644-y",4281 "CorpusId": 276617649,4282 "PubMed": "42428019"4283 },4284 "publicationVenue": {4285 "id": "6c24a0a0-b07d-4d7b-a19b-fd09a3ed453a",4286 "name": "Nature",4287 "type": "journal",4288 "issn": "0028-0836",4289 "url": "https://www.nature.com/",4290 "alternate_urls": [4291 "http://www.nature.com/nature/",4292 "https://www.nature.com/nature/",4293 "http://www.nature.com/nature/archive/index.html"4294 ]4295 },4296 "title": "Accelerating scientific discovery with Co-Scientist",4297 "venue": "Nature",4298 "year": 2025,4299 "citationCount": 593,4300 "openAccessPdf": {4301 "url": "https://www.nature.com/articles/s41586-026-10644-y_reference.pdf",4302 "status": "HYBRID",4303 "license": "other-oa",4304 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2502.18864, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."4305 },4306 "publicationTypes": [4307 "JournalArticle"4308 ],4309 "journal": {4310 "name": "Nature",4311 "pages": "487 - 496",4312 "volume": "655"4313 },4314 "authors": [4315 {4316 "authorId": "2051681709",4317 "name": "Juraj Gottweis"4318 },4319 {4320 "authorId": "2239098855",4321 "name": "Wei-Hung Weng"4322 },4323 {4324 "authorId": "2347345594",4325 "name": "Alexander Daryin"4326 },4327 {4328 "authorId": "2244651851",4329 "name": "Tao Tu"4330 },4331 {4332 "authorId": "2679779",4333 "name": "Petar Sirkovi\u0107"4334 },4335 {4336 "authorId": "2346506169",4337 "name": "Artiom Myaskovsky"4338 },4339 {4340 "authorId": "2520452099",4341 "name": "Grzegorz Glowaty"4342 },4343 {4344 "authorId": "10805574",4345 "name": "Felix Weissenberger"4346 },4347 {4348 "authorId": "2367823022",4349 "name": "Alessio Orlandi"4350 },4351 {4352 "authorId": "2062591154",4353 "name": "D. Popovici"4354 },4355 {4356 "authorId": "2266472373",4357 "name": "Anil Palepu"4358 },4359 {4360 "authorId": "1996199677",4361 "name": "Ke-Ran Rong"4362 },4363 {4364 "authorId": "3480712",4365 "name": "Ryutaro Tanno"4366 },4367 {4368 "authorId": "2279021635",4369 "name": "Khaled Saab"4370 },4371 {4372 "authorId": "2298916585",4373 "name": "Fan Zhang"4374 },4375 {4376 "authorId": "2217621049",4377 "name": "Jacob Blum"4378 },4379 {4380 "authorId": "2336095347",4381 "name": "A. Carroll"4382 },4383 {4384 "authorId": "2269148145",4385 "name": "Kavita Kulkarni"4386 },4387 {4388 "authorId": "2213266",4389 "name": "Nenad Toma\u0161ev"4390 },4391 {4392 "authorId": "115731137",4393 "name": "D. Zverinski"4394 },4395 {4396 "authorId": "2389134",4397 "name": "Ivor Rendulic"4398 },4399 {4400 "authorId": "2298901106",4401 "name": "E. Vedadi"4402 },4403 {4404 "authorId": "113494403",4405 "name": "F. Hasler"4406 },4407 {4408 "authorId": "102638867",4409 "name": "Luka Rimanic"4410 },4411 {4412 "authorId": "36598773",4413 "name": "Marina Boia"4414 },4415 {4416 "authorId": "2784346",4417 "name": "Ivan Budiseli\u0107"4418 },4419 {4420 "authorId": "2074805000",4421 "name": "Ben Feinstein"4422 },4423 {4424 "authorId": "2359151678",4425 "name": "M. Bellaiche"4426 },4427 {4428 "authorId": "2220891317",4429 "name": "Tom Sheffer"4430 },4431 {4432 "authorId": "2279022140",4433 "name": "Jan Freyberg"4434 },4435 {4436 "authorId": "2255314955",4437 "name": "Jeremy D. Ratcliff"4438 },4439 {4440 "authorId": "1789306",4441 "name": "Ottavia Bertolli"4442 },4443 {4444 "authorId": "2269148566",4445 "name": "Katherine Chou"4446 },4447 {4448 "authorId": "1809983",4449 "name": "A. Hassidim"4450 },4451 {4452 "authorId": "2225921342",4453 "name": "Burak Gokturk"4454 },4455 {4456 "authorId": "2257189937",4457 "name": "Amin Vahdat"4458 },4459 {4460 "authorId": "2261405027",4461 "name": "Yuan Guan"4462 },4463 {4464 "authorId": "12268808",4465 "name": "Vikram Dhillon"4466 },4467 {4468 "authorId": "10664723",4469 "name": "E. D. Vaishnav"4470 },4471 {4472 "authorId": "2305695545",4473 "name": "Byron Lee"4474 },4475 {4476 "authorId": "2346514434",4477 "name": "Tiago R D Costa"4478 },4479 {4480 "authorId": "2068738124",4481 "name": "J. Penad\u00e9s"4482 },4483 {4484 "authorId": "2243119549",4485 "name": "Gary Peltz"4486 },4487 {4488 "authorId": "2269148232",4489 "name": "Y. Matias"4490 },4491 {4492 "authorId": "2254252286",4493 "name": "J. Manyika"4494 },4495 {4496 "authorId": "48987704",4497 "name": "D. Hassabis"4498 },4499 {4500 "authorId": "2275191526",4501 "name": "Yunhan Xu"4502 },4503 {4504 "authorId": "143967473",4505 "name": "Pushmeet Kohli"4506 },4507 {4508 "authorId": "2359249680",4509 "name": "Annalisa Pawlosky"4510 },4511 {4512 "authorId": "2213433382",4513 "name": "A. Karthikesalingam"4514 },4515 {4516 "authorId": "144223091",4517 "name": "Vivek Natarajan"4518 }4519 ],4520 "abstract": "Scientific discovery is driven by scientists generating hypotheses for complex problems that undergo rigorous experimental validation. To augment this process, we introduce Co-Scientist, a multi-agent artificial intelligence (AI) system built on Gemini for structured scientific thinking and hypothesis generation. Co-Scientist aims to help scientists discover new original knowledge. Conditioned on their research objectives and previous scientific evidence, it formulates demonstrably novel research hypotheses for experimental verification. The system\u2019s design involves agents continuously generating, critiquing and refining hypotheses accelerated by scaling test-time compute. Key contributions include (1) a multi-agent architecture with an asynchronous task execution framework for flexible compute scaling, and (2) a tournament evolution process for self-improving hypotheses generation. Automated evaluations show continued benefits of test-time compute scaling, improving hypothesis quality over time. Although this is a general-purpose system, we focus the validation in three biomedical applications: drug repurposing; novel-target discovery1; and explaining mechanisms of antimicrobial resistance2. Specifically, Co-Scientist helped to identify new drug-repurposing candidates and synergistic combination therapies for acute myeloid leukaemia that were validated through in vitro experiments. These real-world validations demonstrate the potential of Co-Scientist to accelerate scientific discovery and usher in an era of AI-empowered scientists. Co-Scientist is an AI system for structured scientific thinking and generates research hypotheses for experimental verification."4521 },4522 "arXiv:2504.01848": {4523 "paperId": "52be71605c60d326381a4ddfa1fb476912709504",4524 "externalIds": {4525 "DBLP": "conf/icml/StaraceJSACMDMK25",4526 "ArXiv": "2504.01848",4527 "DOI": "10.48550/arXiv.2504.01848",4528 "CorpusId": 2775019934529 },4530 "publicationVenue": {4531 "id": "fc0a208c-acb7-47dc-a0d4-af8190e21d29",4532 "name": "International Conference on Machine Learning",4533 "type": "conference",4534 "alternate_names": [4535 "ICML",4536 "Int Conf Mach Learn"4537 ],4538 "url": "https://icml.cc/"4539 },4540 "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research",4541 "venue": "International Conference on Machine Learning",4542 "year": 2025,4543 "citationCount": 366,4544 "openAccessPdf": {4545 "url": "",4546 "status": null,4547 "license": null,4548 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2504.01848, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."4549 },4550 "publicationTypes": [4551 "JournalArticle",4552 "Conference"4553 ],4554 "journal": {4555 "name": "ArXiv",4556 "volume": "abs/2504.01848"4557 },4558 "authors": [4559 {4560 "authorId": "2168285763",4561 "name": "Giulio Starace"4562 },4563 {4564 "authorId": "2325151487",4565 "name": "Oliver Jaffe"4566 },4567 {4568 "authorId": "2231409794",4569 "name": "Dane Sherburn"4570 },4571 {4572 "authorId": "2325157923",4573 "name": "James Aung"4574 },4575 {4576 "authorId": "2325723546",4577 "name": "Jun Shern Chan"4578 },4579 {4580 "authorId": "2325157894",4581 "name": "Leon Maksin"4582 },4583 {4584 "authorId": "2328288671",4585 "name": "Rachel Dias"4586 },4587 {4588 "authorId": "2325158061",4589 "name": "E. Mays"4590 },4591 {4592 "authorId": "48208664",4593 "name": "Benjamin Kinsella"4594 },4595 {4596 "authorId": "2353389208",4597 "name": "Wyatt Thompson"4598 },4599 {4600 "authorId": "2151087994",4601 "name": "Johannes Heidecke"4602 },4603 {4604 "authorId": "2355867602",4605 "name": "Amelia Glaese"4606 },4607 {4608 "authorId": "90169542",4609 "name": "Tejal Patwardhan"4610 }4611 ],4612 "abstract": "We introduce PaperBench, a benchmark evaluating the ability of AI agents to replicate state-of-the-art AI research. Agents must replicate 20 ICML 2024 Spotlight and Oral papers from scratch, including understanding paper contributions, developing a codebase, and successfully executing experiments. For objective evaluation, we develop rubrics that hierarchically decompose each replication task into smaller sub-tasks with clear grading criteria. In total, PaperBench contains 8,316 individually gradable tasks. Rubrics are co-developed with the author(s) of each ICML paper for accuracy and realism. To enable scalable evaluation, we also develop an LLM-based judge to automatically grade replication attempts against rubrics, and assess our judge's performance by creating a separate benchmark for judges. We evaluate several frontier models on PaperBench, finding that the best-performing tested agent, Claude 3.5 Sonnet (New) with open-source scaffolding, achieves an average replication score of 21.0%. Finally, we recruit top ML PhDs to attempt a subset of PaperBench, finding that models do not yet outperform the human baseline. We open-source our code (https://github.com/openai/preparedness) to facilitate future research in understanding the AI engineering capabilities of AI agents."4613 },4614 "arXiv:2504.08066": {4615 "paperId": "39983a80f111f7f6e793f02c5725a14bca76b32d",4616 "externalIds": {4617 "DBLP": "journals/corr/abs-2504-08066",4618 "ArXiv": "2504.08066",4619 "DOI": "10.48550/arXiv.2504.08066",4620 "CorpusId": 2777411074621 },4622 "publicationVenue": {4623 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",4624 "name": "arXiv.org",4625 "alternate_names": [4626 "ArXiv"4627 ],4628 "issn": "2331-8422",4629 "url": "https://arxiv.org"4630 },4631 "title": "The AI Scientist-v2: Workshop-Level Automated Scientific Discovery via Agentic Tree Search",4632 "venue": "arXiv.org",4633 "year": 2025,4634 "citationCount": 495,4635 "openAccessPdf": {4636 "url": "",4637 "status": null,4638 "license": null,4639 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2504.08066, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."4640 },4641 "publicationTypes": [4642 "JournalArticle",4643 "Review"4644 ],4645 "journal": {4646 "name": "ArXiv",4647 "volume": "abs/2504.08066"4648 },4649 "authors": [4650 {4651 "authorId": "2111746399",4652 "name": "Yutaro Yamada"4653 },4654 {4655 "authorId": "2266838849",4656 "name": "R. Lange"4657 },4658 {4659 "authorId": "2344954958",4660 "name": "Cong Lu"4661 },4662 {4663 "authorId": "2029238065",4664 "name": "Sheng-Ran Hu"4665 },4666 {4667 "authorId": "4420863",4668 "name": "Chris Lu"4669 },4670 {4671 "authorId": "2275156099",4672 "name": "J. Foerster"4673 },4674 {4675 "authorId": "2303255568",4676 "name": "Jeff Clune"4677 },4678 {4679 "authorId": "2315921540",4680 "name": "David Ha"4681 }4682 ],4683 "abstract": "AI is increasingly playing a pivotal role in transforming how scientific discoveries are made. We introduce The AI Scientist-v2, an end-to-end agentic system capable of producing the first entirely AI generated peer-review-accepted workshop paper. This system iteratively formulates scientific hypotheses, designs and executes experiments, analyzes and visualizes data, and autonomously authors scientific manuscripts. Compared to its predecessor (v1, Lu et al., 2024 arXiv:2408.06292), The AI Scientist-v2 eliminates the reliance on human-authored code templates, generalizes effectively across diverse machine learning domains, and leverages a novel progressive agentic tree-search methodology managed by a dedicated experiment manager agent. Additionally, we enhance the AI reviewer component by integrating a Vision-Language Model (VLM) feedback loop for iterative refinement of content and aesthetics of the figures. We evaluated The AI Scientist-v2 by submitting three fully autonomous manuscripts to a peer-reviewed ICLR workshop. Notably, one manuscript achieved high enough scores to exceed the average human acceptance threshold, marking the first instance of a fully AI-generated paper successfully navigating a peer review. This accomplishment highlights the growing capability of AI in conducting all aspects of scientific research. We anticipate that further advancements in autonomous scientific discovery technologies will profoundly impact human knowledge generation, enabling unprecedented scalability in research productivity and significantly accelerating scientific breakthroughs, greatly benefiting society at large. We have open-sourced the code at https://github.com/SakanaAI/AI-Scientist-v2 to foster the future development of this transformative technology. We also discuss the role of AI in science, including AI safety."4684 },4685 "arXiv:2504.13359": {4686 "paperId": "d120bd154a3ec05c2e5c9f3af7ecbf7b0ae11358",4687 "externalIds": {4688 "DBLP": "journals/corr/abs-2504-13359",4689 "ArXiv": "2504.13359",4690 "DOI": "10.48550/arXiv.2504.13359",4691 "CorpusId": 2779403584692 },4693 "publicationVenue": {4694 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",4695 "name": "arXiv.org",4696 "alternate_names": [4697 "ArXiv"4698 ],4699 "issn": "2331-8422",4700 "url": "https://arxiv.org"4701 },4702 "title": "Cost-of-Pass: An Economic Framework for Evaluating Language Models",4703 "venue": "arXiv.org",4704 "year": 2025,4705 "citationCount": 52,4706 "openAccessPdf": {4707 "url": "",4708 "status": null,4709 "license": null,4710 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2504.13359, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."4711 },4712 "publicationTypes": [4713 "JournalArticle"4714 ],4715 "journal": {4716 "name": "ArXiv",4717 "volume": "abs/2504.13359"4718 },4719 "authors": [4720 {4721 "authorId": "19279031",4722 "name": "Mehmet Hamza Erol"4723 },4724 {4725 "authorId": "2356401680",4726 "name": "Batu El"4727 },4728 {4729 "authorId": "51903517",4730 "name": "M. Suzgun"4731 },4732 {4733 "authorId": "2186981598",4734 "name": "Mert Yuksekgonul"4735 },4736 {4737 "authorId": "2357014651",4738 "name": "James Zou"4739 }4740 ],4741 "abstract": "Widespread adoption of AI systems hinges on their ability to generate economic value that outweighs their inference costs. Evaluating this tradeoff requires metrics accounting for both performance and costs. Building on production theory, we develop an economically grounded framework to evaluate language models'productivity by combining accuracy and inference cost. We formalize cost-of-pass: the expected monetary cost of generating a correct solution. We then define the frontier cost-of-pass: the minimum cost-of-pass achievable across available models or the human-expert(s), using the approx. cost of hiring an expert. Our analysis reveals distinct economic insights. First, lightweight models are most cost-effective for basic quantitative tasks, large models for knowledge-intensive ones, and reasoning models for complex quantitative problems, despite higher per-token costs. Second, tracking the frontier cost-of-pass over the past year reveals significant progress, particularly for complex quant. tasks where the cost roughly halved every few months. Third, to trace key innovations driving this progress, we examine counterfactual frontiers -- estimates of cost-efficiency without specific model classes. We find that innovations in lightweight, large, and reasoning models have been essential for pushing the frontier in basic quant., knowledge-intensive, and complex quant. tasks, respectively. Finally, we assess the cost-reductions from common inference-time techniques (majority voting and self-refinement), and a budget-aware technique (TALE-EP). We find that performance-oriented methods with marginal performance gains rarely justify the costs, while TALE-EP shows some promise. Overall, our findings underscore that complementary model-level innovations are the primary drivers of cost-efficiency and our framework provides a principled tool for measuring this progress and guiding deployment."4742 },4743 "arXiv:2504.18765": {4744 "paperId": "c82f67d2a88d082ee28841f133494e60e8d39433",4745 "externalIds": {4746 "ArXiv": "2504.18765",4747 "DBLP": "journals/corr/abs-2504-18765",4748 "DOI": "10.48550/arXiv.2504.18765",4749 "CorpusId": 2781656044750 },4751 "publicationVenue": {4752 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",4753 "name": "arXiv.org",4754 "alternate_names": [4755 "ArXiv"4756 ],4757 "issn": "2331-8422",4758 "url": "https://arxiv.org"4759 },4760 "title": "A Vision for Auto Research with LLM Agents",4761 "venue": "arXiv.org",4762 "year": 2025,4763 "citationCount": 22,4764 "openAccessPdf": {4765 "url": "",4766 "status": null,4767 "license": null,4768 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2504.18765, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."4769 },4770 "publicationTypes": [4771 "JournalArticle",4772 "Review"4773 ],4774 "journal": {4775 "name": "ArXiv",4776 "volume": "abs/2504.18765"4777 },4778 "authors": [4779 {4780 "authorId": "2158155335",4781 "name": "Cheng-Wei Liu"4782 },4783 {4784 "authorId": "2276510804",4785 "name": "Chong Wang"4786 },4787 {4788 "authorId": "2358168030",4789 "name": "Jia-Yue Cao"4790 },4791 {4792 "authorId": "2309156452",4793 "name": "Jing-Quan Ge"4794 },4795 {4796 "authorId": "2345981921",4797 "name": "Kun Wang"4798 },4799 {4800 "authorId": "2492221389",4801 "name": "Lvye Zhang"4802 },4803 {4804 "authorId": "2362307648",4805 "name": "Ming-Ming Cheng"4806 },4807 {4808 "authorId": "2284827556",4809 "name": "Penghai Zhao"4810 },4811 {4812 "authorId": "2265665080",4813 "name": "Tian-Lin Li"4814 },4815 {4816 "authorId": "2275762052",4817 "name": "Xiao-Jun Jia"4818 },4819 {4820 "authorId": "2359961665",4821 "name": "Xiang Li"4822 },4823 {4824 "authorId": "2144405890",4825 "name": "Xin-Feng Li"4826 },4827 {4828 "authorId": "2320865565",4829 "name": "Yang Liu"4830 },4831 {4832 "authorId": "2279401711",4833 "name": "Yebo Feng"4834 },4835 {4836 "authorId": "2142367654",4837 "name": "Yi-Hao Huang"4838 },4839 {4840 "authorId": "2110643465",4841 "name": "Yi-Jia Xu"4842 },4843 {4844 "authorId": "2281788746",4845 "name": "Yu-Qiang Sun"4846 },4847 {4848 "authorId": "2287501692",4849 "name": "Zhe-Xu Zhou"4850 },4851 {4852 "authorId": "3430657",4853 "name": "Zheng-Zi Xu"4854 }4855 ],4856 "abstract": "This paper introduces Agent-Based Auto Research, a structured multi-agent framework designed to automate, coordinate, and optimize the full lifecycle of scientific research. Leveraging the capabilities of large language models (LLMs) and modular agent collaboration, the system spans all major research phases, including literature review, ideation, methodology planning, experimentation, paper writing, peer review response, and dissemination. By addressing issues such as fragmented workflows, uneven methodological expertise, and cognitive overload, the framework offers a systematic and scalable approach to scientific inquiry. Preliminary explorations demonstrate the feasibility and potential of Auto Research as a promising paradigm for self-improving, AI-driven research processes."4857 },4858 "arXiv:2505.13400": {4859 "paperId": "799017d2818108e91392c541579cbe9d2cc38e35",4860 "externalIds": {4861 "ArXiv": "2505.13400",4862 "DBLP": "journals/corr/abs-2505-13400",4863 "DOI": "10.48550/arXiv.2505.13400",4864 "CorpusId": 2787687994865 },4866 "publicationVenue": {4867 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",4868 "name": "arXiv.org",4869 "alternate_names": [4870 "ArXiv"4871 ],4872 "issn": "2331-8422",4873 "url": "https://arxiv.org"4874 },4875 "title": "Robin: A multi-agent system for automating scientific discovery",4876 "venue": "arXiv.org",4877 "year": 2025,4878 "citationCount": 88,4879 "openAccessPdf": {4880 "url": "",4881 "status": null,4882 "license": null,4883 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2505.13400, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."4884 },4885 "publicationTypes": [4886 "JournalArticle"4887 ],4888 "journal": {4889 "name": "ArXiv",4890 "volume": "abs/2505.13400"4891 },4892 "authors": [4893 {4894 "authorId": "2288198964",4895 "name": "A. Ghareeb"4896 },4897 {4898 "authorId": "2372457795",4899 "name": "Benjamin Chang"4900 },4901 {4902 "authorId": "2115040667",4903 "name": "L. Mitchener"4904 },4905 {4906 "authorId": "2362501842",4907 "name": "Angela Yiu"4908 },4909 {4910 "authorId": "114013157",4911 "name": "Caralyn J. Szostkiewicz"4912 },4913 {4914 "authorId": "2311439808",4915 "name": "Jon M. Laurent"4916 },4917 {4918 "authorId": "2117637436",4919 "name": "Muhammed Razzak"4920 },4921 {4922 "authorId": "2348512359",4923 "name": "Andrew D. White"4924 },4925 {4926 "authorId": "1395364165",4927 "name": "Michaela M. Hinks"4928 },4929 {4930 "authorId": "1924444",4931 "name": "Samuel G. Rodriques"4932 }4933 ],4934 "abstract": "Scientific discovery is driven by the iterative process of background research, hypothesis generation, experimentation, and data analysis. Despite recent advancements in applying artificial intelligence to scientific discovery, no system has yet automated all of these stages in a single workflow. Here, we introduce Robin, the first multi-agent system capable of fully automating the key intellectual steps of the scientific process. By integrating literature search agents with data analysis agents, Robin can generate hypotheses, propose experiments, interpret experimental results, and generate updated hypotheses, achieving a semi-autonomous approach to scientific discovery. By applying this system, we were able to identify a novel treatment for dry age-related macular degeneration (dAMD), the major cause of blindness in the developed world. Robin proposed enhancing retinal pigment epithelium phagocytosis as a therapeutic strategy, and identified and validated a promising therapeutic candidate, ripasudil. Ripasudil is a clinically-used rho kinase (ROCK) inhibitor that has never previously been proposed for treating dAMD. To elucidate the mechanism of ripasudil-induced upregulation of phagocytosis, Robin then proposed and analyzed a follow-up RNA-seq experiment, which revealed upregulation of ABCA1, a critical lipid efflux pump and possible novel target. All hypotheses, experimental plans, data analyses, and data figures in the main text of this report were produced by Robin. As the first AI system to autonomously discover and validate a novel therapeutic candidate within an iterative lab-in-the-loop framework, Robin establishes a new paradigm for AI-driven scientific discovery."4935 },4936 "arXiv:2505.18705": {4937 "paperId": "80a0b76dedc4c3e3d365bbaececcd44a996eb38b",4938 "externalIds": {4939 "ArXiv": "2505.18705",4940 "DBLP": "journals/corr/abs-2505-18705",4941 "DOI": "10.48550/arXiv.2505.18705",4942 "CorpusId": 2789054064943 },4944 "publicationVenue": {4945 "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",4946 "name": "Neural Information Processing Systems",4947 "type": "conference",4948 "alternate_names": [4949 "Neural Inf Process Syst",4950 "NeurIPS",4951 "NIPS"4952 ],4953 "url": "http://neurips.cc/"4954 },4955 "title": "AI-Researcher: Autonomous Scientific Innovation",4956 "venue": "Neural Information Processing Systems",4957 "year": 2025,4958 "citationCount": 114,4959 "openAccessPdf": {4960 "url": "",4961 "status": null,4962 "license": null,4963 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2505.18705, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."4964 },4965 "publicationTypes": [4966 "JournalArticle",4967 "Review"4968 ],4969 "journal": {4970 "name": "ArXiv",4971 "volume": "abs/2505.18705"4972 },4973 "authors": [4974 {4975 "authorId": "2261087798",4976 "name": "Jiabin Tang"4977 },4978 {4979 "authorId": "1830455155",4980 "name": "Lianghao Xia"4981 },4982 {4983 "authorId": "2215613396",4984 "name": "Zhonghang Li"4985 },4986 {4987 "authorId": "2344760658",4988 "name": "Chao Huang"4989 }4990 ],4991 "abstract": "The powerful reasoning capabilities of Large Language Models (LLMs) in mathematics and coding, combined with their ability to automate complex tasks through agentic frameworks, present unprecedented opportunities for accelerating scientific innovation. In this paper, we introduce AI-Researcher, a fully autonomous research system that transforms how AI-driven scientific discovery is conducted and evaluated. Our framework seamlessly orchestrates the complete research pipeline--from literature review and hypothesis generation to algorithm implementation and publication-ready manuscript preparation--with minimal human intervention. To rigorously assess autonomous research capabilities, we develop Scientist-Bench, a comprehensive benchmark comprising state-of-the-art papers across diverse AI research domains, featuring both guided innovation and open-ended exploration tasks. Through extensive experiments, we demonstrate that AI-Researcher achieves remarkable implementation success rates and produces research papers that approach human-level quality. This work establishes new foundations for autonomous scientific innovation that can complement human researchers by systematically exploring solution spaces beyond cognitive limitations."4992 },4993 "arXiv:2507.02825": {4994 "paperId": "eeff713967bfd94da50e4cc8fda888b01f137e90",4995 "externalIds": {4996 "ArXiv": "2507.02825",4997 "DBLP": "journals/corr/abs-2507-02825",4998 "DOI": "10.48550/arXiv.2507.02825",4999 "CorpusId": 2800915435000 },5001 "publicationVenue": {5002 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",5003 "name": "arXiv.org",5004 "alternate_names": [5005 "ArXiv"5006 ],5007 "issn": "2331-8422",5008 "url": "https://arxiv.org"5009 },5010 "title": "Establishing Best Practices for Building Rigorous Agentic Benchmarks",5011 "venue": "arXiv.org",5012 "year": 2025,5013 "citationCount": 124,5014 "openAccessPdf": {5015 "url": "",5016 "status": null,5017 "license": null,5018 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2507.02825, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."5019 },5020 "publicationTypes": [5021 "JournalArticle",5022 "Review"5023 ],5024 "journal": {5025 "name": "ArXiv",5026 "volume": "abs/2507.02825"5027 },5028 "authors": [5029 {5030 "authorId": "2351716372",5031 "name": "Yuxuan Zhu"5032 },5033 {5034 "authorId": "2303902750",5035 "name": "Teng-Jun Jin"5036 },5037 {5038 "authorId": "100984698",5039 "name": "Yada Pruksachatkun"5040 },5041 {5042 "authorId": "2316522831",5043 "name": "Andy K. Zhang"5044 },5045 {5046 "authorId": "2344195930",5047 "name": "Shu Liu"5048 },5049 {5050 "authorId": "2374300501",5051 "name": "Sasha Cui"5052 },5053 {5054 "authorId": "39893263",5055 "name": "Sayash Kapoor"5056 },5057 {5058 "authorId": "2283848744",5059 "name": "Shayne Longpre"5060 },5061 {5062 "authorId": "2350413159",5063 "name": "Kevin Meng"5064 },5065 {5066 "authorId": "2297190630",5067 "name": "Rebecca Weiss"5068 },5069 {5070 "authorId": "2143198655",5071 "name": "Fazl Barez"5072 },5073 {5074 "authorId": "2384860310",5075 "name": "Rahul Gupta"5076 },5077 {5078 "authorId": "3475586",5079 "name": "J. Dhamala"5080 },5081 {5082 "authorId": "2191689389",5083 "name": "Jacob Merizian"5084 },5085 {5086 "authorId": "2372512852",5087 "name": "Mario Giulianelli"5088 },5089 {5090 "authorId": "2043233315",5091 "name": "H. Coppock"5092 },5093 {5094 "authorId": "11285234",5095 "name": "C. Ududec"5096 },5097 {5098 "authorId": "2275054630",5099 "name": "Jasjeet S. Sekhon"5100 },5101 {5102 "authorId": "2327340186",5103 "name": "Jacob Steinhardt"5104 },5105 {5106 "authorId": "2372300510",5107 "name": "A. Kellerman"5108 },5109 {5110 "authorId": "2343632085",5111 "name": "Sarah Schwettmann"5112 },5113 {5114 "authorId": "2221547236",5115 "name": "Matei A. Zaharia"5116 },5117 {5118 "authorId": "2295665819",5119 "name": "Ion Stoica"5120 },5121 {5122 "authorId": "2347224361",5123 "name": "Percy Liang"5124 },5125 {5126 "authorId": "2352791966",5127 "name": "Daniel Kang"5128 }5129 ],5130 "abstract": "Benchmarks are essential for quantitatively tracking progress in AI. As AI agents become increasingly capable, researchers and practitioners have introduced agentic benchmarks to evaluate agents on complex, real-world tasks. These benchmarks typically measure agent capabilities by evaluating task outcomes via specific reward designs. However, we show that many agentic benchmarks have issues in task setup or reward design. For example, SWE-bench Verified uses insufficient test cases, while TAU-bench counts empty responses as successful. Such issues can lead to under- or overestimation of agents'performance by up to 100% in relative terms. To make agentic evaluation rigorous, we introduce the Agentic Benchmark Checklist (ABC), a set of guidelines that we synthesized from our benchmark-building experience, a survey of best practices, and previously reported issues. When applied to CVE-Bench, a benchmark with a particularly complex evaluation design, ABC reduces the performance overestimation by 33%."5131 },5132 "arXiv:2507.09089": {5133 "paperId": "9008680aac5a92b3a089aa1487eea76b8565f0d3",5134 "externalIds": {5135 "DBLP": "journals/corr/abs-2507-09089",5136 "ArXiv": "2507.09089",5137 "DOI": "10.48550/arXiv.2507.09089",5138 "CorpusId": 2800456085139 },5140 "publicationVenue": {5141 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",5142 "name": "arXiv.org",5143 "alternate_names": [5144 "ArXiv"5145 ],5146 "issn": "2331-8422",5147 "url": "https://arxiv.org"5148 },5149 "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",5150 "venue": "arXiv.org",5151 "year": 2025,5152 "citationCount": 232,5153 "openAccessPdf": {5154 "url": "",5155 "status": null,5156 "license": null,5157 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2507.09089, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."5158 },5159 "publicationTypes": [5160 "JournalArticle"5161 ],5162 "journal": {5163 "name": "ArXiv",5164 "volume": "abs/2507.09089"5165 },5166 "authors": [5167 {5168 "authorId": "2331857910",5169 "name": "Joel Becker"5170 },5171 {5172 "authorId": "2350754555",5173 "name": "Nate Rush"5174 },5175 {5176 "authorId": "2331855799",5177 "name": "Elizabeth Barnes"5178 },5179 {5180 "authorId": "2266751602",5181 "name": "David Rein"5182 }5183 ],5184 "abstract": "Despite widespread adoption, the impact of AI tools on software development in the wild remains understudied. We conduct a randomized controlled trial (RCT) to understand how AI tools at the February-June 2025 frontier affect the productivity of experienced open-source developers. 16 developers with moderate AI experience complete 246 tasks in mature projects on which they have an average of 5 years of prior experience. Each task is randomly assigned to allow or disallow usage of early 2025 AI tools. When AI tools are allowed, developers primarily use Cursor Pro, a popular code editor, and Claude 3.5/3.7 Sonnet. Before starting tasks, developers forecast that allowing AI will reduce completion time by 24%. After completing the study, developers estimate that allowing AI reduced completion time by 20%. Surprisingly, we find that allowing AI actually increases completion time by 19%--AI tooling slowed developers down. This slowdown also contradicts predictions from experts in economics (39% shorter) and ML (38% shorter). To understand this result, we collect and evaluate evidence for 20 properties of our setting that a priori could contribute to the observed slowdown effect--for example, the size and quality standards of projects, or prior developer experience with AI tooling. Although the influence of experimental artifacts cannot be entirely ruled out, the robustness of the slowdown effect across our analyses suggests it is unlikely to primarily be a function of our experimental design."5185 },5186 "arXiv:2509.26603": {5187 "paperId": "f3da28bbb488ded8a7d1e1f71a2c4810da38eb20",5188 "externalIds": {5189 "ArXiv": "2509.26603",5190 "DBLP": "journals/corr/abs-2509-26603",5191 "DOI": "10.48550/arXiv.2509.26603",5192 "CorpusId": 2816811965193 },5194 "publicationVenue": {5195 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",5196 "name": "arXiv.org",5197 "alternate_names": [5198 "ArXiv"5199 ],5200 "issn": "2331-8422",5201 "url": "https://arxiv.org"5202 },5203 "title": "DeepScientist: Advancing Frontier-Pushing Scientific Findings Progressively",5204 "venue": "arXiv.org",5205 "year": 2025,5206 "citationCount": 69,5207 "openAccessPdf": {5208 "url": "",5209 "status": null,5210 "license": null,5211 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2509.26603, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."5212 },5213 "publicationTypes": [5214 "JournalArticle"5215 ],5216 "journal": {5217 "name": "ArXiv",5218 "volume": "abs/2509.26603"5219 },5220 "authors": [5221 {5222 "authorId": "2374151012",5223 "name": "Yi-Xuan Weng"5224 },5225 {5226 "authorId": "2316827669",5227 "name": "Min-Jun Zhu"5228 },5229 {5230 "authorId": "2342406856",5231 "name": "Qiu-Jie Xie"5232 },5233 {5234 "authorId": "2383102227",5235 "name": "Qi-Yao Sun"5236 },5237 {5238 "authorId": "2364802150",5239 "name": "Zhen Lin"5240 },5241 {5242 "authorId": "2376557334",5243 "name": "Sifan Liu"5244 },5245 {5246 "authorId": "2374438885",5247 "name": "Yue Zhang"5248 }5249 ],5250 "abstract": "While previous AI Scientist systems can generate novel findings, they often lack the focus to produce scientifically valuable contributions that address pressing human-defined challenges. We introduce DeepScientist, a system designed to overcome this by conducting goal-oriented, fully autonomous scientific discovery over month-long timelines. It formalizes discovery as a Bayesian Optimization problem, operationalized through a hierarchical evaluation process consisting of\"hypothesize, verify, and analyze\". Leveraging a cumulative Findings Memory, this loop intelligently balances the exploration of novel hypotheses with exploitation, selectively promoting the most promising findings to higher-fidelity levels of validation. Consuming over 20,000 GPU hours, the system generated about 5,000 unique scientific ideas and experimentally validated approximately 1100 of them, ultimately surpassing human-designed state-of-the-art (SOTA) methods on three frontier AI tasks by 183.7\\%, 1.9\\%, and 7.9\\%. This work provides the first large-scale evidence of an AI achieving discoveries that progressively surpass human SOTA on scientific tasks, producing valuable findings that genuinely push the frontier of scientific discovery. To facilitate further research into this process, we will open-source all experimental logs and system code at https://github.com/ResearAI/DeepScientist/."5251 },5252 "arXiv:2510.11977": {5253 "paperId": "92f110c93e9c9990129dfbf601b8d7254855dedf",5254 "externalIds": {5255 "DBLP": "journals/corr/abs-2510-11977",5256 "ArXiv": "2510.11977",5257 "DOI": "10.48550/arXiv.2510.11977",5258 "CorpusId": 2820643725259 },5260 "publicationVenue": {5261 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",5262 "name": "arXiv.org",5263 "alternate_names": [5264 "ArXiv"5265 ],5266 "issn": "2331-8422",5267 "url": "https://arxiv.org"5268 },5269 "title": "Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation",5270 "venue": "arXiv.org",5271 "year": 2025,5272 "citationCount": 116,5273 "openAccessPdf": {5274 "url": "",5275 "status": null,5276 "license": null,5277 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2510.11977, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."5278 },5279 "publicationTypes": [5280 "JournalArticle"5281 ],5282 "journal": {5283 "name": "ArXiv",5284 "volume": "abs/2510.11977"5285 },5286 "authors": [5287 {5288 "authorId": "39893263",5289 "name": "Sayash Kapoor"5290 },5291 {5292 "authorId": "2309175570",5293 "name": "Benedikt Stroebl"5294 },5295 {5296 "authorId": "2392881135",5297 "name": "Peter Kirgis"5298 },5299 {5300 "authorId": "2290073765",5301 "name": "Nitya Nadgir"5302 },5303 {5304 "authorId": "2274102473",5305 "name": "Zachary S. Siegel"5306 },5307 {5308 "authorId": "2283309803",5309 "name": "Boyi Wei"5310 },5311 {5312 "authorId": "2353284385",5313 "name": "Tianci Xue"5314 },5315 {5316 "authorId": "2385761588",5317 "name": "Zi-Ru Chen"5318 },5319 {5320 "authorId": "2385732101",5321 "name": "Felix Chen"5322 },5323 {5324 "authorId": "1512255229",5325 "name": "Saiteja Utpala"5326 },5327 {5328 "authorId": "2385560965",5329 "name": "Franck Ndzomga"5330 },5331 {5332 "authorId": "2322217725",5333 "name": "Dheeraj Oruganty"5334 },5335 {5336 "authorId": "2385560963",5337 "name": "Sophie Luskin"5338 },5339 {5340 "authorId": "2386171231",5341 "name": "Kangheng Liu"5342 },5343 {5344 "authorId": "2268658397",5345 "name": "Bo-Tao Yu"5346 },5347 {5348 "authorId": "2283236744",5349 "name": "Amit Arora"5350 },5351 {5352 "authorId": "2327211159",5353 "name": "Dongyoon Hahm"5354 },5355 {5356 "authorId": "6365809",5357 "name": "H. Trivedi"5358 },5359 {5360 "authorId": "2386587363",5361 "name": "Huan Sun"5362 },5363 {5364 "authorId": "2298402035",5365 "name": "Ju-Yong Lee"5366 },5367 {5368 "authorId": "2303902750",5369 "name": "Teng-Jun Jin"5370 },5371 {5372 "authorId": "2054708905",5373 "name": "Yifan Mai"5374 },5375 {5376 "authorId": "2349674686",5377 "name": "Yifei Zhou"5378 },5379 {5380 "authorId": "2351716372",5381 "name": "Yuxuan Zhu"5382 },5383 {5384 "authorId": "2223138553",5385 "name": "Rishi Bommasani"5386 },5387 {5388 "authorId": "2352791966",5389 "name": "Daniel Kang"5390 },5391 {5392 "authorId": "2374189943",5393 "name": "D. Song"5394 },5395 {5396 "authorId": "2363497261",5397 "name": "Peter Henderson"5398 },5399 {5400 "authorId": "1758652",5401 "name": "Yu Su"5402 },5403 {5404 "authorId": "2347512172",5405 "name": "Percy Liang"5406 },5407 {5408 "authorId": "2355353099",5409 "name": "Arvind Narayanan"5410 }5411 ],5412 "abstract": "AI agents have been developed for complex real-world tasks from coding to customer service. But AI agent evaluations suffer from many challenges that undermine our understanding of how well agents really work. We introduce the Holistic Agent Leaderboard (HAL) to address these challenges. We make three main contributions. First, we provide a standardized evaluation harness that orchestrates parallel evaluations across hundreds of VMs, reducing evaluation time from weeks to hours while eliminating common implementation bugs. Second, we conduct three-dimensional analysis spanning models, scaffolds, and benchmarks. We validate the harness by conducting 21,730 agent rollouts across 9 models and 9 benchmarks in coding, web navigation, science, and customer service with a total cost of about $40,000. Our analysis reveals surprising insights, such as higher reasoning effort reducing accuracy in the majority of runs. Third, we use LLM-aided log inspection to uncover previously unreported behaviors, such as searching for the benchmark on HuggingFace instead of solving a task, or misusing credit cards in flight booking tasks. We share all agent logs, comprising 2.5B tokens of language model calls, to incentivize further research into agent behavior. By standardizing how the field evaluates agents and addressing common pitfalls in agent evaluation, we hope to shift the focus from agents that ace benchmarks to agents that work reliably in the real world."5413 },5414 "arXiv:2511.02824": {5415 "paperId": "0fc30d552b48b12817ba6ec4dfa9571b2a262b8d",5416 "externalIds": {5417 "ArXiv": "2511.02824",5418 "DBLP": "journals/corr/abs-2511-02824",5419 "DOI": "10.48550/arXiv.2511.02824",5420 "CorpusId": 2827488275421 },5422 "publicationVenue": {5423 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",5424 "name": "arXiv.org",5425 "alternate_names": [5426 "ArXiv"5427 ],5428 "issn": "2331-8422",5429 "url": "https://arxiv.org"5430 },5431 "title": "Kosmos: An AI Scientist for Autonomous Discovery",5432 "venue": "arXiv.org",5433 "year": 2025,5434 "citationCount": 121,5435 "openAccessPdf": {5436 "url": "",5437 "status": null,5438 "license": null,5439 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2511.02824, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."5440 },5441 "publicationTypes": [5442 "JournalArticle"5443 ],5444 "journal": {5445 "name": "ArXiv",5446 "volume": "abs/2511.02824"5447 },5448 "authors": [5449 {5450 "authorId": "2115040667",5451 "name": "L. Mitchener"5452 },5453 {5454 "authorId": "2362501842",5455 "name": "Angela Yiu"5456 },5457 {5458 "authorId": "2372457795",5459 "name": "Benjamin Chang"5460 },5461 {5462 "authorId": "2337087923",5463 "name": "M. Bourdenx"5464 },5465 {5466 "authorId": "2390494493",5467 "name": "Tyler Nadolski"5468 },5469 {5470 "authorId": "49735802",5471 "name": "Arvis Sulovari"5472 },5473 {5474 "authorId": "6808942",5475 "name": "E. Landsness"5476 },5477 {5478 "authorId": "36956411",5479 "name": "D\u00e1niel L. Barab\u00e1si"5480 },5481 {5482 "authorId": "2311445200",5483 "name": "Siddharth Narayanan"5484 },5485 {5486 "authorId": "2380028900",5487 "name": "Nicky Evans"5488 },5489 {5490 "authorId": "2392220836",5491 "name": "S. Reddy"5492 },5493 {5494 "authorId": "2331311827",5495 "name": "M. Foiani"5496 },5497 {5498 "authorId": "2183418673",5499 "name": "Aizad Kamal"5500 },5501 {5502 "authorId": "4291840",5503 "name": "L. Shriver"5504 },5505 {5506 "authorId": "2390730337",5507 "name": "F. Cao"5508 },5509 {5510 "authorId": "8624819",5511 "name": "A. Wassie"5512 },5513 {5514 "authorId": "2311439808",5515 "name": "Jon M. Laurent"5516 },5517 {5518 "authorId": "2390494299",5519 "name": "Edwin Melville-Green"5520 },5521 {5522 "authorId": "89892165",5523 "name": "M. Caldas"5524 },5525 {5526 "authorId": "2337796843",5527 "name": "Albert Bou"5528 },5529 {5530 "authorId": "2338378584",5531 "name": "Kaleigh F. Roberts"5532 },5533 {5534 "authorId": "49106960",5535 "name": "Sladjana Zagorac"5536 },5537 {5538 "authorId": "2144762732",5539 "name": "Timothy C. Orr"5540 },5541 {5542 "authorId": "2238439648",5543 "name": "Miranda E. Orr"5544 },5545 {5546 "authorId": "4590892",5547 "name": "Kevin J. Zwezdaryk"5548 },5549 {5550 "authorId": "2288198964",5551 "name": "A. Ghareeb"5552 },5553 {5554 "authorId": "2342363627",5555 "name": "L. McCoy"5556 },5557 {5558 "authorId": "2398715364",5559 "name": "B. Gomes"5560 },5561 {5562 "authorId": "2220825270",5563 "name": "Euan A. Ashley"5564 },5565 {5566 "authorId": "2259207100",5567 "name": "K. Duff"5568 },5569 {5570 "authorId": "6018731",5571 "name": "T. Buonassisi"5572 },5573 {5574 "authorId": "2358794",5575 "name": "Tom Rainforth"5576 },5577 {5578 "authorId": "2308325076",5579 "name": "R. Bateman"5580 },5581 {5582 "authorId": "2322440716",5583 "name": "Michael Skarlinski"5584 },5585 {5586 "authorId": "1924444",5587 "name": "Samuel G. Rodriques"5588 },5589 {5590 "authorId": "1395364165",5591 "name": "Michaela M. Hinks"5592 },5593 {5594 "authorId": "2354508425",5595 "name": "Andrew D. White"5596 }5597 ],5598 "abstract": "Data-driven scientific discovery requires iterative cycles of literature search, hypothesis generation, and data analysis. Substantial progress has been made towards AI agents that can automate scientific research, but all such agents remain limited in the number of actions they can take before losing coherence, thus limiting the depth of their findings. Here we present Kosmos, an AI scientist that automates data-driven discovery. Given an open-ended objective and a dataset, Kosmos runs for up to 12 hours performing cycles of parallel data analysis, literature search, and hypothesis generation before synthesizing discoveries into scientific reports. Unlike prior systems, Kosmos uses a structured world model to share information between a data analysis agent and a literature search agent. The world model enables Kosmos to coherently pursue the specified objective over 200 agent rollouts, collectively executing an average of 42,000 lines of code and reading 1,500 papers per run. Kosmos cites all statements in its reports with code or primary literature, ensuring its reasoning is traceable. Independent scientists found 79.4% of statements in Kosmos reports to be accurate, and collaborators reported that a single 20-cycle Kosmos run performed the equivalent of 6 months of their own research time on average. Furthermore, collaborators reported that the number of valuable scientific findings generated scales linearly with Kosmos cycles (tested up to 20 cycles). We highlight seven discoveries made by Kosmos that span metabolomics, materials science, neuroscience, and statistical genetics. Three discoveries independently reproduce findings from preprinted or unpublished manuscripts that were not accessed by Kosmos at runtime, while four make novel contributions to the scientific literature."5599 },5600 "arXiv:2601.11868": {5601 "paperId": "ef0fcf2f99c74918ceed21f529c1b99f782ac33e",5602 "externalIds": {5603 "DBLP": "journals/corr/abs-2601-11868",5604 "ArXiv": "2601.11868",5605 "DOI": "10.48550/arXiv.2601.11868",5606 "CorpusId": 2849118575607 },5608 "publicationVenue": {5609 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",5610 "name": "arXiv.org",5611 "alternate_names": [5612 "ArXiv"5613 ],5614 "issn": "2331-8422",5615 "url": "https://arxiv.org"5616 },5617 "title": "Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces",5618 "venue": "arXiv.org",5619 "year": 2026,5620 "citationCount": 671,5621 "openAccessPdf": {5622 "url": "",5623 "status": null,5624 "license": null,5625 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2601.11868, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."5626 },5627 "publicationTypes": [5628 "JournalArticle"5629 ],5630 "journal": {5631 "name": "ArXiv",5632 "volume": "abs/2601.11868"5633 },5634 "authors": [5635 {5636 "authorId": "2297185552",5637 "name": "Mike A. Merrill"5638 },5639 {5640 "authorId": "2312442525",5641 "name": "Alexander G Shaw"5642 },5643 {5644 "authorId": "2365115789",5645 "name": "Nicholas Carlini"5646 },5647 {5648 "authorId": "2394091201",5649 "name": "Boxuan Li"5650 },5651 {5652 "authorId": "100947321",5653 "name": "H. Raj"5654 },5655 {5656 "authorId": "2366018523",5657 "name": "I. Bercovich"5658 },5659 {5660 "authorId": "2305925735",5661 "name": "Lin Shi"5662 },5663 {5664 "authorId": "2316095027",5665 "name": "J. Shin"5666 },5667 {5668 "authorId": "2389332288",5669 "name": "Thomas Walshe"5670 },5671 {5672 "authorId": "2336877105",5673 "name": "E. K. Buchanan"5674 },5675 {5676 "authorId": "2269118801",5677 "name": "Junhong Shen"5678 },5679 {5680 "authorId": "1753871691",5681 "name": "Guanghao Ye"5682 },5683 {5684 "authorId": "2372664232",5685 "name": "Hao Lin"5686 },5687 {5688 "authorId": "7245508",5689 "name": "Jason Poulos"5690 },5691 {5692 "authorId": "2288752126",5693 "name": "Mao-Yu Wang"5694 },5695 {5696 "authorId": "2174178585",5697 "name": "Marianna Nezhurina"5698 },5699 {5700 "authorId": "2191688",5701 "name": "J. Jitsev"5702 },5703 {5704 "authorId": "2387129493",5705 "name": "Di Lu"5706 },5707 {5708 "authorId": "1978772089",5709 "name": "Orfeas Menis-Mastromichalakis"5710 },5711 {5712 "authorId": "2254325443",5713 "name": "Zhiwei Xu"5714 },5715 {5716 "authorId": "2326326988",5717 "name": "Zizhao Chen"5718 },5719 {5720 "authorId": "2283355537",5721 "name": "Yue Liu"5722 },5723 {5724 "authorId": "2119062534",5725 "name": "Robert Zhang"5726 },5727 {5728 "authorId": "2146073251",5729 "name": "L. Chen"5730 },5731 {5732 "authorId": "2052965045",5733 "name": "Anurag Kashyap"5734 },5735 {5736 "authorId": "2220631332",5737 "name": "Jan-Lucas Uslu"5738 },5739 {5740 "authorId": "2287821949",5741 "name": "Jeffrey Li"5742 },5743 {5744 "authorId": "2302781884",5745 "name": "Jianbo Wu"5746 },5747 {5748 "authorId": "2263988785",5749 "name": "Minghao Yan"5750 },5751 {5752 "authorId": "2342276462",5753 "name": "Song Bian"5754 },5755 {5756 "authorId": "2406484417",5757 "name": "Vedang Sharma"5758 },5759 {5760 "authorId": "2407816422",5761 "name": "Ke Sun"5762 },5763 {5764 "authorId": "2333363742",5765 "name": "S. Dillmann"5766 },5767 {5768 "authorId": "2313160570",5769 "name": "Akshay Anand"5770 },5771 {5772 "authorId": "2312202594",5773 "name": "Andrew Lanpouthakoun"5774 },5775 {5776 "authorId": "2405816209",5777 "name": "Bardia Koopah"5778 },5779 {5780 "authorId": "2295937151",5781 "name": "Chang-Ran Hu"5782 },5783 {5784 "authorId": "2179104354",5785 "name": "E. Guha"5786 },5787 {5788 "authorId": "2325727722",5789 "name": "Gabriel H. S. Dreiman"5790 },5791 {5792 "authorId": "1870353",5793 "name": "Jia-Cheng Zhu"5794 },5795 {5796 "authorId": "2265975127",5797 "name": "Karl Krauth"5798 },5799 {5800 "authorId": "2287937401",5801 "name": "Li Zhong"5802 },5803 {5804 "authorId": "2037383772",5805 "name": "Niklas Muennighoff"5806 },5807 {5808 "authorId": "4798533",5809 "name": "Robert K. Amanfu"5810 },5811 {5812 "authorId": "2275830490",5813 "name": "Shang-Yin Tan"5814 },5815 {5816 "authorId": "2187585049",5817 "name": "Shreyas Pimpalgaonkar"5818 },5819 {5820 "authorId": "2130626178",5821 "name": "Tushar Aggarwal"5822 },5823 {5824 "authorId": "145724540",5825 "name": "Xiang-Ning Lin"5826 },5827 {5828 "authorId": "2405714069",5829 "name": "Xin Lan"5830 },5831 {5832 "authorId": "150345512",5833 "name": "Xuandong Zhao"5834 },5835 {5836 "authorId": "2257367768",5837 "name": "Yi-Qing Liang"5838 },5839 {5840 "authorId": "2444709630",5841 "name": "Yuan-Li Wang"5842 },5843 {5844 "authorId": "2265487043",5845 "name": "Zi-Long Wang"5846 },5847 {5848 "authorId": "2300675202",5849 "name": "Chang-Zhi Zhou"5850 },5851 {5852 "authorId": "2287413575",5853 "name": "D. Heineman"5854 },5855 {5856 "authorId": "2385506498",5857 "name": "Hange Liu"5858 },5859 {5860 "authorId": "6365809",5861 "name": "H. Trivedi"5862 },5863 {5864 "authorId": "2346284658",5865 "name": "John Yang"5866 },5867 {5868 "authorId": "2311427666",5869 "name": "Jun-Hong Lin"5870 },5871 {5872 "authorId": "2275599530",5873 "name": "Manish Shetty"5874 },5875 {5876 "authorId": "2110603190",5877 "name": "Michael Yang"5878 },5879 {5880 "authorId": "2328412793",5881 "name": "Nabil Omi"5882 },5883 {5884 "authorId": "2065899387",5885 "name": "Negin Raoof"5886 },5887 {5888 "authorId": "2257057803",5889 "name": "Shanda Li"5890 },5891 {5892 "authorId": "2080123731",5893 "name": "Terry Yue Zhuo"5894 },5895 {5896 "authorId": "2112406019",5897 "name": "Wu Lin"5898 },5899 {5900 "authorId": "2321354925",5901 "name": "Yi-Wei Dai"5902 },5903 {5904 "authorId": "2504582353",5905 "name": "Yu-Xin Wang"5906 },5907 {5908 "authorId": "2324574706",5909 "name": "Wen-Hao Chai"5910 },5911 {5912 "authorId": "2367092694",5913 "name": "Shang Zhou"5914 },5915 {5916 "authorId": "2230414556",5917 "name": "Dariush Wahdany"5918 },5919 {5920 "authorId": "2126990654",5921 "name": "Zi-Yu She"5922 },5923 {5924 "authorId": "2391004673",5925 "name": "Jia-Ming Hu"5926 },5927 {5928 "authorId": "2386598941",5929 "name": "Zhikang Dong"5930 },5931 {5932 "authorId": "2405833335",5933 "name": "Yuxuan Zhu"5934 },5935 {5936 "authorId": "2374300501",5937 "name": "Sasha Cui"5938 },5939 {5940 "authorId": "2033775348",5941 "name": "Ahson Saiyed"5942 },5943 {5944 "authorId": null,5945 "name": "Arinbj\u00f6rn Kolbeinsson"5946 },5947 {5948 "authorId": "2479368557",5949 "name": "Jesse Hu"5950 },5951 {5952 "authorId": "2131041347",5953 "name": "Christopher Rytting"5954 },5955 {5956 "authorId": "2365261533",5957 "name": "Ryan Marten"5958 },5959 {5960 "authorId": "2445500129",5961 "name": "Yi-Xin Wang"5962 },5963 {5964 "authorId": "2257301126",5965 "name": "A. Dimakis"5966 },5967 {5968 "authorId": "2371549",5969 "name": "A. Konwinski"5970 },5971 {5972 "authorId": "2352216817",5973 "name": "Ludwig Schmidt"5974 }5975 ],5976 "abstract": "AI agents may soon become capable of autonomously completing valuable, long-horizon tasks in diverse domains. Current benchmarks either do not measure real-world tasks, or are not sufficiently difficult to meaningfully measure frontier models. To this end, we present Terminal-Bench 2.0: a carefully curated hard benchmark composed of 89 tasks in computer terminal environments inspired by problems from real workflows. Each task features a unique environment, human-written solution, and comprehensive tests for verification. We show that frontier models and agents score less than 65\\% on the benchmark and conduct an error analysis to identify areas for model and agent improvement. We publish the dataset and evaluation harness to assist developers and researchers in future work at https://www.tbench.ai/ ."5977 },5978 "arXiv:2601.18724": {5979 "paperId": "737546bba8ed4daec51b92de19c1d8c0080aea92",5980 "externalIds": {5981 "DBLP": "journals/corr/abs-2601-18724",5982 "ArXiv": "2601.18724",5983 "DOI": "10.48550/arXiv.2601.18724",5984 "CorpusId": 2850504765985 },5986 "publicationVenue": {5987 "id": "1e33b3be-b2ab-46e9-96e8-d4eb4bad6e44",5988 "name": "Annual Meeting of the Association for Computational Linguistics",5989 "type": "conference",5990 "alternate_names": [5991 "Annu Meet Assoc Comput Linguistics",5992 "Meeting of the Association for Computational Linguistics",5993 "ACL",5994 "Meet Assoc Comput Linguistics"5995 ],5996 "url": "https://www.aclweb.org/anthology/venues/acl/"5997 },5998 "title": "HalluCitation Matters: Revealing the Impact of Hallucinated References with 300 Hallucinated Papers in ACL Conferences",5999 "venue": "Annual Meeting of the Association for Computational Linguistics",6000 "year": 2026,6001 "citationCount": 33,6002 "openAccessPdf": {6003 "url": "",6004 "status": null,6005 "license": null,6006 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2601.18724, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6007 },6008 "publicationTypes": [6009 "JournalArticle",6010 "Conference",6011 "Review"6012 ],6013 "journal": {6014 "pages": "47295-47376"6015 },6016 "authors": [6017 {6018 "authorId": "2204051777",6019 "name": "Yusuke Sakai"6020 },6021 {6022 "authorId": "2391550956",6023 "name": "Hidetaka Kamigaito"6024 },6025 {6026 "authorId": "2266807418",6027 "name": "Taro Watanabe"6028 }6029 ],6030 "abstract": "Recently, we have often observed hallucinated citations or references that do not correspond to any existing work in papers under review, preprints, or published papers. Such hallucinated citations pose a serious concern to scientific reliability. When they appear in accepted papers, they may also negatively affect the credibility of conferences. In this study, we refer to hallucinated citations as\"HalluCitation\"and systematically investigate their prevalence and impact. We analyze all papers published at ACL, NAACL, and EMNLP in 2024 and 2025, including main conference, Findings, and workshop papers. Our analysis reveals that nearly 300 papers contain at least one HalluCitation, most of which were published in 2025. Notably, half of these papers were identified at EMNLP 2025, the most recent conference, indicating that this issue is rapidly increasing. Moreover, more than 100 such papers were accepted as main conference and Findings papers at EMNLP 2025, affecting the credibility."6031 },6032 "arXiv:2602.08990": {6033 "paperId": "79c1e1d6e6c36193f13bf2f1bd8379cf79f1d5ce",6034 "externalIds": {6035 "ArXiv": "2602.08990",6036 "CorpusId": 2854518736037 },6038 "publicationVenue": null,6039 "title": "InternAgent-1.5: A Unified Agentic Framework for Long-Horizon Autonomous Scientific Discovery",6040 "venue": "",6041 "year": 2026,6042 "citationCount": 32,6043 "openAccessPdf": {6044 "url": "",6045 "status": null,6046 "license": null,6047 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2602.08990, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6048 },6049 "publicationTypes": null,6050 "journal": null,6051 "authors": [6052 {6053 "authorId": "2279407901",6054 "name": "Shi-Yang Feng"6055 },6056 {6057 "authorId": "2363407583",6058 "name": "Run-Min Ma"6059 },6060 {6061 "authorId": "2347774321",6062 "name": "Xiang-Yu Yan"6063 },6064 {6065 "authorId": "2385186146",6066 "name": "Yue Fan"6067 },6068 {6069 "authorId": "2363688841",6070 "name": "Yu-Song Hu"6071 },6072 {6073 "authorId": "2344944680",6074 "name": "Song-Tao Huang"6075 },6076 {6077 "authorId": "2300403265",6078 "name": "Shuai Zhang"6079 },6080 {6081 "authorId": "2114000045",6082 "name": "Zong-Sheng Cao"6083 },6084 {6085 "authorId": "2334474829",6086 "name": "Tian-Shuo Peng"6087 },6088 {6089 "authorId": "2174741399",6090 "name": "Jiakang Yuan"6091 },6092 {6093 "authorId": "2353199601",6094 "name": "Zi-Wen Guo"6095 },6096 {6097 "authorId": "2191902766",6098 "name": "Zhi-Jie Zhong"6099 },6100 {6101 "authorId": "2180683671",6102 "name": "Shangheng Du"6103 },6104 {6105 "authorId": "2332666924",6106 "name": "Weida Wang"6107 },6108 {6109 "authorId": "2384794295",6110 "name": "Jin-Xin Shi"6111 },6112 {6113 "authorId": "2363408730",6114 "name": "Yu-Hao Zhou"6115 },6116 {6117 "authorId": "2401837334",6118 "name": "Xiao-Han He"6119 },6120 {6121 "authorId": "2362816316",6122 "name": "Zhiyin Yu"6123 },6124 {6125 "authorId": "2380462504",6126 "name": "Fang-Chen Yu"6127 },6128 {6129 "authorId": "2331607748",6130 "name": "Qi-Hao Zheng"6131 },6132 {6133 "authorId": "2399112319",6134 "name": "Jia-Min Wu"6135 },6136 {6137 "authorId": "2311543366",6138 "name": "Mian-Xin Liu"6139 },6140 {6141 "authorId": "2397692167",6142 "name": "Chi Zhang"6143 },6144 {6145 "authorId": "2363513075",6146 "name": "Shao-Wei Hou"6147 },6148 {6149 "authorId": "2365969557",6150 "name": "Shuya Li"6151 },6152 {6153 "authorId": "2341627617",6154 "name": "Yan-Kai Jiang"6155 },6156 {6157 "authorId": "2367737142",6158 "name": "Wenjie Lou"6159 },6160 {6161 "authorId": "2313042601",6162 "name": "Li-Long Wang"6163 },6164 {6165 "authorId": "2386639981",6166 "name": "Zifu Wang"6167 },6168 {6169 "authorId": "2364093499",6170 "name": "Jiong Wang"6171 },6172 {6173 "authorId": "2363836740",6174 "name": "Wang-Sheng Xu"6175 },6176 {6177 "authorId": "2317496445",6178 "name": "Yue Deng"6179 },6180 {6181 "authorId": "2152508904",6182 "name": "Dong-Rui Liu"6183 },6184 {6185 "authorId": "2359211589",6186 "name": "Yi-Heng Wang"6187 },6188 {6189 "authorId": "2302816941",6190 "name": "Wen-Long Zhang"6191 },6192 {6193 "authorId": "2006398365",6194 "name": "Feng-Hua Ling"6195 },6196 {6197 "authorId": "2303209608",6198 "name": "Shufei Zhang"6199 },6200 {6201 "authorId": "2339975108",6202 "name": "Xiao-Song Wang"6203 },6204 {6205 "authorId": "9493614",6206 "name": "Shu-qi Zheng"6207 },6208 {6209 "authorId": "2118521552",6210 "name": "Xun Huang"6211 },6212 {6213 "authorId": "2368321514",6214 "name": "Si-Qi Sun"6215 },6216 {6217 "authorId": "2362971451",6218 "name": "Shu-Yue Hu"6219 },6220 {6221 "authorId": "2376196253",6222 "name": "Peng Ye"6223 },6224 {6225 "authorId": "2274709444",6226 "name": "Chun-Dong Song"6227 },6228 {6229 "authorId": "2325196565",6230 "name": "Bin Wang"6231 },6232 {6233 "authorId": "2409018870",6234 "name": "Conghui He"6235 },6236 {6237 "authorId": "2361197049",6238 "name": "Yi-Hao Liu"6239 },6240 {6241 "authorId": "2390409960",6242 "name": "Xin Li"6243 },6244 {6245 "authorId": "2061436526",6246 "name": "Qi-Bin Hou"6247 },6248 {6249 "authorId": "2374014076",6250 "name": "Tao Chen"6251 },6252 {6253 "authorId": "2327887452",6254 "name": "Xiang-Yu Yue"6255 },6256 {6257 "authorId": "2298840639",6258 "name": "Liang He"6259 },6260 {6261 "authorId": "2303596290",6262 "name": "Da-Hua Lin"6263 },6264 {6265 "authorId": "2339741350",6266 "name": "Bo-Wen Zhou"6267 },6268 {6269 "authorId": "2141897317",6270 "name": "Bo Zhang"6271 },6272 {6273 "authorId": "2384327830",6274 "name": "Lei Bai"6275 }6276 ],6277 "abstract": "We introduce InternAgent-1.5, a unified system designed for end-to-end scientific discovery across computational and empirical domains. The system is built on a structured architecture composed of three coordinated subsystems for generation, verification, and evolution. These subsystems are supported by foundational capabilities for deep research, solution optimization, and long horizon memory. The architecture allows InternAgent-1.5 to operate continuously across extended discovery cycles while maintaining coherent and improving behavior. It also enables the system to coordinate computational modeling and laboratory experimentation within a single unified system. We evaluate InternAgent-1.5 on scientific reasoning benchmarks such as GAIA, HLE, GPQA, and FrontierScience, and the system achieves leading performance that demonstrates strong foundational capabilities. Beyond these benchmarks, we further assess two categories of discovery tasks. In algorithm discovery tasks, InternAgent-1.5 autonomously designs competitive methods for core machine learning problems. In empirical discovery tasks, it executes complete computational or wet lab experiments and produces scientific findings in earth, life, biological, and physical domains. Overall, these results show that InternAgent-1.5 provides a general and scalable framework for autonomous scientific discovery."6278 },6279 "arXiv:2603.08127": {6280 "paperId": "143a2e1a8fc27f4f523597aa45cb8f2cf5137020",6281 "externalIds": {6282 "ArXiv": "2603.08127",6283 "DBLP": "journals/corr/abs-2603-08127",6284 "DOI": "10.48550/arXiv.2603.08127",6285 "CorpusId": 2863768806286 },6287 "publicationVenue": {6288 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",6289 "name": "arXiv.org",6290 "alternate_names": [6291 "ArXiv"6292 ],6293 "issn": "2331-8422",6294 "url": "https://arxiv.org"6295 },6296 "title": "EvoScientist: Towards Multi-Agent Evolving AI Scientists for End-to-End Scientific Discovery",6297 "venue": "arXiv.org",6298 "year": 2026,6299 "citationCount": 60,6300 "openAccessPdf": {6301 "url": "",6302 "status": null,6303 "license": null,6304 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2603.08127, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6305 },6306 "publicationTypes": [6307 "JournalArticle"6308 ],6309 "journal": {6310 "name": "ArXiv",6311 "volume": "abs/2603.08127"6312 },6313 "authors": [6314 {6315 "authorId": "2141789994",6316 "name": "Yougang Lyu"6317 },6318 {6319 "authorId": "2304527137",6320 "name": "Xi Zhang"6321 },6322 {6323 "authorId": "2341456413",6324 "name": "Yu-Yue Zhao"6325 },6326 {6327 "authorId": "2279542724",6328 "name": "Konstantinos Papakostas"6329 },6330 {6331 "authorId": "2313874751",6332 "name": "Xin-Hao Yi"6333 },6334 {6335 "authorId": "2119112577",6336 "name": "Shuyu Guo"6337 },6338 {6339 "authorId": "2261645232",6340 "name": "Wen-Xiang Hu"6341 },6342 {6343 "authorId": "2273931585",6344 "name": "Jan Piotrowski"6345 },6346 {6347 "authorId": "28999125",6348 "name": "Jakub Kaliski"6349 },6350 {6351 "authorId": "1915156",6352 "name": "Jacopo Urbani"6353 },6354 {6355 "authorId": "2315664370",6356 "name": "Zai-Qiao Meng"6357 },6358 {6359 "authorId": "2238355704",6360 "name": "Lu Zhou"6361 },6362 {6363 "authorId": "2315457268",6364 "name": "Zhao-Chun Ren"6365 },6366 {6367 "authorId": "2374005983",6368 "name": "Xiao-Hu Yan"6369 }6370 ],6371 "abstract": "The increasing adoption of Large Language Models (LLMs) has enabled AI scientists to perform complex end-to-end scientific discovery tasks requiring coordination of specialized roles, including idea generation and experimental execution. However, most state-of-the-art AI scientist systems rely on static, hand-designed pipelines and fail to adapt based on accumulated interaction histories. As a result, these systems overlook promising research directions, repeat failed experiments, and pursue infeasible ideas. To address this, we introduce EvoScientist, an evolving multi-agent AI scientist framework that continuously improves research strategies through persistent memory and self-evolution. EvoScientist comprises three specialized agents: a Researcher Agent (RA) for scientific idea generation, an Engineer Agent (EA) for experiment implementation and execution, and an Evolution Manager Agent (EMA) that distills insights from prior interactions into reusable knowledge. EvoScientist contains two persistent memory modules: (i) an ideation memory, which summarizes feasible research directions from top-ranked ideas while recording previously unsuccessful directions; and (ii) an experimentation memory, which captures effective data processing and model training strategies derived from code search trajectories and best-performing implementations. These modules enable the RA and EA to retrieve relevant prior strategies, improving idea quality and code execution success rates over time. Experiments show that EvoScientist outperforms 7 open-source and commercial state-of-the-art systems in scientific idea generation, achieving higher novelty, feasibility, relevance, and clarity via automatic and human evaluation. EvoScientist also substantially improves code execution success rates through multi-agent evolution, demonstrating persistent memory's effectiveness for end-to-end scientific discovery."6372 },6373 "arXiv:2603.29231": {6374 "paperId": "ab0ffa96b2ef941b7cbe3db785e01981f2682254",6375 "externalIds": {6376 "DBLP": "journals/corr/abs-2603-29231",6377 "ArXiv": "2603.29231",6378 "DOI": "10.48550/arXiv.2603.29231",6379 "CorpusId": 2869770516380 },6381 "publicationVenue": {6382 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",6383 "name": "arXiv.org",6384 "alternate_names": [6385 "ArXiv"6386 ],6387 "issn": "2331-8422",6388 "url": "https://arxiv.org"6389 },6390 "title": "Beyond pass@1: A Reliability Science Framework for Long-Horizon LLM Agents",6391 "venue": "arXiv.org",6392 "year": 2026,6393 "citationCount": 10,6394 "openAccessPdf": {6395 "url": "",6396 "status": null,6397 "license": null,6398 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2603.29231, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6399 },6400 "publicationTypes": [6401 "JournalArticle"6402 ],6403 "journal": {6404 "name": "ArXiv",6405 "volume": "abs/2603.29231"6406 },6407 "authors": [6408 {6409 "authorId": "2363444455",6410 "name": "Aaditya Khanal"6411 },6412 {6413 "authorId": "2105710431",6414 "name": "Yang-Yang Tao"6415 },6416 {6417 "authorId": "2329901586",6418 "name": "Junxiu Zhou"6419 }6420 ],6421 "abstract": "Existing benchmarks measure capability -- whether a model succeeds on a single attempt -- but production deployments require reliability -- consistent success across repeated attempts on tasks of varying duration. We show these properties diverge systematically as task duration grows, and that pass@1 on short tasks is structurally blind to this divergence. We introduce a reliability science framework for long-horizon LLM agents with four metrics: Reliability Decay Curve (RDC), Variance Amplification Factor (VAF), Graceful Degradation Score (GDS), and Meltdown Onset Point (MOP). We evaluate 10 models across 23,392 episodes on a 396-task benchmark spanning four duration buckets and three domains. Key findings: (1) reliability decay is domain-stratified -- SE GDS drops from 0.90 to 0.44 while document processing is nearly flat (0.74 to 0.71); (2) VAF bifurcates by capability tier -- high VAF is a capability signature, not an instability signal; (3) capability and reliability rankings diverge substantially, with multi-rank inversions at long horizons; (4) frontier models have the highest meltdown rates (up to 19%) because they attempt ambitious multi-step strategies that sometimes spiral; and (5) memory scaffolds universally hurt long-horizon performance across all 10 models. These results motivate reliability as a first-class evaluation dimension alongside capability."6422 },6423 "arXiv:2605.07723": {6424 "paperId": "620e5d7f81c94a66f51dc89451e80ea4772a6716",6425 "externalIds": {6426 "DBLP": "journals/corr/abs-2605-07723",6427 "ArXiv": "2605.07723",6428 "DOI": "10.48550/arXiv.2605.07723",6429 "CorpusId": 2881476676430 },6431 "publicationVenue": {6432 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",6433 "name": "arXiv.org",6434 "alternate_names": [6435 "ArXiv"6436 ],6437 "issn": "2331-8422",6438 "url": "https://arxiv.org"6439 },6440 "title": "LLM hallucinations in the wild: Large-scale evidence from non-existent citations",6441 "venue": "arXiv.org",6442 "year": 2026,6443 "citationCount": 16,6444 "openAccessPdf": {6445 "url": "",6446 "status": null,6447 "license": null,6448 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2605.07723, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6449 },6450 "publicationTypes": [6451 "JournalArticle"6452 ],6453 "journal": {6454 "name": "ArXiv",6455 "volume": "abs/2605.07723"6456 },6457 "authors": [6458 {6459 "authorId": "1484816455",6460 "name": "Zhen-Yue Zhao"6461 },6462 {6463 "authorId": "2367001043",6464 "name": "Yihe Wang"6465 },6466 {6467 "authorId": "2334807881",6468 "name": "Toby Stuart"6469 },6470 {6471 "authorId": "2484682509",6472 "name": "M. de Vaan"6473 },6474 {6475 "authorId": "3183325",6476 "name": "P. Ginsparg"6477 },6478 {6479 "authorId": "2256439343",6480 "name": "Yian Yin"6481 }6482 ],6483 "abstract": "Large language models (LLMs) are known to generate plausible but false information across a wide range of contexts, yet the real-world magnitude and consequences of this hallucination problem remain poorly understood. Here we leverage a uniquely verifiable object - scientific citations - to audit 111 million references across 2.5 million papers in arXiv, bioRxiv, SSRN, and PubMed Central. We find a sharp rise in non-existent references following widespread LLM adoption, with a conservative estimate of 146,932 hallucinated citations in 2025 alone. These errors are diffusely embedded across many papers but especially pronounced in fields with rapid AI uptake, in manuscripts with linguistic signatures of AI-assisted writing, and among small and early-career author teams. At the same time, hallucinated references disproportionately assign credit to already prominent and male scholars, suggesting that LLM-generated errors may reinforce existing inequities in scientific recognition. Preprint moderation and journal publication processes capture only a fraction of these errors, suggesting that the spread of hallucinated content has outpaced existing safeguards. Together, these findings demonstrate that LLM hallucinations are infiltrating knowledge production at scale, threatening both the reliability and equity of future scientific discovery as human and AI systems draw on the existing literature."6484 },6485 "arXiv:2605.26340": {6486 "paperId": "1d902a3e54fc9af77c55217b1688c6d3a8734757",6487 "externalIds": {6488 "ArXiv": "2605.26340",6489 "DBLP": "journals/corr/abs-2605-26340",6490 "DOI": "10.48550/arXiv.2605.26340",6491 "CorpusId": 2886729416492 },6493 "publicationVenue": {6494 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",6495 "name": "arXiv.org",6496 "alternate_names": [6497 "ArXiv"6498 ],6499 "issn": "2331-8422",6500 "url": "https://arxiv.org"6501 },6502 "title": "ScientistOne: Towards Human-Level Autonomous Research via Chain-of-Evidence",6503 "venue": "arXiv.org",6504 "year": 2026,6505 "citationCount": 21,6506 "openAccessPdf": {6507 "url": "",6508 "status": null,6509 "license": null,6510 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2605.26340, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6511 },6512 "publicationTypes": [6513 "JournalArticle",6514 "Review"6515 ],6516 "journal": {6517 "name": "ArXiv",6518 "volume": "abs/2605.26340"6519 },6520 "authors": [6521 {6522 "authorId": "2385420268",6523 "name": "Rui Meng"6524 },6525 {6526 "authorId": "40135250",6527 "name": "Bhavana Dalvi"6528 },6529 {6530 "authorId": "2155099104",6531 "name": "Jie-Feng Chen"6532 },6533 {6534 "authorId": "2136342754",6535 "name": "Chun-Liang Li"6536 },6537 {6538 "authorId": "2344611832",6539 "name": "Palash Goyal"6540 },6541 {6542 "authorId": "2384366252",6543 "name": "Mihir Parmar"6544 },6545 {6546 "authorId": "2350143431",6547 "name": "Yi-Wen Song"6548 },6549 {6550 "authorId": "2268777158",6551 "name": "Ya-Le Song"6552 },6553 {6554 "authorId": "2446118",6555 "name": "Rajarishi S. Sinha"6556 },6557 {6558 "authorId": "2266304326",6559 "name": "Parthasarathy Ranganathan"6560 },6561 {6562 "authorId": "2225921342",6563 "name": "Burak Gokturk"6564 },6565 {6566 "authorId": "2392692988",6567 "name": "Jinsung Yoon"6568 },6569 {6570 "authorId": "2382765500",6571 "name": "Tomas Pfister"6572 }6573 ],6574 "abstract": "Autonomous research agents produce competitive solutions and professional-looking manuscripts, yet their outputs contain verifiability failures undetectable by surface-level evaluation: fabricated citations, unreproducible scores, and method descriptions that diverge from the implementation. We address this through three contributions. First, Chain-of-Evidence (CoE), a verifiability framework requiring every claim to be traceable to its evidence source. Second, ScientistOne, an end-to-end autonomous research system that maintains evidence chains by construction throughout literature review, solution discovery, and paper writing. Third, CoE Audit, a post-hoc audit whose four integrity checks -- score verification, specification violation, reference verification, and method-code alignment -- apply uniformly to all systems. Across 75 papers spanning five systems and five frontier research tasks, every baseline exhibits at least one systematic failure mode: hallucinated reference rates reach 21%, score verification passes in as few as 42% of papers, and method-code alignment ranges from 20% to 80%. ScientistOne achieves zero hallucinated references (0/337), perfect score verification (12/12), and the highest method-code alignment (14/15), while matching or exceeding human expert performance on all five tasks. ScientistOne further generalizes to six additional tasks spanning medical imaging, fine-grained recognition, 3D perception, and language modeling, achieving state-of-the-art on Parameter Golf and gold medals on MLE-Bench tasks where baselines fail entirely."6575 },6576 "arXiv:2607.12301": {6577 "paperId": "a71870edb216ebbc78ae21b1b4bb9c9b565613f8",6578 "externalIds": {6579 "DBLP": "journals/corr/abs-2607-12301",6580 "ArXiv": "2607.12301",6581 "DOI": "10.48550/arXiv.2607.12301",6582 "CorpusId": 2901488056583 },6584 "publicationVenue": {6585 "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",6586 "name": "arXiv.org",6587 "alternate_names": [6588 "ArXiv"6589 ],6590 "issn": "2331-8422",6591 "url": "https://arxiv.org"6592 },6593 "title": "XScientist: A Git-Like Research Protocol for Long-Running Autonomous Scientific Discovery",6594 "venue": "arXiv.org",6595 "year": 2026,6596 "citationCount": 1,6597 "openAccessPdf": {6598 "url": "",6599 "status": null,6600 "license": null,6601 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2607.12301, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6602 },6603 "publicationTypes": [6604 "JournalArticle",6605 "Review"6606 ],6607 "journal": {6608 "name": "ArXiv",6609 "volume": "abs/2607.12301"6610 },6611 "authors": [6612 {6613 "authorId": "2111701253",6614 "name": "Ji-Xi Luo"6615 }6616 ],6617 "abstract": "Autonomous research systems can generate plausible papers while losing the decisions, failed branches, and evidence needed to inspect or continue the work. We present XScientist, a local-first, git-like protocol that treats research state, rather than a manuscript, as the unit of continuation. Hypotheses, experiment attempts, observations, claims, reviews, and handoffs are represented as typed, content-addressed objects in an exploration graph. Immutable checkpoints, explicit negative outcomes, claim-evidence closure, replay boundaries, and authority-aware gates make each transition inspectable without treating a passing integrity check as scientific truth. The protocol exports a portable Agent-Native Research Artifact (ARA) that another agent or human can inspect, fork, verify, and extend. A reference implementation integrates planning, execution, review, repair, and supervised long-running operation while preserving provenance across these stages. We evaluate the protocol with controlled artifact-integrity workloads and matched external task pilots, keeping native task performance separate from evidence and audit claims. The result is an interoperability and accountability layer for long-running autonomous science, with explicit boundaries where human judgment and independent evaluation remain necessary."6618 },6619 "arXiv:2608.05179": {6620 "paperId": "9b5f65e0116f686c746f460c719265a71660c2db",6621 "externalIds": {6622 "ArXiv": "2608.05179",6623 "CorpusId": 2908519906624 },6625 "publicationVenue": null,6626 "title": "Autonomous Research Agents: A Survey of AI Scientists and the Verification Gap",6627 "venue": "",6628 "year": 2026,6629 "citationCount": 8,6630 "openAccessPdf": {6631 "url": "",6632 "status": null,6633 "license": null,6634 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2608.05179, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6635 },6636 "publicationTypes": [6637 "Review"6638 ],6639 "journal": null,6640 "authors": [6641 {6642 "authorId": "2184177582",6643 "name": "Tian-Yu Ding"6644 },6645 {6646 "authorId": "2444870503",6647 "name": "Aditya Nannapaneni"6648 },6649 {6650 "authorId": "2445721236",6651 "name": "Bingfan Liu"6652 },6653 {6654 "authorId": "2445719507",6655 "name": "Ling Zhang"6656 }6657 ],6658 "abstract": "Large language model (LLM) agents are increasingly used across the scientific research lifecycle: ideation, literature search, experiment design and execution, analysis, manuscript drafting, and review. End-to-end AI scientist systems can now produce paper-like manuscripts, but their claims are often harder to verify than their code is to run. This survey studies that gap in computational AI/ML research, where code, benchmarks, experiments, and write-ups are most visible. We screen 125 candidate works and include 35, with full-text coding of 26 entries: 24 runnable systems and two study or position papers. We code seven audit dimensions: lifecycle stage, autonomy level, evaluation method, released artifacts, human-in-the-loop points, novelty verification, and result-selection disclosure. The main pattern is that code release is now common, but reproducibility-grade and claim-verification artifacts remain much less common. In the 24 runnable systems, 83 percent release code, while 38 percent release seeds or execution traces and 38 percent report any novelty-verification method. Among nine closed-loop L4 systems, seven are mechanical reruns and one is author-claimed without an external check; no LLM-era system in the corpus demonstrates an externally validated in-loop oracle under our coding rule. We contribute a coded corpus, a lifecycle-by-autonomy map, an auditability-gap analysis, and a reviewer-facing reporting checklist. The survey argues that the field's central bottleneck is no longer only whether agents can complete research tasks, but whether reviewers can verify the claims those agents produce."6659 },6660 "arXiv:2608.14380": {6661 "paperId": "a6c6d02eb805bd1d071df21d84ee2d9605985f9a",6662 "externalIds": {6663 "ArXiv": "2608.14380",6664 "CorpusId": 2911688066665 },6666 "publicationVenue": null,6667 "title": "AgentRewind: Recoverable Execution for Long-Horizon LLM Agents",6668 "venue": "",6669 "year": 2026,6670 "citationCount": 11,6671 "openAccessPdf": {6672 "url": "",6673 "status": null,6674 "license": null,6675 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2608.14380, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6676 },6677 "publicationTypes": null,6678 "journal": null,6679 "authors": [6680 {6681 "authorId": "2464739722",6682 "name": "Yu Zhuang"6683 },6684 {6685 "authorId": "2398377257",6686 "name": "Kefei Chen"6687 },6688 {6689 "authorId": "2368401349",6690 "name": "Yitong Duan"6691 },6692 {6693 "authorId": "2274198400",6694 "name": "Shu-Xin Zheng"6695 },6696 {6697 "authorId": "2351726800",6698 "name": "Jian Li"6699 },6700 {6701 "authorId": "2458017725",6702 "name": "Xu-Yao Zhang"6703 }6704 ],6705 "abstract": "Many real-world tasks require LLM agents to interact with their environments over long execution horizons. Errors that occur early in execution may propagate through both the agent context and environment state, and their effects may be difficult to reverse through subsequent actions. Existing methods mainly seek to reduce such errors through plan refinement and safety checks but provide little support after errors occur. To enable recovery during long-horizon execution, we present AgentRewind, a runtime recovery framework that records aligned checkpoints of the agent context and controlled environment, allowing agents to return to an earlier state and resume execution with information from previous attempts. We also construct MettleBench, a benchmark for evaluating task completion and partial progress on long-horizon engineering assignments containing a series of related requirements. Experiments across tasks, multiple models, execution strategies, and agent harnesses show that AgentRewind improves task success rate and average checklist progress over the compared baselines."6706 },6707 "DOI:10.1002/adma.202413523": {6708 "paperId": "dbbcdb281ed6aa646af0402172cf8a7cfda85d5c",6709 "externalIds": {6710 "PubMedCentral": "12138853",6711 "DOI": "10.1002/adma.202413523",6712 "CorpusId": 274858957,6713 "PubMed": "39696898"6714 },6715 "publicationVenue": {6716 "id": "71697426-616f-45b4-b9d8-d647335a32e6",6717 "name": "Advances in Materials",6718 "type": "journal",6719 "alternate_names": [6720 "Adv Mater",6721 "Advanced Materials"6722 ],6723 "issn": "2327-2503",6724 "alternate_issns": [6725 "0935-9648"6726 ],6727 "url": "http://www.sciencepublishinggroup.com/journal/archive.aspx?amp;issueid=-1&journalid=129",6728 "alternate_urls": [6729 "http://www.advmat.de/",6730 "https://onlinelibrary.wiley.com/journal/15214095",6731 "http://www.sciencepublishinggroup.com/journal/archive.aspx?issueid=-1&journalid=129",6732 "http://www.wiley-vch.de/publish/en/journals/alphabeticIndex/2089/"6733 ]6734 },6735 "title": "SciAgents: Automating Scientific Discovery Through Bioinspired Multi\u2010Agent Intelligent Graph Reasoning",6736 "venue": "Advances in Materials",6737 "year": 2024,6738 "citationCount": 297,6739 "openAccessPdf": {6740 "url": "https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/adma.202413523",6741 "status": "HYBRID",6742 "license": "CCBYNC",6743 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC12138853, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6744 },6745 "publicationTypes": [6746 "JournalArticle"6747 ],6748 "journal": {6749 "name": "Advanced Materials (Deerfield Beach, Fla.)",6750 "volume": "37"6751 },6752 "authors": [6753 {6754 "authorId": "2310665423",6755 "name": "Alireza Ghafarollahi"6756 },6757 {6758 "authorId": "2252265640",6759 "name": "Markus J. Buehler"6760 }6761 ],6762 "abstract": "A key challenge in artificial intelligence (AI) is the creation of systems capable of autonomously advancing scientific understanding by exploring novel domains, identifying complex patterns, and uncovering previously unseen connections in vast scientific data. In this work, SciAgents, an approach that leverages three core concepts is presented: (1) large\u2010scale ontological knowledge graphs to organize and interconnect diverse scientific concepts, (2) a suite of large language models (LLMs) and data retrieval tools, and (3) multi\u2010agent systems with in\u2010situ learning capabilities. Applied to biologically inspired materials, SciAgents reveals hidden interdisciplinary relationships that were previously considered unrelated, achieving a scale, precision, and exploratory power that surpasses human research methods. The framework autonomously generates and refines research hypotheses, elucidating underlying mechanisms, design principles, and unexpected material properties. By integrating these capabilities in a modular fashion, the system yields material discoveries, critiques and improves existing hypotheses, retrieves up\u2010to\u2010date data about existing research, and highlights strengths and limitations. This is achieved by harnessing a \u201cswarm of intelligence\u201d similar to biological systems, providing new avenues for discovery. How this model accelerates the development of advanced materials by unlocking Nature's design principles, resulting in a new biocomposite with enhanced mechanical properties and improved sustainability through energy\u2010efficient production is shown."6763 },6764 "DOI:10.1006/ijhc.1999.0252": {6765 "paperId": "8580dc7ba0f0d7eb1c2495ceb23af9491a805849",6766 "externalIds": {6767 "DBLP": "journals/ijmms/SkitkaMB99",6768 "MAG": "2081011876",6769 "DOI": "10.1006/ijhc.1999.0252",6770 "CorpusId": 18632266771 },6772 "publicationVenue": null,6773 "title": "Does automation bias decision-making?",6774 "venue": "Int. J. Hum. Comput. Stud.",6775 "year": 1999,6776 "citationCount": 702,6777 "openAccessPdf": {6778 "url": "",6779 "status": "CLOSED",6780 "license": null,6781 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1006/ijhc.1999.0252?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1006/ijhc.1999.0252, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6782 },6783 "publicationTypes": [6784 "JournalArticle"6785 ],6786 "journal": {6787 "name": "Int. J. Hum. Comput. Stud.",6788 "pages": "991-1006",6789 "volume": "51"6790 },6791 "authors": [6792 {6793 "authorId": "2622061",6794 "name": "L. Skitka"6795 },6796 {6797 "authorId": "2249256",6798 "name": "K. Mosier"6799 },6800 {6801 "authorId": "10806842",6802 "name": "M. D. Burdick"6803 }6804 ],6805 "abstract": null6806 },6807 "DOI:10.1007/s10994-025-06955-2": {6808 "paperId": "98f166771681acfe51c0f69558d8a977f0782739",6809 "externalIds": {6810 "DBLP": "journals/corr/abs-2305-02251",6811 "ArXiv": "2305.02251",6812 "DOI": "10.1007/s10994-025-06955-2",6813 "CorpusId": 2584616206814 },6815 "publicationVenue": {6816 "id": "22c9862f-a25e-40cd-9d31-d09e68a293e6",6817 "name": "Machine-mediated learning",6818 "type": "journal",6819 "alternate_names": [6820 "Mach learn",6821 "Machine Learning",6822 "Mach Learn"6823 ],6824 "issn": "0732-6718",6825 "alternate_issns": [6826 "0885-6125"6827 ],6828 "url": "http://www.springer.com/computer/artificial/journal/10994",6829 "alternate_urls": [6830 "https://link.springer.com/journal/10994",6831 "http://www.springer.com/west/home/computer/artificial?SGWID=4-147-70-35726603-0"6832 ]6833 },6834 "title": "Automated Scientific Discovery: From Equation Discovery to Autonomous Discovery Systems",6835 "venue": "Machine-mediated learning",6836 "year": 2023,6837 "citationCount": 30,6838 "openAccessPdf": {6839 "url": "https://link.springer.com/content/pdf/10.1007/s10994-025-06955-2.pdf",6840 "status": "HYBRID",6841 "license": "CCBY",6842 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2305.02251, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6843 },6844 "publicationTypes": [6845 "JournalArticle",6846 "Review"6847 ],6848 "journal": {6849 "name": "Machine Learning",6850 "volume": "115"6851 },6852 "authors": [6853 {6854 "authorId": "2058253559",6855 "name": "Stefan Kramer"6856 },6857 {6858 "authorId": "1382852308",6859 "name": "Mattia Cerrato"6860 },6861 {6862 "authorId": "2037478739",6863 "name": "Jannis Brugger"6864 },6865 {6866 "authorId": "1693549",6867 "name": "S. D\u017eeroski"6868 },6869 {6870 "authorId": "33654217",6871 "name": "R. King"6872 }6873 ],6874 "abstract": "The paper surveys automated scientific discovery, from equation discovery and symbolic regression to autonomous discovery systems and agents. It discusses the individual approaches from a \"big picture\" perspective and in context, but also discusses open issues and recent topics like the various roles of deep neural networks in this area, aiding in the discovery of human-interpretable knowledge. Further, we will present closed-loop scientific discovery systems, starting with the pioneering work on the Adam system up to current efforts in fields from material science to astronomy. Finally, we will elaborate on autonomy from a machine learning perspective, but also in analogy to the autonomy levels in autonomous driving. The maximal level, level five, is defined to require no human intervention at all in the production of scientific knowledge. Achieving this is one step towards solving the Nobel Turing Grand Challenge to develop AI Scientists: AI systems capable of making Nobel-quality scientific discoveries highly autonomously at a level comparable, and possibly superior, to the best human scientists by 2050."6875 },6876 "DOI:10.1016/0005-1098(83)90046-8": {6877 "paperId": "0713bb9d9b138e4e0a15406006de9b0cddf68e28",6878 "externalIds": {6879 "MAG": "2008653298",6880 "DBLP": "journals/automatica/Bainbridge83",6881 "DOI": "10.1016/0005-1098(83)90046-8",6882 "CorpusId": 126677426883 },6884 "publicationVenue": {6885 "id": "14f974af-a3bc-407d-92cb-c6cdbd6d94ee",6886 "name": "at - Automatisierungstechnik",6887 "type": "journal",6888 "alternate_names": [6889 " Autom",6890 "At-automatisierungstechnik"6891 ],6892 "issn": "0178-2312",6893 "url": "http://www.degruyter.com/loi/auto",6894 "alternate_urls": [6895 "https://www.degruyter.com/view/j/auto"6896 ]6897 },6898 "title": "Ironies of automation",6899 "venue": "at - Automatisierungstechnik",6900 "year": 1982,6901 "citationCount": 2717,6902 "openAccessPdf": {6903 "url": "",6904 "status": "CLOSED",6905 "license": null,6906 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1016/0005-1098(83)90046-8?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1016/0005-1098(83)90046-8, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6907 },6908 "publicationTypes": [6909 "JournalArticle"6910 ],6911 "journal": {6912 "name": "Autom.",6913 "pages": "775-779",6914 "volume": "19"6915 },6916 "authors": [6917 {6918 "authorId": "10352604",6919 "name": "L. Bainbridge"6920 }6921 ],6922 "abstract": null6923 },6924 "DOI:10.1016/j.cortex.2012.12.016": {6925 "paperId": "ddb5829aec88f87b796171a7ce347aa485fc99bc",6926 "externalIds": {6927 "MAG": "2002030061",6928 "DOI": "10.1016/j.cortex.2012.12.016",6929 "CorpusId": 140204881,6930 "PubMed": "23347556"6931 },6932 "publicationVenue": null,6933 "title": "Registered reports: a new publishing initiative at Cortex.",6934 "venue": "Cortex; a journal devoted to the study of the nervous system and behavior",6935 "year": 2013,6936 "citationCount": 666,6937 "openAccessPdf": {6938 "url": "",6939 "status": "CLOSED",6940 "license": null,6941 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1016/j.cortex.2012.12.016?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1016/j.cortex.2012.12.016, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6942 },6943 "publicationTypes": [6944 "Editorial"6945 ],6946 "journal": {6947 "name": "Cortex; a journal devoted to the study of the nervous system and behavior",6948 "pages": "\n 609-10\n ",6949 "volume": "49 3"6950 },6951 "authors": [6952 {6953 "authorId": "2001975",6954 "name": "C. Chambers"6955 }6956 ],6957 "abstract": null6958 },6959 "DOI:10.1016/j.future.2014.10.008": {6960 "paperId": "4224374796da64e17fce96033d4cd42240d80eaf",6961 "externalIds": {6962 "MAG": "2078962046",6963 "DBLP": "journals/fgcs/DeelmanVJRCMMCS15",6964 "DOI": "10.1016/J.FUTURE.2014.10.008",6965 "CorpusId": 146113456966 },6967 "publicationVenue": {6968 "id": "e3e1f91c-fe26-4f67-8d15-f7acd8abdf42",6969 "name": "Future generations computer systems",6970 "type": "journal",6971 "alternate_names": [6972 "Future gener comput syst",6973 "Future Generation Computer Systems",6974 "Future Gener Comput Syst"6975 ],6976 "issn": "0167-739X",6977 "url": "http://www.elsevier.com/wps/find/journaleditorialboard.cws_home/505611/editorialboard",6978 "alternate_urls": [6979 "https://www.journals.elsevier.com/future-generation-computer-systems",6980 "http://www.sciencedirect.com/science/journal/0167739X"6981 ]6982 },6983 "title": "Pegasus, a workflow management system for science automation",6984 "venue": "Future generations computer systems",6985 "year": 2015,6986 "citationCount": 962,6987 "openAccessPdf": {6988 "url": "https://doi.org/10.1016/j.future.2014.10.008",6989 "status": "BRONZE",6990 "license": null,6991 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1016/J.FUTURE.2014.10.008?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1016/J.FUTURE.2014.10.008, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."6992 },6993 "publicationTypes": [6994 "JournalArticle"6995 ],6996 "journal": {6997 "name": "Future Gener. Comput. Syst.",6998 "pages": "17-35",6999 "volume": "46"7000 },7001 "authors": [7002 {7003 "authorId": "1687896",7004 "name": "E. Deelman"7005 },7006 {7007 "authorId": "1742157",7008 "name": "K. Vahi"7009 },7010 {7011 "authorId": "1711146",7012 "name": "G. Juve"7013 },7014 {7015 "authorId": "1920965",7016 "name": "M. Rynge"7017 },7018 {7019 "authorId": "1711636",7020 "name": "S. Callaghan"7021 },7022 {7023 "authorId": "35376674",7024 "name": "P. Maechling"7025 },7026 {7027 "authorId": "2956722",7028 "name": "R. Mayani"7029 },7030 {7031 "authorId": "48994458",7032 "name": "Weiwei Chen"7033 },7034 {7035 "authorId": "145971089",7036 "name": "Rafael Ferreira da Silva"7037 },7038 {7039 "authorId": "1684336",7040 "name": "M. Livny"7041 },7042 {7043 "authorId": "145559269",7044 "name": "R. K. Wenger"7045 }7046 ],7047 "abstract": null7048 },7049 "DOI:10.1016/j.patter.2021.100322": {7050 "paperId": "1f1297064e405bbcd7552f9d3faf401b2747b814",7051 "externalIds": {7052 "DBLP": "journals/patterns/LeipzigNHRG21",7053 "ArXiv": "2006.08589",7054 "MAG": "3035267492",7055 "PubMedCentral": "8441584",7056 "DOI": "10.1016/j.patter.2021.100322",7057 "CorpusId": 219687399,7058 "PubMed": "34553169"7059 },7060 "publicationVenue": {7061 "id": "17bac89e-3dba-467a-b9d4-71e3baefb08b",7062 "name": "Patterns",7063 "type": "journal",7064 "issn": "2666-3899",7065 "url": "https://www.cell.com/patterns",7066 "alternate_urls": [7067 "https://www.sciencedirect.com/journal/patterns"7068 ]7069 },7070 "title": "The role of metadata in reproducible computational research",7071 "venue": "Patterns",7072 "year": 2020,7073 "citationCount": 128,7074 "openAccessPdf": {7075 "url": "http://www.cell.com/article/S2666389921001707/pdf",7076 "status": "GOLD",7077 "license": "CCBYNCSA",7078 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2006.08589, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."7079 },7080 "publicationTypes": [7081 "Review",7082 "JournalArticle"7083 ],7084 "journal": {7085 "name": "Patterns",7086 "volume": "2"7087 },7088 "authors": [7089 {7090 "authorId": "2057227799",7091 "name": "Jeremy Leipzig"7092 },7093 {7094 "authorId": "2587120",7095 "name": "Daniel N\u00fcst"7096 },7097 {7098 "authorId": "35477117",7099 "name": "Charles Tapley Hoyt"7100 },7101 {7102 "authorId": "2161614",7103 "name": "Stian Soiland-Reyes"7104 },7105 {7106 "authorId": "144686135",7107 "name": "Karthik Ram"7108 },7109 {7110 "authorId": "2057251522",7111 "name": "J. Greenberg"7112 }7113 ],7114 "abstract": "Summary Reproducible computational research (RCR) is the keystone of the scientific method for in silico analyses, packaging the transformation of raw data to published results. In addition to its role in research integrity, improving the reproducibility of scientific studies can accelerate evaluation and reuse. This potential and wide support for the FAIR principles have motivated interest in metadata standards supporting reproducibility. Metadata provide context and provenance to raw data and methods and are essential to both discovery and validation. Despite this shared connection with scientific data, few studies have explicitly described how metadata enable reproducible computational research. This review employs a functional content analysis to identify metadata standards that support reproducibility across an analytic stack consisting of input data, tools, notebooks, pipelines, and publications. Our review provides background context, explores gaps, and discovers component trends of embeddedness and methodology weight from which we derive recommendations for future work."7115 },7116 "DOI:10.1016/j.websem.2015.04.001": {7117 "paperId": "96cce1917fa1c9f6e2c5f7298c5a12cb3dd60dcd",7118 "externalIds": {7119 "MAG": "2029021214",7120 "DBLP": "journals/ws/MoreauGCLM15",7121 "DOI": "10.1016/J.WEBSEM.2015.04.001",7122 "CorpusId": 146726957123 },7124 "publicationVenue": {7125 "id": "78d61825-c1d6-4e7f-ab9d-1525db71105c",7126 "name": "Journal of Web Semantics",7127 "type": "journal",7128 "alternate_names": [7129 "J Web Semant"7130 ],7131 "issn": "1570-8268",7132 "url": "http://www.elsevier.com/locate/websem",7133 "alternate_urls": [7134 "http://www.sciencedirect.com/science/journal/15708268",7135 "http://www.elsevier.com/wps/find/journaldescription.cws_home/671322/description"7136 ]7137 },7138 "title": "The rationale of PROV",7139 "venue": "Journal of Web Semantics",7140 "year": 2015,7141 "citationCount": 120,7142 "openAccessPdf": {7143 "url": "https://www.sciencedirect.com/science/article/pii/S1570826815000177/pdf",7144 "status": "HYBRID",7145 "license": "CCBY",7146 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1016/J.WEBSEM.2015.04.001?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1016/J.WEBSEM.2015.04.001, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."7147 },7148 "publicationTypes": [7149 "JournalArticle"7150 ],7151 "journal": {7152 "name": "J. Web Semant.",7153 "pages": "235-257",7154 "volume": "35"7155 },7156 "authors": [7157 {7158 "authorId": "1773775",7159 "name": "L. Moreau"7160 },7161 {7162 "authorId": "143772714",7163 "name": "Paul Groth"7164 },7165 {7166 "authorId": "144320476",7167 "name": "James Cheney"7168 },7169 {7170 "authorId": "1760030",7171 "name": "Timothy Lebo"7172 },7173 {7174 "authorId": "145116379",7175 "name": "S. Miles"7176 }7177 ],7178 "abstract": "The prov family of documents are the final output of the World Wide Web Consortium Provenance Working Group, chartered to specify a representation of provenance to facilitate its exchange over the Web. This article reflects upon the key requirements, guiding principles, and design decisions that influenced the prov family of documents. A broad range of requirements were found, relating to the key concepts necessary for describing provenance, such as resources, activities, agents and events, and to balancing prov's ease of use with the facility to check its validity. By this retrospective requirement analysis, the article aims to provide some insights into how prov turned out as it did and why. Benefits of this insight include better inter-operability, a roadmap for alternate investigations and improvements, and solid foundations for future standardization activities."7179 },7180 "DOI:10.1038/nbt.3820": {7181 "paperId": "aa02474345b8601509b607ef015ba1d7c7ed7af2",7182 "externalIds": {7183 "MAG": "2605897695",7184 "DOI": "10.1038/nbt.3820",7185 "CorpusId": 9690740,7186 "PubMed": "28398311"7187 },7188 "publicationVenue": {7189 "id": "458166b3-de17-4bf3-bbbb-e53782de2f0f",7190 "name": "Nature Biotechnology",7191 "type": "journal",7192 "alternate_names": [7193 "Nat Biotechnol"7194 ],7195 "issn": "1087-0156",7196 "url": "http://www.nature.com/nbt/",7197 "alternate_urls": [7198 "http://www.nature.com/nbt"7199 ]7200 },7201 "title": "Nextflow enables reproducible computational workflows",7202 "venue": "Nature Biotechnology",7203 "year": 2017,7204 "citationCount": 3722,7205 "openAccessPdf": {7206 "url": "",7207 "status": "CLOSED",7208 "license": null,7209 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1038/nbt.3820?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1038/nbt.3820, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."7210 },7211 "publicationTypes": [7212 "LettersAndComments"7213 ],7214 "journal": {7215 "name": "Nature Biotechnology",7216 "pages": "316-319",7217 "volume": "35"7218 },7219 "authors": [7220 {7221 "authorId": "11059942",7222 "name": "P. Di Tommaso"7223 },7224 {7225 "authorId": "2613149",7226 "name": "Maria Chatzou"7227 },7228 {7229 "authorId": "2141512433",7230 "name": "Evan W. Floden"7231 },7232 {7233 "authorId": "34903918",7234 "name": "Pablo Prieto Barja"7235 },7236 {7237 "authorId": "2083101458",7238 "name": "Emilio Palumbo"7239 },7240 {7241 "authorId": "46726219",7242 "name": "C. Notredame"7243 }7244 ],7245 "abstract": null7246 },7247 "DOI:10.1038/s41562-016-0021": {7248 "paperId": "a68ce412e92d87ca0116519651bbc484d98c76ae",7249 "externalIds": {7250 "PubMedCentral": "7610724",7251 "MAG": "2727329419",7252 "DOI": "10.1038/s41562-016-0021",7253 "CorpusId": 6326747,7254 "PubMed": "33954258"7255 },7256 "publicationVenue": {7257 "id": "b7e5a3b3-2519-4ca2-8870-c2b45ef76ab9",7258 "name": "Nature Human Behaviour",7259 "type": "journal",7260 "alternate_names": [7261 "Nat Hum Behav"7262 ],7263 "issn": "2397-3374",7264 "url": "https://www.nature.com/nathumbehav/"7265 },7266 "title": "A manifesto for reproducible science",7267 "venue": "Nature Human Behaviour",7268 "year": 2017,7269 "citationCount": 2932,7270 "openAccessPdf": {7271 "url": "https://www.nature.com/articles/s41562-016-0021.pdf",7272 "status": "HYBRID",7273 "license": "CCBY",7274 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC7610724, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."7275 },7276 "publicationTypes": [7277 "JournalArticle"7278 ],7279 "journal": {7280 "name": "Nature human behaviour",7281 "volume": "1"7282 },7283 "authors": [7284 {7285 "authorId": "3057626",7286 "name": "M. Munafo"7287 },7288 {7289 "authorId": "2862527",7290 "name": "Brian A. Nosek"7291 },7292 {7293 "authorId": "2064317886",7294 "name": "D. Bishop"7295 },7296 {7297 "authorId": "4058655",7298 "name": "K. Button"7299 },7300 {7301 "authorId": "49625330",7302 "name": "Christopher D. Chambers"7303 },7304 {7305 "authorId": "46181878",7306 "name": "N. P. Sert"7307 },7308 {7309 "authorId": "2130902",7310 "name": "U. Simonsohn"7311 },7312 {7313 "authorId": "2505272",7314 "name": "E. Wagenmakers"7315 },7316 {7317 "authorId": "3959303",7318 "name": "J. Ware"7319 },7320 {7321 "authorId": "145441750",7322 "name": "J. Ioannidis"7323 }7324 ],7325 "abstract": "Improving the reliability and efficiency of scientific research will increase the credibility of the published scientific literature and accelerate discovery. Here we argue for the adoption of measures to optimize key elements of the scientific process: methods, reporting and dissemination, reproducibility, evaluation and incentives. There is some evidence from both simulations and empirical studies supporting the likely effectiveness of these measures, but their broad adoption by researchers, institutions, funders and journals will require iterative evaluation and improvement. We discuss the goals of these measures, and how they can be implemented, in the hope that this will facilitate action toward improving the transparency, reproducibility and efficiency of scientific research."7326 },7327 "DOI:10.1038/s41562-021-01142-4": {7328 "paperId": "b870928f45680afa8e394c85bfaddd487cd6c4d2",7329 "externalIds": {7330 "DOI": "10.1038/s41562-021-01142-4",7331 "CorpusId": 235634310,7332 "PubMed": "34168323"7333 },7334 "publicationVenue": {7335 "id": "b7e5a3b3-2519-4ca2-8870-c2b45ef76ab9",7336 "name": "Nature Human Behaviour",7337 "type": "journal",7338 "alternate_names": [7339 "Nat Hum Behav"7340 ],7341 "issn": "2397-3374",7342 "url": "https://www.nature.com/nathumbehav/"7343 },7344 "title": "Initial evidence of research quality of registered reports compared with the standard publishing model",7345 "venue": "Nature Human Behaviour",7346 "year": 2021,7347 "citationCount": 187,7348 "openAccessPdf": {7349 "url": "https://doi.org/10.31222/osf.io/7x9vy",7350 "status": "GREEN",7351 "license": "public-domain",7352 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1038/s41562-021-01142-4?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1038/s41562-021-01142-4, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."7353 },7354 "publicationTypes": [7355 "JournalArticle",7356 "Review"7357 ],7358 "journal": {7359 "name": "Nature Human Behaviour",7360 "pages": "990 - 997",7361 "volume": "5"7362 },7363 "authors": [7364 {7365 "authorId": "4110370",7366 "name": "C. Soderberg"7367 },7368 {7369 "authorId": "52004348",7370 "name": "Timothy M. Errington"7371 },7372 {7373 "authorId": "40019962",7374 "name": "Sarah R. Schiavone"7375 },7376 {7377 "authorId": "30533087",7378 "name": "Julia G. Bottesini"7379 },7380 {7381 "authorId": "66300351",7382 "name": "Felix Singleton Thorn"7383 },7384 {7385 "authorId": "2809726",7386 "name": "S. Vazire"7387 },7388 {7389 "authorId": "47450657",7390 "name": "Kevin M. Esterling"7391 },7392 {7393 "authorId": "152190711",7394 "name": "B. Nosek"7395 }7396 ],7397 "abstract": "In registered reports (RRs), initial peer review and in-principle acceptance occur before knowing the research outcomes. This combats publication bias and distinguishes planned from unplanned research. How RRs could improve the credibility of research findings is straightforward, but there is little empirical evidence. Also, there could be unintended costs such as reducing novelty. Here, 353 researchers peer reviewed a pair of papers from 29 published RRs from psychology and neuroscience and 57 non-RR comparison papers. RRs numerically outperformed comparison papers on all 19 criteria (mean difference 0.46, scale range \u22124 to +4) with effects ranging from RRs being statistically indistinguishable from comparison papers in novelty (0.13, 95% credible interval [\u22120.24, 0.49]) and creativity (0.22, [\u22120.14, 0.58]) to sizeable improvements in rigour of methodology (0.99, [0.62, 1.35]) and analysis (0.97, [0.60, 1.34]) and overall paper quality (0.66, [0.30, 1.02]). RRs could improve research quality while reducing publication bias and ultimately improve the credibility of the published literature. Soderberg et al. asked scientists to peer review registered reports and standard articles post-publication, after information explicitly identifying the article type had been removed. Registered reports scored higher on some dimensions, including quality and rigour."7398 },7399 "DOI:10.1038/s41562-021-01193-7": {7400 "paperId": "08ed407fcb883c877bcae0b4bde71bb4ddaf4e50",7401 "externalIds": {7402 "DOI": "10.1038/s41562-021-01193-7",7403 "CorpusId": 243200855,7404 "PubMed": "34782730"7405 },7406 "publicationVenue": {7407 "id": "b7e5a3b3-2519-4ca2-8870-c2b45ef76ab9",7408 "name": "Nature Human Behaviour",7409 "type": "journal",7410 "alternate_names": [7411 "Nat Hum Behav"7412 ],7413 "issn": "2397-3374",7414 "url": "https://www.nature.com/nathumbehav/"7415 },7416 "title": "The past, present and future of Registered Reports",7417 "venue": "Nature Human Behaviour",7418 "year": 2020,7419 "citationCount": 366,7420 "openAccessPdf": {7421 "url": "https://www.nature.com/articles/s41562-021-01193-7.pdf",7422 "status": "HYBRID",7423 "license": "other-oa",7424 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1038/s41562-021-01193-7?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1038/s41562-021-01193-7, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."7425 },7426 "publicationTypes": [7427 "Review",7428 "JournalArticle"7429 ],7430 "journal": {7431 "name": "Nature Human Behaviour",7432 "pages": "29 - 42",7433 "volume": "6"7434 },7435 "authors": [7436 {7437 "authorId": "46975669",7438 "name": "C. Chambers"7439 },7440 {7441 "authorId": "52215687",7442 "name": "Loukia Tzavella"7443 }7444 ],7445 "abstract": null7446 },7447 "DOI:10.1038/s41562-024-02024-1": {7448 "paperId": "02aec976011a7d595d61db9639209968f4b93177",7449 "externalIds": {7450 "PubMedCentral": "11659167",7451 "ArXiv": "2405.06087",7452 "DOI": "10.1038/s41562-024-02024-1",7453 "CorpusId": 269741352,7454 "PubMed": "39468277"7455 },7456 "publicationVenue": {7457 "id": "b7e5a3b3-2519-4ca2-8870-c2b45ef76ab9",7458 "name": "Nature Human Behaviour",7459 "type": "journal",7460 "alternate_names": [7461 "Nat Hum Behav"7462 ],7463 "issn": "2397-3374",7464 "url": "https://www.nature.com/nathumbehav/"7465 },7466 "title": "When combinations of humans and AI are useful: A systematic review and meta-analysis",7467 "venue": "Nature Human Behaviour",7468 "year": 2024,7469 "citationCount": 662,7470 "openAccessPdf": {7471 "url": "https://www.nature.com/articles/s41562-024-02024-1.pdf",7472 "status": "HYBRID",7473 "license": "CCBY",7474 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2405.06087, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."7475 },7476 "publicationTypes": [7477 "JournalArticle",7478 "Review"7479 ],7480 "journal": {7481 "name": "Nature Human Behaviour",7482 "pages": "2293 - 2303",7483 "volume": "8"7484 },7485 "authors": [7486 {7487 "authorId": "153024015",7488 "name": "Michelle Vaccaro"7489 },7490 {7491 "authorId": "3085084",7492 "name": "Abdullah Almaatouq"7493 },7494 {7495 "authorId": "2257866472",7496 "name": "Thomas W. Malone"7497 }7498 ],7499 "abstract": "Inspired by the increasing use of artificial intelligence (AI) to augment humans, researchers have studied human\u2013AI systems involving different tasks, systems and populations. Despite such a large body of work, we lack a broad conceptual understanding of when combinations of humans and AI are better than either alone. Here we addressed this question by conducting a preregistered systematic review and meta-analysis of 106 experimental studies reporting 370 effect sizes. We searched an interdisciplinary set of databases (the Association for Computing Machinery Digital Library, the Web of Science and the Association for Information Systems eLibrary) for studies published between 1 January 2020 and 30 June 2023. Each study was required to include an original human-participants experiment that evaluated the performance of humans alone, AI alone and human\u2013AI combinations. First, we found that, on average, human\u2013AI combinations performed significantly worse than the best of humans or AI alone (Hedges\u2019 g\u2009=\u2009\u22120.23; 95% confidence interval, \u22120.39 to \u22120.07). Second, we found performance losses in tasks that involved making decisions and significantly greater gains in tasks that involved creating content. Finally, when humans outperformed AI alone, we found performance gains in the combination, but when AI outperformed humans alone, we found losses. Limitations of the evidence assessed here include possible publication bias and variations in the study designs analysed. Overall, these findings highlight the heterogeneity of the effects of human\u2013AI collaboration and point to promising avenues for improving human\u2013AI systems. Vaccaro et al. present a systematic review and meta-analysis of the performance of human\u2013AI combinations, finding that on average, human\u2013AI combinations performed significantly worse than the best of humans or AI alone. They also found performance losses in decision-making tasks and significantly greater gains in content creation tasks."7500 },7501 "DOI:10.1038/s41586-023-06221-2": {7502 "paperId": "f08060425aa8a212d74185ee23a08329b89abcd2",7503 "externalIds": {7504 "DBLP": "journals/nature/WangFD0HLCLKDAB23",7505 "DOI": "10.1038/s41586-023-06221-2",7506 "CorpusId": 260384616,7507 "PubMed": "37532811"7508 },7509 "publicationVenue": {7510 "id": "6c24a0a0-b07d-4d7b-a19b-fd09a3ed453a",7511 "name": "Nature",7512 "type": "journal",7513 "issn": "0028-0836",7514 "url": "https://www.nature.com/",7515 "alternate_urls": [7516 "http://www.nature.com/nature/",7517 "https://www.nature.com/nature/",7518 "http://www.nature.com/nature/archive/index.html"7519 ]7520 },7521 "title": "Scientific discovery in the age of artificial intelligence",7522 "venue": "Nature",7523 "year": 2023,7524 "citationCount": 2017,7525 "openAccessPdf": {7526 "url": "https://hdl.handle.net/1721.1/165433",7527 "status": "GREEN",7528 "license": "CCBYNC",7529 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1038/s41586-023-06221-2?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1038/s41586-023-06221-2, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."7530 },7531 "publicationTypes": [7532 "Review",7533 "JournalArticle"7534 ],7535 "journal": {7536 "name": "Nature",7537 "pages": "47-60",7538 "volume": "620"7539 },7540 "authors": [7541 {7542 "authorId": "46506460",7543 "name": "Hanchen Wang"7544 },7545 {7546 "authorId": "2427076",7547 "name": "Tianfan Fu"7548 },7549 {7550 "authorId": "93584228",7551 "name": "Yuanqi Du"7552 },7553 {7554 "authorId": "2153575781",7555 "name": "Wen-Hao Gao"7556 },7557 {7558 "authorId": "49454094",7559 "name": "Kexin Huang"7560 },7561 {7562 "authorId": "2145253202",7563 "name": "Zi-Ming Liu"7564 },7565 {7566 "authorId": "89356020",7567 "name": "Payal Chandak"7568 },7569 {7570 "authorId": "1563693999",7571 "name": "Sheng-Chao Liu"7572 },7573 {7574 "authorId": "11295186",7575 "name": "Peter L. Vankatwyk"7576 },7577 {7578 "authorId": "48860334",7579 "name": "Andreea Deac"7580 },7581 {7582 "authorId": "2047844",7583 "name": "Anima Anandkumar"7584 },7585 {7586 "authorId": "46354410",7587 "name": "K. Bergen"7588 },7589 {7590 "authorId": "2188778449",7591 "name": "Carla P. Gomes"7592 },7593 {7594 "authorId": "2226475487",7595 "name": "Shirley Ho"7596 },7597 {7598 "authorId": "143967473",7599 "name": "Pushmeet Kohli"7600 },7601 {7602 "authorId": "1764249",7603 "name": "Joan Lasenby"7604 },7605 {7606 "authorId": "1702139",7607 "name": "J. Leskovec"7608 },7609 {7610 "authorId": "2110264337",7611 "name": "Tie-Yan Liu"7612 },7613 {7614 "authorId": "2613935",7615 "name": "Arjun K. Manrai"7616 },7617 {7618 "authorId": "2226493959",7619 "name": "Debora S. Marks"7620 },7621 {7622 "authorId": "2378027",7623 "name": "B. Ramsundar"7624 },7625 {7626 "authorId": "2159642226",7627 "name": "Le Song"7628 },7629 {7630 "authorId": "2504051310",7631 "name": "Ji-Meng Sun"7632 },7633 {7634 "authorId": "2115854801",7635 "name": "Jian Tang"7636 },7637 {7638 "authorId": "3444569",7639 "name": "Petar Velickovic"7640 },7641 {7642 "authorId": "2228923747",7643 "name": "Max Welling"7644 },7645 {7646 "authorId": "2359205623",7647 "name": "Lin-Feng Zhang"7648 },7649 {7650 "authorId": "13027820",7651 "name": "Connor W. Coley"7652 },7653 {7654 "authorId": "1865800402",7655 "name": "Y. Bengio"7656 },7657 {7658 "authorId": "2095762",7659 "name": "M. Zitnik"7660 }7661 ],7662 "abstract": "Artificial intelligence (AI) is being increasingly integrated into scientific discovery to augment and accelerate research, helping scientists to generate hypotheses, design experiments, collect and interpret large datasets, and gain insights that might not have been possible using traditional scientific methods alone. Here we examine breakthroughs over the past decade that include self-supervised learning, which allows models to be trained on vast amounts of unlabelled data, and geometric deep learning, which leverages knowledge about the structure of scientific data to enhance model accuracy and efficiency. Generative AI methods can create designs, such as small-molecule drugs and proteins, by analysing diverse data modalities, including images and sequences. We discuss how these methods can help scientists throughout the scientific process and the central issues that remain despite such advances. Both developers and users of AI tools need a better understanding of when such approaches need improvement, and challenges posed by poor data quality and stewardship remain. These issues cut across scientific disciplines and require developing foundational algorithmic approaches that can contribute to scientific understanding or acquire it autonomously, making them critical areas of focus for AI innovation. The advances in artificial intelligence over the past decade are examined, with a discussion on how artificial intelligence systems can aid the scientific process and the central issues that remain despite advances."7663 },7664 "DOI:10.1038/s41586-023-06792-0": {7665 "paperId": "6fe3779fe5f2e9402abdd08ad8db41a0f13a99eb",7666 "externalIds": {7667 "DBLP": "journals/nature/BoikoMKG23",7668 "PubMedCentral": "10733136",7669 "DOI": "10.1038/s41586-023-06792-0",7670 "CorpusId": 266432059,7671 "PubMed": "38123806"7672 },7673 "publicationVenue": {7674 "id": "6c24a0a0-b07d-4d7b-a19b-fd09a3ed453a",7675 "name": "Nature",7676 "type": "journal",7677 "issn": "0028-0836",7678 "url": "https://www.nature.com/",7679 "alternate_urls": [7680 "http://www.nature.com/nature/",7681 "https://www.nature.com/nature/",7682 "http://www.nature.com/nature/archive/index.html"7683 ]7684 },7685 "title": "Autonomous chemical research with large language models",7686 "venue": "Nature",7687 "year": 2023,7688 "citationCount": 1510,7689 "openAccessPdf": {7690 "url": "https://www.nature.com/articles/s41586-023-06792-0.pdf",7691 "status": "HYBRID",7692 "license": "CCBY",7693 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC10733136, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."7694 },7695 "publicationTypes": [7696 "JournalArticle"7697 ],7698 "journal": {7699 "name": "Nature",7700 "pages": "570 - 578",7701 "volume": "624"7702 },7703 "authors": [7704 {7705 "authorId": "50771006",7706 "name": "Daniil A. Boiko"7707 },7708 {7709 "authorId": "152587056",7710 "name": "R. MacKnight"7711 },7712 {7713 "authorId": "2399498202",7714 "name": "Benjamin C Kline"7715 },7716 {7717 "authorId": "2181059489",7718 "name": "Gabe Gomes"7719 }7720 ],7721 "abstract": "Transformer-based large language models are making significant strides in various fields, such as natural language processing^ 1 \u2013 5 , biology^ 6 , 7 , chemistry^ 8 \u2013 10 and computer programming^ 11 , 12 . Here, we show the development and capabilities of Coscientist, an artificial intelligence system driven by GPT-4 that autonomously designs, plans and performs complex experiments by incorporating large language models empowered by tools such as internet and documentation search, code execution and experimental automation. Coscientist showcases its potential for accelerating research across six diverse tasks, including the successful reaction optimization of palladium-catalysed cross-couplings, while exhibiting advanced capabilities for (semi-)autonomous experimental design and execution. Our findings demonstrate the versatility, efficacy and explainability of artificial intelligence systems like Coscientist in advancing research. Coscientist is an artificial intelligence system driven by GPT-4 that autonomously designs, plans and performs experiments by incorporating large language models empowered by tools such as internet and documentation search, code execution and experimental automation."7722 },7723 "DOI:10.1038/s41586-025-09442-9": {7724 "paperId": "d24e37aafcf48c76aca30430670bad9a61cd0fca",7725 "externalIds": {7726 "DBLP": "journals/nature/SwansonWBP025",7727 "DOI": "10.1038/s41586-025-09442-9",7728 "CorpusId": 280389333,7729 "PubMed": "40730228"7730 },7731 "publicationVenue": {7732 "id": "6c24a0a0-b07d-4d7b-a19b-fd09a3ed453a",7733 "name": "Nature",7734 "type": "journal",7735 "issn": "0028-0836",7736 "url": "https://www.nature.com/",7737 "alternate_urls": [7738 "http://www.nature.com/nature/",7739 "https://www.nature.com/nature/",7740 "http://www.nature.com/nature/archive/index.html"7741 ]7742 },7743 "title": "The Virtual Lab of AI agents designs new SARS-CoV-2 nanobodies",7744 "venue": "Nature",7745 "year": 2025,7746 "citationCount": 349,7747 "openAccessPdf": {7748 "url": "",7749 "status": null,7750 "license": null,7751 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1038/s41586-025-09442-9?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1038/s41586-025-09442-9, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."7752 },7753 "publicationTypes": [7754 "JournalArticle"7755 ],7756 "journal": {7757 "name": "Nature",7758 "pages": "716 - 723",7759 "volume": "646"7760 },7761 "authors": [7762 {7763 "authorId": "2276778113",7764 "name": "Kyle Swanson"7765 },7766 {7767 "authorId": "2252322743",7768 "name": "Wesley Wu"7769 },7770 {7771 "authorId": "2330818080",7772 "name": "Nash L. Bulaong"7773 },7774 {7775 "authorId": "2248292433",7776 "name": "John E. Pak"7777 },7778 {7779 "authorId": "2204356217",7780 "name": "James Y. Zou"7781 }7782 ],7783 "abstract": null7784 },7785 "DOI:10.1038/s41586-026-10644-y": {7786 "paperId": "390c607723e9767a9a873ded3bc1f66808215e8e",7787 "externalIds": {7788 "ArXiv": "2502.18864",7789 "PubMedCentral": "13345910",7790 "DOI": "10.1038/s41586-026-10644-y",7791 "CorpusId": 276617649,7792 "PubMed": "42428019"7793 },7794 "publicationVenue": {7795 "id": "6c24a0a0-b07d-4d7b-a19b-fd09a3ed453a",7796 "name": "Nature",7797 "type": "journal",7798 "issn": "0028-0836",7799 "url": "https://www.nature.com/",7800 "alternate_urls": [7801 "http://www.nature.com/nature/",7802 "https://www.nature.com/nature/",7803 "http://www.nature.com/nature/archive/index.html"7804 ]7805 },7806 "title": "Accelerating scientific discovery with Co-Scientist",7807 "venue": "Nature",7808 "year": 2025,7809 "citationCount": 593,7810 "openAccessPdf": {7811 "url": "https://www.nature.com/articles/s41586-026-10644-y_reference.pdf",7812 "status": "HYBRID",7813 "license": "other-oa",7814 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2502.18864, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."7815 },7816 "publicationTypes": [7817 "JournalArticle"7818 ],7819 "journal": {7820 "name": "Nature",7821 "pages": "487 - 496",7822 "volume": "655"7823 },7824 "authors": [7825 {7826 "authorId": "2051681709",7827 "name": "Juraj Gottweis"7828 },7829 {7830 "authorId": "2239098855",7831 "name": "Wei-Hung Weng"7832 },7833 {7834 "authorId": "2347345594",7835 "name": "Alexander Daryin"7836 },7837 {7838 "authorId": "2244651851",7839 "name": "Tao Tu"7840 },7841 {7842 "authorId": "2679779",7843 "name": "Petar Sirkovi\u0107"7844 },7845 {7846 "authorId": "2346506169",7847 "name": "Artiom Myaskovsky"7848 },7849 {7850 "authorId": "2520452099",7851 "name": "Grzegorz Glowaty"7852 },7853 {7854 "authorId": "10805574",7855 "name": "Felix Weissenberger"7856 },7857 {7858 "authorId": "2367823022",7859 "name": "Alessio Orlandi"7860 },7861 {7862 "authorId": "2062591154",7863 "name": "D. Popovici"7864 },7865 {7866 "authorId": "2266472373",7867 "name": "Anil Palepu"7868 },7869 {7870 "authorId": "1996199677",7871 "name": "Ke-Ran Rong"7872 },7873 {7874 "authorId": "3480712",7875 "name": "Ryutaro Tanno"7876 },7877 {7878 "authorId": "2279021635",7879 "name": "Khaled Saab"7880 },7881 {7882 "authorId": "2298916585",7883 "name": "Fan Zhang"7884 },7885 {7886 "authorId": "2217621049",7887 "name": "Jacob Blum"7888 },7889 {7890 "authorId": "2336095347",7891 "name": "A. Carroll"7892 },7893 {7894 "authorId": "2269148145",7895 "name": "Kavita Kulkarni"7896 },7897 {7898 "authorId": "2213266",7899 "name": "Nenad Toma\u0161ev"7900 },7901 {7902 "authorId": "115731137",7903 "name": "D. Zverinski"7904 },7905 {7906 "authorId": "2389134",7907 "name": "Ivor Rendulic"7908 },7909 {7910 "authorId": "2298901106",7911 "name": "E. Vedadi"7912 },7913 {7914 "authorId": "113494403",7915 "name": "F. Hasler"7916 },7917 {7918 "authorId": "102638867",7919 "name": "Luka Rimanic"7920 },7921 {7922 "authorId": "36598773",7923 "name": "Marina Boia"7924 },7925 {7926 "authorId": "2784346",7927 "name": "Ivan Budiseli\u0107"7928 },7929 {7930 "authorId": "2074805000",7931 "name": "Ben Feinstein"7932 },7933 {7934 "authorId": "2359151678",7935 "name": "M. Bellaiche"7936 },7937 {7938 "authorId": "2220891317",7939 "name": "Tom Sheffer"7940 },7941 {7942 "authorId": "2279022140",7943 "name": "Jan Freyberg"7944 },7945 {7946 "authorId": "2255314955",7947 "name": "Jeremy D. Ratcliff"7948 },7949 {7950 "authorId": "1789306",7951 "name": "Ottavia Bertolli"7952 },7953 {7954 "authorId": "2269148566",7955 "name": "Katherine Chou"7956 },7957 {7958 "authorId": "1809983",7959 "name": "A. Hassidim"7960 },7961 {7962 "authorId": "2225921342",7963 "name": "Burak Gokturk"7964 },7965 {7966 "authorId": "2257189937",7967 "name": "Amin Vahdat"7968 },7969 {7970 "authorId": "2261405027",7971 "name": "Yuan Guan"7972 },7973 {7974 "authorId": "12268808",7975 "name": "Vikram Dhillon"7976 },7977 {7978 "authorId": "10664723",7979 "name": "E. D. Vaishnav"7980 },7981 {7982 "authorId": "2305695545",7983 "name": "Byron Lee"7984 },7985 {7986 "authorId": "2346514434",7987 "name": "Tiago R D Costa"7988 },7989 {7990 "authorId": "2068738124",7991 "name": "J. Penad\u00e9s"7992 },7993 {7994 "authorId": "2243119549",7995 "name": "Gary Peltz"7996 },7997 {7998 "authorId": "2269148232",7999 "name": "Y. Matias"8000 },8001 {8002 "authorId": "2254252286",8003 "name": "J. Manyika"8004 },8005 {8006 "authorId": "48987704",8007 "name": "D. Hassabis"8008 },8009 {8010 "authorId": "2275191526",8011 "name": "Yunhan Xu"8012 },8013 {8014 "authorId": "143967473",8015 "name": "Pushmeet Kohli"8016 },8017 {8018 "authorId": "2359249680",8019 "name": "Annalisa Pawlosky"8020 },8021 {8022 "authorId": "2213433382",8023 "name": "A. Karthikesalingam"8024 },8025 {8026 "authorId": "144223091",8027 "name": "Vivek Natarajan"8028 }8029 ],8030 "abstract": "Scientific discovery is driven by scientists generating hypotheses for complex problems that undergo rigorous experimental validation. To augment this process, we introduce Co-Scientist, a multi-agent artificial intelligence (AI) system built on Gemini for structured scientific thinking and hypothesis generation. Co-Scientist aims to help scientists discover new original knowledge. Conditioned on their research objectives and previous scientific evidence, it formulates demonstrably novel research hypotheses for experimental verification. The system\u2019s design involves agents continuously generating, critiquing and refining hypotheses accelerated by scaling test-time compute. Key contributions include (1) a multi-agent architecture with an asynchronous task execution framework for flexible compute scaling, and (2) a tournament evolution process for self-improving hypotheses generation. Automated evaluations show continued benefits of test-time compute scaling, improving hypothesis quality over time. Although this is a general-purpose system, we focus the validation in three biomedical applications: drug repurposing; novel-target discovery1; and explaining mechanisms of antimicrobial resistance2. Specifically, Co-Scientist helped to identify new drug-repurposing candidates and synergistic combination therapies for acute myeloid leukaemia that were validated through in vitro experiments. These real-world validations demonstrate the potential of Co-Scientist to accelerate scientific discovery and usher in an era of AI-empowered scientists. Co-Scientist is an AI system for structured scientific thinking and generates research hypotheses for experimental verification."8031 },8032 "DOI:10.1038/s41597-022-01143-6": {8033 "paperId": "172361c8b70a3ce63f7d6177473f2ccd5b5216b2",8034 "externalIds": {8035 "ArXiv": "2103.12793",8036 "DBLP": "journals/corr/abs-2103-12793",8037 "PubMedCentral": "8861064",8038 "DOI": "10.1038/s41597-022-01143-6",8039 "CorpusId": 232335674,8040 "PubMed": "35190569"8041 },8042 "publicationVenue": {8043 "id": "62924b2a-8fb8-4b93-92c6-735516b49af0",8044 "name": "Scientific Data",8045 "type": "journal",8046 "alternate_names": [8047 "Sci Data"8048 ],8049 "issn": "2052-4463",8050 "url": "http://www.nature.com/sdata/"8051 },8052 "title": "A large-scale study on research code quality and execution",8053 "venue": "Scientific Data",8054 "year": 2021,8055 "citationCount": 153,8056 "openAccessPdf": {8057 "url": "https://www.nature.com/articles/s41597-022-01143-6.pdf",8058 "status": "GOLD",8059 "license": "CCBY",8060 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2103.12793, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."8061 },8062 "publicationTypes": [8063 "JournalArticle"8064 ],8065 "journal": {8066 "name": "Scientific Data",8067 "volume": "9"8068 },8069 "authors": [8070 {8071 "authorId": "152233355",8072 "name": "A. Tri\u0161ovi\u0107"8073 },8074 {8075 "authorId": "2621285",8076 "name": "Matthew K. Lau"8077 },8078 {8079 "authorId": "3176695",8080 "name": "Thomas Pasquier"8081 },8082 {8083 "authorId": "3000963",8084 "name": "M. Crosas"8085 }8086 ],8087 "abstract": "This article presents a study on the quality and execution of research code from publicly-available replication datasets at the Harvard Dataverse repository. Research code is typically created by a group of scientists and published together with academic papers to facilitate research transparency and reproducibility. For this study, we define ten questions to address aspects impacting research reproducibility and reuse. First, we retrieve and analyze more than 2000 replication datasets with over 9000 unique R files published from 2010 to 2020. Second, we execute the code in a clean runtime environment to assess its ease of reuse. Common coding errors were identified, and some of them were solved with automatic code cleaning to aid code execution. We find that 74% of R files failed to complete without error in the initial execution, while 56% failed when code cleaning was applied, showing that many errors can be prevented with good coding practices. We also analyze the replication datasets from journals\u2019 collections and discuss the impact of the journal policy strictness on the code re-execution rate. Finally, based on our results, we propose a set of recommendations for code dissemination aimed at researchers, journals, and repositories."8088 },8089 "DOI:10.1038/s41598-023-41032-5": {8090 "paperId": "51dd1ebcebafce801b8856c925dd205f738d8f74",8091 "externalIds": {8092 "PubMedCentral": "10484980",8093 "DOI": "10.1038/s41598-023-41032-5",8094 "CorpusId": 261608321,8095 "PubMed": "37679503"8096 },8097 "publicationVenue": {8098 "id": "f99f77b7-b1b6-44d3-984a-f288e9884b9b",8099 "name": "Scientific Reports",8100 "type": "journal",8101 "alternate_names": [8102 "Sci Rep"8103 ],8104 "issn": "2045-2322",8105 "url": "http://www.nature.com/srep/",8106 "alternate_urls": [8107 "http://www.nature.com/srep/index.html"8108 ]8109 },8110 "title": "Fabrication and errors in the bibliographic citations generated by ChatGPT",8111 "venue": "Scientific Reports",8112 "year": 2023,8113 "citationCount": 556,8114 "openAccessPdf": {8115 "url": "https://www.nature.com/articles/s41598-023-41032-5.pdf",8116 "status": "GOLD",8117 "license": "CCBY",8118 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC10484980, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."8119 },8120 "publicationTypes": [8121 "JournalArticle",8122 "Review"8123 ],8124 "journal": {8125 "name": "Scientific Reports",8126 "volume": "13"8127 },8128 "authors": [8129 {8130 "authorId": "2238416484",8131 "name": "William H. Walters"8132 },8133 {8134 "authorId": "2917043",8135 "name": "E. I. Wilder"8136 }8137 ],8138 "abstract": "Although chatbots such as ChatGPT can facilitate cost-effective text generation and editing, factually incorrect responses (hallucinations) limit their utility. This study evaluates one particular type of hallucination: fabricated bibliographic citations that do not represent actual scholarly works. We used ChatGPT-3.5 and ChatGPT-4 to produce short literature reviews on 42 multidisciplinary topics, compiling data on the 636 bibliographic citations (references) found in the 84 papers. We then searched multiple databases and websites to determine the prevalence of fabricated citations, to identify errors in the citations to non-fabricated papers, and to evaluate adherence to APA citation format. Within this set of documents, 55% of the GPT-3.5 citations but just 18% of the GPT-4 citations are fabricated. Likewise, 43% of the real (non-fabricated) GPT-3.5 citations but just 24% of the real GPT-4 citations include substantive citation errors. Although GPT-4 is a major improvement over GPT-3.5, problems remain."8139 },8140 "DOI:10.1038/s42256-024-00832-8": {8141 "paperId": "354dcdebf3f8b5feeed5c62090e0bc1f0c28db06",8142 "externalIds": {8143 "DBLP": "journals/natmi/BranCSBWS24",8144 "ArXiv": "2304.05376",8145 "PubMedCentral": "11116106",8146 "DOI": "10.1038/s42256-024-00832-8",8147 "CorpusId": 258059792,8148 "PubMed": "38799228"8149 },8150 "publicationVenue": {8151 "id": "6457124b-39bf-4d02-bff4-73752ff21562",8152 "name": "Nature Machine Intelligence",8153 "type": "journal",8154 "alternate_names": [8155 "Nat Mach Intell"8156 ],8157 "issn": "2522-5839",8158 "url": "https://www.nature.com/natmachintell/"8159 },8160 "title": "Augmenting large language models with chemistry tools",8161 "venue": "Nature Machine Intelligence",8162 "year": 2023,8163 "citationCount": 1414,8164 "openAccessPdf": {8165 "url": "https://www.nature.com/articles/s42256-024-00832-8.pdf",8166 "status": "HYBRID",8167 "license": "CCBY",8168 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2304.05376, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."8169 },8170 "publicationTypes": [8171 "JournalArticle"8172 ],8173 "journal": {8174 "name": "Nature Machine Intelligence",8175 "pages": "525 - 535",8176 "volume": "6"8177 },8178 "authors": [8179 {8180 "authorId": "2216007369",8181 "name": "Andr\u00e9s M Bran"8182 },8183 {8184 "authorId": "2161337138",8185 "name": "Sam Cox"8186 },8187 {8188 "authorId": "1820929773",8189 "name": "Oliver Schilter"8190 },8191 {8192 "authorId": "2291607243",8193 "name": "Carlo Baldassari"8194 },8195 {8196 "authorId": "2150199535",8197 "name": "Andrew D. White"8198 },8199 {8200 "authorId": "1379965853",8201 "name": "P. Schwaller"8202 }8203 ],8204 "abstract": "Large language models (LLMs) have shown strong performance in tasks across domains but struggle with chemistry-related problems. These models also lack access to external knowledge sources, limiting their usefulness in scientific applications. We introduce ChemCrow, an LLM chemistry agent designed to accomplish tasks across organic synthesis, drug discovery and materials design. By integrating 18 expert-designed tools and using GPT-4 as the LLM, ChemCrow augments the LLM performance in chemistry, and new capabilities emerge. Our agent autonomously planned and executed the syntheses of an insect repellent and three organocatalysts and guided the discovery of a novel chromophore. Our evaluation, including both LLM and expert assessments, demonstrates ChemCrow\u2019s effectiveness in automating a diverse set of chemical tasks. Our work not only aids expert chemists and lowers barriers for non-experts but also fosters scientific advancement by bridging the gap between experimental and computational chemistry. Large language models can be queried to perform chain-of-thought reasoning on text descriptions of data or computational tools, which can enable flexible and autonomous workflows. Bran et al. developed ChemCrow, a GPT-4-based agent that has access to computational chemistry tools and a robotic chemistry platform, which can autonomously solve tasks for designing or synthesizing chemicals such as drugs or materials."8205 },8206 "DOI:10.1038/sdata.2016.18": {8207 "paperId": "e936f248b2c0489316ed1521656af2564c3502c3",8208 "externalIds": {8209 "MAG": "3083522027",8210 "PubMedCentral": "4792175",8211 "DOI": "10.1038/sdata.2016.18",8212 "CorpusId": 8755162,8213 "PubMed": "26978244"8214 },8215 "publicationVenue": {8216 "id": "62924b2a-8fb8-4b93-92c6-735516b49af0",8217 "name": "Scientific Data",8218 "type": "journal",8219 "alternate_names": [8220 "Sci Data"8221 ],8222 "issn": "2052-4463",8223 "url": "http://www.nature.com/sdata/"8224 },8225 "title": "The FAIR Guiding Principles for scientific data management and stewardship",8226 "venue": "Scientific Data",8227 "year": 2016,8228 "citationCount": 17558,8229 "openAccessPdf": {8230 "url": "https://www.nature.com/articles/sdata201618.pdf",8231 "status": "GOLD",8232 "license": "CCBY",8233 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC4792175, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."8234 },8235 "publicationTypes": [8236 "JournalArticle"8237 ],8238 "journal": {8239 "name": "Scientific Data",8240 "volume": "3"8241 },8242 "authors": [8243 {8244 "authorId": "31764087",8245 "name": "M. Wilkinson"8246 },8247 {8248 "authorId": "1736389",8249 "name": "M. Dumontier"8250 },8251 {8252 "authorId": "1975615",8253 "name": "I. J. Aalbersberg"8254 },8255 {8256 "authorId": "74186014",8257 "name": "G. Appleton"8258 },8259 {8260 "authorId": "4006952",8261 "name": "M. Axton"8262 },8263 {8264 "authorId": "20313879",8265 "name": "A. Baak"8266 },8267 {8268 "authorId": "2776517",8269 "name": "N. Blomberg"8270 },8271 {8272 "authorId": "34945757",8273 "name": "J. Boiten"8274 },8275 {8276 "authorId": "115529154",8277 "name": "Luiz Olavo Bonino da Silva Santos"8278 },8279 {8280 "authorId": "2048644",8281 "name": "P. Bourne"8282 },8283 {8284 "authorId": "47611747",8285 "name": "J. Bouwman"8286 },8287 {8288 "authorId": "2160154",8289 "name": "A. Brookes"8290 },8291 {8292 "authorId": "145469718",8293 "name": "Tim Clark"8294 },8295 {8296 "authorId": "3000963",8297 "name": "M. Crosas"8298 },8299 {8300 "authorId": "145479127",8301 "name": "I. Dillo"8302 },8303 {8304 "authorId": "116235596",8305 "name": "O. Dumon"8306 },8307 {8308 "authorId": "1940242",8309 "name": "S. Edmunds"8310 },8311 {8312 "authorId": "2185817",8313 "name": "C. Evelo"8314 },8315 {8316 "authorId": "2425367",8317 "name": "R. Finkers"8318 },8319 {8320 "authorId": "1401553965",8321 "name": "A. Gonz\u00e1lez-Beltr\u00e1n"8322 },8323 {8324 "authorId": "34071005",8325 "name": "A. Gray"8326 },8327 {8328 "authorId": "143772714",8329 "name": "Paul Groth"8330 },8331 {8332 "authorId": "46555127",8333 "name": "C. Goble"8334 },8335 {8336 "authorId": "1736528",8337 "name": "J. Grethe"8338 },8339 {8340 "authorId": "1699372",8341 "name": "J. Heringa"8342 },8343 {8344 "authorId": "1396879481",8345 "name": "P. 't Hoen"8346 },8347 {8348 "authorId": "2359250",8349 "name": "R. Hooft"8350 },8351 {8352 "authorId": "145727607",8353 "name": "T. Kuhn"8354 },8355 {8356 "authorId": "47044989",8357 "name": "R. Kok"8358 },8359 {8360 "authorId": "1713683",8361 "name": "Joost N. Kok"8362 },8363 {8364 "authorId": "4297975",8365 "name": "S. Lusher"8366 },8367 {8368 "authorId": "2969738",8369 "name": "M. Martone"8370 },8371 {8372 "authorId": "6027579",8373 "name": "Albert Mons"8374 },8375 {8376 "authorId": "2455243",8377 "name": "A. Packer"8378 },8379 {8380 "authorId": "2059300547",8381 "name": "Bengt Persson"8382 },8383 {8384 "authorId": "1397958214",8385 "name": "P. Rocca-Serra"8386 },8387 {8388 "authorId": "144140870",8389 "name": "M. Roos"8390 },8391 {8392 "authorId": "2480475903",8393 "name": "R. V. van Schaik"8394 },8395 {8396 "authorId": "1760490",8397 "name": "S. Sansone"8398 },8399 {8400 "authorId": "1883372",8401 "name": "E. Schultes"8402 },8403 {8404 "authorId": "2510671",8405 "name": "T. Sengstag"8406 },8407 {8408 "authorId": "38261192",8409 "name": "Ted Slater"8410 },8411 {8412 "authorId": "2013713",8413 "name": "G. Strawn"8414 },8415 {8416 "authorId": "144034058",8417 "name": "M. Swertz"8418 },8419 {8420 "authorId": "143962914",8421 "name": "Mark Thompson"8422 },8423 {8424 "authorId": "84257307",8425 "name": "J. van der Lei"8426 },8427 {8428 "authorId": "11563451",8429 "name": "E. V. van Mulligen"8430 },8431 {8432 "authorId": "3261874",8433 "name": "J. Velterop"8434 },8435 {8436 "authorId": "48709451",8437 "name": "A. Waagmeester"8438 },8439 {8440 "authorId": "1681130",8441 "name": "P. Wittenburg"8442 },8443 {8444 "authorId": "143780913",8445 "name": "K. Wolstencroft"8446 },8447 {8448 "authorId": "48019160",8449 "name": "Jun Zhao"8450 },8451 {8452 "authorId": "1995836",8453 "name": "B. Mons"8454 }8455 ],8456 "abstract": "There is an urgent need to improve the infrastructure supporting the reuse of scholarly data. A diverse set of stakeholders\u2014representing academia, industry, funding agencies, and scholarly publishers\u2014have come together to design and jointly endorse a concise and measureable set of principles that we refer to as the FAIR Data Principles. The intent is that these may act as a guideline for those wishing to enhance the reusability of their data holdings. Distinct from peer initiatives that focus on the human scholar, the FAIR Principles put specific emphasis on enhancing the ability of machines to automatically find and use the data, in addition to supporting its reuse by individuals. This Comment is the first formal publication of the FAIR Principles, and includes the rationale behind them, and some exemplar implementations in the community."8457 },8458 "DOI:10.1056/AIoa2400555": null,8459 "DOI:10.1073/pnas.1708274114": {8460 "paperId": "5465884a4aad5dc256ce5b4bfacdfa7cdc2c78ee",8461 "externalIds": {8462 "MAG": "2779812635",8463 "DOI": "10.1073/pnas.1708274114",8464 "CorpusId": 4639380,8465 "PubMed": "29531091"8466 },8467 "publicationVenue": {8468 "id": "bb95bf2e-8383-4748-bf9d-d6906d091085",8469 "name": "Proceedings of the National Academy of Sciences of the United States of America",8470 "type": "journal",8471 "alternate_names": [8472 "PNAS",8473 "PNAS online",8474 "Proceedings of the National Academy of Sciences of the United States of America.",8475 "Proc National Acad Sci",8476 "Proceedings of the National Academy of Sciences",8477 "Proc National Acad Sci u s Am"8478 ],8479 "issn": "0027-8424",8480 "alternate_issns": [8481 "1091-6490"8482 ],8483 "url": "https://www.jstor.org/journal/procnatiacadscie",8484 "alternate_urls": [8485 "http://www.pnas.org/",8486 "https://www.pnas.org/",8487 "http://www.jstor.org/journals/00278424.html",8488 "www.pnas.org/"8489 ]8490 },8491 "title": "The preregistration revolution",8492 "venue": "Proceedings of the National Academy of Sciences of the United States of America",8493 "year": 2018,8494 "citationCount": 1775,8495 "openAccessPdf": {8496 "url": "https://doi.org/10.31219/osf.io/2dxu5",8497 "status": "GREEN",8498 "license": "public-domain",8499 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1073/pnas.1708274114?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1073/pnas.1708274114, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."8500 },8501 "publicationTypes": [8502 "JournalArticle"8503 ],8504 "journal": {8505 "name": "Proceedings of the National Academy of Sciences",8506 "pages": "2600 - 2606",8507 "volume": "115"8508 },8509 "authors": [8510 {8511 "authorId": "2862527",8512 "name": "Brian A. Nosek"8513 },8514 {8515 "authorId": "7004092",8516 "name": "C. Ebersole"8517 },8518 {8519 "authorId": "15910942",8520 "name": "A. DeHaven"8521 },8522 {8523 "authorId": "35395330",8524 "name": "D. Mellor"8525 }8526 ],8527 "abstract": "Progress in science relies in part on generating hypotheses with existing observations and testing hypotheses with new observations. This distinction between postdiction and prediction is appreciated conceptually but is not respected in practice. Mistaking generation of postdictions with testing of predictions reduces the credibility of research findings. However, ordinary biases in human reasoning, such as hindsight bias, make it hard to avoid this mistake. An effective solution is to define the research questions and analysis plan before observing the research outcomes\u2014a process called preregistration. Preregistration distinguishes analyses and outcomes that result from predictions from those that result from postdictions. A variety of practical strategies are available to make the best possible use of preregistration in circumstances that fall short of the ideal application, such as when the data are preexisting. Services are now available for preregistration across all disciplines, facilitating a rapid increase in the practice. Widespread adoption of preregistration will increase distinctiveness between hypothesis generation and hypothesis testing and will improve the credibility of research findings."8528 },8529 "DOI:10.1073/pnas.1708290115": {8530 "paperId": "a7c2d6b4710ea409b14a716616a9caf466adb2f6",8531 "externalIds": {8532 "MAG": "2793993424",8533 "DBLP": "journals/pnas/StoddenSM18",8534 "DOI": "10.1073/pnas.1708290115",8535 "CorpusId": 3850646,8536 "PubMed": "29531050"8537 },8538 "publicationVenue": {8539 "id": "bb95bf2e-8383-4748-bf9d-d6906d091085",8540 "name": "Proceedings of the National Academy of Sciences of the United States of America",8541 "type": "journal",8542 "alternate_names": [8543 "PNAS",8544 "PNAS online",8545 "Proceedings of the National Academy of Sciences of the United States of America.",8546 "Proc National Acad Sci",8547 "Proceedings of the National Academy of Sciences",8548 "Proc National Acad Sci u s Am"8549 ],8550 "issn": "0027-8424",8551 "alternate_issns": [8552 "1091-6490"8553 ],8554 "url": "https://www.jstor.org/journal/procnatiacadscie",8555 "alternate_urls": [8556 "http://www.pnas.org/",8557 "https://www.pnas.org/",8558 "http://www.jstor.org/journals/00278424.html",8559 "www.pnas.org/"8560 ]8561 },8562 "title": "An empirical analysis of journal policy effectiveness for computational reproducibility",8563 "venue": "Proceedings of the National Academy of Sciences of the United States of America",8564 "year": 2018,8565 "citationCount": 342,8566 "openAccessPdf": {8567 "url": "https://www.pnas.org/content/pnas/115/11/2584.full.pdf",8568 "status": "BRONZE",8569 "license": null,8570 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1073/pnas.1708290115?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1073/pnas.1708290115, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."8571 },8572 "publicationTypes": [8573 "JournalArticle"8574 ],8575 "journal": {8576 "name": "Proceedings of the National Academy of Sciences",8577 "pages": "2584 - 2589",8578 "volume": "115"8579 },8580 "authors": [8581 {8582 "authorId": "2598812",8583 "name": "V. Stodden"8584 },8585 {8586 "authorId": "2058677280",8587 "name": "Jennifer Seiler"8588 },8589 {8590 "authorId": "2118105221",8591 "name": "Zhaokun Ma"8592 }8593 ],8594 "abstract": null8595 },8596 "DOI:10.1088/1742-6596/46/1/067": {8597 "paperId": "3f2927d1ea7ff47361b24203ebf62658ffa42cbb",8598 "externalIds": {8599 "MAG": "2116115793",8600 "DOI": "10.1088/1742-6596/46/1/067",8601 "CorpusId": 627620118602 },8603 "publicationVenue": null,8604 "title": "Berkeley Lab Checkpoint/Restart (BLCR) for Linux Clusters",8605 "venue": "",8606 "year": 2006,8607 "citationCount": 511,8608 "openAccessPdf": {8609 "url": "https://iopscience.iop.org/article/10.1088/1742-6596/46/1/067/pdf",8610 "status": "GOLD",8611 "license": null,8612 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1088/1742-6596/46/1/067?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1088/1742-6596/46/1/067, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."8613 },8614 "publicationTypes": null,8615 "journal": {8616 "name": "",8617 "pages": "494-499",8618 "volume": "46"8619 },8620 "authors": [8621 {8622 "authorId": "145980631",8623 "name": "Paul H. Hargrove"8624 },8625 {8626 "authorId": "4871994",8627 "name": "J. Duell"8628 }8629 ],8630 "abstract": "This article describes the motivation, design and implementation of Berkeley Lab Checkpoint/Restart (BLCR), a system-level checkpoint/restart implementation for Linux clusters that targets the space of typical High Performance Computing applications, including MPI. Application-level solutions, including both checkpointing and fault-tolerant algorithms, are recognized as more time and space efficient than system-level checkpoints, which cannot make use of any application-specific knowledge. However, system-level checkpointing allows for preemption, making it suitable for responding to ''fault precursors'' (for instance, elevated error rates from ECC memory or network CRCs, or elevated temperature from sensors). Preemption can also increase the efficiency of batch scheduling; for instance reducing idle cycles (by allowing for shutdown without any queue draining period or reallocation of resources to eliminate idle nodes when better fitting jobs are queued), and reducing the average queued time (by limiting large jobs to running during off-peak hours, without the need to limit the length of such jobs). Each of these potential uses makes BLCR a valuable tool for efficient resource management in Linux clusters."8631 },8632 "DOI:10.1093/bioinformatics/bts480": null,8633 "DOI:10.1098/rsta.2020.0069": {8634 "paperId": "ba7655c2176c35e278eeaae0aab041ab1cb74d0a",8635 "externalIds": {8636 "PubMedCentral": "8059663",8637 "DOI": "10.1098/rsta.2020.0069",8638 "CorpusId": 232387097,8639 "PubMed": "33775145"8640 },8641 "publicationVenue": null,8642 "title": "Learning from reproducing computational results: introducing three principles and the Reproduction Package",8643 "venue": "Philosophical Transactions of the Royal Society A",8644 "year": 2021,8645 "citationCount": 41,8646 "openAccessPdf": {8647 "url": "https://doi.org/10.1098/rsta.2020.0069",8648 "status": "HYBRID",8649 "license": "CCBY",8650 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC8059663, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."8651 },8652 "publicationTypes": [8653 "JournalArticle"8654 ],8655 "journal": {8656 "name": "Philosophical transactions. Series A, Mathematical, physical, and engineering sciences",8657 "volume": "379"8658 },8659 "authors": [8660 {8661 "authorId": "36999319",8662 "name": "Matthew Scott Krafczyk"8663 },8664 {8665 "authorId": "40128780",8666 "name": "A. Shi"8667 },8668 {8669 "authorId": "9383151",8670 "name": "A. Bhaskar"8671 },8672 {8673 "authorId": "1718171",8674 "name": "D. Marinov"8675 },8676 {8677 "authorId": "2598812",8678 "name": "V. Stodden"8679 }8680 ],8681 "abstract": "We carry out efforts to reproduce computational results for seven published articles and identify barriers to computational reproducibility. We then derive three principles to guide the practice and dissemination of reproducible computational research: (i) Provide transparency regarding how computational results are produced; (ii) When writing and releasing research software, aim for ease of (re-)executability; (iii) Make any code upon which the results rely as deterministic as possible. We then exemplify these three principles with 12 specific guidelines for their implementation in practice. We illustrate the three principles of reproducible research with a series of vignettes from our experimental reproducibility work. We define a novel Reproduction Package, a formalism that specifies a structured way to share computational research artifacts that implements the guidelines generated from our reproduction efforts to allow others to build, reproduce and extend computational science. We make our reproduction efforts in this paper publicly available as exemplar Reproduction Packages. This article is part of the theme issue \u2018Reliability and reproducibility in computational science: implementing verification, validation and uncertainty quantification in silico\u2019."8682 },8683 "DOI:10.1109/2.84898": {8684 "paperId": "6953691c2f3c51a2d3cf0905a264673aaa1aca7a",8685 "externalIds": {8686 "DBLP": "journals/computer/GrayS91",8687 "MAG": "2055631879",8688 "DOI": "10.1109/2.84898",8689 "CorpusId": 75507618690 },8691 "publicationVenue": {8692 "id": "f6572f66-2623-4a5e-b0d9-4a5028dea98f",8693 "name": "Computer",8694 "type": "journal",8695 "alternate_names": [8696 "IEEE Computer",8697 "IEEE Comput"8698 ],8699 "issn": "0018-9162",8700 "url": "http://www.computer.org/computer",8701 "alternate_urls": [8702 "https://ieeexplore.ieee.org/servlet/opac?punumber=2",8703 "http://www.computer.org/portal/site/ieeecs/index.jsp",8704 "https://ieeexplore.ieee.org/xpl/RecentIssue.jsp?punumber=2"8705 ]8706 },8707 "title": "High-availability computer systems",8708 "venue": "Computer",8709 "year": 1991,8710 "citationCount": 379,8711 "openAccessPdf": {8712 "url": "http://www.cs.wisc.edu/areas/os/Qual/papers/ha-systems.pdf",8713 "status": "GREEN",8714 "license": null,8715 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1109/2.84898?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1109/2.84898, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."8716 },8717 "publicationTypes": [8718 "JournalArticle"8719 ],8720 "journal": {8721 "name": "Computer",8722 "pages": "39-48",8723 "volume": "24"8724 },8725 "authors": [8726 {8727 "authorId": "39941882",8728 "name": "Jim Gray"8729 },8730 {8731 "authorId": "1742634",8732 "name": "D. Siewiorek"8733 }8734 ],8735 "abstract": null8736 },8737 "DOI:10.1109/3468.844354": {8738 "paperId": "14ae6f2231e09e226b99002aa04b5c70f3c59f2b",8739 "externalIds": {8740 "MAG": "2123179704",8741 "DBLP": "journals/tsmc/ParasuramanSW00",8742 "DOI": "10.1109/3468.844354",8743 "CorpusId": 8664574,8744 "PubMed": "11760769"8745 },8746 "publicationVenue": null,8747 "title": "A model for types and levels of human interaction with automation",8748 "venue": "IEEE Trans. Syst. Man Cybern. Part A",8749 "year": 2000,8750 "citationCount": 4495,8751 "openAccessPdf": {8752 "url": "http://www.cs.uml.edu/~holly/91.549/readings/sheridan-autonomy.pdf",8753 "status": "GREEN",8754 "license": null,8755 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1109/3468.844354?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1109/3468.844354, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."8756 },8757 "publicationTypes": [8758 "JournalArticle"8759 ],8760 "journal": {8761 "name": "IEEE transactions on systems, man, and cybernetics. Part A, Systems and humans : a publication of the IEEE Systems, Man, and Cybernetics Society",8762 "pages": "\n 286-97\n ",8763 "volume": "30 3"8764 },8765 "authors": [8766 {8767 "authorId": "3264674",8768 "name": "R. Parasuraman"8769 },8770 {8771 "authorId": "1712194",8772 "name": "T. Sheridan"8773 },8774 {8775 "authorId": "1772836",8776 "name": "C. Wickens"8777 }8778 ],8779 "abstract": null8780 },8781 "DOI:10.1109/MS.2016.60": {8782 "paperId": "e62070da1b348a7ea439a12b54e3f8b3223ee8a5",8783 "externalIds": {8784 "MAG": "2358431938",8785 "DBLP": "journals/corr/BasiriBRHKRR17",8786 "ArXiv": "1702.05843",8787 "DOI": "10.1109/MS.2016.60",8788 "CorpusId": 162940678789 },8790 "publicationVenue": {8791 "id": "119227a6-1b94-433e-a52d-8445a387dbbe",8792 "name": "IEEE Software",8793 "type": "journal",8794 "alternate_names": [8795 "IEEE Softw"8796 ],8797 "issn": "0740-7459",8798 "url": "http://www.computer.org/software",8799 "alternate_urls": [8800 "http://ieeexplore.ieee.org/servlet/opac?punumber=52"8801 ]8802 },8803 "title": "Chaos Engineering",8804 "venue": "IEEE Software",8805 "year": 2017,8806 "citationCount": 327,8807 "openAccessPdf": {8808 "url": "https://arxiv.org/pdf/1702.05843",8809 "status": "GREEN",8810 "license": null,8811 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/1702.05843, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."8812 },8813 "publicationTypes": [8814 "JournalArticle"8815 ],8816 "journal": {8817 "name": "ArXiv",8818 "volume": "abs/1702.05843"8819 },8820 "authors": [8821 {8822 "authorId": "2040887090",8823 "name": "Ali Basiri"8824 },8825 {8826 "authorId": "3395694",8827 "name": "Niosha Behnam"8828 },8829 {8830 "authorId": "2481279147",8831 "name": "R. D. de Rooij"8832 },8833 {8834 "authorId": "2030762",8835 "name": "L. Hochstein"8836 },8837 {8838 "authorId": "3395609",8839 "name": "Luke Kosewski"8840 },8841 {8842 "authorId": "2068158419",8843 "name": "J. Reynolds"8844 },8845 {8846 "authorId": "40810907",8847 "name": "Casey Rosenthal"8848 }8849 ],8850 "abstract": "Modern software-based services are implemented as distributed systems with complex behavior and failure modes. Many large tech organizations are using experimentation to verify such systems' reliability. Netflix engineers call this approach chaos engineering. They've determined several principles underlying it and have used it to run experiments. This article is part of a theme issue on DevOps."8851 },8852 "DOI:10.1126/science.1213847": {8853 "paperId": "e4625641d1072542315c14083befb43cd7096aba",8854 "externalIds": {8855 "PubMedCentral": "3383002",8856 "MAG": "2008985174",8857 "DOI": "10.1126/science.1213847",8858 "CorpusId": 19755904,8859 "PubMed": "22144613"8860 },8861 "publicationVenue": {8862 "id": "f59506a8-d8bb-4101-b3d4-c4ac3ed03dad",8863 "name": "Science",8864 "type": "journal",8865 "issn": "0193-4511",8866 "alternate_issns": [8867 "0036-8075"8868 ],8869 "url": "https://www.jstor.org/journal/science",8870 "alternate_urls": [8871 "https://www.sciencemag.org/",8872 "http://www.sciencemag.org/",8873 "http://www.jstor.org/journals/00368075.html",8874 "http://www.sciencemag.org/archive/"8875 ]8876 },8877 "title": "Reproducible Research in Computational Science",8878 "venue": "Science",8879 "year": 2011,8880 "citationCount": 1523,8881 "openAccessPdf": {8882 "url": "https://www.ncbi.nlm.nih.gov/pmc/articles/3383002",8883 "status": "GREEN",8884 "license": null,8885 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC3383002, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."8886 },8887 "publicationTypes": [8888 "JournalArticle"8889 ],8890 "journal": {8891 "name": "Science (New York, N.y.)",8892 "pages": "1226 - 1227",8893 "volume": "334"8894 },8895 "authors": [8896 {8897 "authorId": "34814856",8898 "name": "Roger D. Peng"8899 }8900 ],8901 "abstract": "Computational science has led to exciting new developments, but the nature of the work has exposed limitations in our ability to evaluate published findings. Reproducibility has the potential to serve as a minimum standard for judging scientific claims when full independent replication of a study is not possible."8902 },8903 "DOI:10.1126/science.aab2374": {8904 "paperId": "927a32a1b1c69a6041a586b71f52be1b957e5030",8905 "externalIds": {8906 "PubMedCentral": "4550299",8907 "MAG": "2403646253",8908 "DOI": "10.1126/science.aab2374",8909 "CorpusId": 206637955,8910 "PubMed": "26113702"8911 },8912 "publicationVenue": {8913 "id": "f59506a8-d8bb-4101-b3d4-c4ac3ed03dad",8914 "name": "Science",8915 "type": "journal",8916 "issn": "0193-4511",8917 "alternate_issns": [8918 "0036-8075"8919 ],8920 "url": "https://www.jstor.org/journal/science",8921 "alternate_urls": [8922 "https://www.sciencemag.org/",8923 "http://www.sciencemag.org/",8924 "http://www.jstor.org/journals/00368075.html",8925 "http://www.sciencemag.org/archive/"8926 ]8927 },8928 "title": "Promoting an open research culture",8929 "venue": "Science",8930 "year": 2015,8931 "citationCount": 2275,8932 "openAccessPdf": {8933 "url": "https://www.ncbi.nlm.nih.gov/pmc/articles/4550299",8934 "status": "GREEN",8935 "license": null,8936 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC4550299, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."8937 },8938 "publicationTypes": [8939 "JournalArticle"8940 ],8941 "journal": {8942 "name": "Science (New York, N.Y.)",8943 "pages": "1422 - 1425",8944 "volume": "348"8945 },8946 "authors": [8947 {8948 "authorId": "2862527",8949 "name": "Brian A. Nosek"8950 },8951 {8952 "authorId": "2139865103",8953 "name": "G. Alter"8954 },8955 {8956 "authorId": "8611013",8957 "name": "G. Banks"8958 },8959 {8960 "authorId": "3237553",8961 "name": "D. Borsboom"8962 },8963 {8964 "authorId": "72356049",8965 "name": "S. Bowman"8966 },8967 {8968 "authorId": "6240718",8969 "name": "S. J. Breckler"8970 },8971 {8972 "authorId": "2073872541",8973 "name": "S. Buck"8974 },8975 {8976 "authorId": "46975669",8977 "name": "C. Chambers"8978 },8979 {8980 "authorId": "2058346280",8981 "name": "G. Chin"8982 },8983 {8984 "authorId": "2054652938",8985 "name": "G. Christensen"8986 },8987 {8988 "authorId": "50230722",8989 "name": "M. Contestabile"8990 },8991 {8992 "authorId": "2412226244",8993 "name": "A. Dafoe"8994 },8995 {8996 "authorId": "34840029",8997 "name": "E. Eich"8998 },8999 {9000 "authorId": "2364509",9001 "name": "J. Freese"9002 },9003 {9004 "authorId": "4249203",9005 "name": "R. Glennerster"9006 },9007 {9008 "authorId": "13065618",9009 "name": "D. Goroff"9010 },9011 {9012 "authorId": "48997800",9013 "name": "D. P. Green"9014 },9015 {9016 "authorId": "119365490",9017 "name": "B. Hesse"9018 },9019 {9020 "authorId": "32272151",9021 "name": "M. Humphreys"9022 },9023 {9024 "authorId": "46636130",9025 "name": "J. Ishiyama"9026 },9027 {9028 "authorId": "2124080032",9029 "name": "D. Karlan"9030 },9031 {9032 "authorId": "47733218",9033 "name": "A. Kraut"9034 },9035 {9036 "authorId": "47975256",9037 "name": "A. Lupia"9038 },9039 {9040 "authorId": "3154113",9041 "name": "P. Mabry"9042 },9043 {9044 "authorId": "4133783",9045 "name": "T. Madon"9046 },9047 {9048 "authorId": "118685494",9049 "name": "N. Malhotra"9050 },9051 {9052 "authorId": "1390039913",9053 "name": "E. Mayo-Wilson"9054 },9055 {9056 "authorId": "114856072",9057 "name": "M. McNutt"9058 },9059 {9060 "authorId": "144624624",9061 "name": "E. Miguel"9062 },9063 {9064 "authorId": "4659343",9065 "name": "E. Paluck"9066 },9067 {9068 "authorId": "2130902",9069 "name": "U. Simonsohn"9070 },9071 {9072 "authorId": "4110370",9073 "name": "C. Soderberg"9074 },9075 {9076 "authorId": "101916661",9077 "name": "B. Spellman"9078 },9079 {9080 "authorId": "1398590331",9081 "name": "J. Turitto"9082 },9083 {9084 "authorId": "80222152",9085 "name": "G. VandenBos"9086 },9087 {9088 "authorId": "2809726",9089 "name": "S. Vazire"9090 },9091 {9092 "authorId": "2505272",9093 "name": "E. Wagenmakers"9094 },9095 {9096 "authorId": "153160363",9097 "name": "R. Wilson"9098 },9099 {9100 "authorId": "2075675",9101 "name": "T. Yarkoni"9102 }9103 ],9104 "abstract": "Author guidelines for journals could help to promote transparency, openness, and reproducibility Transparency, openness, and reproducibility are readily recognized as vital features of science (1, 2). When asked, most scientists embrace these features as disciplinary norms and values (3). Therefore, one might expect that these valued features would be routine in daily practice. Yet, a growing body of evidence suggests that this is not the case (4\u20136)."9105 },9106 "DOI:10.1126/science.aac4716": {9107 "paperId": "155391192091a99f44b377db1e0e7819f2317498",9108 "externalIds": {9109 "DOI": "10.1126/science.aac4716",9110 "CorpusId": 218065162,9111 "PubMed": "26315443"9112 },9113 "publicationVenue": {9114 "id": "f59506a8-d8bb-4101-b3d4-c4ac3ed03dad",9115 "name": "Science",9116 "type": "journal",9117 "issn": "0193-4511",9118 "alternate_issns": [9119 "0036-8075"9120 ],9121 "url": "https://www.jstor.org/journal/science",9122 "alternate_urls": [9123 "https://www.sciencemag.org/",9124 "http://www.sciencemag.org/",9125 "http://www.jstor.org/journals/00368075.html",9126 "http://www.sciencemag.org/archive/"9127 ]9128 },9129 "title": "Estimating the reproducibility of psychological science",9130 "venue": "Science",9131 "year": 2015,9132 "citationCount": 7539,9133 "openAccessPdf": {9134 "url": "https://research.tilburguniversity.edu/en/publications/4b1e6c37-3ee2-414b-b790-66024d631d4a",9135 "status": "GREEN",9136 "license": "other-oa",9137 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1126/science.aac4716?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1126/science.aac4716, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."9138 },9139 "publicationTypes": [9140 "JournalArticle",9141 "Review"9142 ],9143 "journal": {9144 "name": "Science",9145 "volume": "349"9146 },9147 "authors": [9148 {9149 "authorId": "2231439",9150 "name": "Alexander A. Aarts"9151 },9152 {9153 "authorId": "1400206933",9154 "name": "Joanna E. Anderson"9155 },9156 {9157 "authorId": "96413924",9158 "name": "Christopher J. Anderson"9159 },9160 {9161 "authorId": "113650771",9162 "name": "Peter Raymond Attridge"9163 },9164 {9165 "authorId": "1788796",9166 "name": "A. Attwood"9167 },9168 {9169 "authorId": "152159926",9170 "name": "J. Axt"9171 },9172 {9173 "authorId": "36417745",9174 "name": "Molly Babel"9175 },9176 {9177 "authorId": "5252377",9178 "name": "\u0160. Bahn\u00edk"9179 },9180 {9181 "authorId": "6174975",9182 "name": "E. Baranski"9183 },9184 {9185 "authorId": "1387458734",9186 "name": "M. Barnett-Cowan"9187 },9188 {9189 "authorId": "4867423",9190 "name": "Elizabeth Bartmess"9191 },9192 {9193 "authorId": "3039399",9194 "name": "Jennifer S. Beer"9195 },9196 {9197 "authorId": "47431109",9198 "name": "Raoul Bell"9199 },9200 {9201 "authorId": "2074992746",9202 "name": "Heather Bentley"9203 },9204 {9205 "authorId": "114670541",9206 "name": "Leah Beyan"9207 },9208 {9209 "authorId": "30661345",9210 "name": "Grace Binion"9211 },9212 {9213 "authorId": "3237553",9214 "name": "D. Borsboom"9215 },9216 {9217 "authorId": "116191440",9218 "name": "Annick Bosch"9219 },9220 {9221 "authorId": "8597004",9222 "name": "Frank Bosco"9223 },9224 {9225 "authorId": "72356050",9226 "name": "Sara D. Bowman"9227 },9228 {9229 "authorId": "145470466",9230 "name": "M. Brandt"9231 },9232 {9233 "authorId": "66335862",9234 "name": "E. L. Braswell"9235 },9236 {9237 "authorId": "51904698",9238 "name": "Hilmar Brohmer"9239 },9240 {9241 "authorId": "48120787",9242 "name": "B. T. Brown"9243 },9244 {9245 "authorId": "4197188",9246 "name": "K. Brown"9247 },9248 {9249 "authorId": "31412979",9250 "name": "Jovita Br\u00fcning"9251 },9252 {9253 "authorId": "1400158480",9254 "name": "Ann Calhoun-Sauls"9255 },9256 {9257 "authorId": "36027478",9258 "name": "S. Callahan"9259 },9260 {9261 "authorId": "95354096",9262 "name": "E. Chagnon"9263 },9264 {9265 "authorId": "48719411",9266 "name": "Jesse J. Chandler"9267 },9268 {9269 "authorId": "38049428",9270 "name": "Christopher R. Chartier"9271 },9272 {9273 "authorId": "145787104",9274 "name": "F. Cheung"9275 },9276 {9277 "authorId": "51055585",9278 "name": "Cody Daniel Christopherson"9279 },9280 {9281 "authorId": "4704887",9282 "name": "Linda Cillessen"9283 },9284 {9285 "authorId": "48987469",9286 "name": "Russ Clay"9287 },9288 {9289 "authorId": "46550612",9290 "name": "Hayley M. D. Cleary"9291 },9292 {9293 "authorId": "117283612",9294 "name": "Mark D. Cloud"9295 },9296 {9297 "authorId": "143947719",9298 "name": "M. Conn"9299 },9300 {9301 "authorId": "1795487",9302 "name": "J. Cohoon"9303 },9304 {9305 "authorId": "7293849",9306 "name": "Simon Columbus"9307 },9308 {9309 "authorId": "2052289610",9310 "name": "A. Cordes"9311 },9312 {9313 "authorId": "49251371",9314 "name": "G. Costantini"9315 },9316 {9317 "authorId": "15675192",9318 "name": "Leslie D Cramblet Alvarez"9319 },9320 {9321 "authorId": "51034966",9322 "name": "Edward J. Cremata"9323 },9324 {9325 "authorId": "104792844",9326 "name": "J. Crusius"9327 },9328 {9329 "authorId": "2480852",9330 "name": "J. DeCoster"9331 },9332 {9333 "authorId": "122378013",9334 "name": "M. DeGaetano"9335 },9336 {9337 "authorId": "1665535543",9338 "name": "Nicol\u00e1s Delia Penna"9339 },9340 {9341 "authorId": "1665493156",9342 "name": "Bobby Den Bezemer"9343 },9344 {9345 "authorId": "32368039",9346 "name": "M. Deserno"9347 },9348 {9349 "authorId": "113076847",9350 "name": "Olivia Devitt"9351 },9352 {9353 "authorId": "6629103",9354 "name": "L. Dewitte"9355 },9356 {9357 "authorId": "6958962",9358 "name": "David G. Dobolyi"9359 },9360 {9361 "authorId": "16313368",9362 "name": "Geneva T. Dodson"9363 },9364 {9365 "authorId": "2467192",9366 "name": "M. Donnellan"9367 },9368 {9369 "authorId": "2094220350",9370 "name": "R. Donohue"9371 },9372 {9373 "authorId": "5663433",9374 "name": "Rebecca A. Dore"9375 },9376 {9377 "authorId": "5021006",9378 "name": "A. Dorrough"9379 },9380 {9381 "authorId": "3488772",9382 "name": "A. Dreber"9383 },9384 {9385 "authorId": "5818783",9386 "name": "M. Dugas"9387 },9388 {9389 "authorId": "3100527",9390 "name": "Elizabeth W. Dunn"9391 },9392 {9393 "authorId": "14421983",9394 "name": "K. Easey"9395 },9396 {9397 "authorId": "117807982",9398 "name": "Sylvia Eboigbe"9399 },9400 {9401 "authorId": "34959578",9402 "name": "C. Eggleston"9403 },9404 {9405 "authorId": "123302634",9406 "name": "Jo Embley"9407 },9408 {9409 "authorId": "3018477",9410 "name": "S. Epskamp"9411 },9412 {9413 "authorId": "52004348",9414 "name": "Timothy M. Errington"9415 },9416 {9417 "authorId": "117698833",9418 "name": "Vivien Estel"9419 },9420 {9421 "authorId": "3392676",9422 "name": "Frank J. Farach"9423 },9424 {9425 "authorId": "25452274",9426 "name": "J. Feather"9427 },9428 {9429 "authorId": "2322314",9430 "name": "A. Fedor"9431 },9432 {9433 "authorId": "1400158404",9434 "name": "B. Fern\u00e1ndez-Castilla"9435 },9436 {9437 "authorId": "38548340",9438 "name": "Susann Fiedler"9439 },9440 {9441 "authorId": "32576544",9442 "name": "James G. Field"9443 },9444 {9445 "authorId": "2237526",9446 "name": "Stanka A. Fitneva"9447 },9448 {9449 "authorId": "2387162",9450 "name": "T. Flagan"9451 },9452 {9453 "authorId": "6543598",9454 "name": "A. Forest"9455 },9456 {9457 "authorId": "4193992",9458 "name": "Eskil Forsell"9459 },9460 {9461 "authorId": "153664010",9462 "name": "Joshua D. Foster"9463 },9464 {9465 "authorId": "2536223",9466 "name": "Michael C. Frank"9467 },9468 {9469 "authorId": "144106517",9470 "name": "Rebecca S. Frazier"9471 },9472 {9473 "authorId": "2057921617",9474 "name": "Heather M. Fuchs"9475 },9476 {9477 "authorId": "5807132",9478 "name": "P. Gable"9479 },9480 {9481 "authorId": "4908329",9482 "name": "Jeff Galak"9483 },9484 {9485 "authorId": "52169965",9486 "name": "E. Galliani"9487 },9488 {9489 "authorId": "39007608",9490 "name": "Anup Gampa"9491 },9492 {9493 "authorId": "2072601770",9494 "name": "Sara Garcia"9495 },9496 {9497 "authorId": "4034033",9498 "name": "Douglas Gazarian"9499 },9500 {9501 "authorId": "152595201",9502 "name": "Elizabeth Gilbert"9503 },9504 {9505 "authorId": "1400843054",9506 "name": "Roger Giner-Sorolla"9507 },9508 {9509 "authorId": "2479991061",9510 "name": "A. Gl\u00f6ckner"9511 },9512 {9513 "authorId": "35467332",9514 "name": "Lars Goellner"9515 },9516 {9517 "authorId": "6760964",9518 "name": "Jin X. Goh"9519 },9520 {9521 "authorId": "1402974706",9522 "name": "R. Goldberg"9523 },9524 {9525 "authorId": "6439793",9526 "name": "P. T. Goodbourn"9527 },9528 {9529 "authorId": "1400158332",9530 "name": "S. Gordon-McKeon"9531 },9532 {9533 "authorId": "103288020",9534 "name": "Bryan H. Gorges"9535 },9536 {9537 "authorId": "1677019678",9538 "name": "Jessie Gorges"9539 },9540 {9541 "authorId": "113021959",9542 "name": "J. Goss"9543 },9544 {9545 "authorId": "2250081930",9546 "name": "J. Graham"9547 },9548 {9549 "authorId": "50590791",9550 "name": "J. Grange"9551 },9552 {9553 "authorId": "2556954",9554 "name": "J. Gray"9555 },9556 {9557 "authorId": "2130548269",9558 "name": "C. Hartgerink"9559 },9560 {9561 "authorId": "27816717",9562 "name": "Joshua K. Hartshorne"9563 },9564 {9565 "authorId": "3390722",9566 "name": "F. Hasselman"9567 },9568 {9569 "authorId": "144616859",9570 "name": "Timothy B. Hayes"9571 },9572 {9573 "authorId": "3650446",9574 "name": "Emma Heikensten"9575 },9576 {9577 "authorId": "2196843",9578 "name": "Felix Henninger"9579 },9580 {9581 "authorId": "6941720",9582 "name": "John Hodsoll"9583 },9584 {9585 "authorId": "3388157",9586 "name": "Taylor Holubar"9587 },9588 {9589 "authorId": "92341957",9590 "name": "G. Hoogendoorn"9591 },9592 {9593 "authorId": "1665486287",9594 "name": "D. Humphries"9595 },9596 {9597 "authorId": "70247182",9598 "name": "C. Hung"9599 },9600 {9601 "authorId": "87889800",9602 "name": "N. Immelman"9603 },9604 {9605 "authorId": "6169028",9606 "name": "Vanessa C. Irsik"9607 },9608 {9609 "authorId": "36524750",9610 "name": "G. Jahn"9611 },9612 {9613 "authorId": "80488489",9614 "name": "F. J\u00e4kel"9615 },9616 {9617 "authorId": "2485703",9618 "name": "Marc Jekel"9619 },9620 {9621 "authorId": "145281248",9622 "name": "M. Johannesson"9623 },9624 {9625 "authorId": "117496518",9626 "name": "L. Johnson"9627 },9628 {9629 "authorId": "2150444569",9630 "name": "David J. Johnson"9631 },9632 {9633 "authorId": "10776195",9634 "name": "Kate M. Johnson"9635 },9636 {9637 "authorId": "2071756595",9638 "name": "William Johnston"9639 },9640 {9641 "authorId": "2640241",9642 "name": "K. Jonas"9643 },9644 {9645 "authorId": "1400158286",9646 "name": "Jennifer A. Joy-Gaba"9647 },9648 {9649 "authorId": "4049176",9650 "name": "H. Kappes"9651 },9652 {9653 "authorId": "120498542",9654 "name": "Kim Kelso"9655 },9656 {9657 "authorId": "8633058",9658 "name": "Mallory C. Kidwell"9659 },9660 {9661 "authorId": "2109708345",9662 "name": "Seung K. Kim"9663 },9664 {9665 "authorId": "6395769",9666 "name": "M. Kirkhart"9667 },9668 {9669 "authorId": "6032930",9670 "name": "B. Kleinberg"9671 },9672 {9673 "authorId": "50012037",9674 "name": "G. Kne\u017eevi\u0107"9675 },9676 {9677 "authorId": "38197330",9678 "name": "F. Kolorz"9679 },9680 {9681 "authorId": "39916640",9682 "name": "J. Kossakowski"9683 },9684 {9685 "authorId": "153875794",9686 "name": "Robert W. Krause"9687 },9688 {9689 "authorId": "123143083",9690 "name": "J. Krijnen"9691 },9692 {9693 "authorId": "5199064",9694 "name": "T. Kuhlmann"9695 },9696 {9697 "authorId": "90301659",9698 "name": "Y. Kunkels"9699 },9700 {9701 "authorId": "123804984",9702 "name": "Megan M. Kyc"9703 },9704 {9705 "authorId": "2148764895",9706 "name": "C. Lai"9707 },9708 {9709 "authorId": "1665495033",9710 "name": "A. Laique"9711 },9712 {9713 "authorId": "87792747",9714 "name": "D. Lakens"9715 },9716 {9717 "authorId": "47951763",9718 "name": "Kristin A. Lane"9719 },9720 {9721 "authorId": "6028786",9722 "name": "Bethany Lassetter"9723 },9724 {9725 "authorId": "40526243",9726 "name": "L. Lazarevi\u0107"9727 },9728 {9729 "authorId": "1665487375",9730 "name": "E. Bel"9731 },9732 {9733 "authorId": "2110236533",9734 "name": "Key Jung Lee"9735 },9736 {9737 "authorId": "46293822",9738 "name": "Minha Lee"9739 },9740 {9741 "authorId": "6560078",9742 "name": "K. Lemm"9743 },9744 {9745 "authorId": "2248921",9746 "name": "Carmel A. Levitan"9747 },9748 {9749 "authorId": "48575845",9750 "name": "Melissa Lewis"9751 },9752 {9753 "authorId": "2117106185",9754 "name": "Lin Lin"9755 },9756 {9757 "authorId": "2145198693",9758 "name": "Stephanie C. Lin"9759 },9760 {9761 "authorId": "35780819",9762 "name": "M. Lippold"9763 },9764 {9765 "authorId": "6767826",9766 "name": "Darren Loureiro"9767 },9768 {9769 "authorId": "116957928",9770 "name": "Ilse Luteijn"9771 },9772 {9773 "authorId": "35093146",9774 "name": "S. Mackinnon"9775 },9776 {9777 "authorId": "120566087",9778 "name": "Heather N. Mainard"9779 },9780 {9781 "authorId": "4526963",9782 "name": "Denise C. Marigold"9783 },9784 {9785 "authorId": "150078826",9786 "name": "D. P. Martin"9787 },9788 {9789 "authorId": "123396296",9790 "name": "Tylar Martinez"9791 },9792 {9793 "authorId": "4362855",9794 "name": "E. Masicampo"9795 },9796 {9797 "authorId": "84358116",9798 "name": "Joshua Matacotta"9799 },9800 {9801 "authorId": "32716879",9802 "name": "Maya B. Mathur"9803 },9804 {9805 "authorId": "2055522260",9806 "name": "Michael May"9807 },9808 {9809 "authorId": "5739306",9810 "name": "Nicole C Mechin"9811 },9812 {9813 "authorId": "2796734",9814 "name": "Pranjal H. Mehta"9815 },9816 {9817 "authorId": "3505739",9818 "name": "Johannes M. Meixner"9819 },9820 {9821 "authorId": "2292034",9822 "name": "Alissa Melinger"9823 },9824 {9825 "authorId": "2109507581",9826 "name": "Jeremy K. Miller"9827 },9828 {9829 "authorId": "1664699522",9830 "name": "Mallorie Miller"9831 },9832 {9833 "authorId": "3260369",9834 "name": "K. Moore"9835 },9836 {9837 "authorId": "8173711",9838 "name": "Marcus M\u00f6schl"9839 },9840 {9841 "authorId": "40632770",9842 "name": "M. Motyl"9843 },9844 {9845 "authorId": "49338794",9846 "name": "S. Muller"9847 },9848 {9849 "authorId": "3057626",9850 "name": "M. Munafo"9851 },9852 {9853 "authorId": "6699090",9854 "name": "K. Neijenhuijs"9855 },9856 {9857 "authorId": "122669704",9858 "name": "Taylor Nervi"9859 },9860 {9861 "authorId": "28866192",9862 "name": "Gandalf Nicolas"9863 },9864 {9865 "authorId": "5120903",9866 "name": "G. Nilsonne"9867 },9868 {9869 "authorId": "2862527",9870 "name": "Brian A. Nosek"9871 },9872 {9873 "authorId": "4222695",9874 "name": "M. Nuijten"9875 },9876 {9877 "authorId": "2061321863",9878 "name": "Catherine Olsson"9879 },9880 {9881 "authorId": "21707288",9882 "name": "Colleen M. Osborne"9883 },9884 {9885 "authorId": "15808420",9886 "name": "Lutz Ostkamp"9887 },9888 {9889 "authorId": "1800603",9890 "name": "M. Pavel"9891 },9892 {9893 "authorId": "1398036394",9894 "name": "I. Penton-Voak"9895 },9896 {9897 "authorId": "123910286",9898 "name": "O. Perna"9899 },9900 {9901 "authorId": "1733092",9902 "name": "Pernet Cyril"9903 },9904 {9905 "authorId": "5070565",9906 "name": "M. Perugini"9907 },9908 {9909 "authorId": "145798446",9910 "name": "R. Pipitone"9911 },9912 {9913 "authorId": "7731005",9914 "name": "M. Pitts"9915 },9916 {9917 "authorId": "3086561",9918 "name": "F. Plessow"9919 },9920 {9921 "authorId": "5666302",9922 "name": "J. Prenoveau"9923 },9924 {9925 "authorId": "114309502",9926 "name": "R. Rahal"9927 },9928 {9929 "authorId": "4412299",9930 "name": "Kate A. Ratliff"9931 },9932 {9933 "authorId": "46557202",9934 "name": "David A. Reinhard"9935 },9936 {9937 "authorId": "4137233",9938 "name": "Frank Renkewitz"9939 },9940 {9941 "authorId": "4977822",9942 "name": "Ashley A. Ricker"9943 },9944 {9945 "authorId": "3911950",9946 "name": "A. E. Rigney"9947 },9948 {9949 "authorId": "145836378",9950 "name": "Andrew M Rivers"9951 },9952 {9953 "authorId": "51055550",9954 "name": "Mark A. Roebke"9955 },9956 {9957 "authorId": "6431015",9958 "name": "Abraham M. Rutchick"9959 },9960 {9961 "authorId": "2055552083",9962 "name": "Robert S. Ryan"9963 },9964 {9965 "authorId": "48176355",9966 "name": "O. \u015eahin"9967 },9968 {9969 "authorId": "7144751",9970 "name": "Anondah Saide"9971 },9972 {9973 "authorId": "4225466",9974 "name": "Gillian M. Sandstrom"9975 },9976 {9977 "authorId": "2113520139",9978 "name": "David Santos"9979 },9980 {9981 "authorId": "2055715460",9982 "name": "R. Saxe"9983 },9984 {9985 "authorId": "40063306",9986 "name": "Ren\u00e9 Schlegelmilch"9987 },9988 {9989 "authorId": "49371070",9990 "name": "Kathleen Schmidt"9991 },9992 {9993 "authorId": "49715936",9994 "name": "Sabine Scholz"9995 },9996 {9997 "authorId": "123478162",9998 "name": "L. Seibel"9999 },10000 {10001 "authorId": "4067481",10002 "name": "D. Selterman"10003 },10004 {10005 "authorId": "3397505",10006 "name": "S. Shaki"10007 },10008 {10009 "authorId": "31665223",10010 "name": "W. B. Simpson"10011 },10012 {10013 "authorId": "144232281",10014 "name": "H. Sinclair"10015 },10016 {10017 "authorId": "143804040",10018 "name": "Jeanine L. M. Skorinko"10019 },10020 {10021 "authorId": "2111978802",10022 "name": "A. Slowik"10023 },10024 {10025 "authorId": "2500175",10026 "name": "J. Snyder"10027 },10028 {10029 "authorId": "4110370",10030 "name": "C. Soderberg"10031 },10032 {10033 "authorId": "32218314",10034 "name": "Carina M. Sonnleitner"10035 },10036 {10037 "authorId": "2056814613",10038 "name": "N. Spencer"10039 },10040 {10041 "authorId": "34782684",10042 "name": "Jeffrey R. Spies"10043 },10044 {10045 "authorId": "2889085",10046 "name": "S. Steegen"10047 },10048 {10049 "authorId": "3308834",10050 "name": "S. Stieger"10051 },10052 {10053 "authorId": "3918197",10054 "name": "Nina Strohminger"10055 },10056 {10057 "authorId": "3349937",10058 "name": "G. Sullivan"10059 },10060 {10061 "authorId": "6998374",10062 "name": "Thomas Talhelm"10063 },10064 {10065 "authorId": "123526524",10066 "name": "Megan Tapia"10067 },10068 {10069 "authorId": "120986210",10070 "name": "A. T. Dorsthorst"10071 },10072 {10073 "authorId": "17886332",10074 "name": "M. Thomae"10075 },10076 {10077 "authorId": "2108534776",10078 "name": "Sarah L. Thomas"10079 },10080 {10081 "authorId": "5407737",10082 "name": "Pia Tio"10083 },10084 {10085 "authorId": "26384240",10086 "name": "Frits Traets"10087 },10088 {10089 "authorId": "50343359",10090 "name": "Steve Tsang"10091 },10092 {10093 "authorId": "2109141",10094 "name": "F. Tuerlinckx"10095 },10096 {10097 "authorId": "118528873",10098 "name": "Paul Turchan"10099 },10100 {10101 "authorId": "3738838",10102 "name": "Milan Val\u00e1\u0161ek"10103 },10104 {10105 "authorId": "11366104",10106 "name": "A. Ve\u00e9r"10107 },10108 {10109 "authorId": "81322766",10110 "name": "R. V. Aert"10111 },10112 {10113 "authorId": "13333804",10114 "name": "M. V. Assen"10115 },10116 {10117 "authorId": "2062808617",10118 "name": "R. Bork"10119 },10120 {10121 "authorId": "122303122",10122 "name": "Mathijs van de Ven"10123 },10124 {10125 "authorId": "103097236",10126 "name": "Don van den Bergh"10127 },10128 {10129 "authorId": "151452444",10130 "name": "M. V. D. Hulst"10131 },10132 {10133 "authorId": "51240054",10134 "name": "R. V. Dooren"10135 },10136 {10137 "authorId": "50233092",10138 "name": "J. Doorn"10139 },10140 {10141 "authorId": "4155110",10142 "name": "D. V. Renswoude"10143 },10144 {10145 "authorId": "2392419",10146 "name": "H. van Rijn"10147 },10148 {10149 "authorId": "2409652",10150 "name": "Wolf Vanpaemel"10151 },10152 {10153 "authorId": "2078595605",10154 "name": "Alejandro Echeverr\u00eda"10155 },10156 {10157 "authorId": "2070303231",10158 "name": "Melissa Vazquez"10159 },10160 {10161 "authorId": "6122250",10162 "name": "Natalia V\u00e9lez"10163 },10164 {10165 "authorId": "35039893",10166 "name": "Marieke Vermue"10167 },10168 {10169 "authorId": "88801524",10170 "name": "M. Verschoor"10171 },10172 {10173 "authorId": "46989769",10174 "name": "M. Vianello"10175 },10176 {10177 "authorId": "2176390",10178 "name": "M. Voracek"10179 },10180 {10181 "authorId": "113898223",10182 "name": "Gina Vuu"10183 },10184 {10185 "authorId": "2505272",10186 "name": "E. Wagenmakers"10187 },10188 {10189 "authorId": "6348079",10190 "name": "Joanneke Weerdmeester"10191 },10192 {10193 "authorId": "2058927069",10194 "name": "A. Welsh"10195 },10196 {10197 "authorId": "5900056",10198 "name": "E. Westgate"10199 },10200 {10201 "authorId": "67239528",10202 "name": "Joeri Wissink"10203 },10204 {10205 "authorId": "145322160",10206 "name": "M. Wood"10207 },10208 {10209 "authorId": "48197426",10210 "name": "Andy T. Woods"10211 },10212 {10213 "authorId": "2058458367",10214 "name": "Emily M. Wright"10215 },10216 {10217 "authorId": "151482749",10218 "name": "Sining Wu"10219 },10220 {10221 "authorId": "145525838",10222 "name": "M. Zeelenberg"10223 },10224 {10225 "authorId": "117597026",10226 "name": "Kellylynn Zuni"10227 }10228 ],10229 "abstract": null10230 },10231 "DOI:10.1126/science.aah6168": {10232 "paperId": "2dce110085044c913d6c3bf70aaf27345ffb23d1",10233 "externalIds": {10234 "MAG": "2560471521",10235 "DOI": "10.1126/science.aah6168",10236 "CorpusId": 32551384,10237 "PubMed": "27940837"10238 },10239 "publicationVenue": {10240 "id": "f59506a8-d8bb-4101-b3d4-c4ac3ed03dad",10241 "name": "Science",10242 "type": "journal",10243 "issn": "0193-4511",10244 "alternate_issns": [10245 "0036-8075"10246 ],10247 "url": "https://www.jstor.org/journal/science",10248 "alternate_urls": [10249 "https://www.sciencemag.org/",10250 "http://www.sciencemag.org/",10251 "http://www.jstor.org/journals/00368075.html",10252 "http://www.sciencemag.org/archive/"10253 ]10254 },10255 "title": "Enhancing reproducibility for computational methods",10256 "venue": "Science",10257 "year": 2016,10258 "citationCount": 424,10259 "openAccessPdf": {10260 "url": "https://escholarship.org/uc/item/9wh3k06p",10261 "status": "GREEN",10262 "license": "CCBYNC",10263 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1126/science.aah6168?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1126/science.aah6168, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."10264 },10265 "publicationTypes": [10266 "JournalArticle"10267 ],10268 "journal": {10269 "name": "Science",10270 "pages": "1240 - 1241",10271 "volume": "354"10272 },10273 "authors": [10274 {10275 "authorId": "2598812",10276 "name": "V. Stodden"10277 },10278 {10279 "authorId": "144643621",10280 "name": "M. McNutt"10281 },10282 {10283 "authorId": "1685317",10284 "name": "D. Bailey"10285 },10286 {10287 "authorId": "1687896",10288 "name": "E. Deelman"10289 },10290 {10291 "authorId": "145526918",10292 "name": "Y. Gil"10293 },10294 {10295 "authorId": "47283959",10296 "name": "B. Hanson"10297 },10298 {10299 "authorId": "2135545",10300 "name": "M. Heroux"10301 },10302 {10303 "authorId": "145441750",10304 "name": "J. Ioannidis"10305 },10306 {10307 "authorId": "48478161",10308 "name": "M. Taufer"10309 }10310 ],10311 "abstract": "Data, code, and workflows should be available and cited Over the past two decades, computational methods have radically changed the ability of researchers from all areas of scholarship to process and analyze data and to simulate complex systems. But with these advances come challenges that are contributing to broader concerns over irreproducibility in the scholarly literature, among them the lack of transparency in disclosure of computational methods. Current reporting methods are often uneven, incomplete, and still evolving. We present a novel set of Reproducibility Enhancement Principles (REP) targeting disclosure challenges involving computation. These recommendations, which build upon more general proposals from the Transparency and Openness Promotion (TOP) guidelines (1) and recommendations for field data (2), emerged from workshop discussions among funding agencies, publishers and journal editors, industry participants, and researchers representing a broad range of domains. Although some of these actions may be aspirational, we believe it is important to recognize and move toward ameliorating irreproducibility in computational research."10312 },10313 "DOI:10.1126/scitranslmed.aaf5027": {10314 "paperId": "3728c289e3582f6329a735b3e9882b2a0cabad83",10315 "externalIds": {10316 "MAG": "2414283705",10317 "DOI": "10.1126/scitranslmed.aaf5027",10318 "CorpusId": 848096,10319 "PubMed": "27252173"10320 },10321 "publicationVenue": {10322 "id": "1cd4baea-5ccc-4522-9dbf-0566f39d3b37",10323 "name": "Science Translational Medicine",10324 "type": "journal",10325 "alternate_names": [10326 "Sci Transl Med"10327 ],10328 "issn": "1946-6234",10329 "url": "https://stm.sciencemag.org/",10330 "alternate_urls": [10331 "https://stm.sciencemag.org/site/misc/about.xhtml"10332 ]10333 },10334 "title": "What does research reproducibility mean?",10335 "venue": "Science Translational Medicine",10336 "year": 2016,10337 "citationCount": 1164,10338 "openAccessPdf": {10339 "url": "https://stm.sciencemag.org/content/scitransmed/8/341/341ps12.full.pdf",10340 "status": "BRONZE",10341 "license": null,10342 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1126/scitranslmed.aaf5027?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1126/scitranslmed.aaf5027, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."10343 },10344 "publicationTypes": [10345 "Review",10346 "JournalArticle"10347 ],10348 "journal": {10349 "name": "Science Translational Medicine",10350 "pages": "341ps12 - 341ps12",10351 "volume": "8"10352 },10353 "authors": [10354 {10355 "authorId": "1815823",10356 "name": "S. Goodman"10357 },10358 {10359 "authorId": "30080277",10360 "name": "D. Fanelli"10361 },10362 {10363 "authorId": "145441750",10364 "name": "J. Ioannidis"10365 }10366 ],10367 "abstract": null10368 },10369 "DOI:10.1136/amiajnl-2011-000089": {10370 "paperId": "bbc468e704ff4af72ca9de9ceb8203f7bf930d49",10371 "externalIds": {10372 "MAG": "2281090488",10373 "DBLP": "journals/jamia/GoddardRW12",10374 "DOI": "10.1136/amiajnl-2011-000089",10375 "CorpusId": 2050860,10376 "PubMed": "21685142"10377 },10378 "publicationVenue": null,10379 "title": "Automation bias: a systematic review of frequency, effect mediators, and mitigators",10380 "venue": "J. Am. Medical Informatics Assoc.",10381 "year": 2012,10382 "citationCount": 1130,10383 "openAccessPdf": {10384 "url": "https://academic.oup.com/jamia/article-pdf/19/1/121/5911703/19-1-121.pdf",10385 "status": "BRONZE",10386 "license": null,10387 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1136/amiajnl-2011-000089?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1136/amiajnl-2011-000089, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."10388 },10389 "publicationTypes": [10390 "Review",10391 "JournalArticle"10392 ],10393 "journal": {10394 "name": "Journal of the American Medical Informatics Association : JAMIA",10395 "pages": "\n 121-7\n ",10396 "volume": "19 1"10397 },10398 "authors": [10399 {10400 "authorId": "143911263",10401 "name": "Kate Goddard"10402 },10403 {10404 "authorId": "2839223",10405 "name": "A. Roudsari"10406 },10407 {10408 "authorId": "145984984",10409 "name": "J. Wyatt"10410 }10411 ],10412 "abstract": null10413 },10414 "DOI:10.1145/2160718.2160734": {10415 "paperId": "87fc322b992c1881854c1e2a0e36ef5b6f739071",10416 "externalIds": {10417 "DBLP": "journals/queue/Helland12",10418 "MAG": "2159123115",10419 "DOI": "10.1145/2160718.2160734",10420 "CorpusId": 2854592710421 },10422 "publicationVenue": {10423 "id": "4bd27ad1-f6fa-4340-9d9d-c62ac5be6fc0",10424 "name": "Queue",10425 "type": "journal",10426 "alternate_names": [10427 "ACM Queue"10428 ],10429 "issn": "1542-7730",10430 "url": "http://portal.acm.org/queue",10431 "alternate_urls": [10432 "http://www.acmqueue.org/"10433 ]10434 },10435 "title": "Idempotence is not a medical condition",10436 "venue": "Queue",10437 "year": 2012,10438 "citationCount": 76,10439 "openAccessPdf": {10440 "url": "https://dl.acm.org/doi/pdf/10.1145/2181796.2187821",10441 "status": "BRONZE",10442 "license": null,10443 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/2160718.2160734?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/2160718.2160734, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."10444 },10445 "publicationTypes": [10446 "JournalArticle"10447 ],10448 "journal": {10449 "name": "Communications of the ACM",10450 "pages": "56 - 65",10451 "volume": "55"10452 },10453 "authors": [10454 {10455 "authorId": "2549968",10456 "name": "Pat Helland"10457 }10458 ],10459 "abstract": null10460 },10461 "DOI:10.1145/2408776.2408794": {10462 "paperId": "0831a5baf38c9b3d43c755319a602b15fc01c52d",10463 "externalIds": {10464 "DBLP": "journals/cacm/DeanB13",10465 "MAG": "1982063824",10466 "DOI": "10.1145/2408776.2408794",10467 "CorpusId": 3490052010468 },10469 "publicationVenue": null,10470 "title": "The tail at scale",10471 "venue": "CACM",10472 "year": 2013,10473 "citationCount": 2180,10474 "openAccessPdf": {10475 "url": "",10476 "status": "CLOSED",10477 "license": null,10478 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/2408776.2408794?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/2408776.2408794, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."10479 },10480 "publicationTypes": [10481 "JournalArticle"10482 ],10483 "journal": {10484 "name": "Commun. ACM",10485 "pages": "74-80",10486 "volume": "56"10487 },10488 "authors": [10489 {10490 "authorId": "49959210",10491 "name": "Jeffrey Dean"10492 },10493 {10494 "authorId": "1746646",10495 "name": "L. Barroso"10496 }10497 ],10498 "abstract": null10499 },10500 "DOI:10.1145/2452376.2452478": {10501 "paperId": "e75134cde1477ac341c2197b1882191a972c2b60",10502 "externalIds": {10503 "MAG": "2069028640",10504 "DBLP": "conf/edbt/MissierBC13",10505 "DOI": "10.1145/2452376.2452478",10506 "CorpusId": 1690788110507 },10508 "publicationVenue": {10509 "id": "74f47cf8-2839-4a56-959a-524dd0ed9e3e",10510 "name": "International Conference on Extending Database Technology",10511 "type": "conference",10512 "alternate_names": [10513 "Int Conf Extending Database Technol",10514 "Extending Database Technology",10515 "Extending Database Technol",10516 "EDBT"10517 ],10518 "url": "http://www.edbt.org/"10519 },10520 "title": "The W3C PROV family of specifications for modelling provenance metadata",10521 "venue": "International Conference on Extending Database Technology",10522 "year": 2013,10523 "citationCount": 260,10524 "openAccessPdf": {10525 "url": "",10526 "status": "CLOSED",10527 "license": null,10528 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/2452376.2452478?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/2452376.2452478, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."10529 },10530 "publicationTypes": [10531 "JournalArticle",10532 "Review"10533 ],10534 "journal": {10535 "pages": "773-776"10536 },10537 "authors": [10538 {10539 "authorId": "1755645",10540 "name": "P. Missier"10541 },10542 {10543 "authorId": "3337287",10544 "name": "K. Belhajjame"10545 },10546 {10547 "authorId": "144320476",10548 "name": "James Cheney"10549 }10550 ],10551 "abstract": null10552 },10553 "DOI:10.1145/2670979.2670986": {10554 "paperId": "253818f5625fc74fd2f827dfc82f02ce8f22bef7",10555 "externalIds": {10556 "DBLP": "conf/cloud/GunawiHLPDAELLM14",10557 "MAG": "2006307164",10558 "DOI": "10.1145/2670979.2670986",10559 "CorpusId": 1603876710560 },10561 "publicationVenue": {10562 "id": "d13e941e-4cac-4f1d-bdca-77d927e31f1b",10563 "name": "ACM Symposium on Cloud Computing",10564 "type": "conference",10565 "alternate_names": [10566 "System-on-Chip Conference",10567 "ACM Symp Cloud Comput",10568 "Syst Conf",10569 "Symp Cloud Comput",10570 "Annual IEEE International System-on-Chip Conference",10571 "Symposium on Cloud Computing",10572 "Annu IEEE Int Syst Conf",10573 "SoCC"10574 ],10575 "url": "http://www.ieee-socc.org/"10576 },10577 "title": "What Bugs Live in the Cloud? A Study of 3000+ Issues in Cloud Systems",10578 "venue": "ACM Symposium on Cloud Computing",10579 "year": 2014,10580 "citationCount": 280,10581 "openAccessPdf": {10582 "url": "",10583 "status": "CLOSED",10584 "license": null,10585 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/2670979.2670986?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/2670979.2670986, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."10586 },10587 "publicationTypes": [10588 "Book",10589 "JournalArticle",10590 "Review"10591 ],10592 "journal": {10593 "name": "Proceedings of the ACM Symposium on Cloud Computing"10594 },10595 "authors": [10596 {10597 "authorId": "1738725",10598 "name": "Haryadi S. Gunawi"10599 },10600 {10601 "authorId": "2725752",10602 "name": "M. Hao"10603 },10604 {10605 "authorId": "1709102",10606 "name": "Tanakorn Leesatapornwongsa"10607 },10608 {10609 "authorId": "1405320577",10610 "name": "Tiratat Patana-anake"10611 },10612 {10613 "authorId": "145182862",10614 "name": "Thanh Do"10615 },10616 {10617 "authorId": "2286277",10618 "name": "J. Adityatama"10619 },10620 {10621 "authorId": "2590796",10622 "name": "Kurnia J. Eliazar"10623 },10624 {10625 "authorId": "113194122",10626 "name": "Agung Laksono"10627 },10628 {10629 "authorId": "2235631",10630 "name": "Jeffrey F. Lukman"10631 },10632 {10633 "authorId": "47787880",10634 "name": "V. Martin"10635 },10636 {10637 "authorId": "2382708",10638 "name": "A. Satria"10639 }10640 ],10641 "abstract": null10642 },10643 "DOI:10.1145/2723872.2723882": {10644 "paperId": "e0b34f80227e9c060bc9c37692278e7cefaa9540",10645 "externalIds": {10646 "MAG": "2952640757",10647 "ArXiv": "1410.0846",10648 "DBLP": "journals/sigops/Boettiger15",10649 "DOI": "10.1145/2723872.2723882",10650 "CorpusId": 25558310651 },10652 "publicationVenue": null,10653 "title": "An introduction to Docker for reproducible research",10654 "venue": "OPSR",10655 "year": 2014,10656 "citationCount": 1085,10657 "openAccessPdf": {10658 "url": "https://arxiv.org/pdf/1410.0846",10659 "status": "GREEN",10660 "license": null,10661 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/1410.0846, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."10662 },10663 "publicationTypes": [10664 "JournalArticle",10665 "Review"10666 ],10667 "journal": {10668 "name": "ACM SIGOPS Oper. Syst. Rev.",10669 "pages": "71-79",10670 "volume": "49"10671 },10672 "authors": [10673 {10674 "authorId": "72997413",10675 "name": "C. Boettiger"10676 }10677 ],10678 "abstract": "As computational work becomes more and more integral to many aspects of scientific research, computational reproducibility has become an issue of increasing importance to computer systems researchers and domain scientists alike. Though computational reproducibility seems more straight forward than replicating physical experiments, the complex and rapidly changing nature of computer environments makes being able to reproduce and extend such work a serious challenge. In this paper, I explore common reasons that code developed for one research project cannot be successfully executed or extended by subsequent researchers. I review current approaches to these issues, including virtual machines and workflow systems, and their limitations. I then examine how the popular emerging technology Docker combines several areas from systems research - such as operating system virtualization, cross-platform portability, modular re-usable elements, versioning, and a 'DevOps' philosophy, to address these challenges. I illustrate this with several examples of Docker use with a focus on the R statistical environment."10679 },10680 "DOI:10.1145/2882903.2899401": {10681 "paperId": "4a9811c6ba7c7faacbcbf6a73a4ff40c77885c40",10682 "externalIds": {10683 "MAG": "2431100065",10684 "DBLP": "conf/sigmod/ChirigatiRSF16",10685 "DOI": "10.1145/2882903.2899401",10686 "CorpusId": 1750007510687 },10688 "publicationVenue": null,10689 "title": "ReproZip: Computational Reproducibility With Ease",10690 "venue": "SIGMOD Conference",10691 "year": 2016,10692 "citationCount": 160,10693 "openAccessPdf": {10694 "url": "http://dl.acm.org/ft_gateway.cfm?id=2899401&type=pdf",10695 "status": "BRONZE",10696 "license": null,10697 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/2882903.2899401?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/2882903.2899401, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."10698 },10699 "publicationTypes": [10700 "Book",10701 "JournalArticle",10702 "Conference",10703 "Review"10704 ],10705 "journal": {10706 "name": "Proceedings of the 2016 International Conference on Management of Data"10707 },10708 "authors": [10709 {10710 "authorId": "2541202",10711 "name": "F. Chirigati"10712 },10713 {10714 "authorId": "3416217",10715 "name": "R\u00e9mi Rampin"10716 },10717 {10718 "authorId": "1695878",10719 "name": "D. Shasha"10720 },10721 {10722 "authorId": "144162611",10723 "name": "J. Freire"10724 }10725 ],10726 "abstract": null10727 },10728 "DOI:10.1145/2987550.2987568": {10729 "paperId": "5b60e599096044a5fed372950486ad8eb0ac94ae",10730 "externalIds": {10731 "DBLP": "conf/cloud/SambasivanSMSFG16",10732 "MAG": "2521815480",10733 "DOI": "10.1145/2987550.2987568",10734 "CorpusId": 883504810735 },10736 "publicationVenue": {10737 "id": "d13e941e-4cac-4f1d-bdca-77d927e31f1b",10738 "name": "ACM Symposium on Cloud Computing",10739 "type": "conference",10740 "alternate_names": [10741 "System-on-Chip Conference",10742 "ACM Symp Cloud Comput",10743 "Syst Conf",10744 "Symp Cloud Comput",10745 "Annual IEEE International System-on-Chip Conference",10746 "Symposium on Cloud Computing",10747 "Annu IEEE Int Syst Conf",10748 "SoCC"10749 ],10750 "url": "http://www.ieee-socc.org/"10751 },10752 "title": "Principled workflow-centric tracing of distributed systems",10753 "venue": "ACM Symposium on Cloud Computing",10754 "year": 2016,10755 "citationCount": 72,10756 "openAccessPdf": {10757 "url": "http://dl.acm.org/ft_gateway.cfm?id=2987568&type=pdf",10758 "status": "BRONZE",10759 "license": null,10760 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/2987550.2987568?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/2987550.2987568, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."10761 },10762 "publicationTypes": [10763 "Book",10764 "JournalArticle"10765 ],10766 "journal": {10767 "name": "Proceedings of the Seventh ACM Symposium on Cloud Computing"10768 },10769 "authors": [10770 {10771 "authorId": "2726787",10772 "name": "Raja R. Sambasivan"10773 },10774 {10775 "authorId": "17017774",10776 "name": "Ilari Shafer"10777 },10778 {10779 "authorId": "48532669",10780 "name": "Jonathan Mace"10781 },10782 {10783 "authorId": "2084178",10784 "name": "B. Sigelman"10785 },10786 {10787 "authorId": "145502789",10788 "name": "Rodrigo Fonseca"10789 },10790 {10791 "authorId": "1707164",10792 "name": "G. Ganger"10793 }10794 ],10795 "abstract": null10796 },10797 "DOI:10.1145/3290605.3300233": {10798 "paperId": "ad3cf68bae32d21f25ac142287d4a556155619d2",10799 "externalIds": {10800 "MAG": "2916904544",10801 "DBLP": "conf/chi/AmershiWVFNCSIB19",10802 "DOI": "10.1145/3290605.3300233",10803 "CorpusId": 8686694210804 },10805 "publicationVenue": {10806 "id": "b55b50b1-aae7-47a7-b042-8aecc930073d",10807 "name": "International Conference on Human Factors in Computing Systems",10808 "type": "conference",10809 "alternate_names": [10810 "CHI",10811 "Int Conf Hum Factor Comput Syst",10812 "Human Factors in Computing Systems",10813 "Conference on Human Interface",10814 "Conf Hum Interface",10815 "Hum Factor Comput Syst"10816 ],10817 "url": "http://www.acm.org/sigchi/"10818 },10819 "title": "Guidelines for Human-AI Interaction",10820 "venue": "International Conference on Human Factors in Computing Systems",10821 "year": 2019,10822 "citationCount": 2830,10823 "openAccessPdf": {10824 "url": "",10825 "status": "CLOSED",10826 "license": null,10827 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1145/3290605.3300233?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/3290605.3300233, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."10828 },10829 "publicationTypes": [10830 "Book",10831 "JournalArticle",10832 "Conference"10833 ],10834 "journal": {10835 "name": "Proceedings of the 2019 CHI Conference on Human Factors in Computing Systems"10836 },10837 "authors": [10838 {10839 "authorId": "1719124",10840 "name": "Saleema Amershi"10841 },10842 {10843 "authorId": "1780531",10844 "name": "Daniel S. Weld"10845 },10846 {10847 "authorId": "3109339",10848 "name": "Mihaela Vorvoreanu"10849 },10850 {10851 "authorId": "3318905",10852 "name": "Adam Fourney"10853 },10854 {10855 "authorId": "2571049",10856 "name": "Besmira Nushi"10857 },10858 {10859 "authorId": "9703838",10860 "name": "Penny Collisson"10861 },10862 {10863 "authorId": "38972741",10864 "name": "Ji-Na Suh"10865 },10866 {10867 "authorId": "1730570",10868 "name": "Shamsi T. Iqbal"10869 },10870 {10871 "authorId": "144609235",10872 "name": "Paul N. Bennett"10873 },10874 {10875 "authorId": "51306063",10876 "name": "K. Inkpen"10877 },10878 {10879 "authorId": "144113253",10880 "name": "J. Teevan"10881 },10882 {10883 "authorId": "1405707881",10884 "name": "Ruth Kikin-Gil"10885 },10886 {10887 "authorId": "145479841",10888 "name": "E. Horvitz"10889 }10890 ],10891 "abstract": "Advances in artificial intelligence (AI) frame opportunities and challenges for user interface design. Principles for human-AI interaction have been discussed in the human-computer interaction community for over two decades, but more study and innovation are needed in light of advances in AI and the growing uses of AI technologies in human-facing applications. We propose 18 generally applicable design guidelines for human-AI interaction. These guidelines are validated through multiple rounds of evaluation including a user study with 49 design practitioners who tested the guidelines against 20 popular AI-infused products. The results verify the relevance of the guidelines over a spectrum of interaction scenarios and reveal gaps in our knowledge, highlighting opportunities for further research. Based on the evaluations, we believe the set of design guidelines can serve as a resource to practitioners working on the design of applications and features that harness AI technologies, and to researchers interested in the further development of human-AI interaction design principles."10892 },10893 "DOI:10.1145/3411764.3445717": {10894 "paperId": "ebcbbb8fe297940d79b17aeb6d46bedff9db7fec",10895 "externalIds": {10896 "DBLP": "conf/chi/BansalWZFNKRW21",10897 "ArXiv": "2006.14779",10898 "MAG": "3037634279",10899 "DOI": "10.1145/3411764.3445717",10900 "CorpusId": 22012813810901 },10902 "publicationVenue": {10903 "id": "b55b50b1-aae7-47a7-b042-8aecc930073d",10904 "name": "International Conference on Human Factors in Computing Systems",10905 "type": "conference",10906 "alternate_names": [10907 "CHI",10908 "Int Conf Hum Factor Comput Syst",10909 "Human Factors in Computing Systems",10910 "Conference on Human Interface",10911 "Conf Hum Interface",10912 "Hum Factor Comput Syst"10913 ],10914 "url": "http://www.acm.org/sigchi/"10915 },10916 "title": "Does the Whole Exceed its Parts? The Effect of AI Explanations on Complementary Team Performance",10917 "venue": "International Conference on Human Factors in Computing Systems",10918 "year": 2020,10919 "citationCount": 1161,10920 "openAccessPdf": {10921 "url": "https://arxiv.org/pdf/2006.14779",10922 "status": "GREEN",10923 "license": null,10924 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2006.14779, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."10925 },10926 "publicationTypes": [10927 "JournalArticle",10928 "Book",10929 "Conference"10930 ],10931 "journal": {10932 "name": "Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems"10933 },10934 "authors": [10935 {10936 "authorId": "33340656",10937 "name": "Gagan Bansal"10938 },10939 {10940 "authorId": "35232494",10941 "name": "Tongshuang Sherry Wu"10942 },10943 {10944 "authorId": "153823289",10945 "name": "Joyce Zhou"10946 },10947 {10948 "authorId": "27083453",10949 "name": "Raymond Fok"10950 },10951 {10952 "authorId": "2571049",10953 "name": "Besmira Nushi"10954 },10955 {10956 "authorId": "1783184",10957 "name": "Ece Kamar"10958 },10959 {10960 "authorId": "78846919",10961 "name": "Marco Tulio Ribeiro"10962 },10963 {10964 "authorId": "1780531",10965 "name": "Daniel S. Weld"10966 }10967 ],10968 "abstract": "Many researchers motivate explainable AI with studies showing that human-AI team performance on decision-making tasks improves when the AI explains its recommendations. However, prior studies observed improvements from explanations only when the AI, alone, outperformed both the human and the best team. Can explanations help lead to complementary performance, where team accuracy is higher than either the human or the AI working solo? We conduct mixed-method user studies on three datasets, where an AI with accuracy comparable to humans helps participants solve a task (explaining itself in some conditions). While we observed complementary improvements from AI augmentation, they were not increased by explanations. Rather, explanations increased the chance that humans will accept the AI\u2019s recommendation, regardless of its correctness. Our result poses new challenges for human-centered AI: Can we develop explanatory approaches that encourage appropriate trust in AI, and therefore help generate (or improve) complementary performance?"10969 },10970 "DOI:10.1145/3449287": {10971 "paperId": "65772de56676a6a437c8abaeccef34224f62ee13",10972 "externalIds": {10973 "ArXiv": "2102.09692",10974 "DBLP": "journals/pacmhci/BucincaMG21",10975 "DOI": "10.1145/3449287",10976 "CorpusId": 23197927910977 },10978 "publicationVenue": null,10979 "title": "To Trust or to Think",10980 "venue": "Proc. ACM Hum. Comput. Interact.",10981 "year": 2021,10982 "citationCount": 861,10983 "openAccessPdf": {10984 "url": "https://arxiv.org/pdf/2102.09692",10985 "status": "GREEN",10986 "license": null,10987 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2102.09692, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."10988 },10989 "publicationTypes": [10990 "JournalArticle"10991 ],10992 "journal": {10993 "name": "Proceedings of the ACM on Human-Computer Interaction",10994 "pages": "1 - 21",10995 "volume": "5"10996 },10997 "authors": [10998 {10999 "authorId": null,11000 "name": "Zana Bu\u00e7inca"11001 },11002 {11003 "authorId": "2096042676",11004 "name": "Maja Malaya"11005 },11006 {11007 "authorId": "1770992",11008 "name": "Krzysztof Z Gajos"11009 }11010 ],11011 "abstract": "People supported by AI-powered decision support tools frequently overrely on the AI: they accept an AI's suggestion even when that suggestion is wrong. Adding explanations to the AI decisions does not appear to reduce the overreliance and some studies suggest that it might even increase it. Informed by the dual-process theory of cognition, we posit that people rarely engage analytically with each individual AI recommendation and explanation, and instead develop general heuristics about whether and when to follow the AI suggestions. Building on prior research on medical decision-making, we designed three cognitive forcing interventions to compel people to engage more thoughtfully with the AI-generated explanations. We conducted an experiment (N=199), in which we compared our three cognitive forcing designs to two simple explainable AI approaches and to a no-AI baseline. The results demonstrate that cognitive forcing significantly reduced overreliance compared to the simple explainable AI approaches. However, there was a trade-off: people assigned the least favorable subjective ratings to the designs that reduced the overreliance the most. To audit our work for intervention-generated inequalities, we investigated whether our interventions benefited equally people with different levels of Need for Cognition (i.e., motivation to engage in effortful mental activities). Our results show that, on average, cognitive forcing interventions benefited participants higher in Need for Cognition more. Our research suggests that human cognitive motivation moderates the effectiveness of explainable AI solutions."11012 },11013 "DOI:10.1145/3485510": {11014 "paperId": "fcf4c0ecb2d6b194fb0534fd2e4d5f5a233a5452",11015 "externalIds": {11016 "DBLP": "journals/pacmpl/BurckhardtGJKMM21",11017 "DOI": "10.1145/3485510",11018 "CorpusId": 23864095411019 },11020 "publicationVenue": null,11021 "title": "Durable functions: semantics for stateful serverless",11022 "venue": "Proc. ACM Program. Lang.",11023 "year": 2021,11024 "citationCount": 88,11025 "openAccessPdf": {11026 "url": "https://dl.acm.org/doi/pdf/10.1145/3485510",11027 "status": "GOLD",11028 "license": null,11029 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1145/3485510?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/3485510, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11030 },11031 "publicationTypes": [11032 "JournalArticle"11033 ],11034 "journal": {11035 "name": "Proceedings of the ACM on Programming Languages",11036 "pages": "1 - 27",11037 "volume": "5"11038 },11039 "authors": [11040 {11041 "authorId": "1793711",11042 "name": "S. Burckhardt"11043 },11044 {11045 "authorId": "9414640",11046 "name": "Christoph L. Gillum"11047 },11048 {11049 "authorId": "84064782",11050 "name": "David Justo"11051 },11052 {11053 "authorId": "91973722",11054 "name": "Konstantinos Kallas"11055 },11056 {11057 "authorId": "2064130794",11058 "name": "Connor McMahon"11059 },11060 {11061 "authorId": "1789622",11062 "name": "Christopher S. Meiklejohn"11063 }11064 ],11065 "abstract": "Serverless, or Functions-as-a-Service (FaaS), is an increasingly popular paradigm for application development, as it provides implicit elastic scaling and load based billing. However, the weak execution guarantees and intrinsic compute-storage separation of FaaS create serious challenges when developing applications that require persistent state, reliable progress, or synchronization. This has motivated a new generation of serverless frameworks that provide stateful abstractions. For instance, Azure's Durable Functions (DF) programming model enhances FaaS with actors, workflows, and critical sections. As a programming model, DF is interesting because it combines task and actor parallelism, which makes it suitable for a wide range of serverless applications. We describe DF both informally, using examples, and formally, using an idealized high-level model based on the untyped lambda calculus. Next, we demystify how the DF runtime can (1) execute in a distributed unreliable serverless environment with compute-storage separation, yet still conform to the fault-free high-level model, and (2) persist execution progress without requiring checkpointing support by the language runtime. To this end we define two progressively more complex execution models, which contain the compute-storage separation and the record-replay, and prove that they are equivalent to the high-level model."11066 },11067 "DOI:10.1145/3486897": {11068 "paperId": "f791dce07eb1dd9251f9bffb801d6e075091c98d",11069 "externalIds": {11070 "ArXiv": "2105.07028",11071 "DBLP": "journals/corr/abs-2105-07028",11072 "DOI": "10.1145/3486897",11073 "CorpusId": 23474253611074 },11075 "publicationVenue": {11076 "id": "4d9ce1c4-dc84-46b9-903e-e3751c00c7dd",11077 "name": "Communications of the ACM",11078 "type": "journal",11079 "alternate_names": [11080 "Commun ACM",11081 "Communications of The ACM"11082 ],11083 "issn": "0001-0782",11084 "url": "http://www.acm.org/pubs/cacm/",11085 "alternate_urls": [11086 "http://portal.acm.org/cacm",11087 "http://www.acm.org/pubs/contents/journals/cacm/",11088 "https://cacm.acm.org/"11089 ]11090 },11091 "title": "Methods included",11092 "venue": "Communications of the ACM",11093 "year": 2021,11094 "citationCount": 127,11095 "openAccessPdf": {11096 "url": "https://dl.acm.org/doi/pdf/10.1145/3486897",11097 "status": "HYBRID",11098 "license": "CCBY",11099 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2105.07028, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11100 },11101 "publicationTypes": [11102 "JournalArticle"11103 ],11104 "journal": {11105 "name": "Communications of the ACM",11106 "pages": "54 - 63",11107 "volume": "65"11108 },11109 "authors": [11110 {11111 "authorId": "144361904",11112 "name": "M. Crusoe"11113 },11114 {11115 "authorId": "2922103",11116 "name": "S. Abeln"11117 },11118 {11119 "authorId": "1760940",11120 "name": "A. Iosup"11121 },11122 {11123 "authorId": "37885989",11124 "name": "P. Amstutz"11125 },11126 {11127 "authorId": "113676421",11128 "name": "J. Chilton"11129 },11130 {11131 "authorId": "10671181",11132 "name": "Nebojsa Tijanic"11133 },11134 {11135 "authorId": "2085232797",11136 "name": "Herv'e M'enager"11137 },11138 {11139 "authorId": "2161614",11140 "name": "Stian Soiland-Reyes"11141 },11142 {11143 "authorId": "2254095266",11144 "name": "Bogdan Gavrilovic"11145 },11146 {11147 "authorId": "46555127",11148 "name": "C. Goble"11149 },11150 {11151 "authorId": "2521542905",11152 "name": "The Cwl Community"11153 }11154 ],11155 "abstract": "Standardizing computational reuse and portability with the Common Workflow Language."11156 },11157 "DOI:10.1145/3571730": {11158 "paperId": "3def68bd0f856886d34272840a7f81588f2bc082",11159 "externalIds": {11160 "DBLP": "journals/corr/abs-2202-03629",11161 "ArXiv": "2202.03629",11162 "DOI": "10.1145/3571730",11163 "CorpusId": 24665237211164 },11165 "publicationVenue": {11166 "id": "7b2adce0-d53f-49d6-8784-b0645604fe62",11167 "name": "ACM Computing Surveys",11168 "type": "journal",11169 "alternate_names": [11170 "ACM Comput Surv"11171 ],11172 "issn": "0360-0300",11173 "url": "http://www.acm.org/pubs/surveys/",11174 "alternate_urls": [11175 "http://portal.acm.org/csur",11176 "https://csur.acm.org/",11177 "http://csur.acm.org/"11178 ]11179 },11180 "title": "Survey of Hallucination in Natural Language Generation",11181 "venue": "ACM Computing Surveys",11182 "year": 2022,11183 "citationCount": 6517,11184 "openAccessPdf": {11185 "url": "https://arxiv.org/pdf/2202.03629",11186 "status": "GREEN",11187 "license": null,11188 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2202.03629, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11189 },11190 "publicationTypes": [11191 "JournalArticle",11192 "Review"11193 ],11194 "journal": {11195 "name": "ACM Computing Surveys",11196 "pages": "1 - 38",11197 "volume": "55"11198 },11199 "authors": [11200 {11201 "authorId": "3391272",11202 "name": "Ziwei Ji"11203 },11204 {11205 "authorId": "40221187",11206 "name": "Nayeon Lee"11207 },11208 {11209 "authorId": "1397300094",11210 "name": "Rita Frieske"11211 },11212 {11213 "authorId": "1660855299",11214 "name": "Tiezheng Yu"11215 },11216 {11217 "authorId": "49678544",11218 "name": "D. Su"11219 },11220 {11221 "authorId": "98271906",11222 "name": "Yan Xu"11223 },11224 {11225 "authorId": "38524906",11226 "name": "Etsuko Ishii"11227 },11228 {11229 "authorId": "23672613",11230 "name": "Yejin Bang"11231 },11232 {11233 "authorId": "94210578",11234 "name": "Delong Chen"11235 },11236 {11237 "authorId": "47653392",11238 "name": "Wenliang Dai"11239 },11240 {11241 "authorId": "2111680936",11242 "name": "Andrea Madotto"11243 },11244 {11245 "authorId": "40539650",11246 "name": "Pascale Fung"11247 }11248 ],11249 "abstract": "Natural Language Generation (NLG) has improved exponentially in recent years thanks to the development of sequence-to-sequence deep learning technologies such as Transformer-based language models. This advancement has led to more fluent and coherent NLG, leading to improved development in downstream tasks such as abstractive summarization, dialogue generation, and data-to-text generation. However, it is also apparent that deep learning based generation is prone to hallucinate unintended text, which degrades the system performance and fails to meet user expectations in many real-world scenarios. To address this issue, many studies have been presented in measuring and mitigating hallucinated texts, but these have never been reviewed in a comprehensive manner before. In this survey, we thus provide a broad overview of the research progress and challenges in the hallucination problem in NLG. The survey is organized into two parts: (1) a general overview of metrics, mitigation methods, and future directions, and (2) an overview of task-specific research progress on hallucinations in the following downstream tasks, namely abstractive summarization, dialogue generation, generative question answering, data-to-text generation, and machine translation. This survey serves to facilitate collaborative efforts among researchers in tackling the challenge of hallucinated texts in NLG."11250 },11251 "DOI:10.1145/3593013.3594087": {11252 "paperId": "f46050b4ae991a0c047f9930c8de7c706e61c197",11253 "externalIds": {11254 "DBLP": "conf/fat/LaiCSLT23",11255 "DOI": "10.1145/3593013.3594087",11256 "CorpusId": 25913971411257 },11258 "publicationVenue": {11259 "id": "cdc83875-a82d-445c-b097-cbe91afe99a8",11260 "name": "Conference on Fairness, Accountability and Transparency",11261 "type": "conference",11262 "alternate_names": [11263 "FAccT",11264 "Conf Fairness Account Transpar"11265 ],11266 "url": "https://facctconference.org/"11267 },11268 "title": "Towards a Science of Human-AI Decision Making: An Overview of Design Space in Empirical Human-Subject Studies",11269 "venue": "Conference on Fairness, Accountability and Transparency",11270 "year": 2023,11271 "citationCount": 273,11272 "openAccessPdf": {11273 "url": "https://dl.acm.org/doi/pdf/10.1145/3593013.3594087",11274 "status": "BRONZE",11275 "license": null,11276 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1145/3593013.3594087?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/3593013.3594087, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11277 },11278 "publicationTypes": [11279 "Book",11280 "JournalArticle",11281 "Review"11282 ],11283 "journal": {11284 "name": "Proceedings of the 2023 ACM Conference on Fairness, Accountability, and Transparency"11285 },11286 "authors": [11287 {11288 "authorId": "120801533",11289 "name": "V. Lai"11290 },11291 {11292 "authorId": "2131059444",11293 "name": "Cha-Cha Chen"11294 },11295 {11296 "authorId": "1405364873",11297 "name": "Alison Smith-Renner"11298 },11299 {11300 "authorId": "144921048",11301 "name": "Q. Liao"11302 },11303 {11304 "authorId": "2111727675",11305 "name": "Chenhao Tan"11306 }11307 ],11308 "abstract": "AI systems are adopted in numerous domains due to their increasingly strong predictive performance. However, in high-stakes domains such as criminal justice and healthcare, full automation is often not desirable due to safety, ethical, and legal concerns, yet fully manual approaches can be inaccurate and time-consuming. As a result, there is growing interest in the research community to augment human decision making with AI assistance. Besides developing AI technologies for this purpose, the emerging field of human-AI decision making must embrace empirical approaches to form a foundational understanding of how humans interact and work with AI to make decisions. To invite and help structure research efforts towards a science of understanding and improving human-AI decision making, we survey recent literature of empirical human-subject studies on this topic. We summarize the study design choices made in over 100 papers in three important aspects: (1) decision tasks, (2) AI assistance elements, and (3) evaluation metrics. For each aspect, we summarize current trends, discuss gaps in current practices of the field, and make a list of recommendations for future research. Our work highlights the need to develop common frameworks to account for the design and research spaces of human-AI decision making, so that researchers can make rigorous choices in study design, and the research community can build on each other\u2019s work and produce generalizable scientific knowledge. We also hope this work will serve as a bridge for HCI and AI communities to work together to mutually shape the empirical science and computational technologies for human-AI decision making."11309 },11310 "DOI:10.1145/3805760.3814924": {11311 "paperId": "c3b75af79917c7d268ec15fcfc022dbc8e3d9e13",11312 "externalIds": {11313 "DBLP": "conf/aiware/XueAEUW26",11314 "DOI": "10.1145/3805760.3814924",11315 "CorpusId": 28993023611316 },11317 "publicationVenue": null,11318 "title": "SWE-Bench+: Enhanced LLM Coding Benchmark",11319 "venue": "AIware",11320 "year": 2026,11321 "citationCount": 4,11322 "openAccessPdf": {11323 "url": "https://doi.org/10.1145/3805760.3814924",11324 "status": "HYBRID",11325 "license": "CCBY",11326 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1145/3805760.3814924?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/3805760.3814924, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11327 },11328 "publicationTypes": [11329 "JournalArticle",11330 "Book"11331 ],11332 "journal": {11333 "name": "Proceedings of the 3rd ACM International Conference on AI-Powered Software"11334 },11335 "authors": [11336 {11337 "authorId": "2023500130",11338 "name": "Xu Haoran"11339 },11340 {11341 "authorId": "2115012783",11342 "name": "Reem Aleithan"11343 },11344 {11345 "authorId": "2389401231",11346 "name": "Nafid Enan"11347 },11348 {11349 "authorId": "2325093115",11350 "name": "Gias Uddin"11351 },11352 {11353 "authorId": "2257127006",11354 "name": "Song Wang"11355 }11356 ],11357 "abstract": "SWE-Bench is widely used to evaluate whether large language models can resolve real-world GitHub issues, yet the quality of its benchmark instances remains underexplored. We present SWE-Bench+, an enhanced benchmark framework that improves evaluation reliability by addressing two risks: solution leakage in issue descriptions and weak tests that allow plausible but incorrect patches to pass. We analyze 217 commonly resolved issues from SWE-Bench Lite and SWE-Bench Verified across three top-performing agents, yielding 651 model-generated patches. Our analysis identifies five recurring quality-problem patterns under solution leakage and weak tests. We further develop SoluLeakDetector, which detects solution-leaking content, and TestEnhancer, which strengthens validation tests. We find that 60.83% of commonly resolved issues contain solution leakage and 77.88% are problematic overall. SoluLeakDetector achieves 80.45% accuracy, while TestEnhancer identifies plausible patches for 97.11% of weak-test issues and reduces average resolution rates by 27.00 percentage points on Lite and 36.27 percentage points on Verified."11358 },11359 "DOI:10.1145/568522.568525": {11360 "paperId": "6f9633e70f7027a982fbe6d966b8ca1946212399",11361 "externalIds": {11362 "MAG": "2100970777",11363 "DBLP": "journals/csur/ElnozahyAWJ02",11364 "DOI": "10.1145/568522.568525",11365 "CorpusId": 22129454411366 },11367 "publicationVenue": null,11368 "title": "A survey of rollback-recovery protocols in message-passing systems",11369 "venue": "CSUR",11370 "year": 2002,11371 "citationCount": 1975,11372 "openAccessPdf": {11373 "url": "http://www.cs.utexas.edu/users/lorenzo/papers/SurveyFinal.pdf",11374 "status": "GREEN",11375 "license": null,11376 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/568522.568525?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/568522.568525, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11377 },11378 "publicationTypes": [11379 "JournalArticle",11380 "Review"11381 ],11382 "journal": {11383 "name": "ACM Comput. Surv.",11384 "pages": "375-408",11385 "volume": "34"11386 },11387 "authors": [11388 {11389 "authorId": "1727696",11390 "name": "E. Elnozahy"11391 },11392 {11393 "authorId": "2445753",11394 "name": "L. Alvisi"11395 },11396 {11397 "authorId": "1997552",11398 "name": "Yi-Min Wang"11399 },11400 {11401 "authorId": "2157813956",11402 "name": "David B. Johnson"11403 }11404 ],11405 "abstract": null11406 },11407 "DOI:10.1162/coli_a_00486": {11408 "paperId": "07283e1578a50e15ac66efcc35b4ae0cbf2159ef",11409 "externalIds": {11410 "DBLP": "journals/corr/abs-2112-12870",11411 "ArXiv": "2112.12870",11412 "DOI": "10.1162/coli_a_00486",11413 "CorpusId": 24550276111414 },11415 "publicationVenue": {11416 "id": "ee37a78c-f3d8-407a-bd24-bb97fe6dbab9",11417 "name": "Computational Linguistics",11418 "type": "journal",11419 "alternate_names": [11420 "Comput Linguistics"11421 ],11422 "issn": "0891-2017",11423 "alternate_issns": [11424 "1530-9312",11425 "0362-613x",11426 "0362-613X"11427 ],11428 "url": "http://aclanthology.info/venues/cl",11429 "alternate_urls": [11430 "http://mitpress.mit.edu/catalog/item/default.asp?ttype=4&tid=10",11431 "https://www.mitpressjournals.org/loi/coli"11432 ]11433 },11434 "title": "Measuring Attribution in Natural Language Generation Models",11435 "venue": "Computational Linguistics",11436 "year": 2021,11437 "citationCount": 356,11438 "openAccessPdf": {11439 "url": "https://direct.mit.edu/coli/article-pdf/doi/10.1162/coli_a_00486/2133667/coli_a_00486.pdf",11440 "status": "GOLD",11441 "license": "CCBYNCND",11442 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2112.12870, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11443 },11444 "publicationTypes": [11445 "JournalArticle"11446 ],11447 "journal": {11448 "name": "Computational Linguistics",11449 "pages": "777-840",11450 "volume": "49"11451 },11452 "authors": [11453 {11454 "authorId": "2516777",11455 "name": "Hannah Rashkin"11456 },11457 {11458 "authorId": "48942032",11459 "name": "Vitaly Nikolaev"11460 },11461 {11462 "authorId": "48024953",11463 "name": "Matthew Lamm"11464 },11465 {11466 "authorId": "2257256357",11467 "name": "L. Aroyo"11468 },11469 {11470 "authorId": "123052390",11471 "name": "Michael Collins"11472 },11473 {11474 "authorId": "143790066",11475 "name": "Dipanjan Das"11476 },11477 {11478 "authorId": "1754497",11479 "name": "Slav Petrov"11480 },11481 {11482 "authorId": "32012022",11483 "name": "Gaurav Singh Tomar"11484 },11485 {11486 "authorId": "1388156275",11487 "name": "Iulia Turc"11488 },11489 {11490 "authorId": "1781409",11491 "name": "D. Reitter"11492 }11493 ],11494 "abstract": "Abstract Large neural models have brought a new challenge to natural language generation (NLG): It has become imperative to ensure the safety and reliability of the output of models that generate freely. To this end, we present an evaluation framework, Attributable to Identified Sources (AIS), stipulating that NLG output pertaining to the external world is to be verified against an independent, provided source. We define AIS and a two-stage annotation pipeline for allowing annotators to evaluate model output according to annotation guidelines. We successfully validate this approach on generation datasets spanning three tasks (two conversational QA datasets, a summarization dataset, and a table-to-text dataset). We provide full annotation guidelines in the appendices and publicly release the annotated data at https://github.com/google-research-datasets/AIS."11495 },11496 "DOI:10.1177/0956797611417632": {11497 "paperId": "49bda01e28f3f9fffa4fa14bd909c0a2340aa066",11498 "externalIds": {11499 "MAG": "2161498332",11500 "DOI": "10.1177/0956797611417632",11501 "CorpusId": 13802986,11502 "PubMed": "22006061"11503 },11504 "publicationVenue": {11505 "id": "a990ee76-444c-4bda-b714-df4dad0efa5d",11506 "name": "Psychology Science",11507 "type": "journal",11508 "alternate_names": [11509 "Psychol Sci",11510 "Psychological Science"11511 ],11512 "issn": "1614-9947",11513 "alternate_issns": [11514 "0956-7976"11515 ],11516 "url": "http://www.psychologie-aktuell.com/index.php?id=204",11517 "alternate_urls": [11518 "http://www.jstor.org/journals/09567976.html",11519 "http://www.jstor.org/action/showPublication?journalCode=psychsci",11520 "https://www.jstor.org/journal/psychsci",11521 "http://www.blackwellpublishers.co.uk/online",11522 "http://pss.sagepub.com/"11523 ]11524 },11525 "title": "False-Positive Psychology",11526 "venue": "Psychology Science",11527 "year": 2011,11528 "citationCount": 4563,11529 "openAccessPdf": {11530 "url": "http://journals.sagepub.com/doi/pdf/10.1177/0956797611417632",11531 "status": "BRONZE",11532 "license": null,11533 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1177/0956797611417632?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1177/0956797611417632, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11534 },11535 "publicationTypes": [11536 "JournalArticle",11537 "Review"11538 ],11539 "journal": {11540 "name": "Psychological Science",11541 "pages": "1359 - 1366",11542 "volume": "22"11543 },11544 "authors": [11545 {11546 "authorId": "2118445",11547 "name": "J. Simmons"11548 },11549 {11550 "authorId": "33916832",11551 "name": "Leif D. Nelson"11552 },11553 {11554 "authorId": "2130902",11555 "name": "U. Simonsohn"11556 }11557 ],11558 "abstract": null11559 },11560 "DOI:10.1177/1745691612463078": {11561 "paperId": "406a31a9a208c8bc6489720ce21ef542328d5d4b",11562 "externalIds": {11563 "MAG": "2123263696",11564 "DOI": "10.1177/1745691612463078",11565 "CorpusId": 5096417,11566 "PubMed": "26168122"11567 },11568 "publicationVenue": {11569 "id": "8ec18bc5-9c95-446c-a293-735d7ae1e3e9",11570 "name": "Perspectives on Psychological Science",11571 "type": "journal",11572 "alternate_names": [11573 "Perspect Psychol Sci"11574 ],11575 "issn": "1745-6916",11576 "url": "http://www.sagepub.com/journals/Journal201964/title",11577 "alternate_urls": [11578 "https://uk.sagepub.com/en-gb/eur/journal/perspectives-psychological-science",11579 "http://pps.sagepub.com/",11580 "https://www.jstor.org/journal/perspsycscie",11581 "http://www.jstor.org/action/showPublication?journalCode=perspsycscie"11582 ]11583 },11584 "title": "An Agenda for Purely Confirmatory Research",11585 "venue": "Perspectives on Psychological Science",11586 "year": 2012,11587 "citationCount": 987,11588 "openAccessPdf": {11589 "url": "",11590 "status": "CLOSED",11591 "license": null,11592 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1177/1745691612463078?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1177/1745691612463078, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11593 },11594 "publicationTypes": [11595 "JournalArticle",11596 "Review"11597 ],11598 "journal": {11599 "name": "Perspectives on Psychological Science",11600 "pages": "632 - 638",11601 "volume": "7"11602 },11603 "authors": [11604 {11605 "authorId": "2505272",11606 "name": "E. Wagenmakers"11607 },11608 {11609 "authorId": "2213973960",11610 "name": "Ruud Wetzels"11611 },11612 {11613 "authorId": "3237553",11614 "name": "D. Borsboom"11615 },11616 {11617 "authorId": "2482801",11618 "name": "H.L.J. van der Maas"11619 },11620 {11621 "authorId": "144707887",11622 "name": "R. Kievit"11623 }11624 ],11625 "abstract": null11626 },11627 "DOI:10.1177/17456916231181102": {11628 "paperId": "492202b2502b9d76900370cddd38a4cdcd580897",11629 "externalIds": {11630 "PubMedCentral": "11373149",11631 "DOI": "10.1177/17456916231181102",11632 "CorpusId": 259843975,11633 "PubMed": "37439761"11634 },11635 "publicationVenue": {11636 "id": "8ec18bc5-9c95-446c-a293-735d7ae1e3e9",11637 "name": "Perspectives on Psychological Science",11638 "type": "journal",11639 "alternate_names": [11640 "Perspect Psychol Sci"11641 ],11642 "issn": "1745-6916",11643 "url": "http://www.sagepub.com/journals/Journal201964/title",11644 "alternate_urls": [11645 "https://uk.sagepub.com/en-gb/eur/journal/perspectives-psychological-science",11646 "http://pps.sagepub.com/",11647 "https://www.jstor.org/journal/perspsycscie",11648 "http://www.jstor.org/action/showPublication?journalCode=perspsycscie"11649 ]11650 },11651 "title": "Three Challenges for AI-Assisted Decision-Making",11652 "venue": "Perspectives on Psychological Science",11653 "year": 2023,11654 "citationCount": 224,11655 "openAccessPdf": {11656 "url": "https://journals.sagepub.com/doi/pdf/10.1177/17456916231181102",11657 "status": "HYBRID",11658 "license": "CCBY",11659 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC11373149, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11660 },11661 "publicationTypes": [11662 "JournalArticle"11663 ],11664 "journal": {11665 "name": "Perspectives on Psychological Science",11666 "pages": "722 - 734",11667 "volume": "19"11668 },11669 "authors": [11670 {11671 "authorId": "1804885",11672 "name": "M. Steyvers"11673 },11674 {11675 "authorId": "2110096539",11676 "name": "Aakriti Kumar"11677 }11678 ],11679 "abstract": "Artificial intelligence (AI) has the potential to improve human decision-making by providing decision recommendations and problem-relevant information to assist human decision-makers. However, the full realization of the potential of human\u2013AI collaboration continues to face several challenges. First, the conditions that support complementarity (i.e., situations in which the performance of a human with AI assistance exceeds the performance of an unassisted human or the AI in isolation) must be understood. This task requires humans to be able to recognize situations in which the AI should be leveraged and to develop new AI systems that can learn to complement the human decision-maker. Second, human mental models of the AI, which contain both expectations of the AI and reliance strategies, must be accurately assessed. Third, the effects of different design choices for human-AI interaction must be understood, including both the timing of AI assistance and the amount of model information that should be presented to the human decision-maker to avoid cognitive overload and ineffective reliance strategies. In response to each of these three challenges, we present an interdisciplinary perspective based on recent empirical and theoretical findings and discuss new research directions."11680 },11681 "DOI:10.1177/25152459211007467": {11682 "paperId": "57eb25f55d349caf0ae61fbf10e74496fb2887d1",11683 "externalIds": {11684 "MAG": "3082857103",11685 "DOI": "10.1177/25152459211007467",11686 "CorpusId": 23508010611687 },11688 "publicationVenue": {11689 "id": "db890f0d-5d62-433f-af5a-f99e9c862fd2",11690 "name": "Advances in Methods and Practices in Psychological Science",11691 "alternate_names": [11692 "Adv Method Pract Psychol Sci"11693 ],11694 "issn": "2515-2459",11695 "url": "https://journals.sagepub.com/home/amp",11696 "alternate_urls": [11697 "https://journals.sagepub.com/loi/amp",11698 "https://www.psychologicalscience.org/publications/ampps"11699 ]11700 },11701 "title": "An Excess of Positive Results: Comparing the Standard Psychology Literature With Registered Reports",11702 "venue": "Advances in Methods and Practices in Psychological Science",11703 "year": 2020,11704 "citationCount": 356,11705 "openAccessPdf": {11706 "url": "https://doi.org/10.31234/osf.io/p6e9c",11707 "status": "GREEN",11708 "license": "CCBY",11709 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1177/25152459211007467?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1177/25152459211007467, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11710 },11711 "publicationTypes": [11712 "JournalArticle",11713 "Review"11714 ],11715 "journal": {11716 "name": "Advances in Methods and Practices in Psychological Science",11717 "volume": "4"11718 },11719 "authors": [11720 {11721 "authorId": "152409539",11722 "name": "Anne M. Scheel"11723 },11724 {11725 "authorId": "115920485",11726 "name": "Mitchell Schijen"11727 },11728 {11729 "authorId": "87792747",11730 "name": "D. Lakens"11731 }11732 ],11733 "abstract": "Selectively publishing results that support the tested hypotheses (\u201cpositive\u201d results) distorts the available evidence for scientific claims. For the past decade, psychological scientists have been increasingly concerned about the degree of such distortion in their literature. A new publication format has been developed to prevent selective reporting: In Registered Reports (RRs), peer review and the decision to publish take place before results are known. We compared the results in published RRs (N = 71 as of November 2018) with a random sample of hypothesis-testing studies from the standard literature (N = 152) in psychology. Analyzing the first hypothesis of each article, we found 96% positive results in standard reports but only 44% positive results in RRs. We discuss possible explanations for this large difference and suggest that a plausible factor is the reduction of publication bias and/or Type I error inflation in the RR literature."11734 },11735 "DOI:10.1207/s15327957pspr0203_4": {11736 "paperId": "cdea0d7c29c5ab65dbee126ef9f2b6f28a5f0b94",11737 "externalIds": {11738 "MAG": "2118070418",11739 "DOI": "10.1207/s15327957pspr0203_4",11740 "CorpusId": 22724226,11741 "PubMed": "15647155"11742 },11743 "publicationVenue": {11744 "id": "5b750144-6d7e-45e3-9e2f-44263ea890a0",11745 "name": "Personality and Social Psychology Review",11746 "type": "journal",11747 "alternate_names": [11748 "Personal Soc Psychol Rev"11749 ],11750 "issn": "1532-7957",11751 "url": "https://journals.sagepub.com/loi/psr",11752 "alternate_urls": [11753 "http://psr.sagepub.com/"11754 ]11755 },11756 "title": "HARKing: Hypothesizing After the Results are Known",11757 "venue": "Personality and Social Psychology Review",11758 "year": 1998,11759 "citationCount": 1093,11760 "openAccessPdf": {11761 "url": "http://www2.psych.ubc.ca/~schaller/528Readings/Kerr1998.pdf",11762 "status": "BRONZE",11763 "license": null,11764 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1207/s15327957pspr0203_4?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1207/s15327957pspr0203_4, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11765 },11766 "publicationTypes": [11767 "JournalArticle",11768 "Review"11769 ],11770 "journal": {11771 "name": "Personality and Social Psychology Review",11772 "pages": "196 - 217",11773 "volume": "2"11774 },11775 "authors": [11776 {11777 "authorId": "5255855",11778 "name": "Norbert L. Kerr"11779 }11780 ],11781 "abstract": null11782 },11783 "DOI:10.12688/f1000research.29032.2": null,11784 "DOI:10.12688/f1000research.51738.2": {11785 "paperId": "f7cf15d7591c98824a2c8e15c699787e530322ec",11786 "externalIds": {11787 "MAG": "3151240889",11788 "DBLP": "journals/f1000research/NustE21",11789 "PubMedCentral": "8311796",11790 "DOI": "10.12688/f1000research.51738.2",11791 "CorpusId": 233203273,11792 "PubMed": "34367614"11793 },11794 "publicationVenue": {11795 "id": "dc95b805-f39c-43dd-bf0f-500edd72da12",11796 "name": "F1000Research",11797 "type": "journal",11798 "issn": "2046-1402",11799 "url": "https://f1000research.com/"11800 },11801 "title": "CODECHECK: an Open Science initiative for the independent execution of computations underlying research articles during peer review to improve reproducibility",11802 "venue": "F1000Research",11803 "year": 2021,11804 "citationCount": 44,11805 "openAccessPdf": {11806 "url": "https://f1000research.com/articles/10-253/v2/pdf",11807 "status": "GOLD",11808 "license": "CCBYSA",11809 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC8311796, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11810 },11811 "publicationTypes": [11812 "JournalArticle",11813 "Review"11814 ],11815 "journal": {11816 "name": "F1000Research",11817 "volume": "10"11818 },11819 "authors": [11820 {11821 "authorId": "2587120",11822 "name": "Daniel N\u00fcst"11823 },11824 {11825 "authorId": "2713377",11826 "name": "Stephen J. Eglen"11827 }11828 ],11829 "abstract": "The traditional scientific paper falls short of effectively communicating computational research. To help improve this situation, we propose a system by which the computational workflows underlying research articles are checked. The CODECHECK system uses open infrastructure and tools and can be integrated into review and publication processes in multiple ways. We describe these integrations along multiple dimensions (importance, who, openness, when). In collaboration with academic publishers and conferences, we demonstrate CODECHECK with 25 reproductions of diverse scientific publications. These CODECHECKs show that asking for reproducible workflows during a collaborative review can effectively improve executability. While CODECHECK has clear limitations, it may represent a building block in Open Science and publishing ecosystems for improving the reproducibility, appreciation, and, potentially, the quality of non-textual research artefacts. The CODECHECK website can be accessed here: https://codecheck.org.uk/."11830 },11831 "DOI:10.1287/isre.2021.1079": {11832 "paperId": "73c4417142eea24e290bf574de653fcc9604c591",11833 "externalIds": {11834 "DBLP": "journals/isr/FugenerGGK22",11835 "DOI": "10.1287/isre.2021.1079",11836 "CorpusId": 24506283011837 },11838 "publicationVenue": {11839 "id": "6b25cc59-3bb5-4b47-819e-d95ba292c4f7",11840 "name": "Information systems research",11841 "type": "journal",11842 "alternate_names": [11843 "Information Systems Research",11844 "Inf syst res",11845 "Inf Syst Res"11846 ],11847 "issn": "1047-7047",11848 "url": "http://isr.journal.informs.org/",11849 "alternate_urls": [11850 "http://www.jstor.org/action/showPublication?journalCode=infosysres",11851 "https://pubsonline.informs.org/",11852 "https://www.jstor.org/journal/infosysres"11853 ]11854 },11855 "title": "Cognitive Challenges in Human-Artificial Intelligence Collaboration: Investigating the Path Toward Productive Delegation",11856 "venue": "Information systems research",11857 "year": 2021,11858 "citationCount": 466,11859 "openAccessPdf": {11860 "url": "https://pure.eur.nl/en/publications/7ce28963-ab8a-44f8-a647-b98064150835",11861 "status": "GREEN",11862 "license": "CCBY",11863 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1287/isre.2021.1079?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1287/isre.2021.1079, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11864 },11865 "publicationTypes": [11866 "JournalArticle"11867 ],11868 "journal": {11869 "name": "Inf. Syst. Res.",11870 "pages": "678-696",11871 "volume": "33"11872 },11873 "authors": [11874 {11875 "authorId": "2251595",11876 "name": "Andreas F\u00fcgener"11877 },11878 {11879 "authorId": "1697156",11880 "name": "J\u00f6rn Grahl"11881 },11882 {11883 "authorId": "2109979099",11884 "name": "Alok Gupta"11885 },11886 {11887 "authorId": "1772746",11888 "name": "W. Ketter"11889 }11890 ],11891 "abstract": "A consensus is beginning to emerge that the next phase of artificial intelligence (AI) induction in business organizations will require humans to work with AI in a variety of work arrangements. This article explores the issues related to human capabilities to work with AI. A key to working in many work arrangements is the ability to delegate work to entities that can do them most efficiently. Modern AI can do a remarkable job of efficient delegation to humans because it knows what it knows well and what it does not. Humans, on the other hand, are poor judges of their metaknowledge and are not good at delegating knowledge work to AI\u2014this might prove to be a big stumbling block to create work environments where humans and AI work together. Humans have often created machines to serve them. The sentiment is perhaps exemplified by Oscar Wilde\u2019s statement that \u201ccivilization requires slaves\u2026. Human slavery is wrong, insecure and demoralizing. On mechanical slavery, on the slavery of the machine, the future of the world depends.\u201d However, the time has come when humans might switch roles with machines. Our study highlights capabilities that humans need to effectively work with AI and still be in control rather than just being directed."11892 },11893 "DOI:10.1371/journal.pbio.1002106": {11894 "paperId": "a07a64ba110e0f9f7156f3bd1e376f0d2e1cddf1",11895 "externalIds": {11896 "MAG": "2164005910",11897 "PubMedCentral": "4359000",11898 "DOI": "10.1371/journal.pbio.1002106",11899 "CorpusId": 17475145,11900 "PubMed": "25768323"11901 },11902 "publicationVenue": {11903 "id": "83ff973b-8a0e-4e00-a06a-2cfd9e222de9",11904 "name": "PLoS Biology",11905 "type": "journal",11906 "alternate_names": [11907 "Plo Biology",11908 "PLOS Biology",11909 "PLO Biology"11910 ],11911 "issn": "1544-9173",11912 "url": "http://www.pubmedcentral.nih.gov/tocrender.fcgi?journal=212",11913 "alternate_urls": [11914 "http://www.plosbiology.org/",11915 "https://journals.plos.org/plosbiology/"11916 ]11917 },11918 "title": "The Extent and Consequences of P-Hacking in Science",11919 "venue": "PLoS Biology",11920 "year": 2015,11921 "citationCount": 1156,11922 "openAccessPdf": {11923 "url": "https://journals.plos.org/plosbiology/article/file?id=10.1371/journal.pbio.1002106&type=printable",11924 "status": "GOLD",11925 "license": "CCBY",11926 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC4359000, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11927 },11928 "publicationTypes": [11929 "JournalArticle"11930 ],11931 "journal": {11932 "name": "PLoS Biology",11933 "volume": "13"11934 },11935 "authors": [11936 {11937 "authorId": "6652932",11938 "name": "M. Head"11939 },11940 {11941 "authorId": "3884621",11942 "name": "L. Holman"11943 },11944 {11945 "authorId": "5325261",11946 "name": "R. Lanfear"11947 },11948 {11949 "authorId": "12670573",11950 "name": "Andrew T. Kahn"11951 },11952 {11953 "authorId": "3495787",11954 "name": "M. Jennions"11955 }11956 ],11957 "abstract": "A focus on novel, confirmatory, and statistically significant results leads to substantial bias in the scientific literature. One type of bias, known as \u201cp-hacking,\u201d occurs when researchers collect or select data or statistical analyses until nonsignificant results become significant. Here, we use text-mining to demonstrate that p-hacking is widespread throughout science. We then illustrate how one can test for p-hacking when performing a meta-analysis and show that, while p-hacking is probably common, its effect seems to be weak relative to the real effect sizes being measured. This result suggests that p-hacking probably does not drastically alter scientific consensuses drawn from meta-analyses."11958 },11959 "DOI:10.1371/journal.pcbi.1003285": {11960 "paperId": "12b056fd90c1300c44c4a8cff9f01f0c7a75cab4",11961 "externalIds": {11962 "DBLP": "journals/ploscb/SandveNTH13",11963 "MAG": "2036318837",11964 "PubMedCentral": "3812051",11965 "DOI": "10.1371/journal.pcbi.1003285",11966 "CorpusId": 2180281,11967 "PubMed": "24204232"11968 },11969 "publicationVenue": null,11970 "title": "Ten Simple Rules for Reproducible Computational Research",11971 "venue": "PLoS Comput. Biol.",11972 "year": 2013,11973 "citationCount": 919,11974 "openAccessPdf": {11975 "url": "https://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1003285&type=printable",11976 "status": "GOLD",11977 "license": "CCBY",11978 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC3812051, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."11979 },11980 "publicationTypes": [11981 "JournalArticle"11982 ],11983 "journal": {11984 "name": "PLoS Computational Biology",11985 "volume": "9"11986 },11987 "authors": [11988 {11989 "authorId": "49687105",11990 "name": "G. K. Sandve"11991 },11992 {11993 "authorId": "2686089",11994 "name": "Anton Nekrutenko"11995 },11996 {11997 "authorId": "31436732",11998 "name": "James Taylor"11999 },12000 {12001 "authorId": "1925617",12002 "name": "E. Hovig"12003 }12004 ],12005 "abstract": "Replication is the cornerstone of a cumulative science [1]. However, new tools and technologies, massive amounts of data, interdisciplinary approaches, and the complexity of the questions being asked are complicating replication efforts, as are increased pressures on scientists to advance their research [2]. As full replication of studies on independently collected data is often not feasible, there has recently been a call for reproducible research as an attainable minimum standard for assessing the value of scientific claims [3]. This requires that papers in experimental science describe the results and provide a sufficiently clear protocol to allow successful repetition and extension of analyses based on original data [4]. \n \nThe importance of replication and reproducibility has recently been exemplified through studies showing that scientific papers commonly leave out experimental details essential for reproduction [5], studies showing difficulties with replicating published experimental results [6], an increase in retracted papers [7], and through a high number of failing clinical trials [8], [9]. This has led to discussions on how individual researchers, institutions, funding bodies, and journals can establish routines that increase transparency and reproducibility. In order to foster such aspects, it has been suggested that the scientific community needs to develop a \u201cculture of reproducibility\u201d for computational science, and to require it for published claims [3]. \n \nWe want to emphasize that reproducibility is not only a moral responsibility with respect to the scientific field, but that a lack of reproducibility can also be a burden for you as an individual researcher. As an example, a good practice of reproducibility is necessary in order to allow previously developed methodology to be effectively applied on new data, or to allow reuse of code and results for new projects. In other words, good habits of reproducibility may actually turn out to be a time-saver in the longer run. \n \nWe further note that reproducibility is just as much about the habits that ensure reproducible research as the technologies that can make these processes efficient and realistic. Each of the following ten rules captures a specific aspect of reproducibility, and discusses what is needed in terms of information handling and tracking of procedures. If you are taking a bare-bones approach to bioinformatics analysis, i.e., running various custom scripts from the command line, you will probably need to handle each rule explicitly. If you are instead performing your analyses through an integrated framework (such as GenePattern [10], Galaxy [11], LONI pipeline [12], or Taverna [13]), the system may already provide full or partial support for most of the rules. What is needed on your part is then merely the knowledge of how to exploit these existing possibilities. \n \nIn a pragmatic setting, with publication pressure and deadlines, one may face the need to make a trade-off between the ideals of reproducibility and the need to get the research out while it is still relevant. This trade-off becomes more important when considering that a large part of the analyses being tried out never end up yielding any results. However, frequently one will, with the wisdom of hindsight, contemplate the missed opportunity to ensure reproducibility, as it may already be too late to take the necessary notes from memory (or at least much more difficult than to do it while underway). We believe that the rewards of reproducibility will compensate for the risk of having spent valuable time developing an annotated catalog of analyses that turned out as blind alleys. \n \nAs a minimal requirement, you should at least be able to reproduce the results yourself. This would satisfy the most basic requirements of sound research, allowing any substantial future questioning of the research to be met with a precise explanation. Although it may sound like a very weak requirement, even this level of reproducibility will often require a certain level of care in order to be met. There will for a given analysis be an exponential number of possible combinations of software versions, parameter values, pre-processing steps, and so on, meaning that a failure to take notes may make exact reproduction essentially impossible. \n \nWith this basic level of reproducibility in place, there is much more that can be wished for. An obvious extension is to go from a level where you can reproduce results in case of a critical situation to a level where you can practically and routinely reuse your previous work and increase your productivity. A second extension is to ensure that peers have a practical possibility of reproducing your results, which can lead to increased trust in, interest for, and citations of your work [6], [14]. \n \nWe here present ten simple rules for reproducibility of computational research. These rules can be at your disposal for whenever you want to make your research more accessible\u2014be it for peers or for your future self."12006 },12007 "DOI:10.1371/journal.pcbi.1008316": {12008 "paperId": "317b30303d162ba11f450183bc9c19587bc5bd0e",12009 "externalIds": {12010 "PubMedCentral": "7654784",12011 "DBLP": "journals/ploscb/NustSMEHHE20",12012 "MAG": "3102028509",12013 "DOI": "10.1371/journal.pcbi.1008316",12014 "CorpusId": 226303277,12015 "PubMed": "33170857"12016 },12017 "publicationVenue": null,12018 "title": "Ten simple rules for writing Dockerfiles for reproducible data science",12019 "venue": "PLoS Comput. Biol.",12020 "year": 2020,12021 "citationCount": 85,12022 "openAccessPdf": {12023 "url": "https://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1008316&type=printable",12024 "status": "GOLD",12025 "license": "CCBY",12026 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC7654784, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."12027 },12028 "publicationTypes": [12029 "Editorial",12030 "JournalArticle"12031 ],12032 "journal": {12033 "name": "PLoS Computational Biology",12034 "volume": "16"12035 },12036 "authors": [12037 {12038 "authorId": "2587120",12039 "name": "Daniel N\u00fcst"12040 },12041 {12042 "authorId": "2978615",12043 "name": "V. Sochat"12044 },12045 {12046 "authorId": "8755947",12047 "name": "B. Marwick"12048 },12049 {12050 "authorId": "2713377",12051 "name": "Stephen J. Eglen"12052 },12053 {12054 "authorId": "2169785071",12055 "name": "T. Head"12056 },12057 {12058 "authorId": "5754713",12059 "name": "T. Hirst"12060 },12061 {12062 "authorId": "2055340502",12063 "name": "Benjamin D. Evans"12064 }12065 ],12066 "abstract": "Computational science has been greatly improved by the use of containers for packaging software and data dependencies. In a scholarly context, the main drivers for using these containers are transparency and support of reproducibility; in turn, a workflow\u2019s reproducibility can be greatly affected by the choices that are made with respect to building containers. In many cases, the build process for the container\u2019s image is created from instructions provided in a Dockerfile format. In support of this approach, we present a set of rules to help researchers write understandable Dockerfiles for typical data science workflows. By following the rules in this article, researchers can create containers suitable for sharing with fellow scientists, for including in scholarly communication such as education or scientific papers, and for effective and sustainable personal workflows."12067 },12068 "DOI:10.1371/journal.pmed.0020124": null,12069 "DOI:10.14778/3529337.3529344": null,12070 "DOI:10.1511/2014.111.460": {12071 "paperId": "822447fab9e8bf5b7b58e5380527a5a746de5694",12072 "externalIds": {12073 "MAG": "2322006099",12074 "DOI": "10.1511/2014.111.460",12075 "CorpusId": 14685523512076 },12077 "publicationVenue": null,12078 "title": "The statistical crisis in science",12079 "venue": "",12080 "year": 2014,12081 "citationCount": 856,12082 "openAccessPdf": {12083 "url": "",12084 "status": "CLOSED",12085 "license": null,12086 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1511/2014.111.460?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1511/2014.111.460, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."12087 },12088 "publicationTypes": null,12089 "journal": {12090 "name": "American Scientist",12091 "pages": "460",12092 "volume": "102"12093 },12094 "authors": [12095 {12096 "authorId": "144389145",12097 "name": "A. Gelman"12098 },12099 {12100 "authorId": "145092174",12101 "name": "E. Loken"12102 }12103 ],12104 "abstract": null12105 },12106 "DOI:10.1518/001872097778543886": {12107 "paperId": "1cf890ce8391ca958fbc173533d575efcb51f70a",12108 "externalIds": {12109 "MAG": "2063052894",12110 "DBLP": "journals/hf/ParasuramanR97",12111 "DOI": "10.1518/001872097778543886",12112 "CorpusId": 4114907812113 },12114 "publicationVenue": null,12115 "title": "Humans and Automation: Use, Misuse, Disuse, Abuse",12116 "venue": "Hum. Factors",12117 "year": 1997,12118 "citationCount": 4879,12119 "openAccessPdf": {12120 "url": "",12121 "status": "CLOSED",12122 "license": null,12123 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1518/001872097778543886?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1518/001872097778543886, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."12124 },12125 "publicationTypes": [12126 "JournalArticle"12127 ],12128 "journal": {12129 "name": "Human Factors: The Journal of Human Factors and Ergonomics Society",12130 "pages": "230 - 253",12131 "volume": "39"12132 },12133 "authors": [12134 {12135 "authorId": "3264674",12136 "name": "R. Parasuraman"12137 },12138 {12139 "authorId": "50615483",12140 "name": "V. Riley"12141 }12142 ],12143 "abstract": null12144 },12145 "DOI:10.1518/hfes.46.1.50_30392": {12146 "paperId": "7dd86508438657ac7a704a5d952a2a4422808975",12147 "externalIds": {12148 "DBLP": "journals/hf/LeeS04",12149 "MAG": "2110171129",12150 "DOI": "10.1518/hfes.46.1.50_30392",12151 "CorpusId": 5210390,12152 "PubMed": "15151155"12153 },12154 "publicationVenue": null,12155 "title": "Trust in Automation: Designing for Appropriate Reliance",12156 "venue": "Hum. Factors",12157 "year": 2004,12158 "citationCount": 6989,12159 "openAccessPdf": {12160 "url": "",12161 "status": "CLOSED",12162 "license": null,12163 "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1518/hfes.46.1.50_30392?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1518/hfes.46.1.50_30392, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."12164 },12165 "publicationTypes": [12166 "Review",12167 "JournalArticle"12168 ],12169 "journal": {12170 "name": "Human Factors: The Journal of Human Factors and Ergonomics Society",12171 "pages": "50 - 80",12172 "volume": "46"12173 },12174 "authors": [12175 {12176 "authorId": "1520031837",12177 "name": "John D. Lee"12178 },12179 {12180 "authorId": "49748104",12181 "name": "Katrina A. See"12182 }12183 ],12184 "abstract": null12185 },12186 "DOI:10.17226/25303": {12187 "paperId": "655be23c570656c5fcee09c1a7e0ccabc12be1fb",12188 "externalIds": {12189 "MAG": "2944196828",12190 "DOI": "10.17226/25303",12191 "CorpusId": 27615751612192 },12193 "publicationVenue": null,12194 "title": "Reproducibility and Replicability in Science",12195 "venue": "",12196 "year": 2019,12197 "citationCount": 73,12198 "openAccessPdf": {12199 "url": "",12200 "status": "CLOSED",12201 "license": null,12202 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.17226/25303?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.17226/25303, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."12203 },12204 "publicationTypes": null,12205 "journal": {12206 "name": "",12207 "volume": ""12208 },12209 "authors": [12210 {12211 "authorId": "83862054",12212 "name": "Division on Earth"12213 },12214 {12215 "authorId": "71242706",12216 "name": "C. Behavioral"12217 }12218 ],12219 "abstract": "Jennifer Heimberg, Study Director, Senior Program Officer, Nuclear and Radiation Studies Board, Division on Earth and Life Studies Adrienne Stith Butler, Associate Director, Board on Behavioral, Cognitive, and Sensory Sciences, Division of Behavioral and Social Sciences and Education Tom Arrison, Board/Program Director, Policy and Global Affairs Michael Cohen, Senior Program Officer, Committee on National Statistics, Division of Behavioral and Social Sciences and Education Michelle Schwalbe, Director, Board on Mathematical Sciences and Analytics, Division on Engineering and Physical Sciences Tina Winters, Associate Program Officer, Board on Behavioral, Cognitive, and Sensory Sciences, Division of Behavioral and Social Sciences and Education Study Committee About Consensus Studies Statement of Task"12220 },12221 "DOI:10.18653/v1/2020.emnlp-main.609": {12222 "paperId": "b770d84055c32febe922be9931c453fdbebe9002",12223 "externalIds": {12224 "MAG": "3023035014",12225 "ArXiv": "2004.14974",12226 "DBLP": "conf/emnlp/WaddenLLWZCH20",12227 "ACL": "2020.emnlp-main.609",12228 "DOI": "10.18653/v1/2020.emnlp-main.609",12229 "CorpusId": 21686713312230 },12231 "publicationVenue": {12232 "id": "41bf9ed3-85b3-4c90-b015-150e31690253",12233 "name": "Conference on Empirical Methods in Natural Language Processing",12234 "type": "conference",12235 "alternate_names": [12236 "Empir Method Nat Lang Process",12237 "Empirical Methods in Natural Language Processing",12238 "Conf Empir Method Nat Lang Process",12239 "EMNLP"12240 ],12241 "url": "https://www.aclweb.org/portal/emnlp"12242 },12243 "title": "Fact or Fiction: Verifying Scientific Claims",12244 "venue": "Conference on Empirical Methods in Natural Language Processing",12245 "year": 2020,12246 "citationCount": 912,12247 "openAccessPdf": {12248 "url": "https://www.aclweb.org/anthology/2020.emnlp-main.609.pdf",12249 "status": "HYBRID",12250 "license": "CCBY",12251 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2004.14974, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."12252 },12253 "publicationTypes": [12254 "JournalArticle",12255 "Conference"12256 ],12257 "journal": {12258 "name": "ArXiv",12259 "volume": "abs/2004.14974"12260 },12261 "authors": [12262 {12263 "authorId": "30051202",12264 "name": "David Wadden"12265 },12266 {12267 "authorId": "32370203",12268 "name": "Shanchuan Lin"12269 },12270 {12271 "authorId": "46258841",12272 "name": "Kyle Lo"12273 },12274 {12275 "authorId": "31860505",12276 "name": "Lucy Lu Wang"12277 },12278 {12279 "authorId": "15292561",12280 "name": "Madeleine van Zuylen"12281 },12282 {12283 "authorId": "2527954",12284 "name": "Arman Cohan"12285 },12286 {12287 "authorId": "2548384",12288 "name": "Hannaneh Hajishirzi"12289 }12290 ],12291 "abstract": "We introduce scientific claim verification, a new task to select abstracts from the research literature containing evidence that supports or refutes a given scientific claim, and to identify rationales justifying each decision. To study this task, we construct SciFact, a dataset of 1.4K expert-written scientific claims paired with evidence-containing abstracts annotated with labels and rationales. We develop baseline models for SciFact, and demonstrate that these models benefit from combined training on a large dataset of claims about Wikipedia articles, together with the new SciFact data. We show that our claim verification system is able to identify plausible evidence for 23 / 36 claims relevant to COVID-19 on the CORD-19 corpus. Our results and experiments strongly suggest that our new task and data will support significant future research efforts."12292 },12293 "DOI:10.18653/v1/2022.acl-short.18": null,12294 "DOI:10.18653/v1/2023.acl-long.910": {12295 "paperId": "66242baf48b0f6b828e7547ac39ffaa5e1b2cb3e",12296 "externalIds": {12297 "ArXiv": "2210.08726",12298 "DBLP": "conf/acl/GaoDPCCFZLLJG23",12299 "ACL": "2023.acl-long.910",12300 "DOI": "10.18653/v1/2023.acl-long.910",12301 "CorpusId": 25424726012302 },12303 "publicationVenue": {12304 "id": "1e33b3be-b2ab-46e9-96e8-d4eb4bad6e44",12305 "name": "Annual Meeting of the Association for Computational Linguistics",12306 "type": "conference",12307 "alternate_names": [12308 "Annu Meet Assoc Comput Linguistics",12309 "Meeting of the Association for Computational Linguistics",12310 "ACL",12311 "Meet Assoc Comput Linguistics"12312 ],12313 "url": "https://www.aclweb.org/anthology/venues/acl/"12314 },12315 "title": "RARR: Researching and Revising What Language Models Say, Using Language Models",12316 "venue": "Annual Meeting of the Association for Computational Linguistics",12317 "year": 2022,12318 "citationCount": 407,12319 "openAccessPdf": {12320 "url": "https://aclanthology.org/2023.acl-long.910.pdf",12321 "status": "HYBRID",12322 "license": "CCBY",12323 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2210.08726, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."12324 },12325 "publicationTypes": [12326 "JournalArticle",12327 "Conference"12328 ],12329 "journal": {12330 "pages": "16477-16508"12331 },12332 "authors": [12333 {12334 "authorId": "49715441",12335 "name": "Luyu Gao"12336 },12337 {12338 "authorId": "2475437",12339 "name": "Zhuyun Dai"12340 },12341 {12342 "authorId": "2616463",12343 "name": "Panupong Pasupat"12344 },12345 {12346 "authorId": "52151521",12347 "name": "Anthony Chen"12348 },12349 {12350 "authorId": "2719924",12351 "name": "Arun Tejasvi Chaganty"12352 },12353 {12354 "authorId": "2051969653",12355 "name": "Yicheng Fan"12356 },12357 {12358 "authorId": "2664737",12359 "name": "Vincent Zhao"12360 },12361 {12362 "authorId": "1914797",12363 "name": "N. Lao"12364 },12365 {12366 "authorId": "8386466",12367 "name": "Hongrae Lee"12368 },12369 {12370 "authorId": "50270386",12371 "name": "Da-Cheng Juan"12372 },12373 {12374 "authorId": "2091768",12375 "name": "Kelvin Guu"12376 }12377 ],12378 "abstract": "Language models (LMs) now excel at many tasks such as question answering, reasoning, and dialog. However, they sometimes generate unsupported or misleading content. A user cannot easily determine whether their outputs are trustworthy or not, because most LMs do not have any built-in mechanism for attribution to external evidence. To enable attribution while still preserving all the powerful advantages of recent generation models, we propose RARR (Retrofit Attribution using Research and Revision), a system that 1) automatically finds attribution for the output of any text generation model, and 2) post-edits the output to fix unsupported content while preserving the original output as much as possible. When applied to the output of several state-of-the-art LMs on a diverse set of generation tasks, we find that RARR significantly improves attribution while otherwise preserving the original input to a much greater degree than previously explored edit models. Furthermore, the implementation of RARR requires only a handful of training examples, a large language model, and standard web search."12379 },12380 "DOI:10.18653/v1/2023.emnlp-main.398": null,12381 "DOI:10.18653/v1/2023.emnlp-main.741": null,12382 "DOI:10.18653/v1/2023.findings-emnlp.722": null,12383 "DOI:10.18653/v1/N18-1074": {12384 "paperId": "b1d24e8e08435b7c52335485a0d635abf9bc604c",12385 "externalIds": {12386 "MAG": "2789566302",12387 "ACL": "N18-1074",12388 "ArXiv": "1803.05355",12389 "DBLP": "conf/naacl/ThorneVCM18",12390 "DOI": "10.18653/v1/N18-1074",12391 "CorpusId": 471142512392 },12393 "publicationVenue": {12394 "id": "01103732-3808-4930-b8e4-7e9e68d5c68d",12395 "name": "North American Chapter of the Association for Computational Linguistics",12396 "type": "conference",12397 "alternate_names": [12398 "North Am Chapter Assoc Comput Linguistics",12399 "NAACL"12400 ],12401 "url": "https://www.aclweb.org/portal/naacl"12402 },12403 "title": "FEVER: a Large-scale Dataset for Fact Extraction and VERification",12404 "venue": "North American Chapter of the Association for Computational Linguistics",12405 "year": 2018,12406 "citationCount": 2590,12407 "openAccessPdf": {12408 "url": "https://www.aclweb.org/anthology/N18-1074.pdf",12409 "status": "HYBRID",12410 "license": "CCBY",12411 "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/1803.05355, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."12412 },12413 "publicationTypes": [12414 "JournalArticle",12415 "Conference"12416 ],12417 "journal": {12418 "pages": "809-819"12419 },12420 "authors": [12421 {12422 "authorId": "144603330",12423 "name": "James Thorne"12424 },12425 {12426 "authorId": "2064056928",12427 "name": "Andreas Vlachos"12428 },12429 {12430 "authorId": "2718039",12431 "name": "Christos Christodoulopoulos"12432 },12433 {12434 "authorId": "2008596",12435 "name": "Arpit Mittal"12436 }12437 ],12438 "abstract": "In this paper we introduce a new publicly available dataset for verification against textual sources, FEVER: Fact Extraction and VERification. It consists of 185,445 claims generated by altering sentences extracted from Wikipedia and subsequently verified without knowledge of the sentence they were derived from. The claims are classified as Supported, Refuted or NotEnoughInfo by annotators achieving 0.6841 in Fleiss kappa. For the first two classes, the annotators also recorded the sentence(s) forming the necessary evidence for their judgment. To characterize the challenge of the dataset presented, we develop a pipeline approach and compare it to suitably designed oracles. The best accuracy we achieve on labeling a claim accompanied by the correct evidence is 31.87%, while if we ignore the evidence we achieve 50.91%. Thus we believe that FEVER is a challenging testbed that will help stimulate progress on claim verification against textual sources."12439 },12440 "DOI:10.2139/ssrn.4573321": {12441 "paperId": "409dc9f1801947f85c53231cba346f0e314cb208",12442 "externalIds": {12443 "DOI": "10.2139/ssrn.4573321",12444 "CorpusId": 26219947812445 },12446 "publicationVenue": {12447 "id": "75d7a8c1-d871-42db-a8e4-7cf5146fdb62",12448 "name": "Social Science Research Network",12449 "type": "journal",12450 "alternate_names": [12451 "SSRN, Social Science Research Network (SSRN) home page",12452 "SSRN Electronic Journal",12453 "Soc Sci Res Netw",12454 "SSRN",12455 "SSRN Home Page",12456 "SSRN Electron J",12457 "Social Science Electronic Publishing presents Social Science Research Network"12458 ],12459 "issn": "1556-5068",12460 "url": "http://www.ssrn.com/",12461 "alternate_urls": [12462 "www.ssrn.com/",12463 "https://fatcat.wiki/container/tol7woxlqjeg5bmzadeg6qrg3e",12464 "https://www.wikidata.org/wiki/Q53949192",12465 "www.ssrn.com/en",12466 "http://www.ssrn.com/en/",12467 "http://umlib.nl/ssrn",12468 "umlib.nl/ssrn"12469 ]12470 },12471 "title": "Navigating the Jagged Technological Frontier: Field Experimental Evidence of the Effects of AI on Knowledge Worker Productivity and Quality",12472 "venue": "Social Science Research Network",12473 "year": 2023,12474 "citationCount": 804,12475 "openAccessPdf": {12476 "url": "https://doi.org/10.2139/ssrn.4573321",12477 "status": "GREEN",12478 "license": null,12479 "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.2139/ssrn.4573321?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.2139/ssrn.4573321, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."12480 },12481 "publicationTypes": [12482 "JournalArticle"12483 ],12484 "journal": {12485 "name": "SSRN Electronic Journal"12486 },12487 "authors": [12488 {12489 "authorId": "2244496752",12490 "name": "Fabrizio Dell'Acqua"12491 },12492 {12493 "authorId": "2244497063",12494 "name": "Edward McFowland"12495 },12496 {12497 "authorId": "1722774",12498 "name": "Ethan Mollick"12499 },12500 {12501 "authorId": "1397350576",12502 "name": "Hila Lifshitz-Assaf"12503 },12504 {12505 "authorId": "2244502347",12506 "name": "Katherine C. Kellogg"12507 },12508 {12509 "authorId": "2244358073",12510 "name": "Saran Rajendran"12511 },12512 {12513 "authorId": "2244506815",12514 "name": "Lisa Krayer"12515 },12516 {12517 "authorId": "2244506911",12518 "name": "F. Candelon"12519 },12520 {12521 "authorId": "2505610",12522 "name": "Karim R. Lakhani"12523 }12524 ],12525 "abstract": null12526 },12527 "DOI:10.2196/53164": {12528 "paperId": "9a0e8c101b72e9094ea554194698daa3aeb3bc6f",12529 "externalIds": {12530 "PubMedCentral": "11153973",12531 "DOI": "10.2196/53164",12532 "CorpusId": 269949485,12533 "PubMed": "38776130"12534 },12535 "publicationVenue": {12536 "id": "2baad992-2268-4c38-9120-e453622f2eeb",12537 "name": "Journal of Medical Internet Research",12538 "type": "journal",12539 "alternate_names": [12540 "J Med Internet Res"12541 ],12542 "issn": "1438-8871",12543 "url": "http://www.symposion.com/jmir/index.htm",12544 "alternate_urls": [12545 "http://www.jmir.org/",12546 "https://www.jmir.org/"12547 ]12548 },12549 "title": "Hallucination Rates and Reference Accuracy of ChatGPT and Bard for Systematic Reviews: Comparative Analysis",12550 "venue": "Journal of Medical Internet Research",12551 "year": 2024,12552 "citationCount": 475,12553 "openAccessPdf": {12554 "url": "https://doi.org/10.2196/53164",12555 "status": "GOLD",12556 "license": "CCBY",12557 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC11153973, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."12558 },12559 "publicationTypes": [12560 "Study",12561 "JournalArticle",12562 "Review"12563 ],12564 "journal": {12565 "name": "Journal of Medical Internet Research",12566 "volume": "26"12567 },12568 "authors": [12569 {12570 "authorId": "2287298900",12571 "name": "Micha\u00ebl Chelli"12572 },12573 {12574 "authorId": "123164546",12575 "name": "J. Descamps"12576 },12577 {12578 "authorId": "2278002609",12579 "name": "V. Lavou\u00e9"12580 },12581 {12582 "authorId": "4193798",12583 "name": "C. Trojani"12584 },12585 {12586 "authorId": "2065302847",12587 "name": "Michel Azar"12588 },12589 {12590 "authorId": "2302548215",12591 "name": "M. Deckert"12592 },12593 {12594 "authorId": "27673403",12595 "name": "J. Raynier"12596 },12597 {12598 "authorId": "17722605",12599 "name": "Gilles Clowez"12600 },12601 {12602 "authorId": "2279472796",12603 "name": "Pascal Boileau"12604 },12605 {12606 "authorId": "2058782562",12607 "name": "C. Ruetsch-Chelli"12608 }12609 ],12610 "abstract": "Background Large language models (LLMs) have raised both interest and concern in the academic community. They offer the potential for automating literature search and synthesis for systematic reviews but raise concerns regarding their reliability, as the tendency to generate unsupported (hallucinated) content persist. Objective The aim of the study is to assess the performance of LLMs such as ChatGPT and Bard (subsequently rebranded Gemini) to produce references in the context of scientific writing. Methods The performance of ChatGPT and Bard in replicating the results of human-conducted systematic reviews was assessed. Using systematic reviews pertaining to shoulder rotator cuff pathology, these LLMs were tested by providing the same inclusion criteria and comparing the results with original systematic review references, serving as gold standards. The study used 3 key performance metrics: recall, precision, and F1-score, alongside the hallucination rate. Papers were considered \u201challucinated\u201d if any 2 of the following information were wrong: title, first author, or year of publication. Results In total, 11 systematic reviews across 4 fields yielded 33 prompts to LLMs (3 LLMs\u00d711 reviews), with 471 references analyzed. Precision rates for GPT-3.5, GPT-4, and Bard were 9.4% (13/139), 13.4% (16/119), and 0% (0/104) respectively (P<.001). Recall rates were 11.9% (13/109) for GPT-3.5 and 13.7% (15/109) for GPT-4, with Bard failing to retrieve any relevant papers (P<.001). Hallucination rates stood at 39.6% (55/139) for GPT-3.5, 28.6% (34/119) for GPT-4, and 91.4% (95/104) for Bard (P<.001). Further analysis of nonhallucinated papers retrieved by GPT models revealed significant differences in identifying various criteria, such as randomized studies, participant criteria, and intervention criteria. The study also noted the geographical and open-access biases in the papers retrieved by the LLMs. Conclusions Given their current performance, it is not recommended for LLMs to be deployed as the primary or exclusive tool for conducting systematic reviews. Any references generated by such models warrant thorough validation by researchers. The high occurrence of hallucinations in LLMs highlights the necessity for refining their training and functionality before confidently using them for rigorous academic purposes."12611 },12612 "DOI:10.3389/fpsyg.2016.01832": {12613 "paperId": "5e8902d504c03c7513f29ade98931189165a8a62",12614 "externalIds": {12615 "MAG": "2547503455",12616 "PubMedCentral": "5122713",12617 "DOI": "10.3389/fpsyg.2016.01832",12618 "CorpusId": 12942653,12619 "PubMed": "27933012"12620 },12621 "publicationVenue": {12622 "id": "89097a03-8be6-4e2d-ae2c-a6df64c77a06",12623 "name": "Frontiers in Psychology",12624 "type": "journal",12625 "alternate_names": [12626 "Front Psychol"12627 ],12628 "issn": "1664-1078",12629 "url": "http://www.frontiersin.org/Cultural_Psychology",12630 "alternate_urls": [12631 "http://frontiersin.org/psychology/",12632 "https://www.frontiersin.org/journals/psychology",12633 "http://journal.frontiersin.org/journal/psychology",12634 "http://www.frontiersin.org/psychology"12635 ]12636 },12637 "title": "Degrees of Freedom in Planning, Running, Analyzing, and Reporting Psychological Studies: A Checklist to Avoid p-Hacking",12638 "venue": "Frontiers in Psychology",12639 "year": 2016,12640 "citationCount": 632,12641 "openAccessPdf": {12642 "url": "https://www.frontiersin.org/articles/10.3389/fpsyg.2016.01832/pdf",12643 "status": "GOLD",12644 "license": "CCBY",12645 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC5122713, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."12646 },12647 "publicationTypes": [12648 "JournalArticle",12649 "Review"12650 ],12651 "journal": {12652 "name": "Frontiers in Psychology",12653 "volume": "7"12654 },12655 "authors": [12656 {12657 "authorId": "2748685",12658 "name": "J. Wicherts"12659 },12660 {12661 "authorId": "3642852",12662 "name": "C. L. S. Veldkamp"12663 },12664 {12665 "authorId": "8505119",12666 "name": "H. Augusteijn"12667 },12668 {12669 "authorId": "40021672",12670 "name": "Marjan Bakker"12671 },12672 {12673 "authorId": "5646115",12674 "name": "R. V. van Aert"12675 },12676 {12677 "authorId": "1707089",12678 "name": "M. van Assen"12679 }12680 ],12681 "abstract": "The designing, collecting, analyzing, and reporting of psychological studies entail many choices that are often arbitrary. The opportunistic use of these so-called researcher degrees of freedom aimed at obtaining statistically significant results is problematic because it enhances the chances of false positive results and may inflate effect size estimates. In this review article, we present an extensive list of 34 degrees of freedom that researchers have in formulating hypotheses, and in designing, running, analyzing, and reporting of psychological research. The list can be used in research methods education, and as a checklist to assess the quality of preregistrations and to determine the potential for bias due to (arbitrary) choices in unregistered studies."12682 },12683 "DOI:10.7759/cureus.35179": {12684 "paperId": "7be169a70f5db74f40adfd2790021aa3fbf3cb87",12685 "externalIds": {12686 "PubMedCentral": "9939079",12687 "DOI": "10.7759/cureus.35179",12688 "CorpusId": 257037938,12689 "PubMed": "36811129"12690 },12691 "publicationVenue": {12692 "id": "87c855fc-c59a-4443-832a-e5b1ca14b9f3",12693 "name": "Cureus",12694 "type": "journal",12695 "issn": "2168-8184",12696 "url": "https://www.cureus.com/"12697 },12698 "title": "Artificial Hallucinations in ChatGPT: Implications in Scientific Writing",12699 "venue": "Cureus",12700 "year": 2023,12701 "citationCount": 1603,12702 "openAccessPdf": {12703 "url": "https://assets.cureus.com/uploads/editorial/pdf/138667/20230219-28928-6kcyip.pdf",12704 "status": "GOLD",12705 "license": null,12706 "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC9939079, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."12707 },12708 "publicationTypes": [12709 "Editorial"12710 ],12711 "journal": {12712 "name": "Cureus",12713 "volume": "15"12714 },12715 "authors": [12716 {12717 "authorId": "1412759796",12718 "name": "H. Alkaissi"12719 },12720 {12721 "authorId": "81301990",12722 "name": "Samy I McFarlane"12723 }12724 ],12725 "abstract": "While still in its infancy, ChatGPT (Generative Pretrained Transformer), introduced in November 2022, is bound to hugely impact many industries, including healthcare, medical education, biomedical research, and scientific writing. Implications of ChatGPT, that new chatbot introduced by OpenAI on academic writing, is largely unknown. In response to the Journal of Medical Science (Cureus) Turing Test - call for case reports written with the assistance of ChatGPT, we present two cases one of homocystinuria-associated osteoporosis, and the other is on late-onset Pompe disease (LOPD), a rare metabolic disorder. We tested ChatGPT to write about the pathogenesis of these conditions. We documented the positive, negative, and rather troubling aspects of our newly introduced chatbot\u2019s performance."12726 }12727}