1#!/usr/bin/env python32"""Build literature_matrix.csv and references_verified.bib for WP29 from cached3primary-metadata records (Crossref, arXiv API, Semantic Scholar, OpenReview).45Every entry below was checked against the cached API response in verify_cache/6(fetched 2026-10-29/30 UTC). Nothing is emitted unless the cache record exists.7"""8import csv, json, os, re, sys, html, urllib.parse, datetime9import xml.etree.ElementTree as ET1011HERE = os.path.dirname(os.path.abspath(__file__))12CACHE = os.path.join(HERE, "verify_cache")13S2 = json.load(open(os.path.join(CACHE, "semanticscholar_batch.json")))14DOIRES = json.load(open(os.path.join(CACHE, "doi_resolution.json")))15NS = {"a": "http://www.w3.org/2005/Atom", "ar": "http://arxiv.org/schemas/atom"}161718def cache_path(url):19 return os.path.join(CACHE, re.sub(r"[^A-Za-z0-9]+", "_", url)[:150])202122def mtime_utc(path):23 return datetime.datetime.fromtimestamp(os.path.getmtime(path), datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")242526def crossref(doi):27 url = "https://api.crossref.org/works/" + urllib.parse.quote(doi, safe="")28 p = cache_path(url)29 m = json.load(open(p))["message"]30 yr = (m.get("issued") or m.get("published-print") or m.get("published-online"))["date-parts"][0][0]31 authors = [(a.get("family", ""), a.get("given", "")) for a in m.get("author", [])]32 return dict(title=" ".join(m["title"][0].split()), authors=authors, year=yr,33 container=(m.get("container-title") or [""])[0], crtype=m["type"],34 volume=m.get("volume"), issue=m.get("issue"), pages=m.get("page"),35 publisher=m.get("publisher"), url="https://api.crossref.org/works/" + doi,36 verified=mtime_utc(p))373839def arxiv(aid):40 url = "https://export.arxiv.org/api/query?id_list=" + aid41 p = cache_path(url)42 e = ET.parse(p).getroot().find("a:entry", NS)43 names = [a.find("a:name", NS).text for a in e.findall("a:author", NS)]44 authors = [(n.split()[-1], " ".join(n.split()[:-1])) for n in names]45 jr = e.find("ar:journal_ref", NS)46 return dict(title=" ".join(e.find("a:title", NS).text.split()), authors=authors,47 year=int(e.find("a:published", NS).text[:4]), container="arXiv preprint",48 jref=jr.text if jr is not None else None, url=url, verified=mtime_utc(p))495051def s2_venue(key):52 v = S2.get(key) or {}53 pv = v.get("publicationVenue") or {}54 return v.get("venue"), pv.get("type"), v.get("year")555657# ----------------------------------------------------------------------------58# Entry list. Fields: key, stream, doi, arxiv, vtype, peer (yes/no/unknown),59# peer_src (extra verification URL for the venue), rel (relevance), claim.60# ----------------------------------------------------------------------------61E = []62def add(**k):63 E.append(k)6465# ---- Stream 1: autonomous scientific discovery / agentic research systems ----66add(key="lu2024aiscientist", stream=1, arxiv="2408.06292", vtype="preprint", peer="no",67 rel="Reference end-to-end autonomous research system (idea to paper plus simulated review) against which WP29's protocol-guided design is positioned.",68 claim="Introduces The AI Scientist, which generates research ideas, writes code, executes experiments, visualizes results, writes a full paper and runs a simulated review, with each paper developed at a cost of less than $15 and an automated reviewer reported to achieve near-human performance on paper scores.")69add(key="lu2026aiscientist", stream=1, doi="10.1038/s41586-026-10265-5", vtype="journal", peer="yes",70 rel="Peer-reviewed (Nature, 2026) publication of The AI Scientist line of work, establishing that fully automated ML research pipelines are now part of the refereed literature.",71 claim="n/a (Nature abstract not retrieved behind the publisher login; see lu2024aiscientist for the system description).")72add(key="yamada2025aiscientistv2", stream=1, arxiv="2504.08066", vtype="preprint", peer="no",73 rel="Closest fully autonomous end-to-end system; its tree-search workflow and in-system AI reviewer are the counterpoint to WP29's executable gates and separated review.",74 claim="Describes an end-to-end agentic system with progressive agentic tree search managed by an experiment-manager agent and a VLM feedback loop for figures; of three fully autonomous manuscripts submitted to an ICLR workshop, one scored above the average human acceptance threshold.")75add(key="schmidgall2025agentlab", stream=1, doi="10.18653/v1/2025.findings-emnlp.320", arxiv="2501.04227", vtype="conference", peer="yes",76 rel="Staged (literature review, experimentation, report writing) autonomous research framework with optional human feedback; provides the human-in-the-loop and cost comparisons WP29 builds on.",77 claim="Agent Laboratory takes a human-provided idea through literature review, experimentation and report writing; the authors report that o1-preview gives the best outcomes, that human feedback at each stage significantly improves research quality, and an 84% decrease in research expenses relative to previous autonomous research methods.")78add(key="boiko2023coscientist", stream=1, doi="10.1038/s41586-023-06792-0", vtype="journal", peer="yes",79 rel="Landmark peer-reviewed demonstration of an LLM agent planning and executing laboratory experiments; defines the 'autonomous research' baseline WP29 compares to.",80 claim="Coscientist, driven by GPT-4 with internet/documentation search, code execution and experimental automation tools, autonomously designs, plans and performs experiments across six tasks, including successful reaction optimization of palladium-catalysed cross-couplings.")81add(key="bran2024chemcrow", stream=1, doi="10.1038/s42256-024-00832-8", vtype="journal", peer="yes",82 rel="Tool-augmented LLM research agent whose evaluation combined LLM and expert assessment, an early instance of the evaluator-validity question WP29 addresses with protected oracles.",83 claim="ChemCrow integrates 18 expert-designed tools with GPT-4, autonomously planned and executed syntheses of an insect repellent and three organocatalysts, and was evaluated with both LLM and expert assessments.")84add(key="ifargan2025datatopaper", stream=1, doi="10.1056/AIoa2400555", arxiv="2404.17605", vtype="journal", peer="yes",85 rel="Closest prior work on traceability: an autonomous data-to-paper platform that programmatically backtraces information flow for human verification, akin to WP29's persistent evidence records.",86 claim="data-to-paper guides interacting LLM agents through a stepwise research process from annotated data to papers while programmatically backtracing information flow and allowing human oversight, and can run fully autonomously ('autopilot') or with human intervention.")87add(key="gottweis2026coscientist", stream=1, doi="10.1038/s41586-026-10644-y", arxiv="2502.18864", vtype="journal", peer="yes",88 rel="Multi-agent hypothesis-generation system with generate/critique/refine loops; illustrates in-system self-critique rather than independent verification.",89 claim="Co-Scientist is a Gemini-based multi-agent system that continuously generates, critiques and refines research hypotheses using an asynchronous task framework and a tournament evolution process, scaled by test-time compute, to propose demonstrably novel hypotheses for experimental verification.")90add(key="swanson2025virtuallab", stream=1, doi="10.1038/s41586-025-09442-9", vtype="journal", peer="yes",91 rel="AI-human research collaboration with an LLM principal-investigator agent and human oversight; a human-involvement design point for the novelty matrix.",92 claim="The Virtual Lab is an AI-human research collaboration in which an LLM Principal Investigator agent guides a team of LLM scientist agents to perform interdisciplinary research, applied to designing SARS-CoV-2 nanobodies.")93add(key="mitchener2025kosmos", stream=1, arxiv="2511.02824", vtype="preprint", peer="no",94 rel="Recent long-horizon AI scientist (up to 12 h, 200 rollouts); documents the coherence/verification challenges of long autonomous runs that WP29's gates target.",95 claim="Kosmos runs for up to 12 hours performing cycles of parallel data analysis, literature search and hypothesis generation, using a structured world model shared between agents to pursue an objective coherently over 200 agent rollouts before synthesizing reports.")96add(key="luo2025pitfalls", stream=1, arxiv="2509.08713", vtype="preprint", peer="no",97 rel="Directly motivates WP29: shows AI scientist systems commit benchmark selection, leakage, metric-misuse and post-hoc selection errors that only traces reveal.",98 claim="Identifies four failure modes in AI scientist systems (inappropriate benchmark selection, data leakage, metric misuse and post-hoc selection bias) and, via controlled experiments on two open-source systems, finds several of them present.")99add(key="beel2025sakana", stream=1, doi="10.1145/3769733.3769747", arxiv="2502.14297", vtype="journal", peer="unknown",100 rel="Independent third-party evaluation of The AI Scientist; evidence that unverified autonomous outputs contain novelty and execution errors.",101 claim="An independent evaluation of the AI Scientist found several critical shortcomings, including a literature review based on simplistic keyword searches that led to poor novelty assessments, with several generated ideas incorrectly classified as novel.")102add(key="luo2026xscientist", stream=1, arxiv="2607.12301", vtype="preprint", peer="no",103 rel="Closest 2026 design to a research protocol: git-like typed research state with checkpoints, claim-evidence closure and gates; differs from WP29 in being a protocol artifact rather than a controlled comparison of protocol arms.",104 claim="Proposes a local-first, git-like protocol that treats research state rather than a manuscript as the unit of continuation, representing hypotheses, attempts, claims, reviews and handoffs as typed content-addressed objects with immutable checkpoints, claim-evidence closure and authority-aware gates, without treating a passing integrity check as scientific truth.")105add(key="ding2026verificationgap", stream=1, arxiv="2608.05179", vtype="preprint", peer="no",106 rel="Survey that names the 'verification gap' in AI scientist systems; supplies the audit dimensions (evaluation method, artifacts, human-in-the-loop points, result-selection disclosure) WP29 reports.",107 claim="Screens 125 works and codes 26 (24 runnable systems) on seven audit dimensions, finding that code release is now common but reproducibility-grade and claim-verification artifacts remain rare.")108109# ---- Stream 2: language agents for data analysis and scientific programming ----110add(key="chen2025scienceagentbench", stream=2, arxiv="2410.05080", vtype="conference", peer="yes",111 peer_src="https://api2.openreview.net/notes/search?term=ScienceAgentBench (ICLR 2025 Poster record)",112 rel="Benchmark of expert-validated data-driven discovery tasks with program-level outputs, execution metrics and cost; the task style and contamination mitigations inform WP29's task bank.",113 claim="Presents 102 tasks extracted from 44 publications in four disciplines and validated by nine experts; the best agent solves 32.4% of tasks independently (34.3% with expert knowledge), o1-preview reaches 42.2% at more than 10 times the cost, and two data-contamination mitigation strategies are proposed.")114add(key="majumder2025discoverybench", stream=2, arxiv="2407.01725", vtype="conference", peer="yes",115 peer_src="https://api2.openreview.net/notes/search?term=DiscoveryBench (ICLR 2025 Poster record)",116 rel="Formalizes multi-step data-driven discovery as dataset + metadata + goal, the same unit of analysis as WP29's research tasks.",117 claim="DiscoveryBench formalizes the multi-step process of data-driven discovery with 264 tasks across six domains derived from published papers plus 903 synthetic tasks, each defined by a dataset, its metadata and a natural-language discovery goal.")118add(key="jing2025dsbench", stream=2, arxiv="2409.07703", vtype="conference", peer="yes",119 peer_src="https://api2.openreview.net/notes/search?term=DSBench (ICLR 2025 Poster record)",120 rel="Realistic data-science agent benchmark (long contexts, large data) used to characterize executor capability independent of protocol.",121 claim="DSBench comprises 466 data-analysis and 74 data-modeling tasks sourced from Eloquence and Kaggle competitions, with long contexts, multimodal task backgrounds and reasoning over large data files.")122add(key="huang2024mlagentbench", stream=2, arxiv="2310.03302", vtype="conference", peer="yes",123 rel="Evaluates agents on ML experimentation loops with file/code actions; a precursor of WP29's executor arms.",124 claim="MLAgentBench is a suite of 13 ML experimentation tasks (from CIFAR-10 improvement to BabyLM) in which a ReAct-based agent can read/write files, execute code and inspect outputs; a Claude 3 Opus agent had the best success rate.")125add(key="gu2024blade", stream=2, arxiv="2408.09667", vtype="conference", peer="yes",126 rel="Evaluates analysis decisions (variables, transformations, models) against independent expert analyses, the multi-path ground truth WP29's oracles must handle.",127 claim="BLADE evaluates agents' multifaceted approaches to open-ended research questions using 12 datasets and questions from the literature, with ground truth collected from independent analyses by expert data scientists.")128add(key="jimenez2024swebench", stream=2, arxiv="2310.06770", vtype="conference", peer="yes",129 rel="Canonical execution-verified coding benchmark; its test-based grading is the model for WP29's executable verification gates.",130 claim="SWE-bench consists of 2,294 software-engineering problems drawn from real GitHub issues and pull requests across 12 Python repositories, where resolving an issue frequently requires coordinating changes across multiple functions, classes and files.")131add(key="chan2025mlebench", stream=2, arxiv="2410.07095", vtype="conference", peer="yes",132 peer_src="https://proceedings.iclr.cc/paper_files/paper/2025/hash/7e3767db483c942b883eb4f8cfb74e31-Abstract-Conference.html",133 rel="Agentic ML-engineering benchmark with human baselines, resource-scaling and contamination analyses that WP29's resource accounting parallels.",134 claim="MLE-bench curates 75 Kaggle competitions with human leaderboard baselines; the best setup (o1-preview with AIDE scaffolding) reaches at least bronze-medal level in 16.9% of competitions, and the paper also studies resource scaling and pre-training contamination.")135add(key="siegel2024corebench", stream=2, arxiv="2409.11363", vtype="journal", peer="yes",136 rel="Benchmarks agents on computational reproducibility of published papers, linking agent evaluation to the reproducibility stream.",137 claim="CORE-Bench contains 270 tasks based on 90 scientific papers across computer science, social science and medicine, measuring agents' accuracy at reproducing published results from provided code and data at three difficulty levels.")138add(key="starace2025paperbench", stream=2, arxiv="2504.01848", vtype="conference", peer="yes",139 rel="Rubric-decomposed replication grading with a validated LLM judge; closest precedent for WP29's rubric oracles and judge validation.",140 claim="PaperBench asks agents to replicate 20 ICML 2024 papers from scratch, graded against 8,316 rubric items co-developed with the papers' authors, with an LLM-based judge whose performance is assessed on a separate judge benchmark.")141142# ---- Stream 3: workflows, provenance, computational reproducibility ----143add(key="koster2012snakemake", stream=3, doi="10.1093/bioinformatics/bts480", vtype="journal", peer="yes",144 rel="Rule-based workflow engine; WP29's protocol stages are an analogue of declarative, re-executable workflow rules.",145 claim="Snakemake provides a readable Python-based workflow definition language and an execution environment that scales from single-core workstations to compute clusters without modifying the workflow.")146add(key="ditommaso2017nextflow", stream=3, doi="10.1038/nbt.3820", vtype="journal", peer="yes",147 rel="Container-based dataflow workflow manager for reproducible pipelines; reference for execution isolation in WP29's executors.",148 claim="Argues that combining dataflow programming with software containers (Nextflow) yields computational workflows that are reproducible across platforms.")149add(key="missier2013prov", stream=3, doi="10.1145/2452376.2452478", vtype="conference", peer="yes",150 rel="W3C PROV family defines the provenance vocabulary (entities, activities, agents) that WP29's evidence records instantiate.",151 claim="Overview of the W3C PROV family of specifications (data model, ontology, notation and constraints) for interoperable representation and exchange of provenance metadata.")152add(key="chirigati2016reprozip", stream=3, doi="10.1145/2882903.2899401", vtype="conference", peer="yes",153 rel="Automatic capture of execution provenance into re-executable packages; a model for evidence capture without relying on the agent's self-report.",154 claim="ReproZip automatically captures the provenance of a command-line experiment (files, libraries, environment) and packages it so that it can be re-executed elsewhere.")155add(key="sandve2013tensimplerules", stream=3, doi="10.1371/journal.pcbi.1003285", vtype="journal", peer="yes",156 rel="Widely cited rules (track provenance, avoid manual steps, record versions and seeds) that WP29 operationalizes as machine-checked gates.",157 claim="Proposes ten rules for reproducible computational research, including tracking how every result was produced, avoiding manual data manipulation, archiving exact program versions and random seeds, and providing public access to scripts, runs and results.")158add(key="stodden2018journalpolicy", stream=3, doi="10.1073/pnas.1708290115", vtype="journal", peer="yes",159 rel="Empirical reproducibility failure rates under policy-only enforcement; motivates executable rather than declarative verification.",160 claim="In a random sample of 204 Science papers subject to a data/code-on-request policy, the authors obtained artifacts from 44% of papers and could reproduce the findings for 26%.")161add(key="peng2011reproducible", stream=3, doi="10.1126/science.1213847", vtype="journal", peer="yes",162 rel="Defines reproducibility (data + code available) as the minimum standard for computational claims, the standard WP29's evidence records target.",163 claim="Reproducibility has the potential to serve as a minimum standard for judging scientific claims when full independent replication of a study is not possible.")164add(key="nasem2019reproducibility", stream=3, doi="10.17226/25303", vtype="report", peer="unknown",165 rel="Authoritative definitions of reproducibility vs replicability used in WP29's outcome definitions.",166 claim="Defines reproducibility as obtaining consistent computational results using the same input data, computational steps, methods, code and conditions of analysis, and replicability as obtaining consistent results across studies aimed at the same question with new data.")167add(key="krafczyk2021reproduction", stream=3, doi="10.1098/rsta.2020.0069", vtype="journal", peer="yes",168 rel="Principles (transparency, re-executability, determinism) and a Reproduction Package formalism close to WP29's release bundle.",169 claim="From reproducing seven articles the authors derive three principles (transparency about how results are produced, ease of re-executability, determinism), 12 guidelines, and a Reproduction Package formalism for sharing artifacts.")170add(key="trisovic2022codequality", stream=3, doi="10.1038/s41597-022-01143-6", vtype="journal", peer="yes",171 rel="Large-scale evidence that deposited research code usually fails to run, justifying execution-based verification gates.",172 claim="Executing over 9,000 R files from more than 2,000 Harvard Dataverse replication datasets, 74% failed to complete without error initially and 56% still failed after automatic code cleaning.")173174# ---- Stream 4: preregistration, registered reports, specification discipline ----175add(key="nosek2018preregistration", stream=4, doi="10.1073/pnas.1708274114", vtype="journal", peer="yes",176 rel="Preregistration as prediction/postdiction separation is the direct analogue of WP29's protocol fixed before execution.",177 claim="Defining research questions and the analysis plan before observing outcomes (preregistration) distinguishes analyses that result from predictions from those that result from postdictions, and practical strategies exist for cases that fall short of the ideal.")178add(key="chambers2013registeredreports", stream=4, doi="10.1016/j.cortex.2012.12.016", vtype="journal", peer="unknown",179 rel="Origin of Registered Reports: review of the protocol before results, the publishing analogue of WP29's pre-execution gate.",180 claim="Introduces Registered Reports at Cortex, in which study protocols are peer reviewed and provisionally accepted before data collection.")181add(key="chambers2022registeredreports", stream=4, doi="10.1038/s41562-021-01193-7", vtype="journal", peer="yes",182 rel="Reviews evidence that pre-acceptance of protocols reduces reporting bias; supports the 'specification before execution' arm.",183 claim="Registered Reports are a publication format in which study proposals are peer reviewed and pre-accepted before research is undertaken; early evidence indicates they work as intended although they are not a universal solution for irreproducibility.")184add(key="simmons2011falsepositive", stream=4, doi="10.1177/0956797611417632", vtype="journal", peer="yes",185 rel="Researcher degrees of freedom inflate false positives; WP29's locked analysis specification removes these from the agent.",186 claim="Flexibility in data collection, analysis and reporting dramatically increases actual false-positive rates, so that a researcher is often more likely to falsely find evidence that an effect exists than to correctly find evidence that it does not; six disclosure requirements for authors and four reviewer guidelines are proposed.")187add(key="munafo2017manifesto", stream=4, doi="10.1038/s41562-016-0021", vtype="journal", peer="yes",188 rel="Umbrella statement of reproducibility measures (methods, reporting, evaluation, incentives) that WP29 adapts to autonomous agents.",189 claim="Argues for measures optimizing methods, reporting and dissemination, reproducibility, evaluation and incentives, noting evidence from simulations and empirical studies but the need for iterative evaluation of their adoption.")190add(key="gelman2014statisticalcrisis", stream=4, doi="10.1511/2014.111.460", vtype="journal", peer="unknown",191 rel="Garden of forking paths: data-dependent analysis choices invalidate inference even without intent, which an agent in an open instruction arm is free to make.",192 claim="Argues that data-dependent choices of analysis (the 'garden of forking paths') can produce spurious statistically significant findings even when researchers do not deliberately fish for significance.")193add(key="nosek2015top", stream=4, doi="10.1126/science.aab2374", vtype="journal", peer="yes",194 rel="TOP guidelines define graded transparency standards (preregistration, data, code) that WP29's protocol tiers mirror.",195 claim="Proposes journal author guidelines (the TOP guidelines) to promote transparency, openness and reproducibility, noting that although scientists embrace these norms they are not routine in practice.")196add(key="scheel2021excess", stream=4, doi="10.1177/25152459211007467", vtype="journal", peer="yes",197 rel="Quantifies how pre-specified protocols change the result distribution; benchmark for interpreting WP29's arm differences.",198 claim="Comparing 71 published Registered Reports with 152 standard hypothesis-testing papers, the first hypothesis was positive in 96% of standard reports but only 44% of Registered Reports.")199add(key="osc2015reproducibility", stream=4, doi="10.1126/science.aac4716", vtype="journal", peer="yes",200 rel="Baseline replication rates in a human literature; provides a human reference for WP29's validity outcomes.",201 claim="Replications of 100 experiments from 2008 papers in three psychology journals found that about one-third to one-half of original findings were also observed in the replication, depending on the criterion.")202add(key="kerr1998harking", stream=4, doi="10.1207/s15327957pspr0203_4", vtype="journal", peer="yes",203 rel="HARKing (presenting post hoc hypotheses as a priori) is a failure mode an unconstrained agent can reproduce when writing up.",204 claim="Defines HARKing (hypothesizing after the results are known) as presenting post hoc hypotheses as if they were a priori, and discusses its prevalence and costs.")205206# ---- Stream 5: agent evaluation, contamination, evaluator validity ----207add(key="liu2024agentbench", stream=5, arxiv="2308.03688", vtype="conference", peer="yes",208 rel="Multi-environment agent benchmark identifying long-horizon reasoning and instruction following as main failure causes, relevant to protocol adherence measurement.",209 claim="AgentBench evaluates LLMs as agents across 8 environments; top commercial LLMs act well in complex environments but a significant gap separates them from open-source models up to 70B, with poor long-term reasoning, decision-making and instruction following as main obstacles.")210add(key="yao2024taubench", stream=5, arxiv="2406.12045", vtype="preprint", peer="unknown",211 rel="State-based outcome checking and the pass^k reliability metric are reused in WP29's repeated-run reliability analysis.",212 claim="tau-bench evaluates agents in simulated user conversations with API tools and policy guidelines by comparing the final database state to an annotated goal state, and proposes pass^k for reliability over multiple trials; gpt-4o succeeds on fewer than 50% of tasks with pass^8 below 25% in retail.")213add(key="kapoor2024agentsthatmatter", stream=5, arxiv="2407.01502", vtype="journal", peer="yes",214 rel="Core methodological critique (cost reporting, holdouts, conflated benchmarking needs) that WP29's evaluation design follows.",215 claim="Analysis of agent benchmarks finds a narrow focus on accuracy without cost, leading to needlessly complex and costly agents; proposes jointly optimizing accuracy and cost, and notes that model- and downstream-developer needs are conflated and that many benchmarks have inadequate holdout sets.")216add(key="sainz2023contamination", stream=5, doi="10.18653/v1/2023.findings-emnlp.722", arxiv="2310.18018", vtype="conference", peer="yes",217 rel="Defines contamination levels; WP29's task bank must document contamination risk for each task.",218 claim="Position paper arguing that training on a benchmark's test split causes overestimation of performance with potentially harmful scientific conclusions, defining levels of data contamination and calling for community measurement for each benchmark.")219add(key="zheng2023mtbench", stream=5, arxiv="2306.05685", vtype="conference", peer="yes",220 peer_src="https://api2.openreview.net/notes/search?term=Judging%20LLM-as-a-judge (NeurIPS 2023 Datasets and Benchmarks record)",221 rel="Establishes LLM-as-a-judge and its biases (position, verbosity, self-enhancement), which WP29's protected oracles are designed to avoid.",222 claim="Examines LLM-as-a-judge limitations including position, verbosity and self-enhancement biases, and finds that strong judges such as GPT-4 agree with human preferences over 80% of the time on MT-bench and Chatbot Arena, the same level as human-human agreement.")223add(key="thakur2025judgingjudges", stream=5, arxiv="2406.12624", vtype="conference", peer="yes",224 rel="Shows even the best judge models lag inter-human agreement, cautioning against unprotected LLM grading of research outputs.",225 claim="Across thirteen judge models and nine exam-taker models in a high-human-agreement setting, only the best and largest judges reach reasonable alignment with humans, remain well behind inter-human agreement, and may differ from human scores by up to 5 points.")226add(key="panickssery2024selfpreference", stream=5, arxiv="2404.13076", vtype="conference", peer="yes",227 rel="Self-preference bias is the specific threat WP29 mitigates by separating executor and reviewer and protecting oracles.",228 claim="LLM evaluators such as GPT-4 and Llama 2 have non-trivial accuracy at recognizing their own outputs, and fine-tuning reveals a linear correlation between self-recognition capability and the strength of self-preference.")229add(key="zhu2025abc", stream=5, arxiv="2507.02825", vtype="preprint", peer="no",230 rel="Agentic Benchmark Checklist (task validity, outcome validity, reporting) used as a checklist for WP29's task and oracle construction.",231 claim="Shows that many agentic benchmarks have task-setup or reward-design flaws (e.g., SWE-bench Verified uses insufficient test cases, TAU-bench counts empty responses as successful) that can mis-estimate performance by up to 100% relative, and introduces the Agentic Benchmark Checklist, which reduced overestimation on CVE-Bench by 33%.")232add(key="kapoor2025hal", stream=5, arxiv="2510.11977", vtype="preprint", peer="no",233 rel="Standardized harness with full cost accounting across models, scaffolds and benchmarks, the infrastructure model for WP29's isolated executors and resource logs.",234 claim="Introduces a standardized evaluation harness and reports 21,730 agent rollouts across 9 models and 9 benchmarks at a total cost of about $40,000, finding among other things that higher reasoning effort can reduce accuracy.")235add(key="li2025llmreval", stream=5, arxiv="2510.12367", vtype="preprint", peer="no",236 rel="Simulated research-and-review loop showing LLM reviewers misaligned with humans; motivates WP29's separate, protected review arm rather than in-system AI review.",237 claim="Using a simulation with a research agent that writes and revises papers and a review agent that assesses them, followed by human annotation, the authors identify pronounced misalignment between LLM reviewers and human judgments.")238239# ---- Stream 6: human-AI delegation, automation bias, intervention measurement ----240add(key="parasuraman1997automation", stream=6, doi="10.1518/001872097778543886", vtype="journal", peer="yes",241 rel="Use/misuse/disuse/abuse taxonomy frames how a researcher delegates to an autonomous agent and over-relies on its outputs.",242 claim="Distinguishes use, misuse (over-reliance producing monitoring failures or decision biases), disuse (often caused by false alarms) and abuse of automation, with trust, workload and reliability as key influences.")243add(key="parasuraman2000levels", stream=6, doi="10.1109/3468.844354", vtype="journal", peer="yes",244 rel="Levels-of-automation model used to locate WP29's four arms on the acquisition/analysis/decision/implementation dimensions.",245 claim="Proposes that automation can be applied to four function classes (information acquisition, information analysis, decision and action selection, action implementation), each across a continuum from fully manual to fully automatic, and that automation changes rather than merely supplants human activity.")246add(key="goddard2012automationbias", stream=6, doi="10.1136/amiajnl-2011-000089", vtype="journal", peer="yes",247 rel="Systematic review of automation bias frequency and mitigators informing how human review of agent outputs is measured in WP29.",248 claim="A systematic review (74 of 13,821 retrieved papers) finds automation bias mediated by user factors such as cognitive style, decision-support design and task experience, discussed alongside automation-induced complacency.")249add(key="lai2023humanai", stream=6, doi="10.1145/3593013.3594087", arxiv="2112.11471", vtype="conference", peer="yes",250 rel="Design-space survey of empirical human-AI decision-making studies used to position WP29's intervention measurement.",251 claim="Surveys empirical human-subject studies of AI-assisted decision making and summarizes the design space of tasks, AI assistance elements and evaluation metrics to structure a science of human-AI decision making.")252add(key="bansal2021whole", stream=6, doi="10.1145/3411764.3445717", vtype="conference", peer="yes",253 rel="Shows explanations raise acceptance of AI recommendations regardless of correctness, a risk for human review of agent-generated research.",254 claim="In studies on three datasets, AI augmentation produced complementary improvements but explanations did not increase them; instead explanations increased the chance that humans accept the AI's recommendation regardless of its correctness.")255add(key="lubars2019delegability", stream=6, arxiv="1902.03245", vtype="conference", peer="yes",256 rel="Task-delegability framework (motivation, difficulty, risk, trust) used to characterize which research sub-tasks are delegated in each WP29 arm.",257 claim="Proposes a delegability framework with four factors (motivation, difficulty, risk, trust) and, surveying 100 tasks, finds little preference for full AI control and a strong preference for machine-in-the-loop designs, with trust most correlated with preferences.")258add(key="vaccaro2024combinations", stream=6, doi="10.1038/s41562-024-02024-1", vtype="journal", peer="yes",259 rel="Meta-analytic evidence on when human-AI combinations beat either alone; reference point for interpreting WP29's human-review arm.",260 claim="A preregistered meta-analysis of 106 experiments (370 effect sizes, 2020-2023) finds that on average human-AI combinations performed worse than the best of humans or AI alone, with losses in decision tasks and gains in content-creation tasks.")261add(key="bainbridge1983ironies", stream=6, doi="10.1016/0005-1098(83)90046-8", vtype="journal", peer="yes",262 rel="Classic argument that automation makes the residual human role harder, relevant to reviewers of autonomous research output.",263 claim="Argues that the more advanced a control system is, the more crucial the contribution of the human operator becomes, because the operator is left with the tasks the designer could not automate and must monitor a system that rarely needs intervention.")264add(key="bucinca2021trust", stream=6, doi="10.1145/3449287", vtype="journal", peer="yes",265 rel="Cognitive forcing as a mitigation of overreliance; analogous to WP29's requirement that reviewers execute checks rather than read summaries.",266 claim="In an experiment with 199 participants, three cognitive forcing interventions reduced overreliance on AI relative to simple explainable-AI approaches.")267add(key="becker2025metr", stream=6, arxiv="2507.09089", vtype="preprint", peer="no",268 rel="Randomized measurement of AI-tool effects on expert developers showing perception-reality gaps; methodological model for WP29's time/cost outcomes.",269 claim="In a randomized controlled trial with 16 experienced open-source developers on 246 tasks, allowing early-2025 AI tools increased completion time by 19%, although developers forecast a 24% speedup beforehand and still believed in a 20% speedup afterwards.")270271# ---- Stream 7: durable execution, failure recovery, observability, cost ----272add(key="dean2013tail", stream=7, doi="10.1145/2408776.2408794", vtype="journal", peer="unknown",273 rel="Tail-tolerance techniques (hedging, timeouts) underlie WP29's executor time budgets and retry policy.",274 claim="Software techniques that tolerate latency variability are vital to building responsive large-scale Web services.")275add(key="gray1986whystop", stream=7, s2="87c9e34c39dd1fbd470010dc1ea98ba3517f5a2a", vtype="conference", peer="unknown",276 rel="Foundational failure analysis (software and operations dominate outages; process pairs, transactions) behind WP29's checkpoint/idempotent step design.",277 claim="Reports Tandem field data attributing most system outages to software and operations/administration rather than hardware, and advocates process pairs and transactions as the means to tolerate faults.")278add(key="basiri2016chaos", stream=7, doi="10.1109/MS.2016.60", vtype="journal", peer="yes",279 rel="Chaos engineering as experimentation on failure injection motivates WP29's fault-injection tests of the executors.",280 claim="Describes chaos engineering as Netflix's practice of experimenting on distributed systems to verify their reliability, together with the principles underlying it.")281add(key="helland2012idempotence", stream=7, doi="10.1145/2160718.2160734", vtype="journal", peer="unknown",282 rel="Idempotent operations make retries safe, a requirement for WP29's resumable protocol steps.",283 claim="Messages may be retried; idempotence means that is acceptable.")284add(key="burckhardt2021durablefunctions", stream=7, doi="10.1145/3485510", vtype="journal", peer="yes",285 rel="Formal semantics for durable workflow execution (persistent state, reliable progress) that WP29's run ledger approximates.",286 claim="Describes Azure Durable Functions, which enhance FaaS with actors, workflows and critical sections to provide persistent state and reliable progress, and gives a formal semantics based on an idealized untyped lambda-calculus model.")287add(key="erol2025costofpass", stream=7, arxiv="2504.13359", vtype="preprint", peer="no",288 rel="Cost-of-pass (expected cost per correct solution) is the economic outcome metric WP29 reports per arm.",289 claim="Defines cost-of-pass as the expected monetary cost of generating a correct solution and the frontier cost-of-pass relative to a human expert, finding lightweight models most cost-effective for basic quantitative tasks and reasoning models for complex quantitative problems.")290add(key="dong2024agentops", stream=7, arxiv="2411.05285", vtype="preprint", peer="no",291 rel="Taxonomy of agent-lifecycle artifacts to trace; WP29's evidence records cover these observability artifacts.",292 claim="Presents a taxonomy of AgentOps artifacts and data that should be traced throughout the agent lifecycle to achieve observability, derived from a systematic mapping study of existing AgentOps tools.")293add(key="khanal2026reliability", stream=7, arxiv="2603.29231", vtype="preprint", peer="no",294 rel="Shows capability (pass@1) and reliability diverge with task duration, supporting WP29's repeated-run reliability outcomes.",295 claim="Introduces four reliability metrics (Reliability Decay Curve, Variance Amplification Factor, Graceful Degradation Score, Meltdown Onset Point) and, over 23,392 episodes of 10 models on 396 tasks, finds reliability decay is domain-stratified (software-engineering GDS falls from 0.90 to 0.44) while document processing is nearly flat.")296add(key="zhuang2026agentrewind", stream=7, arxiv="2608.14380", vtype="preprint", peer="no",297 rel="Checkpoint/rewind recovery for long-horizon agents; the failure-recovery mechanism WP29's protocol runs implement at stage boundaries.",298 claim="AgentRewind records aligned checkpoints of agent context and a controlled environment so that agents can return to an earlier state and resume with information from previous attempts, evaluated on the MettleBench long-horizon engineering benchmark.")299add(key="sigelman2010dapper", stream=7, s2="003d5a65de0ac72daaf105ded903cb3eb88585b3", vtype="report", peer="no",300 rel="Classic distributed tracing design (sampling, application-level transparency) referenced for WP29's run tracing.",301 claim="Describes Google's production distributed tracing infrastructure, designed for low overhead through sampling and application-level transparency.")302303# ---- Stream 8: evidence-grounded generation and claim verification ----304add(key="wadden2020scifact", stream=8, doi="10.18653/v1/2020.emnlp-main.609", vtype="conference", peer="yes",305 rel="Scientific claim verification task and dataset; model for WP29's claim-to-evidence checks in generated reports.",306 claim="Introduces scientific claim verification with SciFact, 1.4K expert-written claims paired with evidence abstracts annotated with labels and rationales; the system identified plausible evidence for 23 of 36 COVID-19 claims on CORD-19.")307add(key="thorne2018fever", stream=8, doi="10.18653/v1/N18-1074", vtype="conference", peer="yes",308 rel="Canonical fact extraction and verification benchmark defining Supported/Refuted/NotEnoughInfo labels adopted in WP29's citation audit.",309 claim="FEVER contains 185,445 claims derived from Wikipedia and labeled Supported, Refuted or NotEnoughInfo (Fleiss kappa 0.6841), with a best pipeline accuracy of 31.87% when the correct evidence is required.")310add(key="bohnet2022attributedqa", stream=8, arxiv="2212.08037", vtype="preprint", peer="no",311 rel="Attributed QA frames generation that must cite supporting sources; WP29 requires attributed claims in agent reports.",312 claim="Formulates Attributed QA with a reproducible evaluation framework that uses human annotations as gold standard and shows that a correlated automatic metric is suitable for development, benchmarking a broad set of architectures.")313add(key="walters2023fabrication", stream=8, doi="10.1038/s41598-023-41032-5", vtype="journal", peer="yes",314 rel="Quantifies fabricated references in LLM literature reviews; justifies WP29's mandatory citation verification gate.",315 claim="In 84 short literature reviews on 42 topics containing 636 citations, 55% of GPT-3.5 citations and 18% of GPT-4 citations were fabricated, and 43% versus 24% of the real citations contained substantive errors.")316add(key="alkaissi2023hallucinations", stream=8, doi="10.7759/cureus.35179", vtype="journal", peer="unknown",317 rel="Early clinical report of fabricated references ('artificial hallucinations') in scientific writing.",318 claim="Reports two case studies in which ChatGPT produced plausible but fabricated content including references when asked to write on rare conditions, documenting positive, negative and troubling aspects of its performance.")319add(key="gao2023rarr", stream=8, doi="10.18653/v1/2023.acl-long.910", vtype="conference", peer="yes",320 rel="Post-hoc attribution and revision pipeline; a model for WP29's independent evidence-grounding pass over agent output.",321 claim="RARR automatically finds attribution for the output of any text generator and post-edits unsupported content, significantly improving attribution while otherwise preserving the original output.")322add(key="wang2023selfconsistency", stream=8, arxiv="2203.11171", vtype="conference", peer="yes",323 rel="Self-consistency is an in-model verification heuristic; WP29 contrasts it with external executable verification.",324 claim="Self-consistency samples diverse reasoning paths and selects the most consistent answer, improving chain-of-thought accuracy on GSM8K (+17.9%), SVAMP (+11.0%) and AQuA (+12.2%).")325add(key="min2023factscore", stream=8, doi="10.18653/v1/2023.emnlp-main.741", arxiv="2305.14251", vtype="conference", peer="yes",326 rel="Atomic-fact precision scoring is the template for WP29's claim-level verification of generated reports.",327 claim="FActScore decomposes a generation into atomic facts and computes the percentage supported by a reliable knowledge source; ChatGPT biographies reach only 58%, and an automated estimator is introduced.")328add(key="huang2024selfcorrect", stream=8, arxiv="2310.01798", vtype="conference", peer="yes",329 rel="Evidence that intrinsic self-correction fails without external feedback, supporting WP29's external gates over agent self-review.",330 claim="Finds that LLMs struggle to self-correct their reasoning without external feedback, and that performance can even degrade after intrinsic self-correction.")331add(key="sakai2026hallucitation", stream=8, arxiv="2601.18724", vtype="conference", peer="yes",332 rel="Documents hallucinated citations reaching accepted NLP papers, showing the scale of the problem WP29's citation gate addresses.",333 claim="Analyzing all ACL, NAACL and EMNLP papers of 2024-2025, nearly 300 papers contain at least one hallucinated citation, half of them at EMNLP 2025, and more than 100 such papers were accepted as main-conference or Findings papers.")334add(key="zhao2026hallucinationswild", stream=8, arxiv="2605.07723", vtype="preprint", peer="no",335 rel="Large-scale audit of non-existent citations in the literature after LLM adoption; external evidence of the citation-fabrication risk.",336 claim="Auditing 111 million references across 2.5 million papers in arXiv, bioRxiv, SSRN and PubMed Central shows a sharp rise in non-existent references after widespread LLM adoption, with a conservative estimate of 146,932 hallucinated citations in 2025 alone.")337338339# ----------------------------------------------------------------------------340def authors_short(authors, key):341 fam = [a[0] for a in authors if a[0]]342 if key == "osc2015reproducibility":343 return "Open Science Collaboration"344 if key == "nasem2019reproducibility":345 return "National Academies of Sciences, Engineering, and Medicine"346 if not fam:347 return "n/a"348 if len(fam) == 1:349 return fam[0]350 if len(fam) == 2:351 return f"{fam[0]} & {fam[1]}"352 return f"{fam[0]} et al."353354355def bibesc(s):356 return (s.replace("&", "\\&").replace("%", "\\%").replace("#", "\\#").replace("_", "\\_")357 .replace("τ", "$\\tau$").replace("$τ$", "$\\tau$"))358359360rows, bibs, counts = [], [], {}361for e in E:362 doi, aid, s2id = e.get("doi"), e.get("arxiv"), e.get("s2")363 src = []364 if doi:365 m = crossref(doi); src.append(m["url"])366 res = DOIRES.get(doi); src.append(f"doi.org HEAD {res}")367 if aid:368 a = arxiv(aid); src.append(a["url"])369 venue = m["container"] or ("National Academies Press" if e["key"] == "nasem2019reproducibility" else "")370 verified = m["verified"]371 meta = m372 elif aid:373 meta = arxiv(aid); src.append(meta["url"]); verified = meta["verified"]374 vn, vt, _ = s2_venue("arXiv:" + aid)375 s2url = "https://api.semanticscholar.org/graph/v1/paper/arXiv:" + aid376 if e["vtype"] == "conference":377 venue = {"chen2025scienceagentbench": "ICLR 2025", "majumder2025discoverybench": "ICLR 2025",378 "jing2025dsbench": "ICLR 2025", "chan2025mlebench": "ICLR 2025",379 "huang2024mlagentbench": "ICML 2024", "gu2024blade": "EMNLP 2024",380 "jimenez2024swebench": "ICLR 2024", "starace2025paperbench": "ICML 2025",381 "liu2024agentbench": "ICLR 2024", "zheng2023mtbench": "NeurIPS 2023 Datasets and Benchmarks",382 "thakur2025judgingjudges": "GEM2 Workshop, ACL 2025",383 "panickssery2024selfpreference": "NeurIPS 2024", "lubars2019delegability": "NeurIPS 2019",384 "wang2023selfconsistency": "ICLR 2023", "huang2024selfcorrect": "ICLR 2024",385 "sakai2026hallucitation": "ACL 2026 (per Semantic Scholar venue record)"}[e["key"]]386 src.append(s2url + f" (venue={vn!r}, type={vt!r})")387 if meta.get("jref"):388 src.append("arXiv journal_ref: " + meta["jref"])389 elif e["vtype"] == "journal":390 venue = {"kapoor2024agentsthatmatter": "Transactions on Machine Learning Research",391 "siegel2024corebench": "Transactions on Machine Learning Research"}[e["key"]]392 src.append(s2url + f" (venue={vn!r})")393 else:394 venue = "arXiv preprint"395 else: # Semantic Scholar only396 fn = {"gray1986whystop": "s2_gray1986_search_observed.json",397 "sigelman2010dapper": "s2_dapper2010.json"}[e["key"]]398 p = os.path.join(CACHE, fn)399 if fn.endswith(".jsonl"):400 rec = None401 for line in open(p):402 line = line.strip()403 if not line: continue404 try:405 d = json.loads(line)406 except Exception:407 continue408 for x in d.get("data", []):409 if x.get("paperId") == s2id: rec = x410 else:411 rec = json.load(open(p))412 meta = dict(title=rec["title"], authors=[(a["name"].split()[-1], " ".join(a["name"].split()[:-1])) for a in rec["authors"]],413 year=rec["year"], container=rec.get("venue") or "")414 venue = rec.get("venue") or ("Google Technical Report dapper-2010-1" if e["key"] == "sigelman2010dapper" else "")415 src.append("https://api.semanticscholar.org/graph/v1/paper/" + s2id + f" (record cached in {fn})")416 verified = mtime_utc(p)417 if e.get("peer_src"):418 src.append(e["peer_src"])419 ident = ("doi:" + doi) if doi else (("arXiv:" + aid) if aid else "s2:" + s2id)420 counts[e["stream"]] = counts.get(e["stream"], 0) + 1421 rows.append(dict(ref_key=e["key"], stream=e["stream"], authors_short=authors_short(meta["authors"], e["key"]),422 year=meta["year"], title=meta["title"], venue=venue, venue_type=e["vtype"],423 doi_or_arxiv=ident, verification_source=" ; ".join(src), verified_utc=verified,424 relevance_to_wp29=e["rel"], key_claim_or_finding=e["claim"], peer_reviewed=e["peer"]))425426 # ---- BibTeX ----427 if e["key"] == "osc2015reproducibility":428 au = "{Open Science Collaboration}"429 elif e["key"] == "nasem2019reproducibility":430 au = "{National Academies of Sciences, Engineering, and Medicine}"431 elif doi:432 au = " and ".join(f"{f}, {g}" if g else f for f, g in meta["authors"])433 else:434 au = " and ".join(f"{g} {f}".strip() for f, g in meta["authors"])435 au = bibesc(au)436 ti = bibesc(meta["title"])437 f = [f" author = {{{au}}}", f" title = {{{{{ti}}}}}", f" year = {{{meta['year']}}}"]438 if e["vtype"] == "journal":439 btype = "article"; f.append(f" journal = {{{bibesc(venue)}}}")440 if doi and meta.get("volume"): f.append(f" volume = {{{meta['volume']}}}")441 if doi and meta.get("issue"): f.append(f" number = {{{meta['issue']}}}")442 if doi and meta.get("pages"): f.append(f" pages = {{{meta['pages'].replace('-', '--')}}}")443 elif e["vtype"] == "conference":444 btype = "inproceedings"; f.append(f" booktitle = {{{bibesc(venue)}}}")445 if doi and meta.get("pages"): f.append(f" pages = {{{meta['pages'].replace('-', '--')}}}")446 elif e["vtype"] == "report" and e["key"] == "nasem2019reproducibility":447 btype = "book"; f.append(" publisher = {The National Academies Press}"); f.append(" address = {Washington, DC}")448 elif e["vtype"] == "report":449 btype = "techreport"; f.append(" institution = {Google}"); f.append(" number = {dapper-2010-1}")450 else:451 btype = "misc"; f.append(f" howpublished = {{arXiv preprint arXiv:{aid}}}")452 if doi:453 f.append(f" doi = {{{doi}}}")454 if aid:455 f.append(f" eprint = {{{aid}}}"); f.append(" archivePrefix = {arXiv}")456 comment = f"% verified via {src[0].split(' (')[0]} on {verified[:10]}"457 bibs.append((comment, f"@{btype}{{{e['key']},\n" + ",\n".join(f) + "\n}"))458459cols = ["ref_key", "stream", "authors_short", "year", "title", "venue", "venue_type", "doi_or_arxiv",460 "verification_source", "verified_utc", "relevance_to_wp29", "key_claim_or_finding", "peer_reviewed"]461with open(os.path.join(HERE, "literature_matrix.csv"), "w", newline="", encoding="utf-8") as fh:462 w = csv.DictWriter(fh, fieldnames=cols); w.writeheader(); w.writerows(rows)463464with open(os.path.join(HERE, "references_verified.bib"), "w", encoding="utf-8") as fh:465 for i, (c, b) in enumerate(bibs):466 if i == 0:467 # No '%' line may precede the first entry: its verification comment follows it.468 fh.write(b + "\n" + c + " (comment placed after the first entry so that no % line precedes it)\n\n")469 else:470 fh.write(c + "\n" + b + "\n\n")471472print("entries:", len(rows), "per stream:", dict(sorted(counts.items())))473print("peer_reviewed:", {k: sum(1 for r in rows if r["peer_reviewed"] == k) for k in ("yes", "no", "unknown")})474