#!/usr/bin/env python3
"""Build literature_matrix.csv and references_verified.bib for WP29 from cached
primary-metadata records (Crossref, arXiv API, Semantic Scholar, OpenReview).

Every entry below was checked against the cached API response in verify_cache/
(fetched 2026-10-29/30 UTC).  Nothing is emitted unless the cache record exists.
"""
import csv, json, os, re, sys, html, urllib.parse, datetime
import xml.etree.ElementTree as ET

HERE = os.path.dirname(os.path.abspath(__file__))
CACHE = os.path.join(HERE, "verify_cache")
S2 = json.load(open(os.path.join(CACHE, "semanticscholar_batch.json")))
DOIRES = json.load(open(os.path.join(CACHE, "doi_resolution.json")))
NS = {"a": "http://www.w3.org/2005/Atom", "ar": "http://arxiv.org/schemas/atom"}


def cache_path(url):
    return os.path.join(CACHE, re.sub(r"[^A-Za-z0-9]+", "_", url)[:150])


def mtime_utc(path):
    return datetime.datetime.fromtimestamp(os.path.getmtime(path), datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")


def crossref(doi):
    url = "https://api.crossref.org/works/" + urllib.parse.quote(doi, safe="")
    p = cache_path(url)
    m = json.load(open(p))["message"]
    yr = (m.get("issued") or m.get("published-print") or m.get("published-online"))["date-parts"][0][0]
    authors = [(a.get("family", ""), a.get("given", "")) for a in m.get("author", [])]
    return dict(title=" ".join(m["title"][0].split()), authors=authors, year=yr,
                container=(m.get("container-title") or [""])[0], crtype=m["type"],
                volume=m.get("volume"), issue=m.get("issue"), pages=m.get("page"),
                publisher=m.get("publisher"), url="https://api.crossref.org/works/" + doi,
                verified=mtime_utc(p))


def arxiv(aid):
    url = "https://export.arxiv.org/api/query?id_list=" + aid
    p = cache_path(url)
    e = ET.parse(p).getroot().find("a:entry", NS)
    names = [a.find("a:name", NS).text for a in e.findall("a:author", NS)]
    authors = [(n.split()[-1], " ".join(n.split()[:-1])) for n in names]
    jr = e.find("ar:journal_ref", NS)
    return dict(title=" ".join(e.find("a:title", NS).text.split()), authors=authors,
                year=int(e.find("a:published", NS).text[:4]), container="arXiv preprint",
                jref=jr.text if jr is not None else None, url=url, verified=mtime_utc(p))


def s2_venue(key):
    v = S2.get(key) or {}
    pv = v.get("publicationVenue") or {}
    return v.get("venue"), pv.get("type"), v.get("year")


# ----------------------------------------------------------------------------
# Entry list.  Fields: key, stream, doi, arxiv, vtype, peer (yes/no/unknown),
# peer_src (extra verification URL for the venue), rel (relevance), claim.
# ----------------------------------------------------------------------------
E = []
def add(**k):
    E.append(k)

# ---- Stream 1: autonomous scientific discovery / agentic research systems ----
add(key="lu2024aiscientist", stream=1, arxiv="2408.06292", vtype="preprint", peer="no",
    rel="Reference end-to-end autonomous research system (idea to paper plus simulated review) against which WP29's protocol-guided design is positioned.",
    claim="Introduces The AI Scientist, which generates research ideas, writes code, executes experiments, visualizes results, writes a full paper and runs a simulated review, with each paper developed at a cost of less than $15 and an automated reviewer reported to achieve near-human performance on paper scores.")
add(key="lu2026aiscientist", stream=1, doi="10.1038/s41586-026-10265-5", vtype="journal", peer="yes",
    rel="Peer-reviewed (Nature, 2026) publication of The AI Scientist line of work, establishing that fully automated ML research pipelines are now part of the refereed literature.",
    claim="n/a (Nature abstract not retrieved behind the publisher login; see lu2024aiscientist for the system description).")
add(key="yamada2025aiscientistv2", stream=1, arxiv="2504.08066", vtype="preprint", peer="no",
    rel="Closest fully autonomous end-to-end system; its tree-search workflow and in-system AI reviewer are the counterpoint to WP29's executable gates and separated review.",
    claim="Describes an end-to-end agentic system with progressive agentic tree search managed by an experiment-manager agent and a VLM feedback loop for figures; of three fully autonomous manuscripts submitted to an ICLR workshop, one scored above the average human acceptance threshold.")
add(key="schmidgall2025agentlab", stream=1, doi="10.18653/v1/2025.findings-emnlp.320", arxiv="2501.04227", vtype="conference", peer="yes",
    rel="Staged (literature review, experimentation, report writing) autonomous research framework with optional human feedback; provides the human-in-the-loop and cost comparisons WP29 builds on.",
    claim="Agent Laboratory takes a human-provided idea through literature review, experimentation and report writing; the authors report that o1-preview gives the best outcomes, that human feedback at each stage significantly improves research quality, and an 84% decrease in research expenses relative to previous autonomous research methods.")
add(key="boiko2023coscientist", stream=1, doi="10.1038/s41586-023-06792-0", vtype="journal", peer="yes",
    rel="Landmark peer-reviewed demonstration of an LLM agent planning and executing laboratory experiments; defines the 'autonomous research' baseline WP29 compares to.",
    claim="Coscientist, driven by GPT-4 with internet/documentation search, code execution and experimental automation tools, autonomously designs, plans and performs experiments across six tasks, including successful reaction optimization of palladium-catalysed cross-couplings.")
add(key="bran2024chemcrow", stream=1, doi="10.1038/s42256-024-00832-8", vtype="journal", peer="yes",
    rel="Tool-augmented LLM research agent whose evaluation combined LLM and expert assessment, an early instance of the evaluator-validity question WP29 addresses with protected oracles.",
    claim="ChemCrow integrates 18 expert-designed tools with GPT-4, autonomously planned and executed syntheses of an insect repellent and three organocatalysts, and was evaluated with both LLM and expert assessments.")
add(key="ifargan2025datatopaper", stream=1, doi="10.1056/AIoa2400555", arxiv="2404.17605", vtype="journal", peer="yes",
    rel="Closest prior work on traceability: an autonomous data-to-paper platform that programmatically backtraces information flow for human verification, akin to WP29's persistent evidence records.",
    claim="data-to-paper guides interacting LLM agents through a stepwise research process from annotated data to papers while programmatically backtracing information flow and allowing human oversight, and can run fully autonomously ('autopilot') or with human intervention.")
add(key="gottweis2026coscientist", stream=1, doi="10.1038/s41586-026-10644-y", arxiv="2502.18864", vtype="journal", peer="yes",
    rel="Multi-agent hypothesis-generation system with generate/critique/refine loops; illustrates in-system self-critique rather than independent verification.",
    claim="Co-Scientist is a Gemini-based multi-agent system that continuously generates, critiques and refines research hypotheses using an asynchronous task framework and a tournament evolution process, scaled by test-time compute, to propose demonstrably novel hypotheses for experimental verification.")
add(key="swanson2025virtuallab", stream=1, doi="10.1038/s41586-025-09442-9", vtype="journal", peer="yes",
    rel="AI-human research collaboration with an LLM principal-investigator agent and human oversight; a human-involvement design point for the novelty matrix.",
    claim="The Virtual Lab is an AI-human research collaboration in which an LLM Principal Investigator agent guides a team of LLM scientist agents to perform interdisciplinary research, applied to designing SARS-CoV-2 nanobodies.")
add(key="mitchener2025kosmos", stream=1, arxiv="2511.02824", vtype="preprint", peer="no",
    rel="Recent long-horizon AI scientist (up to 12 h, 200 rollouts); documents the coherence/verification challenges of long autonomous runs that WP29's gates target.",
    claim="Kosmos runs for up to 12 hours performing cycles of parallel data analysis, literature search and hypothesis generation, using a structured world model shared between agents to pursue an objective coherently over 200 agent rollouts before synthesizing reports.")
add(key="luo2025pitfalls", stream=1, arxiv="2509.08713", vtype="preprint", peer="no",
    rel="Directly motivates WP29: shows AI scientist systems commit benchmark selection, leakage, metric-misuse and post-hoc selection errors that only traces reveal.",
    claim="Identifies four failure modes in AI scientist systems (inappropriate benchmark selection, data leakage, metric misuse and post-hoc selection bias) and, via controlled experiments on two open-source systems, finds several of them present.")
add(key="beel2025sakana", stream=1, doi="10.1145/3769733.3769747", arxiv="2502.14297", vtype="journal", peer="unknown",
    rel="Independent third-party evaluation of The AI Scientist; evidence that unverified autonomous outputs contain novelty and execution errors.",
    claim="An independent evaluation of the AI Scientist found several critical shortcomings, including a literature review based on simplistic keyword searches that led to poor novelty assessments, with several generated ideas incorrectly classified as novel.")
add(key="luo2026xscientist", stream=1, arxiv="2607.12301", vtype="preprint", peer="no",
    rel="Closest 2026 design to a research protocol: git-like typed research state with checkpoints, claim-evidence closure and gates; differs from WP29 in being a protocol artifact rather than a controlled comparison of protocol arms.",
    claim="Proposes a local-first, git-like protocol that treats research state rather than a manuscript as the unit of continuation, representing hypotheses, attempts, claims, reviews and handoffs as typed content-addressed objects with immutable checkpoints, claim-evidence closure and authority-aware gates, without treating a passing integrity check as scientific truth.")
add(key="ding2026verificationgap", stream=1, arxiv="2608.05179", vtype="preprint", peer="no",
    rel="Survey that names the 'verification gap' in AI scientist systems; supplies the audit dimensions (evaluation method, artifacts, human-in-the-loop points, result-selection disclosure) WP29 reports.",
    claim="Screens 125 works and codes 26 (24 runnable systems) on seven audit dimensions, finding that code release is now common but reproducibility-grade and claim-verification artifacts remain rare.")

# ---- Stream 2: language agents for data analysis and scientific programming ----
add(key="chen2025scienceagentbench", stream=2, arxiv="2410.05080", vtype="conference", peer="yes",
    peer_src="https://api2.openreview.net/notes/search?term=ScienceAgentBench (ICLR 2025 Poster record)",
    rel="Benchmark of expert-validated data-driven discovery tasks with program-level outputs, execution metrics and cost; the task style and contamination mitigations inform WP29's task bank.",
    claim="Presents 102 tasks extracted from 44 publications in four disciplines and validated by nine experts; the best agent solves 32.4% of tasks independently (34.3% with expert knowledge), o1-preview reaches 42.2% at more than 10 times the cost, and two data-contamination mitigation strategies are proposed.")
add(key="majumder2025discoverybench", stream=2, arxiv="2407.01725", vtype="conference", peer="yes",
    peer_src="https://api2.openreview.net/notes/search?term=DiscoveryBench (ICLR 2025 Poster record)",
    rel="Formalizes multi-step data-driven discovery as dataset + metadata + goal, the same unit of analysis as WP29's research tasks.",
    claim="DiscoveryBench formalizes the multi-step process of data-driven discovery with 264 tasks across six domains derived from published papers plus 903 synthetic tasks, each defined by a dataset, its metadata and a natural-language discovery goal.")
add(key="jing2025dsbench", stream=2, arxiv="2409.07703", vtype="conference", peer="yes",
    peer_src="https://api2.openreview.net/notes/search?term=DSBench (ICLR 2025 Poster record)",
    rel="Realistic data-science agent benchmark (long contexts, large data) used to characterize executor capability independent of protocol.",
    claim="DSBench comprises 466 data-analysis and 74 data-modeling tasks sourced from Eloquence and Kaggle competitions, with long contexts, multimodal task backgrounds and reasoning over large data files.")
add(key="huang2024mlagentbench", stream=2, arxiv="2310.03302", vtype="conference", peer="yes",
    rel="Evaluates agents on ML experimentation loops with file/code actions; a precursor of WP29's executor arms.",
    claim="MLAgentBench is a suite of 13 ML experimentation tasks (from CIFAR-10 improvement to BabyLM) in which a ReAct-based agent can read/write files, execute code and inspect outputs; a Claude 3 Opus agent had the best success rate.")
add(key="gu2024blade", stream=2, arxiv="2408.09667", vtype="conference", peer="yes",
    rel="Evaluates analysis decisions (variables, transformations, models) against independent expert analyses, the multi-path ground truth WP29's oracles must handle.",
    claim="BLADE evaluates agents' multifaceted approaches to open-ended research questions using 12 datasets and questions from the literature, with ground truth collected from independent analyses by expert data scientists.")
add(key="jimenez2024swebench", stream=2, arxiv="2310.06770", vtype="conference", peer="yes",
    rel="Canonical execution-verified coding benchmark; its test-based grading is the model for WP29's executable verification gates.",
    claim="SWE-bench consists of 2,294 software-engineering problems drawn from real GitHub issues and pull requests across 12 Python repositories, where resolving an issue frequently requires coordinating changes across multiple functions, classes and files.")
add(key="chan2025mlebench", stream=2, arxiv="2410.07095", vtype="conference", peer="yes",
    peer_src="https://proceedings.iclr.cc/paper_files/paper/2025/hash/7e3767db483c942b883eb4f8cfb74e31-Abstract-Conference.html",
    rel="Agentic ML-engineering benchmark with human baselines, resource-scaling and contamination analyses that WP29's resource accounting parallels.",
    claim="MLE-bench curates 75 Kaggle competitions with human leaderboard baselines; the best setup (o1-preview with AIDE scaffolding) reaches at least bronze-medal level in 16.9% of competitions, and the paper also studies resource scaling and pre-training contamination.")
add(key="siegel2024corebench", stream=2, arxiv="2409.11363", vtype="journal", peer="yes",
    rel="Benchmarks agents on computational reproducibility of published papers, linking agent evaluation to the reproducibility stream.",
    claim="CORE-Bench contains 270 tasks based on 90 scientific papers across computer science, social science and medicine, measuring agents' accuracy at reproducing published results from provided code and data at three difficulty levels.")
add(key="starace2025paperbench", stream=2, arxiv="2504.01848", vtype="conference", peer="yes",
    rel="Rubric-decomposed replication grading with a validated LLM judge; closest precedent for WP29's rubric oracles and judge validation.",
    claim="PaperBench asks agents to replicate 20 ICML 2024 papers from scratch, graded against 8,316 rubric items co-developed with the papers' authors, with an LLM-based judge whose performance is assessed on a separate judge benchmark.")

# ---- Stream 3: workflows, provenance, computational reproducibility ----
add(key="koster2012snakemake", stream=3, doi="10.1093/bioinformatics/bts480", vtype="journal", peer="yes",
    rel="Rule-based workflow engine; WP29's protocol stages are an analogue of declarative, re-executable workflow rules.",
    claim="Snakemake provides a readable Python-based workflow definition language and an execution environment that scales from single-core workstations to compute clusters without modifying the workflow.")
add(key="ditommaso2017nextflow", stream=3, doi="10.1038/nbt.3820", vtype="journal", peer="yes",
    rel="Container-based dataflow workflow manager for reproducible pipelines; reference for execution isolation in WP29's executors.",
    claim="Argues that combining dataflow programming with software containers (Nextflow) yields computational workflows that are reproducible across platforms.")
add(key="missier2013prov", stream=3, doi="10.1145/2452376.2452478", vtype="conference", peer="yes",
    rel="W3C PROV family defines the provenance vocabulary (entities, activities, agents) that WP29's evidence records instantiate.",
    claim="Overview of the W3C PROV family of specifications (data model, ontology, notation and constraints) for interoperable representation and exchange of provenance metadata.")
add(key="chirigati2016reprozip", stream=3, doi="10.1145/2882903.2899401", vtype="conference", peer="yes",
    rel="Automatic capture of execution provenance into re-executable packages; a model for evidence capture without relying on the agent's self-report.",
    claim="ReproZip automatically captures the provenance of a command-line experiment (files, libraries, environment) and packages it so that it can be re-executed elsewhere.")
add(key="sandve2013tensimplerules", stream=3, doi="10.1371/journal.pcbi.1003285", vtype="journal", peer="yes",
    rel="Widely cited rules (track provenance, avoid manual steps, record versions and seeds) that WP29 operationalizes as machine-checked gates.",
    claim="Proposes ten rules for reproducible computational research, including tracking how every result was produced, avoiding manual data manipulation, archiving exact program versions and random seeds, and providing public access to scripts, runs and results.")
add(key="stodden2018journalpolicy", stream=3, doi="10.1073/pnas.1708290115", vtype="journal", peer="yes",
    rel="Empirical reproducibility failure rates under policy-only enforcement; motivates executable rather than declarative verification.",
    claim="In a random sample of 204 Science papers subject to a data/code-on-request policy, the authors obtained artifacts from 44% of papers and could reproduce the findings for 26%.")
add(key="peng2011reproducible", stream=3, doi="10.1126/science.1213847", vtype="journal", peer="yes",
    rel="Defines reproducibility (data + code available) as the minimum standard for computational claims, the standard WP29's evidence records target.",
    claim="Reproducibility has the potential to serve as a minimum standard for judging scientific claims when full independent replication of a study is not possible.")
add(key="nasem2019reproducibility", stream=3, doi="10.17226/25303", vtype="report", peer="unknown",
    rel="Authoritative definitions of reproducibility vs replicability used in WP29's outcome definitions.",
    claim="Defines reproducibility as obtaining consistent computational results using the same input data, computational steps, methods, code and conditions of analysis, and replicability as obtaining consistent results across studies aimed at the same question with new data.")
add(key="krafczyk2021reproduction", stream=3, doi="10.1098/rsta.2020.0069", vtype="journal", peer="yes",
    rel="Principles (transparency, re-executability, determinism) and a Reproduction Package formalism close to WP29's release bundle.",
    claim="From reproducing seven articles the authors derive three principles (transparency about how results are produced, ease of re-executability, determinism), 12 guidelines, and a Reproduction Package formalism for sharing artifacts.")
add(key="trisovic2022codequality", stream=3, doi="10.1038/s41597-022-01143-6", vtype="journal", peer="yes",
    rel="Large-scale evidence that deposited research code usually fails to run, justifying execution-based verification gates.",
    claim="Executing over 9,000 R files from more than 2,000 Harvard Dataverse replication datasets, 74% failed to complete without error initially and 56% still failed after automatic code cleaning.")

# ---- Stream 4: preregistration, registered reports, specification discipline ----
add(key="nosek2018preregistration", stream=4, doi="10.1073/pnas.1708274114", vtype="journal", peer="yes",
    rel="Preregistration as prediction/postdiction separation is the direct analogue of WP29's protocol fixed before execution.",
    claim="Defining research questions and the analysis plan before observing outcomes (preregistration) distinguishes analyses that result from predictions from those that result from postdictions, and practical strategies exist for cases that fall short of the ideal.")
add(key="chambers2013registeredreports", stream=4, doi="10.1016/j.cortex.2012.12.016", vtype="journal", peer="unknown",
    rel="Origin of Registered Reports: review of the protocol before results, the publishing analogue of WP29's pre-execution gate.",
    claim="Introduces Registered Reports at Cortex, in which study protocols are peer reviewed and provisionally accepted before data collection.")
add(key="chambers2022registeredreports", stream=4, doi="10.1038/s41562-021-01193-7", vtype="journal", peer="yes",
    rel="Reviews evidence that pre-acceptance of protocols reduces reporting bias; supports the 'specification before execution' arm.",
    claim="Registered Reports are a publication format in which study proposals are peer reviewed and pre-accepted before research is undertaken; early evidence indicates they work as intended although they are not a universal solution for irreproducibility.")
add(key="simmons2011falsepositive", stream=4, doi="10.1177/0956797611417632", vtype="journal", peer="yes",
    rel="Researcher degrees of freedom inflate false positives; WP29's locked analysis specification removes these from the agent.",
    claim="Flexibility in data collection, analysis and reporting dramatically increases actual false-positive rates, so that a researcher is often more likely to falsely find evidence that an effect exists than to correctly find evidence that it does not; six disclosure requirements for authors and four reviewer guidelines are proposed.")
add(key="munafo2017manifesto", stream=4, doi="10.1038/s41562-016-0021", vtype="journal", peer="yes",
    rel="Umbrella statement of reproducibility measures (methods, reporting, evaluation, incentives) that WP29 adapts to autonomous agents.",
    claim="Argues for measures optimizing methods, reporting and dissemination, reproducibility, evaluation and incentives, noting evidence from simulations and empirical studies but the need for iterative evaluation of their adoption.")
add(key="gelman2014statisticalcrisis", stream=4, doi="10.1511/2014.111.460", vtype="journal", peer="unknown",
    rel="Garden of forking paths: data-dependent analysis choices invalidate inference even without intent, which an agent in an open instruction arm is free to make.",
    claim="Argues that data-dependent choices of analysis (the 'garden of forking paths') can produce spurious statistically significant findings even when researchers do not deliberately fish for significance.")
add(key="nosek2015top", stream=4, doi="10.1126/science.aab2374", vtype="journal", peer="yes",
    rel="TOP guidelines define graded transparency standards (preregistration, data, code) that WP29's protocol tiers mirror.",
    claim="Proposes journal author guidelines (the TOP guidelines) to promote transparency, openness and reproducibility, noting that although scientists embrace these norms they are not routine in practice.")
add(key="scheel2021excess", stream=4, doi="10.1177/25152459211007467", vtype="journal", peer="yes",
    rel="Quantifies how pre-specified protocols change the result distribution; benchmark for interpreting WP29's arm differences.",
    claim="Comparing 71 published Registered Reports with 152 standard hypothesis-testing papers, the first hypothesis was positive in 96% of standard reports but only 44% of Registered Reports.")
add(key="osc2015reproducibility", stream=4, doi="10.1126/science.aac4716", vtype="journal", peer="yes",
    rel="Baseline replication rates in a human literature; provides a human reference for WP29's validity outcomes.",
    claim="Replications of 100 experiments from 2008 papers in three psychology journals found that about one-third to one-half of original findings were also observed in the replication, depending on the criterion.")
add(key="kerr1998harking", stream=4, doi="10.1207/s15327957pspr0203_4", vtype="journal", peer="yes",
    rel="HARKing (presenting post hoc hypotheses as a priori) is a failure mode an unconstrained agent can reproduce when writing up.",
    claim="Defines HARKing (hypothesizing after the results are known) as presenting post hoc hypotheses as if they were a priori, and discusses its prevalence and costs.")

# ---- Stream 5: agent evaluation, contamination, evaluator validity ----
add(key="liu2024agentbench", stream=5, arxiv="2308.03688", vtype="conference", peer="yes",
    rel="Multi-environment agent benchmark identifying long-horizon reasoning and instruction following as main failure causes, relevant to protocol adherence measurement.",
    claim="AgentBench evaluates LLMs as agents across 8 environments; top commercial LLMs act well in complex environments but a significant gap separates them from open-source models up to 70B, with poor long-term reasoning, decision-making and instruction following as main obstacles.")
add(key="yao2024taubench", stream=5, arxiv="2406.12045", vtype="preprint", peer="unknown",
    rel="State-based outcome checking and the pass^k reliability metric are reused in WP29's repeated-run reliability analysis.",
    claim="tau-bench evaluates agents in simulated user conversations with API tools and policy guidelines by comparing the final database state to an annotated goal state, and proposes pass^k for reliability over multiple trials; gpt-4o succeeds on fewer than 50% of tasks with pass^8 below 25% in retail.")
add(key="kapoor2024agentsthatmatter", stream=5, arxiv="2407.01502", vtype="journal", peer="yes",
    rel="Core methodological critique (cost reporting, holdouts, conflated benchmarking needs) that WP29's evaluation design follows.",
    claim="Analysis of agent benchmarks finds a narrow focus on accuracy without cost, leading to needlessly complex and costly agents; proposes jointly optimizing accuracy and cost, and notes that model- and downstream-developer needs are conflated and that many benchmarks have inadequate holdout sets.")
add(key="sainz2023contamination", stream=5, doi="10.18653/v1/2023.findings-emnlp.722", arxiv="2310.18018", vtype="conference", peer="yes",
    rel="Defines contamination levels; WP29's task bank must document contamination risk for each task.",
    claim="Position paper arguing that training on a benchmark's test split causes overestimation of performance with potentially harmful scientific conclusions, defining levels of data contamination and calling for community measurement for each benchmark.")
add(key="zheng2023mtbench", stream=5, arxiv="2306.05685", vtype="conference", peer="yes",
    peer_src="https://api2.openreview.net/notes/search?term=Judging%20LLM-as-a-judge (NeurIPS 2023 Datasets and Benchmarks record)",
    rel="Establishes LLM-as-a-judge and its biases (position, verbosity, self-enhancement), which WP29's protected oracles are designed to avoid.",
    claim="Examines LLM-as-a-judge limitations including position, verbosity and self-enhancement biases, and finds that strong judges such as GPT-4 agree with human preferences over 80% of the time on MT-bench and Chatbot Arena, the same level as human-human agreement.")
add(key="thakur2025judgingjudges", stream=5, arxiv="2406.12624", vtype="conference", peer="yes",
    rel="Shows even the best judge models lag inter-human agreement, cautioning against unprotected LLM grading of research outputs.",
    claim="Across thirteen judge models and nine exam-taker models in a high-human-agreement setting, only the best and largest judges reach reasonable alignment with humans, remain well behind inter-human agreement, and may differ from human scores by up to 5 points.")
add(key="panickssery2024selfpreference", stream=5, arxiv="2404.13076", vtype="conference", peer="yes",
    rel="Self-preference bias is the specific threat WP29 mitigates by separating executor and reviewer and protecting oracles.",
    claim="LLM evaluators such as GPT-4 and Llama 2 have non-trivial accuracy at recognizing their own outputs, and fine-tuning reveals a linear correlation between self-recognition capability and the strength of self-preference.")
add(key="zhu2025abc", stream=5, arxiv="2507.02825", vtype="preprint", peer="no",
    rel="Agentic Benchmark Checklist (task validity, outcome validity, reporting) used as a checklist for WP29's task and oracle construction.",
    claim="Shows that many agentic benchmarks have task-setup or reward-design flaws (e.g., SWE-bench Verified uses insufficient test cases, TAU-bench counts empty responses as successful) that can mis-estimate performance by up to 100% relative, and introduces the Agentic Benchmark Checklist, which reduced overestimation on CVE-Bench by 33%.")
add(key="kapoor2025hal", stream=5, arxiv="2510.11977", vtype="preprint", peer="no",
    rel="Standardized harness with full cost accounting across models, scaffolds and benchmarks, the infrastructure model for WP29's isolated executors and resource logs.",
    claim="Introduces a standardized evaluation harness and reports 21,730 agent rollouts across 9 models and 9 benchmarks at a total cost of about $40,000, finding among other things that higher reasoning effort can reduce accuracy.")
add(key="li2025llmreval", stream=5, arxiv="2510.12367", vtype="preprint", peer="no",
    rel="Simulated research-and-review loop showing LLM reviewers misaligned with humans; motivates WP29's separate, protected review arm rather than in-system AI review.",
    claim="Using a simulation with a research agent that writes and revises papers and a review agent that assesses them, followed by human annotation, the authors identify pronounced misalignment between LLM reviewers and human judgments.")

# ---- Stream 6: human-AI delegation, automation bias, intervention measurement ----
add(key="parasuraman1997automation", stream=6, doi="10.1518/001872097778543886", vtype="journal", peer="yes",
    rel="Use/misuse/disuse/abuse taxonomy frames how a researcher delegates to an autonomous agent and over-relies on its outputs.",
    claim="Distinguishes use, misuse (over-reliance producing monitoring failures or decision biases), disuse (often caused by false alarms) and abuse of automation, with trust, workload and reliability as key influences.")
add(key="parasuraman2000levels", stream=6, doi="10.1109/3468.844354", vtype="journal", peer="yes",
    rel="Levels-of-automation model used to locate WP29's four arms on the acquisition/analysis/decision/implementation dimensions.",
    claim="Proposes that automation can be applied to four function classes (information acquisition, information analysis, decision and action selection, action implementation), each across a continuum from fully manual to fully automatic, and that automation changes rather than merely supplants human activity.")
add(key="goddard2012automationbias", stream=6, doi="10.1136/amiajnl-2011-000089", vtype="journal", peer="yes",
    rel="Systematic review of automation bias frequency and mitigators informing how human review of agent outputs is measured in WP29.",
    claim="A systematic review (74 of 13,821 retrieved papers) finds automation bias mediated by user factors such as cognitive style, decision-support design and task experience, discussed alongside automation-induced complacency.")
add(key="lai2023humanai", stream=6, doi="10.1145/3593013.3594087", arxiv="2112.11471", vtype="conference", peer="yes",
    rel="Design-space survey of empirical human-AI decision-making studies used to position WP29's intervention measurement.",
    claim="Surveys empirical human-subject studies of AI-assisted decision making and summarizes the design space of tasks, AI assistance elements and evaluation metrics to structure a science of human-AI decision making.")
add(key="bansal2021whole", stream=6, doi="10.1145/3411764.3445717", vtype="conference", peer="yes",
    rel="Shows explanations raise acceptance of AI recommendations regardless of correctness, a risk for human review of agent-generated research.",
    claim="In studies on three datasets, AI augmentation produced complementary improvements but explanations did not increase them; instead explanations increased the chance that humans accept the AI's recommendation regardless of its correctness.")
add(key="lubars2019delegability", stream=6, arxiv="1902.03245", vtype="conference", peer="yes",
    rel="Task-delegability framework (motivation, difficulty, risk, trust) used to characterize which research sub-tasks are delegated in each WP29 arm.",
    claim="Proposes a delegability framework with four factors (motivation, difficulty, risk, trust) and, surveying 100 tasks, finds little preference for full AI control and a strong preference for machine-in-the-loop designs, with trust most correlated with preferences.")
add(key="vaccaro2024combinations", stream=6, doi="10.1038/s41562-024-02024-1", vtype="journal", peer="yes",
    rel="Meta-analytic evidence on when human-AI combinations beat either alone; reference point for interpreting WP29's human-review arm.",
    claim="A preregistered meta-analysis of 106 experiments (370 effect sizes, 2020-2023) finds that on average human-AI combinations performed worse than the best of humans or AI alone, with losses in decision tasks and gains in content-creation tasks.")
add(key="bainbridge1983ironies", stream=6, doi="10.1016/0005-1098(83)90046-8", vtype="journal", peer="yes",
    rel="Classic argument that automation makes the residual human role harder, relevant to reviewers of autonomous research output.",
    claim="Argues that the more advanced a control system is, the more crucial the contribution of the human operator becomes, because the operator is left with the tasks the designer could not automate and must monitor a system that rarely needs intervention.")
add(key="bucinca2021trust", stream=6, doi="10.1145/3449287", vtype="journal", peer="yes",
    rel="Cognitive forcing as a mitigation of overreliance; analogous to WP29's requirement that reviewers execute checks rather than read summaries.",
    claim="In an experiment with 199 participants, three cognitive forcing interventions reduced overreliance on AI relative to simple explainable-AI approaches.")
add(key="becker2025metr", stream=6, arxiv="2507.09089", vtype="preprint", peer="no",
    rel="Randomized measurement of AI-tool effects on expert developers showing perception-reality gaps; methodological model for WP29's time/cost outcomes.",
    claim="In a randomized controlled trial with 16 experienced open-source developers on 246 tasks, allowing early-2025 AI tools increased completion time by 19%, although developers forecast a 24% speedup beforehand and still believed in a 20% speedup afterwards.")

# ---- Stream 7: durable execution, failure recovery, observability, cost ----
add(key="dean2013tail", stream=7, doi="10.1145/2408776.2408794", vtype="journal", peer="unknown",
    rel="Tail-tolerance techniques (hedging, timeouts) underlie WP29's executor time budgets and retry policy.",
    claim="Software techniques that tolerate latency variability are vital to building responsive large-scale Web services.")
add(key="gray1986whystop", stream=7, s2="87c9e34c39dd1fbd470010dc1ea98ba3517f5a2a", vtype="conference", peer="unknown",
    rel="Foundational failure analysis (software and operations dominate outages; process pairs, transactions) behind WP29's checkpoint/idempotent step design.",
    claim="Reports Tandem field data attributing most system outages to software and operations/administration rather than hardware, and advocates process pairs and transactions as the means to tolerate faults.")
add(key="basiri2016chaos", stream=7, doi="10.1109/MS.2016.60", vtype="journal", peer="yes",
    rel="Chaos engineering as experimentation on failure injection motivates WP29's fault-injection tests of the executors.",
    claim="Describes chaos engineering as Netflix's practice of experimenting on distributed systems to verify their reliability, together with the principles underlying it.")
add(key="helland2012idempotence", stream=7, doi="10.1145/2160718.2160734", vtype="journal", peer="unknown",
    rel="Idempotent operations make retries safe, a requirement for WP29's resumable protocol steps.",
    claim="Messages may be retried; idempotence means that is acceptable.")
add(key="burckhardt2021durablefunctions", stream=7, doi="10.1145/3485510", vtype="journal", peer="yes",
    rel="Formal semantics for durable workflow execution (persistent state, reliable progress) that WP29's run ledger approximates.",
    claim="Describes Azure Durable Functions, which enhance FaaS with actors, workflows and critical sections to provide persistent state and reliable progress, and gives a formal semantics based on an idealized untyped lambda-calculus model.")
add(key="erol2025costofpass", stream=7, arxiv="2504.13359", vtype="preprint", peer="no",
    rel="Cost-of-pass (expected cost per correct solution) is the economic outcome metric WP29 reports per arm.",
    claim="Defines cost-of-pass as the expected monetary cost of generating a correct solution and the frontier cost-of-pass relative to a human expert, finding lightweight models most cost-effective for basic quantitative tasks and reasoning models for complex quantitative problems.")
add(key="dong2024agentops", stream=7, arxiv="2411.05285", vtype="preprint", peer="no",
    rel="Taxonomy of agent-lifecycle artifacts to trace; WP29's evidence records cover these observability artifacts.",
    claim="Presents a taxonomy of AgentOps artifacts and data that should be traced throughout the agent lifecycle to achieve observability, derived from a systematic mapping study of existing AgentOps tools.")
add(key="khanal2026reliability", stream=7, arxiv="2603.29231", vtype="preprint", peer="no",
    rel="Shows capability (pass@1) and reliability diverge with task duration, supporting WP29's repeated-run reliability outcomes.",
    claim="Introduces four reliability metrics (Reliability Decay Curve, Variance Amplification Factor, Graceful Degradation Score, Meltdown Onset Point) and, over 23,392 episodes of 10 models on 396 tasks, finds reliability decay is domain-stratified (software-engineering GDS falls from 0.90 to 0.44) while document processing is nearly flat.")
add(key="zhuang2026agentrewind", stream=7, arxiv="2608.14380", vtype="preprint", peer="no",
    rel="Checkpoint/rewind recovery for long-horizon agents; the failure-recovery mechanism WP29's protocol runs implement at stage boundaries.",
    claim="AgentRewind records aligned checkpoints of agent context and a controlled environment so that agents can return to an earlier state and resume with information from previous attempts, evaluated on the MettleBench long-horizon engineering benchmark.")
add(key="sigelman2010dapper", stream=7, s2="003d5a65de0ac72daaf105ded903cb3eb88585b3", vtype="report", peer="no",
    rel="Classic distributed tracing design (sampling, application-level transparency) referenced for WP29's run tracing.",
    claim="Describes Google's production distributed tracing infrastructure, designed for low overhead through sampling and application-level transparency.")

# ---- Stream 8: evidence-grounded generation and claim verification ----
add(key="wadden2020scifact", stream=8, doi="10.18653/v1/2020.emnlp-main.609", vtype="conference", peer="yes",
    rel="Scientific claim verification task and dataset; model for WP29's claim-to-evidence checks in generated reports.",
    claim="Introduces scientific claim verification with SciFact, 1.4K expert-written claims paired with evidence abstracts annotated with labels and rationales; the system identified plausible evidence for 23 of 36 COVID-19 claims on CORD-19.")
add(key="thorne2018fever", stream=8, doi="10.18653/v1/N18-1074", vtype="conference", peer="yes",
    rel="Canonical fact extraction and verification benchmark defining Supported/Refuted/NotEnoughInfo labels adopted in WP29's citation audit.",
    claim="FEVER contains 185,445 claims derived from Wikipedia and labeled Supported, Refuted or NotEnoughInfo (Fleiss kappa 0.6841), with a best pipeline accuracy of 31.87% when the correct evidence is required.")
add(key="bohnet2022attributedqa", stream=8, arxiv="2212.08037", vtype="preprint", peer="no",
    rel="Attributed QA frames generation that must cite supporting sources; WP29 requires attributed claims in agent reports.",
    claim="Formulates Attributed QA with a reproducible evaluation framework that uses human annotations as gold standard and shows that a correlated automatic metric is suitable for development, benchmarking a broad set of architectures.")
add(key="walters2023fabrication", stream=8, doi="10.1038/s41598-023-41032-5", vtype="journal", peer="yes",
    rel="Quantifies fabricated references in LLM literature reviews; justifies WP29's mandatory citation verification gate.",
    claim="In 84 short literature reviews on 42 topics containing 636 citations, 55% of GPT-3.5 citations and 18% of GPT-4 citations were fabricated, and 43% versus 24% of the real citations contained substantive errors.")
add(key="alkaissi2023hallucinations", stream=8, doi="10.7759/cureus.35179", vtype="journal", peer="unknown",
    rel="Early clinical report of fabricated references ('artificial hallucinations') in scientific writing.",
    claim="Reports two case studies in which ChatGPT produced plausible but fabricated content including references when asked to write on rare conditions, documenting positive, negative and troubling aspects of its performance.")
add(key="gao2023rarr", stream=8, doi="10.18653/v1/2023.acl-long.910", vtype="conference", peer="yes",
    rel="Post-hoc attribution and revision pipeline; a model for WP29's independent evidence-grounding pass over agent output.",
    claim="RARR automatically finds attribution for the output of any text generator and post-edits unsupported content, significantly improving attribution while otherwise preserving the original output.")
add(key="wang2023selfconsistency", stream=8, arxiv="2203.11171", vtype="conference", peer="yes",
    rel="Self-consistency is an in-model verification heuristic; WP29 contrasts it with external executable verification.",
    claim="Self-consistency samples diverse reasoning paths and selects the most consistent answer, improving chain-of-thought accuracy on GSM8K (+17.9%), SVAMP (+11.0%) and AQuA (+12.2%).")
add(key="min2023factscore", stream=8, doi="10.18653/v1/2023.emnlp-main.741", arxiv="2305.14251", vtype="conference", peer="yes",
    rel="Atomic-fact precision scoring is the template for WP29's claim-level verification of generated reports.",
    claim="FActScore decomposes a generation into atomic facts and computes the percentage supported by a reliable knowledge source; ChatGPT biographies reach only 58%, and an automated estimator is introduced.")
add(key="huang2024selfcorrect", stream=8, arxiv="2310.01798", vtype="conference", peer="yes",
    rel="Evidence that intrinsic self-correction fails without external feedback, supporting WP29's external gates over agent self-review.",
    claim="Finds that LLMs struggle to self-correct their reasoning without external feedback, and that performance can even degrade after intrinsic self-correction.")
add(key="sakai2026hallucitation", stream=8, arxiv="2601.18724", vtype="conference", peer="yes",
    rel="Documents hallucinated citations reaching accepted NLP papers, showing the scale of the problem WP29's citation gate addresses.",
    claim="Analyzing all ACL, NAACL and EMNLP papers of 2024-2025, nearly 300 papers contain at least one hallucinated citation, half of them at EMNLP 2025, and more than 100 such papers were accepted as main-conference or Findings papers.")
add(key="zhao2026hallucinationswild", stream=8, arxiv="2605.07723", vtype="preprint", peer="no",
    rel="Large-scale audit of non-existent citations in the literature after LLM adoption; external evidence of the citation-fabrication risk.",
    claim="Auditing 111 million references across 2.5 million papers in arXiv, bioRxiv, SSRN and PubMed Central shows a sharp rise in non-existent references after widespread LLM adoption, with a conservative estimate of 146,932 hallucinated citations in 2025 alone.")


# ----------------------------------------------------------------------------
def authors_short(authors, key):
    fam = [a[0] for a in authors if a[0]]
    if key == "osc2015reproducibility":
        return "Open Science Collaboration"
    if key == "nasem2019reproducibility":
        return "National Academies of Sciences, Engineering, and Medicine"
    if not fam:
        return "n/a"
    if len(fam) == 1:
        return fam[0]
    if len(fam) == 2:
        return f"{fam[0]} & {fam[1]}"
    return f"{fam[0]} et al."


def bibesc(s):
    return (s.replace("&", "\\&").replace("%", "\\%").replace("#", "\\#").replace("_", "\\_")
             .replace("τ", "$\\tau$").replace("$τ$", "$\\tau$"))


rows, bibs, counts = [], [], {}
for e in E:
    doi, aid, s2id = e.get("doi"), e.get("arxiv"), e.get("s2")
    src = []
    if doi:
        m = crossref(doi); src.append(m["url"])
        res = DOIRES.get(doi); src.append(f"doi.org HEAD {res}")
        if aid:
            a = arxiv(aid); src.append(a["url"])
        venue = m["container"] or ("National Academies Press" if e["key"] == "nasem2019reproducibility" else "")
        verified = m["verified"]
        meta = m
    elif aid:
        meta = arxiv(aid); src.append(meta["url"]); verified = meta["verified"]
        vn, vt, _ = s2_venue("arXiv:" + aid)
        s2url = "https://api.semanticscholar.org/graph/v1/paper/arXiv:" + aid
        if e["vtype"] == "conference":
            venue = {"chen2025scienceagentbench": "ICLR 2025", "majumder2025discoverybench": "ICLR 2025",
                     "jing2025dsbench": "ICLR 2025", "chan2025mlebench": "ICLR 2025",
                     "huang2024mlagentbench": "ICML 2024", "gu2024blade": "EMNLP 2024",
                     "jimenez2024swebench": "ICLR 2024", "starace2025paperbench": "ICML 2025",
                     "liu2024agentbench": "ICLR 2024", "zheng2023mtbench": "NeurIPS 2023 Datasets and Benchmarks",
                     "thakur2025judgingjudges": "GEM2 Workshop, ACL 2025",
                     "panickssery2024selfpreference": "NeurIPS 2024", "lubars2019delegability": "NeurIPS 2019",
                     "wang2023selfconsistency": "ICLR 2023", "huang2024selfcorrect": "ICLR 2024",
                     "sakai2026hallucitation": "ACL 2026 (per Semantic Scholar venue record)"}[e["key"]]
            src.append(s2url + f" (venue={vn!r}, type={vt!r})")
            if meta.get("jref"):
                src.append("arXiv journal_ref: " + meta["jref"])
        elif e["vtype"] == "journal":
            venue = {"kapoor2024agentsthatmatter": "Transactions on Machine Learning Research",
                     "siegel2024corebench": "Transactions on Machine Learning Research"}[e["key"]]
            src.append(s2url + f" (venue={vn!r})")
        else:
            venue = "arXiv preprint"
    else:  # Semantic Scholar only
        fn = {"gray1986whystop": "s2_gray1986_search_observed.json",
              "sigelman2010dapper": "s2_dapper2010.json"}[e["key"]]
        p = os.path.join(CACHE, fn)
        if fn.endswith(".jsonl"):
            rec = None
            for line in open(p):
                line = line.strip()
                if not line: continue
                try:
                    d = json.loads(line)
                except Exception:
                    continue
                for x in d.get("data", []):
                    if x.get("paperId") == s2id: rec = x
        else:
            rec = json.load(open(p))
        meta = dict(title=rec["title"], authors=[(a["name"].split()[-1], " ".join(a["name"].split()[:-1])) for a in rec["authors"]],
                    year=rec["year"], container=rec.get("venue") or "")
        venue = rec.get("venue") or ("Google Technical Report dapper-2010-1" if e["key"] == "sigelman2010dapper" else "")
        src.append("https://api.semanticscholar.org/graph/v1/paper/" + s2id + f" (record cached in {fn})")
        verified = mtime_utc(p)
    if e.get("peer_src"):
        src.append(e["peer_src"])
    ident = ("doi:" + doi) if doi else (("arXiv:" + aid) if aid else "s2:" + s2id)
    counts[e["stream"]] = counts.get(e["stream"], 0) + 1
    rows.append(dict(ref_key=e["key"], stream=e["stream"], authors_short=authors_short(meta["authors"], e["key"]),
                     year=meta["year"], title=meta["title"], venue=venue, venue_type=e["vtype"],
                     doi_or_arxiv=ident, verification_source=" ; ".join(src), verified_utc=verified,
                     relevance_to_wp29=e["rel"], key_claim_or_finding=e["claim"], peer_reviewed=e["peer"]))

    # ---- BibTeX ----
    if e["key"] == "osc2015reproducibility":
        au = "{Open Science Collaboration}"
    elif e["key"] == "nasem2019reproducibility":
        au = "{National Academies of Sciences, Engineering, and Medicine}"
    elif doi:
        au = " and ".join(f"{f}, {g}" if g else f for f, g in meta["authors"])
    else:
        au = " and ".join(f"{g} {f}".strip() for f, g in meta["authors"])
    au = bibesc(au)
    ti = bibesc(meta["title"])
    f = [f"  author = {{{au}}}", f"  title = {{{{{ti}}}}}", f"  year = {{{meta['year']}}}"]
    if e["vtype"] == "journal":
        btype = "article"; f.append(f"  journal = {{{bibesc(venue)}}}")
        if doi and meta.get("volume"): f.append(f"  volume = {{{meta['volume']}}}")
        if doi and meta.get("issue"): f.append(f"  number = {{{meta['issue']}}}")
        if doi and meta.get("pages"): f.append(f"  pages = {{{meta['pages'].replace('-', '--')}}}")
    elif e["vtype"] == "conference":
        btype = "inproceedings"; f.append(f"  booktitle = {{{bibesc(venue)}}}")
        if doi and meta.get("pages"): f.append(f"  pages = {{{meta['pages'].replace('-', '--')}}}")
    elif e["vtype"] == "report" and e["key"] == "nasem2019reproducibility":
        btype = "book"; f.append("  publisher = {The National Academies Press}"); f.append("  address = {Washington, DC}")
    elif e["vtype"] == "report":
        btype = "techreport"; f.append("  institution = {Google}"); f.append("  number = {dapper-2010-1}")
    else:
        btype = "misc"; f.append(f"  howpublished = {{arXiv preprint arXiv:{aid}}}")
    if doi:
        f.append(f"  doi = {{{doi}}}")
    if aid:
        f.append(f"  eprint = {{{aid}}}"); f.append("  archivePrefix = {arXiv}")
    comment = f"% verified via {src[0].split(' (')[0]} on {verified[:10]}"
    bibs.append((comment, f"@{btype}{{{e['key']},\n" + ",\n".join(f) + "\n}"))

cols = ["ref_key", "stream", "authors_short", "year", "title", "venue", "venue_type", "doi_or_arxiv",
        "verification_source", "verified_utc", "relevance_to_wp29", "key_claim_or_finding", "peer_reviewed"]
with open(os.path.join(HERE, "literature_matrix.csv"), "w", newline="", encoding="utf-8") as fh:
    w = csv.DictWriter(fh, fieldnames=cols); w.writeheader(); w.writerows(rows)

with open(os.path.join(HERE, "references_verified.bib"), "w", encoding="utf-8") as fh:
    for i, (c, b) in enumerate(bibs):
        if i == 0:
            # No '%' line may precede the first entry: its verification comment follows it.
            fh.write(b + "\n" + c + "  (comment placed after the first entry so that no % line precedes it)\n\n")
        else:
            fh.write(c + "\n" + b + "\n\n")

print("entries:", len(rows), "per stream:", dict(sorted(counts.items())))
print("peer_reviewed:", {k: sum(1 for r in rows if r["peer_reviewed"] == k) for k in ("yes", "no", "unknown")})