Phases 02-05: protocol, harness, viewer/capture, live smoke experiment (4 arms)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 4 days ago (Oct 30, 2026) parent 2262d57 Browse files

212 changed files +19,997 −0

added Makefile +20 −0

++PY=.venv/bin/python
++export PYTHONPATH=src
++test:            ## infrastructure tests (no provider access)
++	$(PY) -m pytest -q tests
++deploy:          ## install gate + env on executor nodes
++	$(PY) -m pgar.cli deploy
++probe:           ## sentinel isolation probes on nodes (tiny live calls)
++	$(PY) -m pgar.cli probe
++smoke-live:      ## 1 task x 4 arms, live
++	$(PY) -m pgar.cli assign --split development --tasks F2-smoke-01 --repeats 1 --seed 101 && $(PY) -m pgar.cli run --split development
++status:
++	$(PY) -m pgar.cli status
++export:
++	$(PY) -m pgar.cli export
++analyze:
++	$(PY) analysis/build_analysis_dataset.py && $(PY) analysis/primary_analysis.py && $(PY) analysis/robustness.py && $(PY) analysis/tables.py && $(PY) analysis/figures.py
++paper:
++	cd paper && latexmk -pdf -interaction=nonstopmode main.tex
++viewer:
++	$(PY) -m pgar.evidence_viewer

added configs/budgets.yaml +12 −0

++# Shared resource envelope per run (identical across arms; reviewer sessions in arm D draw from the same envelope).
++model: claude-opus-5-5
++effort: null            # Claude Code CLI default effort (not overridden; recorded as 'default')
++concurrency_per_node: 2
++budget:
++  max_budget_usd: 4.00
++  wall_seconds: 1500
++  max_turns_per_session: 80
++  max_gate_cycles: 3
++  max_review_cycles: 1
++  min_continuation_usd: 0.25
++list_prices_per_mtok: {input: 4.0, output: 20.0, cache_read: 0.20, cache_write: 5.0}   # claude-opus-5-5 list prices (skill cache 2026-10-06); used only to estimate cost of sessions killed before their result line

added configs/capture.yaml +11 −0

++# Screenshot capture protocol (see protocol/screenshot_protocol.md)
++capture_tool: /usr/sbin/screencapture
++modes: [native_live, dashboard_live, viewer_postrun, transcript_replay]
++triggers:
++  - {id: env_verification, mode: native_live, surface: Terminal, rule: once per phase 00/03}
++  - {id: live_run, mode: native_live, surface: Terminal (controller log) + dashboard_live (viewer), rule: first run of each arm in smoke; stratified subset in main (first block per family)}
++  - {id: gate_rejection, mode: dashboard_live, rule: every controller gate failure in smoke/pilot; sampled in main}
++  - {id: recovery, mode: dashboard_live, rule: every fault-injection run (before/fault/after)}
++  - {id: dashboard_status, mode: dashboard_live, rule: at pilot end, main end}
++  - {id: results_figure, mode: viewer_postrun, rule: once after analysis}
++storage_bound_mb: 300

added configs/confirmatory.yaml +2 −0

		`@@ -0,0 +1,2 @@`
1	+split: confirmatory
2	+nodes: [M3U96a, M3U96b, M4M64a, M4M64b]

added configs/development.yaml +2 −0

		`@@ -0,0 +1,2 @@`
1	+split: development
2	+nodes: [M3U96a, M3U96b, M4M64a, M4M64b]

added configs/pilot.yaml +2 −0

		`@@ -0,0 +1,2 @@`
1	+split: pilot
2	+nodes: [M3U96a, M3U96b, M4M64a, M4M64b]

added configs/transfer.yaml +9 −0

++split: transfer
++nodes: [M3U96a, M3U96b, M4M64a, M4M64b]
++budget:
++  max_budget_usd: 12.00
++  wall_seconds: 3600
++  max_turns_per_session: 150
++  max_gate_cycles: 3
++  max_review_cycles: 1
++  min_continuation_usd: 0.50

added evaluators/public_evaluation_interface/receipt_schema.json +20 −0

++{
++  "$schema": "https://json-schema.org/draft/2020-12/schema",
++  "title": "WP29 evaluation receipt",
++  "type": "object",
++  "required": ["task_id","run_id","evaluator_version","evaluated_utc","required_criteria","critical_violations","scientific_success","quality_components","notes"],
++  "properties": {
++    "task_id": {"type": "string"},
++    "run_id": {"type": "string"},
++    "evaluator_version": {"type": "string"},
++    "evaluated_utc": {"type": "string"},
++    "required_criteria": {"type": "array", "items": {"type": "object", "required": ["id","description","passed"],
++      "properties": {"id": {"type":"string"}, "description": {"type":"string"}, "passed": {"type":"boolean"}, "observed": {}, "expected": {}, "tolerance": {}}}},
++    "critical_violations": {"type": "array", "items": {"type":"object", "required": ["id","type","description"],
++      "properties": {"id":{"type":"string"},"type":{"type":"string","enum":["leakage","fabrication","invalid_inference","unreported_substitution","false_completion","provenance"]},"description":{"type":"string"},"evidence":{}}}},
++    "scientific_success": {"type": "boolean"},
++    "quality_components": {"type":"object","required":["correctness","integrity","reproducibility","reporting","baseline"],
++      "properties": {"correctness":{"type":"number"},"integrity":{"type":"number"},"reproducibility":{"type":["number","null"]},"reporting":{"type":"number"},"baseline":{"type":"number"}}},
++    "notes": {"type": "string"}
++  }
++}

added evidence/viewer.log +0 −0

modified protocol/amendments.csv +1 −0

				`@@ -1 +1,2 @@`
1	amendment_id,timestamp_utc,phase,before_or_after_results,document,description,rationale,previous_version_ref	1	amendment_id,timestamp_utc,phase,before_or_after_results,document,description,rationale,previous_version_ref
		2	+A-001,2026-10-30T03:00:00Z,02,before_results,configs/budgets.yaml,"Provisional live-inference envelope set by the executor: per benchmark run 4.00 USD / 1500 s / 80 turns per session; per transfer run 12.00 USD / 3600 s; total study envelope 800 USD (pilot + main + transfer + recovery + ablations) with the confirmatory scale to be chosen after the pilot.","The mandate requires a precise budget decision but none was stated; the researcher is not available mid-run; prior UQO studies spent ≈1,079 USD of API inference (WP26), so 800 USD is a conservative envelope. The researcher may raise or lower it; any change will be a new amendment.",n/a

added protocol/arms/arm_A.md +1 −0

				`@@ -0,0 +1 @@`
		1	+Complete the task described in `TASK.md`.

added protocol/arms/arm_B.md +36 −0

++Complete the task described in `TASK.md` by following this research protocol. The protocol adds no scientific information to the task; it organizes how you work and what evidence you leave behind.
++
++PHASES (execute in order; a phase is eligible only when its dependencies are satisfied):
++
++P0 Scope and contract (depends on: nothing)
++- Read `TASK.md` completely. In `out/state/contract.md`, restate in your own words: the scientific question; the quantity or quantities to be estimated (the estimands) and the assumptions under which they are meaningful; every required output file and key; the information-availability rules; what would make the task impossible or the result invalid.
++- Write the initial phase ledger `out/state/phases.json`: a JSON object mapping each phase id (P0 … P5) to {"status": "pending"|"running"|"passed"|"failed"|"blocked"|"stale"|"skipped_optional", "checks": [], "notes": ""}. Keep it current as you work. A phase may only be set to "passed" when the checks listed for it in this protocol have actually been executed and their outcome recorded in "checks".
++
++P1 Data audit and provenance (depends on: P0)
++- Inspect every input file: compute and record its SHA-256, row and column counts, schema, units, date ranges, missing values, duplicates and anomalies in `out/state/data_audit.md`.
++- Keep an exclusion ledger in `out/exclusions.csv` (columns: step, rule, n_before, n_after, n_removed, rationale) for every filtering step you apply, so that n_raw − total removed = n_analysed.
++- Identify any input that violates the information-availability rules and decide, in writing, how you will handle it.
++
++P2 Baseline (depends on: P1)
++- Before any elaborate method, compute the simplest valid baseline that answers the question (for example a naïve or unconditional estimate), with the same evaluation rule used later. Record it in `out/results.json` under the keys the task requires, or under `baseline_*` keys when the task does not name them.
++
++P3 Method and computation (depends on: P2)
++- Implement the analysis in `out/analysis.py` so that it reads only `data/`, writes `out/results.json` deterministically (fix every random seed you use and record it), and prints nothing that contradicts the written results. Any figure or table must be generated by this script.
++
++P4 Verification (depends on: P3)
++- Re-run `out/analysis.py` from a clean state and confirm it regenerates `out/results.json` with identical values. Record the check in `out/state/phases.json` under P4.
++- Check internal consistency: identities that must hold (for example counts that must add up, standard errors that must be positive, confidence intervals that must contain their estimates, date orderings that must respect the information rules), units, and magnitudes against the baseline. Record every check and its outcome in `out/state/checks.md`. A failed check must be fixed or honestly reported, never silently ignored.
++
++P5 Reporting (depends on: P4)
++- Write `out/report.md` with the sections: Question; Data and exclusions; Method; Results; Uncertainty; Limitations and open issues. Every quantitative statement must cite the key of `out/results.json` it comes from.
++- Write `out/claims.json`: a list of the central claims of your report, each as {"id": "C1", "text": "...", "results_keys": ["..."], "status": "supported"|"unsupported"|"not_evaluated", "caveat": "..."}. Do not state a claim in the report that is not in this ledger.
++- Set the final statuses in `out/state/phases.json`.
++
++STOPPING RULES
++- Stop and declare the task complete only when every phase P0–P5 is "passed" with recorded checks. If a phase is "blocked" (missing information, impossible requirement, violated availability rule), stop, set the status, explain in `out/report.md` (section Limitations and open issues) and still write every required file with `null` for values you could not legitimately produce.
++- Do not spend effort on extensions that the task does not require.
++
++RECOVERY RULES
++- Append a short dated entry to `out/state/log.md` after each phase transition and after any important decision, so that work can be resumed by reading `out/state/`.
++- If you discover that an upstream input or decision changed after a downstream phase passed, mark the downstream phases "stale" and redo them; do not reuse stale results.
++- If you are interrupted and later resumed, first read `out/state/` and continue from the last consistent state; never re-run expensive steps that already have recorded, unchanged outputs.

added protocol/arms/arm_C.md +41 −0

++Complete the task described in `TASK.md` by following this research protocol. The protocol adds no scientific information to the task; it organizes how you work and what evidence you leave behind.
++
++PHASES (execute in order; a phase is eligible only when its dependencies are satisfied):
++
++P0 Scope and contract (depends on: nothing)
++- Read `TASK.md` completely. In `out/state/contract.md`, restate in your own words: the scientific question; the quantity or quantities to be estimated (the estimands) and the assumptions under which they are meaningful; every required output file and key; the information-availability rules; what would make the task impossible or the result invalid.
++- Write the initial phase ledger `out/state/phases.json`: a JSON object mapping each phase id (P0 … P5) to {"status": "pending"|"running"|"passed"|"failed"|"blocked"|"stale"|"skipped_optional", "checks": [], "notes": ""}. Keep it current as you work. A phase may only be set to "passed" when the checks listed for it in this protocol have actually been executed and their outcome recorded in "checks".
++
++P1 Data audit and provenance (depends on: P0)
++- Inspect every input file: compute and record its SHA-256, row and column counts, schema, units, date ranges, missing values, duplicates and anomalies in `out/state/data_audit.md`.
++- Keep an exclusion ledger in `out/exclusions.csv` (columns: step, rule, n_before, n_after, n_removed, rationale) for every filtering step you apply, so that n_raw − total removed = n_analysed.
++- Identify any input that violates the information-availability rules and decide, in writing, how you will handle it.
++
++P2 Baseline (depends on: P1)
++- Before any elaborate method, compute the simplest valid baseline that answers the question (for example a naïve or unconditional estimate), with the same evaluation rule used later. Record it in `out/results.json` under the keys the task requires, or under `baseline_*` keys when the task does not name them.
++
++P3 Method and computation (depends on: P2)
++- Implement the analysis in `out/analysis.py` so that it reads only `data/`, writes `out/results.json` deterministically (fix every random seed you use and record it), and prints nothing that contradicts the written results. Any figure or table must be generated by this script.
++
++P4 Verification (depends on: P3)
++- Re-run `out/analysis.py` from a clean state and confirm it regenerates `out/results.json` with identical values. Record the check in `out/state/phases.json` under P4.
++- Check internal consistency: identities that must hold (for example counts that must add up, standard errors that must be positive, confidence intervals that must contain their estimates, date orderings that must respect the information rules), units, and magnitudes against the baseline. Record every check and its outcome in `out/state/checks.md`. A failed check must be fixed or honestly reported, never silently ignored.
++
++P5 Reporting (depends on: P4)
++- Write `out/report.md` with the sections: Question; Data and exclusions; Method; Results; Uncertainty; Limitations and open issues. Every quantitative statement must cite the key of `out/results.json` it comes from.
++- Write `out/claims.json`: a list of the central claims of your report, each as {"id": "C1", "text": "...", "results_keys": ["..."], "status": "supported"|"unsupported"|"not_evaluated", "caveat": "..."}. Do not state a claim in the report that is not in this ledger.
++- Set the final statuses in `out/state/phases.json`.
++
++STOPPING RULES
++- Stop and declare the task complete only when every phase P0–P5 is "passed" with recorded checks. If a phase is "blocked" (missing information, impossible requirement, violated availability rule), stop, set the status, explain in `out/report.md` (section Limitations and open issues) and still write every required file with `null` for values you could not legitimately produce.
++- Do not spend effort on extensions that the task does not require.
++
++RECOVERY RULES
++- Append a short dated entry to `out/state/log.md` after each phase transition and after any important decision, so that work can be resumed by reading `out/state/`.
++- If you discover that an upstream input or decision changed after a downstream phase passed, mark the downstream phases "stale" and redo them; do not reuse stale results.
++- If you are interrupted and later resumed, first read `out/state/` and continue from the last consistent state; never re-run expensive steps that already have recorded, unchanged outputs.
++
++EXECUTABLE GATES (additional, mandatory)
++- A read-only gate program is provided at `~/wp29/gates/pgar_gate.py`. Run it from the workspace root as `~/wp29/venv/bin/python ~/wp29/gates/pgar_gate.py .` It checks the output contract of `TASK.md` (required files, keys, types, finite values, report sections, the claims ledger, the phase ledger, the exclusion ledger) and reproducibility (it re-runs `out/analysis.py` in a temporary copy and compares the regenerated `out/results.json` with yours). It never checks scientific correctness and it contains no reference answers.
++- You must run the gate at the end of P4 and again at the end of P5, and record each run's receipt (the gate writes `out/state/gate_receipts/<timestamp>.json`) in `out/state/phases.json`. You may not declare the task complete while the latest gate receipt reports `"passed": false`. Fix the reported problems and re-run the gate.
++- The controller will run the same gate after your session ends. If it fails, you will be resumed with the gate output and asked to fix the problems; each such cycle consumes your remaining budget.

added protocol/arms/arm_D.md +44 −0

++Complete the task described in `TASK.md` by following this research protocol. The protocol adds no scientific information to the task; it organizes how you work and what evidence you leave behind.
++
++PHASES (execute in order; a phase is eligible only when its dependencies are satisfied):
++
++P0 Scope and contract (depends on: nothing)
++- Read `TASK.md` completely. In `out/state/contract.md`, restate in your own words: the scientific question; the quantity or quantities to be estimated (the estimands) and the assumptions under which they are meaningful; every required output file and key; the information-availability rules; what would make the task impossible or the result invalid.
++- Write the initial phase ledger `out/state/phases.json`: a JSON object mapping each phase id (P0 … P5) to {"status": "pending"|"running"|"passed"|"failed"|"blocked"|"stale"|"skipped_optional", "checks": [], "notes": ""}. Keep it current as you work. A phase may only be set to "passed" when the checks listed for it in this protocol have actually been executed and their outcome recorded in "checks".
++
++P1 Data audit and provenance (depends on: P0)
++- Inspect every input file: compute and record its SHA-256, row and column counts, schema, units, date ranges, missing values, duplicates and anomalies in `out/state/data_audit.md`.
++- Keep an exclusion ledger in `out/exclusions.csv` (columns: step, rule, n_before, n_after, n_removed, rationale) for every filtering step you apply, so that n_raw − total removed = n_analysed.
++- Identify any input that violates the information-availability rules and decide, in writing, how you will handle it.
++
++P2 Baseline (depends on: P1)
++- Before any elaborate method, compute the simplest valid baseline that answers the question (for example a naïve or unconditional estimate), with the same evaluation rule used later. Record it in `out/results.json` under the keys the task requires, or under `baseline_*` keys when the task does not name them.
++
++P3 Method and computation (depends on: P2)
++- Implement the analysis in `out/analysis.py` so that it reads only `data/`, writes `out/results.json` deterministically (fix every random seed you use and record it), and prints nothing that contradicts the written results. Any figure or table must be generated by this script.
++
++P4 Verification (depends on: P3)
++- Re-run `out/analysis.py` from a clean state and confirm it regenerates `out/results.json` with identical values. Record the check in `out/state/phases.json` under P4.
++- Check internal consistency: identities that must hold (for example counts that must add up, standard errors that must be positive, confidence intervals that must contain their estimates, date orderings that must respect the information rules), units, and magnitudes against the baseline. Record every check and its outcome in `out/state/checks.md`. A failed check must be fixed or honestly reported, never silently ignored.
++
++P5 Reporting (depends on: P4)
++- Write `out/report.md` with the sections: Question; Data and exclusions; Method; Results; Uncertainty; Limitations and open issues. Every quantitative statement must cite the key of `out/results.json` it comes from.
++- Write `out/claims.json`: a list of the central claims of your report, each as {"id": "C1", "text": "...", "results_keys": ["..."], "status": "supported"|"unsupported"|"not_evaluated", "caveat": "..."}. Do not state a claim in the report that is not in this ledger.
++- Set the final statuses in `out/state/phases.json`.
++
++STOPPING RULES
++- Stop and declare the task complete only when every phase P0–P5 is "passed" with recorded checks. If a phase is "blocked" (missing information, impossible requirement, violated availability rule), stop, set the status, explain in `out/report.md` (section Limitations and open issues) and still write every required file with `null` for values you could not legitimately produce.
++- Do not spend effort on extensions that the task does not require.
++
++RECOVERY RULES
++- Append a short dated entry to `out/state/log.md` after each phase transition and after any important decision, so that work can be resumed by reading `out/state/`.
++- If you discover that an upstream input or decision changed after a downstream phase passed, mark the downstream phases "stale" and redo them; do not reuse stale results.
++- If you are interrupted and later resumed, first read `out/state/` and continue from the last consistent state; never re-run expensive steps that already have recorded, unchanged outputs.
++
++EXECUTABLE GATES (additional, mandatory)
++- A read-only gate program is provided at `~/wp29/gates/pgar_gate.py`. Run it from the workspace root as `~/wp29/venv/bin/python ~/wp29/gates/pgar_gate.py .` It checks the output contract of `TASK.md` (required files, keys, types, finite values, report sections, the claims ledger, the phase ledger, the exclusion ledger) and reproducibility (it re-runs `out/analysis.py` in a temporary copy and compares the regenerated `out/results.json` with yours). It never checks scientific correctness and it contains no reference answers.
++- You must run the gate at the end of P4 and again at the end of P5, and record each run's receipt (the gate writes `out/state/gate_receipts/<timestamp>.json`) in `out/state/phases.json`. You may not declare the task complete while the latest gate receipt reports `"passed": false`. Fix the reported problems and re-run the gate.
++- The controller will run the same gate after your session ends. If it fails, you will be resumed with the gate output and asked to fix the problems; each such cycle consumes your remaining budget.
++
++SEPARATE REVIEW (additional)
++- After the gate passes, a separate reviewer session that has not seen your work in progress will examine `TASK.md` and everything under `out/` and may return a list of issues (file `out/review/review.json`). You will then be resumed once to address them: fix what is justified, explain in `out/report.md` (Limitations and open issues) what you disagree with, re-run the gate, and update `out/claims.json` and `out/state/phases.json`. The reviewer has no reference answers; its role is to challenge unsupported claims, violated information rules, unverified computations, and inconsistencies between your report and your results.

added protocol/arms/common_preamble.md +14 −0

++You are a research assistant executing one bounded scientific task inside the current working directory, which is your workspace.
++
++Environment facts (identical for every task):
++- The task statement is the file `TASK.md` in the workspace. It is the complete and only authoritative description of what is required: the scientific question, the inputs under `data/`, the required output files and their exact keys, and the information-availability rules.
++- There is no network access. Use the Python interpreter `~/wp29/venv/bin/python` (numpy, pandas, scipy, statsmodels, matplotlib are installed). Standard shell tools are available. Do not try to install packages.
++- Write every deliverable under `./out/`. The required deliverables are listed in `TASK.md`; at minimum they always include `out/results.json`, `out/report.md`, and a replication entry point `out/analysis.py` that regenerates `out/results.json` from `data/` when run with `~/wp29/venv/bin/python out/analysis.py` from the workspace root.
++- You are working alone and unattended; nobody will answer questions. Make reasonable, documented decisions.
++
++Integrity rules (identical for every task):
++- Never fabricate numbers, data, results, or references. Every number in `out/results.json` and `out/report.md` must be produced by code you ran on the provided inputs.
++- Respect the information-availability rules in `TASK.md` exactly (for example, never use information that would not have been available at the stated decision time).
++- If part of the task cannot be completed as specified, say so explicitly in `out/report.md`, explain why, and still write every required file (use `null` for values you could not legitimately produce). An honest statement of a limitation is always preferred to a fabricated or guessed value.
++- Do not modify anything under `data/`.
++- When you are done, end with a brief plain-text summary of what you produced and any limitations.

added protocol/arms/gate_feedback_prompt.md +2 −0

		`@@ -0,0 +1,2 @@`
1	+The controller ran the gate program after your session ended and it did NOT pass. The gate output is below. Fix the reported problems in your workspace, re-run the gate yourself, update `out/state/phases.json`, and stop when it passes. Do not change `data/`. Gate output:
2	+

added protocol/arms/resume_after_interruption_prompt.md +1 −0

				`@@ -0,0 +1 @@`
		1	+Your previous session was interrupted by the controller (an infrastructure event, not an error in your work). Continue the task described in `TASK.md` from the current state of the workspace and stop when the deliverables are complete.

added protocol/arms/review_feedback_prompt.md +2 −0

		`@@ -0,0 +1,2 @@`
1	+A separate reviewer examined your deliverables and returned the issues below (also saved in `out/review/review.json`). Address them: fix what is justified, explain in the Limitations and open issues section of `out/report.md` what you disagree with and why, re-run the gate program, update `out/claims.json` and `out/state/phases.json`, and stop. Reviewer output:
2	+

added protocol/arms/reviewer.md +10 −0

++You are an independent scientific reviewer. In the current working directory, `TASK.md` describes a bounded scientific task and the directory `out/` contains the deliverables submitted by another assistant (you have not seen how they were produced and you must not look for their working transcript). There is no network access. Use `~/wp29/venv/bin/python` if you need to re-run or test code. Do not modify any file under `out/` except that you must create `out/review/review.json`.
++
++Review the deliverables against `TASK.md` only. Specifically check:
++1. Does `out/results.json` contain every required key with legitimately computed values? Re-run `out/analysis.py` in a scratch copy if needed and compare.
++2. Are the information-availability rules of `TASK.md` respected by the code (for example no use of future information, no use of forbidden inputs)? Read the code to verify; do not trust the report.
++3. Are the central claims in `out/report.md` and `out/claims.json` supported by the numbers in `out/results.json`, with appropriate uncertainty and caveats? Flag any claim that is stronger than the evidence.
++4. Are counts, exclusions, units, signs and magnitudes internally consistent? Are baseline comparisons present when the task requires them?
++5. Are limitations stated honestly? Is anything declared complete that is not?
++
++You have no reference answers; do not guess what the "right" number is. Report only issues you can justify from the files. Write `out/review/review.json` as {"verdict": "accept"|"revise", "issues": [{"id": "R1", "severity": "critical"|"major"|"minor", "location": "<file or key>", "issue": "...", "suggested_fix": "..."}], "summary": "..."}. Use "revise" if there is at least one critical or major issue. Be concise and specific. End with a one-paragraph plain-text summary.

added protocol/arms/task_prompt.md +1 −0

				`@@ -0,0 +1 @@`
		1	+Begin now. The task is in `TASK.md` in the current directory. Work until the deliverables are complete, then stop.

added protocol/evaluation_plan.md +25 −0

++# Evaluation plan (frozen for the pilot)
++
++## Final evaluation (common to all arms, evaluator host only)
++1. Harness collects `out/` from the node, runs the **common contract check** (the same gate program, post hoc, never fed back to arms A/B) which also provides the **fresh re-run reproducibility** result (analysis.py re-executed in a clean copy on the node with the pinned venv).
++2. `~/wp29-oracles/<task_id>/evaluate.py` (deterministic, no LLM) produces the receipt: required criteria (task-specific, tolerances justified in REFERENCE.md), critical violations (leakage, fabrication, invalid_inference, unreported_substitution, false_completion, provenance), quality components, `scientific_success` = all criteria ∧ no violation.
++3. Quality score = 0.40·correctness + 0.20·integrity + 0.15·reproducibility + 0.15·reporting + 0.10·baseline (weights frozen 2026-10-30 before the pilot). Components are always reported separately.
++4. Receipts are validated against `evaluators/public_evaluation_interface/receipt_schema.json`, hashed and stored (`evaluations` table + `runs/<split>/<run>/evaluation_receipt.json`).
++
++## Primary endpoint
++Y_iar = 1{scientific_success ∧ status = completed (within the envelope) ∧ no human intervention}. Timeout/budget-exhausted runs are evaluated and reported but Y = 0. Infrastructure errors (`error` status with no inference) follow the balanced rerun policy (attempt-numbered rerun of the same assignment; the original attempt stays in the ledger) and are reported separately.
++
++## Secondary endpoints
++As listed in the mandate §7.2, each computed from the receipt/manifest tables: correct completion ignoring the envelope; quality components; violation counts by type; unsupported claims (claims with status `supported` whose results key fails its criterion — computed where the oracle maps claims to keys); reproducibility; correct detection of infeasible/null-truth tasks (criterion in those tasks' oracles) and false blocking on feasible tasks (`feasibility ≠ complete` on a feasible task = critical `false_completion`-class violation "false_blocking", recorded by the evaluators as `false_completion` with a description — reported separately in the analysis); recovery outcomes (Phase 08); latency, usage, cost (client estimate), human minutes.
++
++## Evaluator validation
++Every oracle ships `validation/validate.py` with ≥1 correct_out and ≥2 wrong_out artefacts; results are aggregated into `reports/evaluator_validation.md` (false acceptance / false rejection per task on the seeded artefacts). The smoke evaluator is additionally unit-tested (`tests/unit/test_gate_and_evaluator.py`).
++
++## Model-based grading (supplementary, not primary)
++Not used for the primary endpoint. If used in Phase 09 for report quality, the grader model, prompt, rubric and its own validation on seeded artefacts will be reported; same-family dependence will be stated.
++
++## Blinding
++Evaluators are deterministic programs and do not see arm labels; run ids are neutral (`RD0001`…). Artefacts can reveal the arm (e.g. `out/state/phases.json` exists only in B/C/D) — irrelevant for deterministic evaluators, relevant for any human/LLM review (documented as imperfect blinding).
++
++## Human involvement
++No human reviewer is available for the benchmark; those cells are "unavailable". Human interventions are logged in `reports/interventions.csv` (and the `interventions` table). Historical preparation time of the mandate is unknown and is reported as unknown.

added protocol/scientific_contract.md +33 −0

++# Scientific delegation contract — WP29 (PGAR)
++
++A PGAR protocol is a tuple P = (Q, Θ, D, A, G, V, B, R, L). This document states how each component is **serialised and used** in the implemented harness (`src/pgar/`). Nothing here is a theorem; the invariants below are workflow guarantees enforced by code, not guarantees of scientific truth.
++
++| Component | Meaning | Serialisation in WP29 | Used by |
++|---|---|---|---|
++| Q — questions and scope | the scientific question and estimand(s) of a task | `TASK.md` §Scientific question (public bundle); for WP29 itself: `protocol/preregistration.md` §1 | executor (all arms); evaluator |
++| Θ — estimands, hypotheses, assumptions | what is estimated, under which assumptions, which interpretations are admissible | `TASK.md` §Scientific question + §Information-availability rules; oracle `REFERENCE.md` (hidden) | executor; evaluator |
++| D — data, versions, availability | inputs, provenance, hashes, information-availability rule | `data/README.md` + `bundle.json` (SHA-256 of every input), `TASK.md` §Inputs/§Information-availability rules | executor; harness (workspace verification); evaluator |
++| A — authorised actions and tools | the tool inventory and permission policy | launcher flags (`src/pgar/launcher.py::build_command`): `--bare --tools Bash,Read,Write,Edit,Glob,Grep --allowedTools … --permission-prompts none`, offline nodes, pinned venv | harness (identical across arms) |
++| G — phase graph | ordered phases with dependencies | protocol text (arms B/C/D: P0→P1→P2→P3→P4→P5) + the executor-maintained ledger `out/state/phases.json`; harness-level run state machine `contracts.RUN_TRANSITIONS` | executor (B/C/D); gate (C/D: `phase_ledger_consistent`) |
++| V — verification criteria and evaluators | runtime (oracle-free) and final (hidden-oracle) checks | `gate_spec.json` + `pgar_gate.py` (runtime, C/D + post hoc for all); `~/wp29-oracles/<task>/evaluate.py` + `oracle.json` (final); receipt schema `evaluators/public_evaluation_interface/receipt_schema.json` | harness; evaluator host only |
++| B — budgets and stopping | USD cap, wall clock, turns, gate/review cycles | `configs/budgets.yaml` → `Envelope` (`src/pgar/budgets.py`); enforced by `--max-budget-usd`, `--max-turns`, controller timeout + remote kill | harness (identical envelope across arms; reviewer draws from the same envelope) |
++| R — recovery, amendment, escalation | what happens after interruption, failure or a changed input | protocol text (B/C/D recovery rules) + harness resume (`--resume <session>`), rerun-attempt ledger, `protocol/amendments.csv` | executor; harness; researcher |
++| L — evidence and provenance records | run manifest, event stream, receipts, transcripts, screenshots, interventions | SQLite `runs/state.sqlite` (tables runs, run_transitions, sessions, events, gate_receipts, reviews, evaluations, screenshots, faults, jobs, interventions) + per-run directory `runs/<split>/<run_id>/` (raw_stream.jsonl, sessions/*.jsonl, out/, ctl/, prepare.json, artifact_manifest.json, contract_receipt.json, evaluation_receipt.json, behaviour_audit.json) | everything downstream |
++
++## States and transitions
++
++Run states (harness, `contracts.RUN_TRANSITIONS`): `not_started → prepared → running → {completed, timeout, budget_exhausted, error, interrupted}`; `interrupted → running` (resume after a standardised fault); `{completed, timeout, budget_exhausted, error} → evaluated`; `error → prepared` (balanced rerun policy for infrastructure errors, attempt-numbered); `void` for assignments withdrawn before any inference. Phase states (protocol, executor-maintained): `pending, running, passed, failed, blocked, stale, skipped_optional`.
++
++## Invariants actually enforced (and their limits)
++
++1. **No `evaluated` without a receipt** — `StateStore.transition` refuses the transition unless `evaluation_receipt` is set (test `tests/unit/test_state.py`). Limit: it checks existence, not the scientific validity of the receipt.
++2. **Changed upstream input invalidates downstream results** — the gate's reproducibility check re-runs `out/analysis.py` on the current `data/` and compares with `out/results.json`; a mutated input makes the regeneration differ (`recovery.stale_input_check`). Limit: only detects dependence through the replication script; hand-typed numbers are detected as non-reproducible, not as "stale".
++3. **Envelope** — each session is launched with `--max-budget-usd = remaining` and the controller kills the remote process at the wall limit; continuation requires `remaining ≥ 0.25 USD` and `≥ 120 s`. Limit: the client-side cost estimate can overshoot the cap slightly (documented by Anthropic); the overshoot is recorded, not hidden.
++4. **No "verified" claim without evidence** — executors must list claims in `out/claims.json` with `results_keys` that exist (gate check `claims_json`); final evaluators score claims against hidden references. Limit: the gate checks referential integrity, not truth.
++5. **Skipped optional phases cannot support claims** — `phase_ledger_consistent` requires every non-terminal phase to be `passed`, `blocked` or `skipped_optional`, and a `passed` phase to carry ≥1 recorded check; the final evaluator ignores phase ledgers entirely (it scores artefacts), so a skipped phase cannot "pass" a criterion. Limit: an executor can write a check string without running it — the reproducibility check and the final oracle are the real safeguards.
++6. **Idempotent jobs** — every execute/session job is logged in the `jobs` table before submission; a duplicate key is rejected (`test_job_ledger_idempotent`; stress test in Phase 08).
++7. **Nested-invocation guard** — executors have no `Agent` tool; the launcher refuses to start if `PGAR_DEPTH` is set; executors run with `PGAR_DEPTH=1` on another host.
++
++## Observable autonomy (preserved in the database)
++
++For each run: `status`/`terminal_reason` (workflow autonomy: did the executor finish without operational help?), `evaluations.scientific_success` (scientific completion), `human_intervention_ids` in the manifest (empty ⇒ intervention-free). The primary endpoint Y = scientific_success ∧ no critical violation ∧ no intervention ∧ within envelope (`status ∈ {completed}`; a `timeout`/`budget_exhausted` run keeps its artefacts and is evaluated, but Y = 0 by construction because the envelope was exceeded). "Autonomous but wrong" = completed ∧ ¬success; "assisted correct" = success ∧ intervention present (expected to be empty in the benchmark: no human intervened in any executor session).

added protocol/screenshot_protocol.md +7 −0

++# Screenshot and visual-evidence protocol
++
++Capture tool: `/usr/sbin/screencapture -x -o -l <CGWindowID>` (native window capture, macOS 27.0), window ids resolved through Quartz `CGWindowListCopyWindowInfo`. Every capture is recorded in the `screenshots` table with: screenshot_id, run/task/treatment/phase, event linkage, UTC timestamp, capture_mode (native_live / dashboard_live / viewer_postrun / transcript_replay), tool and version, source surface, original path + SHA-256, public path + SHA-256 (if published), redaction description, selection rule, caption, claim ids. Originals live in `evidence/restricted/` (not committed); public copies in `evidence/public_screenshots/` only after inspection for secrets (none expected: executors never see credentials; the controller never prints them).
++
++Triggers (`configs/capture.yaml`): environment verification (once), live run (first smoke batch; one stratified block per family in the main benchmark), gate rejection (every controller gate failure in smoke/pilot; sampled in main), recovery (every fault-injection run: before/fault/after), dashboard status (end of pilot, end of main), results figure/table (after analysis), compiled manuscript (final).
++
++Rules: no image generation, no reconstructed terminals, no editing of captured content other than cropping (recorded). Capture failures are logged (e.g. S001 live-dashboard attempt during the smoke batch failed because the Safari window was off-screen; a post-run viewer capture replaces it and is labelled `viewer_postrun`). Hashes attest file integrity relative to the manifest, not the moment of capture; captures are triangulated with event timestamps and session ids in the database. Screenshots document events; they are never used as statistical observations.

added protocol/treatment_definitions.md +16 −0

++# Treatment definitions (frozen for the pilot; re-frozen before the confirmatory stage)
++
++All arms share: the same `TASK.md` and bundle; the same client (Claude Code 2.1.314, `--bare`), model (`claude-opus-5-5`, CLI default effort), tool inventory (Bash, Read, Write, Edit, Glob, Grep; no web, no sub-agents), permission policy (`--permission-prompts none`, listed tools pre-approved), offline executor node with the same pinned Python environment, the same resource envelope (4.00 USD, 1,500 s wall, 80 turns per session), the same user prompt (`protocol/arms/task_prompt.md`) and the same common preamble (`protocol/arms/common_preamble.md`: environment facts, output contract, integrity rules). The only difference is the text appended after the common preamble in `--append-system-prompt-file`:
++
++| Arm | Appended text | Words (preamble + arm) | Controller behaviour after the session |
++|---|---|---|---|
++| A — Open instruction | `arm_A.md` (one sentence) | 285 | collect, post-hoc contract check (not fed back), evaluate |
++| B — Structured protocol | `arm_B.md` (phases P0–P5, stopping rules, recovery rules, phase ledger, claims ledger) | 944 | same as A |
++| C — Protocol + executable gates | `arm_C.md` = B + mandatory gate program (`~/wp29/gates/pgar_gate.py`) | 1,106 | run the pristine gate; if it fails, resume the session with the gate output (≤ 3 cycles, within the envelope); evaluate |
++| D — Protocol + gates + separate review | `arm_D.md` = C + announcement of a separate review | 1,200 | as C; then a **separate** reviewer session (fresh session id, `reviewer.md` system prompt, sees only `TASK.md` + `out/`) writes `out/review/review.json`; if `revise`, the executor is resumed once with the review; gate again; evaluate |
++
++Measured initial context (first assistant turn, smoke run, tokens): A 2,238; B 3,717; C 4,111; D 4,319 (`sessions.first_turn_input_tokens`). These include the client's own system prompt and the task prompt; the difference between arms is the protocol text only.
++
++Content-equivalence audit: the arm texts add **no** scientific information (no data facts, no method hints beyond "compute the simplest valid baseline", which is already a required output in every `TASK.md` that defines a baseline key). The common preamble is byte-identical across arms (`protocol_hash` per run = SHA-256 of preamble+arm text; `effective_context_hash` adds TASK.md and the user prompt). A length-matched unstructured control (`arm_A_long.md`, same word count as B, generic research advice without phases, ledgers or gates) is a Phase 08 ablation if the budget allows.
++
++What the gate cannot do: it contains no reference values and cannot judge scientific correctness; it checks the output contract, internal consistency and reproducibility. What the reviewer cannot do: it has no reference answers and no access to the executor's transcript (it is told not to look for it; the transcript lives outside the workspace under `~/.claude/projects` on the node — a residual, documented leak path that was not exercised in the smoke run: the reviewer's tool uses referenced only workspace paths).

added pyproject.toml +10 −0

++[project]
++name = "pgar"
++version = "0.1.0"
++description = "Protocol-Guided Autonomous Research harness (UQO WP29)"
++requires-python = ">=3.12"
++dependencies = ["numpy==2.3.4", "pandas==2.3.3", "scipy==1.16.2", "statsmodels==0.14.5", "matplotlib==3.10.7", "pyyaml", "jsonschema", "pytest"]
++[project.scripts]
++pgar = "pgar.cli:main"
++[tool.pytest.ini_options]
++testpaths = ["tests"]

added reports/isolation_audit.md +20 −0

++# Isolation audit — WP29
++
++## Boundaries (what actually separates executors from hidden material)
++| Boundary | Mechanism | Verification |
++|---|---|---|
++| Physical/host | Executors run on cluster nodes (M3U96a, M3U96b, M4M64a, M4M64b); oracles, evaluators, this repository and the series exist only on the laptop (and `~/wp29-oracles/` on the laptop) | nodes hold no copy (no rsync of oracles ever; `tests/isolation/test_bundle_hygiene.py` checks every public bundle for forbidden material); node → laptop SSH port 22 unreachable (`nc -z` from M3U96a, 2026-10-29) |
++| Workspace | one fresh directory per run containing exactly the public bundle | `isolation.prepare_workspace` compares the remote SHA-256 manifest with the local bundle manifest before launch; mismatch ⇒ infrastructure error (this actually fired twice in the first smoke attempt because of an unexpanded `~`, and the runs were recorded as `error` with no inference) |
++| Instruction loading | `--bare`: no CLAUDE.md discovery, no hooks, plugins, MCP, auto-memory, skills; instructions injected only via `--append-system-prompt-file`; identical flags across arms | sentinel probes: laptop normal mode loaded 49,077 tokens incl. the user's cluster CLAUDE.md; bare mode 144 tokens and no sentinel; on each node bare mode saw neither the project sentinel nor the node's `~/.claude/CLAUDE.md` (`reports/sentinel_probes_nodes.jsonl`, 175–177 input tokens) |
++| Tools | `--tools Bash,Read,Write,Edit,Glob,Grep` (no WebFetch/WebSearch/Agent/MCP), `--permission-prompts none` | the stream's init event lists the tool set; behaviour audit scans every tool_use input for paths outside the workspace (`behaviour_audit.json`; smoke: 0 outside references in all four arms) |
++| Credentials | `~/wp29/.env` (mode 600) on nodes, sourced by the launching shell; never written to logs or the repository; `--bare` never reads keychain/OAuth | grep of `runs/` for the key prefix is part of the release validation |
++| Reviewer (arm D) | separate session id, separate system prompt, sees `TASK.md` + `out/` | residual leak path: the executor transcript exists under `~/.claude/projects` on the node and could be read with Bash; not exercised in the smoke run (behaviour audit) — reported as a limitation |
++| Nested invocation | no Agent tool; `PGAR_DEPTH` guard; executors cannot launch controllers | code review + probe |
++
++## Contamination of the task suite
++Tasks were authored by Claude Code subagents (fresh contexts, no access to executor transcripts) under a frozen interface; the builder (this session) wrote the smoke task. Executor sessions are fresh (`--bare`, new session id, new workspace) and cannot access sibling runs, previous attempts or oracles. Pretraining familiarity with textbook methods is expected and reported per task (`contamination_risk` in each task card). Reference answers were accessible to: the authoring subagent (designer) and the evaluator host. No executor ever had access.
++
++## Residual risks
++1. Same user account on the nodes: an executor could read other runs' workspaces under `~/wp29/runs/` (sibling outputs of *other* runs) via Bash. Behaviour audit flags any such path; none occurred in the smoke. A stricter per-run OS identity was not implemented (documented limitation).
++2. The reviewer can in principle read the executor transcript (see above).
++3. The gate program is world-readable on the node; arms A/B were not told about it and did not call it (0 invocations in the smoke), which the audit records for every run.

modified reports/phase_status.md +30 −0

+ - **Unresolved**: packaged artefacts absent (replaced by live retrieval); budget envelope set provisionally by the executor (Phase 02).
+ - **Gate**: identity protected ✔; capabilities vs missing inputs distinguished ✔; no credentials exposed ✔. **PASSED.**
+ - **Next eligible**: Phase 01 (literature + corpus coding, launched as two background subagents at ≈ 03:25 UTC) and Phase 02 (protocol, tasks, evaluators) in parallel.
++
++## Phase 01 — Literature and scientific blueprint (in progress → closed with Phase 02 artefacts)
++
++- **Started** 2026-10-29 ≈ 03:25 UTC. Corpus coding completed ≈ 03:45 UTC by a Claude Code subagent (`research/corpus_coding_protocol.md`, `code_corpus.py`, `corpus_automated.csv`, `corpus_coding.csv`, `corpus_summary.md`; single coder, no intercoder reliability). Literature review delegated to a second subagent (still running at the time of this entry; outputs `research/literature_matrix.csv`, `novelty_matrix.csv`, `source_notes.md`, `references_verified.bib`).
++- **Blueprint**: `protocol/scientific_contract.md`, `protocol/treatment_definitions.md`, `protocol/evaluation_plan.md` (this phase and Phase 02 overlap by design: the smallest executable study was built and smoke-tested before the literature files landed).
++- **Gate**: closest prior work identified — pending the literature subagent's report; novelty qualified in `research/source_notes.md` when it lands. PARTIALLY PASSED (blueprint ✔, literature pending).
++
++## Phase 02 — Protocol, task suite and evaluator specification
++
++- **Started** 2026-10-30 ≈ 03:05 UTC. Wrote `tasks/TASK_AUTHORING_SPEC.md` (frozen interface v1), `evaluators/public_evaluation_interface/receipt_schema.json`, the four arm texts + reviewer + feedback prompts (`protocol/arms/`), `configs/*.yaml`, `protocol/screenshot_protocol.md`, the smoke task `F2-smoke-01` (bundle, card, hidden oracle with independent numpy reference), and the family brief `tasks/FAMILY_AGENT_BRIEF.md`; six task-family subagents launched ≈ 03:20 UTC (1 dev + 4 conf tasks each, with hidden oracles and validation sets).
++- **Checks**: content-equivalence audit of arm texts (no task facts in protocols); endpoint/estimand definitions written before any pilot run.
++- **Gate**: information matched across arms ✔; success/failure definitions frozen for the pilot ✔ (`protocol/evaluation_plan.md`); hidden-answer protection plan ✔ (`reports/isolation_audit.md`); negative/infeasible cases required in F5/F6 briefs ✔ (verification when the agents report). PASSED for the pilot design.
++
++## Phase 03 — Minimal harness
++
++- **Started** 2026-10-30 ≈ 03:10 UTC. Implemented `src/pgar/{contracts,state,events,launcher,isolation,provenance,budgets,runtime_gates,recovery,scheduler,evaluation,exports,cli}.py` (≈1,300 lines, standard library + numpy/pandas/jsonschema/yaml on the controller; the gate is standard-library only). Deployed the gate (`sha256 1760d13e…`) and a mode-600 key file on the four nodes; pinned venvs; `make test`: 7 infrastructure tests pass (state machine, idempotent job ledger, invariant "no evaluated without receipt", event normalisation, gate accepts a correct artefact and rejects three planted defects, smoke evaluator accepts correct_out and rejects a wrong-SE/fabricated-count artefact). The bundle-hygiene test is pending until all task agents finish.
++- **Failures fixed**: (i) `sqlite3` connection shared across threads returned a spurious `KeyError` → all reads serialised behind the store lock; (ii) `shlex.quote("~/…")` prevented tilde expansion on nodes → absolute remote root; (iii) behaviour audit false positives from truncated summaries → audit now parses the verbatim stream with a path regex.
++- **Gate**: harness rejects known defects ✔ (tests); worker access to hidden material prevented by tested controls ✔ (`reports/isolation_audit.md`, sentinel probes on all nodes). PASSED.
++
++## Phase 04 — Capture system and evidence viewer
++
++- Implemented `src/pgar/evidence_viewer.py` (read-only HTTP dashboard over `runs/state.sqlite`, port 8765) and `src/pgar/capture.py` (Quartz window lookup + `screencapture -l`, manifest in the `screenshots` table). Captures so far: **S002** native_live Terminal window tailing the live stream of RD0002 (arm D) during the smoke batch; **S003** native_live Terminal window of the builder session itself (incidental, kept for the reflexive trace); **S001** dashboard capture *failed* during the batch (Safari window off-screen; `screencapture` could not create the image) — a post-run viewer capture will be taken and labelled `viewer_postrun`. No synthetic imagery exists.
++- **Gate**: ≥1 genuine capture linked to an actual event ✔ (S002 ↔ RD0002 session b425e9f5…); no synthetic UI ✔; public evidence not yet published (originals in `evidence/restricted/`, to be inspected before copying). PASSED (with the recorded capture failure).
++
++## Phase 05 — Real smoke experiment
++
++- **Executed** 2026-10-30 03:30–03:36 UTC: assignment seed 101 → RD0001 (A, M3U96a), RD0002 (D, M3U96b), RD0003 (C, M4M64a), RD0004 (B, M4M64b), task `F2-smoke-01`, model `claude-opus-5-5` (reported by the client in `modelUsage`), client 2.1.314. First attempt: RD0003/RD0004 failed at workspace verification (tilde bug, no inference) and RD0001/RD0002 hit the thread-safety bug before launch — all four recorded in the ledger (`run_transitions`) and rerun under the attempt-numbered rerun policy after the fixes.
++- **Outcomes** (all live): 4 of 4 executor sessions finished (`result.subtype = success`); scientific_success = 1 in all four arms (5/5 criteria, 0 violations, reproducible). Costs (client estimate, USD): A 0.120 (6 turns, 53 s), B 0.311 (11 turns, 103 s), C 0.237 (7 turns, 85 s; gate run 3× by the executor, controller gate passed at cycle 1), D 0.418 = executor 0.303 + reviewer 0.114 (11 + 5 turns; reviewer verdict `accept` with 2 minor issues; no repair cycle). First-turn input tokens A 2,238 / B 3,717 / C 4,111 / D 4,319.
++- **Checks**: workspace manifests verified identical before each launch; behaviour audit: 0 outside-workspace references in all arms; gate invocations by executors: A 0, B 0, C 3, D 4; transcripts collected (`runs/development/RD000x/sessions/*.jsonl`).
++- **Gate**: four genuine task executions accounted for incl. the four infrastructure failures ✔; isolation and treatment delivery verified ✔ (effective-context hashes differ by arm only through the protocol text; token counts consistent); results are development observations ✔. PASSED. Next: pilot on the six development tasks when the family agents deliver them.

added reports/sentinel_probes_nodes.jsonl +4 −0

++{"node": "M3U96a", "result": "(1) NONE; (2) NONE.", "input_tokens": 176, "cost": 6.81e-05, "ok": true}
++{"node": "M3U96b", "result": "(1) NONE; (2) NONE.", "input_tokens": 176, "cost": 6.96e-05, "ok": true}
++{"node": "M4M64a", "result": "(1) NONE; (2) NONE.", "input_tokens": 177, "cost": 7.27e-05, "ok": true}
++{"node": "M4M64b", "result": "(1) NONE; (2) NONE.", "input_tokens": 175, "cost": 7.2e-05, "ok": true}

added research/build_literature_matrix.py +473 −0

++#!/usr/bin/env python3
++"""Build literature_matrix.csv and references_verified.bib for WP29 from cached
++primary-metadata records (Crossref, arXiv API, Semantic Scholar, OpenReview).
++
++Every entry below was checked against the cached API response in verify_cache/
++(fetched 2026-10-29/30 UTC).  Nothing is emitted unless the cache record exists.
++"""
++import csv, json, os, re, sys, html, urllib.parse, datetime
++import xml.etree.ElementTree as ET
++
++HERE = os.path.dirname(os.path.abspath(__file__))
++CACHE = os.path.join(HERE, "verify_cache")
++S2 = json.load(open(os.path.join(CACHE, "semanticscholar_batch.json")))
++DOIRES = json.load(open(os.path.join(CACHE, "doi_resolution.json")))
++NS = {"a": "http://www.w3.org/2005/Atom", "ar": "http://arxiv.org/schemas/atom"}
++
++
++def cache_path(url):
++    return os.path.join(CACHE, re.sub(r"[^A-Za-z0-9]+", "_", url)[:150])
++
++
++def mtime_utc(path):
++    return datetime.datetime.fromtimestamp(os.path.getmtime(path), datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
++
++
++def crossref(doi):
++    url = "https://api.crossref.org/works/" + urllib.parse.quote(doi, safe="")
++    p = cache_path(url)
++    m = json.load(open(p))["message"]
++    yr = (m.get("issued") or m.get("published-print") or m.get("published-online"))["date-parts"][0][0]
++    authors = [(a.get("family", ""), a.get("given", "")) for a in m.get("author", [])]
++    return dict(title=" ".join(m["title"][0].split()), authors=authors, year=yr,
++                container=(m.get("container-title") or [""])[0], crtype=m["type"],
++                volume=m.get("volume"), issue=m.get("issue"), pages=m.get("page"),
++                publisher=m.get("publisher"), url="https://api.crossref.org/works/" + doi,
++                verified=mtime_utc(p))
++
++
++def arxiv(aid):
++    url = "https://export.arxiv.org/api/query?id_list=" + aid
++    p = cache_path(url)
++    e = ET.parse(p).getroot().find("a:entry", NS)
++    names = [a.find("a:name", NS).text for a in e.findall("a:author", NS)]
++    authors = [(n.split()[-1], " ".join(n.split()[:-1])) for n in names]
++    jr = e.find("ar:journal_ref", NS)
++    return dict(title=" ".join(e.find("a:title", NS).text.split()), authors=authors,
++                year=int(e.find("a:published", NS).text[:4]), container="arXiv preprint",
++                jref=jr.text if jr is not None else None, url=url, verified=mtime_utc(p))
++
++
++def s2_venue(key):
++    v = S2.get(key) or {}
++    pv = v.get("publicationVenue") or {}
++    return v.get("venue"), pv.get("type"), v.get("year")
++
++
++# ----------------------------------------------------------------------------
++# Entry list.  Fields: key, stream, doi, arxiv, vtype, peer (yes/no/unknown),
++# peer_src (extra verification URL for the venue), rel (relevance), claim.
++# ----------------------------------------------------------------------------
++E = []
++def add(**k):
++    E.append(k)
++
++# ---- Stream 1: autonomous scientific discovery / agentic research systems ----
++add(key="lu2024aiscientist", stream=1, arxiv="2408.06292", vtype="preprint", peer="no",
++    rel="Reference end-to-end autonomous research system (idea to paper plus simulated review) against which WP29's protocol-guided design is positioned.",
++    claim="Introduces The AI Scientist, which generates research ideas, writes code, executes experiments, visualizes results, writes a full paper and runs a simulated review, with each paper developed at a cost of less than $15 and an automated reviewer reported to achieve near-human performance on paper scores.")
++add(key="lu2026aiscientist", stream=1, doi="10.1038/s41586-026-10265-5", vtype="journal", peer="yes",
++    rel="Peer-reviewed (Nature, 2026) publication of The AI Scientist line of work, establishing that fully automated ML research pipelines are now part of the refereed literature.",
++    claim="n/a (Nature abstract not retrieved behind the publisher login; see lu2024aiscientist for the system description).")
++add(key="yamada2025aiscientistv2", stream=1, arxiv="2504.08066", vtype="preprint", peer="no",
++    rel="Closest fully autonomous end-to-end system; its tree-search workflow and in-system AI reviewer are the counterpoint to WP29's executable gates and separated review.",
++    claim="Describes an end-to-end agentic system with progressive agentic tree search managed by an experiment-manager agent and a VLM feedback loop for figures; of three fully autonomous manuscripts submitted to an ICLR workshop, one scored above the average human acceptance threshold.")
++add(key="schmidgall2025agentlab", stream=1, doi="10.18653/v1/2025.findings-emnlp.320", arxiv="2501.04227", vtype="conference", peer="yes",
++    rel="Staged (literature review, experimentation, report writing) autonomous research framework with optional human feedback; provides the human-in-the-loop and cost comparisons WP29 builds on.",
++    claim="Agent Laboratory takes a human-provided idea through literature review, experimentation and report writing; the authors report that o1-preview gives the best outcomes, that human feedback at each stage significantly improves research quality, and an 84% decrease in research expenses relative to previous autonomous research methods.")
++add(key="boiko2023coscientist", stream=1, doi="10.1038/s41586-023-06792-0", vtype="journal", peer="yes",
++    rel="Landmark peer-reviewed demonstration of an LLM agent planning and executing laboratory experiments; defines the 'autonomous research' baseline WP29 compares to.",
++    claim="Coscientist, driven by GPT-4 with internet/documentation search, code execution and experimental automation tools, autonomously designs, plans and performs experiments across six tasks, including successful reaction optimization of palladium-catalysed cross-couplings.")
++add(key="bran2024chemcrow", stream=1, doi="10.1038/s42256-024-00832-8", vtype="journal", peer="yes",
++    rel="Tool-augmented LLM research agent whose evaluation combined LLM and expert assessment, an early instance of the evaluator-validity question WP29 addresses with protected oracles.",
++    claim="ChemCrow integrates 18 expert-designed tools with GPT-4, autonomously planned and executed syntheses of an insect repellent and three organocatalysts, and was evaluated with both LLM and expert assessments.")
++add(key="ifargan2025datatopaper", stream=1, doi="10.1056/AIoa2400555", arxiv="2404.17605", vtype="journal", peer="yes",
++    rel="Closest prior work on traceability: an autonomous data-to-paper platform that programmatically backtraces information flow for human verification, akin to WP29's persistent evidence records.",
++    claim="data-to-paper guides interacting LLM agents through a stepwise research process from annotated data to papers while programmatically backtracing information flow and allowing human oversight, and can run fully autonomously ('autopilot') or with human intervention.")
++add(key="gottweis2026coscientist", stream=1, doi="10.1038/s41586-026-10644-y", arxiv="2502.18864", vtype="journal", peer="yes",
++    rel="Multi-agent hypothesis-generation system with generate/critique/refine loops; illustrates in-system self-critique rather than independent verification.",
++    claim="Co-Scientist is a Gemini-based multi-agent system that continuously generates, critiques and refines research hypotheses using an asynchronous task framework and a tournament evolution process, scaled by test-time compute, to propose demonstrably novel hypotheses for experimental verification.")
++add(key="swanson2025virtuallab", stream=1, doi="10.1038/s41586-025-09442-9", vtype="journal", peer="yes",
++    rel="AI-human research collaboration with an LLM principal-investigator agent and human oversight; a human-involvement design point for the novelty matrix.",
++    claim="The Virtual Lab is an AI-human research collaboration in which an LLM Principal Investigator agent guides a team of LLM scientist agents to perform interdisciplinary research, applied to designing SARS-CoV-2 nanobodies.")
++add(key="mitchener2025kosmos", stream=1, arxiv="2511.02824", vtype="preprint", peer="no",
++    rel="Recent long-horizon AI scientist (up to 12 h, 200 rollouts); documents the coherence/verification challenges of long autonomous runs that WP29's gates target.",
++    claim="Kosmos runs for up to 12 hours performing cycles of parallel data analysis, literature search and hypothesis generation, using a structured world model shared between agents to pursue an objective coherently over 200 agent rollouts before synthesizing reports.")
++add(key="luo2025pitfalls", stream=1, arxiv="2509.08713", vtype="preprint", peer="no",
++    rel="Directly motivates WP29: shows AI scientist systems commit benchmark selection, leakage, metric-misuse and post-hoc selection errors that only traces reveal.",
++    claim="Identifies four failure modes in AI scientist systems (inappropriate benchmark selection, data leakage, metric misuse and post-hoc selection bias) and, via controlled experiments on two open-source systems, finds several of them present.")
++add(key="beel2025sakana", stream=1, doi="10.1145/3769733.3769747", arxiv="2502.14297", vtype="journal", peer="unknown",
++    rel="Independent third-party evaluation of The AI Scientist; evidence that unverified autonomous outputs contain novelty and execution errors.",
++    claim="An independent evaluation of the AI Scientist found several critical shortcomings, including a literature review based on simplistic keyword searches that led to poor novelty assessments, with several generated ideas incorrectly classified as novel.")
++add(key="luo2026xscientist", stream=1, arxiv="2607.12301", vtype="preprint", peer="no",
++    rel="Closest 2026 design to a research protocol: git-like typed research state with checkpoints, claim-evidence closure and gates; differs from WP29 in being a protocol artifact rather than a controlled comparison of protocol arms.",
++    claim="Proposes a local-first, git-like protocol that treats research state rather than a manuscript as the unit of continuation, representing hypotheses, attempts, claims, reviews and handoffs as typed content-addressed objects with immutable checkpoints, claim-evidence closure and authority-aware gates, without treating a passing integrity check as scientific truth.")
++add(key="ding2026verificationgap", stream=1, arxiv="2608.05179", vtype="preprint", peer="no",
++    rel="Survey that names the 'verification gap' in AI scientist systems; supplies the audit dimensions (evaluation method, artifacts, human-in-the-loop points, result-selection disclosure) WP29 reports.",
++    claim="Screens 125 works and codes 26 (24 runnable systems) on seven audit dimensions, finding that code release is now common but reproducibility-grade and claim-verification artifacts remain rare.")
++
++# ---- Stream 2: language agents for data analysis and scientific programming ----
++add(key="chen2025scienceagentbench", stream=2, arxiv="2410.05080", vtype="conference", peer="yes",
++    peer_src="https://api2.openreview.net/notes/search?term=ScienceAgentBench (ICLR 2025 Poster record)",
++    rel="Benchmark of expert-validated data-driven discovery tasks with program-level outputs, execution metrics and cost; the task style and contamination mitigations inform WP29's task bank.",
++    claim="Presents 102 tasks extracted from 44 publications in four disciplines and validated by nine experts; the best agent solves 32.4% of tasks independently (34.3% with expert knowledge), o1-preview reaches 42.2% at more than 10 times the cost, and two data-contamination mitigation strategies are proposed.")
++add(key="majumder2025discoverybench", stream=2, arxiv="2407.01725", vtype="conference", peer="yes",
++    peer_src="https://api2.openreview.net/notes/search?term=DiscoveryBench (ICLR 2025 Poster record)",
++    rel="Formalizes multi-step data-driven discovery as dataset + metadata + goal, the same unit of analysis as WP29's research tasks.",
++    claim="DiscoveryBench formalizes the multi-step process of data-driven discovery with 264 tasks across six domains derived from published papers plus 903 synthetic tasks, each defined by a dataset, its metadata and a natural-language discovery goal.")
++add(key="jing2025dsbench", stream=2, arxiv="2409.07703", vtype="conference", peer="yes",
++    peer_src="https://api2.openreview.net/notes/search?term=DSBench (ICLR 2025 Poster record)",
++    rel="Realistic data-science agent benchmark (long contexts, large data) used to characterize executor capability independent of protocol.",
++    claim="DSBench comprises 466 data-analysis and 74 data-modeling tasks sourced from Eloquence and Kaggle competitions, with long contexts, multimodal task backgrounds and reasoning over large data files.")
++add(key="huang2024mlagentbench", stream=2, arxiv="2310.03302", vtype="conference", peer="yes",
++    rel="Evaluates agents on ML experimentation loops with file/code actions; a precursor of WP29's executor arms.",
++    claim="MLAgentBench is a suite of 13 ML experimentation tasks (from CIFAR-10 improvement to BabyLM) in which a ReAct-based agent can read/write files, execute code and inspect outputs; a Claude 3 Opus agent had the best success rate.")
++add(key="gu2024blade", stream=2, arxiv="2408.09667", vtype="conference", peer="yes",
++    rel="Evaluates analysis decisions (variables, transformations, models) against independent expert analyses, the multi-path ground truth WP29's oracles must handle.",
++    claim="BLADE evaluates agents' multifaceted approaches to open-ended research questions using 12 datasets and questions from the literature, with ground truth collected from independent analyses by expert data scientists.")
++add(key="jimenez2024swebench", stream=2, arxiv="2310.06770", vtype="conference", peer="yes",
++    rel="Canonical execution-verified coding benchmark; its test-based grading is the model for WP29's executable verification gates.",
++    claim="SWE-bench consists of 2,294 software-engineering problems drawn from real GitHub issues and pull requests across 12 Python repositories, where resolving an issue frequently requires coordinating changes across multiple functions, classes and files.")
++add(key="chan2025mlebench", stream=2, arxiv="2410.07095", vtype="conference", peer="yes",
++    peer_src="https://proceedings.iclr.cc/paper_files/paper/2025/hash/7e3767db483c942b883eb4f8cfb74e31-Abstract-Conference.html",
++    rel="Agentic ML-engineering benchmark with human baselines, resource-scaling and contamination analyses that WP29's resource accounting parallels.",
++    claim="MLE-bench curates 75 Kaggle competitions with human leaderboard baselines; the best setup (o1-preview with AIDE scaffolding) reaches at least bronze-medal level in 16.9% of competitions, and the paper also studies resource scaling and pre-training contamination.")
++add(key="siegel2024corebench", stream=2, arxiv="2409.11363", vtype="journal", peer="yes",
++    rel="Benchmarks agents on computational reproducibility of published papers, linking agent evaluation to the reproducibility stream.",
++    claim="CORE-Bench contains 270 tasks based on 90 scientific papers across computer science, social science and medicine, measuring agents' accuracy at reproducing published results from provided code and data at three difficulty levels.")
++add(key="starace2025paperbench", stream=2, arxiv="2504.01848", vtype="conference", peer="yes",
++    rel="Rubric-decomposed replication grading with a validated LLM judge; closest precedent for WP29's rubric oracles and judge validation.",
++    claim="PaperBench asks agents to replicate 20 ICML 2024 papers from scratch, graded against 8,316 rubric items co-developed with the papers' authors, with an LLM-based judge whose performance is assessed on a separate judge benchmark.")
++
++# ---- Stream 3: workflows, provenance, computational reproducibility ----
++add(key="koster2012snakemake", stream=3, doi="10.1093/bioinformatics/bts480", vtype="journal", peer="yes",
++    rel="Rule-based workflow engine; WP29's protocol stages are an analogue of declarative, re-executable workflow rules.",
++    claim="Snakemake provides a readable Python-based workflow definition language and an execution environment that scales from single-core workstations to compute clusters without modifying the workflow.")
++add(key="ditommaso2017nextflow", stream=3, doi="10.1038/nbt.3820", vtype="journal", peer="yes",
++    rel="Container-based dataflow workflow manager for reproducible pipelines; reference for execution isolation in WP29's executors.",
++    claim="Argues that combining dataflow programming with software containers (Nextflow) yields computational workflows that are reproducible across platforms.")
++add(key="missier2013prov", stream=3, doi="10.1145/2452376.2452478", vtype="conference", peer="yes",
++    rel="W3C PROV family defines the provenance vocabulary (entities, activities, agents) that WP29's evidence records instantiate.",
++    claim="Overview of the W3C PROV family of specifications (data model, ontology, notation and constraints) for interoperable representation and exchange of provenance metadata.")
++add(key="chirigati2016reprozip", stream=3, doi="10.1145/2882903.2899401", vtype="conference", peer="yes",
++    rel="Automatic capture of execution provenance into re-executable packages; a model for evidence capture without relying on the agent's self-report.",
++    claim="ReproZip automatically captures the provenance of a command-line experiment (files, libraries, environment) and packages it so that it can be re-executed elsewhere.")
++add(key="sandve2013tensimplerules", stream=3, doi="10.1371/journal.pcbi.1003285", vtype="journal", peer="yes",
++    rel="Widely cited rules (track provenance, avoid manual steps, record versions and seeds) that WP29 operationalizes as machine-checked gates.",
++    claim="Proposes ten rules for reproducible computational research, including tracking how every result was produced, avoiding manual data manipulation, archiving exact program versions and random seeds, and providing public access to scripts, runs and results.")
++add(key="stodden2018journalpolicy", stream=3, doi="10.1073/pnas.1708290115", vtype="journal", peer="yes",
++    rel="Empirical reproducibility failure rates under policy-only enforcement; motivates executable rather than declarative verification.",
++    claim="In a random sample of 204 Science papers subject to a data/code-on-request policy, the authors obtained artifacts from 44% of papers and could reproduce the findings for 26%.")
++add(key="peng2011reproducible", stream=3, doi="10.1126/science.1213847", vtype="journal", peer="yes",
++    rel="Defines reproducibility (data + code available) as the minimum standard for computational claims, the standard WP29's evidence records target.",
++    claim="Reproducibility has the potential to serve as a minimum standard for judging scientific claims when full independent replication of a study is not possible.")
++add(key="nasem2019reproducibility", stream=3, doi="10.17226/25303", vtype="report", peer="unknown",
++    rel="Authoritative definitions of reproducibility vs replicability used in WP29's outcome definitions.",
++    claim="Defines reproducibility as obtaining consistent computational results using the same input data, computational steps, methods, code and conditions of analysis, and replicability as obtaining consistent results across studies aimed at the same question with new data.")
++add(key="krafczyk2021reproduction", stream=3, doi="10.1098/rsta.2020.0069", vtype="journal", peer="yes",
++    rel="Principles (transparency, re-executability, determinism) and a Reproduction Package formalism close to WP29's release bundle.",
++    claim="From reproducing seven articles the authors derive three principles (transparency about how results are produced, ease of re-executability, determinism), 12 guidelines, and a Reproduction Package formalism for sharing artifacts.")
++add(key="trisovic2022codequality", stream=3, doi="10.1038/s41597-022-01143-6", vtype="journal", peer="yes",
++    rel="Large-scale evidence that deposited research code usually fails to run, justifying execution-based verification gates.",
++    claim="Executing over 9,000 R files from more than 2,000 Harvard Dataverse replication datasets, 74% failed to complete without error initially and 56% still failed after automatic code cleaning.")
++
++# ---- Stream 4: preregistration, registered reports, specification discipline ----
++add(key="nosek2018preregistration", stream=4, doi="10.1073/pnas.1708274114", vtype="journal", peer="yes",
++    rel="Preregistration as prediction/postdiction separation is the direct analogue of WP29's protocol fixed before execution.",
++    claim="Defining research questions and the analysis plan before observing outcomes (preregistration) distinguishes analyses that result from predictions from those that result from postdictions, and practical strategies exist for cases that fall short of the ideal.")
++add(key="chambers2013registeredreports", stream=4, doi="10.1016/j.cortex.2012.12.016", vtype="journal", peer="unknown",
++    rel="Origin of Registered Reports: review of the protocol before results, the publishing analogue of WP29's pre-execution gate.",
++    claim="Introduces Registered Reports at Cortex, in which study protocols are peer reviewed and provisionally accepted before data collection.")
++add(key="chambers2022registeredreports", stream=4, doi="10.1038/s41562-021-01193-7", vtype="journal", peer="yes",
++    rel="Reviews evidence that pre-acceptance of protocols reduces reporting bias; supports the 'specification before execution' arm.",
++    claim="Registered Reports are a publication format in which study proposals are peer reviewed and pre-accepted before research is undertaken; early evidence indicates they work as intended although they are not a universal solution for irreproducibility.")
++add(key="simmons2011falsepositive", stream=4, doi="10.1177/0956797611417632", vtype="journal", peer="yes",
++    rel="Researcher degrees of freedom inflate false positives; WP29's locked analysis specification removes these from the agent.",
++    claim="Flexibility in data collection, analysis and reporting dramatically increases actual false-positive rates, so that a researcher is often more likely to falsely find evidence that an effect exists than to correctly find evidence that it does not; six disclosure requirements for authors and four reviewer guidelines are proposed.")
++add(key="munafo2017manifesto", stream=4, doi="10.1038/s41562-016-0021", vtype="journal", peer="yes",
++    rel="Umbrella statement of reproducibility measures (methods, reporting, evaluation, incentives) that WP29 adapts to autonomous agents.",
++    claim="Argues for measures optimizing methods, reporting and dissemination, reproducibility, evaluation and incentives, noting evidence from simulations and empirical studies but the need for iterative evaluation of their adoption.")
++add(key="gelman2014statisticalcrisis", stream=4, doi="10.1511/2014.111.460", vtype="journal", peer="unknown",
++    rel="Garden of forking paths: data-dependent analysis choices invalidate inference even without intent, which an agent in an open instruction arm is free to make.",
++    claim="Argues that data-dependent choices of analysis (the 'garden of forking paths') can produce spurious statistically significant findings even when researchers do not deliberately fish for significance.")
++add(key="nosek2015top", stream=4, doi="10.1126/science.aab2374", vtype="journal", peer="yes",
++    rel="TOP guidelines define graded transparency standards (preregistration, data, code) that WP29's protocol tiers mirror.",
++    claim="Proposes journal author guidelines (the TOP guidelines) to promote transparency, openness and reproducibility, noting that although scientists embrace these norms they are not routine in practice.")
++add(key="scheel2021excess", stream=4, doi="10.1177/25152459211007467", vtype="journal", peer="yes",
++    rel="Quantifies how pre-specified protocols change the result distribution; benchmark for interpreting WP29's arm differences.",
++    claim="Comparing 71 published Registered Reports with 152 standard hypothesis-testing papers, the first hypothesis was positive in 96% of standard reports but only 44% of Registered Reports.")
++add(key="osc2015reproducibility", stream=4, doi="10.1126/science.aac4716", vtype="journal", peer="yes",
++    rel="Baseline replication rates in a human literature; provides a human reference for WP29's validity outcomes.",
++    claim="Replications of 100 experiments from 2008 papers in three psychology journals found that about one-third to one-half of original findings were also observed in the replication, depending on the criterion.")
++add(key="kerr1998harking", stream=4, doi="10.1207/s15327957pspr0203_4", vtype="journal", peer="yes",
++    rel="HARKing (presenting post hoc hypotheses as a priori) is a failure mode an unconstrained agent can reproduce when writing up.",
++    claim="Defines HARKing (hypothesizing after the results are known) as presenting post hoc hypotheses as if they were a priori, and discusses its prevalence and costs.")
++
++# ---- Stream 5: agent evaluation, contamination, evaluator validity ----
++add(key="liu2024agentbench", stream=5, arxiv="2308.03688", vtype="conference", peer="yes",
++    rel="Multi-environment agent benchmark identifying long-horizon reasoning and instruction following as main failure causes, relevant to protocol adherence measurement.",
++    claim="AgentBench evaluates LLMs as agents across 8 environments; top commercial LLMs act well in complex environments but a significant gap separates them from open-source models up to 70B, with poor long-term reasoning, decision-making and instruction following as main obstacles.")
++add(key="yao2024taubench", stream=5, arxiv="2406.12045", vtype="preprint", peer="unknown",
++    rel="State-based outcome checking and the pass^k reliability metric are reused in WP29's repeated-run reliability analysis.",
++    claim="tau-bench evaluates agents in simulated user conversations with API tools and policy guidelines by comparing the final database state to an annotated goal state, and proposes pass^k for reliability over multiple trials; gpt-4o succeeds on fewer than 50% of tasks with pass^8 below 25% in retail.")
++add(key="kapoor2024agentsthatmatter", stream=5, arxiv="2407.01502", vtype="journal", peer="yes",
++    rel="Core methodological critique (cost reporting, holdouts, conflated benchmarking needs) that WP29's evaluation design follows.",
++    claim="Analysis of agent benchmarks finds a narrow focus on accuracy without cost, leading to needlessly complex and costly agents; proposes jointly optimizing accuracy and cost, and notes that model- and downstream-developer needs are conflated and that many benchmarks have inadequate holdout sets.")
++add(key="sainz2023contamination", stream=5, doi="10.18653/v1/2023.findings-emnlp.722", arxiv="2310.18018", vtype="conference", peer="yes",
++    rel="Defines contamination levels; WP29's task bank must document contamination risk for each task.",
++    claim="Position paper arguing that training on a benchmark's test split causes overestimation of performance with potentially harmful scientific conclusions, defining levels of data contamination and calling for community measurement for each benchmark.")
++add(key="zheng2023mtbench", stream=5, arxiv="2306.05685", vtype="conference", peer="yes",
++    peer_src="https://api2.openreview.net/notes/search?term=Judging%20LLM-as-a-judge (NeurIPS 2023 Datasets and Benchmarks record)",
++    rel="Establishes LLM-as-a-judge and its biases (position, verbosity, self-enhancement), which WP29's protected oracles are designed to avoid.",
++    claim="Examines LLM-as-a-judge limitations including position, verbosity and self-enhancement biases, and finds that strong judges such as GPT-4 agree with human preferences over 80% of the time on MT-bench and Chatbot Arena, the same level as human-human agreement.")
++add(key="thakur2025judgingjudges", stream=5, arxiv="2406.12624", vtype="conference", peer="yes",
++    rel="Shows even the best judge models lag inter-human agreement, cautioning against unprotected LLM grading of research outputs.",
++    claim="Across thirteen judge models and nine exam-taker models in a high-human-agreement setting, only the best and largest judges reach reasonable alignment with humans, remain well behind inter-human agreement, and may differ from human scores by up to 5 points.")
++add(key="panickssery2024selfpreference", stream=5, arxiv="2404.13076", vtype="conference", peer="yes",
++    rel="Self-preference bias is the specific threat WP29 mitigates by separating executor and reviewer and protecting oracles.",
++    claim="LLM evaluators such as GPT-4 and Llama 2 have non-trivial accuracy at recognizing their own outputs, and fine-tuning reveals a linear correlation between self-recognition capability and the strength of self-preference.")
++add(key="zhu2025abc", stream=5, arxiv="2507.02825", vtype="preprint", peer="no",
++    rel="Agentic Benchmark Checklist (task validity, outcome validity, reporting) used as a checklist for WP29's task and oracle construction.",
++    claim="Shows that many agentic benchmarks have task-setup or reward-design flaws (e.g., SWE-bench Verified uses insufficient test cases, TAU-bench counts empty responses as successful) that can mis-estimate performance by up to 100% relative, and introduces the Agentic Benchmark Checklist, which reduced overestimation on CVE-Bench by 33%.")
++add(key="kapoor2025hal", stream=5, arxiv="2510.11977", vtype="preprint", peer="no",
++    rel="Standardized harness with full cost accounting across models, scaffolds and benchmarks, the infrastructure model for WP29's isolated executors and resource logs.",
++    claim="Introduces a standardized evaluation harness and reports 21,730 agent rollouts across 9 models and 9 benchmarks at a total cost of about $40,000, finding among other things that higher reasoning effort can reduce accuracy.")
++add(key="li2025llmreval", stream=5, arxiv="2510.12367", vtype="preprint", peer="no",
++    rel="Simulated research-and-review loop showing LLM reviewers misaligned with humans; motivates WP29's separate, protected review arm rather than in-system AI review.",
++    claim="Using a simulation with a research agent that writes and revises papers and a review agent that assesses them, followed by human annotation, the authors identify pronounced misalignment between LLM reviewers and human judgments.")
++
++# ---- Stream 6: human-AI delegation, automation bias, intervention measurement ----
++add(key="parasuraman1997automation", stream=6, doi="10.1518/001872097778543886", vtype="journal", peer="yes",
++    rel="Use/misuse/disuse/abuse taxonomy frames how a researcher delegates to an autonomous agent and over-relies on its outputs.",
++    claim="Distinguishes use, misuse (over-reliance producing monitoring failures or decision biases), disuse (often caused by false alarms) and abuse of automation, with trust, workload and reliability as key influences.")
++add(key="parasuraman2000levels", stream=6, doi="10.1109/3468.844354", vtype="journal", peer="yes",
++    rel="Levels-of-automation model used to locate WP29's four arms on the acquisition/analysis/decision/implementation dimensions.",
++    claim="Proposes that automation can be applied to four function classes (information acquisition, information analysis, decision and action selection, action implementation), each across a continuum from fully manual to fully automatic, and that automation changes rather than merely supplants human activity.")
++add(key="goddard2012automationbias", stream=6, doi="10.1136/amiajnl-2011-000089", vtype="journal", peer="yes",
++    rel="Systematic review of automation bias frequency and mitigators informing how human review of agent outputs is measured in WP29.",
++    claim="A systematic review (74 of 13,821 retrieved papers) finds automation bias mediated by user factors such as cognitive style, decision-support design and task experience, discussed alongside automation-induced complacency.")
++add(key="lai2023humanai", stream=6, doi="10.1145/3593013.3594087", arxiv="2112.11471", vtype="conference", peer="yes",
++    rel="Design-space survey of empirical human-AI decision-making studies used to position WP29's intervention measurement.",
++    claim="Surveys empirical human-subject studies of AI-assisted decision making and summarizes the design space of tasks, AI assistance elements and evaluation metrics to structure a science of human-AI decision making.")
++add(key="bansal2021whole", stream=6, doi="10.1145/3411764.3445717", vtype="conference", peer="yes",
++    rel="Shows explanations raise acceptance of AI recommendations regardless of correctness, a risk for human review of agent-generated research.",
++    claim="In studies on three datasets, AI augmentation produced complementary improvements but explanations did not increase them; instead explanations increased the chance that humans accept the AI's recommendation regardless of its correctness.")
++add(key="lubars2019delegability", stream=6, arxiv="1902.03245", vtype="conference", peer="yes",
++    rel="Task-delegability framework (motivation, difficulty, risk, trust) used to characterize which research sub-tasks are delegated in each WP29 arm.",
++    claim="Proposes a delegability framework with four factors (motivation, difficulty, risk, trust) and, surveying 100 tasks, finds little preference for full AI control and a strong preference for machine-in-the-loop designs, with trust most correlated with preferences.")
++add(key="vaccaro2024combinations", stream=6, doi="10.1038/s41562-024-02024-1", vtype="journal", peer="yes",
++    rel="Meta-analytic evidence on when human-AI combinations beat either alone; reference point for interpreting WP29's human-review arm.",
++    claim="A preregistered meta-analysis of 106 experiments (370 effect sizes, 2020-2023) finds that on average human-AI combinations performed worse than the best of humans or AI alone, with losses in decision tasks and gains in content-creation tasks.")
++add(key="bainbridge1983ironies", stream=6, doi="10.1016/0005-1098(83)90046-8", vtype="journal", peer="yes",
++    rel="Classic argument that automation makes the residual human role harder, relevant to reviewers of autonomous research output.",
++    claim="Argues that the more advanced a control system is, the more crucial the contribution of the human operator becomes, because the operator is left with the tasks the designer could not automate and must monitor a system that rarely needs intervention.")
++add(key="bucinca2021trust", stream=6, doi="10.1145/3449287", vtype="journal", peer="yes",
++    rel="Cognitive forcing as a mitigation of overreliance; analogous to WP29's requirement that reviewers execute checks rather than read summaries.",
++    claim="In an experiment with 199 participants, three cognitive forcing interventions reduced overreliance on AI relative to simple explainable-AI approaches.")
++add(key="becker2025metr", stream=6, arxiv="2507.09089", vtype="preprint", peer="no",
++    rel="Randomized measurement of AI-tool effects on expert developers showing perception-reality gaps; methodological model for WP29's time/cost outcomes.",
++    claim="In a randomized controlled trial with 16 experienced open-source developers on 246 tasks, allowing early-2025 AI tools increased completion time by 19%, although developers forecast a 24% speedup beforehand and still believed in a 20% speedup afterwards.")
++
++# ---- Stream 7: durable execution, failure recovery, observability, cost ----
++add(key="dean2013tail", stream=7, doi="10.1145/2408776.2408794", vtype="journal", peer="unknown",
++    rel="Tail-tolerance techniques (hedging, timeouts) underlie WP29's executor time budgets and retry policy.",
++    claim="Software techniques that tolerate latency variability are vital to building responsive large-scale Web services.")
++add(key="gray1986whystop", stream=7, s2="87c9e34c39dd1fbd470010dc1ea98ba3517f5a2a", vtype="conference", peer="unknown",
++    rel="Foundational failure analysis (software and operations dominate outages; process pairs, transactions) behind WP29's checkpoint/idempotent step design.",
++    claim="Reports Tandem field data attributing most system outages to software and operations/administration rather than hardware, and advocates process pairs and transactions as the means to tolerate faults.")
++add(key="basiri2016chaos", stream=7, doi="10.1109/MS.2016.60", vtype="journal", peer="yes",
++    rel="Chaos engineering as experimentation on failure injection motivates WP29's fault-injection tests of the executors.",
++    claim="Describes chaos engineering as Netflix's practice of experimenting on distributed systems to verify their reliability, together with the principles underlying it.")
++add(key="helland2012idempotence", stream=7, doi="10.1145/2160718.2160734", vtype="journal", peer="unknown",
++    rel="Idempotent operations make retries safe, a requirement for WP29's resumable protocol steps.",
++    claim="Messages may be retried; idempotence means that is acceptable.")
++add(key="burckhardt2021durablefunctions", stream=7, doi="10.1145/3485510", vtype="journal", peer="yes",
++    rel="Formal semantics for durable workflow execution (persistent state, reliable progress) that WP29's run ledger approximates.",
++    claim="Describes Azure Durable Functions, which enhance FaaS with actors, workflows and critical sections to provide persistent state and reliable progress, and gives a formal semantics based on an idealized untyped lambda-calculus model.")
++add(key="erol2025costofpass", stream=7, arxiv="2504.13359", vtype="preprint", peer="no",
++    rel="Cost-of-pass (expected cost per correct solution) is the economic outcome metric WP29 reports per arm.",
++    claim="Defines cost-of-pass as the expected monetary cost of generating a correct solution and the frontier cost-of-pass relative to a human expert, finding lightweight models most cost-effective for basic quantitative tasks and reasoning models for complex quantitative problems.")
++add(key="dong2024agentops", stream=7, arxiv="2411.05285", vtype="preprint", peer="no",
++    rel="Taxonomy of agent-lifecycle artifacts to trace; WP29's evidence records cover these observability artifacts.",
++    claim="Presents a taxonomy of AgentOps artifacts and data that should be traced throughout the agent lifecycle to achieve observability, derived from a systematic mapping study of existing AgentOps tools.")
++add(key="khanal2026reliability", stream=7, arxiv="2603.29231", vtype="preprint", peer="no",
++    rel="Shows capability (pass@1) and reliability diverge with task duration, supporting WP29's repeated-run reliability outcomes.",
++    claim="Introduces four reliability metrics (Reliability Decay Curve, Variance Amplification Factor, Graceful Degradation Score, Meltdown Onset Point) and, over 23,392 episodes of 10 models on 396 tasks, finds reliability decay is domain-stratified (software-engineering GDS falls from 0.90 to 0.44) while document processing is nearly flat.")
++add(key="zhuang2026agentrewind", stream=7, arxiv="2608.14380", vtype="preprint", peer="no",
++    rel="Checkpoint/rewind recovery for long-horizon agents; the failure-recovery mechanism WP29's protocol runs implement at stage boundaries.",
++    claim="AgentRewind records aligned checkpoints of agent context and a controlled environment so that agents can return to an earlier state and resume with information from previous attempts, evaluated on the MettleBench long-horizon engineering benchmark.")
++add(key="sigelman2010dapper", stream=7, s2="003d5a65de0ac72daaf105ded903cb3eb88585b3", vtype="report", peer="no",
++    rel="Classic distributed tracing design (sampling, application-level transparency) referenced for WP29's run tracing.",
++    claim="Describes Google's production distributed tracing infrastructure, designed for low overhead through sampling and application-level transparency.")
++
++# ---- Stream 8: evidence-grounded generation and claim verification ----
++add(key="wadden2020scifact", stream=8, doi="10.18653/v1/2020.emnlp-main.609", vtype="conference", peer="yes",
++    rel="Scientific claim verification task and dataset; model for WP29's claim-to-evidence checks in generated reports.",
++    claim="Introduces scientific claim verification with SciFact, 1.4K expert-written claims paired with evidence abstracts annotated with labels and rationales; the system identified plausible evidence for 23 of 36 COVID-19 claims on CORD-19.")
++add(key="thorne2018fever", stream=8, doi="10.18653/v1/N18-1074", vtype="conference", peer="yes",
++    rel="Canonical fact extraction and verification benchmark defining Supported/Refuted/NotEnoughInfo labels adopted in WP29's citation audit.",
++    claim="FEVER contains 185,445 claims derived from Wikipedia and labeled Supported, Refuted or NotEnoughInfo (Fleiss kappa 0.6841), with a best pipeline accuracy of 31.87% when the correct evidence is required.")
++add(key="bohnet2022attributedqa", stream=8, arxiv="2212.08037", vtype="preprint", peer="no",
++    rel="Attributed QA frames generation that must cite supporting sources; WP29 requires attributed claims in agent reports.",
++    claim="Formulates Attributed QA with a reproducible evaluation framework that uses human annotations as gold standard and shows that a correlated automatic metric is suitable for development, benchmarking a broad set of architectures.")
++add(key="walters2023fabrication", stream=8, doi="10.1038/s41598-023-41032-5", vtype="journal", peer="yes",
++    rel="Quantifies fabricated references in LLM literature reviews; justifies WP29's mandatory citation verification gate.",
++    claim="In 84 short literature reviews on 42 topics containing 636 citations, 55% of GPT-3.5 citations and 18% of GPT-4 citations were fabricated, and 43% versus 24% of the real citations contained substantive errors.")
++add(key="alkaissi2023hallucinations", stream=8, doi="10.7759/cureus.35179", vtype="journal", peer="unknown",
++    rel="Early clinical report of fabricated references ('artificial hallucinations') in scientific writing.",
++    claim="Reports two case studies in which ChatGPT produced plausible but fabricated content including references when asked to write on rare conditions, documenting positive, negative and troubling aspects of its performance.")
++add(key="gao2023rarr", stream=8, doi="10.18653/v1/2023.acl-long.910", vtype="conference", peer="yes",
++    rel="Post-hoc attribution and revision pipeline; a model for WP29's independent evidence-grounding pass over agent output.",
++    claim="RARR automatically finds attribution for the output of any text generator and post-edits unsupported content, significantly improving attribution while otherwise preserving the original output.")
++add(key="wang2023selfconsistency", stream=8, arxiv="2203.11171", vtype="conference", peer="yes",
++    rel="Self-consistency is an in-model verification heuristic; WP29 contrasts it with external executable verification.",
++    claim="Self-consistency samples diverse reasoning paths and selects the most consistent answer, improving chain-of-thought accuracy on GSM8K (+17.9%), SVAMP (+11.0%) and AQuA (+12.2%).")
++add(key="min2023factscore", stream=8, doi="10.18653/v1/2023.emnlp-main.741", arxiv="2305.14251", vtype="conference", peer="yes",
++    rel="Atomic-fact precision scoring is the template for WP29's claim-level verification of generated reports.",
++    claim="FActScore decomposes a generation into atomic facts and computes the percentage supported by a reliable knowledge source; ChatGPT biographies reach only 58%, and an automated estimator is introduced.")
++add(key="huang2024selfcorrect", stream=8, arxiv="2310.01798", vtype="conference", peer="yes",
++    rel="Evidence that intrinsic self-correction fails without external feedback, supporting WP29's external gates over agent self-review.",
++    claim="Finds that LLMs struggle to self-correct their reasoning without external feedback, and that performance can even degrade after intrinsic self-correction.")
++add(key="sakai2026hallucitation", stream=8, arxiv="2601.18724", vtype="conference", peer="yes",
++    rel="Documents hallucinated citations reaching accepted NLP papers, showing the scale of the problem WP29's citation gate addresses.",
++    claim="Analyzing all ACL, NAACL and EMNLP papers of 2024-2025, nearly 300 papers contain at least one hallucinated citation, half of them at EMNLP 2025, and more than 100 such papers were accepted as main-conference or Findings papers.")
++add(key="zhao2026hallucinationswild", stream=8, arxiv="2605.07723", vtype="preprint", peer="no",
++    rel="Large-scale audit of non-existent citations in the literature after LLM adoption; external evidence of the citation-fabrication risk.",
++    claim="Auditing 111 million references across 2.5 million papers in arXiv, bioRxiv, SSRN and PubMed Central shows a sharp rise in non-existent references after widespread LLM adoption, with a conservative estimate of 146,932 hallucinated citations in 2025 alone.")
++
++
++# ----------------------------------------------------------------------------
++def authors_short(authors, key):
++    fam = [a[0] for a in authors if a[0]]
++    if key == "osc2015reproducibility":
++        return "Open Science Collaboration"
++    if key == "nasem2019reproducibility":
++        return "National Academies of Sciences, Engineering, and Medicine"
++    if not fam:
++        return "n/a"
++    if len(fam) == 1:
++        return fam[0]
++    if len(fam) == 2:
++        return f"{fam[0]} & {fam[1]}"
++    return f"{fam[0]} et al."
++
++
++def bibesc(s):
++    return (s.replace("&", "\\&").replace("%", "\\%").replace("#", "\\#").replace("_", "\\_")
++             .replace("τ", "$\\tau$").replace("$τ$", "$\\tau$"))
++
++
++rows, bibs, counts = [], [], {}
++for e in E:
++    doi, aid, s2id = e.get("doi"), e.get("arxiv"), e.get("s2")
++    src = []
++    if doi:
++        m = crossref(doi); src.append(m["url"])
++        res = DOIRES.get(doi); src.append(f"doi.org HEAD {res}")
++        if aid:
++            a = arxiv(aid); src.append(a["url"])
++        venue = m["container"] or ("National Academies Press" if e["key"] == "nasem2019reproducibility" else "")
++        verified = m["verified"]
++        meta = m
++    elif aid:
++        meta = arxiv(aid); src.append(meta["url"]); verified = meta["verified"]
++        vn, vt, _ = s2_venue("arXiv:" + aid)
++        s2url = "https://api.semanticscholar.org/graph/v1/paper/arXiv:" + aid
++        if e["vtype"] == "conference":
++            venue = {"chen2025scienceagentbench": "ICLR 2025", "majumder2025discoverybench": "ICLR 2025",
++                     "jing2025dsbench": "ICLR 2025", "chan2025mlebench": "ICLR 2025",
++                     "huang2024mlagentbench": "ICML 2024", "gu2024blade": "EMNLP 2024",
++                     "jimenez2024swebench": "ICLR 2024", "starace2025paperbench": "ICML 2025",
++                     "liu2024agentbench": "ICLR 2024", "zheng2023mtbench": "NeurIPS 2023 Datasets and Benchmarks",
++                     "thakur2025judgingjudges": "GEM2 Workshop, ACL 2025",
++                     "panickssery2024selfpreference": "NeurIPS 2024", "lubars2019delegability": "NeurIPS 2019",
++                     "wang2023selfconsistency": "ICLR 2023", "huang2024selfcorrect": "ICLR 2024",
++                     "sakai2026hallucitation": "ACL 2026 (per Semantic Scholar venue record)"}[e["key"]]
++            src.append(s2url + f" (venue={vn!r}, type={vt!r})")
++            if meta.get("jref"):
++                src.append("arXiv journal_ref: " + meta["jref"])
++        elif e["vtype"] == "journal":
++            venue = {"kapoor2024agentsthatmatter": "Transactions on Machine Learning Research",
++                     "siegel2024corebench": "Transactions on Machine Learning Research"}[e["key"]]
++            src.append(s2url + f" (venue={vn!r})")
++        else:
++            venue = "arXiv preprint"
++    else:  # Semantic Scholar only
++        fn = {"gray1986whystop": "s2_gray1986_search_observed.json",
++              "sigelman2010dapper": "s2_dapper2010.json"}[e["key"]]
++        p = os.path.join(CACHE, fn)
++        if fn.endswith(".jsonl"):
++            rec = None
++            for line in open(p):
++                line = line.strip()
++                if not line: continue
++                try:
++                    d = json.loads(line)
++                except Exception:
++                    continue
++                for x in d.get("data", []):
++                    if x.get("paperId") == s2id: rec = x
++        else:
++            rec = json.load(open(p))
++        meta = dict(title=rec["title"], authors=[(a["name"].split()[-1], " ".join(a["name"].split()[:-1])) for a in rec["authors"]],
++                    year=rec["year"], container=rec.get("venue") or "")
++        venue = rec.get("venue") or ("Google Technical Report dapper-2010-1" if e["key"] == "sigelman2010dapper" else "")
++        src.append("https://api.semanticscholar.org/graph/v1/paper/" + s2id + f" (record cached in {fn})")
++        verified = mtime_utc(p)
++    if e.get("peer_src"):
++        src.append(e["peer_src"])
++    ident = ("doi:" + doi) if doi else (("arXiv:" + aid) if aid else "s2:" + s2id)
++    counts[e["stream"]] = counts.get(e["stream"], 0) + 1
++    rows.append(dict(ref_key=e["key"], stream=e["stream"], authors_short=authors_short(meta["authors"], e["key"]),
++                     year=meta["year"], title=meta["title"], venue=venue, venue_type=e["vtype"],
++                     doi_or_arxiv=ident, verification_source=" ; ".join(src), verified_utc=verified,
++                     relevance_to_wp29=e["rel"], key_claim_or_finding=e["claim"], peer_reviewed=e["peer"]))
++
++    # ---- BibTeX ----
++    if e["key"] == "osc2015reproducibility":
++        au = "{Open Science Collaboration}"
++    elif e["key"] == "nasem2019reproducibility":
++        au = "{National Academies of Sciences, Engineering, and Medicine}"
++    elif doi:
++        au = " and ".join(f"{f}, {g}" if g else f for f, g in meta["authors"])
++    else:
++        au = " and ".join(f"{g} {f}".strip() for f, g in meta["authors"])
++    au = bibesc(au)
++    ti = bibesc(meta["title"])
++    f = [f"  author = {{{au}}}", f"  title = {{{{{ti}}}}}", f"  year = {{{meta['year']}}}"]
++    if e["vtype"] == "journal":
++        btype = "article"; f.append(f"  journal = {{{bibesc(venue)}}}")
++        if doi and meta.get("volume"): f.append(f"  volume = {{{meta['volume']}}}")
++        if doi and meta.get("issue"): f.append(f"  number = {{{meta['issue']}}}")
++        if doi and meta.get("pages"): f.append(f"  pages = {{{meta['pages'].replace('-', '--')}}}")
++    elif e["vtype"] == "conference":
++        btype = "inproceedings"; f.append(f"  booktitle = {{{bibesc(venue)}}}")
++        if doi and meta.get("pages"): f.append(f"  pages = {{{meta['pages'].replace('-', '--')}}}")
++    elif e["vtype"] == "report" and e["key"] == "nasem2019reproducibility":
++        btype = "book"; f.append("  publisher = {The National Academies Press}"); f.append("  address = {Washington, DC}")
++    elif e["vtype"] == "report":
++        btype = "techreport"; f.append("  institution = {Google}"); f.append("  number = {dapper-2010-1}")
++    else:
++        btype = "misc"; f.append(f"  howpublished = {{arXiv preprint arXiv:{aid}}}")
++    if doi:
++        f.append(f"  doi = {{{doi}}}")
++    if aid:
++        f.append(f"  eprint = {{{aid}}}"); f.append("  archivePrefix = {arXiv}")
++    comment = f"% verified via {src[0].split(' (')[0]} on {verified[:10]}"
++    bibs.append((comment, f"@{btype}{{{e['key']},\n" + ",\n".join(f) + "\n}"))
++
++cols = ["ref_key", "stream", "authors_short", "year", "title", "venue", "venue_type", "doi_or_arxiv",
++        "verification_source", "verified_utc", "relevance_to_wp29", "key_claim_or_finding", "peer_reviewed"]
++with open(os.path.join(HERE, "literature_matrix.csv"), "w", newline="", encoding="utf-8") as fh:
++    w = csv.DictWriter(fh, fieldnames=cols); w.writeheader(); w.writerows(rows)
++
++with open(os.path.join(HERE, "references_verified.bib"), "w", encoding="utf-8") as fh:
++    for i, (c, b) in enumerate(bibs):
++        if i == 0:
++            # No '%' line may precede the first entry: its verification comment follows it.
++            fh.write(b + "\n" + c + "  (comment placed after the first entry so that no % line precedes it)\n\n")
++        else:
++            fh.write(c + "\n" + b + "\n\n")
++
++print("entries:", len(rows), "per stream:", dict(sorted(counts.items())))
++print("peer_reviewed:", {k: sum(1 for r in rows if r["peer_reviewed"] == k) for k in ("yes", "no", "unknown")})

added research/code_corpus.py +278 −0

++#!/usr/bin/env python3
++"""
++code_corpus.py -- automated (script-computed) content-coding variables for the
++WP29 mandate corpus (research/corpus/*.md).
++
++Standard library only.  Run from anywhere:
++
++    python3 -I research/code_corpus.py
++
++Writes research/corpus_automated.csv (one row per file, including the
++byte-identical WP15 duplicate, which is flagged in `duplicate_of`).
++
++Every variable here is AUTOMATED (computed from the bytes of the file).  The
++definitions below are the authoritative ones referenced by
++research/corpus_coding_protocol.md.  Keyword counts are *mentions*, not
++judgements: a document that says "no budget is specified" would still count
++one mention of the `budget` family.
++"""
++
++import csv
++import hashlib
++import os
++import re
++import sys
++
++HERE = os.path.dirname(os.path.abspath(__file__))
++CORPUS_DIR = os.path.join(HERE, "corpus")
++MANIFEST = os.path.join(HERE, "corpus_manifest.csv")
++OUT = os.path.join(HERE, "corpus_automated.csv")
++
++# --------------------------------------------------------------------------
++# Regular expressions (documented for the codebook)
++# --------------------------------------------------------------------------
++
++# Markdown ATX headings: 1-6 '#' at line start followed by a space.
++RE_HEADING = re.compile(r"^#{1,6}\s+\S", re.MULTILINE)
++
++# Explicit phase labels anywhere in the text, e.g. "Phase 0", "PHASE 3 —",
++# "phase_XX" is NOT matched (requires digits).  Distinct numbers are counted.
++RE_PHASE_LABEL = re.compile(r"\bphase\s*(\d+)", re.IGNORECASE)
++
++# Lettered stage labels ("Stage A", "Stage G") -- WP15 uses stages, not phases.
++# Case-insensitive on the word "stage", but the label itself must be a capital
++# letter.  NB: WP28 also uses "Stage A..G" for its coordinator/worker message
++# protocol; because WP28 has explicit "Phase N" labels those stages are not
++# used as its phase count.
++RE_STAGE_LABEL = re.compile(r"(?i:\bstage)\s+([A-Z])\b")
++
++# Gate labels ("Gate 1", "Gate 12") -- distinct numbers counted.
++RE_GATE_LABEL = re.compile(r"\bgate\s*(\d+)", re.IGNORECASE)
++
++# Numbered list items at line start ("1. ", "  12. ").
++RE_NUMBERED_ITEM = re.compile(r"^\s*\d+\.\s", re.MULTILINE)
++
++# Headings whose text indicates a phase/sequence/execution-order section.  The
++# numbered items *inside* such sections are counted as declared phases when the
++# document has no explicit "Phase N" labels.
++RE_PHASE_SECTION_HEADING = re.compile(
++    r"^#{1,6}\s+.*\b(phase|phases|stage|stages|sequence|execution order|"
++    r"work sequence|execution protocol|execution phases|milestones)\b",
++    re.IGNORECASE | re.MULTILINE,
++)
++
++# Markdown table separator rows: |---|---:| etc.  One separator row == one table.
++RE_TABLE_SEP = re.compile(r"^\s*\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$", re.MULTILINE)
++
++# Fenced code blocks: pairs of ``` fences.
++RE_FENCE = re.compile(r"^\s*```", re.MULTILINE)
++
++# Keyword families (case-insensitive).  Each is a mention count + boolean.
++KEYWORD_FAMILIES = {
++    "preregist": r"pre-?regist\w*",
++    "estimand": r"\bestimands?\b",
++    "hypothes": r"\bhypothes\w*",
++    "falsif": r"\bfalsif\w*",
++    "baseline": r"\bbaselines?\b",
++    "leakage": r"\bleakage\b|\blook-?ahead\b|\blook ahead\b|\bpoint-in-time\b|\bpoint in time\b",
++    "provenance": r"\bprovenance\b|\bhash(?:es|ed|ing)?\b|\bsha-?256\b|\bchecksums?\b",
++    "reproduc": r"\breproduc\w*",
++    "stopping": r"\bstopping\b|\bstop rules?\b|\bstop conditions?\b|\btermination\b|\bstop and\b|\bstop if\b|\bstop only\b",
++    "recovery": r"\brecover\w*|\bresum\w*|\bcheckpoint\w*",
++    "gate": r"\bgates?\b|\bacceptance\b|\bcriteria\b",
++    "exhibit": r"\bscreenshots?\b|\bfigures?\b|\btables?\b",
++    "disclos": r"\bdisclos\w*|\bAI-use\b|\bAI use\b|\bAI contribution\b",
++    "budget": r"\bbudgets?\b|\bcosts?\b|\bUSD\b|\$",
++    "license": r"\blicen[cs]e[sd]?\b|\blicensing\b",
++    "seed": r"\bseeds?\b|\bseeded\b",
++}
++KEYWORD_RE = {k: re.compile(v, re.IGNORECASE) for k, v in KEYWORD_FAMILIES.items()}
++
++# Language heuristic: tokens that are common in French prose and rare in
++# English technical prose ("et" is excluded because of "et al.").
++# Tokens shared with English usage in this corpus ("conventions", "journal",
++# "du" as in "Universite du Quebec", "et" as in "et al.") are deliberately excluded.
++FR_TOKENS = {
++    "le", "la", "les", "des", "est", "une", "ce", "cette", "dans", "pour",
++    "sur", "avec", "pas", "aux", "qui", "que", "ci-dessous", "dossier",
++    "appelle", "correspond", "auteur",
++}
++EN_TOKENS = {
++    "the", "and", "of", "to", "in", "is", "for", "with", "that", "not", "must",
++    "do", "never", "each", "every", "all", "or", "as", "from", "if", "use",
++}
++RE_WORD = re.compile(r"[A-Za-zÀ-ÿ][A-Za-zÀ-ÿ'\-]*")
++
++
++def sha256_of(path):
++    h = hashlib.sha256()
++    with open(path, "rb") as f:
++        for chunk in iter(lambda: f.read(1 << 20), b""):
++            h.update(chunk)
++    return h.hexdigest()
++
++
++def language_of(text):
++    """Return (label, fr_share).  fr_share = FR hits / (FR hits + EN hits).
++
++    en    : fr_share < 0.015
++    mixed : 0.015 <= fr_share < 0.50  (e.g. a French preamble on an English body)
++    fr    : fr_share >= 0.50
++    (threshold chosen after inspection: WP18 = 0.043 and WP20 = 0.020 carry a
++    French prefatory paragraph; all other files score 0.000)
++    """
++    toks = [t.lower() for t in RE_WORD.findall(text)]
++    fr = sum(1 for t in toks if t in FR_TOKENS)
++    en = sum(1 for t in toks if t in EN_TOKENS)
++    total = fr + en
++    share = (fr / total) if total else 0.0
++    if share >= 0.50:
++        label = "fr"
++    elif share >= 0.015:
++        label = "mixed"
++    else:
++        label = "en"
++    return label, round(share, 4)
++
++
++def numbered_items_in_phase_sections(text):
++    """Count numbered list items inside sections whose heading mentions
++    phase/stage/sequence/execution order.  A section runs from its heading to
++    the next heading of the same or higher level."""
++    lines = text.splitlines()
++    heading_idx = []
++    for i, line in enumerate(lines):
++        m = re.match(r"^(#{1,6})\s+(.*)$", line)
++        if m:
++            heading_idx.append((i, len(m.group(1)), m.group(2)))
++    count = 0
++    for n, (i, level, title) in enumerate(heading_idx):
++        if not RE_PHASE_SECTION_HEADING.match("#" * level + " " + title):
++            continue
++        end = len(lines)
++        for j, lvl, _ in heading_idx[n + 1:]:
++            if lvl <= level:
++                end = j
++                break
++        block = "\n".join(lines[i + 1:end])
++        count += len(RE_NUMBERED_ITEM.findall(block))
++    return count
++
++
++def analyse(path):
++    raw = open(path, "rb").read()
++    text = raw.decode("utf-8", errors="replace")
++    name = os.path.basename(path)
++    wp = re.match(r"(WP\d+)", name).group(1)
++
++    phase_numbers = {int(x) for x in RE_PHASE_LABEL.findall(text)}
++    stage_letters = set(RE_STAGE_LABEL.findall(text))
++    gate_numbers = {int(x) for x in RE_GATE_LABEL.findall(text)}
++    items_in_phase_sections = numbered_items_in_phase_sections(text)
++
++    # n_numbered_phases: distinct "Phase N" labels if any; else distinct
++    # "Stage X" labels if any; else distinct "Gate N" labels if any; else
++    # numbered items within phase/sequence sections.  (Analyst-coded
++    # n_phases_declared in corpus_coding.csv is the authoritative value; this
++    # is the mechanical proxy.)
++    if phase_numbers:
++        n_numbered_phases = len(phase_numbers)
++        phase_rule = "distinct 'Phase N' labels"
++    elif stage_letters:
++        n_numbered_phases = len(stage_letters)
++        phase_rule = "distinct 'Stage X' labels"
++    elif gate_numbers:
++        n_numbered_phases = len(gate_numbers)
++        phase_rule = "distinct 'Gate N' labels"
++    else:
++        n_numbered_phases = items_in_phase_sections
++        phase_rule = "numbered items inside phase/sequence sections"
++
++    lang, fr_share = language_of(text)
++    words = len(text.split())
++    row = {
++        "file": name,
++        "wp": wp,
++        "sha256": hashlib.sha256(raw).hexdigest(),
++        "bytes": len(raw),
++        "lines": text.count("\n") + (0 if text.endswith("\n") else 1),
++        "words": words,
++        "n_headings": len(RE_HEADING.findall(text)),
++        "n_numbered_phases": n_numbered_phases,
++        "n_numbered_phases_rule": phase_rule,
++        "n_phase_labels_distinct": len(phase_numbers),
++        "n_stage_labels_distinct": len(stage_letters),
++        "n_gate_labels_distinct": len(gate_numbers),
++        "n_numbered_items_in_phase_sections": items_in_phase_sections,
++        "n_numbered_items_total": len(RE_NUMBERED_ITEM.findall(text)),
++        "n_tables": len(RE_TABLE_SEP.findall(text)),
++        "n_code_blocks": len(RE_FENCE.findall(text)) // 2,
++        "language_of_document": lang,
++        "fr_token_share": fr_share,
++    }
++    for fam, rx in KEYWORD_RE.items():
++        n = len(rx.findall(text))
++        row["mentions_%s_n" % fam] = n
++        row["mentions_%s" % fam] = "yes" if n else "no"
++    return row
++
++
++def main():
++    files = sorted(
++        os.path.join(CORPUS_DIR, f) for f in os.listdir(CORPUS_DIR) if f.endswith(".md")
++    )
++    if not files:
++        sys.exit("no .md files in %s" % CORPUS_DIR)
++
++    # manifest lookup (url, retrieved_utc, expected sha)
++    manifest = {}
++    if os.path.exists(MANIFEST):
++        with open(MANIFEST, newline="", encoding="utf-8") as f:
++            for r in csv.DictReader(f):
++                if r.get("local_file"):
++                    manifest[os.path.basename(r["local_file"])] = r
++
++    rows = [analyse(p) for p in files]
++
++    # duplicates by sha256 (first file in sorted order is the canonical one)
++    seen = {}
++    for r in rows:
++        r["duplicate_of"] = seen.get(r["sha256"], "")
++        seen.setdefault(r["sha256"], r["file"])
++        m = manifest.get(r["file"], {})
++        r["manifest_url"] = m.get("url", "")
++        r["manifest_retrieved_utc"] = m.get("retrieved_utc", "")
++        r["sha256_matches_manifest"] = (
++            "yes" if m and m.get("sha256") == r["sha256"] else ("no" if m else "not_in_manifest")
++        )
++        r["document_type"] = (
++            "design_brief" if "BRIEF" in r["file"]
++            else ("duplicate_claude_md" if r["duplicate_of"] else "claude_md")
++        )
++
++    fieldnames = (
++        ["file", "wp", "document_type", "duplicate_of", "sha256", "sha256_matches_manifest",
++         "manifest_url", "manifest_retrieved_utc", "bytes", "lines", "words", "n_headings",
++         "n_numbered_phases", "n_numbered_phases_rule", "n_phase_labels_distinct",
++         "n_stage_labels_distinct", "n_gate_labels_distinct",
++         "n_numbered_items_in_phase_sections", "n_numbered_items_total", "n_tables",
++         "n_code_blocks", "language_of_document", "fr_token_share"]
++        + ["mentions_%s" % k for k in KEYWORD_FAMILIES]
++        + ["mentions_%s_n" % k for k in KEYWORD_FAMILIES]
++    )
++    with open(OUT, "w", newline="", encoding="utf-8") as f:
++        w = csv.DictWriter(f, fieldnames=fieldnames, quoting=csv.QUOTE_MINIMAL)
++        w.writeheader()
++        for r in rows:
++            w.writerow({k: r.get(k, "") for k in fieldnames})
++
++    # console summary
++    print("wrote %s (%d rows)" % (OUT, len(rows)))
++    for r in rows:
++        print("%-38s %6d w  %3d h  phases=%-2s tables=%d code=%d lang=%s dup=%s" % (
++            r["file"], r["words"], r["n_headings"], r["n_numbered_phases"], r["n_tables"],
++            r["n_code_blocks"], r["language_of_document"], r["duplicate_of"] or "-"))
++
++
++if __name__ == "__main__":
++    main()

added research/corpus_automated.csv +14 −0

++file,wp,document_type,duplicate_of,sha256,sha256_matches_manifest,manifest_url,manifest_retrieved_utc,bytes,lines,words,n_headings,n_numbered_phases,n_numbered_phases_rule,n_phase_labels_distinct,n_stage_labels_distinct,n_gate_labels_distinct,n_numbered_items_in_phase_sections,n_numbered_items_total,n_tables,n_code_blocks,language_of_document,fr_token_share,mentions_preregist,mentions_estimand,mentions_hypothes,mentions_falsif,mentions_baseline,mentions_leakage,mentions_provenance,mentions_reproduc,mentions_stopping,mentions_recovery,mentions_gate,mentions_exhibit,mentions_disclos,mentions_budget,mentions_license,mentions_seed,mentions_preregist_n,mentions_estimand_n,mentions_hypothes_n,mentions_falsif_n,mentions_baseline_n,mentions_leakage_n,mentions_provenance_n,mentions_reproduc_n,mentions_stopping_n,mentions_recovery_n,mentions_gate_n,mentions_exhibit_n,mentions_disclos_n,mentions_budget_n,mentions_license_n,mentions_seed_n
++WP15_CLAUDE.md,WP15,claude_md,,2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77,yes,https://git.spboucher.ai/raw/wp15_uqo/main/docs/CLAUDE.md,2026-10-30T03:01:13Z,56305,2165,7685,151,7,distinct 'Stage X' labels,0,7,0,99,180,0,12,en,0.0,no,yes,yes,yes,yes,yes,yes,yes,yes,yes,yes,yes,no,yes,yes,yes,0,2,3,6,15,5,8,7,3,7,1,69,0,2,2,2
++WP15_PROMPT_PROMPT_INITIAL_CLAUDE.md,WP15,duplicate_claude_md,WP15_CLAUDE.md,2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77,yes,https://git.spboucher.ai/raw/wp15_uqo/main/docs/PROMPT_INITIAL_CLAUDE.md,2026-10-30T03:01:13Z,56305,2165,7685,151,7,distinct 'Stage X' labels,0,7,0,99,180,0,12,en,0.0,no,yes,yes,yes,yes,yes,yes,yes,yes,yes,yes,yes,no,yes,yes,yes,0,2,3,6,15,5,8,7,3,7,1,69,0,2,2,2
++WP18_CLAUDE.md,WP18,claude_md,,f538d6fe2483c975e45a0eed10b12e506ef2d7e20d48b16985c97aad17bc6779,yes,https://git.spboucher.ai/raw/wp18_uqo/main/CLAUDE.md,2026-10-30T03:01:13Z,12000,98,1358,9,5,distinct 'Phase N' labels,5,0,0,25,25,0,1,mixed,0.043,no,no,yes,no,yes,no,no,yes,no,no,yes,yes,no,yes,no,yes,0,0,2,0,5,0,0,6,0,0,1,24,0,5,0,1
++WP19_CLAUDE.md,WP19,claude_md,,2afdcd653423f83a24bc34cedc69589dc90cd372064134f28fddb6e771404a3b,yes,https://git.spboucher.ai/raw/wp19_uqo/main/CLAUDE.md,2026-10-30T03:01:14Z,9590,49,1092,12,8,distinct 'Phase N' labels,8,0,0,3,3,0,0,en,0.0,yes,no,yes,no,yes,no,yes,yes,no,yes,yes,yes,no,yes,no,yes,1,0,1,0,6,0,1,3,0,1,2,17,0,4,0,1
++WP20_CLAUDE.md,WP20,claude_md,,388ca2431215e4894f6b935a379a6db34a2f1d7d72cd42d188d75ed6c6fefddf,yes,https://git.spboucher.ai/raw/wp20_uqo/main/CLAUDE.md,2026-10-30T03:01:14Z,18756,179,2314,12,7,distinct 'Gate N' labels,0,0,7,0,41,0,1,mixed,0.0199,no,no,no,yes,yes,yes,yes,yes,no,no,yes,yes,yes,yes,yes,yes,0,0,0,1,2,2,2,8,0,0,9,21,1,42,2,1
++WP21_CLAUDE.md,WP21,claude_md,,692a6e50b3cf1d026a4bc12201759527ea3336e299bfd4786d3f635fff292b98,yes,https://git.spboucher.ai/raw/wp21_uqo/main/CLAUDE.md,2026-10-30T03:01:14Z,9870,100,1114,1,0,numbered items inside phase/sequence sections,0,0,0,0,15,0,0,en,0.0,yes,no,no,no,no,no,no,yes,no,no,yes,yes,no,no,no,yes,1,0,0,0,0,0,0,4,0,0,2,17,0,0,0,1
++WP22_CLAUDE.md,WP22,claude_md,,b9eb747485cff5a1caa115f0d9138ed925d86915aa77813862b4059bd6d45da5,yes,https://git.spboucher.ai/raw/wp22_uqo/main/CLAUDE.md,2026-10-30T03:01:14Z,16077,127,1966,25,8,distinct 'Phase N' labels,8,0,0,0,0,0,0,en,0.0,no,yes,yes,no,no,yes,yes,yes,no,no,yes,yes,yes,yes,no,yes,0,3,1,0,0,1,3,6,0,0,1,11,1,1,0,1
++WP23_CLAUDE.md,WP23,claude_md,,0f5ebcb5ec63d04392a4872530a43706535f1f8c00e6a3f61bdaaa34d1096cf0,yes,https://git.spboucher.ai/raw/wp23_uqo/main/CLAUDE.md,2026-10-30T03:01:14Z,9474,43,1132,20,0,numbered items inside phase/sequence sections,0,0,0,0,0,0,0,en,0.0,no,no,no,no,no,yes,no,yes,no,yes,yes,yes,no,yes,no,yes,0,0,0,0,0,3,0,4,0,1,1,7,0,1,0,1
++WP24_CLAUDE.md,WP24,claude_md,,02ae00f871fffd7937e9cd7ec121c53367addd9b5b709915bf6921adfa705347,yes,https://git.spboucher.ai/raw/wp24_uqo/main/CLAUDE.md,2026-10-30T03:01:14Z,18802,107,2285,20,7,distinct 'Phase N' labels,7,0,0,11,21,0,0,en,0.0,no,no,no,yes,yes,no,yes,yes,yes,no,no,yes,no,yes,yes,yes,0,0,0,1,5,0,1,6,1,0,0,10,0,10,3,1
++WP25_CLAUDE.md,WP25,claude_md,,69f5ee82532d14513986c8f53f1b0981a2f7311dc0fc3b26d844610ccc80bf8b,yes,https://git.spboucher.ai/raw/wp25_uqo/main/CLAUDE.md,2026-10-30T03:01:14Z,14054,96,1671,19,6,numbered items inside phase/sequence sections,0,0,0,6,6,0,1,en,0.0,yes,no,yes,yes,yes,yes,yes,yes,no,yes,yes,yes,no,yes,no,yes,1,0,1,2,8,2,1,4,0,1,11,9,0,10,0,3
++WP26_CLAUDE.md,WP26,claude_md,,49afcce91475e5e145b4bd49d29486065c8c613a402b51fdb9baa43c88f4b330,yes,https://git.spboucher.ai/raw/wp26_uqo/main/CLAUDE.md,2026-10-30T03:01:15Z,18482,135,2240,19,9,distinct 'Phase N' labels,9,0,0,0,19,0,0,en,0.0,yes,yes,yes,yes,yes,yes,yes,yes,yes,no,yes,yes,no,yes,yes,yes,4,1,2,1,4,7,6,7,2,0,2,6,0,20,4,2
++WP27_BRIEF_DESIGN-BRIEF.md,WP27,design_brief,,72dc7ab846c6d8f8e0cc4101b6bdb600884179e9467c43fddfd3aece395e236e,yes,https://git.spboucher.ai/raw/wp27_uqo/main/docs/DESIGN-BRIEF.md,2026-10-30T03:01:15Z,5231,59,820,3,0,numbered items inside phase/sequence sections,0,0,0,0,0,0,0,en,0.0,no,no,yes,no,no,no,no,no,no,no,no,no,no,yes,no,yes,0,0,1,0,0,0,0,0,0,0,0,0,0,3,0,3
++WP28_CLAUDE.md,WP28,claude_md,,c6c50e319f971b01fa0662840feef177d7bc896526956b0b7a3cf7a83f36246d,yes,https://git.spboucher.ai/raw/wp28_uqo/main/CLAUDE.md,2026-10-30T03:01:15Z,67200,1055,8809,76,13,distinct 'Phase N' labels,13,7,13,50,71,2,27,en,0.0,no,no,no,no,yes,no,yes,yes,yes,yes,yes,yes,yes,yes,no,yes,0,0,0,0,9,0,7,18,1,1,16,49,9,13,0,9

added research/corpus_coding.csv +14 −0

++"wp","file","document_type","source_url","retrieved_utc","sha256","research_domain","research_type","explicit_question","explicit_estimand_or_hypotheses","stated_assumptions","falsification_or_null_result_conditions","literature_or_novelty_requirement","data_access_controls","temporal_information_controls","baseline_requirement","n_phases_declared","dependency_or_gate_structure","stopping_rules","recovery_or_resume_instructions","artifact_requirements","provenance_requirements","review_or_self_audit_stage","disclosure_or_AI_use_statement","budget_or_resource_limits","integrity_prohibitions","contradictory_or_non_operational_requirements","statements_added_during_implementation","analyst_notes"
++"WP15","WP15_CLAUDE.md","claude_md","https://git.spboucher.ai/raw/wp15_uqo/main/docs/CLAUDE.md","2026-10-30T03:01:13Z","2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77","finance / market microstructure (intraday shock propagation, U.S. equities, OHLCV data)","empirical (with ML extension and synthetic-data tests)","""How have the speed, breadth, persistence, and asymmetry of intraday shock propagation across U.S. equities evolved over the past three decades, and under which market conditions does an initially idiosyncratic shock become sectoral or market-wide?""","yes — central hypothesis stated and outcome measures formally defined: ""Modern markets may absorb ordinary shocks more efficiently while transmitting extreme shocks more rapidly and discontinuously."" ; ""Define the Intraday Shock Amplification Ratio: ISAR_{i,t}(h) = …"" ; ""Optimization must not change estimands or silently alter precision.""","partial — assumptions must be stated and threats listed, and data assumptions are flagged as unverified: ""Explain: … identification assumptions; threats."" ; ""The natural key is initially assumed to be: (symbol, timestamp) But this assumption must be verified.""","yes — ""The project must attempt to disprove its own central claim."" ; ""If a falsification fails, do not hide it. Reassess the claim."" ; ""A rigorous negative assessment is preferable to a polished but invalid paper.""","yes — ""Do not overclaim uniqueness until a documented literature search confirms it."" ; ""never cite a work based only on another paper's bibliography."" ; literature map with ""verification date"" column required","yes — ""stop if the schema is unusable or if legal/access restrictions prevent research use."" ; ""authenticated access only if required"" ; ""No user-specific absolute path may appear in committed code."" ; "".env.example"" in tree","yes — ""For a shock at time t, network weights W_{ij,t} must be estimated only from data dated strictly before t."" ; ""Any liquidity-based inclusion criterion must use lagged data to avoid look-ahead bias."" ; ""Never randomly split minute observations across train and test."" ; ""Future information may be used only as a falsification test, never in the baseline estimator.""","yes — ""Compare with: unconditional historical mean; linear and logistic regressions; penalized regression; random forest; …"" ; ""one transparent baseline regression"" in first-phase deliverables ; ""The primary nonlinear result must also be represented with a transparent non-ML specification.""","7 (Stages A–G in ""3. Execution protocol""; plus ""30. Minimum deliverables for the first phase"" and ""33. Immediate actions"" 1–16)","explicit — ""Only after Stage A passes:"" ; ""Do not run `make release` until the pilot audit and full-data checks pass."" ; executable release gate: ""The release script must fail if: a table references a missing result; a figure is missing source data; LaTeX does not compile; … manuscript claims disagree with the results registry; data revision is not pinned; tests fail.""","yes — dedicated section: ""# 31. Stop conditions Stop and write a diagnostic report rather than proceeding if: timestamps cannot be interpreted reliably; OHLC values are extensively inconsistent; … data licensing prevents the intended use; …"" ; also ""stop if the schema is unusable"" (Stage A) and ""Escalate only issues that materially alter the research question, data legality, identification, or interpretation.""","yes — ""use resumable transfers"" ; ""permit partial reruns and node-level parallelism"" ; ""Every distributed task must: be idempotent; write to temporary paths; atomically rename on success; save checkpoints; … support retry; validate output before success."" ; ""never silently overwrite an existing file""","paper (main.tex + sections, anonymous and named PDFs); Internet Appendix; tables (standalone .tex AND machine-readable .csv/.parquet); figures (vector PDF + PNG, source data in outputs/figures/data/); results registry (outputs/estimates/results_registry.json) feeding LaTeX macros; code (src package, 22 numbered scripts, CLI, Makefile with ~19 targets); tests (8 files); data manifests (raw_files.parquet/csv with sha256, revision, row counts); docs (DATA_DICTIONARY, DATA_AUDIT, EMPIRICAL_DESIGN, LITERATURE_MAP, REPRODUCIBILITY, COMPUTE_LOG, DECISION_LOG, LIMITATIONS); STATUS.md per stage; README, LICENSE, CITATION.cff, pyproject/uv.lock/environment.yml/.env.example; release bundle (cover letter, source_archive.zip, replication_manifest.json, submission_checklist.md); 4 notebooks","yes — ""pin the exact dataset revision or commit hash"" ; manifest columns include ""sha256; status; download_started_at; … code_commit"" ; ""Each processed file must record: parent inputs; transformation version; filters; row counts before and after; creation timestamp; code commit."" ; ""Record: Git commit; Python version; package versions; OS; hardware; dataset revision; …""","yes — ""Stage C — Raw-data audit"" with docs/DATA_AUDIT.md ; ""28.3 Release validation"" ; decision log ""whether pre-specified or data-driven"" ; no adversarial referee stage, but ""Report blockers and proposed solutions before launching the full download.""","no — only a style instruction: ""The paper must read like a finance paper, not AI-generated prose."" (not a disclosure requirement)","partial — resource estimation required but no cap: ""Before a full run: 1. benchmark one month; 2. benchmark one year; 3. estimate CPU hours; 4. estimate memory; 5. estimate temporary storage; 6. estimate final storage; 7. record in docs/COMPUTE_LOG.md."" ; ""disk-space checks"" ; ""Avoid loading the full dataset into RAM."" ; no monetary or token budget","yes — ""Never fabricate data, results, citations, institutional details, variable definitions, statistical significance, sample sizes, or conclusions. Never write a result in the paper before it has been generated and verified from a saved machine-readable output."" ; ""Never manually type numerical results into LaTeX."" ; ""Never winsorize away a possible split without logging it.""","Non-operational success criteria that no script can check: ""the paper documents a new and economically important fact""; ""the writing is disciplined""; ""prepared to the standard expected in a top finance journal""; ""must read like a finance paper, not AI-generated prose"". Judgement-dependent trade-off with no threshold: ""Do not define isolation so strictly that only trivial events remain. Report the trade-off."" External dependency: ""main manuscript no longer than the current JF limit"" requires checking an external source at run time. Dataset facts are presented as unverified claims (""Claimed coverage: 1992–2026"") and must be audited, which is consistent. No internal contradiction found between autonomy (""Do not ask for confirmation for routine implementation choices"") and the mandated pause (""Report blockers and proposed solutions before launching the full download"") since the latter is an explicit escalation point.","none identified (no dated additions or reconstruction notes; the WP15 PROMPT_INITIAL copy is byte-identical)","Longest and most engineering-oriented mandate (7 685 words, 151 headings, 12 code blocks). The only mandate with a dedicated stop-condition section and an executable release-validation gate (""must fail if""). No UQO series framing (targets Journal of Finance), no AI-use disclosure, no page-count target other than JF limit. Cluster-compute section (27) presupposes distributed execution."
++"WP18","WP18_CLAUDE.md","claude_md","https://git.spboucher.ai/raw/wp18_uqo/main/CLAUDE.md","2026-10-30T03:01:13Z","f538d6fe2483c975e45a0eed10b12e506ef2d7e20d48b16985c97aad17bc6779","real estate / housing macro-finance (OLG housing economy, leverage, intergenerational inequality)","theoretical (with numerical general-equilibrium simulation)","""Under what conditions do inherited wealth, mortgage leverage, nominal debt, housing supply and down-payment constraints generate persistent owner-renter wealth inequality among households with similar labor income? What changes under unexpected inflation?""","partial — candidate contributions to be treated as hypotheses, not formal H statements: ""Candidate contributions: (i) an endogenous ownership-entry wealth threshold jointly shaped by bequests, collateral and price feedback; (ii) conditions for persistent ownership/wealth gaps and path dependence; … Treat each as a hypothesis, not a preordained result."" ; ten required experiments listed","partial — assumptions must accompany every proposition but are not stated in the mandate: ""For every proposition provide exact assumptions, statement, proof or clearly labeled proof gap."" ; one substantive non-assumption imposed: ""Do not assume inflation necessarily raises real house prices or enriches every owner.""","partial — failed conjectures and contrary outcomes must be recorded, no disproof condition defined: ""Save proofs in `paper/appendix_proofs.tex` and failed conjectures in `research/failed_hypotheses.md`."" ; ""Investigate contrary outcomes."" ; ""conditions for persistence/multiple equilibria (only if established)""","yes — ""Produce `research/novelty_matrix.csv` comparing each claimed contribution with the closest prior papers. Redesign weak contributions. Do not assert novelty before verifying it."" ; ""target of 50–80 **verified** references if genuinely relevant and accessible (never pad)""","partial — generic honesty about access only: ""Log unavailable files rather than claiming to have read them."" ; ""ask for essential permissions or missing credentials rather than pretending access"" ; calibration ""from traceable Canada/Québec sources when justified, or use clearly marked illustrative parameters""","no — theoretical paper; only ""Record units, observation dates, transformations and sources."" (no look-ahead rule)","partial — benchmark model and baseline scenario, no comparison methods: ""First solve a two-/three-period benchmark with known behavior; unit-test budget identities"" ; experiment ""1. Baseline wealth and ownership by age, income and inheritance quantile.""","5 (Phase 0–4) plus ""Final reproducibility and QA""","explicit — ""**Mandatory order:** inspect previous UQO WPs and review literature → specify mathematical model → derive propositions → implement and validate numerical model → … → audit results → **only then write and compile the LaTeX paper**."" ; ""**Gate:** do not start manuscript drafting until the baseline model is validated, claimed GE equilibria clear markets, principal simulations and exhibits reproduce, bibliography is verified, and theoretical claims have proofs or appropriate qualifications.""","partial — fallback only, no stop condition: ""If blocked, document the issue honestly and deliver a scoped partial result.""","partial — progress log after every phase, no resume/checkpoint instruction: ""Maintain `research/progress.md` after every phase with completed work, evidence, blockers and next steps.""","paper/main.tex + main.pdf; references.bib; appendix_proofs.tex; appendix_computation.tex; 10–14 figures (vector PDF + PNG in results/figures/); 6–9 auto-generated LaTeX tables (results/tables/); raw CSV/Parquet (results/data/); logs; code modules src/{model,household_solver,equilibrium,distributions,simulation,calibration,robustness,figures,tables}.py; tests/; configs YAML; Makefile (setup/test/simulate/figures/tables/paper/all); requirements.txt; research/{progress,wp_style_audit,novelty_matrix,literature_matrix,literature_review,model_specification,failed_hypotheses,results_audit}; README","partial — traceability without hashes/manifests: ""Every paper number must map to a result file."" ; ""Record units, observation dates, transformations and sources."" ; ""pinned dependencies, fixed seeds, tests, structured experiment logs""","yes — ""## Phase 4 — Integrity audit, then write the manuscript Before writing the paper, complete `research/results_audit.md`: experiment, hypotheses, parameters, algorithm, convergence, results, limitations, robustness, and claims actually supported.""","no — ""Do not invent coauthors or acknowledgments."" concerns authorship metadata, not AI-use disclosure","no","yes — ""Never invent results, citations, proofs, or data."" ; ""If general equilibrium does not solve, **never label partial equilibrium as GE**."" ; ""No fabricated calibration targets or empirical validation."" ; ""Never manipulate assumptions to force a preferred conclusion."" ; ""Do not call simulation findings theorems.""","Quantity targets coexist with anti-padding clauses and are resolved only by a priority statement: ""30–45-page English main text"", ""10–14 meaningful publication-quality figures and 6–9 substantive tables, adjusted to genuine findings rather than padded"", versus ""**Priority: scientific correctness, originality and reproducibility over speed, page count or positive findings.**"" Non-verifiable from the document: ""Rerun from a clean environment"" (no environment specification), ""Check PDFs visually"". The gate condition ""bibliography is verified"" has no operational verification procedure. The French preamble re-maps the directory name used in the plan (""le nom `wp18_uqo/` du plan ci-dessous désigne ce dossier""), i.e. the plan's paths are not literal.","Prefatory note (French) placed before the mandate: ""(Spécification de l'auteur, 2026-10-20. Conventions de forme : ~/Desktop/Recherche-UQO/HARMONISATION.md ; ce dossier s'appelle `immobilier/housing-leverage-intergenerational-inequality` et correspond au dépôt spbgit `wp18_uqo.git` — le nom `wp18_uqo/` du plan ci-dessous désigne ce dossier.)"" — a repository-mapping note dated with the specification, not marked as post-implementation.","Language: English body with a French prefatory paragraph (automated: mixed). Role framing ""Act as an autonomous team of housing economists, …"". Requires reading WP1–WP17 for style conventions (research/wp_style_audit.md)."
++"WP19","WP19_CLAUDE.md","claude_md","https://git.spboucher.ai/raw/wp19_uqo/main/CLAUDE.md","2026-10-30T03:01:14Z","2afdcd653423f83a24bc34cedc69589dc90cd372064134f28fddb6e771404a3b","real estate / housing market microstructure (search-and-matching, liquidity premium)","theoretical (continuous-time search model with numerical simulation)","""How do market tightness, heterogeneous seller reservation values, financing frictions, time-on-market costs, and bargaining power jointly determine an endogenous housing liquidity discount, transaction prices, selling hazards, and price dispersion?""","yes — ""Candidate propositions to PROVE: higher carrying cost lowers reservation price; decline in arrival/qualification widens the discount and increases TOM (with caveats); liquidity shocks move prices and volume without a fundamental shock; dispersion from heterogeneous urgency; selection bias in transaction indices."" ; defined object ""precise liquidity premium/discount definition with sign conventions (D = (p* − p)/p*)"" ; ""Each: preregistered hypothesis, exact config, output CSV, entry point, diagnostics, interpretation.""","partial — required, not stated: ""explicit timing, assumptions, parameter restrictions, identification limits, boundary cases"" ; ""Full assumptions, statements, proofs or counterexamples, intuition; label numerical patterns as conjectures.""","partial — counterexamples and failures must be reported; no disproof condition: ""proofs or counterexamples"" ; ""record failed experiments and regions without equilibrium"" ; ""Report failures.""","yes — ""Novelty gate: if the mechanism is already established, adjust the model and contribution before proceeding."" ; ""Authors to investigate only with independent bibliographic verification""","partial — ""No proprietary data, secrets, or unlicensed material."" ; ""If an endpoint is unavailable, log the failure and proceed"" ; ""do not claim to have read inaccessible material""","no — not applicable (theoretical, synthetic experiments)","partial — baseline scenario and frictionless benchmark, not comparison methods: ""1 baseline; 2 urgency gradient; …"" ; ""frictionless benchmark p*"" ; ""11 Nash vs posted prices""","8 (Phase 0–7); also an 8-step ""Mandatory execution sequence: (0) … (7)""","explicit — ""Mandatory execution sequence: (0) inspect prior UQO papers … (7) only then draft and compile the full LaTeX manuscript."" ; ""Novelty gate"" ; ""## Phase 6 — Independent audit before prose drafting"" ; ""## Research integrity and publication quality gates"" ; ""Introduction and abstract written last.""","no — ""Report failures."" only; no condition under which work stops","no","full repository; paper/main.pdf, main.tex, references.bib; scripts (src/{model,solver,simulation,calibration,diagnostics,plotting,tables}.py); tests (pytest); config/baseline.yaml; 10–14 figures (PNG + PDF); 6–9 tables; research/{wp_style_audit,literature_review,literature_matrix.csv,novelty_matrix.csv,model_specification,proofs,validation,results_audit,limitations}; README ""with commands and output inventory""; ""final report (contributions, proved propositions, strongest results with figure/table references, literature coverage, unresolved items, reproducibility status, file locations)""","yes — ""Fix random seeds; save environment and configuration hashes."" ; ""verify every number against result files"" ; ""`research/results_audit.md` maps every claim to proposition/proof, simulation file/figure/table, or citation""","yes — ""## Phase 6 — Independent audit before prose drafting `research/results_audit.md` maps every claim … classify as proved, numerically illustrated, empirically documented, or speculative""","no","no","yes — ""Never invent a proof, result, reference, DOI, dataset, or empirical validation."" ; ""Never fabricate outputs, citations, proofs, parameters or significance. No empirical language for synthetic experiments."" ; ""No \""publishable / first-of-its-kind\"" claims.""","""preregistered hypothesis"" is required for each of the 12 experiments but no registration mechanism, timestamp or location is specified, so pre-registration cannot be verified from the document. ""Independent audit"" (Phase 6) is to be performed by the same agent that produced the results — independence is not operational. Reference target ""Collect and verify 50–80 genuinely relevant references, not a quota"" and length ""30–45-page main paper"" are targets with built-in escape clauses. ""compile with latexmk; resolve warnings; inspect the rendered PDF"" presumes a TeX toolchain not checked for.","none identified","Most compact fully phased mandate (1 092 words, 8 phases). Explicitly scoped against WP18 (""Key distinction from WP18""). Role framing ""You are an autonomous interdisciplinary research team""."
++"WP20","WP20_CLAUDE.md","claude_md","https://git.spboucher.ai/raw/wp20_uqo/main/CLAUDE.md","2026-10-30T03:01:14Z","388ca2431215e4894f6b935a379a6db34a2f1d7d72cd42d188d75ed6c6fefddf","real estate appraisal / multimodal AI (cost approach from listing images and text; Vrai Prix workflow)","mixed (empirical evaluation + design-science pipeline, with a clearly labeled synthetic fallback)","""RQ1: How accurately can multimodal AI extract appraisal-relevant building attributes from listing photographs and descriptions relative to verified property records or expert annotations?"" (RQ1 of five)","partial — five research questions and a formal valuation identity, no hypotheses: ""\widehat V_i^{cost}=\widehat L_i + \widehat{RCN}_i - \widehat D_i + \widehat S_i"" ; metrics ""attribute extraction precision/recall/F1 for categorical features, MAE/RMSE for continuous features, calibration and abstention metrics, and cost-approach accuracy versus independent credible cost or appraisal benchmarks""","partial — ""Prove only results supported by stated assumptions."" ; ""Separate observable visual features, inferred proxy features, verified administrative fields, text-claimed features, and latent/unobservable features.""","partial — ""Favor transparent uncertainty and falsifiable evidence over marketing language."" ; ""If evidence for novelty is weak, revise the contribution statement accordingly."" ; ""never describe synthetic findings as real-world validation"" ; no disproof condition for RQ1–RQ5","yes — ""aiming for 60–100 genuinely verified scholarly references if sufficient relevant work exists; quality and relevance take priority over quota."" ; ""Establish novelty only after examining existing literature."" ; ""Gate 2 — Literature: verified bibliography, mapped novelty and gap.""","yes — ""Respect website terms, robots policies where applicable, copyright, privacy, and licensing; do not assume scraping or publication permission."" ; ""Keep secrets in environment variables, never in tracked files."" ; ""Never bypass access controls or website restrictions."" ; ""Avoid any use of private API keys in paper, logs, screenshots, or Git history."" ; docs/data_rights.md","yes — ""Prevent train/test leakage by property, duplicate listing, and time."" ; ""Where feasible, use temporal and geographic holdouts."" ; experiment ""10. Geographic and temporal out-of-sample performance.""","yes — ""Compare at least: - B0: traditional cost approach using verified structured fields and standard cost schedules. - B1: listing-text-only extracted attributes + cost model. - B2: image-only … - B3: multimodal image+text … - B4: multimodal + verified administrative fields + uncertainty-aware reconciliation."" ; ""Do not make comparisons unfair by giving one method privileged information without identifying it.""","6 (six-step ""Mandatory sequence: (1) … (6) ONLY THEN write""); organised in sections A–J with 7 numbered quality gates","explicit — ""Gate 1 — Repository audit … Gate 2 — Literature … Gate 3 — Theory … Gate 4 — Implementation … Gate 5 — Experiments … Gate 6 — Manuscript: only then draft and compile … Gate 7 — Integrity"" ; ""Hard gate: Do not draft Results or Conclusions until numerical outputs have been executed, independently checked, and saved.""","partial — blocker handling, no stop: ""When missing permissions, source data, or access makes a claim impossible to validate, clearly identify the blocker, proceed with the portions that are feasible, and provide the exact minimal information needed to finish.""","partial — ""caching, retry limits"" ; ""Maintain a TODO list and research log."" ; no resume instruction","paper/main.pdf + main.tex + sections/ + references.bib + figures/ + tables/; ~12–16 vector PDF figures; 8–12 LaTeX tables generated from saved data; src/{ingestion,extraction,feature_schema,cost_model,evaluation,simulations,visualization}; configs/; data/{README,raw,processed,synthetic}; results/manifest.json; docs/{implementation_audit,literature_matrix.csv,experiment_plan,results_validation,data_rights,limitations,research_log}; tests/; requirements.txt; Makefile; data dictionary + data-flow diagram; annotated extraction schema; reproducibility instructions; limitations report; ""final checklist distinguishing completed, blocked, and proposed items""","yes — ""Store prompts, model versions, temperatures, and response schemas."" ; ""generate a machine-readable results/manifest.json"" ; ""Maintain docs/implementation_audit.md with citations to local paths or source lines"" ; ""Every substantive factual claim must have a traceable source."" ; ""Avoid hardcoded table results; generate them from actual outputs.""","yes — ""Gate 7 — Integrity: audit every numerical claim against saved outputs and every citation against source metadata."" ; ""Write docs/results_validation.md documenting each check.""","no","no — ""retry limits"" only; experiment 16 mentions ""annotation time/cost"" as an outcome, not a limit","yes — ""Never invent datasets, numerical findings, literature references, citations, software functionality, API results, human labels, or Vrai Prix implementation details. Never fabricate significance."" ; ""Do NOT infer foundation integrity, hidden mold, structural soundness, exact year built, precise area, code compliance, permit status, or legal tenure from images alone.""","Human-dependent steps inside an autonomous mandate: ""Propose a rigorous annotation protocol with independent expert raters"" and experiment ""4. Human expert agreement and AI-human disagreement"" cannot be executed by the agent (mitigated by ""subject to data availability""). ""independently checked"" in the hard gate has no independent checker. Verification of 60–100 references ""against primary records"" presumes network access not confirmed. Figure/table count targets (""approximately 12–16 figures and 8–12 tables"") versus ""never pad with meaningless exhibits"". The French preamble re-maps the directory used in the plan (""le nom `wp20_uqo/` du plan ci-dessous désigne ce dossier"").","Prefatory note (French): ""(Spécification de l'auteur, 2026-10-20. Conventions de forme : ~/Desktop/Recherche-UQO/HARMONISATION.md ; ce dossier s'appelle `immobilier/multimodal-cost-approach-valuation` et correspond au dépôt spbgit `wp20_uqo.git` — le nom `wp20_uqo/` du plan ci-dessous désigne ce dossier. Journal : `docs/research_log.md`.)"" — repository mapping and log location, dated with the specification.","Language mixed (French preamble). Unusually explicit ethics/rights section (E) and a closing principle: ""Produce the strongest defensible scientific contribution possible, not the most optimistic portrayal of AI or Vrai Prix."" Keyword family `budget` is inflated by ""cost approach"" (42 hits of ""cost"")."
++"WP21","WP21_CLAUDE.md","claude_md","https://git.spboucher.ai/raw/wp21_uqo/main/CLAUDE.md","2026-10-30T03:01:14Z","692a6e50b3cf1d026a4bc12201759527ea3336e299bfd4786d3f635fff292b98","real estate economics / applied econometrics (systematic review and meta-analysis of hedonic pricing models)","empirical (meta-analysis with automated full-text extraction pipeline)","""RQ1 characteristics that consistently affect prices across countries and periods; RQ2 size and uncertainty of marginal contributions; RQ3 variation with geography, market conditions, property types, socioeconomics"" (RQ1–RQ3 of eight, written as fragments)","partial — effect-size families and pooled quantities defined, no hypotheses: ""Families: percentage premiums, elasticities, semi-elasticities, standardised effects, partial correlations, monetary marginal prices."" ; ""8.1 random-effects (REML) with pooled estimate, SE, 95% CI, 95% PI, τ², I², k studies, n estimates""","partial — methodological constraints stated as rules rather than assumptions: ""no monetary-to-percentage conversion without a defensible reference price"" ; ""Pool only within comparable groups; document and propagate uncertainty."" ; ""protocol before synthesis""","no — robustness and publication-selection diagnostics only (""Leave-one-study-out, leave-one-country-out, …""; ""funnel, small-study effects, PET-PEESE where appropriate""); no condition under which a claim is abandoned, except ""never relax criteria to reach targets""","yes — the paper is a literature synthesis; ""Compare with competing reviews before any \""largest\"" claim."" ; ""Explain precisely how the paper extends Sirmans et al. (2005, 2006)""","partial — ""read in full when legally accessible"" ; ""Google Scholar (compliant)"" ; no secrets/credential rules","no — a search cutoff date is set (""Reproducible search 1970–2026-10-20"") but no information-leakage rule applies","partial — comparison to a prior synthesis rather than baseline methods: ""comparison with Sirmans et al."" (required table) ; ""RQ5 traditional vs spatial vs machine-learning estimates""","7 (execution order A–G, inline)","implicit — ordering only, no gate conditions: ""15. EXECUTION ORDER. A research design → B literature acquisition → C data construction → D empirical analysis → E scientific interpretation → F manuscript → G final audit. No results written before analysis""","no — ""document obstacles and continue with feasible work.""","no","LaTeX paper (main.tex, sections, tables, figures, appendices, references.bib, reproducibility statement); structured research database data/{raw,metadata,extracted,validated,final}; scripts/{discovery,screening,extraction,harmonization,validation,econometrics,visualization}; protocol/{preregistration,eligibility,search_strategy}.md; reports/{search_audit,extraction_audit,methodological_decisions,limitations}.md; PRISMA flow diagram; 13 required tables and 12 required figures; tests; requirements.txt; README","yes — ""full records of queries, dates, sources, screening decisions, retrieval failures"" ; ""Store raw and harmonised values; never infer missing SEs except transparently from reported statistics; preserve every conversion formula; every number traceable to source table."" ; ""coefficient-level information (… page and table, extraction confidence)""","yes — ""G final audit"" ; ""Extraction validation: random sample of tables independently verified; report precision, recall, numerical accuracy, error rates; unverified or low-confidence extractions excluded from the primary synthesis."" ; ""flag ambiguous studies for human review""","no","no","yes — ""Never invent studies, citations, coefficients, standard errors, observations, statistical results, or claims of comprehensiveness."" ; ""never relax criteria to reach targets"" ; ""Never synthetic results."" ; ""no promotional language""","Aspirational targets (""≥5,000 unique records screened, ≥1,000 full texts, several hundred eligible studies"") are explicitly non-binding (""never relax criteria to reach targets""), so compliance cannot be judged. ""protocol/preregistration.md"" is required but no mechanism makes it precede the synthesis verifiably. ""random sample of tables independently verified"" has no independent verifier for a single agent; ""flag ambiguous studies for human review"" introduces a human-in-the-loop step without specifying who or when. ""vision/LLM-assisted classification"" implies paid model calls with no budget or recording requirement. ""read in full when legally accessible"" for ≥1,000 full texts is not checkable. R ""where superior"" is a judgement call.","none identified","Densest format: 15 numbered paragraphs, a single Markdown heading (automated n_headings = 1), 1 114 words. Only mandate whose primary object is the literature itself; ""Autonomous scientific research assignment, not a narrative review."""
++"WP22","WP22_CLAUDE.md","claude_md","https://git.spboucher.ai/raw/wp22_uqo/main/CLAUDE.md","2026-10-30T03:01:14Z","b9eb747485cff5a1caa115f0d9138ed925d86915aa77813862b4059bd6d45da5","real estate economics / price-index econometrics (quality-adjusted housing price indices, Quebec transactions)","mixed (empirical index construction + Monte Carlo simulation + methodological contribution)","none — stated as objectives, e.g. ""Investigate an adaptive estimator that optimizes the trade-off between local information, borrowing strength, temporal responsiveness, bias, and variance."" (not phrased as a question)","yes — dedicated section: ""## 4. Core estimand Define I[g,k,t] as the quality-constant market price index for geography g, building type k and month t. Establish a reference property distribution and common-support rules. Set 2021 annual average = 100 for the primary index"" ; no hypotheses as such","partial — ""Enforce identifiability constraints and explicitly describe priors, likelihood, covariance, temporal evolution and estimation algorithm."" ; ""Make explicit whether the headline is a price index for the stock, for transacted properties, or for a representative constant-quality basket.""","yes — ""Include deliberate counterexamples where SHARP-HPI underperforms simpler alternatives."" ; ""Report when reconciliation introduces bias or excessive smoothing."" ; ""Conduct placebo tests, synthetic quality-composition shocks, …"" ; ""Test whether estimated changes reflect new transaction coverage rather than market dynamics.""","yes — ""Derive and test an adaptive pooling/shrinkage mechanism; do not call it novel unless a literature review establishes novelty."" ; ""Explain exactly how the proposed contribution differs from each close predecessor."" ; anchor references named (Hill et al. 2022; Ishaak et al. 2024; Hill 2026)","yes — ""Discover local Vrai Prix sources, read-only credentials, schemas, backups … Never assume credentials, copy secrets into logs, or modify production."" ; ""Keep raw transaction records private; publish only legally permissible aggregate results. Never commit credentials or personally identifying records.""","yes — ""Never use post-transaction information or future data in real-time estimates."" ; ""(2) real-time vintage series estimated using only information available at each historical publication cutoff."" ; ""Use forward-chaining and rolling-origin backtesting, never random train-test splits for temporal index claims."" ; ""Prevent image-derived signals from encoding the sale price, future information, listing outcome or date-dependent model artifacts.""","yes — ""## 5. Benchmark estimators Implement and compare: A. Raw median and arithmetic mean prices. B. Stratified median. C. Standard pooled hedonic time-dummy OLS. D. Rolling time-dummy … K. Existing QHPI method as a documented benchmark.""","8 (Phase 0–7 in ""21. Work sequence"")","explicit — ""## 12. Simulations — mandatory before paper claims"" ; ""## 22. Final quality gate No paper can be declared complete until all figures and tables are reproducible, every empirical claim is linked to a generated result, all citations are verified, benchmarks have run, Monte Carlo tests are documented, uncertainty is calibrated, geographic coverage is disclosed, sparse cells are labeled, leakage tests pass, the manuscript compiles without substantive errors, and the results have been critically reviewed for overclaiming."" (one terminal gate; phases 0–7 are ordered without per-phase pass criteria)","partial — fallback, no stop: ""If data access is unavailable, implement testable code and synthetic-data simulations, but never pretend real-data analysis has been completed.""","partial — ""cached intermediate artifacts"" ; ""At every phase save machine-readable artifacts and an explicit decision log."" ; no resume instruction","data_dictionary.md; provenance_report.md; cleaned transaction pipeline + exclusion audit; model specification and derivation; reproducible benchmark and SHARP estimators; Monte Carlo framework and results; monthly index panel in Parquet and CSV with a 23-field row schema; coverage/reliability tables; index vintage/revision database; tables and figures from actual outputs; optional interactive geographic diagnostics; limitations and reproducibility documentation; English LaTeX manuscript with BibTeX; unit/integration tests; versioned configs; environment lockfiles; decision log","yes — ""Build an immutable extraction snapshot."" ; ""provenance_report.md"" ; index rows carry ""data_cutoff, publication_vintage, model_version, index_method"" ; ""versioned configurations, unit tests, integration tests, data contracts, environment lockfiles and cached intermediate artifacts""","yes — ""## 1. Audit and provenance"" (data audit) ; final gate requires ""the results have been critically reviewed for overclaiming"" ; ""final reproducibility audit"" in Phase 7","no","no — ""Profile memory and runtime."" and ""Use streaming/chunked computation when necessary."" are engineering guidance, not limits","yes — ""Do not invent data, empirical results, references, statistical significance, or novelty claims."" ; ""Never equate predictive accuracy with index accuracy."" ; ""A model estimate with zero local transactions is not an observed monthly price movement."" ; ""Avoid fabricated precision."" ; ""Do not silently substitute assessment values for market transactions.""","Terminal-gate conditions that cannot be checked from the document alone: ""uncertainty is calibrated"" (requires truth, available only in simulation), ""critically reviewed for overclaiming"" (no reviewer defined), ""a publishable contribution"". Data-dependent requirements that may be unsatisfiable: ""aggregation weights, ideally grounded in eligible housing stock"", ""Compare aggregate series to suitable external indices"", ""historical boundary crosswalks"". Optional/unverifiable: ""interactive geographic diagnostics if practical"". Factual assertion inside an instruction: ""The October 2026 observation is partial through the actual last reliable transaction date"". No explicit escalation or stop rule despite ""Begin immediately … Continue autonomously"".","none identified","Only mandate with a dedicated ""Core estimand"" section and a prescribed output data schema. 22 numbered sections, 1 966 words. Strongest temporal-information discipline in the corpus (real-time vintages, revision policy)."
++"WP23","WP23_CLAUDE.md","claude_md","https://git.spboucher.ai/raw/wp23_uqo/main/CLAUDE.md","2026-10-30T03:01:14Z","0f5ebcb5ec63d04392a4872530a43706535f1f8c00e6a3f61bdaaa34d1096cf0","real estate economics / machine-learning forecasting (spatiotemporal, hierarchical and probabilistic price forecasting, Quebec)","mixed (empirical forecasting evaluation + simulation study)","""do newly designed forecasting architectures substantially improve residential real estate price predictions?""","partial — forecasting targets defined, no hypotheses: ""Task A — future transaction price: P(i,t+h) = f(X(i,t), S(i,t), M(t), h); address transaction selection explicitly."" ; ""Task B … horizons 1, 3, 6, 12 months"" ; ""Task C — probabilistic forecasting: predictive distributions (mean, median, intervals, quantiles, downside risk, P(decline), …)""","no — none stated or required beyond ""Never assume a variable exists.""","partial — the question admits a negative answer and the simulation must show failures, but no disproof condition: ""(8) when complex models beat conventional ones"" ; ""when methods succeed or fail; explain mechanisms"" ; ""honest novelty statement""","yes — ""Never manufacture findings or claim novelty without verifying the literature."" ; ""# 13. LITERATURE REVIEW — … verify every reference; classify contributions (existing / adapted / new combination / original)."" ; ""literature search first""","partial — read-only rule on the upstream project only: ""WP22 (`../sharp-hpi-quebec`) is read-only for this project."" ; ""never modify WP22"" ; no credential/secret rules","yes — ""Every feature computed from information available at the forecast origin; no leakage."" ; ""strict pseudo-real-time expanding/rolling windows; for each origin: information set → estimate → forecast → store → compare. Never random splits."" ; ""regimes from information available at the origin only"" ; ""Prevent leakage from WP22: reconstruct index vintages from transactions observable at the origin; audit hedonics, clustering, normalisation, feature selection, imputation, target encoding, graph estimation, tuning, revisions.""","yes — ""# 6. BENCHMARKS — econometric (hedonic OLS, ridge/EN, SAR, ARIMA/SARIMA/ARIMAX, VAR/BVAR, DFM, local linear trend, state space), ML (RF, XGBoost, LightGBM, CatBoost, SVR, MLP), deep (LSTM, GRU, TCN, TFT, modern TS), simple (random walk, no change, mean growth, seasonal naive, momentum); fair common origins/information sets""","none (18 numbered topical sections, no execution phases; only ""literature search first"" and ""SIMULATION STUDY"" ordering cues)","explicit (terminal gate only) — ""# 18. QUALITY GATE — reproducible figures/tables, verified references, leakage audit passed, strict chronology, uncertainty reported, honest novelty statement, manuscript compiles.""","no","no","complete academic LaTeX manuscript (15 sections + appendices on architectures, hyper-parameters, tests, supplementary tables); reproducible Python research pipeline; original forecasting models; empirical results; 15 listed figure types (vector, reproducible); 13 listed table types; supplementary material; simulation study","partial — ""reproducible Python research pipeline"", ""vector; reproducible"" figures and a ""leakage audit"", but no manifest, hash, seed-logging or number-to-file traceability requirement","partial — ""leakage audit passed"" inside the terminal quality gate; no dedicated results audit or referee stage","no","no — a ""computational cost"" table is required as a reported outcome, not as a limit","yes — ""Never manufacture findings or claim novelty without verifying the literature."" ; ""no granular results for inadequate cells"" ; ""predictive importance ≠ causality"" ; ""no over-complex benchmarks the data cannot estimate""","The document itself declares that it is partly reconstructed: ""the pasted text ended at section 17 \""11. Prob\""; sections 17–end were completed from the UQO working-paper conventions"" and ""[Sections 11–15 and the remainder of the protocol were reconstructed by the author's assistant because the pasted protocol was truncated.]"" — part of the mandate is therefore not the author's original instruction. Six named ""original"" architectures (HSHN, DSGFN, RAREF, BHPF, QANSS, QREFA) are prescribed in advance while novelty is to be ""verified"" afterwards and ""keep only components with incremental out-of-sample value"" — pre-specified novelty versus post-hoc selection. ""substantially improve"" has no effect-size threshold. Heavy methods (""VI / Laplace / HMC on subsets"") with no compute budget. Section numbering is internally inconsistent (headings ""# 17."" and ""# 18."" versus the note ""ended at section 17 \""11. Prob\"""").","Two explicit markers: (1) header ""> Protocol as received on 2026-10-21 (the pasted text ended at section 17 \""11. Prob\""; sections 17–end were completed from the UQO working-paper conventions, see docs/PLAN.md). WP22 (`../sharp-hpi-quebec`) is read-only for this project."" ; (2) in section 17: ""[Sections 11–15 and the remainder of the protocol were reconstructed by the author's assistant because the pasted protocol was truncated.]""","Telegraphic style (1 132 words, 20 headings). The only mandate whose provenance is partly non-authorial by its own admission. No phases, no stop or recovery rules, no provenance manifest, no self-audit beyond the leakage audit."
++"WP24","WP24_CLAUDE.md","claude_md","https://git.spboucher.ai/raw/wp24_uqo/main/CLAUDE.md","2026-10-30T03:01:14Z","02ae00f871fffd7937e9cd7ec121c53367addd9b5b709915bf6921adfa705347","macro-finance theory (AGI, task-based automation, general equilibrium and asset valuation; extension of Restrepo 2025)","mixed (theoretical model + quantitative simulation + empirical calibration/short-run design)","""How does increasingly capable and abundant AI reallocate factor income from labor to reproducible and scarce capital, change the ownership distribution of aggregate wealth, and jointly determine the prices and expected returns of financial, real-estate, and other assets?""","yes — nine target results: ""T1. Conditions under which Restrepo's labor-share asymptote survives multi-sector production with complementary physical inputs and scarce factors; counterexamples with binding energy/land. T2. Sufficient conditions for factor-income reallocation …"" through T9 ; ""Each result needs explicit assumptions, mathematical statement, rigorous proof or counterexample, economic interpretation, and simulation check.""","yes — primitives are stated and a numbered assumptions protocol and register are required: ""Restrepo's central primitives include task output X_t(ω)=L_t(ω)+Q_t(ω)/α_t(ω), …"" ; ""Proof protocol: write assumptions as numbered statements; verify differentiability, convexity/concavity, equilibrium existence, transversality, boundedness, market clearing, timing, stochastic integrability, and no-arbitrage."" ; `docs/assumptions_register.md`","yes — ""Do not assume that AI automatically raises profits, all asset prices, inequality, or aggregate welfare. Derive conditions and counterexamples."" ; ""Tests must fail on deliberately inconsistent accounting."" ; ""identify parameter regimes where asset valuations fall despite higher output"" ; ""Provide a falsifiable short-run empirical design: … with pre-trends, confounding, measurement error and alternative mechanisms addressed.""","yes — ""Prepare a literature matrix: … Verify exact references from authoritative sources. Mark all unverified claims."" ; referee question ""(1) is this genuinely new relative to Restrepo and task models?"" ; ""Explicitly separate the original author's conclusions from our proposed extensions.""","yes — ""Do not present proprietary datasets as accessible unless actually available."" ; ""company annual reports/Compustat if licensed"" ; ""`data/README.md`, `data/raw/`, `data/processed/`: sources, licenses, provenance and transformations; no fabricated observed data.""","partial — vintage awareness and pre-trend checks, no leakage rule: ""Check exact definitions, vintages, geographic scope, units, time frequency and licenses."" ; ""Validate against pre-AGI historical patterns without claiming the historical sample identifies post-AGI structural parameters."" ; ""Do not use speculative AGI dates as facts.""","yes — ""Reproduce a minimal baseline numerically or analytically before adding features."" ; ""Implement a real baseline first."" ; paper section ""3. Benchmark Restrepo replication and limitations."" ; ""Define a transparent benchmark and add layers with tests at every stage.""","7 (Phase 0–6) plus a 12-step ""EXECUTION ORDER"" A–L","explicit — ""## EXECUTION ORDER AND STOP CONDITIONS A. Audit source PDF and write source notes; B. derive smallest exact benchmark; C. … L. perform referee audit and final replication."" ; ""Never declare a theorem proven without a proof; never declare a model solved without numerical equilibrium checks; never declare a PDF compiled without verifying the file."" ; ""Implement deterministic perfect-foresight transition first, then aggregate shocks"" (no numbered gates)","partial — the heading says ""STOP CONDITIONS"" but the content is a fallback: ""If a block cannot be solved, deliver a verified simpler model and clearly document the limitation."" ; ""start implementing without waiting for further permission unless a genuinely blocking external input is required.""","partial — ""At each stage update `docs/research_log.md` with completed steps, assumptions, residuals, unresolved problems and next actions."" ; ""Record seeds, solver tolerances and convergence diagnostics."" ; no resume/checkpoint instruction","paper/main.tex (35–60+ pages); paper/appendix.tex; paper/references.bib; slides/main.tex (30–45 Beamer slides); src/ (solver, simulations, calibration, tests); data/README.md + raw/ + processed/; results/ (tables, figures PDF+PNG, parameter files, machine-readable outputs); docs/{model_dictionary,assumptions_register,identification,research_log,replication,referee_report}.md; Makefile; requirements.txt or pyproject.toml; environment lockfile; README with one-command reproduction; compiled PDFs ""if TeX available""","yes — ""sources, licenses, provenance and transformations"" for data ; ""Build a source-to-parameter table with observed moments, uncertainty and identification limits."" ; ""Clearly separate externally identified, calibrated and scenario-only parameters."" ; ""every result traceable to output"" ; ""Record seeds, solver tolerances and convergence diagnostics.""","yes — ""## PHASE 6 — CRITICAL SELF-REVIEW Write an adversarial referee report addressing: (1) is this genuinely new relative to Restrepo and task models? (2) is the model too large to identify? … (12) can the core mechanisms be established in a simpler model? Revise after review."" ; docs/referee_report.md","no","no","yes — ""no invented citations, DOIs or results"" ; ""no fabricated empirical estimates, no invented statistical significance"" ; ""Never declare a theorem proven without a proof; never declare a model solved without numerical equilibrium checks; never declare a PDF compiled without verifying the file."" ; ""Label conjectures as conjectures, not theorems."" ; ""Avoid hype words like 'revolutionary' or 'legendary' in manuscript.""","Scope tension: an 8–12-sector IO economy with heterogeneous households, full asset-pricing block, optional monetary/fiscal/open-economy modules and a 2026–2100 simulation versus ""Use a nested, modular economy, not an intractable all-at-once model"" and ""Prioritize a correct publishable contribution over a grand but incoherent system"" (resolved only by priority ordering). Length targets (""35–60+ page"", ""30–45 Beamer"") with no anti-padding clause other than the priority statement. External dependency: ""Download and read the complete original PDF"" presumes network access. ""STOP CONDITIONS"" heading without any stop condition. Conditional deliverables (""Compile PDFs if TeX available"") make completion ambiguous. Non-verifiable: ""publication-quality"", ""disciplined findings"".","Explicit post-hoc section: ""## UQO conventions (added 2026-10-27) This project is **UQO Working Paper No. 24** in `Recherche-UQO/finance/`. Follow `Recherche-UQO/HARMONISATION.md` (canonical title page, `\WP…` macros, README template, `paper/Makefile` + `.latexmkrc`, git remote `gitsrv:srv/git/wp24_uqo.git`).""","Mandate was written without UQO framing and later attached to the series (addition dated 2026-10-27). Contains a ""FIRST RESPONSE EXPECTED FROM CLAUDE CODE"" section specifying the first turn. Only mandate requiring seminar slides."
++"WP25","WP25_CLAUDE.md","claude_md","https://git.spboucher.ai/raw/wp25_uqo/main/CLAUDE.md","2026-10-30T03:01:14Z","69f5ee82532d14513986c8f53f1b0981a2f7311dc0fc3b26d844610ccc80bf8b","economic theory / computational economics (agentic general equilibrium, LLM agents as delegated decision-makers)","mixed (theoretical model + heterogeneous-agent numerics + LLM-agent simulation laboratory)","""How does delegation of economic decisions to autonomous AI agents change equilibrium wages, interest rates, asset prices, capital accumulation, production, credit, systemic risk, and wealth inequality?""","yes — six falsifiable hypotheses: ""H1: Delegation creates an endogenous adoption threshold determined by information gains, execution costs, compute prices and agency losses. H2: Shared foundation models correlate trades and may reduce market depth under stress, conditional on common information. H3: …"" through H6 ; ""Pre-register outcomes.""","partial — required at every stage and sketched as a non-binding starting point: ""(b) write down assumptions"" ; ""Provide propositions and rigorous proofs under explicit assumptions, or label unsolved statements conjectures."" ; ""Illustrative starting point, NOT an assumed final model: V_i(s)=max_{c,l,a',d}{…}""","yes — ""For each: mathematical mechanism, formal or numerical prediction, falsification condition, isolation experiment and limitation."" ; ""Science gate: controlled experiments, uncertainty, negative results, sensitivity and mechanism isolation."" ; ""Supply counterexamples and boundary cases.""","yes — ""Before developing the model, search and verify literature on … Create `docs/literature_matrix.md` with verified bibliographic details, URLs/DOIs, modeling differences, and the precise remaining gap. No invented references."" ; ""1. **Novelty gate:** verified literature matrix, precise contribution and competing explanations.""","yes — ""Support local OpenAI-compatible inference and optional BYOK providers; no embedded credentials."" ; ""Request user input only for indispensable private data, external credentials, expensive compute authorization, or major scope changes."" ; ""Use verified public macro, labor, wealth, interest rate and financial data where possible.""","yes — ""Prevent look-ahead and information leakage."" ; ""validate surrogates on held-out actions"" ; ""No final-test-informed"" logic implied by ""Pre-register outcomes.""","yes — ""Controls: rational optimized agents, rules, reinforcement learning, single LLM, multiple heterogeneous LLMs, shared LLM with varied prompts, with/without tools, memory and negotiation."" ; ""Benchmark model with AI switched off against known analytical cases."" ; ""zero-AI limiting case"" (Numerics gate) ; scenario ""no AI""","6 (six numbered ""Milestones and acceptance gates""; the ""Start now"" paragraph adds an ordered sequence; no ""Phase N"" labels)","explicit — ""## Milestones and acceptance gates 1. **Novelty gate:** … 2. **Theory gate:** … 3. **Numerics gate:** … 4. **Agent gate:** … 5. **Science gate:** … 6. **Writing gate:** … Do not proceed to extravagant multi-agent simulations before gates 1–3 are met. If a gate fails, report why and repair the design."" ; ""Build tiny verified economies first; scale only after tests pass.""","partial — gate-failure and escalation rules, no stop: ""If a gate fails, report why and repair the design."" ; ""Request user input only for indispensable private data, external credentials, expensive compute authorization, or major scope changes."" ; ""If internet or TeX is unavailable, state the limitation and continue with honest local deliverables.""","yes — ""Support deterministic seeds, config-driven CLI, checkpointing, experiment manifests, Git revision and reproducible environment locks."" ; ""Cache reproducibly.""","repository scaffold (configs/, data/raw+processed, src/age/{economy,agents,equilibrium,markets,banking,llm,experiments,analysis}, tests/{unit,integration,property}); docs/{literature_matrix.md,model_specification.md,parameter_ledger.csv,research_log.md,limitations.md}; experiments/pre_registration.md; results/{figures,tables} (13 named figure types, 9 named table types, vector quality); paper/main.tex + sections/ + appendices/ (proofs, algorithms, prompts, schemas, replication instructions) + references.bib; README; pyproject.toml; ""replication package""; ""2–3 page research blueprint""","yes — ""Record model/version, prompt, context, temperature, seed, response, latency, tokens and cost. Cache reproducibly."" ; ""experiment manifests, Git revision and reproducible environment locks"" ; ""figure/table provenance"" (Writing gate) ; ""`docs/parameter_ledger.csv`: parameter, units, source, year/geography, moment, method, range, uncertainty.""","yes — ""6. **Writing gate:** complete paper, figure/table provenance, verified references, replication package and hostile-referee critique."" ; execution protocol step ""(e) discuss threats to interpretation""","no","partial — costs recorded and expensive compute requires authorisation, no numeric budget: ""expensive compute authorization"" ; ""Record … tokens and cost"" ; figure ""inference cost frontier"" ; ""Start on a single workstation; optional distributed inference only after a working local baseline.""","yes — ""Never fabricate results, citations, proofs, datasets, or calibrations. Label conjectures and illustrative simulations. Verify novelty before claiming it."" ; ""LLM output is never trusted as accounting truth."" ; ""Never claim a test passed unless it ran."" ; ""Do not call arbitrary LLM-generated market outcomes general equilibrium without verifying feasibility, market clearing and convergence."" ; ""Never treat simulated LLM preferences as verified human preferences.""","""Pre-register outcomes."" and `experiments/pre_registration.md` without any timestamping/registry mechanism — pre-registration by the same agent that runs the experiments is not verifiable. ""hostile-referee critique"" is produced by the same model. Scope tension between ""Aim for a parsimonious groundbreaking mechanism, not complexity for its own sake"" and three nested models (A, B, C) plus an LLM laboratory with eight roles and banking. Negative design constraint stated as judgement: ""The key innovation MUST NOT be simply adding AI to a Cobb–Douglas production function."" Non-verifiable: ""Aim at top economics/finance/computational economics journals without assuming acceptance.""","Prefatory note: ""(UQO Working Paper No. 25 — repository `Recherche-UQO/finance/agentic-general-equilibrium`, spbgit `wp25_uqo.git`. Programme supplied by the author on 2026-10-27; form follows `Recherche-UQO/HARMONISATION.md`.)"" — repository mapping added around the author's programme (dated 2026-10-27, same day as the programme).","Gate structure is the most formal of the theory mandates (six named acceptance gates with contents). Explicit per-stage protocol ""(a) state the precise scientific question, (b) write down assumptions, (c) implement and test, (d) show numerical evidence or mathematical proof, (e) discuss threats to interpretation, (f) log decisions and next tasks."""
++"WP26","WP26_CLAUDE.md","claude_md","https://git.spboucher.ai/raw/wp26_uqo/main/CLAUDE.md","2026-10-30T03:01:15Z","49afcce91475e5e145b4bd49d29486065c8c613a402b51fdb9baa43c88f4b330","real estate valuation / multi-agent AI (LLM agent deliberation for property valuation, Quebec transactions)","mixed (empirical evaluation + design-science multi-agent system + stylised welfare model)","""RQ1: Do specialized multi-agent systems outperform single LLMs, hedonic models, repeat-sales approaches, gradient boosting, and conventional AVMs on genuinely unseen Quebec transactions?"" (RQ1 of six)","yes — pre-specified primary outcome: ""Pre-register primary endpoints and hypotheses before running final test evaluations. Primary outcome: out-of-time absolute log-price error on arm's-length residential transactions. Co-primary or secondary: MAE in CAD, MdAPE, RMSE log, tail error (P90/P95), interval coverage, interval width, CRPS or pinball loss, cost and latency."" ; ""Propose falsifiable propositions about diversity, correlation, communication topology, and aggregation weights.""","partial — ""State identification assumptions and develop multiple operational definitions of value."" ; ""Transaction prices are noisy proxies, not ground-truth fundamental values."" ; ""Do not claim general existence/uniqueness theorems without proof.""","yes — ""A negative finding is publishable if the design is rigorous."" ; ""Report failure as rigorously as success."" ; placebo arm ""I. Random/shuffled-role or placebo deliberation"" ; ""Explicitly separate improvements from more tokens, more tools, more models, or more data."" ; ""Derive testable conditions under which deliberation reduces or increases mean-squared error""","yes — ""Phase 1: literature review and novelty map; identify close prior multi-agent real-estate/AVM papers and avoid unsupported priority claims."" ; ""Search and verify the relevant academic literature if browsing is available; otherwise flag bibliography for verification.""","yes — ""These are potential sources, not automatically accessible. Begin with a data-access audit"" ; ""Do not bypass authentication, scrape restricted endpoints, or infer consent. Do not send personal identifiers, private listing remarks, occupant information, or confidential records to external LLM APIs without authorization and suitable safeguards."" ; ""use environment variables and a local secrets manager, never put secrets in source control or manuscript"" ; ""Publish only de-identified or synthetic examples if redistribution is not authorized.""","yes — ""Only use features observable as of the valuation timestamp; prevent post-sale information and target leakage."" ; ""Comparable retrieval must be strictly as-of-date; no same-property future transfers, post-sale listings, or near-duplicate records leaking across partitions."" ; ""Use a chronological split with train/validation/test and a final untouched holdout"" ; ""Build a leakage-audit suite and adversarial tests."" ; ""No final-test-informed aggregation weights.""","yes — ""Implement and evaluate at least: A. Hedonic OLS / regularized regression (proper spatial/time controls) B. Repeat-sales or hierarchical hedonic … C. CatBoost/XGBoost/LightGBM tuned on validation only D. Strong tabular AVM … E. Single LLM, evidence-matched F. Independent LLM agents aggregated without communication … J. Cost-matched single-agent repeated reasoning and cost-matched ensemble""","9 (Phase 0–8 in ""Execution phases and acceptance gates"") plus seven ""Immediate first actions""","explicit — ""## Execution phases and acceptance gates Phase 0: inventory … Phase 5: run provider-balanced, information-matched, compute-matched randomized ablations on validation; freeze protocol. Phase 6: evaluate final temporal/spatial holdouts once, with clustered uncertainty and cost accounting. … Phase 8: produce manuscript"" ; ""After each phase, produce a concise `reports/phase_XX.md` with completed artifacts, failed checks, decisions, outstanding assumptions, and exact commands."" (ordering and freeze/once-only conditions; no per-phase pass criteria beyond the phase reports)","partial — escalation rule and debate-round limits only: ""Stop and ask targeted questions only for indispensable access or decisions; otherwise proceed with explicitly documented defaults. Never fabricate completion."" ; ""Limit debate to pre-specified rounds (e.g. 0, 1, 2, 4)"" ; src/debate/ ""stopping rules""","partial — ""privacy-aware caching"" ; ""bounded retries"" ; ""store response artifacts securely and measure run-to-run variation"" ; no resume/checkpoint instruction","README.md, CLAUDE.md, pyproject.toml, .env.example, LICENSE; configs/ (splits, model IDs, treatments, budgets, seeds); src/{data,features,baselines,agents,debate,evaluation,experiments}; tests/ (unit/integration/leakage/mock-provider); notebooks/; paper/ (LaTeX manuscript, BibTeX, figures, tables, appendices, compilation script, PDF); reports/ (experiment cards, failure analyses, model cards, limitations, phase_XX.md); research_plan.md, data_access_checklist.md, preregistration.md, architecture.md; data dictionaries; clearly labeled synthetic dataset and mock LLM adapters; CLI (ingest, validate, train, run agents, evaluate, generate tables, build PDF); limitations register; API-cost budget","yes — ""Store raw snapshots immutable with data dictionaries, versioned transformations, provenance, licenses, and record-level lineage."" ; ""Record model IDs, API parameters, prompt hashes, temperature, tool calls, token usage, cost estimates, and latency."" ; ""All figures/tables must come from versioned outputs."" ; ""Pin dependencies, set seeds, capture versions and hardware""","yes — roles ""11. Data-quality, leakage, and provenance auditor 12. Independent skeptical reviewer / red-team agent"" ; ""Phase 7: human benchmark if authorized, welfare model, robustness, interpretability audits."" ; ""Build a leakage-audit suite and adversarial tests.""","no — ""Agents are **simulated analytic roles**, not licensed Quebec appraisers or real lenders; never misrepresent professional certification."" concerns role representation, not AI-use disclosure for the paper","yes — ""Present an API-cost budget with small pilot, medium validation, and final experiment tiers, without inventing vendor prices."" ; ""token budgets"" ; ""compare all-Claude, all-OpenAI, and mixed-provider committees at matched cost and information budgets"" ; ""async concurrency with strict limits""","yes — ""Never invent data, results, citations, human assessments, API capabilities, or model performance."" ; ""Never fabricate empirical tables: use placeholders clearly marked `NOT YET ESTIMATED` until generated by scripts."" ; ""No p-hacking or test-set prompt tuning."" ; ""do not fabricate a human benchmark"" ; ""Do not claim deterministic reproducibility for hosted LLMs""","""Pre-register primary endpoints and hypotheses before running final test evaluations"" and `preregistration.md` have no registry/timestamp mechanism; the same agent writes and executes. ""evaluate final temporal/spatial holdouts once"" — the once-only rule has no enforcement mechanism. Human-dependent arm: ""recruit licensed appraisal professionals and/or trained human appraisers under an approved protocol"" cannot be executed autonomously (mitigated: ""mark this arm as pending and do not claim it was performed""). ""Respect Quebec privacy requirements and organizational research-ethics review where applicable"" is not operational for an agent. Tension between ""make outputs reproducible"" and ""Do not claim deterministic reproducibility for hosted LLMs"" (resolved by storing response artifacts). Scope (12 default roles, up to 20, ten treatment arms, three provider configurations) versus an API budget that is to be proposed rather than given.","Explicit post-hoc section: ""## Repository conventions (added by the implementation, 2026-10-27) - UQO Working Paper No. 26, folder `Recherche-UQO/immobilier/agentic-valuation-quebec`, git remote `gitsrv:srv/git/wp26_uqo.git`. - Python package: `src/agentic_valuation/` (installed editable: `pip install -e .`). CLI: `av <command>` (see README). - Secrets only in `.env` (gitignored). Snapshot provenance in `data/raw/README.md`. Every result file is produced by a numbered script in `scripts/`.""","Most complete empirical-design mandate in the corpus: pre-specified primary outcome, placebo arm, cost-matched controls, leakage suite, API budget tiers, phase reports. Research-only clause: ""No live autonomous trading, lending decisions, or real estate offers""."
++"WP28","WP28_CLAUDE.md","claude_md","https://git.spboucher.ai/raw/wp28_uqo/main/CLAUDE.md","2026-10-30T03:01:15Z","c6c50e319f971b01fa0662840feef177d7bc896526956b0b7a3cf7a83f36246d","econometric theory / statistical computing (distributed maximum likelihood, higher-order bias correction, communication-computation trade-off)","mixed (mathematical theory + Monte Carlo simulation + distributed-systems benchmark)","""For regular nonlinear likelihood models with `N` observations and `K` distributed partitions, how can additional local computation replace global network synchronization while preserving the first-order inferential properties of centralized maximum likelihood?""","yes — candidate theorem and proof queue: ""K = o( N^[ 1 - 1 / ( 2^(q+1) * r ) ] )"" ; ""**Theorem 3 — Exact exponential boundary.** Derive all Gamma moments needed and establish the nonzero limit `-4 theta_0 c^4` for half-block jackknife + one global Newton step when `K ~ c N^(7/8)`."" ; Lemmas 1–7, Theorems 1–4, a failure Proposition and a Corollary are enumerated","yes — baseline model assumptions are listed in the mandate and an assumptions file is required: ""### 5.1 Baseline model Start with i.i.d. observations and a correctly specified fixed-dimensional regular likelihood: - `X_1,...,X_N ~ P_{theta_0}` independent; - `theta_0` interior to an open parameter set in `R^p`; - information matrix `I(theta_0)` positive definite; …"" ; ""State precisely which theorem uses which assumption."" ; theory/ASSUMPTIONS.md","yes — ""A dead end with a mathematical explanation is better than a fabricated breakthrough. A strong negative result or counterexample may itself become the contribution."" ; ""**Gate 4:** at least one nontrivial theorem beyond a rephrasing of known one-step-plus-jackknife results is fully proved, **or** the report explicitly concludes that novelty is not yet proved and reframes the paper around validated insights and counterexamples."" ; ""a counterexample or violation for at least one essential assumption"" ; ""**Proposition — Failure/counterexample.**"" ; ""Track E — Lower bounds, counterexamples and impossibility""","yes — ""Do not claim that distributed gradients, one-step Newton, split-sample jackknife, Richardson extrapolation, `K = o(N^{7/8})`, or `K = o(N^{11/12})` are intrinsically novel. Investigate who has already obtained each idea or bound."" ; `literature/PRIOR_ART_MATRIX.csv` with ""novelty_threat"" column ; ""**Gate 1:** no unqualified novelty language is allowed until the closest competing theorems have been inspected and cited."" ; six anchor references with DOIs to verify","yes — ""never scrape or transmit private material to external services without authorization"" ; ""Do not expose API keys, SSH keys, passwords, endpoint secrets or private datasets in source files, logs, paper, artifacts or commits."" ; ""Use the researcher's available CPU/GPU/cluster systems only when provisioned and authorized."" ; ""Avoid destructive commands, forced pushes and unrelated repository modifications.""","no — not applicable (synthetic Monte Carlo); only a literature cutoff: ""**Current date for literature cutoff:** 2026-10-28; search current material rather than assuming bibliographic currency.""","yes — ""Full centralized maximum likelihood, using a trusted optimizer and tight tolerances."" ; ""Credible implementations of at least two strong literature baselines where practical, such as surrogate likelihood/CSL and another relevant distributed estimator."" ; audit items ""New method comparisons include ordinary local averaging + two Newton rounds."" and ""A baseline with exact sufficient-statistic aggregation is shown for applicable families.""","13 (Phase 0–12, each closed by Gate 0–12)","explicit — ""**Execute in order, completing one phase's acceptance tests before starting the next.**"" ; ""**Gate 0: pass only when** all old claims and artifacts are inventoried, every existing number is tagged as replicated / not replicated / unavailable, and a WP28 workspace exists without clobbering other work."" ; ""**Gate 6:** all critical tests pass and the code reconstructs original-paper figures within stated tolerances before large experiments begin."" ; pass/fail rubric in section 16","yes — ""Complete the project from initial inspection through PDF validation; stop only when a real blocker makes an indispensable step impossible. Document blocked items, and continue with whatever can be completed."" ; ""**Gate 11:** produce `FINAL_REVIEW.md` with a final verdict `ready for human review / theoretically incomplete / novelty unresolved / computational evidence insufficient`"" ; ""Never assert results from jobs that have not finished."" ; ""Do not publish a preprint or submit to a journal without explicit instruction.""","partial — phase status and run manifests, failure tracking, no resume instruction: ""After completing each phase, append to `PHASE_STATUS.md`: Phase: Start/finish timestamp: What was actually done: Artifacts created: Tests passed: Tests failed: …"" ; ""Track optimization failures rather than discard failed replications silently."" ; ""straggler failures"" to be bounded","article (~25–40 pages) + mathematical appendix; paper/{main.tex,preamble.tex,sections,appendices,references.bib,figures}; literature/{PRIOR_ART_MATRIX.csv,PRIORITY_AND_GAP.md}; theory/{THEOREM_AUDIT.md,ASSUMPTIONS.md,proofs_check.py,symbolic_checks.py}; src/pamle/ (10 modules) with common-interface estimators; experiments/{configs,run_monte_carlo.py,run_benchmarks.py,run_ablations.py,EXPERIMENT_MANIFEST.csv}; tests/ (7 files); results/{raw,processed,tables,figures,resource_profiles}; scripts/{setup.sh,reproduce_all.sh,compile_paper.sh,validate_pdf.sh}; PROJECT_MAP.md, SOURCE_PROVENANCE.md, PHASE_STATUS.md, research_log.md, LIMITATIONS_AND_CLAIMS.md, FINAL_REVIEW.md, FINAL_DELIVERY.md; proof ledger; 8 figures + 6 tables programme; delivery bundle (PDF, .tex, source-and-reproducibility zip, Research_Summary, Theorem_Audit, Prior_Art_and_Novelty, Experiment_Results.csv, Final_Review); nine-point closing report","yes — ""Extract the existing archive to an immutable `references/initial_pamle/` directory if available; record checksums."" ; run manifest ""run_id,git_commit,python_version,dependency_lock_sha256, seed,N,K,m,p,DGP,method,optimizer,tolerance,replicates, parallel_backend,node_count,hardware,OS,wall_seconds, communicated_bytes,rounds,status,raw_output_sha256"" ; ""Every table value has a traceable input/output path."" ; ""Project checksums and commit state are captured.""","yes — ""## 14. PHASE 11 — Adversarial referee and proof audit Simulate at least three independent review perspectives; log objections and responses"" (Referees A, B, C) ; ""For every objection, respond with (a) exact location in paper/code, (b) correction made, (c) new proof/test if needed, and (d) residual limitation."" ; THEOREM_AUDIT.md with ""proved / conditional / open / false"" labels","yes — ""Confirm the applicable journal/venue authorship and AI-disclosure policy before a submission version; never falsely represent an AI system as a legally accountable human author."" ; ""The journal submission version may need GPT-6.1 removed from the formal author list and placed in transparent AI-use disclosure; **flag this for the human author rather than concealing the difference**."" ; audit item ""AI contribution disclosure and human review requirements are explicit."" ; rubric row ""Attribution | Simon-Pierre Boucher properly named; AI role transparently recorded""","partial — resources bounded by availability, no numeric cap: ""Continue to a publishable-quality, internally consistent draft within the resources and access actually available."" ; ""`N`: logarithmic sequence … chosen based on available RAM/compute"" ; blocker category ""(iv) exhausted compute budget"" ; ""where measured, electricity and dollar-cost assumptions""","yes — ""No invented theorem, proof, assumption, dataset, benchmark, reference, DOI, parameter estimate, speedup, significance level, or journal acceptance."" ; ""Do not call an in-memory mathematical partition a distributed cluster benchmark."" ; ""Never assert results from jobs that have not finished."" ; ""Do not silently describe these as 1,000 replications or as physical-cluster experiments."" ; ""Never explicitly invert the Hessian in production code""","(1) ""Simulate at least three independent review perspectives"" versus ""do not have the same model merely declare its own work accepted"" — the simulated referees are by construction the same model, so independence is not operational. (2) Authorship: ""Preserve this provenance and the user's requested GPT-6.1 attribution in the working draft"" (GPT-6.1 in the author block) versus ""never falsely represent an AI system as a legally accountable human author"" — resolved only by deferring to the human for the submission version. (3) ""Research program funding line to include **only if confirmed** for this particular WP28 project"" — confirmation requires the human; the agent cannot satisfy it. (4) Compute: ""target at least 1,000 per **main statistical design**"" across ≥4 N values, several K, several p, ≥3 models and DGP conditions with no budget beyond ""where practical"". (5) Conditional bonuses (""If Lean is installed, formalize at least one deterministic algebraic lemma as a bonus, not as a blocker""). (6) Rubric criteria such as ""substantive new result or defensible negative discovery"" and ""journal-style typography"" are not script-checkable. (7) ""The researcher may have access to a MacLustr collection"" together with ""never assume the existence … of a MacLustr node"" is consistent but leaves hardware scope undetermined until run time.","none identified as post-hoc; the header records prior provenance rather than an addition: ""**Prior exploratory draft credits:** Simon-Pierre Boucher and GPT-6.1. Preserve this provenance and the user's requested GPT-6.1 attribution in the working draft.""","Longest mandate in the corpus (8 809 words, 76 headings, 27 code blocks, 2 Markdown tables). The only mandate with an AI-use/authorship disclosure instruction, a per-phase acceptance-test rule (""completing one phase's acceptance tests before starting the next""), a pass/fail rubric table, a multi-referee audit phase and a verdict vocabulary. Explicitly a re-examination of a prior AI-assisted draft (""This is NOT an instruction to paraphrase the exploratory note."")."
++"WP27_BRIEF","WP27_BRIEF_DESIGN-BRIEF.md","design_brief","https://git.spboucher.ai/raw/wp27_uqo/main/docs/DESIGN-BRIEF.md","2026-10-30T03:01:15Z","72dc7ab846c6d8f8e0cc4101b6bdb600884179e9467c43fddfd3aece395e236e","finance / market microstructure (agent-based limit-order-book simulation with LLM trading agents)","simulation (agent-based market simulator with a stylised one-period theoretical model)","none — the brief describes ""what the simulator *actually* implements"" and lists hypotheses H1–H6; no research question is stated","yes — ""Hypotheses the simulator tests: H1 informativeness falls in the share using a single shared model (holding information quality fixed) and rises in M_eff; H2 order-flow correlation and synchrony rise in φ; H3 bubble/crash frequency and liquidity spirals rise with φ × leverage; H4 prompt/model diversity restores part of the loss; H5 communication channels raise synchrony; H6 circuit breakers and position limits cut crash depth at a cost in time-to-absorb."" ; objects defined: ""price informativeness τ = 1 − Var(v | Y)/σ_v²; pricing error Var(P − v); order-flow correlation ρ_x = Corr(x_i, x_j); synchrony; effective number of models M_eff = 1/Σ_g φ_g²""","yes — model assumptions stated in the text: ""One-period, one asset, liquidation value v ~ N(0, σ_v²) around the public PV. N traders receive s_i = v + ε_i with ε_i = η_{g(i)} + u_i"" ; ""β > 0 a fixed response intensity; the behavioral assumption mirrors rule-like LLM policies and is relaxed in a CARA extension"" ; ""A competitive, risk-neutral market maker sets P = E[v | Y]""","partial — directional hypotheses are testable and a seed/power design is sketched (""≥ 30 seeds exploratory, power-based for confirmatory""), but no null-result condition is stated","no — the brief is addressed to ""the sub-agents writing the literature review, theory and pre-analysis plan"" but imposes no literature or novelty requirement","no — synthetic data; providers listed without access rules: ""Providers: OpenAI (Responses API, json_schema), Anthropic (Messages API, json_schema), mock (seeded rule with injected invalid answers).""","yes — information-set discipline inside the simulator: ""Agents never see V_t."" ; ""No agent sees future draws."" ; ""orders arrive at decision time + latency and are processed in arrival order""","partial — control arm and matched design rather than named baseline methods: ""Matched pairs across `llm_share` share every stream except the LLM slot draw."" ; ""mock (seeded rule …)"" provider ; ""frozen decision library replay vs online calls""","none","none","no","no","none stated explicitly; implied: literature review, theory write-up (""to be written up by the theory agent, must map to the above"") and a pre-analysis plan; the simulator reports ""Realized LLM share of agents, orders and volume""","partial — ""Randomization: master seed → independent streams (fundamentals, news, slots, policy assignment, LLM slot assignment, activation, tie-breaks, per-agent RNG)."" ; ""Invalid decisions = hold, logged."" ; ""exact conservation checked after every fill"" ; no manifest/hash requirement","no","no","partial — budgets appear as experimental treatments, not as resource limits: ""equal token / USD / decision-opportunity budgets""","no","Not a mandate, so few directives. Non-operational: ""power-based for confirmatory"" without a power target or effect size; ""must map to the above"" asserts a correspondence between the one-period theory and the multi-tick simulator (fixed β market orders versus a continuous double auction) that the brief does not check.","none identified — but the whole document is by nature post-implementation (""It describes what the simulator *actually* implements (src/market_agents)"").","Supplementary design brief, not a CLAUDE.md; coded separately and excluded from all ""of 11"" counts. WP27's own CLAUDE.md returned HTTP 404 and is not represented in the corpus. 820 words, 3 headings."
++"WP15_PROMPT","WP15_PROMPT_PROMPT_INITIAL_CLAUDE.md","duplicate","https://git.spboucher.ai/raw/wp15_uqo/main/docs/PROMPT_INITIAL_CLAUDE.md","2026-10-30T03:01:13Z","2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77","see WP15","see WP15","see WP15 (byte-identical duplicate; not coded separately)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)","docs/PROMPT_INITIAL_CLAUDE.md in the wp15_uqo repository is byte-identical to docs/CLAUDE.md in the retrieved snapshot. Recorded once in the corpus manifest for completeness; counted as one mandate (WP15) in all tallies. The two paths suggest the initial prompt was preserved verbatim as the working CLAUDE.md, but the snapshot alone cannot establish which file came first."

added research/corpus_coding_protocol.md +98 −0

++# WP29 — Content-coding protocol for the mandate corpus (codebook)
++
++Version 1.0 — 2026-10-29. Applies to `research/corpus/*.md` (13 files, hashes in `research/corpus_manifest.csv`).
++
++## 1. Scope, provenance and limits of the coding
++
++- **Corpus.** Thirteen files retrieved on 2026-10-29 (UTC 2026-10-30T03:01). They contain **eleven distinct CLAUDE.md mandates** (WP15, WP18, WP19, WP20, WP21, WP22, WP23, WP24, WP25, WP26, WP28), **one supplementary design brief** (WP27 `DESIGN-BRIEF.md`, which is not a CLAUDE.md) and **one byte-identical duplicate** (`WP15_PROMPT_PROMPT_INITIAL_CLAUDE.md`, SHA-256 `2dd19c59…1b77`, identical to `WP15_CLAUDE.md`). The duplicate is recorded but coded once. WP27's own CLAUDE.md returned HTTP 404 and is therefore absent; nothing is inferred about it.
++- **Who coded.** A **single automated + LLM-analyst pass** (Claude Code subagent, model Claude Fable 5.1, 2026-10-29). The automated variables were computed by `research/code_corpus.py`; the analyst variables were produced by the same subagent after reading each document in full. **There was no human second coder, so no intercoder reliability (kappa, agreement rate) is reported and none should be inferred.** The analyst columns are the subagent's reading of the text and may contain judgement errors; every yes/partial/no judgement is accompanied by a verbatim quote so that it can be re-checked against the source.
++- **What the coding can and cannot support.** The corpus is **small (11 mandates) and researcher-selected** (the author's own instruction files for earlier UQO working papers). It is used to **motivate design patterns** for WP29's protocol, **not to estimate the prevalence** of practices across autonomous-research mandates in general. Denominators are reported as "x of 11 mandates" and no inferential statistics are computed.
++- **What is coded.** Only what the instruction documents say. The coding does **not** assess whether the earlier working papers were good, whether the instructions were followed, or whether the papers were produced without human assistance. A mandate that *requires* a provenance manifest is coded as requiring one; whether such a manifest exists is outside this coding.
++- **Verbatim vs interpretation.** In `corpus_coding.csv`, verbatim material is always enclosed in double quotes `"…"`; everything outside the quotes is analyst interpretation. Quotes are copied exactly (including capitalisation, spelling and Markdown markup) and truncated with `…` only when the sentence exceeds roughly 40 words; nothing is paraphrased inside quotation marks.
++
++## 2. Files produced
++
++| File | Produced by | Content |
++|---|---|---|
++| `research/code_corpus.py` | script (stdlib only) | computes the automated variables |
++| `research/corpus_automated.csv` | script | 13 rows (one per file, duplicate flagged) |
++| `research/corpus_coding.csv` | analyst (written with `csv` module, RFC-4180) | 13 rows: 11 mandates + 1 design brief + 1 duplicate-note row |
++| `research/corpus_summary.md` | analyst | counts, patterns, excerpts |
++
++## 3. Variable types
++
++- **(A) automated** — computed by `code_corpus.py` from the file bytes; fully reproducible by re-running the script.
++- **(I) analyst-interpreted** — a categorical judgement made after reading the whole document; always accompanied by a supporting verbatim quote or by `none`.
++- **(V) verbatim extraction** — text copied from the document, or metadata copied from `corpus_manifest.csv`.
++
++## 4. Automated variables (`corpus_automated.csv`) — type (A)
++
++| Variable | Definition |
++|---|---|
++| `file` | file name in `research/corpus/` |
++| `wp` | leading `WP\d+` token of the file name |
++| `document_type` | `claude_md`; `design_brief` if the name contains `BRIEF`; `duplicate_claude_md` if the SHA-256 equals that of an earlier file in sorted order |
++| `duplicate_of` | file name of the first file with the same SHA-256, else empty |
++| `sha256`, `sha256_matches_manifest` | SHA-256 of the bytes; `yes` if equal to the manifest value |
++| `manifest_url`, `manifest_retrieved_utc` | copied from `corpus_manifest.csv` |
++| `bytes`, `lines`, `words` | byte length; newline count (+1 if no trailing newline); `len(text.split())` |
++| `n_headings` | lines matching `^#{1,6}\s+\S` (ATX Markdown headings) |
++| `n_phase_labels_distinct` | number of distinct N in matches of `\bphase\s*(\d+)` (case-insensitive) |
++| `n_stage_labels_distinct` | distinct capital letters X in `(?i:\bstage)\s+([A-Z])\b` |
++| `n_gate_labels_distinct` | distinct N in `\bgate\s*(\d+)` (case-insensitive) |
++| `n_numbered_items_in_phase_sections` | numbered list items (`^\s*\d+\.\s`) located inside a section whose heading contains one of *phase(s), stage(s), sequence, execution order, work sequence, execution protocol, execution phases, milestones*; a section extends to the next heading of equal or higher level |
++| `n_numbered_items_total` | all numbered list items in the file |
++| `n_numbered_phases` | mechanical phase count. Rule, in order: distinct `Phase N` labels if any; else distinct `Stage X` labels; else distinct `Gate N` labels; else numbered items inside phase/sequence sections. `n_numbered_phases_rule` records which branch fired. Known limits: WP20 states a 6-step "Mandatory sequence: (1) … (6)" inline with parentheses (not matched) and 7 numbered gates (matched → 7); WP21 lists steps "A … G" inline (→ 0); WP28's "Stage A–G" are message-protocol steps, not phases, but its 13 `Phase N` labels take precedence. The analyst variable `n_phases_declared` is authoritative. |
++| `n_tables` | Markdown table separator rows (`|---|---|`): one per table |
++| `n_code_blocks` | pairs of ``` fences |
++| `language_of_document`, `fr_token_share` | heuristic: share of French function-word tokens (le, la, les, des, est, une, ce, cette, dans, pour, sur, avec, pas, aux, qui, que, ci-dessous, dossier, appelle, correspond, auteur) among French + English function-word tokens; `en` < 0.015 ≤ `mixed` < 0.50 ≤ `fr`. Tokens shared with English in this corpus (*conventions, journal, du, et*) are excluded. Result: all files `en` except WP18 and WP20 (`mixed`: a French prefatory paragraph on an English body). |
++| `mentions_<family>` / `mentions_<family>_n` | boolean / count of case-insensitive regex matches. Families: `preregist` `pre-?regist\w*`; `estimand` `\bestimands?\b`; `hypothes` `\bhypothes\w*`; `falsif` `\bfalsif\w*`; `baseline` `\bbaselines?\b`; `leakage` `leakage|look-?ahead|look ahead|point-in-time|point in time`; `provenance` `provenance|hash(es|ed|ing)?|sha-?256|checksums?`; `reproduc` `\breproduc\w*`; `stopping` `stopping|stop rules?|stop conditions?|termination|stop and|stop if|stop only`; `recovery` `recover\w*|resum\w*|checkpoint\w*`; `gate` `gates?|acceptance|criteria`; `exhibit` `screenshots?|figures?|tables?`; `disclos` `disclos\w*|AI-use|AI use|AI contribution`; `budget` `budgets?|costs?|USD|\$`; `license` `licen[cs]e[sd]?|licensing`; `seed` `seeds?|seeded`. **Caveat:** these are mention counts, not judgements. The `budget` family is inflated where "cost" is a research object (WP20: the *cost approach* to appraisal, 42 hits; WP24: adjustment *costs*); the `gate` family also catches "criteria" used in "inclusion criteria" (WP21). The analyst variables, not these counts, carry the substantive coding. |
++
++## 5. Analyst variables (`corpus_coding.csv`)
++
++Allowed values for every yes/no-type variable are **`yes` / `partial` / `no`** (plus `n/a` for the duplicate row). `partial` means the mandate addresses the feature but only indirectly, conditionally, or without an operational mechanism (e.g. it *requires* that assumptions be stated but does not state them). Each value is followed, in the same cell, by a supporting verbatim quote `"…"` or by an analyst remark outside quotes.
++
++| Variable | Type | Definition and allowed values |
++|---|---|---|
++| `wp` | V | WP15 … WP28; `WP27_BRIEF` for the design brief; `WP15_PROMPT` for the duplicate row |
++| `file` | V | corpus file name |
++| `document_type` | I | `claude_md` (an instruction file addressed to Claude Code); `design_brief` (WP27, an internal description of the simulator, not a mandate); `duplicate` |
++| `source_url`, `retrieved_utc`, `sha256` | V | copied from `corpus_manifest.csv` |
++| `research_domain` | I | free label: real estate economics; finance / market microstructure; macro-finance theory; econometric theory / statistical computing; applied econometrics (meta-analysis); etc. |
++| `research_type` | I | `empirical`, `theoretical`, `simulation`, `design-science`, or `mixed (…)` with the components in parentheses |
++| `explicit_question` | V | the research question verbatim (≤ 40 words; the first RQ when several are listed), or `none` when the document states objectives but no question |
++| `explicit_estimand_or_hypotheses` | I + V | `yes` = named estimand(s) / hypotheses / theorems to prove, or a pre-specified primary outcome; `partial` = research questions or candidate contributions treated "as hypotheses" without formal statements; `no` |
++| `stated_assumptions` | I + V | `yes` = the mandate itself states substantive model/identification assumptions or requires a numbered assumptions register; `partial` = requires that assumptions be stated with each result but does not state them; `no` |
++| `falsification_or_null_result_conditions` | I + V | `yes` = explicit instruction to attempt disproof, required counterexamples/placebos, or a declared acceptable null outcome; `partial` = failures/negative results must be reported but no disproof condition is defined; `no` |
++| `literature_or_novelty_requirement` | I + V | `yes` = verified literature review and/or novelty gate required before claiming a contribution; `no` |
++| `data_access_controls` | I + V | `yes` = explicit rules on credentials, secrets, scraping, licensing, privacy or permission; `partial` = only generic "do not pretend access / log unavailable files"; `no` |
++| `temporal_information_controls` | I + V | `yes` = look-ahead / as-of-date / point-in-time / vintage / chronological-split rules; `partial` = vintage awareness without a leakage rule; `no` (including not applicable) |
++| `baseline_requirement` | I + V | `yes` = named comparison baselines/benchmark methods; `partial` = a "baseline scenario" or benchmark model, not comparison methods; `no` |
++| `n_phases_declared` | I | integer count of execution phases/stages the document declares, or `none`; the labelling scheme is noted in parentheses |
++| `dependency_or_gate_structure` | I + V | `explicit` = named gates or "do not start X until Y" conditions; `implicit` = ordering only; `none` |
++| `stopping_rules` | I + V | `yes` = conditions under which work must stop / a verdict be issued; `partial` = escalation or fallback rules only ("if blocked, document and deliver partial result"); `no` |
++| `recovery_or_resume_instructions` | I + V | `yes` = resumable/idempotent/checkpointed execution required; `partial` = phase logs, caching or retry limits that enable resumption only indirectly; `no` |
++| `artifact_requirements` | V (list) | semicolon-separated list of required deliverables (paper, tables, figures, code, data, README, logs, audits …) |
++| `provenance_requirements` | I + V | `yes` = hashes/manifests/pinned revisions/traceable number-to-file mapping/versioned outputs; `partial` = "every number must map to a result file" or logs without hashes/manifests; `no` |
++| `review_or_self_audit_stage` | I + V | `yes` = a dedicated audit/referee/validation stage; `partial` = audit items inside a final checklist only; `no` |
++| `disclosure_or_AI_use_statement` | I + V | `yes` = an AI-use / authorship-disclosure instruction; `no` (statements about prose style, e.g. "not AI-generated prose", are **not** disclosure) |
++| `budget_or_resource_limits` | I + V | `yes` = an explicit budget (API cost tiers, token budgets) or numeric resource cap; `partial` = resource estimation/profiling or "within available resources" without a cap; `no` |
++| `integrity_prohibitions` | I + V | `yes` = explicit "never invent / fabricate …" prohibitions; `no` |
++| `contradictory_or_non_operational_requirements` | I | free text listing instructions that cannot be verified from the document's own criteria, or that conflict with another instruction; `none found` otherwise |
++| `statements_added_during_implementation` | V | passages that the document itself marks as added later or reconstructed (headers such as "added 2026-10-27", bracketed reconstruction notes, prefatory mapping notes dated after the programme); `none identified` otherwise |
++| `analyst_notes` | I | anything else relevant |
++
++## 6. Procedure
++
++1. Compute hashes and automated variables (`python3 -I research/code_corpus.py`); confirm every hash equals the manifest (`sha256_matches_manifest = yes` for all 13 files) and identify duplicates.
++2. Read each distinct document in full (the two long documents, WP15 at 2 165 lines and WP28 at 1 055 lines, were read in consecutive line ranges).
++3. For each analyst variable, locate the supporting passage, copy it verbatim, then assign the value. Where no passage exists, assign `no`/`none` without quote.
++4. Write `corpus_coding.csv` with Python's `csv` module (`QUOTE_ALL`), re-read it with the same module and check the row count (13) and column count.
++5. Tabulate counts over the 11 mandates only (brief and duplicate excluded) for `corpus_summary.md`.
++
++## 7. Known weaknesses of this coding
++
++- Single coder; no reliability statistics (see §1).
++- Keyword counts are crude (see the `budget`/`gate` caveats in §4) and are reported only as descriptive context.
++- The `partial` category absorbs heterogeneous cases; the quotes allow a reader to re-cut the categories.
++- The design brief (WP27) is a different genre; its row is kept separate and excluded from all "of 11" counts.

added research/corpus_summary.md +86 −0

++# WP29 — Descriptive summary of the mandate corpus
++
++Coding date 2026-10-29. Inputs: `research/corpus/` (13 files, hashes verified against `corpus_manifest.csv`), `research/corpus_automated.csv` (script), `research/corpus_coding.csv` (analyst). Codebook: `research/corpus_coding_protocol.md`.
++
++**Denominator.** Eleven distinct CLAUDE.md mandates (WP15, WP18–WP26, WP28). `WP15_PROMPT_PROMPT_INITIAL_CLAUDE.md` is byte-identical to `WP15_CLAUDE.md` (same SHA-256) and is counted once. The WP27 design brief is a different genre (a post-implementation description of a simulator, not an instruction file) and is excluded from every "of 11" count; WP27's CLAUDE.md was not retrievable (HTTP 404) and nothing is inferred about it. Coding was a single automated + LLM-analyst pass with no human second coder; no intercoder reliability is claimed. The corpus is small and researcher-selected: the counts motivate design patterns for WP29 and do not estimate prevalence.
++
++**Important limit.** These documents are instructions. They say what the author asked Claude Code to do. They do **not** establish how the earlier working papers were actually executed, whether the instructions were followed, or whether the papers were produced without human assistance (several mandates in fact anticipate human steps: expert raters, human review of ambiguous studies, credential provision, confirmation of funding lines).
++
++## 1. Length and form
++
++| Statistic (11 mandates) | Value |
++|---|---|
++| Words — min / median / max / mean | 1 092 (WP19) / 1 966 (WP22) / 8 809 (WP28) / 2 879 |
++| Sorted word counts | 1 092, 1 114, 1 132, 1 358, 1 671, 1 966, 2 240, 2 285, 2 314, 7 685, 8 809 |
++| Markdown headings — min / median / max | 1 (WP21) / 19 (WP25, WP26) / 151 (WP15) |
++| Code blocks | 0 in six mandates; 1 in WP18, WP20, WP25; 12 in WP15; 27 in WP28 |
++| Markdown tables | 0 except WP28 (2) |
++| Language | English body in all; WP18 and WP20 carry a French prefatory paragraph (heuristic label `mixed`) |
++
++Two mandates (WP15, WP28) are an order of magnitude longer than the other nine, which cluster between 1 100 and 2 300 words. Length tracks the amount of engineering prescription (directory trees, manifests, CLI commands, rubrics), not the number of substantive research rules.
++
++## 2. Feature counts
++
++Values are the analyst codes from `corpus_coding.csv` (first token of each cell). "partial" is defined in the codebook as present but indirect, conditional or without an operational mechanism.
++
++| Feature | yes | partial | no | Note |
++|---|---|---|---|---|
++| Explicit research question (verbatim) | 10 of 11 | — | 1 (WP22 states objectives, not a question) | |
++| Explicit estimand or hypotheses | 7 | 4 | 0 | H1–H6 lists (WP25), theorem queues (WP24, WP28), pre-specified primary outcome (WP26), core-estimand section (WP22) |
++| Assumptions stated in the mandate | 2 (WP24, WP28) | 8 | 1 (WP23) | most mandates *require* assumptions to accompany every result without stating them |
++| Falsification / null-result conditions | 6 | 4 | 1 (WP21) | "attempt to disprove its own central claim" (WP15); "A negative finding is publishable" (WP26) |
++| Literature / novelty requirement | 11 | 0 | 0 | universal; usually a "novelty gate" plus a verified literature matrix |
++| Data-access controls | 7 | 4 | 0 | credentials, scraping, privacy, licensing rules; `partial` = theory papers with only "do not pretend access" |
++| Temporal-information controls | 6 | 1 | 4 | universal among the data-based mandates (WP15, WP20, WP22, WP23, WP25, WP26); not applicable to the pure-theory ones |
++| Baseline / benchmark requirement | 8 | 3 | 0 | `partial` = baseline scenario or benchmark model rather than comparison methods |
++| Phases declared | 10 of 11 | — | 1 (WP23) | 5 to 13 phases; median 7 |
++| Dependency / gate structure | 10 explicit | 1 implicit (WP21) | 0 none | but only 2 (WP15, WP28) tie a gate to an executable test |
++| Stopping rules | 2 (WP15, WP28) | 6 | 3 | `partial` = "if blocked, document and deliver a partial result" |
++| Recovery / resume instructions | 2 (WP15, WP25) | 6 | 3 | `partial` = progress logs, caching, retry limits |
++| Provenance requirements | 9 | 2 | 0 | manifests with hashes in WP15, WP28; prompt/model/seed logging in WP25, WP26 |
++| Review / self-audit stage | 10 | 1 (WP23) | 0 | results audits, adversarial referee reports (WP24, WP28), red-team agent (WP26) |
++| AI-use / disclosure statement | 1 (WP28) | 0 | 10 | |
++| Budget / resource limits | 1 (WP26) | 3 | 7 | WP26 asks for API-cost tiers; others estimate or profile without a cap |
++| Integrity prohibitions ("never invent …") | 11 | 0 | 0 | universal, usually in the opening paragraph |
++| Pre-registration mentioned (automated keyword) | 4 (WP19, WP21, WP25, WP26) | — | 7 | none specifies a registry or timestamp mechanism |
++| Random seeds mentioned (automated keyword) | 11 | — | 0 | |
++| Statements marked as added/reconstructed | 6 | — | 5 | explicit "added" sections in WP24, WP26; reconstruction notes in WP23; prefatory repository-mapping notes in WP18, WP20, WP25 |
++
++## 3. Recurring design patterns (what the mandates consistently require)
++
++1. **Role framing + integrity preamble.** Every mandate opens by casting Claude Code as a multidisciplinary "research team" and immediately prohibits fabrication of data, results, citations, proofs or significance (11 of 11). The prohibition is the single most stable element of the corpus.
++2. **Ordered phases ending in "only then write the paper".** Ten of eleven declare 5–13 phases; nine make manuscript drafting conditional on validated results ("Never write a result in the paper before it has been generated and verified from a saved machine-readable output", WP15). Gates are named in ten mandates, most explicitly as numbered acceptance gates (WP20: Gates 1–7; WP25: six named gates; WP28: Gates 0–12).
++3. **Verified-literature / novelty gate before contribution claims** (11 of 11), typically with a literature matrix file and a "do not call it novel unless …" clause.
++4. **Audit and self-review as a phase** (10 of 11): a results audit mapping every claim to a file (WP18, WP19), an adversarial referee report (WP24), three simulated referees with a verdict vocabulary (WP28), or a red-team agent inside the architecture (WP26).
++5. **Traceability of every number** (9 of 11 "yes"): "Every paper number must map to a result file" (WP18), results registries feeding LaTeX macros (WP15), run manifests with hashes (WP28), versioned outputs (WP26).
++6. **Information-set discipline wherever data exist** (6 of 6 data-based mandates): look-ahead, as-of-date, vintage and chronological-split rules.
++7. **Named baselines** (8 of 11), from "random walk, no change" (WP23) to cost-matched single-agent controls (WP26).
++8. **Honest-failure fallbacks**: "If blocked, document the issue honestly and deliver a scoped partial result" (WP18); "A dead end with a mathematical explanation is better than a fabricated breakthrough" (WP28).
++
++## 4. What the mandates consistently lack
++
++1. **Externally enforced or executable gates.** Gates are declarative. Only WP15 ("The release script must fail if …") and WP28 ("completing one phase's acceptance tests before starting the next") attach a gate to something that can fail mechanically; none specifies who or what checks a gate other than the executing agent itself.
++2. **Independent verification.** "Independent audit" (WP19), "independently checked" (WP20), "independently verified" (WP21), "hostile-referee critique" (WP25) and "three independent review perspectives" (WP28) are all to be performed by the same agent; WP28 is the only mandate that names this tension ("do not have the same model merely declare its own work accepted").
++3. **Operational pre-registration.** Four mandates ask for pre-registration; none specifies a registry, a timestamp, a hash or a freeze mechanism that would make it verifiable.
++4. **Stopping rules.** Only WP15 (a dedicated "Stop conditions" section) and WP28 (stop only on a real blocker; mandatory final verdict) define when to stop; most others say "continue with feasible work".
++5. **Recovery / resumption.** Only WP15 (resumable, idempotent, checkpointed jobs) and WP25 (checkpointing, manifests) require it; the rest rely on progress logs.
++6. **Budgets.** Seven of eleven set no resource limit; only WP26 asks for an API-cost budget, and no mandate states a numeric token, dollar or compute cap.
++7. **AI-use disclosure.** Ten of eleven say nothing about disclosing AI involvement in the paper; WP28 is the exception because it inherits a prior "GPT-6.1" co-author credit that it must reconcile with journal policy.
++8. **Quantified success criteria.** Targets are phrased as "substantially improve" (WP23), "publication-quality", "publishable contribution", "the standard expected in a top finance journal" (WP15) — none is checkable from the document. Page, figure, table and reference counts are given as ranges with anti-padding escape clauses (WP18, WP19, WP20, WP24, WP28).
++9. **Authorial provenance of the instruction itself.** WP23 states that part of its text was reconstructed by an assistant after truncation; WP24 and WP26 carry sections "added" on 2026-10-27; WP18, WP20 and WP25 carry prefatory repository-mapping notes. The mandate as retrieved is therefore not always the mandate as originally issued.
++
++## 5. Verbatim excerpts illustrating the patterns
++
++1. Integrity preamble — WP15_CLAUDE.md: "Never fabricate data, results, citations, institutional details, variable definitions, statistical significance, sample sizes, or conclusions. Never write a result in the paper before it has been generated and verified from a saved machine-readable output."
++2. Phase gate before drafting — WP18_CLAUDE.md: "**Gate:** do not start manuscript drafting until the baseline model is validated, claimed GE equilibria clear markets, principal simulations and exhibits reproduce, bibliography is verified, and theoretical claims have proofs or appropriate qualifications. If blocked, document the issue honestly and deliver a scoped partial result."
++3. Executable release gate — WP15_CLAUDE.md: "The release script must fail if: a table references a missing result; a figure is missing source data; LaTeX does not compile; citations are unresolved; labels are duplicated; page count exceeds the current target; anonymous PDF contains author-identifying text; manuscript claims disagree with the results registry; data revision is not pinned; tests fail."
++4. Novelty gate — WP28_CLAUDE.md: "Do not claim that distributed gradients, one-step Newton, split-sample jackknife, Richardson extrapolation, `K = o(N^{7/8})`, or `K = o(N^{11/12})` are intrinsically novel. Investigate who has already obtained each idea or bound." and "**Gate 1:** no unqualified novelty language is allowed until the closest competing theorems have been inspected and cited."
++5. Temporal-information control — WP22_CLAUDE.md: "Never use post-transaction information or future data in real-time estimates." / WP26_CLAUDE.md: "Comparable retrieval must be strictly as-of-date; no same-property future transfers, post-sale listings, or near-duplicate records leaking across partitions."
++6. Pre-specified primary outcome — WP26_CLAUDE.md: "Pre-register primary endpoints and hypotheses before running final test evaluations. Primary outcome: out-of-time absolute log-price error on arm's-length residential transactions."
++7. Falsification and null results — WP15_CLAUDE.md: "The project must attempt to disprove its own central claim." / WP28_CLAUDE.md: "A dead end with a mathematical explanation is better than a fabricated breakthrough. A strong negative result or counterexample may itself become the contribution."
++8. Self-review by the same model, acknowledged — WP28_CLAUDE.md: "Simulate at least three independent review perspectives; log objections and responses … Record unresolved issues candidly; do not have the same model merely declare its own work accepted."
++9. Instruction provenance — WP23_CLAUDE.md: "[Sections 11–15 and the remainder of the protocol were reconstructed by the author's assistant because the pasted protocol was truncated.]" / WP26_CLAUDE.md: "## Repository conventions (added by the implementation, 2026-10-27)".
++
++## 6. Implications for the WP29 protocol design (motivation only)
++
++The corpus shows a stable *rhetorical* protocol — role, integrity rules, phases, gates, audits, traceability — and an equally stable absence of *mechanisms*: gates are not executable, audits are not independent, pre-registration is not timestamped, budgets and stopping rules are rare, and the instruction text itself is sometimes amended after the fact. WP29's protocol should therefore turn the recurring declarative elements into verifiable ones (hashed and frozen mandates, machine-checked gates, external evaluators, explicit budgets and stop conditions, recorded human interventions) rather than add more prose rules. Nothing in this corpus demonstrates that the earlier papers were produced without human assistance; that question requires run-level evidence, not instruction files.

added research/literature_matrix.csv +87 −0

++ref_key,stream,authors_short,year,title,venue,venue_type,doi_or_arxiv,verification_source,verified_utc,relevance_to_wp29,key_claim_or_finding,peer_reviewed
++lu2024aiscientist,1,Lu et al.,2024,The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery,arXiv preprint,preprint,arXiv:2408.06292,https://export.arxiv.org/api/query?id_list=2408.06292,2026-10-30T03:12:38Z,Reference end-to-end autonomous research system (idea to paper plus simulated review) against which WP29's protocol-guided design is positioned.,"Introduces The AI Scientist, which generates research ideas, writes code, executes experiments, visualizes results, writes a full paper and runs a simulated review, with each paper developed at a cost of less than $15 and an automated reviewer reported to achieve near-human performance on paper scores.",no
++lu2026aiscientist,1,Lu et al.,2026,Towards end-to-end automation of AI research,Nature,journal,doi:10.1038/s41586-026-10265-5,https://api.crossref.org/works/10.1038/s41586-026-10265-5 ; doi.org HEAD None,2026-10-30T03:18:58Z,"Peer-reviewed (Nature, 2026) publication of The AI Scientist line of work, establishing that fully automated ML research pipelines are now part of the refereed literature.",n/a (Nature abstract not retrieved behind the publisher login; see lu2024aiscientist for the system description).,yes
++yamada2025aiscientistv2,1,Yamada et al.,2025,The AI Scientist-v2: Workshop-Level Automated Scientific Discovery via Agentic Tree Search,arXiv preprint,preprint,arXiv:2504.08066,https://export.arxiv.org/api/query?id_list=2504.08066,2026-10-30T03:12:39Z,Closest fully autonomous end-to-end system; its tree-search workflow and in-system AI reviewer are the counterpoint to WP29's executable gates and separated review.,"Describes an end-to-end agentic system with progressive agentic tree search managed by an experiment-manager agent and a VLM feedback loop for figures; of three fully autonomous manuscripts submitted to an ICLR workshop, one scored above the average human acceptance threshold.",no
++schmidgall2025agentlab,1,Schmidgall et al.,2025,Agent Laboratory: Using LLM Agents as Research Assistants,Findings of the Association for Computational Linguistics: EMNLP 2025,conference,doi:10.18653/v1/2025.findings-emnlp.320,https://api.crossref.org/works/10.18653/v1/2025.findings-emnlp.320 ; doi.org HEAD None ; https://export.arxiv.org/api/query?id_list=2501.04227,2026-10-30T03:18:58Z,"Staged (literature review, experimentation, report writing) autonomous research framework with optional human feedback; provides the human-in-the-loop and cost comparisons WP29 builds on.","Agent Laboratory takes a human-provided idea through literature review, experimentation and report writing; the authors report that o1-preview gives the best outcomes, that human feedback at each stage significantly improves research quality, and an 84% decrease in research expenses relative to previous autonomous research methods.",yes
++boiko2023coscientist,1,Boiko et al.,2023,Autonomous chemical research with large language models,Nature,journal,doi:10.1038/s41586-023-06792-0,https://api.crossref.org/works/10.1038/s41586-023-06792-0 ; doi.org HEAD 302,2026-10-30T03:12:40Z,Landmark peer-reviewed demonstration of an LLM agent planning and executing laboratory experiments; defines the 'autonomous research' baseline WP29 compares to.,"Coscientist, driven by GPT-4 with internet/documentation search, code execution and experimental automation tools, autonomously designs, plans and performs experiments across six tasks, including successful reaction optimization of palladium-catalysed cross-couplings.",yes
++bran2024chemcrow,1,M. Bran et al.,2024,Augmenting large language models with chemistry tools,Nature Machine Intelligence,journal,doi:10.1038/s42256-024-00832-8,https://api.crossref.org/works/10.1038/s42256-024-00832-8 ; doi.org HEAD 302,2026-10-30T03:15:06Z,"Tool-augmented LLM research agent whose evaluation combined LLM and expert assessment, an early instance of the evaluator-validity question WP29 addresses with protected oracles.","ChemCrow integrates 18 expert-designed tools with GPT-4, autonomously planned and executed syntheses of an insect repellent and three organocatalysts, and was evaluated with both LLM and expert assessments.",yes
++ifargan2025datatopaper,1,Ifargan et al.,2025,Autonomous LLM-Driven Research — from Data to Human-Verifiable Research Papers,NEJM AI,journal,doi:10.1056/AIoa2400555,https://api.crossref.org/works/10.1056/AIoa2400555 ; doi.org HEAD 302 ; https://export.arxiv.org/api/query?id_list=2404.17605,2026-10-30T03:13:47Z,"Closest prior work on traceability: an autonomous data-to-paper platform that programmatically backtraces information flow for human verification, akin to WP29's persistent evidence records.","data-to-paper guides interacting LLM agents through a stepwise research process from annotated data to papers while programmatically backtracing information flow and allowing human oversight, and can run fully autonomously ('autopilot') or with human intervention.",yes
++gottweis2026coscientist,1,Gottweis et al.,2026,Accelerating scientific discovery with Co-Scientist,Nature,journal,doi:10.1038/s41586-026-10644-y,https://api.crossref.org/works/10.1038/s41586-026-10644-y ; doi.org HEAD 302 ; https://export.arxiv.org/api/query?id_list=2502.18864,2026-10-30T03:13:46Z,Multi-agent hypothesis-generation system with generate/critique/refine loops; illustrates in-system self-critique rather than independent verification.,"Co-Scientist is a Gemini-based multi-agent system that continuously generates, critiques and refines research hypotheses using an asynchronous task framework and a tournament evolution process, scaled by test-time compute, to propose demonstrably novel hypotheses for experimental verification.",yes
++swanson2025virtuallab,1,Swanson et al.,2025,The Virtual Lab of AI agents designs new SARS-CoV-2 nanobodies,Nature,journal,doi:10.1038/s41586-025-09442-9,https://api.crossref.org/works/10.1038/s41586-025-09442-9 ; doi.org HEAD 302,2026-10-30T03:12:48Z,AI-human research collaboration with an LLM principal-investigator agent and human oversight; a human-involvement design point for the novelty matrix.,"The Virtual Lab is an AI-human research collaboration in which an LLM Principal Investigator agent guides a team of LLM scientist agents to perform interdisciplinary research, applied to designing SARS-CoV-2 nanobodies.",yes
++mitchener2025kosmos,1,Mitchener et al.,2025,Kosmos: An AI Scientist for Autonomous Discovery,arXiv preprint,preprint,arXiv:2511.02824,https://export.arxiv.org/api/query?id_list=2511.02824,2026-10-30T03:12:57Z,"Recent long-horizon AI scientist (up to 12 h, 200 rollouts); documents the coherence/verification challenges of long autonomous runs that WP29's gates target.","Kosmos runs for up to 12 hours performing cycles of parallel data analysis, literature search and hypothesis generation, using a structured world model shared between agents to pursue an objective coherently over 200 agent rollouts before synthesizing reports.",no
++luo2025pitfalls,1,Luo et al.,2025,"The More You Automate, the Less You See: Hidden Pitfalls of AI Scientist Systems",arXiv preprint,preprint,arXiv:2509.08713,https://export.arxiv.org/api/query?id_list=2509.08713,2026-10-30T03:16:07Z,"Directly motivates WP29: shows AI scientist systems commit benchmark selection, leakage, metric-misuse and post-hoc selection errors that only traces reveal.","Identifies four failure modes in AI scientist systems (inappropriate benchmark selection, data leakage, metric misuse and post-hoc selection bias) and, via controlled experiments on two open-source systems, finds several of them present.",no
++beel2025sakana,1,Beel et al.,2025,"Evaluating Sakana's AI Scientist: Bold Claims, Mixed Results, and a Promising Future?",ACM SIGIR Forum,journal,doi:10.1145/3769733.3769747,https://api.crossref.org/works/10.1145/3769733.3769747 ; doi.org HEAD None ; https://export.arxiv.org/api/query?id_list=2502.14297,2026-10-30T03:18:59Z,Independent third-party evaluation of The AI Scientist; evidence that unverified autonomous outputs contain novelty and execution errors.,"An independent evaluation of the AI Scientist found several critical shortcomings, including a literature review based on simplistic keyword searches that led to poor novelty assessments, with several generated ideas incorrectly classified as novel.",unknown
++luo2026xscientist,1,Luo,2026,XScientist: A Git-Like Research Protocol for Long-Running Autonomous Scientific Discovery,arXiv preprint,preprint,arXiv:2607.12301,https://export.arxiv.org/api/query?id_list=2607.12301,2026-10-30T03:13:42Z,"Closest 2026 design to a research protocol: git-like typed research state with checkpoints, claim-evidence closure and gates; differs from WP29 in being a protocol artifact rather than a controlled comparison of protocol arms.","Proposes a local-first, git-like protocol that treats research state rather than a manuscript as the unit of continuation, representing hypotheses, attempts, claims, reviews and handoffs as typed content-addressed objects with immutable checkpoints, claim-evidence closure and authority-aware gates, without treating a passing integrity check as scientific truth.",no
++ding2026verificationgap,1,Ding et al.,2026,Autonomous Research Agents: A Survey of AI Scientists and the Verification Gap,arXiv preprint,preprint,arXiv:2608.05179,https://export.arxiv.org/api/query?id_list=2608.05179,2026-10-30T03:13:43Z,"Survey that names the 'verification gap' in AI scientist systems; supplies the audit dimensions (evaluation method, artifacts, human-in-the-loop points, result-selection disclosure) WP29 reports.","Screens 125 works and codes 26 (24 runnable systems) on seven audit dimensions, finding that code release is now common but reproducibility-grade and claim-verification artifacts remain rare.",no
++chen2025scienceagentbench,2,Chen et al.,2024,ScienceAgentBench: Toward Rigorous Assessment of Language Agents for Data-Driven Scientific Discovery,ICLR 2025,conference,arXiv:2410.05080,"https://export.arxiv.org/api/query?id_list=2410.05080 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2410.05080 (venue='arXiv.org', type=None) ; https://api2.openreview.net/notes/search?term=ScienceAgentBench (ICLR 2025 Poster record)",2026-10-30T03:13:01Z,"Benchmark of expert-validated data-driven discovery tasks with program-level outputs, execution metrics and cost; the task style and contamination mitigations inform WP29's task bank.","Presents 102 tasks extracted from 44 publications in four disciplines and validated by nine experts; the best agent solves 32.4% of tasks independently (34.3% with expert knowledge), o1-preview reaches 42.2% at more than 10 times the cost, and two data-contamination mitigation strategies are proposed.",yes
++majumder2025discoverybench,2,Majumder et al.,2024,DiscoveryBench: Towards Data-Driven Discovery with Large Language Models,ICLR 2025,conference,arXiv:2407.01725,"https://export.arxiv.org/api/query?id_list=2407.01725 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2407.01725 (venue='International Conference on Learning Representations', type='conference') ; https://api2.openreview.net/notes/search?term=DiscoveryBench (ICLR 2025 Poster record)",2026-10-30T03:15:06Z,"Formalizes multi-step data-driven discovery as dataset + metadata + goal, the same unit of analysis as WP29's research tasks.","DiscoveryBench formalizes the multi-step process of data-driven discovery with 264 tasks across six domains derived from published papers plus 903 synthetic tasks, each defined by a dataset, its metadata and a natural-language discovery goal.",yes
++jing2025dsbench,2,Jing et al.,2024,DSBench: How Far Are Data Science Agents from Becoming Data Science Experts?,ICLR 2025,conference,arXiv:2409.07703,"https://export.arxiv.org/api/query?id_list=2409.07703 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2409.07703 (venue='', type=None) ; https://api2.openreview.net/notes/search?term=DSBench (ICLR 2025 Poster record)",2026-10-30T03:13:02Z,"Realistic data-science agent benchmark (long contexts, large data) used to characterize executor capability independent of protocol.","DSBench comprises 466 data-analysis and 74 data-modeling tasks sourced from Eloquence and Kaggle competitions, with long contexts, multimodal task backgrounds and reasoning over large data files.",yes
++huang2024mlagentbench,2,Huang et al.,2023,MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation,ICML 2024,conference,arXiv:2310.03302,"https://export.arxiv.org/api/query?id_list=2310.03302 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2310.03302 (venue='International Conference on Machine Learning', type='conference')",2026-10-30T03:13:03Z,Evaluates agents on ML experimentation loops with file/code actions; a precursor of WP29's executor arms.,"MLAgentBench is a suite of 13 ML experimentation tasks (from CIFAR-10 improvement to BabyLM) in which a ReAct-based agent can read/write files, execute code and inspect outputs; a Claude 3 Opus agent had the best success rate.",yes
++gu2024blade,2,Gu et al.,2024,BLADE: Benchmarking Language Model Agents for Data-Driven Science,EMNLP 2024,conference,arXiv:2408.09667,"https://export.arxiv.org/api/query?id_list=2408.09667 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2408.09667 (venue='Conference on Empirical Methods in Natural Language Processing', type='conference')",2026-10-30T03:13:05Z,"Evaluates analysis decisions (variables, transformations, models) against independent expert analyses, the multi-path ground truth WP29's oracles must handle.","BLADE evaluates agents' multifaceted approaches to open-ended research questions using 12 datasets and questions from the literature, with ground truth collected from independent analyses by expert data scientists.",yes
++jimenez2024swebench,2,Jimenez et al.,2023,SWE-bench: Can Language Models Resolve Real-World GitHub Issues?,ICLR 2024,conference,arXiv:2310.06770,"https://export.arxiv.org/api/query?id_list=2310.06770 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2310.06770 (venue='International Conference on Learning Representations', type='conference')",2026-10-30T03:13:08Z,Canonical execution-verified coding benchmark; its test-based grading is the model for WP29's executable verification gates.,"SWE-bench consists of 2,294 software-engineering problems drawn from real GitHub issues and pull requests across 12 Python repositories, where resolving an issue frequently requires coordinating changes across multiple functions, classes and files.",yes
++chan2025mlebench,2,Chan et al.,2024,MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering,ICLR 2025,conference,arXiv:2410.07095,"https://export.arxiv.org/api/query?id_list=2410.07095 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2410.07095 (venue='arXiv.org', type=None) ; https://proceedings.iclr.cc/paper_files/paper/2025/hash/7e3767db483c942b883eb4f8cfb74e31-Abstract-Conference.html",2026-10-30T03:13:09Z,"Agentic ML-engineering benchmark with human baselines, resource-scaling and contamination analyses that WP29's resource accounting parallels.","MLE-bench curates 75 Kaggle competitions with human leaderboard baselines; the best setup (o1-preview with AIDE scaffolding) reaches at least bronze-medal level in 16.9% of competitions, and the paper also studies resource scaling and pre-training contamination.",yes
++siegel2024corebench,2,Siegel et al.,2024,CORE-Bench: Fostering the Credibility of Published Research Through a Computational Reproducibility Agent Benchmark,Transactions on Machine Learning Research,journal,arXiv:2409.11363,https://export.arxiv.org/api/query?id_list=2409.11363 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2409.11363 (venue='Trans. Mach. Learn. Res.'),2026-10-30T03:13:10Z,"Benchmarks agents on computational reproducibility of published papers, linking agent evaluation to the reproducibility stream.","CORE-Bench contains 270 tasks based on 90 scientific papers across computer science, social science and medicine, measuring agents' accuracy at reproducing published results from provided code and data at three difficulty levels.",yes
++starace2025paperbench,2,Starace et al.,2025,PaperBench: Evaluating AI's Ability to Replicate AI Research,ICML 2025,conference,arXiv:2504.01848,"https://export.arxiv.org/api/query?id_list=2504.01848 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2504.01848 (venue='International Conference on Machine Learning', type='conference')",2026-10-30T03:13:11Z,Rubric-decomposed replication grading with a validated LLM judge; closest precedent for WP29's rubric oracles and judge validation.,"PaperBench asks agents to replicate 20 ICML 2024 papers from scratch, graded against 8,316 rubric items co-developed with the papers' authors, with an LLM-based judge whose performance is assessed on a separate judge benchmark.",yes
++koster2012snakemake,3,Köster & Rahmann,2012,Snakemake—a scalable bioinformatics workflow engine,Bioinformatics,journal,doi:10.1093/bioinformatics/bts480,https://api.crossref.org/works/10.1093/bioinformatics/bts480 ; doi.org HEAD 302,2026-10-30T03:13:47Z,"Rule-based workflow engine; WP29's protocol stages are an analogue of declarative, re-executable workflow rules.",Snakemake provides a readable Python-based workflow definition language and an execution environment that scales from single-core workstations to compute clusters without modifying the workflow.,yes
++ditommaso2017nextflow,3,Di Tommaso et al.,2017,Nextflow enables reproducible computational workflows,Nature Biotechnology,journal,doi:10.1038/nbt.3820,https://api.crossref.org/works/10.1038/nbt.3820 ; doi.org HEAD 302,2026-10-30T03:13:48Z,Container-based dataflow workflow manager for reproducible pipelines; reference for execution isolation in WP29's executors.,Argues that combining dataflow programming with software containers (Nextflow) yields computational workflows that are reproducible across platforms.,yes
++missier2013prov,3,Missier et al.,2013,The W3C PROV family of specifications for modelling provenance metadata,Proceedings of the 16th International Conference on Extending Database Technology,conference,doi:10.1145/2452376.2452478,https://api.crossref.org/works/10.1145/2452376.2452478 ; doi.org HEAD 302,2026-10-30T03:13:49Z,"W3C PROV family defines the provenance vocabulary (entities, activities, agents) that WP29's evidence records instantiate.","Overview of the W3C PROV family of specifications (data model, ontology, notation and constraints) for interoperable representation and exchange of provenance metadata.",yes
++chirigati2016reprozip,3,Chirigati et al.,2016,ReproZip,Proceedings of the 2016 International Conference on Management of Data,conference,doi:10.1145/2882903.2899401,https://api.crossref.org/works/10.1145/2882903.2899401 ; doi.org HEAD 302,2026-10-30T03:13:49Z,Automatic capture of execution provenance into re-executable packages; a model for evidence capture without relying on the agent's self-report.,"ReproZip automatically captures the provenance of a command-line experiment (files, libraries, environment) and packages it so that it can be re-executed elsewhere.",yes
++sandve2013tensimplerules,3,Sandve et al.,2013,Ten Simple Rules for Reproducible Computational Research,PLoS Computational Biology,journal,doi:10.1371/journal.pcbi.1003285,https://api.crossref.org/works/10.1371/journal.pcbi.1003285 ; doi.org HEAD 302,2026-10-30T03:13:50Z,"Widely cited rules (track provenance, avoid manual steps, record versions and seeds) that WP29 operationalizes as machine-checked gates.","Proposes ten rules for reproducible computational research, including tracking how every result was produced, avoiding manual data manipulation, archiving exact program versions and random seeds, and providing public access to scripts, runs and results.",yes
++stodden2018journalpolicy,3,Stodden et al.,2018,An empirical analysis of journal policy effectiveness for computational reproducibility,Proceedings of the National Academy of Sciences,journal,doi:10.1073/pnas.1708290115,https://api.crossref.org/works/10.1073/pnas.1708290115 ; doi.org HEAD 302,2026-10-30T03:13:50Z,Empirical reproducibility failure rates under policy-only enforcement; motivates executable rather than declarative verification.,"In a random sample of 204 Science papers subject to a data/code-on-request policy, the authors obtained artifacts from 44% of papers and could reproduce the findings for 26%.",yes
++peng2011reproducible,3,Peng,2011,Reproducible Research in Computational Science,Science,journal,doi:10.1126/science.1213847,https://api.crossref.org/works/10.1126/science.1213847 ; doi.org HEAD 302,2026-10-30T03:13:51Z,"Defines reproducibility (data + code available) as the minimum standard for computational claims, the standard WP29's evidence records target.",Reproducibility has the potential to serve as a minimum standard for judging scientific claims when full independent replication of a study is not possible.,yes
++nasem2019reproducibility,3,"National Academies of Sciences, Engineering, and Medicine",2019,Reproducibility and Replicability in Science,National Academies Press,report,doi:10.17226/25303,https://api.crossref.org/works/10.17226/25303 ; doi.org HEAD 302,2026-10-30T03:13:51Z,Authoritative definitions of reproducibility vs replicability used in WP29's outcome definitions.,"Defines reproducibility as obtaining consistent computational results using the same input data, computational steps, methods, code and conditions of analysis, and replicability as obtaining consistent results across studies aimed at the same question with new data.",unknown
++krafczyk2021reproduction,3,Krafczyk et al.,2021,Learning from reproducing computational results: introducing three principles and the <i>Reproduction Package</i>,"Philosophical Transactions of the Royal Society A: Mathematical, Physical and Engineering Sciences",journal,doi:10.1098/rsta.2020.0069,https://api.crossref.org/works/10.1098/rsta.2020.0069 ; doi.org HEAD 302,2026-10-30T03:13:52Z,"Principles (transparency, re-executability, determinism) and a Reproduction Package formalism close to WP29's release bundle.","From reproducing seven articles the authors derive three principles (transparency about how results are produced, ease of re-executability, determinism), 12 guidelines, and a Reproduction Package formalism for sharing artifacts.",yes
++trisovic2022codequality,3,Trisovic et al.,2022,A large-scale study on research code quality and execution,Scientific Data,journal,doi:10.1038/s41597-022-01143-6,https://api.crossref.org/works/10.1038/s41597-022-01143-6 ; doi.org HEAD 302,2026-10-30T03:13:53Z,"Large-scale evidence that deposited research code usually fails to run, justifying execution-based verification gates.","Executing over 9,000 R files from more than 2,000 Harvard Dataverse replication datasets, 74% failed to complete without error initially and 56% still failed after automatic code cleaning.",yes
++nosek2018preregistration,4,Nosek et al.,2018,The preregistration revolution,Proceedings of the National Academy of Sciences,journal,doi:10.1073/pnas.1708274114,https://api.crossref.org/works/10.1073/pnas.1708274114 ; doi.org HEAD 302,2026-10-30T03:13:55Z,Preregistration as prediction/postdiction separation is the direct analogue of WP29's protocol fixed before execution.,"Defining research questions and the analysis plan before observing outcomes (preregistration) distinguishes analyses that result from predictions from those that result from postdictions, and practical strategies exist for cases that fall short of the ideal.",yes
++chambers2013registeredreports,4,Chambers,2013,Registered Reports: A new publishing initiative at Cortex,Cortex,journal,doi:10.1016/j.cortex.2012.12.016,https://api.crossref.org/works/10.1016/j.cortex.2012.12.016 ; doi.org HEAD 302,2026-10-30T03:13:56Z,"Origin of Registered Reports: review of the protocol before results, the publishing analogue of WP29's pre-execution gate.","Introduces Registered Reports at Cortex, in which study protocols are peer reviewed and provisionally accepted before data collection.",unknown
++chambers2022registeredreports,4,Chambers & Tzavella,2021,"The past, present and future of Registered Reports",Nature Human Behaviour,journal,doi:10.1038/s41562-021-01193-7,https://api.crossref.org/works/10.1038/s41562-021-01193-7 ; doi.org HEAD 302,2026-10-30T03:13:57Z,Reviews evidence that pre-acceptance of protocols reduces reporting bias; supports the 'specification before execution' arm.,Registered Reports are a publication format in which study proposals are peer reviewed and pre-accepted before research is undertaken; early evidence indicates they work as intended although they are not a universal solution for irreproducibility.,yes
++simmons2011falsepositive,4,Simmons et al.,2011,False-Positive Psychology,Psychological Science,journal,doi:10.1177/0956797611417632,https://api.crossref.org/works/10.1177/0956797611417632 ; doi.org HEAD 302,2026-10-30T03:13:57Z,Researcher degrees of freedom inflate false positives; WP29's locked analysis specification removes these from the agent.,"Flexibility in data collection, analysis and reporting dramatically increases actual false-positive rates, so that a researcher is often more likely to falsely find evidence that an effect exists than to correctly find evidence that it does not; six disclosure requirements for authors and four reviewer guidelines are proposed.",yes
++munafo2017manifesto,4,Munafò et al.,2017,A manifesto for reproducible science,Nature Human Behaviour,journal,doi:10.1038/s41562-016-0021,https://api.crossref.org/works/10.1038/s41562-016-0021 ; doi.org HEAD 302,2026-10-30T03:13:57Z,"Umbrella statement of reproducibility measures (methods, reporting, evaluation, incentives) that WP29 adapts to autonomous agents.","Argues for measures optimizing methods, reporting and dissemination, reproducibility, evaluation and incentives, noting evidence from simulations and empirical studies but the need for iterative evaluation of their adoption.",yes
++gelman2014statisticalcrisis,4,Gelman & Loken,2014,The Statistical Crisis in Science,American Scientist,journal,doi:10.1511/2014.111.460,https://api.crossref.org/works/10.1511/2014.111.460 ; doi.org HEAD 302,2026-10-30T03:13:58Z,"Garden of forking paths: data-dependent analysis choices invalidate inference even without intent, which an agent in an open instruction arm is free to make.",Argues that data-dependent choices of analysis (the 'garden of forking paths') can produce spurious statistically significant findings even when researchers do not deliberately fish for significance.,unknown
++nosek2015top,4,Nosek et al.,2015,Promoting an open research culture,Science,journal,doi:10.1126/science.aab2374,https://api.crossref.org/works/10.1126/science.aab2374 ; doi.org HEAD 302,2026-10-30T03:13:59Z,"TOP guidelines define graded transparency standards (preregistration, data, code) that WP29's protocol tiers mirror.","Proposes journal author guidelines (the TOP guidelines) to promote transparency, openness and reproducibility, noting that although scientists embrace these norms they are not routine in practice.",yes
++scheel2021excess,4,Scheel et al.,2021,An Excess of Positive Results: Comparing the Standard Psychology Literature With Registered Reports,Advances in Methods and Practices in Psychological Science,journal,doi:10.1177/25152459211007467,https://api.crossref.org/works/10.1177/25152459211007467 ; doi.org HEAD 302,2026-10-30T03:14:00Z,Quantifies how pre-specified protocols change the result distribution; benchmark for interpreting WP29's arm differences.,"Comparing 71 published Registered Reports with 152 standard hypothesis-testing papers, the first hypothesis was positive in 96% of standard reports but only 44% of Registered Reports.",yes
++osc2015reproducibility,4,Open Science Collaboration,2015,Estimating the reproducibility of psychological science,Science,journal,doi:10.1126/science.aac4716,https://api.crossref.org/works/10.1126/science.aac4716 ; doi.org HEAD 302,2026-10-30T03:14:00Z,Baseline replication rates in a human literature; provides a human reference for WP29's validity outcomes.,"Replications of 100 experiments from 2008 papers in three psychology journals found that about one-third to one-half of original findings were also observed in the replication, depending on the criterion.",yes
++kerr1998harking,4,Kerr,1998,HARKing: Hypothesizing After the Results are Known,Personality and Social Psychology Review,journal,doi:10.1207/s15327957pspr0203_4,https://api.crossref.org/works/10.1207/s15327957pspr0203_4 ; doi.org HEAD 302,2026-10-30T03:14:01Z,HARKing (presenting post hoc hypotheses as a priori) is a failure mode an unconstrained agent can reproduce when writing up.,"Defines HARKing (hypothesizing after the results are known) as presenting post hoc hypotheses as if they were a priori, and discusses its prevalence and costs.",yes
++liu2024agentbench,5,Liu et al.,2023,AgentBench: Evaluating LLMs as Agents,ICLR 2024,conference,arXiv:2308.03688,"https://export.arxiv.org/api/query?id_list=2308.03688 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2308.03688 (venue='International Conference on Learning Representations', type='conference')",2026-10-30T03:14:02Z,"Multi-environment agent benchmark identifying long-horizon reasoning and instruction following as main failure causes, relevant to protocol adherence measurement.","AgentBench evaluates LLMs as agents across 8 environments; top commercial LLMs act well in complex environments but a significant gap separates them from open-source models up to 70B, with poor long-term reasoning, decision-making and instruction following as main obstacles.",yes
++yao2024taubench,5,Yao et al.,2024,$τ$-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains,arXiv preprint,preprint,arXiv:2406.12045,https://export.arxiv.org/api/query?id_list=2406.12045,2026-10-30T03:14:02Z,State-based outcome checking and the pass^k reliability metric are reused in WP29's repeated-run reliability analysis.,"tau-bench evaluates agents in simulated user conversations with API tools and policy guidelines by comparing the final database state to an annotated goal state, and proposes pass^k for reliability over multiple trials; gpt-4o succeeds on fewer than 50% of tasks with pass^8 below 25% in retail.",unknown
++kapoor2024agentsthatmatter,5,Kapoor et al.,2024,AI Agents That Matter,Transactions on Machine Learning Research,journal,arXiv:2407.01502,https://export.arxiv.org/api/query?id_list=2407.01502 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2407.01502 (venue='Trans. Mach. Learn. Res.'),2026-10-30T03:14:03Z,"Core methodological critique (cost reporting, holdouts, conflated benchmarking needs) that WP29's evaluation design follows.","Analysis of agent benchmarks finds a narrow focus on accuracy without cost, leading to needlessly complex and costly agents; proposes jointly optimizing accuracy and cost, and notes that model- and downstream-developer needs are conflated and that many benchmarks have inadequate holdout sets.",yes
++sainz2023contamination,5,Sainz et al.,2023,NLP Evaluation in trouble: On the Need to Measure LLM Data Contamination for each Benchmark,Findings of the Association for Computational Linguistics: EMNLP 2023,conference,doi:10.18653/v1/2023.findings-emnlp.722,https://api.crossref.org/works/10.18653/v1/2023.findings-emnlp.722 ; doi.org HEAD 302 ; https://export.arxiv.org/api/query?id_list=2310.18018,2026-10-30T03:14:04Z,Defines contamination levels; WP29's task bank must document contamination risk for each task.,"Position paper arguing that training on a benchmark's test split causes overestimation of performance with potentially harmful scientific conclusions, defining levels of data contamination and calling for community measurement for each benchmark.",yes
++zheng2023mtbench,5,Zheng et al.,2023,Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena,NeurIPS 2023 Datasets and Benchmarks,conference,arXiv:2306.05685,"https://export.arxiv.org/api/query?id_list=2306.05685 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2306.05685 (venue='Neural Information Processing Systems', type='conference') ; https://api2.openreview.net/notes/search?term=Judging%20LLM-as-a-judge (NeurIPS 2023 Datasets and Benchmarks record)",2026-10-30T03:14:05Z,"Establishes LLM-as-a-judge and its biases (position, verbosity, self-enhancement), which WP29's protected oracles are designed to avoid.","Examines LLM-as-a-judge limitations including position, verbosity and self-enhancement biases, and finds that strong judges such as GPT-4 agree with human preferences over 80% of the time on MT-bench and Chatbot Arena, the same level as human-human agreement.",yes
++thakur2025judgingjudges,5,Thakur et al.,2024,Judging the Judges: Evaluating Alignment and Vulnerabilities in LLMs-as-Judges,"GEM2 Workshop, ACL 2025",conference,arXiv:2406.12624,"https://export.arxiv.org/api/query?id_list=2406.12624 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2406.12624 (venue='arXiv.org', type=None) ; arXiv journal_ref: Proceedings of the Fourth Workshop on Generation Evaluation and Metrics GEM2 2025 pages 404 to 430; July 31 August 1 2025; 2025 Association for Computational Linguistics",2026-10-30T03:14:06Z,"Shows even the best judge models lag inter-human agreement, cautioning against unprotected LLM grading of research outputs.","Across thirteen judge models and nine exam-taker models in a high-human-agreement setting, only the best and largest judges reach reasonable alignment with humans, remain well behind inter-human agreement, and may differ from human scores by up to 5 points.",yes
++panickssery2024selfpreference,5,Panickssery et al.,2024,LLM Evaluators Recognize and Favor Their Own Generations,NeurIPS 2024,conference,arXiv:2404.13076,"https://export.arxiv.org/api/query?id_list=2404.13076 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2404.13076 (venue='Neural Information Processing Systems', type='conference')",2026-10-30T03:14:06Z,Self-preference bias is the specific threat WP29 mitigates by separating executor and reviewer and protecting oracles.,"LLM evaluators such as GPT-4 and Llama 2 have non-trivial accuracy at recognizing their own outputs, and fine-tuning reveals a linear correlation between self-recognition capability and the strength of self-preference.",yes
++zhu2025abc,5,Zhu et al.,2025,Establishing Best Practices for Building Rigorous Agentic Benchmarks,arXiv preprint,preprint,arXiv:2507.02825,https://export.arxiv.org/api/query?id_list=2507.02825,2026-10-30T03:13:45Z,"Agentic Benchmark Checklist (task validity, outcome validity, reporting) used as a checklist for WP29's task and oracle construction.","Shows that many agentic benchmarks have task-setup or reward-design flaws (e.g., SWE-bench Verified uses insufficient test cases, TAU-bench counts empty responses as successful) that can mis-estimate performance by up to 100% relative, and introduces the Agentic Benchmark Checklist, which reduced overestimation on CVE-Bench by 33%.",no
++kapoor2025hal,5,Kapoor et al.,2025,Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation,arXiv preprint,preprint,arXiv:2510.11977,https://export.arxiv.org/api/query?id_list=2510.11977,2026-10-30T03:13:45Z,"Standardized harness with full cost accounting across models, scaffolds and benchmarks, the infrastructure model for WP29's isolated executors and resource logs.","Introduces a standardized evaluation harness and reports 21,730 agent rollouts across 9 models and 9 benchmarks at a total cost of about $40,000, finding among other things that higher reasoning effort can reduce accuracy.",no
++li2025llmreval,5,Li et al.,2025,LLM-REVal: Can We Trust LLM Reviewers Yet?,arXiv preprint,preprint,arXiv:2510.12367,https://export.arxiv.org/api/query?id_list=2510.12367,2026-10-30T03:16:08Z,"Simulated research-and-review loop showing LLM reviewers misaligned with humans; motivates WP29's separate, protected review arm rather than in-system AI review.","Using a simulation with a research agent that writes and revises papers and a review agent that assesses them, followed by human annotation, the authors identify pronounced misalignment between LLM reviewers and human judgments.",no
++parasuraman1997automation,6,Parasuraman & Riley,1997,"Humans and Automation: Use, Misuse, Disuse, Abuse",Human Factors: The Journal of the Human Factors and Ergonomics Society,journal,doi:10.1518/001872097778543886,https://api.crossref.org/works/10.1518/001872097778543886 ; doi.org HEAD 302,2026-10-30T03:14:09Z,Use/misuse/disuse/abuse taxonomy frames how a researcher delegates to an autonomous agent and over-relies on its outputs.,"Distinguishes use, misuse (over-reliance producing monitoring failures or decision biases), disuse (often caused by false alarms) and abuse of automation, with trust, workload and reliability as key influences.",yes
++parasuraman2000levels,6,Parasuraman et al.,2000,A model for types and levels of human interaction with automation,"IEEE Transactions on Systems, Man, and Cybernetics - Part A: Systems and Humans",journal,doi:10.1109/3468.844354,https://api.crossref.org/works/10.1109/3468.844354 ; doi.org HEAD 302,2026-10-30T03:14:10Z,Levels-of-automation model used to locate WP29's four arms on the acquisition/analysis/decision/implementation dimensions.,"Proposes that automation can be applied to four function classes (information acquisition, information analysis, decision and action selection, action implementation), each across a continuum from fully manual to fully automatic, and that automation changes rather than merely supplants human activity.",yes
++goddard2012automationbias,6,Goddard et al.,2012,"Automation bias: a systematic review of frequency, effect mediators, and mitigators",Journal of the American Medical Informatics Association,journal,doi:10.1136/amiajnl-2011-000089,https://api.crossref.org/works/10.1136/amiajnl-2011-000089 ; doi.org HEAD 302,2026-10-30T03:14:10Z,Systematic review of automation bias frequency and mitigators informing how human review of agent outputs is measured in WP29.,"A systematic review (74 of 13,821 retrieved papers) finds automation bias mediated by user factors such as cognitive style, decision-support design and task experience, discussed alongside automation-induced complacency.",yes
++lai2023humanai,6,Lai et al.,2023,Towards a Science of Human-AI Decision Making: An Overview of Design Space in Empirical Human-Subject Studies,2023 ACM Conference on Fairness Accountability and Transparency,conference,doi:10.1145/3593013.3594087,https://api.crossref.org/works/10.1145/3593013.3594087 ; doi.org HEAD 302 ; https://export.arxiv.org/api/query?id_list=2112.11471,2026-10-30T03:14:11Z,Design-space survey of empirical human-AI decision-making studies used to position WP29's intervention measurement.,"Surveys empirical human-subject studies of AI-assisted decision making and summarizes the design space of tasks, AI assistance elements and evaluation metrics to structure a science of human-AI decision making.",yes
++bansal2021whole,6,Bansal et al.,2021,Does the Whole Exceed its Parts? The Effect of AI Explanations on Complementary Team Performance,Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems,conference,doi:10.1145/3411764.3445717,https://api.crossref.org/works/10.1145/3411764.3445717 ; doi.org HEAD 302,2026-10-30T03:14:11Z,"Shows explanations raise acceptance of AI recommendations regardless of correctness, a risk for human review of agent-generated research.","In studies on three datasets, AI augmentation produced complementary improvements but explanations did not increase them; instead explanations increased the chance that humans accept the AI's recommendation regardless of its correctness.",yes
++lubars2019delegability,6,Lubars & Tan,2019,"Ask Not What AI Can Do, But What AI Should Do: Towards a Framework of Task Delegability",NeurIPS 2019,conference,arXiv:1902.03245,"https://export.arxiv.org/api/query?id_list=1902.03245 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:1902.03245 (venue='Neural Information Processing Systems', type='conference')",2026-10-30T03:14:12Z,"Task-delegability framework (motivation, difficulty, risk, trust) used to characterize which research sub-tasks are delegated in each WP29 arm.","Proposes a delegability framework with four factors (motivation, difficulty, risk, trust) and, surveying 100 tasks, finds little preference for full AI control and a strong preference for machine-in-the-loop designs, with trust most correlated with preferences.",yes
++vaccaro2024combinations,6,Vaccaro et al.,2024,When combinations of humans and AI are useful: A systematic review and meta-analysis,Nature Human Behaviour,journal,doi:10.1038/s41562-024-02024-1,https://api.crossref.org/works/10.1038/s41562-024-02024-1 ; doi.org HEAD 302,2026-10-30T03:14:12Z,Meta-analytic evidence on when human-AI combinations beat either alone; reference point for interpreting WP29's human-review arm.,"A preregistered meta-analysis of 106 experiments (370 effect sizes, 2020-2023) finds that on average human-AI combinations performed worse than the best of humans or AI alone, with losses in decision tasks and gains in content-creation tasks.",yes
++bainbridge1983ironies,6,Bainbridge,1983,Ironies of automation,Automatica,journal,doi:10.1016/0005-1098(83)90046-8,https://api.crossref.org/works/10.1016/0005-1098(83)90046-8 ; doi.org HEAD 302,2026-10-30T03:14:13Z,"Classic argument that automation makes the residual human role harder, relevant to reviewers of autonomous research output.","Argues that the more advanced a control system is, the more crucial the contribution of the human operator becomes, because the operator is left with the tasks the designer could not automate and must monitor a system that rarely needs intervention.",yes
++bucinca2021trust,6,Buçinca et al.,2021,To Trust or to Think,Proceedings of the ACM on Human-Computer Interaction,journal,doi:10.1145/3449287,https://api.crossref.org/works/10.1145/3449287 ; doi.org HEAD 302,2026-10-30T03:14:14Z,Cognitive forcing as a mitigation of overreliance; analogous to WP29's requirement that reviewers execute checks rather than read summaries.,"In an experiment with 199 participants, three cognitive forcing interventions reduced overreliance on AI relative to simple explainable-AI approaches.",yes
++becker2025metr,6,Becker et al.,2025,Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity,arXiv preprint,preprint,arXiv:2507.09089,https://export.arxiv.org/api/query?id_list=2507.09089,2026-10-30T03:14:16Z,Randomized measurement of AI-tool effects on expert developers showing perception-reality gaps; methodological model for WP29's time/cost outcomes.,"In a randomized controlled trial with 16 experienced open-source developers on 246 tasks, allowing early-2025 AI tools increased completion time by 19%, although developers forecast a 24% speedup beforehand and still believed in a 20% speedup afterwards.",no
++dean2013tail,7,Dean & Barroso,2013,The tail at scale,Communications of the ACM,journal,doi:10.1145/2408776.2408794,https://api.crossref.org/works/10.1145/2408776.2408794 ; doi.org HEAD 302,2026-10-30T03:14:17Z,"Tail-tolerance techniques (hedging, timeouts) underlie WP29's executor time budgets and retry policy.",Software techniques that tolerate latency variability are vital to building responsive large-scale Web services.,unknown
++gray1986whystop,7,Gray,1986,Why Do Computers Stop and What Can Be Done About It?,Symposium on Reliability in Distributed Software and Database Systems,conference,s2:87c9e34c39dd1fbd470010dc1ea98ba3517f5a2a,https://api.semanticscholar.org/graph/v1/paper/87c9e34c39dd1fbd470010dc1ea98ba3517f5a2a (record cached in s2_gray1986_search_observed.json),2026-10-30T03:31:46Z,"Foundational failure analysis (software and operations dominate outages; process pairs, transactions) behind WP29's checkpoint/idempotent step design.","Reports Tandem field data attributing most system outages to software and operations/administration rather than hardware, and advocates process pairs and transactions as the means to tolerate faults.",unknown
++basiri2016chaos,7,Basiri et al.,2016,Chaos Engineering,IEEE Software,journal,doi:10.1109/MS.2016.60,https://api.crossref.org/works/10.1109/MS.2016.60 ; doi.org HEAD 302,2026-10-30T03:14:17Z,Chaos engineering as experimentation on failure injection motivates WP29's fault-injection tests of the executors.,"Describes chaos engineering as Netflix's practice of experimenting on distributed systems to verify their reliability, together with the principles underlying it.",yes
++helland2012idempotence,7,Helland,2012,Idempotence is not a medical condition,Communications of the ACM,journal,doi:10.1145/2160718.2160734,https://api.crossref.org/works/10.1145/2160718.2160734 ; doi.org HEAD 302,2026-10-30T03:14:18Z,"Idempotent operations make retries safe, a requirement for WP29's resumable protocol steps.",Messages may be retried; idempotence means that is acceptable.,unknown
++burckhardt2021durablefunctions,7,Burckhardt et al.,2021,Durable functions: semantics for stateful serverless,Proceedings of the ACM on Programming Languages,journal,doi:10.1145/3485510,https://api.crossref.org/works/10.1145/3485510 ; doi.org HEAD 302,2026-10-30T03:14:20Z,"Formal semantics for durable workflow execution (persistent state, reliable progress) that WP29's run ledger approximates.","Describes Azure Durable Functions, which enhance FaaS with actors, workflows and critical sections to provide persistent state and reliable progress, and gives a formal semantics based on an idealized untyped lambda-calculus model.",yes
++erol2025costofpass,7,Erol et al.,2025,Cost-of-Pass: An Economic Framework for Evaluating Language Models,arXiv preprint,preprint,arXiv:2504.13359,https://export.arxiv.org/api/query?id_list=2504.13359,2026-10-30T03:14:21Z,Cost-of-pass (expected cost per correct solution) is the economic outcome metric WP29 reports per arm.,"Defines cost-of-pass as the expected monetary cost of generating a correct solution and the frontier cost-of-pass relative to a human expert, finding lightweight models most cost-effective for basic quantitative tasks and reasoning models for complex quantitative problems.",no
++dong2024agentops,7,Dong et al.,2024,AgentOps: Enabling Observability of LLM Agents,arXiv preprint,preprint,arXiv:2411.05285,https://export.arxiv.org/api/query?id_list=2411.05285,2026-10-30T03:14:22Z,Taxonomy of agent-lifecycle artifacts to trace; WP29's evidence records cover these observability artifacts.,"Presents a taxonomy of AgentOps artifacts and data that should be traced throughout the agent lifecycle to achieve observability, derived from a systematic mapping study of existing AgentOps tools.",no
++khanal2026reliability,7,Khanal et al.,2026,Beyond pass@1: A Reliability Science Framework for Long-Horizon LLM Agents,arXiv preprint,preprint,arXiv:2603.29231,https://export.arxiv.org/api/query?id_list=2603.29231,2026-10-30T03:15:09Z,"Shows capability (pass@1) and reliability diverge with task duration, supporting WP29's repeated-run reliability outcomes.","Introduces four reliability metrics (Reliability Decay Curve, Variance Amplification Factor, Graceful Degradation Score, Meltdown Onset Point) and, over 23,392 episodes of 10 models on 396 tasks, finds reliability decay is domain-stratified (software-engineering GDS falls from 0.90 to 0.44) while document processing is nearly flat.",no
++zhuang2026agentrewind,7,Zhuang et al.,2026,AgentRewind: Recoverable Execution for Long-Horizon LLM Agents,arXiv preprint,preprint,arXiv:2608.14380,https://export.arxiv.org/api/query?id_list=2608.14380,2026-10-30T03:15:09Z,Checkpoint/rewind recovery for long-horizon agents; the failure-recovery mechanism WP29's protocol runs implement at stage boundaries.,"AgentRewind records aligned checkpoints of agent context and a controlled environment so that agents can return to an earlier state and resume with information from previous attempts, evaluated on the MettleBench long-horizon engineering benchmark.",no
++sigelman2010dapper,7,Sigelman et al.,2010,"Dapper, a Large-Scale Distributed Systems Tracing Infrastructure",Google Technical Report dapper-2010-1,report,s2:003d5a65de0ac72daaf105ded903cb3eb88585b3,https://api.semanticscholar.org/graph/v1/paper/003d5a65de0ac72daaf105ded903cb3eb88585b3 (record cached in s2_dapper2010.json),2026-10-30T03:38:36Z,"Classic distributed tracing design (sampling, application-level transparency) referenced for WP29's run tracing.","Describes Google's production distributed tracing infrastructure, designed for low overhead through sampling and application-level transparency.",no
++wadden2020scifact,8,Wadden et al.,2020,Fact or Fiction: Verifying Scientific Claims,Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP),conference,doi:10.18653/v1/2020.emnlp-main.609,https://api.crossref.org/works/10.18653/v1/2020.emnlp-main.609 ; doi.org HEAD 302,2026-10-30T03:14:22Z,Scientific claim verification task and dataset; model for WP29's claim-to-evidence checks in generated reports.,"Introduces scientific claim verification with SciFact, 1.4K expert-written claims paired with evidence abstracts annotated with labels and rationales; the system identified plausible evidence for 23 of 36 COVID-19 claims on CORD-19.",yes
++thorne2018fever,8,Thorne et al.,2018,FEVER: a Large-scale Dataset for Fact Extraction and VERification,"Proceedings of the 2018 Conference of the North American Chapter of
++          the Association for Computational Linguistics: Human Language
++          Technologies, Volume 1 (Long Papers)",conference,doi:10.18653/v1/N18-1074,https://api.crossref.org/works/10.18653/v1/N18-1074 ; doi.org HEAD 302,2026-10-30T03:14:23Z,Canonical fact extraction and verification benchmark defining Supported/Refuted/NotEnoughInfo labels adopted in WP29's citation audit.,"FEVER contains 185,445 claims derived from Wikipedia and labeled Supported, Refuted or NotEnoughInfo (Fleiss kappa 0.6841), with a best pipeline accuracy of 31.87% when the correct evidence is required.",yes
++bohnet2022attributedqa,8,Bohnet et al.,2022,Attributed Question Answering: Evaluation and Modeling for Attributed Large Language Models,arXiv preprint,preprint,arXiv:2212.08037,https://export.arxiv.org/api/query?id_list=2212.08037,2026-10-30T03:14:23Z,Attributed QA frames generation that must cite supporting sources; WP29 requires attributed claims in agent reports.,"Formulates Attributed QA with a reproducible evaluation framework that uses human annotations as gold standard and shows that a correlated automatic metric is suitable for development, benchmarking a broad set of architectures.",no
++walters2023fabrication,8,Walters & Wilder,2023,Fabrication and errors in the bibliographic citations generated by ChatGPT,Scientific Reports,journal,doi:10.1038/s41598-023-41032-5,https://api.crossref.org/works/10.1038/s41598-023-41032-5 ; doi.org HEAD 302,2026-10-30T03:14:24Z,Quantifies fabricated references in LLM literature reviews; justifies WP29's mandatory citation verification gate.,"In 84 short literature reviews on 42 topics containing 636 citations, 55% of GPT-3.5 citations and 18% of GPT-4 citations were fabricated, and 43% versus 24% of the real citations contained substantive errors.",yes
++alkaissi2023hallucinations,8,Alkaissi & McFarlane,2023,Artificial Hallucinations in ChatGPT: Implications in Scientific Writing,Cureus,journal,doi:10.7759/cureus.35179,https://api.crossref.org/works/10.7759/cureus.35179 ; doi.org HEAD 302,2026-10-30T03:14:24Z,Early clinical report of fabricated references ('artificial hallucinations') in scientific writing.,"Reports two case studies in which ChatGPT produced plausible but fabricated content including references when asked to write on rare conditions, documenting positive, negative and troubling aspects of its performance.",unknown
++gao2023rarr,8,Gao et al.,2023,"RARR: Researching and Revising What Language Models Say, Using Language Models",Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers),conference,doi:10.18653/v1/2023.acl-long.910,https://api.crossref.org/works/10.18653/v1/2023.acl-long.910 ; doi.org HEAD 302,2026-10-30T03:14:25Z,Post-hoc attribution and revision pipeline; a model for WP29's independent evidence-grounding pass over agent output.,"RARR automatically finds attribution for the output of any text generator and post-edits unsupported content, significantly improving attribution while otherwise preserving the original output.",yes
++wang2023selfconsistency,8,Wang et al.,2022,Self-Consistency Improves Chain of Thought Reasoning in Language Models,ICLR 2023,conference,arXiv:2203.11171,"https://export.arxiv.org/api/query?id_list=2203.11171 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2203.11171 (venue='International Conference on Learning Representations', type='conference')",2026-10-30T03:14:25Z,Self-consistency is an in-model verification heuristic; WP29 contrasts it with external executable verification.,"Self-consistency samples diverse reasoning paths and selects the most consistent answer, improving chain-of-thought accuracy on GSM8K (+17.9%), SVAMP (+11.0%) and AQuA (+12.2%).",yes
++min2023factscore,8,Min et al.,2023,FActScore: Fine-grained Atomic Evaluation of Factual Precision in Long Form Text Generation,Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing,conference,doi:10.18653/v1/2023.emnlp-main.741,https://api.crossref.org/works/10.18653/v1/2023.emnlp-main.741 ; doi.org HEAD 302 ; https://export.arxiv.org/api/query?id_list=2305.14251,2026-10-30T03:14:25Z,Atomic-fact precision scoring is the template for WP29's claim-level verification of generated reports.,"FActScore decomposes a generation into atomic facts and computes the percentage supported by a reliable knowledge source; ChatGPT biographies reach only 58%, and an automated estimator is introduced.",yes
++huang2024selfcorrect,8,Huang et al.,2023,Large Language Models Cannot Self-Correct Reasoning Yet,ICLR 2024,conference,arXiv:2310.01798,"https://export.arxiv.org/api/query?id_list=2310.01798 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2310.01798 (venue='International Conference on Learning Representations', type='conference')",2026-10-30T03:14:26Z,"Evidence that intrinsic self-correction fails without external feedback, supporting WP29's external gates over agent self-review.","Finds that LLMs struggle to self-correct their reasoning without external feedback, and that performance can even degrade after intrinsic self-correction.",yes
++sakai2026hallucitation,8,Sakai et al.,2026,HalluCitation Matters: Revealing the Impact of Hallucinated References with 300 Hallucinated Papers in ACL Conferences,ACL 2026 (per Semantic Scholar venue record),conference,arXiv:2601.18724,"https://export.arxiv.org/api/query?id_list=2601.18724 ; https://api.semanticscholar.org/graph/v1/paper/arXiv:2601.18724 (venue='Annual Meeting of the Association for Computational Linguistics', type='conference')",2026-10-30T03:15:08Z,"Documents hallucinated citations reaching accepted NLP papers, showing the scale of the problem WP29's citation gate addresses.","Analyzing all ACL, NAACL and EMNLP papers of 2024-2025, nearly 300 papers contain at least one hallucinated citation, half of them at EMNLP 2025, and more than 100 such papers were accepted as main-conference or Findings papers.",yes
++zhao2026hallucinationswild,8,Zhao et al.,2026,LLM hallucinations in the wild: Large-scale evidence from non-existent citations,arXiv preprint,preprint,arXiv:2605.07723,https://export.arxiv.org/api/query?id_list=2605.07723,2026-10-30T03:15:08Z,Large-scale audit of non-existent citations in the literature after LLM adoption; external evidence of the citation-fabrication risk.,"Auditing 111 million references across 2.5 million papers in arXiv, bioRxiv, SSRN and PubMed Central shows a sharp rise in non-existent references after widespread LLM adoption, with a conservative estimate of 146,932 hallucinated citations in 2025 alone.",no

added research/make_coding.py +329 −0

++import csv, os
++ROOT="/Users/simon-pierreboucher/Desktop/Recherche-UQO/methodes/autonomous-research-claude-code/research"
++man={r["wp"]:r for r in csv.DictReader(open(os.path.join(ROOT,"corpus_manifest.csv"),newline="",encoding="utf-8"))}
++F=["wp","file","document_type","source_url","retrieved_utc","sha256","research_domain","research_type","explicit_question","explicit_estimand_or_hypotheses","stated_assumptions","falsification_or_null_result_conditions","literature_or_novelty_requirement","data_access_controls","temporal_information_controls","baseline_requirement","n_phases_declared","dependency_or_gate_structure","stopping_rules","recovery_or_resume_instructions","artifact_requirements","provenance_requirements","review_or_self_audit_stage","disclosure_or_AI_use_statement","budget_or_resource_limits","integrity_prohibitions","contradictory_or_non_operational_requirements","statements_added_during_implementation","analyst_notes"]
++rows=[]
++def R(wp,**k):
++    m=man[wp]; d={"wp":wp,"file":os.path.basename(m["local_file"]),"source_url":m["url"],"retrieved_utc":m["retrieved_utc"],"sha256":m["sha256"]}
++    d.update(k); rows.append(d)
++
++R("WP15",document_type="claude_md",
++research_domain="finance / market microstructure (intraday shock propagation, U.S. equities, OHLCV data)",
++research_type="empirical (with ML extension and synthetic-data tests)",
++explicit_question='"How have the speed, breadth, persistence, and asymmetry of intraday shock propagation across U.S. equities evolved over the past three decades, and under which market conditions does an initially idiosyncratic shock become sectoral or market-wide?"',
++explicit_estimand_or_hypotheses='yes — central hypothesis stated and outcome measures formally defined: "Modern markets may absorb ordinary shocks more efficiently while transmitting extreme shocks more rapidly and discontinuously." ; "Define the Intraday Shock Amplification Ratio: ISAR_{i,t}(h) = …" ; "Optimization must not change estimands or silently alter precision."',
++stated_assumptions='partial — assumptions must be stated and threats listed, and data assumptions are flagged as unverified: "Explain: … identification assumptions; threats." ; "The natural key is initially assumed to be: (symbol, timestamp) But this assumption must be verified."',
++falsification_or_null_result_conditions='yes — "The project must attempt to disprove its own central claim." ; "If a falsification fails, do not hide it. Reassess the claim." ; "A rigorous negative assessment is preferable to a polished but invalid paper."',
++literature_or_novelty_requirement='yes — "Do not overclaim uniqueness until a documented literature search confirms it." ; "never cite a work based only on another paper\'s bibliography." ; literature map with "verification date" column required',
++data_access_controls='yes — "stop if the schema is unusable or if legal/access restrictions prevent research use." ; "authenticated access only if required" ; "No user-specific absolute path may appear in committed code." ; ".env.example" in tree',
++temporal_information_controls='yes — "For a shock at time t, network weights W_{ij,t} must be estimated only from data dated strictly before t." ; "Any liquidity-based inclusion criterion must use lagged data to avoid look-ahead bias." ; "Never randomly split minute observations across train and test." ; "Future information may be used only as a falsification test, never in the baseline estimator."',
++baseline_requirement='yes — "Compare with: unconditional historical mean; linear and logistic regressions; penalized regression; random forest; …" ; "one transparent baseline regression" in first-phase deliverables ; "The primary nonlinear result must also be represented with a transparent non-ML specification."',
++n_phases_declared='7 (Stages A–G in "3. Execution protocol"; plus "30. Minimum deliverables for the first phase" and "33. Immediate actions" 1–16)',
++dependency_or_gate_structure='explicit — "Only after Stage A passes:" ; "Do not run `make release` until the pilot audit and full-data checks pass." ; executable release gate: "The release script must fail if: a table references a missing result; a figure is missing source data; LaTeX does not compile; … manuscript claims disagree with the results registry; data revision is not pinned; tests fail."',
++stopping_rules='yes — dedicated section: "# 31. Stop conditions Stop and write a diagnostic report rather than proceeding if: timestamps cannot be interpreted reliably; OHLC values are extensively inconsistent; … data licensing prevents the intended use; …" ; also "stop if the schema is unusable" (Stage A) and "Escalate only issues that materially alter the research question, data legality, identification, or interpretation."',
++recovery_or_resume_instructions='yes — "use resumable transfers" ; "permit partial reruns and node-level parallelism" ; "Every distributed task must: be idempotent; write to temporary paths; atomically rename on success; save checkpoints; … support retry; validate output before success." ; "never silently overwrite an existing file"',
++artifact_requirements='paper (main.tex + sections, anonymous and named PDFs); Internet Appendix; tables (standalone .tex AND machine-readable .csv/.parquet); figures (vector PDF + PNG, source data in outputs/figures/data/); results registry (outputs/estimates/results_registry.json) feeding LaTeX macros; code (src package, 22 numbered scripts, CLI, Makefile with ~19 targets); tests (8 files); data manifests (raw_files.parquet/csv with sha256, revision, row counts); docs (DATA_DICTIONARY, DATA_AUDIT, EMPIRICAL_DESIGN, LITERATURE_MAP, REPRODUCIBILITY, COMPUTE_LOG, DECISION_LOG, LIMITATIONS); STATUS.md per stage; README, LICENSE, CITATION.cff, pyproject/uv.lock/environment.yml/.env.example; release bundle (cover letter, source_archive.zip, replication_manifest.json, submission_checklist.md); 4 notebooks',
++provenance_requirements='yes — "pin the exact dataset revision or commit hash" ; manifest columns include "sha256; status; download_started_at; … code_commit" ; "Each processed file must record: parent inputs; transformation version; filters; row counts before and after; creation timestamp; code commit." ; "Record: Git commit; Python version; package versions; OS; hardware; dataset revision; …"',
++review_or_self_audit_stage='yes — "Stage C — Raw-data audit" with docs/DATA_AUDIT.md ; "28.3 Release validation" ; decision log "whether pre-specified or data-driven" ; no adversarial referee stage, but "Report blockers and proposed solutions before launching the full download."',
++disclosure_or_AI_use_statement='no — only a style instruction: "The paper must read like a finance paper, not AI-generated prose." (not a disclosure requirement)',
++budget_or_resource_limits='partial — resource estimation required but no cap: "Before a full run: 1. benchmark one month; 2. benchmark one year; 3. estimate CPU hours; 4. estimate memory; 5. estimate temporary storage; 6. estimate final storage; 7. record in docs/COMPUTE_LOG.md." ; "disk-space checks" ; "Avoid loading the full dataset into RAM." ; no monetary or token budget',
++integrity_prohibitions='yes — "Never fabricate data, results, citations, institutional details, variable definitions, statistical significance, sample sizes, or conclusions. Never write a result in the paper before it has been generated and verified from a saved machine-readable output." ; "Never manually type numerical results into LaTeX." ; "Never winsorize away a possible split without logging it."',
++contradictory_or_non_operational_requirements='Non-operational success criteria that no script can check: "the paper documents a new and economically important fact"; "the writing is disciplined"; "prepared to the standard expected in a top finance journal"; "must read like a finance paper, not AI-generated prose". Judgement-dependent trade-off with no threshold: "Do not define isolation so strictly that only trivial events remain. Report the trade-off." External dependency: "main manuscript no longer than the current JF limit" requires checking an external source at run time. Dataset facts are presented as unverified claims ("Claimed coverage: 1992–2026") and must be audited, which is consistent. No internal contradiction found between autonomy ("Do not ask for confirmation for routine implementation choices") and the mandated pause ("Report blockers and proposed solutions before launching the full download") since the latter is an explicit escalation point.',
++statements_added_during_implementation='none identified (no dated additions or reconstruction notes; the WP15 PROMPT_INITIAL copy is byte-identical)',
++analyst_notes='Longest and most engineering-oriented mandate (7 685 words, 151 headings, 12 code blocks). The only mandate with a dedicated stop-condition section and an executable release-validation gate ("must fail if"). No UQO series framing (targets Journal of Finance), no AI-use disclosure, no page-count target other than JF limit. Cluster-compute section (27) presupposes distributed execution.')
++
++R("WP18",document_type="claude_md",
++research_domain="real estate / housing macro-finance (OLG housing economy, leverage, intergenerational inequality)",
++research_type="theoretical (with numerical general-equilibrium simulation)",
++explicit_question='"Under what conditions do inherited wealth, mortgage leverage, nominal debt, housing supply and down-payment constraints generate persistent owner-renter wealth inequality among households with similar labor income? What changes under unexpected inflation?"',
++explicit_estimand_or_hypotheses='partial — candidate contributions to be treated as hypotheses, not formal H statements: "Candidate contributions: (i) an endogenous ownership-entry wealth threshold jointly shaped by bequests, collateral and price feedback; (ii) conditions for persistent ownership/wealth gaps and path dependence; … Treat each as a hypothesis, not a preordained result." ; ten required experiments listed',
++stated_assumptions='partial — assumptions must accompany every proposition but are not stated in the mandate: "For every proposition provide exact assumptions, statement, proof or clearly labeled proof gap." ; one substantive non-assumption imposed: "Do not assume inflation necessarily raises real house prices or enriches every owner."',
++falsification_or_null_result_conditions='partial — failed conjectures and contrary outcomes must be recorded, no disproof condition defined: "Save proofs in `paper/appendix_proofs.tex` and failed conjectures in `research/failed_hypotheses.md`." ; "Investigate contrary outcomes." ; "conditions for persistence/multiple equilibria (only if established)"',
++literature_or_novelty_requirement='yes — "Produce `research/novelty_matrix.csv` comparing each claimed contribution with the closest prior papers. Redesign weak contributions. Do not assert novelty before verifying it." ; "target of 50–80 **verified** references if genuinely relevant and accessible (never pad)"',
++data_access_controls='partial — generic honesty about access only: "Log unavailable files rather than claiming to have read them." ; "ask for essential permissions or missing credentials rather than pretending access" ; calibration "from traceable Canada/Québec sources when justified, or use clearly marked illustrative parameters"',
++temporal_information_controls='no — theoretical paper; only "Record units, observation dates, transformations and sources." (no look-ahead rule)',
++baseline_requirement='partial — benchmark model and baseline scenario, no comparison methods: "First solve a two-/three-period benchmark with known behavior; unit-test budget identities" ; experiment "1. Baseline wealth and ownership by age, income and inheritance quantile."',
++n_phases_declared='5 (Phase 0–4) plus "Final reproducibility and QA"',
++dependency_or_gate_structure='explicit — "**Mandatory order:** inspect previous UQO WPs and review literature → specify mathematical model → derive propositions → implement and validate numerical model → … → audit results → **only then write and compile the LaTeX paper**." ; "**Gate:** do not start manuscript drafting until the baseline model is validated, claimed GE equilibria clear markets, principal simulations and exhibits reproduce, bibliography is verified, and theoretical claims have proofs or appropriate qualifications."',
++stopping_rules='partial — fallback only, no stop condition: "If blocked, document the issue honestly and deliver a scoped partial result."',
++recovery_or_resume_instructions='partial — progress log after every phase, no resume/checkpoint instruction: "Maintain `research/progress.md` after every phase with completed work, evidence, blockers and next steps."',
++artifact_requirements='paper/main.tex + main.pdf; references.bib; appendix_proofs.tex; appendix_computation.tex; 10–14 figures (vector PDF + PNG in results/figures/); 6–9 auto-generated LaTeX tables (results/tables/); raw CSV/Parquet (results/data/); logs; code modules src/{model,household_solver,equilibrium,distributions,simulation,calibration,robustness,figures,tables}.py; tests/; configs YAML; Makefile (setup/test/simulate/figures/tables/paper/all); requirements.txt; research/{progress,wp_style_audit,novelty_matrix,literature_matrix,literature_review,model_specification,failed_hypotheses,results_audit}; README',
++provenance_requirements='partial — traceability without hashes/manifests: "Every paper number must map to a result file." ; "Record units, observation dates, transformations and sources." ; "pinned dependencies, fixed seeds, tests, structured experiment logs"',
++review_or_self_audit_stage='yes — "## Phase 4 — Integrity audit, then write the manuscript Before writing the paper, complete `research/results_audit.md`: experiment, hypotheses, parameters, algorithm, convergence, results, limitations, robustness, and claims actually supported."',
++disclosure_or_AI_use_statement='no — "Do not invent coauthors or acknowledgments." concerns authorship metadata, not AI-use disclosure',
++budget_or_resource_limits='no',
++integrity_prohibitions='yes — "Never invent results, citations, proofs, or data." ; "If general equilibrium does not solve, **never label partial equilibrium as GE**." ; "No fabricated calibration targets or empirical validation." ; "Never manipulate assumptions to force a preferred conclusion." ; "Do not call simulation findings theorems."',
++contradictory_or_non_operational_requirements='Quantity targets coexist with anti-padding clauses and are resolved only by a priority statement: "30–45-page English main text", "10–14 meaningful publication-quality figures and 6–9 substantive tables, adjusted to genuine findings rather than padded", versus "**Priority: scientific correctness, originality and reproducibility over speed, page count or positive findings.**" Non-verifiable from the document: "Rerun from a clean environment" (no environment specification), "Check PDFs visually". The gate condition "bibliography is verified" has no operational verification procedure. The French preamble re-maps the directory name used in the plan ("le nom `wp18_uqo/` du plan ci-dessous désigne ce dossier"), i.e. the plan\'s paths are not literal.',
++statements_added_during_implementation='Prefatory note (French) placed before the mandate: "(Spécification de l\'auteur, 2026-10-20. Conventions de forme : ~/Desktop/Recherche-UQO/HARMONISATION.md ; ce dossier s\'appelle `immobilier/housing-leverage-intergenerational-inequality` et correspond au dépôt spbgit `wp18_uqo.git` — le nom `wp18_uqo/` du plan ci-dessous désigne ce dossier.)" — a repository-mapping note dated with the specification, not marked as post-implementation.',
++analyst_notes='Language: English body with a French prefatory paragraph (automated: mixed). Role framing "Act as an autonomous team of housing economists, …". Requires reading WP1–WP17 for style conventions (research/wp_style_audit.md).')
++
++R("WP19",document_type="claude_md",
++research_domain="real estate / housing market microstructure (search-and-matching, liquidity premium)",
++research_type="theoretical (continuous-time search model with numerical simulation)",
++explicit_question='"How do market tightness, heterogeneous seller reservation values, financing frictions, time-on-market costs, and bargaining power jointly determine an endogenous housing liquidity discount, transaction prices, selling hazards, and price dispersion?"',
++explicit_estimand_or_hypotheses='yes — "Candidate propositions to PROVE: higher carrying cost lowers reservation price; decline in arrival/qualification widens the discount and increases TOM (with caveats); liquidity shocks move prices and volume without a fundamental shock; dispersion from heterogeneous urgency; selection bias in transaction indices." ; defined object "precise liquidity premium/discount definition with sign conventions (D = (p* − p)/p*)" ; "Each: preregistered hypothesis, exact config, output CSV, entry point, diagnostics, interpretation."',
++stated_assumptions='partial — required, not stated: "explicit timing, assumptions, parameter restrictions, identification limits, boundary cases" ; "Full assumptions, statements, proofs or counterexamples, intuition; label numerical patterns as conjectures."',
++falsification_or_null_result_conditions='partial — counterexamples and failures must be reported; no disproof condition: "proofs or counterexamples" ; "record failed experiments and regions without equilibrium" ; "Report failures."',
++literature_or_novelty_requirement='yes — "Novelty gate: if the mechanism is already established, adjust the model and contribution before proceeding." ; "Authors to investigate only with independent bibliographic verification"',
++data_access_controls='partial — "No proprietary data, secrets, or unlicensed material." ; "If an endpoint is unavailable, log the failure and proceed" ; "do not claim to have read inaccessible material"',
++temporal_information_controls='no — not applicable (theoretical, synthetic experiments)',
++baseline_requirement='partial — baseline scenario and frictionless benchmark, not comparison methods: "1 baseline; 2 urgency gradient; …" ; "frictionless benchmark p*" ; "11 Nash vs posted prices"',
++n_phases_declared='8 (Phase 0–7); also an 8-step "Mandatory execution sequence: (0) … (7)"',
++dependency_or_gate_structure='explicit — "Mandatory execution sequence: (0) inspect prior UQO papers … (7) only then draft and compile the full LaTeX manuscript." ; "Novelty gate" ; "## Phase 6 — Independent audit before prose drafting" ; "## Research integrity and publication quality gates" ; "Introduction and abstract written last."',
++stopping_rules='no — "Report failures." only; no condition under which work stops',
++recovery_or_resume_instructions='no',
++artifact_requirements='full repository; paper/main.pdf, main.tex, references.bib; scripts (src/{model,solver,simulation,calibration,diagnostics,plotting,tables}.py); tests (pytest); config/baseline.yaml; 10–14 figures (PNG + PDF); 6–9 tables; research/{wp_style_audit,literature_review,literature_matrix.csv,novelty_matrix.csv,model_specification,proofs,validation,results_audit,limitations}; README "with commands and output inventory"; "final report (contributions, proved propositions, strongest results with figure/table references, literature coverage, unresolved items, reproducibility status, file locations)"',
++provenance_requirements='yes — "Fix random seeds; save environment and configuration hashes." ; "verify every number against result files" ; "`research/results_audit.md` maps every claim to proposition/proof, simulation file/figure/table, or citation"',
++review_or_self_audit_stage='yes — "## Phase 6 — Independent audit before prose drafting `research/results_audit.md` maps every claim … classify as proved, numerically illustrated, empirically documented, or speculative"',
++disclosure_or_AI_use_statement='no',
++budget_or_resource_limits='no',
++integrity_prohibitions='yes — "Never invent a proof, result, reference, DOI, dataset, or empirical validation." ; "Never fabricate outputs, citations, proofs, parameters or significance. No empirical language for synthetic experiments." ; "No \\"publishable / first-of-its-kind\\" claims."',
++contradictory_or_non_operational_requirements='"preregistered hypothesis" is required for each of the 12 experiments but no registration mechanism, timestamp or location is specified, so pre-registration cannot be verified from the document. "Independent audit" (Phase 6) is to be performed by the same agent that produced the results — independence is not operational. Reference target "Collect and verify 50–80 genuinely relevant references, not a quota" and length "30–45-page main paper" are targets with built-in escape clauses. "compile with latexmk; resolve warnings; inspect the rendered PDF" presumes a TeX toolchain not checked for.',
++statements_added_during_implementation='none identified',
++analyst_notes='Most compact fully phased mandate (1 092 words, 8 phases). Explicitly scoped against WP18 ("Key distinction from WP18"). Role framing "You are an autonomous interdisciplinary research team".')
++
++R("WP20",document_type="claude_md",
++research_domain="real estate appraisal / multimodal AI (cost approach from listing images and text; Vrai Prix workflow)",
++research_type="mixed (empirical evaluation + design-science pipeline, with a clearly labeled synthetic fallback)",
++explicit_question='"RQ1: How accurately can multimodal AI extract appraisal-relevant building attributes from listing photographs and descriptions relative to verified property records or expert annotations?" (RQ1 of five)',
++explicit_estimand_or_hypotheses='partial — five research questions and a formal valuation identity, no hypotheses: "\\widehat V_i^{cost}=\\widehat L_i + \\widehat{RCN}_i - \\widehat D_i + \\widehat S_i" ; metrics "attribute extraction precision/recall/F1 for categorical features, MAE/RMSE for continuous features, calibration and abstention metrics, and cost-approach accuracy versus independent credible cost or appraisal benchmarks"',
++stated_assumptions='partial — "Prove only results supported by stated assumptions." ; "Separate observable visual features, inferred proxy features, verified administrative fields, text-claimed features, and latent/unobservable features."',
++falsification_or_null_result_conditions='partial — "Favor transparent uncertainty and falsifiable evidence over marketing language." ; "If evidence for novelty is weak, revise the contribution statement accordingly." ; "never describe synthetic findings as real-world validation" ; no disproof condition for RQ1–RQ5',
++literature_or_novelty_requirement='yes — "aiming for 60–100 genuinely verified scholarly references if sufficient relevant work exists; quality and relevance take priority over quota." ; "Establish novelty only after examining existing literature." ; "Gate 2 — Literature: verified bibliography, mapped novelty and gap."',
++data_access_controls='yes — "Respect website terms, robots policies where applicable, copyright, privacy, and licensing; do not assume scraping or publication permission." ; "Keep secrets in environment variables, never in tracked files." ; "Never bypass access controls or website restrictions." ; "Avoid any use of private API keys in paper, logs, screenshots, or Git history." ; docs/data_rights.md',
++temporal_information_controls='yes — "Prevent train/test leakage by property, duplicate listing, and time." ; "Where feasible, use temporal and geographic holdouts." ; experiment "10. Geographic and temporal out-of-sample performance."',
++baseline_requirement='yes — "Compare at least: - B0: traditional cost approach using verified structured fields and standard cost schedules. - B1: listing-text-only extracted attributes + cost model. - B2: image-only … - B3: multimodal image+text … - B4: multimodal + verified administrative fields + uncertainty-aware reconciliation." ; "Do not make comparisons unfair by giving one method privileged information without identifying it."',
++n_phases_declared='6 (six-step "Mandatory sequence: (1) … (6) ONLY THEN write"); organised in sections A–J with 7 numbered quality gates',
++dependency_or_gate_structure='explicit — "Gate 1 — Repository audit … Gate 2 — Literature … Gate 3 — Theory … Gate 4 — Implementation … Gate 5 — Experiments … Gate 6 — Manuscript: only then draft and compile … Gate 7 — Integrity" ; "Hard gate: Do not draft Results or Conclusions until numerical outputs have been executed, independently checked, and saved."',
++stopping_rules='partial — blocker handling, no stop: "When missing permissions, source data, or access makes a claim impossible to validate, clearly identify the blocker, proceed with the portions that are feasible, and provide the exact minimal information needed to finish."',
++recovery_or_resume_instructions='partial — "caching, retry limits" ; "Maintain a TODO list and research log." ; no resume instruction',
++artifact_requirements='paper/main.pdf + main.tex + sections/ + references.bib + figures/ + tables/; ~12–16 vector PDF figures; 8–12 LaTeX tables generated from saved data; src/{ingestion,extraction,feature_schema,cost_model,evaluation,simulations,visualization}; configs/; data/{README,raw,processed,synthetic}; results/manifest.json; docs/{implementation_audit,literature_matrix.csv,experiment_plan,results_validation,data_rights,limitations,research_log}; tests/; requirements.txt; Makefile; data dictionary + data-flow diagram; annotated extraction schema; reproducibility instructions; limitations report; "final checklist distinguishing completed, blocked, and proposed items"',
++provenance_requirements='yes — "Store prompts, model versions, temperatures, and response schemas." ; "generate a machine-readable results/manifest.json" ; "Maintain docs/implementation_audit.md with citations to local paths or source lines" ; "Every substantive factual claim must have a traceable source." ; "Avoid hardcoded table results; generate them from actual outputs."',
++review_or_self_audit_stage='yes — "Gate 7 — Integrity: audit every numerical claim against saved outputs and every citation against source metadata." ; "Write docs/results_validation.md documenting each check."',
++disclosure_or_AI_use_statement='no',
++budget_or_resource_limits='no — "retry limits" only; experiment 16 mentions "annotation time/cost" as an outcome, not a limit',
++integrity_prohibitions='yes — "Never invent datasets, numerical findings, literature references, citations, software functionality, API results, human labels, or Vrai Prix implementation details. Never fabricate significance." ; "Do NOT infer foundation integrity, hidden mold, structural soundness, exact year built, precise area, code compliance, permit status, or legal tenure from images alone."',
++contradictory_or_non_operational_requirements='Human-dependent steps inside an autonomous mandate: "Propose a rigorous annotation protocol with independent expert raters" and experiment "4. Human expert agreement and AI-human disagreement" cannot be executed by the agent (mitigated by "subject to data availability"). "independently checked" in the hard gate has no independent checker. Verification of 60–100 references "against primary records" presumes network access not confirmed. Figure/table count targets ("approximately 12–16 figures and 8–12 tables") versus "never pad with meaningless exhibits". The French preamble re-maps the directory used in the plan ("le nom `wp20_uqo/` du plan ci-dessous désigne ce dossier").',
++statements_added_during_implementation='Prefatory note (French): "(Spécification de l\'auteur, 2026-10-20. Conventions de forme : ~/Desktop/Recherche-UQO/HARMONISATION.md ; ce dossier s\'appelle `immobilier/multimodal-cost-approach-valuation` et correspond au dépôt spbgit `wp20_uqo.git` — le nom `wp20_uqo/` du plan ci-dessous désigne ce dossier. Journal : `docs/research_log.md`.)" — repository mapping and log location, dated with the specification.',
++analyst_notes='Language mixed (French preamble). Unusually explicit ethics/rights section (E) and a closing principle: "Produce the strongest defensible scientific contribution possible, not the most optimistic portrayal of AI or Vrai Prix." Keyword family `budget` is inflated by "cost approach" (42 hits of "cost").')
++
++R("WP21",document_type="claude_md",
++research_domain="real estate economics / applied econometrics (systematic review and meta-analysis of hedonic pricing models)",
++research_type="empirical (meta-analysis with automated full-text extraction pipeline)",
++explicit_question='"RQ1 characteristics that consistently affect prices across countries and periods; RQ2 size and uncertainty of marginal contributions; RQ3 variation with geography, market conditions, property types, socioeconomics" (RQ1–RQ3 of eight, written as fragments)',
++explicit_estimand_or_hypotheses='partial — effect-size families and pooled quantities defined, no hypotheses: "Families: percentage premiums, elasticities, semi-elasticities, standardised effects, partial correlations, monetary marginal prices." ; "8.1 random-effects (REML) with pooled estimate, SE, 95% CI, 95% PI, τ², I², k studies, n estimates"',
++stated_assumptions='partial — methodological constraints stated as rules rather than assumptions: "no monetary-to-percentage conversion without a defensible reference price" ; "Pool only within comparable groups; document and propagate uncertainty." ; "protocol before synthesis"',
++falsification_or_null_result_conditions='no — robustness and publication-selection diagnostics only ("Leave-one-study-out, leave-one-country-out, …"; "funnel, small-study effects, PET-PEESE where appropriate"); no condition under which a claim is abandoned, except "never relax criteria to reach targets"',
++literature_or_novelty_requirement='yes — the paper is a literature synthesis; "Compare with competing reviews before any \\"largest\\" claim." ; "Explain precisely how the paper extends Sirmans et al. (2005, 2006)"',
++data_access_controls='partial — "read in full when legally accessible" ; "Google Scholar (compliant)" ; no secrets/credential rules',
++temporal_information_controls='no — a search cutoff date is set ("Reproducible search 1970–2026-10-20") but no information-leakage rule applies',
++baseline_requirement='partial — comparison to a prior synthesis rather than baseline methods: "comparison with Sirmans et al." (required table) ; "RQ5 traditional vs spatial vs machine-learning estimates"',
++n_phases_declared='7 (execution order A–G, inline)',
++dependency_or_gate_structure='implicit — ordering only, no gate conditions: "15. EXECUTION ORDER. A research design → B literature acquisition → C data construction → D empirical analysis → E scientific interpretation → F manuscript → G final audit. No results written before analysis"',
++stopping_rules='no — "document obstacles and continue with feasible work."',
++recovery_or_resume_instructions='no',
++artifact_requirements='LaTeX paper (main.tex, sections, tables, figures, appendices, references.bib, reproducibility statement); structured research database data/{raw,metadata,extracted,validated,final}; scripts/{discovery,screening,extraction,harmonization,validation,econometrics,visualization}; protocol/{preregistration,eligibility,search_strategy}.md; reports/{search_audit,extraction_audit,methodological_decisions,limitations}.md; PRISMA flow diagram; 13 required tables and 12 required figures; tests; requirements.txt; README',
++provenance_requirements='yes — "full records of queries, dates, sources, screening decisions, retrieval failures" ; "Store raw and harmonised values; never infer missing SEs except transparently from reported statistics; preserve every conversion formula; every number traceable to source table." ; "coefficient-level information (… page and table, extraction confidence)"',
++review_or_self_audit_stage='yes — "G final audit" ; "Extraction validation: random sample of tables independently verified; report precision, recall, numerical accuracy, error rates; unverified or low-confidence extractions excluded from the primary synthesis." ; "flag ambiguous studies for human review"',
++disclosure_or_AI_use_statement='no',
++budget_or_resource_limits='no',
++integrity_prohibitions='yes — "Never invent studies, citations, coefficients, standard errors, observations, statistical results, or claims of comprehensiveness." ; "never relax criteria to reach targets" ; "Never synthetic results." ; "no promotional language"',
++contradictory_or_non_operational_requirements='Aspirational targets ("≥5,000 unique records screened, ≥1,000 full texts, several hundred eligible studies") are explicitly non-binding ("never relax criteria to reach targets"), so compliance cannot be judged. "protocol/preregistration.md" is required but no mechanism makes it precede the synthesis verifiably. "random sample of tables independently verified" has no independent verifier for a single agent; "flag ambiguous studies for human review" introduces a human-in-the-loop step without specifying who or when. "vision/LLM-assisted classification" implies paid model calls with no budget or recording requirement. "read in full when legally accessible" for ≥1,000 full texts is not checkable. R "where superior" is a judgement call.',
++statements_added_during_implementation='none identified',
++analyst_notes='Densest format: 15 numbered paragraphs, a single Markdown heading (automated n_headings = 1), 1 114 words. Only mandate whose primary object is the literature itself; "Autonomous scientific research assignment, not a narrative review."')
++
++R("WP22",document_type="claude_md",
++research_domain="real estate economics / price-index econometrics (quality-adjusted housing price indices, Quebec transactions)",
++research_type="mixed (empirical index construction + Monte Carlo simulation + methodological contribution)",
++explicit_question='none — stated as objectives, e.g. "Investigate an adaptive estimator that optimizes the trade-off between local information, borrowing strength, temporal responsiveness, bias, and variance." (not phrased as a question)',
++explicit_estimand_or_hypotheses='yes — dedicated section: "## 4. Core estimand Define I[g,k,t] as the quality-constant market price index for geography g, building type k and month t. Establish a reference property distribution and common-support rules. Set 2021 annual average = 100 for the primary index" ; no hypotheses as such',
++stated_assumptions='partial — "Enforce identifiability constraints and explicitly describe priors, likelihood, covariance, temporal evolution and estimation algorithm." ; "Make explicit whether the headline is a price index for the stock, for transacted properties, or for a representative constant-quality basket."',
++falsification_or_null_result_conditions='yes — "Include deliberate counterexamples where SHARP-HPI underperforms simpler alternatives." ; "Report when reconciliation introduces bias or excessive smoothing." ; "Conduct placebo tests, synthetic quality-composition shocks, …" ; "Test whether estimated changes reflect new transaction coverage rather than market dynamics."',
++literature_or_novelty_requirement='yes — "Derive and test an adaptive pooling/shrinkage mechanism; do not call it novel unless a literature review establishes novelty." ; "Explain exactly how the proposed contribution differs from each close predecessor." ; anchor references named (Hill et al. 2022; Ishaak et al. 2024; Hill 2026)',
++data_access_controls='yes — "Discover local Vrai Prix sources, read-only credentials, schemas, backups … Never assume credentials, copy secrets into logs, or modify production." ; "Keep raw transaction records private; publish only legally permissible aggregate results. Never commit credentials or personally identifying records."',
++temporal_information_controls='yes — "Never use post-transaction information or future data in real-time estimates." ; "(2) real-time vintage series estimated using only information available at each historical publication cutoff." ; "Use forward-chaining and rolling-origin backtesting, never random train-test splits for temporal index claims." ; "Prevent image-derived signals from encoding the sale price, future information, listing outcome or date-dependent model artifacts."',
++baseline_requirement='yes — "## 5. Benchmark estimators Implement and compare: A. Raw median and arithmetic mean prices. B. Stratified median. C. Standard pooled hedonic time-dummy OLS. D. Rolling time-dummy … K. Existing QHPI method as a documented benchmark."',
++n_phases_declared='8 (Phase 0–7 in "21. Work sequence")',
++dependency_or_gate_structure='explicit — "## 12. Simulations — mandatory before paper claims" ; "## 22. Final quality gate No paper can be declared complete until all figures and tables are reproducible, every empirical claim is linked to a generated result, all citations are verified, benchmarks have run, Monte Carlo tests are documented, uncertainty is calibrated, geographic coverage is disclosed, sparse cells are labeled, leakage tests pass, the manuscript compiles without substantive errors, and the results have been critically reviewed for overclaiming." (one terminal gate; phases 0–7 are ordered without per-phase pass criteria)',
++stopping_rules='partial — fallback, no stop: "If data access is unavailable, implement testable code and synthetic-data simulations, but never pretend real-data analysis has been completed."',
++recovery_or_resume_instructions='partial — "cached intermediate artifacts" ; "At every phase save machine-readable artifacts and an explicit decision log." ; no resume instruction',
++artifact_requirements='data_dictionary.md; provenance_report.md; cleaned transaction pipeline + exclusion audit; model specification and derivation; reproducible benchmark and SHARP estimators; Monte Carlo framework and results; monthly index panel in Parquet and CSV with a 23-field row schema; coverage/reliability tables; index vintage/revision database; tables and figures from actual outputs; optional interactive geographic diagnostics; limitations and reproducibility documentation; English LaTeX manuscript with BibTeX; unit/integration tests; versioned configs; environment lockfiles; decision log',
++provenance_requirements='yes — "Build an immutable extraction snapshot." ; "provenance_report.md" ; index rows carry "data_cutoff, publication_vintage, model_version, index_method" ; "versioned configurations, unit tests, integration tests, data contracts, environment lockfiles and cached intermediate artifacts"',
++review_or_self_audit_stage='yes — "## 1. Audit and provenance" (data audit) ; final gate requires "the results have been critically reviewed for overclaiming" ; "final reproducibility audit" in Phase 7',
++disclosure_or_AI_use_statement='no',
++budget_or_resource_limits='no — "Profile memory and runtime." and "Use streaming/chunked computation when necessary." are engineering guidance, not limits',
++integrity_prohibitions='yes — "Do not invent data, empirical results, references, statistical significance, or novelty claims." ; "Never equate predictive accuracy with index accuracy." ; "A model estimate with zero local transactions is not an observed monthly price movement." ; "Avoid fabricated precision." ; "Do not silently substitute assessment values for market transactions."',
++contradictory_or_non_operational_requirements='Terminal-gate conditions that cannot be checked from the document alone: "uncertainty is calibrated" (requires truth, available only in simulation), "critically reviewed for overclaiming" (no reviewer defined), "a publishable contribution". Data-dependent requirements that may be unsatisfiable: "aggregation weights, ideally grounded in eligible housing stock", "Compare aggregate series to suitable external indices", "historical boundary crosswalks". Optional/unverifiable: "interactive geographic diagnostics if practical". Factual assertion inside an instruction: "The October 2026 observation is partial through the actual last reliable transaction date". No explicit escalation or stop rule despite "Begin immediately … Continue autonomously".',
++statements_added_during_implementation='none identified',
++analyst_notes='Only mandate with a dedicated "Core estimand" section and a prescribed output data schema. 22 numbered sections, 1 966 words. Strongest temporal-information discipline in the corpus (real-time vintages, revision policy).')
++
++R("WP23",document_type="claude_md",
++research_domain="real estate economics / machine-learning forecasting (spatiotemporal, hierarchical and probabilistic price forecasting, Quebec)",
++research_type="mixed (empirical forecasting evaluation + simulation study)",
++explicit_question='"do newly designed forecasting architectures substantially improve residential real estate price predictions?"',
++explicit_estimand_or_hypotheses='partial — forecasting targets defined, no hypotheses: "Task A — future transaction price: P(i,t+h) = f(X(i,t), S(i,t), M(t), h); address transaction selection explicitly." ; "Task B … horizons 1, 3, 6, 12 months" ; "Task C — probabilistic forecasting: predictive distributions (mean, median, intervals, quantiles, downside risk, P(decline), …)"',
++stated_assumptions='no — none stated or required beyond "Never assume a variable exists."',
++falsification_or_null_result_conditions='partial — the question admits a negative answer and the simulation must show failures, but no disproof condition: "(8) when complex models beat conventional ones" ; "when methods succeed or fail; explain mechanisms" ; "honest novelty statement"',
++literature_or_novelty_requirement='yes — "Never manufacture findings or claim novelty without verifying the literature." ; "# 13. LITERATURE REVIEW — … verify every reference; classify contributions (existing / adapted / new combination / original)." ; "literature search first"',
++data_access_controls='partial — read-only rule on the upstream project only: "WP22 (`../sharp-hpi-quebec`) is read-only for this project." ; "never modify WP22" ; no credential/secret rules',
++temporal_information_controls='yes — "Every feature computed from information available at the forecast origin; no leakage." ; "strict pseudo-real-time expanding/rolling windows; for each origin: information set → estimate → forecast → store → compare. Never random splits." ; "regimes from information available at the origin only" ; "Prevent leakage from WP22: reconstruct index vintages from transactions observable at the origin; audit hedonics, clustering, normalisation, feature selection, imputation, target encoding, graph estimation, tuning, revisions."',
++baseline_requirement='yes — "# 6. BENCHMARKS — econometric (hedonic OLS, ridge/EN, SAR, ARIMA/SARIMA/ARIMAX, VAR/BVAR, DFM, local linear trend, state space), ML (RF, XGBoost, LightGBM, CatBoost, SVR, MLP), deep (LSTM, GRU, TCN, TFT, modern TS), simple (random walk, no change, mean growth, seasonal naive, momentum); fair common origins/information sets"',
++n_phases_declared='none (18 numbered topical sections, no execution phases; only "literature search first" and "SIMULATION STUDY" ordering cues)',
++dependency_or_gate_structure='explicit (terminal gate only) — "# 18. QUALITY GATE — reproducible figures/tables, verified references, leakage audit passed, strict chronology, uncertainty reported, honest novelty statement, manuscript compiles."',
++stopping_rules='no',
++recovery_or_resume_instructions='no',
++artifact_requirements='complete academic LaTeX manuscript (15 sections + appendices on architectures, hyper-parameters, tests, supplementary tables); reproducible Python research pipeline; original forecasting models; empirical results; 15 listed figure types (vector, reproducible); 13 listed table types; supplementary material; simulation study',
++provenance_requirements='partial — "reproducible Python research pipeline", "vector; reproducible" figures and a "leakage audit", but no manifest, hash, seed-logging or number-to-file traceability requirement',
++review_or_self_audit_stage='partial — "leakage audit passed" inside the terminal quality gate; no dedicated results audit or referee stage',
++disclosure_or_AI_use_statement='no',
++budget_or_resource_limits='no — a "computational cost" table is required as a reported outcome, not as a limit',
++integrity_prohibitions='yes — "Never manufacture findings or claim novelty without verifying the literature." ; "no granular results for inadequate cells" ; "predictive importance ≠ causality" ; "no over-complex benchmarks the data cannot estimate"',
++contradictory_or_non_operational_requirements='The document itself declares that it is partly reconstructed: "the pasted text ended at section 17 \\"11. Prob\\"; sections 17–end were completed from the UQO working-paper conventions" and "[Sections 11–15 and the remainder of the protocol were reconstructed by the author\'s assistant because the pasted protocol was truncated.]" — part of the mandate is therefore not the author\'s original instruction. Six named "original" architectures (HSHN, DSGFN, RAREF, BHPF, QANSS, QREFA) are prescribed in advance while novelty is to be "verified" afterwards and "keep only components with incremental out-of-sample value" — pre-specified novelty versus post-hoc selection. "substantially improve" has no effect-size threshold. Heavy methods ("VI / Laplace / HMC on subsets") with no compute budget. Section numbering is internally inconsistent (headings "# 17." and "# 18." versus the note "ended at section 17 \\"11. Prob\\"").',
++statements_added_during_implementation='Two explicit markers: (1) header "> Protocol as received on 2026-10-21 (the pasted text ended at section 17 \\"11. Prob\\"; sections 17–end were completed from the UQO working-paper conventions, see docs/PLAN.md). WP22 (`../sharp-hpi-quebec`) is read-only for this project." ; (2) in section 17: "[Sections 11–15 and the remainder of the protocol were reconstructed by the author\'s assistant because the pasted protocol was truncated.]"',
++analyst_notes='Telegraphic style (1 132 words, 20 headings). The only mandate whose provenance is partly non-authorial by its own admission. No phases, no stop or recovery rules, no provenance manifest, no self-audit beyond the leakage audit.')
++
++R("WP24",document_type="claude_md",
++research_domain="macro-finance theory (AGI, task-based automation, general equilibrium and asset valuation; extension of Restrepo 2025)",
++research_type="mixed (theoretical model + quantitative simulation + empirical calibration/short-run design)",
++explicit_question='"How does increasingly capable and abundant AI reallocate factor income from labor to reproducible and scarce capital, change the ownership distribution of aggregate wealth, and jointly determine the prices and expected returns of financial, real-estate, and other assets?"',
++explicit_estimand_or_hypotheses='yes — nine target results: "T1. Conditions under which Restrepo\'s labor-share asymptote survives multi-sector production with complementary physical inputs and scarce factors; counterexamples with binding energy/land. T2. Sufficient conditions for factor-income reallocation …" through T9 ; "Each result needs explicit assumptions, mathematical statement, rigorous proof or counterexample, economic interpretation, and simulation check."',
++stated_assumptions='yes — primitives are stated and a numbered assumptions protocol and register are required: "Restrepo\'s central primitives include task output X_t(ω)=L_t(ω)+Q_t(ω)/α_t(ω), …" ; "Proof protocol: write assumptions as numbered statements; verify differentiability, convexity/concavity, equilibrium existence, transversality, boundedness, market clearing, timing, stochastic integrability, and no-arbitrage." ; `docs/assumptions_register.md`',
++falsification_or_null_result_conditions='yes — "Do not assume that AI automatically raises profits, all asset prices, inequality, or aggregate welfare. Derive conditions and counterexamples." ; "Tests must fail on deliberately inconsistent accounting." ; "identify parameter regimes where asset valuations fall despite higher output" ; "Provide a falsifiable short-run empirical design: … with pre-trends, confounding, measurement error and alternative mechanisms addressed."',
++literature_or_novelty_requirement='yes — "Prepare a literature matrix: … Verify exact references from authoritative sources. Mark all unverified claims." ; referee question "(1) is this genuinely new relative to Restrepo and task models?" ; "Explicitly separate the original author\'s conclusions from our proposed extensions."',
++data_access_controls='yes — "Do not present proprietary datasets as accessible unless actually available." ; "company annual reports/Compustat if licensed" ; "`data/README.md`, `data/raw/`, `data/processed/`: sources, licenses, provenance and transformations; no fabricated observed data."',
++temporal_information_controls='partial — vintage awareness and pre-trend checks, no leakage rule: "Check exact definitions, vintages, geographic scope, units, time frequency and licenses." ; "Validate against pre-AGI historical patterns without claiming the historical sample identifies post-AGI structural parameters." ; "Do not use speculative AGI dates as facts."',
++baseline_requirement='yes — "Reproduce a minimal baseline numerically or analytically before adding features." ; "Implement a real baseline first." ; paper section "3. Benchmark Restrepo replication and limitations." ; "Define a transparent benchmark and add layers with tests at every stage."',
++n_phases_declared='7 (Phase 0–6) plus a 12-step "EXECUTION ORDER" A–L',
++dependency_or_gate_structure='explicit — "## EXECUTION ORDER AND STOP CONDITIONS A. Audit source PDF and write source notes; B. derive smallest exact benchmark; C. … L. perform referee audit and final replication." ; "Never declare a theorem proven without a proof; never declare a model solved without numerical equilibrium checks; never declare a PDF compiled without verifying the file." ; "Implement deterministic perfect-foresight transition first, then aggregate shocks" (no numbered gates)',
++stopping_rules='partial — the heading says "STOP CONDITIONS" but the content is a fallback: "If a block cannot be solved, deliver a verified simpler model and clearly document the limitation." ; "start implementing without waiting for further permission unless a genuinely blocking external input is required."',
++recovery_or_resume_instructions='partial — "At each stage update `docs/research_log.md` with completed steps, assumptions, residuals, unresolved problems and next actions." ; "Record seeds, solver tolerances and convergence diagnostics." ; no resume/checkpoint instruction',
++artifact_requirements='paper/main.tex (35–60+ pages); paper/appendix.tex; paper/references.bib; slides/main.tex (30–45 Beamer slides); src/ (solver, simulations, calibration, tests); data/README.md + raw/ + processed/; results/ (tables, figures PDF+PNG, parameter files, machine-readable outputs); docs/{model_dictionary,assumptions_register,identification,research_log,replication,referee_report}.md; Makefile; requirements.txt or pyproject.toml; environment lockfile; README with one-command reproduction; compiled PDFs "if TeX available"',
++provenance_requirements='yes — "sources, licenses, provenance and transformations" for data ; "Build a source-to-parameter table with observed moments, uncertainty and identification limits." ; "Clearly separate externally identified, calibrated and scenario-only parameters." ; "every result traceable to output" ; "Record seeds, solver tolerances and convergence diagnostics."',
++review_or_self_audit_stage='yes — "## PHASE 6 — CRITICAL SELF-REVIEW Write an adversarial referee report addressing: (1) is this genuinely new relative to Restrepo and task models? (2) is the model too large to identify? … (12) can the core mechanisms be established in a simpler model? Revise after review." ; docs/referee_report.md',
++disclosure_or_AI_use_statement='no',
++budget_or_resource_limits='no',
++integrity_prohibitions='yes — "no invented citations, DOIs or results" ; "no fabricated empirical estimates, no invented statistical significance" ; "Never declare a theorem proven without a proof; never declare a model solved without numerical equilibrium checks; never declare a PDF compiled without verifying the file." ; "Label conjectures as conjectures, not theorems." ; "Avoid hype words like \'revolutionary\' or \'legendary\' in manuscript."',
++contradictory_or_non_operational_requirements='Scope tension: an 8–12-sector IO economy with heterogeneous households, full asset-pricing block, optional monetary/fiscal/open-economy modules and a 2026–2100 simulation versus "Use a nested, modular economy, not an intractable all-at-once model" and "Prioritize a correct publishable contribution over a grand but incoherent system" (resolved only by priority ordering). Length targets ("35–60+ page", "30–45 Beamer") with no anti-padding clause other than the priority statement. External dependency: "Download and read the complete original PDF" presumes network access. "STOP CONDITIONS" heading without any stop condition. Conditional deliverables ("Compile PDFs if TeX available") make completion ambiguous. Non-verifiable: "publication-quality", "disciplined findings".',
++statements_added_during_implementation='Explicit post-hoc section: "## UQO conventions (added 2026-10-27) This project is **UQO Working Paper No. 24** in `Recherche-UQO/finance/`. Follow `Recherche-UQO/HARMONISATION.md` (canonical title page, `\\WP…` macros, README template, `paper/Makefile` + `.latexmkrc`, git remote `gitsrv:srv/git/wp24_uqo.git`)."',
++analyst_notes='Mandate was written without UQO framing and later attached to the series (addition dated 2026-10-27). Contains a "FIRST RESPONSE EXPECTED FROM CLAUDE CODE" section specifying the first turn. Only mandate requiring seminar slides.')
++
++R("WP25",document_type="claude_md",
++research_domain="economic theory / computational economics (agentic general equilibrium, LLM agents as delegated decision-makers)",
++research_type="mixed (theoretical model + heterogeneous-agent numerics + LLM-agent simulation laboratory)",
++explicit_question='"How does delegation of economic decisions to autonomous AI agents change equilibrium wages, interest rates, asset prices, capital accumulation, production, credit, systemic risk, and wealth inequality?"',
++explicit_estimand_or_hypotheses='yes — six falsifiable hypotheses: "H1: Delegation creates an endogenous adoption threshold determined by information gains, execution costs, compute prices and agency losses. H2: Shared foundation models correlate trades and may reduce market depth under stress, conditional on common information. H3: …" through H6 ; "Pre-register outcomes."',
++stated_assumptions='partial — required at every stage and sketched as a non-binding starting point: "(b) write down assumptions" ; "Provide propositions and rigorous proofs under explicit assumptions, or label unsolved statements conjectures." ; "Illustrative starting point, NOT an assumed final model: V_i(s)=max_{c,l,a\',d}{…}"',
++falsification_or_null_result_conditions='yes — "For each: mathematical mechanism, formal or numerical prediction, falsification condition, isolation experiment and limitation." ; "Science gate: controlled experiments, uncertainty, negative results, sensitivity and mechanism isolation." ; "Supply counterexamples and boundary cases."',
++literature_or_novelty_requirement='yes — "Before developing the model, search and verify literature on … Create `docs/literature_matrix.md` with verified bibliographic details, URLs/DOIs, modeling differences, and the precise remaining gap. No invented references." ; "1. **Novelty gate:** verified literature matrix, precise contribution and competing explanations."',
++data_access_controls='yes — "Support local OpenAI-compatible inference and optional BYOK providers; no embedded credentials." ; "Request user input only for indispensable private data, external credentials, expensive compute authorization, or major scope changes." ; "Use verified public macro, labor, wealth, interest rate and financial data where possible."',
++temporal_information_controls='yes — "Prevent look-ahead and information leakage." ; "validate surrogates on held-out actions" ; "No final-test-informed" logic implied by "Pre-register outcomes."',
++baseline_requirement='yes — "Controls: rational optimized agents, rules, reinforcement learning, single LLM, multiple heterogeneous LLMs, shared LLM with varied prompts, with/without tools, memory and negotiation." ; "Benchmark model with AI switched off against known analytical cases." ; "zero-AI limiting case" (Numerics gate) ; scenario "no AI"',
++n_phases_declared='6 (six numbered "Milestones and acceptance gates"; the "Start now" paragraph adds an ordered sequence; no "Phase N" labels)',
++dependency_or_gate_structure='explicit — "## Milestones and acceptance gates 1. **Novelty gate:** … 2. **Theory gate:** … 3. **Numerics gate:** … 4. **Agent gate:** … 5. **Science gate:** … 6. **Writing gate:** … Do not proceed to extravagant multi-agent simulations before gates 1–3 are met. If a gate fails, report why and repair the design." ; "Build tiny verified economies first; scale only after tests pass."',
++stopping_rules='partial — gate-failure and escalation rules, no stop: "If a gate fails, report why and repair the design." ; "Request user input only for indispensable private data, external credentials, expensive compute authorization, or major scope changes." ; "If internet or TeX is unavailable, state the limitation and continue with honest local deliverables."',
++recovery_or_resume_instructions='yes — "Support deterministic seeds, config-driven CLI, checkpointing, experiment manifests, Git revision and reproducible environment locks." ; "Cache reproducibly."',
++artifact_requirements='repository scaffold (configs/, data/raw+processed, src/age/{economy,agents,equilibrium,markets,banking,llm,experiments,analysis}, tests/{unit,integration,property}); docs/{literature_matrix.md,model_specification.md,parameter_ledger.csv,research_log.md,limitations.md}; experiments/pre_registration.md; results/{figures,tables} (13 named figure types, 9 named table types, vector quality); paper/main.tex + sections/ + appendices/ (proofs, algorithms, prompts, schemas, replication instructions) + references.bib; README; pyproject.toml; "replication package"; "2–3 page research blueprint"',
++provenance_requirements='yes — "Record model/version, prompt, context, temperature, seed, response, latency, tokens and cost. Cache reproducibly." ; "experiment manifests, Git revision and reproducible environment locks" ; "figure/table provenance" (Writing gate) ; "`docs/parameter_ledger.csv`: parameter, units, source, year/geography, moment, method, range, uncertainty."',
++review_or_self_audit_stage='yes — "6. **Writing gate:** complete paper, figure/table provenance, verified references, replication package and hostile-referee critique." ; execution protocol step "(e) discuss threats to interpretation"',
++disclosure_or_AI_use_statement='no',
++budget_or_resource_limits='partial — costs recorded and expensive compute requires authorisation, no numeric budget: "expensive compute authorization" ; "Record … tokens and cost" ; figure "inference cost frontier" ; "Start on a single workstation; optional distributed inference only after a working local baseline."',
++integrity_prohibitions='yes — "Never fabricate results, citations, proofs, datasets, or calibrations. Label conjectures and illustrative simulations. Verify novelty before claiming it." ; "LLM output is never trusted as accounting truth." ; "Never claim a test passed unless it ran." ; "Do not call arbitrary LLM-generated market outcomes general equilibrium without verifying feasibility, market clearing and convergence." ; "Never treat simulated LLM preferences as verified human preferences."',
++contradictory_or_non_operational_requirements='"Pre-register outcomes." and `experiments/pre_registration.md` without any timestamping/registry mechanism — pre-registration by the same agent that runs the experiments is not verifiable. "hostile-referee critique" is produced by the same model. Scope tension between "Aim for a parsimonious groundbreaking mechanism, not complexity for its own sake" and three nested models (A, B, C) plus an LLM laboratory with eight roles and banking. Negative design constraint stated as judgement: "The key innovation MUST NOT be simply adding AI to a Cobb–Douglas production function." Non-verifiable: "Aim at top economics/finance/computational economics journals without assuming acceptance."',
++statements_added_during_implementation='Prefatory note: "(UQO Working Paper No. 25 — repository `Recherche-UQO/finance/agentic-general-equilibrium`, spbgit `wp25_uqo.git`. Programme supplied by the author on 2026-10-27; form follows `Recherche-UQO/HARMONISATION.md`.)" — repository mapping added around the author\'s programme (dated 2026-10-27, same day as the programme).',
++analyst_notes='Gate structure is the most formal of the theory mandates (six named acceptance gates with contents). Explicit per-stage protocol "(a) state the precise scientific question, (b) write down assumptions, (c) implement and test, (d) show numerical evidence or mathematical proof, (e) discuss threats to interpretation, (f) log decisions and next tasks."')
++
++R("WP26",document_type="claude_md",
++research_domain="real estate valuation / multi-agent AI (LLM agent deliberation for property valuation, Quebec transactions)",
++research_type="mixed (empirical evaluation + design-science multi-agent system + stylised welfare model)",
++explicit_question='"RQ1: Do specialized multi-agent systems outperform single LLMs, hedonic models, repeat-sales approaches, gradient boosting, and conventional AVMs on genuinely unseen Quebec transactions?" (RQ1 of six)',
++explicit_estimand_or_hypotheses='yes — pre-specified primary outcome: "Pre-register primary endpoints and hypotheses before running final test evaluations. Primary outcome: out-of-time absolute log-price error on arm\'s-length residential transactions. Co-primary or secondary: MAE in CAD, MdAPE, RMSE log, tail error (P90/P95), interval coverage, interval width, CRPS or pinball loss, cost and latency." ; "Propose falsifiable propositions about diversity, correlation, communication topology, and aggregation weights."',
++stated_assumptions='partial — "State identification assumptions and develop multiple operational definitions of value." ; "Transaction prices are noisy proxies, not ground-truth fundamental values." ; "Do not claim general existence/uniqueness theorems without proof."',
++falsification_or_null_result_conditions='yes — "A negative finding is publishable if the design is rigorous." ; "Report failure as rigorously as success." ; placebo arm "I. Random/shuffled-role or placebo deliberation" ; "Explicitly separate improvements from more tokens, more tools, more models, or more data." ; "Derive testable conditions under which deliberation reduces or increases mean-squared error"',
++literature_or_novelty_requirement='yes — "Phase 1: literature review and novelty map; identify close prior multi-agent real-estate/AVM papers and avoid unsupported priority claims." ; "Search and verify the relevant academic literature if browsing is available; otherwise flag bibliography for verification."',
++data_access_controls='yes — "These are potential sources, not automatically accessible. Begin with a data-access audit" ; "Do not bypass authentication, scrape restricted endpoints, or infer consent. Do not send personal identifiers, private listing remarks, occupant information, or confidential records to external LLM APIs without authorization and suitable safeguards." ; "use environment variables and a local secrets manager, never put secrets in source control or manuscript" ; "Publish only de-identified or synthetic examples if redistribution is not authorized."',
++temporal_information_controls='yes — "Only use features observable as of the valuation timestamp; prevent post-sale information and target leakage." ; "Comparable retrieval must be strictly as-of-date; no same-property future transfers, post-sale listings, or near-duplicate records leaking across partitions." ; "Use a chronological split with train/validation/test and a final untouched holdout" ; "Build a leakage-audit suite and adversarial tests." ; "No final-test-informed aggregation weights."',
++baseline_requirement='yes — "Implement and evaluate at least: A. Hedonic OLS / regularized regression (proper spatial/time controls) B. Repeat-sales or hierarchical hedonic … C. CatBoost/XGBoost/LightGBM tuned on validation only D. Strong tabular AVM … E. Single LLM, evidence-matched F. Independent LLM agents aggregated without communication … J. Cost-matched single-agent repeated reasoning and cost-matched ensemble"',
++n_phases_declared='9 (Phase 0–8 in "Execution phases and acceptance gates") plus seven "Immediate first actions"',
++dependency_or_gate_structure='explicit — "## Execution phases and acceptance gates Phase 0: inventory … Phase 5: run provider-balanced, information-matched, compute-matched randomized ablations on validation; freeze protocol. Phase 6: evaluate final temporal/spatial holdouts once, with clustered uncertainty and cost accounting. … Phase 8: produce manuscript" ; "After each phase, produce a concise `reports/phase_XX.md` with completed artifacts, failed checks, decisions, outstanding assumptions, and exact commands." (ordering and freeze/once-only conditions; no per-phase pass criteria beyond the phase reports)',
++stopping_rules='partial — escalation rule and debate-round limits only: "Stop and ask targeted questions only for indispensable access or decisions; otherwise proceed with explicitly documented defaults. Never fabricate completion." ; "Limit debate to pre-specified rounds (e.g. 0, 1, 2, 4)" ; src/debate/ "stopping rules"',
++recovery_or_resume_instructions='partial — "privacy-aware caching" ; "bounded retries" ; "store response artifacts securely and measure run-to-run variation" ; no resume/checkpoint instruction',
++artifact_requirements='README.md, CLAUDE.md, pyproject.toml, .env.example, LICENSE; configs/ (splits, model IDs, treatments, budgets, seeds); src/{data,features,baselines,agents,debate,evaluation,experiments}; tests/ (unit/integration/leakage/mock-provider); notebooks/; paper/ (LaTeX manuscript, BibTeX, figures, tables, appendices, compilation script, PDF); reports/ (experiment cards, failure analyses, model cards, limitations, phase_XX.md); research_plan.md, data_access_checklist.md, preregistration.md, architecture.md; data dictionaries; clearly labeled synthetic dataset and mock LLM adapters; CLI (ingest, validate, train, run agents, evaluate, generate tables, build PDF); limitations register; API-cost budget',
++provenance_requirements='yes — "Store raw snapshots immutable with data dictionaries, versioned transformations, provenance, licenses, and record-level lineage." ; "Record model IDs, API parameters, prompt hashes, temperature, tool calls, token usage, cost estimates, and latency." ; "All figures/tables must come from versioned outputs." ; "Pin dependencies, set seeds, capture versions and hardware"',
++review_or_self_audit_stage='yes — roles "11. Data-quality, leakage, and provenance auditor 12. Independent skeptical reviewer / red-team agent" ; "Phase 7: human benchmark if authorized, welfare model, robustness, interpretability audits." ; "Build a leakage-audit suite and adversarial tests."',
++disclosure_or_AI_use_statement='no — "Agents are **simulated analytic roles**, not licensed Quebec appraisers or real lenders; never misrepresent professional certification." concerns role representation, not AI-use disclosure for the paper',
++budget_or_resource_limits='yes — "Present an API-cost budget with small pilot, medium validation, and final experiment tiers, without inventing vendor prices." ; "token budgets" ; "compare all-Claude, all-OpenAI, and mixed-provider committees at matched cost and information budgets" ; "async concurrency with strict limits"',
++integrity_prohibitions='yes — "Never invent data, results, citations, human assessments, API capabilities, or model performance." ; "Never fabricate empirical tables: use placeholders clearly marked `NOT YET ESTIMATED` until generated by scripts." ; "No p-hacking or test-set prompt tuning." ; "do not fabricate a human benchmark" ; "Do not claim deterministic reproducibility for hosted LLMs"',
++contradictory_or_non_operational_requirements='"Pre-register primary endpoints and hypotheses before running final test evaluations" and `preregistration.md` have no registry/timestamp mechanism; the same agent writes and executes. "evaluate final temporal/spatial holdouts once" — the once-only rule has no enforcement mechanism. Human-dependent arm: "recruit licensed appraisal professionals and/or trained human appraisers under an approved protocol" cannot be executed autonomously (mitigated: "mark this arm as pending and do not claim it was performed"). "Respect Quebec privacy requirements and organizational research-ethics review where applicable" is not operational for an agent. Tension between "make outputs reproducible" and "Do not claim deterministic reproducibility for hosted LLMs" (resolved by storing response artifacts). Scope (12 default roles, up to 20, ten treatment arms, three provider configurations) versus an API budget that is to be proposed rather than given.',
++statements_added_during_implementation='Explicit post-hoc section: "## Repository conventions (added by the implementation, 2026-10-27) - UQO Working Paper No. 26, folder `Recherche-UQO/immobilier/agentic-valuation-quebec`, git remote `gitsrv:srv/git/wp26_uqo.git`. - Python package: `src/agentic_valuation/` (installed editable: `pip install -e .`). CLI: `av <command>` (see README). - Secrets only in `.env` (gitignored). Snapshot provenance in `data/raw/README.md`. Every result file is produced by a numbered script in `scripts/`."',
++analyst_notes='Most complete empirical-design mandate in the corpus: pre-specified primary outcome, placebo arm, cost-matched controls, leakage suite, API budget tiers, phase reports. Research-only clause: "No live autonomous trading, lending decisions, or real estate offers".')
++
++R("WP28",document_type="claude_md",
++research_domain="econometric theory / statistical computing (distributed maximum likelihood, higher-order bias correction, communication-computation trade-off)",
++research_type="mixed (mathematical theory + Monte Carlo simulation + distributed-systems benchmark)",
++explicit_question='"For regular nonlinear likelihood models with `N` observations and `K` distributed partitions, how can additional local computation replace global network synchronization while preserving the first-order inferential properties of centralized maximum likelihood?"',
++explicit_estimand_or_hypotheses='yes — candidate theorem and proof queue: "K = o( N^[ 1 - 1 / ( 2^(q+1) * r ) ] )" ; "**Theorem 3 — Exact exponential boundary.** Derive all Gamma moments needed and establish the nonzero limit `-4 theta_0 c^4` for half-block jackknife + one global Newton step when `K ~ c N^(7/8)`." ; Lemmas 1–7, Theorems 1–4, a failure Proposition and a Corollary are enumerated',
++stated_assumptions='yes — baseline model assumptions are listed in the mandate and an assumptions file is required: "### 5.1 Baseline model Start with i.i.d. observations and a correctly specified fixed-dimensional regular likelihood: - `X_1,...,X_N ~ P_{theta_0}` independent; - `theta_0` interior to an open parameter set in `R^p`; - information matrix `I(theta_0)` positive definite; …" ; "State precisely which theorem uses which assumption." ; theory/ASSUMPTIONS.md',
++falsification_or_null_result_conditions='yes — "A dead end with a mathematical explanation is better than a fabricated breakthrough. A strong negative result or counterexample may itself become the contribution." ; "**Gate 4:** at least one nontrivial theorem beyond a rephrasing of known one-step-plus-jackknife results is fully proved, **or** the report explicitly concludes that novelty is not yet proved and reframes the paper around validated insights and counterexamples." ; "a counterexample or violation for at least one essential assumption" ; "**Proposition — Failure/counterexample.**" ; "Track E — Lower bounds, counterexamples and impossibility"',
++literature_or_novelty_requirement='yes — "Do not claim that distributed gradients, one-step Newton, split-sample jackknife, Richardson extrapolation, `K = o(N^{7/8})`, or `K = o(N^{11/12})` are intrinsically novel. Investigate who has already obtained each idea or bound." ; `literature/PRIOR_ART_MATRIX.csv` with "novelty_threat" column ; "**Gate 1:** no unqualified novelty language is allowed until the closest competing theorems have been inspected and cited." ; six anchor references with DOIs to verify',
++data_access_controls='yes — "never scrape or transmit private material to external services without authorization" ; "Do not expose API keys, SSH keys, passwords, endpoint secrets or private datasets in source files, logs, paper, artifacts or commits." ; "Use the researcher\'s available CPU/GPU/cluster systems only when provisioned and authorized." ; "Avoid destructive commands, forced pushes and unrelated repository modifications."',
++temporal_information_controls='no — not applicable (synthetic Monte Carlo); only a literature cutoff: "**Current date for literature cutoff:** 2026-10-28; search current material rather than assuming bibliographic currency."',
++baseline_requirement='yes — "Full centralized maximum likelihood, using a trusted optimizer and tight tolerances." ; "Credible implementations of at least two strong literature baselines where practical, such as surrogate likelihood/CSL and another relevant distributed estimator." ; audit items "New method comparisons include ordinary local averaging + two Newton rounds." and "A baseline with exact sufficient-statistic aggregation is shown for applicable families."',
++n_phases_declared='13 (Phase 0–12, each closed by Gate 0–12)',
++dependency_or_gate_structure='explicit — "**Execute in order, completing one phase\'s acceptance tests before starting the next.**" ; "**Gate 0: pass only when** all old claims and artifacts are inventoried, every existing number is tagged as replicated / not replicated / unavailable, and a WP28 workspace exists without clobbering other work." ; "**Gate 6:** all critical tests pass and the code reconstructs original-paper figures within stated tolerances before large experiments begin." ; pass/fail rubric in section 16',
++stopping_rules='yes — "Complete the project from initial inspection through PDF validation; stop only when a real blocker makes an indispensable step impossible. Document blocked items, and continue with whatever can be completed." ; "**Gate 11:** produce `FINAL_REVIEW.md` with a final verdict `ready for human review / theoretically incomplete / novelty unresolved / computational evidence insufficient`" ; "Never assert results from jobs that have not finished." ; "Do not publish a preprint or submit to a journal without explicit instruction."',
++recovery_or_resume_instructions='partial — phase status and run manifests, failure tracking, no resume instruction: "After completing each phase, append to `PHASE_STATUS.md`: Phase: Start/finish timestamp: What was actually done: Artifacts created: Tests passed: Tests failed: …" ; "Track optimization failures rather than discard failed replications silently." ; "straggler failures" to be bounded',
++artifact_requirements='article (~25–40 pages) + mathematical appendix; paper/{main.tex,preamble.tex,sections,appendices,references.bib,figures}; literature/{PRIOR_ART_MATRIX.csv,PRIORITY_AND_GAP.md}; theory/{THEOREM_AUDIT.md,ASSUMPTIONS.md,proofs_check.py,symbolic_checks.py}; src/pamle/ (10 modules) with common-interface estimators; experiments/{configs,run_monte_carlo.py,run_benchmarks.py,run_ablations.py,EXPERIMENT_MANIFEST.csv}; tests/ (7 files); results/{raw,processed,tables,figures,resource_profiles}; scripts/{setup.sh,reproduce_all.sh,compile_paper.sh,validate_pdf.sh}; PROJECT_MAP.md, SOURCE_PROVENANCE.md, PHASE_STATUS.md, research_log.md, LIMITATIONS_AND_CLAIMS.md, FINAL_REVIEW.md, FINAL_DELIVERY.md; proof ledger; 8 figures + 6 tables programme; delivery bundle (PDF, .tex, source-and-reproducibility zip, Research_Summary, Theorem_Audit, Prior_Art_and_Novelty, Experiment_Results.csv, Final_Review); nine-point closing report',
++provenance_requirements='yes — "Extract the existing archive to an immutable `references/initial_pamle/` directory if available; record checksums." ; run manifest "run_id,git_commit,python_version,dependency_lock_sha256, seed,N,K,m,p,DGP,method,optimizer,tolerance,replicates, parallel_backend,node_count,hardware,OS,wall_seconds, communicated_bytes,rounds,status,raw_output_sha256" ; "Every table value has a traceable input/output path." ; "Project checksums and commit state are captured."',
++review_or_self_audit_stage='yes — "## 14. PHASE 11 — Adversarial referee and proof audit Simulate at least three independent review perspectives; log objections and responses" (Referees A, B, C) ; "For every objection, respond with (a) exact location in paper/code, (b) correction made, (c) new proof/test if needed, and (d) residual limitation." ; THEOREM_AUDIT.md with "proved / conditional / open / false" labels',
++disclosure_or_AI_use_statement='yes — "Confirm the applicable journal/venue authorship and AI-disclosure policy before a submission version; never falsely represent an AI system as a legally accountable human author." ; "The journal submission version may need GPT-6.1 removed from the formal author list and placed in transparent AI-use disclosure; **flag this for the human author rather than concealing the difference**." ; audit item "AI contribution disclosure and human review requirements are explicit." ; rubric row "Attribution | Simon-Pierre Boucher properly named; AI role transparently recorded"',
++budget_or_resource_limits='partial — resources bounded by availability, no numeric cap: "Continue to a publishable-quality, internally consistent draft within the resources and access actually available." ; "`N`: logarithmic sequence … chosen based on available RAM/compute" ; blocker category "(iv) exhausted compute budget" ; "where measured, electricity and dollar-cost assumptions"',
++integrity_prohibitions='yes — "No invented theorem, proof, assumption, dataset, benchmark, reference, DOI, parameter estimate, speedup, significance level, or journal acceptance." ; "Do not call an in-memory mathematical partition a distributed cluster benchmark." ; "Never assert results from jobs that have not finished." ; "Do not silently describe these as 1,000 replications or as physical-cluster experiments." ; "Never explicitly invert the Hessian in production code"',
++contradictory_or_non_operational_requirements='(1) "Simulate at least three independent review perspectives" versus "do not have the same model merely declare its own work accepted" — the simulated referees are by construction the same model, so independence is not operational. (2) Authorship: "Preserve this provenance and the user\'s requested GPT-6.1 attribution in the working draft" (GPT-6.1 in the author block) versus "never falsely represent an AI system as a legally accountable human author" — resolved only by deferring to the human for the submission version. (3) "Research program funding line to include **only if confirmed** for this particular WP28 project" — confirmation requires the human; the agent cannot satisfy it. (4) Compute: "target at least 1,000 per **main statistical design**" across ≥4 N values, several K, several p, ≥3 models and DGP conditions with no budget beyond "where practical". (5) Conditional bonuses ("If Lean is installed, formalize at least one deterministic algebraic lemma as a bonus, not as a blocker"). (6) Rubric criteria such as "substantive new result or defensible negative discovery" and "journal-style typography" are not script-checkable. (7) "The researcher may have access to a MacLustr collection" together with "never assume the existence … of a MacLustr node" is consistent but leaves hardware scope undetermined until run time.',
++statements_added_during_implementation='none identified as post-hoc; the header records prior provenance rather than an addition: "**Prior exploratory draft credits:** Simon-Pierre Boucher and GPT-6.1. Preserve this provenance and the user\'s requested GPT-6.1 attribution in the working draft."',
++analyst_notes='Longest mandate in the corpus (8 809 words, 76 headings, 27 code blocks, 2 Markdown tables). The only mandate with an AI-use/authorship disclosure instruction, a per-phase acceptance-test rule ("completing one phase\'s acceptance tests before starting the next"), a pass/fail rubric table, a multi-referee audit phase and a verdict vocabulary. Explicitly a re-examination of a prior AI-assisted draft ("This is NOT an instruction to paraphrase the exploratory note.").')
++
++R("WP27_BRIEF",document_type="design_brief",
++research_domain="finance / market microstructure (agent-based limit-order-book simulation with LLM trading agents)",
++research_type="simulation (agent-based market simulator with a stylised one-period theoretical model)",
++explicit_question='none — the brief describes "what the simulator *actually* implements" and lists hypotheses H1–H6; no research question is stated',
++explicit_estimand_or_hypotheses='yes — "Hypotheses the simulator tests: H1 informativeness falls in the share using a single shared model (holding information quality fixed) and rises in M_eff; H2 order-flow correlation and synchrony rise in φ; H3 bubble/crash frequency and liquidity spirals rise with φ × leverage; H4 prompt/model diversity restores part of the loss; H5 communication channels raise synchrony; H6 circuit breakers and position limits cut crash depth at a cost in time-to-absorb." ; objects defined: "price informativeness τ = 1 − Var(v | Y)/σ_v²; pricing error Var(P − v); order-flow correlation ρ_x = Corr(x_i, x_j); synchrony; effective number of models M_eff = 1/Σ_g φ_g²"',
++stated_assumptions='yes — model assumptions stated in the text: "One-period, one asset, liquidation value v ~ N(0, σ_v²) around the public PV. N traders receive s_i = v + ε_i with ε_i = η_{g(i)} + u_i" ; "β > 0 a fixed response intensity; the behavioral assumption mirrors rule-like LLM policies and is relaxed in a CARA extension" ; "A competitive, risk-neutral market maker sets P = E[v | Y]"',
++falsification_or_null_result_conditions='partial — directional hypotheses are testable and a seed/power design is sketched ("≥ 30 seeds exploratory, power-based for confirmatory"), but no null-result condition is stated',
++literature_or_novelty_requirement='no — the brief is addressed to "the sub-agents writing the literature review, theory and pre-analysis plan" but imposes no literature or novelty requirement',
++data_access_controls='no — synthetic data; providers listed without access rules: "Providers: OpenAI (Responses API, json_schema), Anthropic (Messages API, json_schema), mock (seeded rule with injected invalid answers)."',
++temporal_information_controls='yes — information-set discipline inside the simulator: "Agents never see V_t." ; "No agent sees future draws." ; "orders arrive at decision time + latency and are processed in arrival order"',
++baseline_requirement='partial — control arm and matched design rather than named baseline methods: "Matched pairs across `llm_share` share every stream except the LLM slot draw." ; "mock (seeded rule …)" provider ; "frozen decision library replay vs online calls"',
++n_phases_declared='none',
++dependency_or_gate_structure='none',
++stopping_rules='no',
++recovery_or_resume_instructions='no',
++artifact_requirements='none stated explicitly; implied: literature review, theory write-up ("to be written up by the theory agent, must map to the above") and a pre-analysis plan; the simulator reports "Realized LLM share of agents, orders and volume"',
++provenance_requirements='partial — "Randomization: master seed → independent streams (fundamentals, news, slots, policy assignment, LLM slot assignment, activation, tie-breaks, per-agent RNG)." ; "Invalid decisions = hold, logged." ; "exact conservation checked after every fill" ; no manifest/hash requirement',
++review_or_self_audit_stage='no',
++disclosure_or_AI_use_statement='no',
++budget_or_resource_limits='partial — budgets appear as experimental treatments, not as resource limits: "equal token / USD / decision-opportunity budgets"',
++integrity_prohibitions='no',
++contradictory_or_non_operational_requirements='Not a mandate, so few directives. Non-operational: "power-based for confirmatory" without a power target or effect size; "must map to the above" asserts a correspondence between the one-period theory and the multi-tick simulator (fixed β market orders versus a continuous double auction) that the brief does not check.',
++statements_added_during_implementation='none identified — but the whole document is by nature post-implementation ("It describes what the simulator *actually* implements (src/market_agents)").',
++analyst_notes='Supplementary design brief, not a CLAUDE.md; coded separately and excluded from all "of 11" counts. WP27\'s own CLAUDE.md returned HTTP 404 and is not represented in the corpus. 820 words, 3 headings.')
++
++R("WP15_PROMPT",document_type="duplicate",
++research_domain="see WP15", research_type="see WP15",
++explicit_question="see WP15 (byte-identical duplicate; not coded separately)",
++**{k:"n/a — duplicate of WP15_CLAUDE.md (same SHA-256 2dd19c59fd320ee5fcdbb25f9ebd41b08b612703573cd8af119858e21a521b77)" for k in F[9:28]},
++analyst_notes='docs/PROMPT_INITIAL_CLAUDE.md in the wp15_uqo repository is byte-identical to docs/CLAUDE.md in the retrieved snapshot. Recorded once in the corpus manifest for completeness; counted as one mandate (WP15) in all tallies. The two paths suggest the initial prompt was preserved verbatim as the working CLAUDE.md, but the snapshot alone cannot establish which file came first.')
++
++order=["WP15","WP18","WP19","WP20","WP21","WP22","WP23","WP24","WP25","WP26","WP28","WP27_BRIEF","WP15_PROMPT"]
++rows.sort(key=lambda r:order.index(r["wp"]))
++out=os.path.join(ROOT,"corpus_coding.csv")
++with open(out,"w",newline="",encoding="utf-8") as f:
++    w=csv.DictWriter(f,fieldnames=F,quoting=csv.QUOTE_ALL,lineterminator="\r\n")
++    w.writeheader()
++    for r in rows:
++        missing=[k for k in F if k not in r]
++        assert not missing,(r["wp"],missing)
++        w.writerow(r)
++# validate
++back=list(csv.DictReader(open(out,newline="",encoding="utf-8")))
++assert len(back)==13 and all(len(r)==len(F) for r in back)
++print("ok",len(back),"rows",len(F),"columns ->",out)

added research/references_verified.bib +915 −0

++@misc{lu2024aiscientist,
++  author = {Chris Lu and Cong Lu and Robert Tjarko Lange and Jakob Foerster and Jeff Clune and David Ha},
++  title = {{The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery}},
++  year = {2024},
++  howpublished = {arXiv preprint arXiv:2408.06292},
++  eprint = {2408.06292},
++  archivePrefix = {arXiv}
++}
++% verified via https://export.arxiv.org/api/query?id_list=2408.06292 on 2026-10-30  (comment placed after the first entry so that no % line precedes it)
++
++% verified via https://api.crossref.org/works/10.1038/s41586-026-10265-5 on 2026-10-30
++@article{lu2026aiscientist,
++  author = {Lu, Chris and Lu, Cong and Lange, Robert Tjarko and Yamada, Yutaro and Hu, Shengran and Foerster, Jakob and Ha, David and Clune, Jeff},
++  title = {{Towards end-to-end automation of AI research}},
++  year = {2026},
++  journal = {Nature},
++  volume = {651},
++  number = {8107},
++  pages = {914--919},
++  doi = {10.1038/s41586-026-10265-5}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2504.08066 on 2026-10-30
++@misc{yamada2025aiscientistv2,
++  author = {Yutaro Yamada and Robert Tjarko Lange and Cong Lu and Shengran Hu and Chris Lu and Jakob Foerster and Jeff Clune and David Ha},
++  title = {{The AI Scientist-v2: Workshop-Level Automated Scientific Discovery via Agentic Tree Search}},
++  year = {2025},
++  howpublished = {arXiv preprint arXiv:2504.08066},
++  eprint = {2504.08066},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.crossref.org/works/10.18653/v1/2025.findings-emnlp.320 on 2026-10-30
++@inproceedings{schmidgall2025agentlab,
++  author = {Schmidgall, Samuel and Su, Yusheng and Wang, Ze and Sun, Ximeng and Wu, Jialian and Yu, Xiaodong and Liu, Jiang and Moor, Michael and Liu, Zicheng and Barsoum, Emad},
++  title = {{Agent Laboratory: Using LLM Agents as Research Assistants}},
++  year = {2025},
++  booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2025},
++  pages = {5977--6043},
++  doi = {10.18653/v1/2025.findings-emnlp.320},
++  eprint = {2501.04227},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.crossref.org/works/10.1038/s41586-023-06792-0 on 2026-10-30
++@article{boiko2023coscientist,
++  author = {Boiko, Daniil A. and MacKnight, Robert and Kline, Ben and Gomes, Gabe},
++  title = {{Autonomous chemical research with large language models}},
++  year = {2023},
++  journal = {Nature},
++  volume = {624},
++  number = {7992},
++  pages = {570--578},
++  doi = {10.1038/s41586-023-06792-0}
++}
++
++% verified via https://api.crossref.org/works/10.1038/s42256-024-00832-8 on 2026-10-30
++@article{bran2024chemcrow,
++  author = {M. Bran, Andres and Cox, Sam and Schilter, Oliver and Baldassari, Carlo and White, Andrew D. and Schwaller, Philippe},
++  title = {{Augmenting large language models with chemistry tools}},
++  year = {2024},
++  journal = {Nature Machine Intelligence},
++  volume = {6},
++  number = {5},
++  pages = {525--535},
++  doi = {10.1038/s42256-024-00832-8}
++}
++
++% verified via https://api.crossref.org/works/10.1056/AIoa2400555 on 2026-10-30
++@article{ifargan2025datatopaper,
++  author = {Ifargan, Tal and Hafner, Lukas and Kern, Maor and Alcalay, Ori and Kishony, Roy},
++  title = {{Autonomous LLM-Driven Research — from Data to Human-Verifiable Research Papers}},
++  year = {2025},
++  journal = {NEJM AI},
++  volume = {2},
++  number = {1},
++  doi = {10.1056/AIoa2400555},
++  eprint = {2404.17605},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.crossref.org/works/10.1038/s41586-026-10644-y on 2026-10-30
++@article{gottweis2026coscientist,
++  author = {Gottweis, Juraj and Weng, Wei-Hung and Daryin, Alexander and Tu, Tao and Sirkovic, Petar and Myaskovsky, Artiom and Glowaty, Grzegorz and Weissenberger, Felix and Orlandi, Alessio and Popovici, Dan and Palepu, Anil and Rong, Keran and Tanno, Ryutaro and Saab, Khaled and Zhang, Fan and Blum, Jacob and Carroll, Andrew and Kulkarni, Kavita and Tomašev, Nenad and Zverinski, Dina and Rendulic, Ivor and Vedadi, Elahe and Hasler, Florian and Rimanic, Luka and Boia, Marina and Budiselic, Ivan and Feinstein, Ben and Bellaiche, Mathias and Sheffer, Tom and Freyberg, Jan and Ratcliff, Jeremy and Bertolli, Ottavia and Chou, Katherine and Hassidim, Avinatan and Gokturk, Burak and Vahdat, Amin and Guan, Yuan and Dhillon, Vikram and Vaishnav, Eeshit Dhaval and Lee, Byron and Costa, Tiago R. D. and Penadés, José R. and Peltz, Gary and Matias, Yossi and Manyika, James and Hassabis, Demis and Xu, Yunhan and Kohli, Pushmeet and Pawlosky, Annalisa and Karthikesalingam, Alan and Natarajan, Vivek},
++  title = {{Accelerating scientific discovery with Co-Scientist}},
++  year = {2026},
++  journal = {Nature},
++  volume = {655},
++  number = {8122},
++  pages = {487--496},
++  doi = {10.1038/s41586-026-10644-y},
++  eprint = {2502.18864},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.crossref.org/works/10.1038/s41586-025-09442-9 on 2026-10-30
++@article{swanson2025virtuallab,
++  author = {Swanson, Kyle and Wu, Wesley and Bulaong, Nash L. and Pak, John E. and Zou, James},
++  title = {{The Virtual Lab of AI agents designs new SARS-CoV-2 nanobodies}},
++  year = {2025},
++  journal = {Nature},
++  volume = {646},
++  number = {8085},
++  pages = {716--723},
++  doi = {10.1038/s41586-025-09442-9}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2511.02824 on 2026-10-30
++@misc{mitchener2025kosmos,
++  author = {Ludovico Mitchener and Angela Yiu and Benjamin Chang and Mathieu Bourdenx and Tyler Nadolski and Arvis Sulovari and Eric C. Landsness and Daniel L. Barabasi and Siddharth Narayanan and Nicky Evans and Shriya Reddy and Martha Foiani and Aizad Kamal and Leah P. Shriver and Fang Cao and Asmamaw T. Wassie and Jon M. Laurent and Edwin Melville-Green and Mayk Caldas and Albert Bou and Kaleigh F. Roberts and Sladjana Zagorac and Timothy C. Orr and Miranda E. Orr and Kevin J. Zwezdaryk and Ali E. Ghareeb and Laurie McCoy and Bruna Gomes and Euan A. Ashley and Karen E. Duff and Tonio Buonassisi and Tom Rainforth and Randall J. Bateman and Michael Skarlinski and Samuel G. Rodriques and Michaela M. Hinks and Andrew D. White},
++  title = {{Kosmos: An AI Scientist for Autonomous Discovery}},
++  year = {2025},
++  howpublished = {arXiv preprint arXiv:2511.02824},
++  eprint = {2511.02824},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2509.08713 on 2026-10-30
++@misc{luo2025pitfalls,
++  author = {Ziming Luo and Atoosa Kasirzadeh and Nihar B. Shah},
++  title = {{The More You Automate, the Less You See: Hidden Pitfalls of AI Scientist Systems}},
++  year = {2025},
++  howpublished = {arXiv preprint arXiv:2509.08713},
++  eprint = {2509.08713},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.crossref.org/works/10.1145/3769733.3769747 on 2026-10-30
++@article{beel2025sakana,
++  author = {Beel, Joeran and Kan, Min-Yen and Baumgart, Moritz},
++  title = {{Evaluating Sakana's AI Scientist: Bold Claims, Mixed Results, and a Promising Future?}},
++  year = {2025},
++  journal = {ACM SIGIR Forum},
++  volume = {59},
++  number = {1},
++  pages = {1--20},
++  doi = {10.1145/3769733.3769747},
++  eprint = {2502.14297},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2607.12301 on 2026-10-30
++@misc{luo2026xscientist,
++  author = {Jixiang Luo},
++  title = {{XScientist: A Git-Like Research Protocol for Long-Running Autonomous Scientific Discovery}},
++  year = {2026},
++  howpublished = {arXiv preprint arXiv:2607.12301},
++  eprint = {2607.12301},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2608.05179 on 2026-10-30
++@misc{ding2026verificationgap,
++  author = {Tianyu Ding and Aditya Nannapaneni and Bingfan Liu and Ling Zhang},
++  title = {{Autonomous Research Agents: A Survey of AI Scientists and the Verification Gap}},
++  year = {2026},
++  howpublished = {arXiv preprint arXiv:2608.05179},
++  eprint = {2608.05179},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2410.05080 on 2026-10-30
++@inproceedings{chen2025scienceagentbench,
++  author = {Ziru Chen and Shijie Chen and Yuting Ning and Qianheng Zhang and Boshi Wang and Botao Yu and Yifei Li and Zeyi Liao and Chen Wei and Zitong Lu and Vishal Dey and Mingyi Xue and Frazier N. Baker and Benjamin Burns and Daniel Adu-Ampratwum and Xuhui Huang and Xia Ning and Song Gao and Yu Su and Huan Sun},
++  title = {{ScienceAgentBench: Toward Rigorous Assessment of Language Agents for Data-Driven Scientific Discovery}},
++  year = {2024},
++  booktitle = {ICLR 2025},
++  eprint = {2410.05080},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2407.01725 on 2026-10-30
++@inproceedings{majumder2025discoverybench,
++  author = {Bodhisattwa Prasad Majumder and Harshit Surana and Dhruv Agarwal and Bhavana Dalvi Mishra and Abhijeetsingh Meena and Aryan Prakhar and Tirth Vora and Tushar Khot and Ashish Sabharwal and Peter Clark},
++  title = {{DiscoveryBench: Towards Data-Driven Discovery with Large Language Models}},
++  year = {2024},
++  booktitle = {ICLR 2025},
++  eprint = {2407.01725},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2409.07703 on 2026-10-30
++@inproceedings{jing2025dsbench,
++  author = {Liqiang Jing and Zhehui Huang and Xiaoyang Wang and Wenlin Yao and Wenhao Yu and Kaixin Ma and Hongming Zhang and Xinya Du and Dong Yu},
++  title = {{DSBench: How Far Are Data Science Agents from Becoming Data Science Experts?}},
++  year = {2024},
++  booktitle = {ICLR 2025},
++  eprint = {2409.07703},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2310.03302 on 2026-10-30
++@inproceedings{huang2024mlagentbench,
++  author = {Qian Huang and Jian Vora and Percy Liang and Jure Leskovec},
++  title = {{MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation}},
++  year = {2023},
++  booktitle = {ICML 2024},
++  eprint = {2310.03302},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2408.09667 on 2026-10-30
++@inproceedings{gu2024blade,
++  author = {Ken Gu and Ruoxi Shang and Ruien Jiang and Keying Kuang and Richard-John Lin and Donghe Lyu and Yue Mao and Youran Pan and Teng Wu and Jiaqian Yu and Yikun Zhang and Tianmai M. Zhang and Lanyi Zhu and Mike A. Merrill and Jeffrey Heer and Tim Althoff},
++  title = {{BLADE: Benchmarking Language Model Agents for Data-Driven Science}},
++  year = {2024},
++  booktitle = {EMNLP 2024},
++  eprint = {2408.09667},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2310.06770 on 2026-10-30
++@inproceedings{jimenez2024swebench,
++  author = {Carlos E. Jimenez and John Yang and Alexander Wettig and Shunyu Yao and Kexin Pei and Ofir Press and Karthik Narasimhan},
++  title = {{SWE-bench: Can Language Models Resolve Real-World GitHub Issues?}},
++  year = {2023},
++  booktitle = {ICLR 2024},
++  eprint = {2310.06770},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2410.07095 on 2026-10-30
++@inproceedings{chan2025mlebench,
++  author = {Jun Shern Chan and Neil Chowdhury and Oliver Jaffe and James Aung and Dane Sherburn and Evan Mays and Giulio Starace and Kevin Liu and Leon Maksin and Tejal Patwardhan and Lilian Weng and Aleksander Mądry},
++  title = {{MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering}},
++  year = {2024},
++  booktitle = {ICLR 2025},
++  eprint = {2410.07095},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2409.11363 on 2026-10-30
++@article{siegel2024corebench,
++  author = {Zachary S. Siegel and Sayash Kapoor and Nitya Nadgir and Benedikt Stroebl and Arvind Narayanan},
++  title = {{CORE-Bench: Fostering the Credibility of Published Research Through a Computational Reproducibility Agent Benchmark}},
++  year = {2024},
++  journal = {Transactions on Machine Learning Research},
++  eprint = {2409.11363},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2504.01848 on 2026-10-30
++@inproceedings{starace2025paperbench,
++  author = {Giulio Starace and Oliver Jaffe and Dane Sherburn and James Aung and Jun Shern Chan and Leon Maksin and Rachel Dias and Evan Mays and Benjamin Kinsella and Wyatt Thompson and Johannes Heidecke and Amelia Glaese and Tejal Patwardhan},
++  title = {{PaperBench: Evaluating AI's Ability to Replicate AI Research}},
++  year = {2025},
++  booktitle = {ICML 2025},
++  eprint = {2504.01848},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.crossref.org/works/10.1093/bioinformatics/bts480 on 2026-10-30
++@article{koster2012snakemake,
++  author = {Köster, Johannes and Rahmann, Sven},
++  title = {{Snakemake—a scalable bioinformatics workflow engine}},
++  year = {2012},
++  journal = {Bioinformatics},
++  volume = {28},
++  number = {19},
++  pages = {2520--2522},
++  doi = {10.1093/bioinformatics/bts480}
++}
++
++% verified via https://api.crossref.org/works/10.1038/nbt.3820 on 2026-10-30
++@article{ditommaso2017nextflow,
++  author = {Di Tommaso, Paolo and Chatzou, Maria and Floden, Evan W and Barja, Pablo Prieto and Palumbo, Emilio and Notredame, Cedric},
++  title = {{Nextflow enables reproducible computational workflows}},
++  year = {2017},
++  journal = {Nature Biotechnology},
++  volume = {35},
++  number = {4},
++  pages = {316--319},
++  doi = {10.1038/nbt.3820}
++}
++
++% verified via https://api.crossref.org/works/10.1145/2452376.2452478 on 2026-10-30
++@inproceedings{missier2013prov,
++  author = {Missier, Paolo and Belhajjame, Khalid and Cheney, James},
++  title = {{The W3C PROV family of specifications for modelling provenance metadata}},
++  year = {2013},
++  booktitle = {Proceedings of the 16th International Conference on Extending Database Technology},
++  pages = {773--776},
++  doi = {10.1145/2452376.2452478}
++}
++
++% verified via https://api.crossref.org/works/10.1145/2882903.2899401 on 2026-10-30
++@inproceedings{chirigati2016reprozip,
++  author = {Chirigati, Fernando and Rampin, Rémi and Shasha, Dennis and Freire, Juliana},
++  title = {{ReproZip}},
++  year = {2016},
++  booktitle = {Proceedings of the 2016 International Conference on Management of Data},
++  pages = {2085--2088},
++  doi = {10.1145/2882903.2899401}
++}
++
++% verified via https://api.crossref.org/works/10.1371/journal.pcbi.1003285 on 2026-10-30
++@article{sandve2013tensimplerules,
++  author = {Sandve, Geir Kjetil and Nekrutenko, Anton and Taylor, James and Hovig, Eivind},
++  title = {{Ten Simple Rules for Reproducible Computational Research}},
++  year = {2013},
++  journal = {PLoS Computational Biology},
++  volume = {9},
++  number = {10},
++  pages = {e1003285},
++  doi = {10.1371/journal.pcbi.1003285}
++}
++
++% verified via https://api.crossref.org/works/10.1073/pnas.1708290115 on 2026-10-30
++@article{stodden2018journalpolicy,
++  author = {Stodden, Victoria and Seiler, Jennifer and Ma, Zhaokun},
++  title = {{An empirical analysis of journal policy effectiveness for computational reproducibility}},
++  year = {2018},
++  journal = {Proceedings of the National Academy of Sciences},
++  volume = {115},
++  number = {11},
++  pages = {2584--2589},
++  doi = {10.1073/pnas.1708290115}
++}
++
++% verified via https://api.crossref.org/works/10.1126/science.1213847 on 2026-10-30
++@article{peng2011reproducible,
++  author = {Peng, Roger D.},
++  title = {{Reproducible Research in Computational Science}},
++  year = {2011},
++  journal = {Science},
++  volume = {334},
++  number = {6060},
++  pages = {1226--1227},
++  doi = {10.1126/science.1213847}
++}
++
++% verified via https://api.crossref.org/works/10.17226/25303 on 2026-10-30
++@book{nasem2019reproducibility,
++  author = {{National Academies of Sciences, Engineering, and Medicine}},
++  title = {{Reproducibility and Replicability in Science}},
++  year = {2019},
++  publisher = {The National Academies Press},
++  address = {Washington, DC},
++  doi = {10.17226/25303}
++}
++
++% verified via https://api.crossref.org/works/10.1098/rsta.2020.0069 on 2026-10-30
++@article{krafczyk2021reproduction,
++  author = {Krafczyk, M. S. and Shi, A. and Bhaskar, A. and Marinov, D. and Stodden, V.},
++  title = {{Learning from reproducing computational results: introducing three principles and the <i>Reproduction Package</i>}},
++  year = {2021},
++  journal = {Philosophical Transactions of the Royal Society A: Mathematical, Physical and Engineering Sciences},
++  volume = {379},
++  number = {2197},
++  doi = {10.1098/rsta.2020.0069}
++}
++
++% verified via https://api.crossref.org/works/10.1038/s41597-022-01143-6 on 2026-10-30
++@article{trisovic2022codequality,
++  author = {Trisovic, Ana and Lau, Matthew K. and Pasquier, Thomas and Crosas, Mercè},
++  title = {{A large-scale study on research code quality and execution}},
++  year = {2022},
++  journal = {Scientific Data},
++  volume = {9},
++  number = {1},
++  doi = {10.1038/s41597-022-01143-6}
++}
++
++% verified via https://api.crossref.org/works/10.1073/pnas.1708274114 on 2026-10-30
++@article{nosek2018preregistration,
++  author = {Nosek, Brian A. and Ebersole, Charles R. and DeHaven, Alexander C. and Mellor, David T.},
++  title = {{The preregistration revolution}},
++  year = {2018},
++  journal = {Proceedings of the National Academy of Sciences},
++  volume = {115},
++  number = {11},
++  pages = {2600--2606},
++  doi = {10.1073/pnas.1708274114}
++}
++
++% verified via https://api.crossref.org/works/10.1016/j.cortex.2012.12.016 on 2026-10-30
++@article{chambers2013registeredreports,
++  author = {Chambers, Christopher D.},
++  title = {{Registered Reports: A new publishing initiative at Cortex}},
++  year = {2013},
++  journal = {Cortex},
++  volume = {49},
++  number = {3},
++  pages = {609--610},
++  doi = {10.1016/j.cortex.2012.12.016}
++}
++
++% verified via https://api.crossref.org/works/10.1038/s41562-021-01193-7 on 2026-10-30
++@article{chambers2022registeredreports,
++  author = {Chambers, Christopher D. and Tzavella, Loukia},
++  title = {{The past, present and future of Registered Reports}},
++  year = {2021},
++  journal = {Nature Human Behaviour},
++  volume = {6},
++  number = {1},
++  pages = {29--42},
++  doi = {10.1038/s41562-021-01193-7}
++}
++
++% verified via https://api.crossref.org/works/10.1177/0956797611417632 on 2026-10-30
++@article{simmons2011falsepositive,
++  author = {Simmons, Joseph P. and Nelson, Leif D. and Simonsohn, Uri},
++  title = {{False-Positive Psychology}},
++  year = {2011},
++  journal = {Psychological Science},
++  volume = {22},
++  number = {11},
++  pages = {1359--1366},
++  doi = {10.1177/0956797611417632}
++}
++
++% verified via https://api.crossref.org/works/10.1038/s41562-016-0021 on 2026-10-30
++@article{munafo2017manifesto,
++  author = {Munafò, Marcus R. and Nosek, Brian A. and Bishop, Dorothy V. M. and Button, Katherine S. and Chambers, Christopher D. and Percie du Sert, Nathalie and Simonsohn, Uri and Wagenmakers, Eric-Jan and Ware, Jennifer J. and Ioannidis, John P. A.},
++  title = {{A manifesto for reproducible science}},
++  year = {2017},
++  journal = {Nature Human Behaviour},
++  volume = {1},
++  number = {1},
++  doi = {10.1038/s41562-016-0021}
++}
++
++% verified via https://api.crossref.org/works/10.1511/2014.111.460 on 2026-10-30
++@article{gelman2014statisticalcrisis,
++  author = {Gelman, Andrew and Loken, Eric},
++  title = {{The Statistical Crisis in Science}},
++  year = {2014},
++  journal = {American Scientist},
++  volume = {102},
++  number = {6},
++  pages = {460},
++  doi = {10.1511/2014.111.460}
++}
++
++% verified via https://api.crossref.org/works/10.1126/science.aab2374 on 2026-10-30
++@article{nosek2015top,
++  author = {Nosek, B. A. and Alter, G. and Banks, G. C. and Borsboom, D. and Bowman, S. D. and Breckler, S. J. and Buck, S. and Chambers, C. D. and Chin, G. and Christensen, G. and Contestabile, M. and Dafoe, A. and Eich, E. and Freese, J. and Glennerster, R. and Goroff, D. and Green, D. P. and Hesse, B. and Humphreys, M. and Ishiyama, J. and Karlan, D. and Kraut, A. and Lupia, A. and Mabry, P. and Madon, T. and Malhotra, N. and Mayo-Wilson, E. and McNutt, M. and Miguel, E. and Paluck, E. Levy and Simonsohn, U. and Soderberg, C. and Spellman, B. A. and Turitto, J. and VandenBos, G. and Vazire, S. and Wagenmakers, E. J. and Wilson, R. and Yarkoni, T.},
++  title = {{Promoting an open research culture}},
++  year = {2015},
++  journal = {Science},
++  volume = {348},
++  number = {6242},
++  pages = {1422--1425},
++  doi = {10.1126/science.aab2374}
++}
++
++% verified via https://api.crossref.org/works/10.1177/25152459211007467 on 2026-10-30
++@article{scheel2021excess,
++  author = {Scheel, Anne M. and Schijen, Mitchell R. M. J. and Lakens, Daniël},
++  title = {{An Excess of Positive Results: Comparing the Standard Psychology Literature With Registered Reports}},
++  year = {2021},
++  journal = {Advances in Methods and Practices in Psychological Science},
++  volume = {4},
++  number = {2},
++  doi = {10.1177/25152459211007467}
++}
++
++% verified via https://api.crossref.org/works/10.1126/science.aac4716 on 2026-10-30
++@article{osc2015reproducibility,
++  author = {{Open Science Collaboration}},
++  title = {{Estimating the reproducibility of psychological science}},
++  year = {2015},
++  journal = {Science},
++  volume = {349},
++  number = {6251},
++  doi = {10.1126/science.aac4716}
++}
++
++% verified via https://api.crossref.org/works/10.1207/s15327957pspr0203_4 on 2026-10-30
++@article{kerr1998harking,
++  author = {Kerr, Norbert L.},
++  title = {{HARKing: Hypothesizing After the Results are Known}},
++  year = {1998},
++  journal = {Personality and Social Psychology Review},
++  volume = {2},
++  number = {3},
++  pages = {196--217},
++  doi = {10.1207/s15327957pspr0203_4}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2308.03688 on 2026-10-30
++@inproceedings{liu2024agentbench,
++  author = {Xiao Liu and Hao Yu and Hanchen Zhang and Yifan Xu and Xuanyu Lei and Hanyu Lai and Yu Gu and Hangliang Ding and Kaiwen Men and Kejuan Yang and Shudan Zhang and Xiang Deng and Aohan Zeng and Zhengxiao Du and Chenhui Zhang and Sheng Shen and Tianjun Zhang and Yu Su and Huan Sun and Minlie Huang and Yuxiao Dong and Jie Tang},
++  title = {{AgentBench: Evaluating LLMs as Agents}},
++  year = {2023},
++  booktitle = {ICLR 2024},
++  eprint = {2308.03688},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2406.12045 on 2026-10-30
++@misc{yao2024taubench,
++  author = {Shunyu Yao and Noah Shinn and Pedram Razavi and Karthik Narasimhan},
++  title = {{$$\tau$$-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains}},
++  year = {2024},
++  howpublished = {arXiv preprint arXiv:2406.12045},
++  eprint = {2406.12045},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2407.01502 on 2026-10-30
++@article{kapoor2024agentsthatmatter,
++  author = {Sayash Kapoor and Benedikt Stroebl and Zachary S. Siegel and Nitya Nadgir and Arvind Narayanan},
++  title = {{AI Agents That Matter}},
++  year = {2024},
++  journal = {Transactions on Machine Learning Research},
++  eprint = {2407.01502},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.crossref.org/works/10.18653/v1/2023.findings-emnlp.722 on 2026-10-30
++@inproceedings{sainz2023contamination,
++  author = {Sainz, Oscar and Campos, Jon and García-Ferrero, Iker and Etxaniz, Julen and de Lacalle, Oier Lopez and Agirre, Eneko},
++  title = {{NLP Evaluation in trouble: On the Need to Measure LLM Data Contamination for each Benchmark}},
++  year = {2023},
++  booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2023},
++  pages = {10776--10787},
++  doi = {10.18653/v1/2023.findings-emnlp.722},
++  eprint = {2310.18018},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2306.05685 on 2026-10-30
++@inproceedings{zheng2023mtbench,
++  author = {Lianmin Zheng and Wei-Lin Chiang and Ying Sheng and Siyuan Zhuang and Zhanghao Wu and Yonghao Zhuang and Zi Lin and Zhuohan Li and Dacheng Li and Eric P. Xing and Hao Zhang and Joseph E. Gonzalez and Ion Stoica},
++  title = {{Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena}},
++  year = {2023},
++  booktitle = {NeurIPS 2023 Datasets and Benchmarks},
++  eprint = {2306.05685},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2406.12624 on 2026-10-30
++@inproceedings{thakur2025judgingjudges,
++  author = {Aman Singh Thakur and Kartik Choudhary and Venkat Srinik Ramayapally and Sankaran Vaidyanathan and Dieuwke Hupkes},
++  title = {{Judging the Judges: Evaluating Alignment and Vulnerabilities in LLMs-as-Judges}},
++  year = {2024},
++  booktitle = {GEM2 Workshop, ACL 2025},
++  eprint = {2406.12624},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2404.13076 on 2026-10-30
++@inproceedings{panickssery2024selfpreference,
++  author = {Arjun Panickssery and Samuel R. Bowman and Shi Feng},
++  title = {{LLM Evaluators Recognize and Favor Their Own Generations}},
++  year = {2024},
++  booktitle = {NeurIPS 2024},
++  eprint = {2404.13076},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2507.02825 on 2026-10-30
++@misc{zhu2025abc,
++  author = {Yuxuan Zhu and Tengjun Jin and Yada Pruksachatkun and Andy Zhang and Shu Liu and Sasha Cui and Sayash Kapoor and Shayne Longpre and Kevin Meng and Rebecca Weiss and Fazl Barez and Rahul Gupta and Jwala Dhamala and Jacob Merizian and Mario Giulianelli and Harry Coppock and Cozmin Ududec and Jasjeet Sekhon and Jacob Steinhardt and Antony Kellermann and Sarah Schwettmann and Matei Zaharia and Ion Stoica and Percy Liang and Daniel Kang},
++  title = {{Establishing Best Practices for Building Rigorous Agentic Benchmarks}},
++  year = {2025},
++  howpublished = {arXiv preprint arXiv:2507.02825},
++  eprint = {2507.02825},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2510.11977 on 2026-10-30
++@misc{kapoor2025hal,
++  author = {Sayash Kapoor and Benedikt Stroebl and Peter Kirgis and Nitya Nadgir and Zachary S Siegel and Boyi Wei and Tianci Xue and Ziru Chen and Felix Chen and Saiteja Utpala and Franck Ndzomga and Dheeraj Oruganty and Sophie Luskin and Kangheng Liu and Botao Yu and Amit Arora and Dongyoon Hahm and Harsh Trivedi and Huan Sun and Juyong Lee and Tengjun Jin and Yifan Mai and Yifei Zhou and Yuxuan Zhu and Rishi Bommasani and Daniel Kang and Dawn Song and Peter Henderson and Yu Su and Percy Liang and Arvind Narayanan},
++  title = {{Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation}},
++  year = {2025},
++  howpublished = {arXiv preprint arXiv:2510.11977},
++  eprint = {2510.11977},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2510.12367 on 2026-10-30
++@misc{li2025llmreval,
++  author = {Rui Li and Jia-Chen Gu and Po-Nien Kung and Heming Xia and Junfeng liu and Xiangwen Kong and Zhifang Sui and Nanyun Peng},
++  title = {{LLM-REVal: Can We Trust LLM Reviewers Yet?}},
++  year = {2025},
++  howpublished = {arXiv preprint arXiv:2510.12367},
++  eprint = {2510.12367},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.crossref.org/works/10.1518/001872097778543886 on 2026-10-30
++@article{parasuraman1997automation,
++  author = {Parasuraman, Raja and Riley, Victor},
++  title = {{Humans and Automation: Use, Misuse, Disuse, Abuse}},
++  year = {1997},
++  journal = {Human Factors: The Journal of the Human Factors and Ergonomics Society},
++  volume = {39},
++  number = {2},
++  pages = {230--253},
++  doi = {10.1518/001872097778543886}
++}
++
++% verified via https://api.crossref.org/works/10.1109/3468.844354 on 2026-10-30
++@article{parasuraman2000levels,
++  author = {Parasuraman, R. and Sheridan, T.B. and Wickens, C.D.},
++  title = {{A model for types and levels of human interaction with automation}},
++  year = {2000},
++  journal = {IEEE Transactions on Systems, Man, and Cybernetics - Part A: Systems and Humans},
++  volume = {30},
++  number = {3},
++  pages = {286--297},
++  doi = {10.1109/3468.844354}
++}
++
++% verified via https://api.crossref.org/works/10.1136/amiajnl-2011-000089 on 2026-10-30
++@article{goddard2012automationbias,
++  author = {Goddard, Kate and Roudsari, Abdul and Wyatt, Jeremy C},
++  title = {{Automation bias: a systematic review of frequency, effect mediators, and mitigators}},
++  year = {2012},
++  journal = {Journal of the American Medical Informatics Association},
++  volume = {19},
++  number = {1},
++  pages = {121--127},
++  doi = {10.1136/amiajnl-2011-000089}
++}
++
++% verified via https://api.crossref.org/works/10.1145/3593013.3594087 on 2026-10-30
++@inproceedings{lai2023humanai,
++  author = {Lai, Vivian and Chen, Chacha and Smith-Renner, Alison and Liao, Q. Vera and Tan, Chenhao},
++  title = {{Towards a Science of Human-AI Decision Making: An Overview of Design Space in Empirical Human-Subject Studies}},
++  year = {2023},
++  booktitle = {2023 ACM Conference on Fairness Accountability and Transparency},
++  pages = {1369--1385},
++  doi = {10.1145/3593013.3594087},
++  eprint = {2112.11471},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.crossref.org/works/10.1145/3411764.3445717 on 2026-10-30
++@inproceedings{bansal2021whole,
++  author = {Bansal, Gagan and Wu, Tongshuang and Zhou, Joyce and Fok, Raymond and Nushi, Besmira and Kamar, Ece and Ribeiro, Marco Tulio and Weld, Daniel},
++  title = {{Does the Whole Exceed its Parts? The Effect of AI Explanations on Complementary Team Performance}},
++  year = {2021},
++  booktitle = {Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems},
++  pages = {1--16},
++  doi = {10.1145/3411764.3445717}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=1902.03245 on 2026-10-30
++@inproceedings{lubars2019delegability,
++  author = {Brian Lubars and Chenhao Tan},
++  title = {{Ask Not What AI Can Do, But What AI Should Do: Towards a Framework of Task Delegability}},
++  year = {2019},
++  booktitle = {NeurIPS 2019},
++  eprint = {1902.03245},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.crossref.org/works/10.1038/s41562-024-02024-1 on 2026-10-30
++@article{vaccaro2024combinations,
++  author = {Vaccaro, Michelle and Almaatouq, Abdullah and Malone, Thomas},
++  title = {{When combinations of humans and AI are useful: A systematic review and meta-analysis}},
++  year = {2024},
++  journal = {Nature Human Behaviour},
++  volume = {8},
++  number = {12},
++  pages = {2293--2303},
++  doi = {10.1038/s41562-024-02024-1}
++}
++
++% verified via https://api.crossref.org/works/10.1016/0005-1098(83)90046-8 on 2026-10-30
++@article{bainbridge1983ironies,
++  author = {Bainbridge, Lisanne},
++  title = {{Ironies of automation}},
++  year = {1983},
++  journal = {Automatica},
++  volume = {19},
++  number = {6},
++  pages = {775--779},
++  doi = {10.1016/0005-1098(83)90046-8}
++}
++
++% verified via https://api.crossref.org/works/10.1145/3449287 on 2026-10-30
++@article{bucinca2021trust,
++  author = {Buçinca, Zana and Malaya, Maja Barbara and Gajos, Krzysztof Z.},
++  title = {{To Trust or to Think}},
++  year = {2021},
++  journal = {Proceedings of the ACM on Human-Computer Interaction},
++  volume = {5},
++  number = {CSCW1},
++  pages = {1--21},
++  doi = {10.1145/3449287}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2507.09089 on 2026-10-30
++@misc{becker2025metr,
++  author = {Joel Becker and Nate Rush and Elizabeth Barnes and David Rein},
++  title = {{Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity}},
++  year = {2025},
++  howpublished = {arXiv preprint arXiv:2507.09089},
++  eprint = {2507.09089},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.crossref.org/works/10.1145/2408776.2408794 on 2026-10-30
++@article{dean2013tail,
++  author = {Dean, Jeffrey and Barroso, Luiz André},
++  title = {{The tail at scale}},
++  year = {2013},
++  journal = {Communications of the ACM},
++  volume = {56},
++  number = {2},
++  pages = {74--80},
++  doi = {10.1145/2408776.2408794}
++}
++
++% verified via https://api.semanticscholar.org/graph/v1/paper/87c9e34c39dd1fbd470010dc1ea98ba3517f5a2a on 2026-10-30
++@inproceedings{gray1986whystop,
++  author = {Jim Gray},
++  title = {{Why Do Computers Stop and What Can Be Done About It?}},
++  year = {1986},
++  booktitle = {Symposium on Reliability in Distributed Software and Database Systems}
++}
++
++% verified via https://api.crossref.org/works/10.1109/MS.2016.60 on 2026-10-30
++@article{basiri2016chaos,
++  author = {Basiri, Ali and Behnam, Niosha and de Rooij, Ruud and Hochstein, Lorin and Kosewski, Luke and Reynolds, Justin and Rosenthal, Casey},
++  title = {{Chaos Engineering}},
++  year = {2016},
++  journal = {IEEE Software},
++  volume = {33},
++  number = {3},
++  pages = {35--41},
++  doi = {10.1109/MS.2016.60}
++}
++
++% verified via https://api.crossref.org/works/10.1145/2160718.2160734 on 2026-10-30
++@article{helland2012idempotence,
++  author = {Helland, Pat},
++  title = {{Idempotence is not a medical condition}},
++  year = {2012},
++  journal = {Communications of the ACM},
++  volume = {55},
++  number = {5},
++  pages = {56--65},
++  doi = {10.1145/2160718.2160734}
++}
++
++% verified via https://api.crossref.org/works/10.1145/3485510 on 2026-10-30
++@article{burckhardt2021durablefunctions,
++  author = {Burckhardt, Sebastian and Gillum, Chris and Justo, David and Kallas, Konstantinos and McMahon, Connor and Meiklejohn, Christopher S.},
++  title = {{Durable functions: semantics for stateful serverless}},
++  year = {2021},
++  journal = {Proceedings of the ACM on Programming Languages},
++  volume = {5},
++  number = {OOPSLA},
++  pages = {1--27},
++  doi = {10.1145/3485510}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2504.13359 on 2026-10-30
++@misc{erol2025costofpass,
++  author = {Mehmet Hamza Erol and Batu El and Mirac Suzgun and Mert Yuksekgonul and James Zou},
++  title = {{Cost-of-Pass: An Economic Framework for Evaluating Language Models}},
++  year = {2025},
++  howpublished = {arXiv preprint arXiv:2504.13359},
++  eprint = {2504.13359},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2411.05285 on 2026-10-30
++@misc{dong2024agentops,
++  author = {Liming Dong and Qinghua Lu and Liming Zhu},
++  title = {{AgentOps: Enabling Observability of LLM Agents}},
++  year = {2024},
++  howpublished = {arXiv preprint arXiv:2411.05285},
++  eprint = {2411.05285},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2603.29231 on 2026-10-30
++@misc{khanal2026reliability,
++  author = {Aaditya Khanal and Yangyang Tao and Junxiu Zhou},
++  title = {{Beyond pass@1: A Reliability Science Framework for Long-Horizon LLM Agents}},
++  year = {2026},
++  howpublished = {arXiv preprint arXiv:2603.29231},
++  eprint = {2603.29231},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2608.14380 on 2026-10-30
++@misc{zhuang2026agentrewind,
++  author = {Yu Zhuang and Kefei Chen and Yitong Duan and Shuxin Zheng and Jian Li and Xu-Yao Zhang},
++  title = {{AgentRewind: Recoverable Execution for Long-Horizon LLM Agents}},
++  year = {2026},
++  howpublished = {arXiv preprint arXiv:2608.14380},
++  eprint = {2608.14380},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.semanticscholar.org/graph/v1/paper/003d5a65de0ac72daaf105ded903cb3eb88585b3 on 2026-10-30
++@techreport{sigelman2010dapper,
++  author = {B. Sigelman and L. Barroso and M. Burrows and Patrick Stephenson and M. Plakal and Donald Beaver and Saul Jaspan and C. Shanbhag},
++  title = {{Dapper, a Large-Scale Distributed Systems Tracing Infrastructure}},
++  year = {2010},
++  institution = {Google},
++  number = {dapper-2010-1}
++}
++
++% verified via https://api.crossref.org/works/10.18653/v1/2020.emnlp-main.609 on 2026-10-30
++@inproceedings{wadden2020scifact,
++  author = {Wadden, David and Lin, Shanchuan and Lo, Kyle and Wang, Lucy Lu and van Zuylen, Madeleine and Cohan, Arman and Hajishirzi, Hannaneh},
++  title = {{Fact or Fiction: Verifying Scientific Claims}},
++  year = {2020},
++  booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
++  pages = {7534--7550},
++  doi = {10.18653/v1/2020.emnlp-main.609}
++}
++
++% verified via https://api.crossref.org/works/10.18653/v1/N18-1074 on 2026-10-30
++@inproceedings{thorne2018fever,
++  author = {Thorne, James and Vlachos, Andreas and Christodoulopoulos, Christos and Mittal, Arpit},
++  title = {{FEVER: a Large-scale Dataset for Fact Extraction and VERification}},
++  year = {2018},
++  booktitle = {Proceedings of the 2018 Conference of the North American Chapter of
++          the Association for Computational Linguistics: Human Language
++          Technologies, Volume 1 (Long Papers)},
++  pages = {809--819},
++  doi = {10.18653/v1/N18-1074}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2212.08037 on 2026-10-30
++@misc{bohnet2022attributedqa,
++  author = {Bernd Bohnet and Vinh Q. Tran and Pat Verga and Roee Aharoni and Daniel Andor and Livio Baldini Soares and Massimiliano Ciaramita and Jacob Eisenstein and Kuzman Ganchev and Jonathan Herzig and Kai Hui and Tom Kwiatkowski and Ji Ma and Jianmo Ni and Lierni Sestorain Saralegui and Tal Schuster and William W. Cohen and Michael Collins and Dipanjan Das and Donald Metzler and Slav Petrov and Kellie Webster},
++  title = {{Attributed Question Answering: Evaluation and Modeling for Attributed Large Language Models}},
++  year = {2022},
++  howpublished = {arXiv preprint arXiv:2212.08037},
++  eprint = {2212.08037},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.crossref.org/works/10.1038/s41598-023-41032-5 on 2026-10-30
++@article{walters2023fabrication,
++  author = {Walters, William H. and Wilder, Esther Isabelle},
++  title = {{Fabrication and errors in the bibliographic citations generated by ChatGPT}},
++  year = {2023},
++  journal = {Scientific Reports},
++  volume = {13},
++  number = {1},
++  doi = {10.1038/s41598-023-41032-5}
++}
++
++% verified via https://api.crossref.org/works/10.7759/cureus.35179 on 2026-10-30
++@article{alkaissi2023hallucinations,
++  author = {Alkaissi, Hussam and McFarlane, Samy I},
++  title = {{Artificial Hallucinations in ChatGPT: Implications in Scientific Writing}},
++  year = {2023},
++  journal = {Cureus},
++  doi = {10.7759/cureus.35179}
++}
++
++% verified via https://api.crossref.org/works/10.18653/v1/2023.acl-long.910 on 2026-10-30
++@inproceedings{gao2023rarr,
++  author = {Gao, Luyu and Dai, Zhuyun and Pasupat, Panupong and Chen, Anthony and Chaganty, Arun Tejasvi and Fan, Yicheng and Zhao, Vincent and Lao, Ni and Lee, Hongrae and Juan, Da-Cheng and Guu, Kelvin},
++  title = {{RARR: Researching and Revising What Language Models Say, Using Language Models}},
++  year = {2023},
++  booktitle = {Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
++  pages = {16477--16508},
++  doi = {10.18653/v1/2023.acl-long.910}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2203.11171 on 2026-10-30
++@inproceedings{wang2023selfconsistency,
++  author = {Xuezhi Wang and Jason Wei and Dale Schuurmans and Quoc Le and Ed Chi and Sharan Narang and Aakanksha Chowdhery and Denny Zhou},
++  title = {{Self-Consistency Improves Chain of Thought Reasoning in Language Models}},
++  year = {2022},
++  booktitle = {ICLR 2023},
++  eprint = {2203.11171},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://api.crossref.org/works/10.18653/v1/2023.emnlp-main.741 on 2026-10-30
++@inproceedings{min2023factscore,
++  author = {Min, Sewon and Krishna, Kalpesh and Lyu, Xinxi and Lewis, Mike and Yih, Wen-tau and Koh, Pang and Iyyer, Mohit and Zettlemoyer, Luke and Hajishirzi, Hannaneh},
++  title = {{FActScore: Fine-grained Atomic Evaluation of Factual Precision in Long Form Text Generation}},
++  year = {2023},
++  booktitle = {Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},
++  pages = {12076--12100},
++  doi = {10.18653/v1/2023.emnlp-main.741},
++  eprint = {2305.14251},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2310.01798 on 2026-10-30
++@inproceedings{huang2024selfcorrect,
++  author = {Jie Huang and Xinyun Chen and Swaroop Mishra and Huaixiu Steven Zheng and Adams Wei Yu and Xinying Song and Denny Zhou},
++  title = {{Large Language Models Cannot Self-Correct Reasoning Yet}},
++  year = {2023},
++  booktitle = {ICLR 2024},
++  eprint = {2310.01798},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2601.18724 on 2026-10-30
++@inproceedings{sakai2026hallucitation,
++  author = {Yusuke Sakai and Hidetaka Kamigaito and Taro Watanabe},
++  title = {{HalluCitation Matters: Revealing the Impact of Hallucinated References with 300 Hallucinated Papers in ACL Conferences}},
++  year = {2026},
++  booktitle = {ACL 2026 (per Semantic Scholar venue record)},
++  eprint = {2601.18724},
++  archivePrefix = {arXiv}
++}
++
++% verified via https://export.arxiv.org/api/query?id_list=2605.07723 on 2026-10-30
++@misc{zhao2026hallucinationswild,
++  author = {Zhenyue Zhao and Yihe Wang and Toby Stuart and Mathijs De Vaan and Paul Ginsparg and Yian Yin},
++  title = {{LLM hallucinations in the wild: Large-scale evidence from non-existent citations}},
++  year = {2026},
++  howpublished = {arXiv preprint arXiv:2605.07723},
++  eprint = {2605.07723},
++  archivePrefix = {arXiv}
++}
++

added research/verify_cache/doi_resolution.json +84 −0

++{
++ "10.1002/adma.202413523": 302,
++ "10.1006/ijhc.1999.0252": 302,
++ "10.1007/s10994-025-06955-2": 302,
++ "10.1016/0005-1098(83)90046-8": 302,
++ "10.1016/j.cortex.2012.12.016": 302,
++ "10.1016/j.future.2014.10.008": 302,
++ "10.1016/j.patter.2021.100322": 302,
++ "10.1016/j.websem.2015.04.001": 302,
++ "10.1038/nbt.3820": 302,
++ "10.1038/s41562-016-0021": 302,
++ "10.1038/s41562-021-01142-4": 302,
++ "10.1038/s41562-021-01193-7": 302,
++ "10.1038/s41562-024-02024-1": 302,
++ "10.1038/s41586-023-06221-2": 302,
++ "10.1038/s41586-023-06792-0": 302,
++ "10.1038/s41586-025-09442-9": 302,
++ "10.1038/s41586-026-10644-y": 302,
++ "10.1038/s41597-022-01143-6": 302,
++ "10.1038/s41598-023-41032-5": 302,
++ "10.1038/s42256-024-00832-8": 302,
++ "10.1038/sdata.2016.18": 302,
++ "10.1056/AIoa2400555": 302,
++ "10.1073/pnas.1708274114": 302,
++ "10.1073/pnas.1708290115": 302,
++ "10.1088/1742-6596/46/1/067": 302,
++ "10.1093/bioinformatics/bts480": 302,
++ "10.1098/rsta.2020.0069": 302,
++ "10.1109/2.84898": 302,
++ "10.1109/3468.844354": 302,
++ "10.1109/MS.2016.60": 302,
++ "10.1126/science.1213847": 302,
++ "10.1126/science.aab2374": 302,
++ "10.1126/science.aac4716": 302,
++ "10.1126/science.aah6168": 302,
++ "10.1126/scitranslmed.aaf5027": 302,
++ "10.1136/amiajnl-2011-000089": 302,
++ "10.1145/2160718.2160734": 302,
++ "10.1145/2408776.2408794": 302,
++ "10.1145/2452376.2452478": 302,
++ "10.1145/2670979.2670986": 302,
++ "10.1145/2723872.2723882": 302,
++ "10.1145/2882903.2899401": 302,
++ "10.1145/2987550.2987568": 302,
++ "10.1145/3290605.3300233": 302,
++ "10.1145/3411764.3445717": 302,
++ "10.1145/3449287": 302,
++ "10.1145/3485510": 302,
++ "10.1145/3486897": 302,
++ "10.1145/3571730": 302,
++ "10.1145/3593013.3594087": 302,
++ "10.1145/3805760.3814924": 302,
++ "10.1145/568522.568525": 302,
++ "10.1162/coli_a_00486": 302,
++ "10.1177/0956797611417632": 302,
++ "10.1177/1745691612463078": 302,
++ "10.1177/17456916231181102": 302,
++ "10.1177/25152459211007467": 302,
++ "10.1207/s15327957pspr0203_4": 302,
++ "10.12688/f1000research.29032.2": 302,
++ "10.12688/f1000research.51738.2": 302,
++ "10.1287/isre.2021.1079": 302,
++ "10.1371/journal.pbio.1002106": 302,
++ "10.1371/journal.pcbi.1003285": 302,
++ "10.1371/journal.pcbi.1008316": 302,
++ "10.1371/journal.pmed.0020124": 302,
++ "10.14778/3529337.3529344": 302,
++ "10.1511/2014.111.460": 302,
++ "10.1518/001872097778543886": 302,
++ "10.1518/hfes.46.1.50_30392": 302,
++ "10.17226/25303": 302,
++ "10.18653/v1/2020.emnlp-main.609": 302,
++ "10.18653/v1/2022.acl-short.18": 302,
++ "10.18653/v1/2023.acl-long.910": 302,
++ "10.18653/v1/2023.emnlp-main.398": 302,
++ "10.18653/v1/2023.emnlp-main.741": 302,
++ "10.18653/v1/2023.findings-emnlp.722": 302,
++ "10.18653/v1/N18-1074": 302,
++ "10.2139/ssrn.4573321": 302,
++ "10.2196/53164": 302,
++ "10.3389/fpsyg.2016.01832": 302,
++ "10.7759/cureus.35179": 302,
++ "10.1145/3769733.376974": 404
++}
+ \ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1006_2Fijhc_1999_0252 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T02:15:55Z","timestamp":1793240155170,"version":"4.5.0"},"reference-count":34,"publisher":"Elsevier BV","issue":"5","license":[{"start":{"date-parts":[[1999,11,1]],"date-time":"1999-11-01T00:00:00Z","timestamp":941414400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["International Journal of Human-Computer Studies"],"published-print":{"date-parts":[[1999,11]]},"DOI":"10.1006\/ijhc.1999.0252","type":"journal-article","created":{"date-parts":[[2002,9,17]],"date-time":"2002-09-17T19:19:12Z","timestamp":1032290352000},"page":"991-1006","source":"Crossref","is-referenced-by-count":561,"title":["Does automation bias decision-making?"],"prefix":"10.1006","volume":"51","author":[{"given":"LINDA J.","family":"SKITKA","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"KATHLEEN L.","family":"MOSIER","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"MARK","family":"BURDICK","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1006\/ijhc.1999.0252_RF1","first-page":"17","volume":"138","year":"1993","journal-title":"Aviation and Space Technology"},{"key":"10.1006\/ijhc.1999.0252_RF2","series-title":"Human-centered aircraft automation: principles and guidelines","author":"BILLINGS","year":"1996"},{"key":"10.1006\/ijhc.1999.0252_RF3","series-title":"Social Psychology","author":"BREWER","year":"1994"},{"key":"10.1006\/ijhc.1999.0252_RF4","first-page":"167","article-title":"Task specific effects of automation and automation failure on performance, workload, and situation awareness","author":"CARMODY","year":"1993"},{"key":"10.1006\/ijhc.1999.0252_RF5","doi-asserted-by":"crossref","first-page":"20","DOI":"10.1037\/0022-3514.44.1.20","article-title":"A hypothesis-confirming bias in labeling effects","volume":"44","author":"DARLEY","year":"1983","journal-title":"Journal of Personality and Social Psychology"},{"key":"10.1006\/ijhc.1999.0252_RF6","article-title":"The effectiveness of training programs for preventing air crew \u201cerror\u201d","author":"DIEHL","year":"1991"},{"key":"10.1006\/ijhc.1999.0252_RF7","series-title":"Social Cognition","author":"FISKE","year":"1994"},{"key":"10.1006\/ijhc.1999.0252_RF8","doi-asserted-by":"crossref","first-page":"178","DOI":"10.1037\/0022-3514.55.2.178","article-title":"What mediates sex discrimination in hiring decisions?","volume":"55","author":"GLICK","year":"1988","journal-title":"Journal of Personality and Social Psychology"},{"key":"10.1006\/ijhc.1999.0252_RF9","series-title":"Cognitive Processes in Stereotyping and intergroup Behaviour","author":"HAMILTON","year":"1981"},{"key":"10.1006\/ijhc.1999.0252_RF10","doi-asserted-by":"crossref","first-page":"171","DOI":"10.1097\/00005053-196608000-00008","article-title":"An experimental study in nurse-physician relationships","volume":"143","author":"HOFLING","year":"1966","journal-title":"Journal of Nervous and Mental Disease"},{"key":"10.1006\/ijhc.1999.0252_RF11","series-title":"Judgment under Uncertainty: Heuristics and Biases","author":"KAHNEMAN","year":"1982"},{"key":"10.1006\/ijhc.1999.0252_RF12","doi-asserted-by":"crossref","first-page":"681","DOI":"10.1037\/0022-3514.65.4.681","article-title":"Social loafing: a meta-analytic review and theoretical integration","volume":"65","author":"KARAU","year":"1993","journal-title":"Journal of Personality and Social Psychology"},{"key":"10.1006\/ijhc.1999.0252_RF13","series-title":"Judgment under Uncertainty: Heuristics and Biases","author":"KAHNEMAN","year":"1982"},{"key":"10.1006\/ijhc.1999.0252_RF14","doi-asserted-by":"crossref","first-page":"822","DOI":"10.1037\/0022-3514.37.6.822","article-title":"Many hands make light the work: the causes and consequences of social loafing","volume":"37","author":"LATAN\u00c9","year":"1979","journal-title":"Journal of Personality and Social Psychology"},{"key":"10.1006\/ijhc.1999.0252_RF15","doi-asserted-by":"crossref","first-page":"94","DOI":"10.1177\/001872089403600106","article-title":"Design of a cooperative problem-solving system for en-route flight planning: An empirical evaluation","volume":"36","author":"LAYTON","year":"1994","journal-title":"Human Factors"},{"key":"10.1006\/ijhc.1999.0252_RF16","doi-asserted-by":"crossref","first-page":"133","DOI":"10.1177\/108602669100500203","article-title":"Human factors in large-scale technological systems' accidents: three Miles Island, Bhopal, and Chernobyl","volume":"5","author":"MESHKATI","year":"1991","journal-title":"Industrial Crisis Quarterly"},{"key":"10.1006\/ijhc.1999.0252_RF17","first-page":"427","article-title":"Organizational and safety factors in automated oil and gas pipeline systems","author":"MESHKATI","year":"1996"},{"key":"10.1006\/ijhc.1999.0252_RF18","first-page":"201","article-title":"Human decision-makers and automated decision aids: Made for each other?","author":"MOSIER","year":"1996"},{"key":"10.1006\/ijhc.1999.0252_RF19","doi-asserted-by":"crossref","DOI":"10.1177\/154193129804200304","article-title":"Automation bias and errors: Are terms better than individuals?","author":"MOSIER","year":"1998"},{"key":"10.1006\/ijhc.1999.0252_RF20","doi-asserted-by":"crossref","first-page":"47","DOI":"10.1207\/s15327108ijap0801_3","article-title":"Automation bias: decision making and performance in high-tech cockpits","volume":"8","author":"MOSIER","year":"1998","journal-title":"The International Journal of Aviation Psychology"},{"key":"10.1006\/ijhc.1999.0252_RF21","doi-asserted-by":"crossref","first-page":"669","DOI":"10.1006\/ijhc.1996.0073","article-title":"Can computers be teammates?","volume":"45","author":"NASS","year":"1996","journal-title":"International Journal of Human\u2013 Computer Studies"},{"key":"10.1006\/ijhc.1999.0252_RF22","first-page":"147","article-title":"Effects of adaptive function allocation on human performance","author":"PARASURAMAN","year":"1993"},{"key":"10.1006\/ijhc.1999.0252_RF23","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1207\/s15327108ijap0301_1","article-title":"Performance consequences of automation-induced \u201ccomplacency\u201d","volume":"3","author":"PARASURAMAN","year":"1993","journal-title":"International Journal of Aviation Psychology."},{"key":"10.1006\/ijhc.1999.0252_RF24","first-page":"45","article-title":"Monitoring automation failures in human-machine systems","author":"PARASURAMAN","year":"1994"},{"key":"10.1006\/ijhc.1999.0252_RF25","doi-asserted-by":"crossref","first-page":"230","DOI":"10.1518\/001872097778543886","article-title":"Humans and automation: use, misuse, disuse, abuse","volume":"39","author":"PARASURAMAN","year":"1997","journal-title":"Human Factors"},{"key":"10.1006\/ijhc.1999.0252_RF26","doi-asserted-by":"crossref","first-page":"455","DOI":"10.2466\/pms.1994.78.2.455","article-title":"Effects of social loafing on word recognition","volume":"78","author":"PRATARELLI","year":"1994","journal-title":"Perceptual and Motor Skills"},{"key":"10.1006\/ijhc.1999.0252_RF27","doi-asserted-by":"crossref","first-page":"87","DOI":"10.1007\/BF00995116","article-title":"The HELP system","volume":"7","author":"PRYOR","year":"1983","journal-title":"Journal of Medical Systems"},{"key":"10.1006\/ijhc.1999.0252_RF28","doi-asserted-by":"crossref","first-page":"188","DOI":"10.2307\/2955382","article-title":"Hospital nurses' compliance with medication overdose orders: a failure to replicate","volume":"18","author":"RANK","year":"1997","journal-title":"Journal of Health and Social Behaviour"},{"key":"10.1006\/ijhc.1999.0252_RF29","first-page":"19","article-title":"Operator reliance on automation: theory and data","author":"RILEY","year":"1996"},{"key":"10.1006\/ijhc.1999.0252_RF30","unstructured":"RILEY, V. LYALL, E. WEINER, E. 1993, Analytic methods for flight-deck automation design and evaluation, phase two report: pilot use of automation, Honeywell Technology Centre, Minneapolis, MN"},{"key":"10.1006\/ijhc.1999.0252_RF31","unstructured":"SARTER, N. R. WOODS, D. D. 1993, Cognitive engineering in aerospace application: Pilot interaction with cockpit automation, 177617, NASA Ames Research Centre, Moffett Field, CA"},{"key":"10.1006\/ijhc.1999.0252_RF32","doi-asserted-by":"crossref","unstructured":"SKITKA, L. J. MOSIER, K. L. BURDICK, M. ROSENBLATT, B. 1998, Automation bias and errors: are teams better than individuals?","DOI":"10.1177\/154193129804200304"},{"key":"10.1006\/ijhc.1999.0252_RF33","unstructured":"WEINER, E. 1989, Human factors of advanced technology (\u201cglass cockpit\u201d) aircraft, NASA Contractor Report #177528, Moffett Field, CA"},{"key":"10.1006\/ijhc.1999.0252_RF34","first-page":"111","article-title":"Information processing","author":"WICKENES","year":"1988"}],"container-title":["International Journal of Human-Computer Studies"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1071581999902525?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1071581999902525?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2019,12,15]],"date-time":"2019-12-15T23:53:25Z","timestamp":1576454005000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S1071581999902525"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[1999,11]]},"references-count":34,"journal-issue":{"issue":"5","published-print":{"date-parts":[[1999,11]]}},"alternative-id":["S1071581999902525"],"URL":"https:\/\/doi.org\/10.1006\/ijhc.1999.0252","relation":{},"ISSN":["1071-5819"],"issn-type":[{"value":"1071-5819","type":"print"}],"subject":[],"published":{"date-parts":[[1999,11]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1007_2Fs10994_025_06955_2 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,11]],"date-time":"2026-09-11T06:38:43Z","timestamp":1789108723043,"version":"build-2803163510"},"reference-count":143,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2026,4,29]],"date-time":"2026-04-29T00:00:00Z","timestamp":1777420800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2026,4,29]],"date-time":"2026-04-29T00:00:00Z","timestamp":1777420800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"name":"Johannes Gutenberg-Universit\u00e4t Mainz"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Mach Learn"],"published-print":{"date-parts":[[2026,5]]},"abstract":"<jats:title>Abstract<\/jats:title>\n <jats:p>The paper surveys automated scientific discovery, from equation discovery and symbolic regression to autonomous discovery systems and agents. It discusses the individual approaches from a \"big picture\" perspective and in context, but also discusses open issues and recent topics like the various roles of deep neural networks in this area, aiding in the discovery of human-interpretable knowledge. Further, we will present closed-loop scientific discovery systems, starting with the pioneering work on the Adam system up to current efforts in fields from material science to astronomy. Finally, we will elaborate on autonomy from a machine learning perspective, but also in analogy to the autonomy levels in autonomous driving. The maximal level, level five, is defined to require no human intervention at all in the production of scientific knowledge. Achieving this is one step towards solving the Nobel Turing Grand Challenge to develop AI Scientists: AI systems capable of making Nobel-quality scientific discoveries highly autonomously at a level comparable, and possibly superior, to the best human scientists by 2050.<\/jats:p>","DOI":"10.1007\/s10994-025-06955-2","type":"journal-article","created":{"date-parts":[[2026,4,29]],"date-time":"2026-04-29T19:32:40Z","timestamp":1777491160000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":6,"title":["Automated Scientific Discovery: From Equation Discovery to Autonomous Discovery Systems"],"prefix":"10.1007","volume":"115","author":[{"given":"Stefan","family":"Kramer","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mattia","family":"Cerrato","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jannis","family":"Brugger","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sa\u0161o","family":"D\u017eeroski","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ross D.","family":"King","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,4,29]]},"reference":[{"key":"6955_CR1","doi-asserted-by":"publisher","unstructured":"Al-Helali, B., Chen, Q., Xue, B., & Zhang, M. (2020). GP-based feature selection and weighted KNN-based instance selection for symbolic regression with incomplete data. In 2020 IEEE symposium series on computational intelligence, SSCI 2020, Canberra, Australia, December 1\u20134, 2020, pp. 905\u2013912. IEEE Press, Piscataway, NJ. https:\/\/doi.org\/10.1109\/SSCI47803.2020.9308382","DOI":"10.1109\/SSCI47803.2020.9308382"},{"issue":"3","key":"6955_CR2","doi-asserted-by":"publisher","first-page":"2269","DOI":"10.1109\/TETCI.2024.3369407","volume":"8","author":"B Al-Helali","year":"2024","unstructured":"Al-Helali, B., Chen, Q., Xue, B., & Zhang, M. (2024). Genetic programming for feature selection based on feature removal impact in high-dimensional symbolic regression. IEEE Transactions on Emerging Topics in Computational Intelligence, 8(3), 2269\u20132282. https:\/\/doi.org\/10.1109\/TETCI.2024.3369407","journal-title":"IEEE Transactions on Emerging Topics in Computational Intelligence"},{"key":"6955_CR3","doi-asserted-by":"publisher","unstructured":"Ali, M.S., Kshirsagar, M., Naredo, E., & Ryan, C. (2022). Automated grammar-based feature selection in symbolic regression. In Proceedings of the 2022 Genetic and Evolutionary Computation Conference (GECCO), pp. 902\u2013910. https:\/\/doi.org\/10.1145\/3512290.3528852","DOI":"10.1145\/3512290.3528852"},{"issue":"12","key":"6955_CR4","doi-asserted-by":"publisher","first-page":"1315","DOI":"10.1038\/s41592-019-0598-1","volume":"16","author":"EC Alley","year":"2019","unstructured":"Alley, E. C., Khimulya, G., Biswas, S., AlQuraishi, M., & Church, G. M. (2019). Unified rational protein engineering with sequence-based deep representation learning. Nature Methods, 16(12), 1315\u20131322. https:\/\/doi.org\/10.1038\/s41592-019-0598-1","journal-title":"Nature Methods"},{"key":"6955_CR5","doi-asserted-by":"publisher","first-page":"742","DOI":"10.1039\/D3DD00183K","volume":"3","author":"C Beeler","year":"2024","unstructured":"Beeler, C., Subramanian, S. G., Sprague, K., Baula, M., Chatti, N., Dawit, A., Li, X., Paquin, N., Shahen, M., Yang, Z., Bellinger, C., Crowley, M., & Tamblyn, I. (2024). ChemGymRL: A customizable interactive framework for reinforcement learning for digital chemistry. Digital Discovery, 3, 742\u2013758. https:\/\/doi.org\/10.1039\/D3DD00183K","journal-title":"Digital Discovery"},{"key":"6955_CR6","unstructured":"Bendinelli, T., Biggio, L., & Kamienny, P. (2023). Controllable neural symbolic regression. In: Krause, A., Brunskill, E., Cho, K., Engelhardt, B., Sabato, S., Scarlett, J. (eds.) International Conference on Machine Learning, ICML 2023, 23-29 July 2023, Honolulu, Hawaii, USA. Proceedings of Machine Learning Research, 202, 2063\u20132077."},{"key":"6955_CR7","doi-asserted-by":"crossref","unstructured":"Berghaus, D., Cvejoski, K., Seifner, P., Ojeda, C., & Sanchez, R. (2024). Foundation inference models for markov jump processes. In: Advances in Neural Information Processing Systems (NeurIPS). https:\/\/papers.nips.cc\/paper_files\/paper\/2024\/hash\/e9df36b21ff4ee211a8b71ee8b7e9f57-Abstract-Conference.html","DOI":"10.52202\/079017-4112"},{"key":"6955_CR8","unstructured":"Biggio, L., Bendinelli, T., Neitz, A., Lucchi, A., & Parascandolo, G. (2021). Neural symbolic regression that scales. In: Meila, M., Zhang, T. (eds.) Proceedings of the 38th International Conference on Machine Learning, ICML 2021, 18\u201324 July 2021, Virtual Event. Proceedings of Machine Learning Research, vol. 139, pp. 936\u2013945. Virtual event. http:\/\/proceedings.mlr.press\/v139\/biggio21a.html"},{"key":"6955_CR9","doi-asserted-by":"publisher","unstructured":"Bomarito, G.F., Leser, P.E., Strauss, N.C.M., Garbrecht, K.M., & Hochhalter, J.D. (2022). Bayesian model selection for reducing bloat and overfitting in genetic programming for symbolic regression. In Proceedings of the 2022 genetic and evolutionary computation conference companion (GECCO), pp. 526\u2013529.https:\/\/doi.org\/10.1145\/3520304.3528899.","DOI":"10.1145\/3520304.3528899"},{"key":"6955_CR11","doi-asserted-by":"publisher","first-page":"742","DOI":"10.1016\/j.ins.2023.03.073","volume":"632","author":"J Brence","year":"2023","unstructured":"Brence, J., Dzeroski, S., & Todorovski, L. (2023). Dimensionally-consistent equation discovery through probabilistic attribute grammars. Information Sciences, 632, 742\u2013756. https:\/\/doi.org\/10.1016\/j.ins.2023.03.073","journal-title":"Information Sciences"},{"key":"6955_CR12","doi-asserted-by":"publisher","DOI":"10.1016\/j.knosys.2021.107077","volume":"224","author":"J Brence","year":"2021","unstructured":"Brence, J., Todorovski, L., & D\u017eeroski, S. (2021). Probabilistic grammars for equation discovery. Knowledge Based Systems, 224, Article 107077.","journal-title":"Knowledge Based Systems"},{"key":"6955_CR13","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s10994-007-5042-6","volume":"71","author":"W Bridewell","year":"2008","unstructured":"Bridewell, W., Langley, P., Todorovski, L., & D\u017eeroski, S. (2008). Inductive process modeling. Machine Learning, 71, 1\u201332.","journal-title":"Machine Learning"},{"key":"6955_CR14","unstructured":"Brugger, J., Cerrato, M., Richter, D., Derstroff, C., Maninger, D., Mezini, M., & Kramer, S. (2025). Neural-guided equation discovery. https:\/\/arxiv.org\/abs\/2503.16953"},{"key":"6955_CR15","doi-asserted-by":"crossref","unstructured":"Brugger, J., Pfanschilling, V., Richter, D., Mezini, M., & Kramer, S. (2025). Prompting neural-guided equation discovery based on residuals. In Proceedings of the 28th international conference on discovery science 2025. Accepted for publication.","DOI":"10.1007\/978-3-032-05461-6_7"},{"key":"6955_CR16","doi-asserted-by":"publisher","first-page":"3932","DOI":"10.1073\/pnas.1517384113","volume":"113","author":"SL Brunton","year":"2016","unstructured":"Brunton, S. L., Proctor, J. L., & Kutz, J. N. (2016). Discovering governing equations from data by sparse identification of nonlinear dynamical systems. PNAS, 113, 3932\u20133937.","journal-title":"PNAS"},{"issue":"6","key":"6955_CR17","doi-asserted-by":"publisher","first-page":"2526","DOI":"10.1214\/14-AOS1260","volume":"42","author":"P B\u00fchlmann","year":"2014","unstructured":"B\u00fchlmann, P., Peters, J., & Ernest, J. (2014). CAM: Causal additive models, high-dimensional order search and penalized regression. Annals of Statistics, 42(6), 2526\u20132556.","journal-title":"Annals of Statistics"},{"issue":"7815","key":"6955_CR18","doi-asserted-by":"publisher","first-page":"237","DOI":"10.1038\/s41586-020-2442-2","volume":"583","author":"B Burger","year":"2020","unstructured":"Burger, B., Maffettone, P. M., Gusev, V. V., Aitchison, C. M., Bai, Y., Wang, X., Li, X., Alston, B. M., Li, B., Clowes, R., Rankin, N., Harris, B., Sprick, R. S., & Cooper, A. I. (2020). A mobile robotic chemist. Nature, 583(7815), 237\u2013241. https:\/\/doi.org\/10.1038\/s41586-020-2442-2","journal-title":"Nature"},{"issue":"1","key":"6955_CR19","doi-asserted-by":"publisher","first-page":"982","DOI":"10.1137\/23M1561129","volume":"23","author":"A Bychkov","year":"2024","unstructured":"Bychkov, A., Issan, I., Pogudin, G., & Kr\u00e4mer, B. (2024). Exact and optimal quadratization of nonlinear finite-dimensional non-autonomous dynamical systems. SIAM Journal on Applied Dynamical Systems, 23(1), 982\u20131016.","journal-title":"SIAM Journal on Applied Dynamical Systems"},{"key":"6955_CR20","doi-asserted-by":"publisher","first-page":"136","DOI":"10.1016\/j.ecolmodel.2012.06.001","volume":"45","author":"D \u010cerepnalkoski","year":"2012","unstructured":"\u010cerepnalkoski, D., Ta\u0161kova, K., Todorovski, L., Atanasova, N., & D\u017eeroski, S. (2012). The influence of parameter fitting methods on model structure selection in automated modeling of aquatic ecosystems. Ecological Modelling, 45, 136\u2013165.","journal-title":"Ecological Modelling"},{"key":"6955_CR21","doi-asserted-by":"publisher","unstructured":"Cerrato, M., Schmitt, N., Baur, L., Finkelstein, E., Jukic, S., M\u00fcnzel, L., Paul, F.P., Pfannes, P., Rohr, B., Schellenberg, J., Wolf, P., & Kramer, S. (2024). Science-Gym: A simple testbed for ai-driven scientific discovery. In: Proceedings of the 26th International Conference on Discovery Science (DS). Lecture Notes in Computer Science, vol. 15243, pp. 229\u2013243. Springer, Pisa, Italy. https:\/\/doi.org\/10.1007\/978-3-031-78977-9_15","DOI":"10.1007\/978-3-031-78977-9_15"},{"issue":"10","key":"6955_CR22","doi-asserted-by":"publisher","first-page":"6059","DOI":"10.1021\/acscatal.0c04525","volume":"11","author":"Lea Chanussot","year":"2021","unstructured":"Chanussot, Lea. (2021). The open catalyst 2020 (oc20) dataset and community challenges. ACS Catalysis, 11(10), 6059\u20136072.","journal-title":"ACS Catalysis"},{"key":"6955_CR23","unstructured":"Chaushevska, M., Todorovski, L., Brence, J., & D\u017eeroski, S. (2022). Learning the probabilities in probabilistic context-free grammars for arithmetical expressions from equation corpora. In Proceedings of the Slovenian conference on artificial intelligence."},{"key":"6955_CR24","doi-asserted-by":"publisher","first-page":"433","DOI":"10.1038\/s43588-022-00281-6","volume":"2","author":"B Chen","year":"2022","unstructured":"Chen, B., Huang, K., Raghupathi, S., Chandratreya, I., Du, Q., & Lipson, H. (2022). Automated discovery of fundamental variables hidden in experimental data. Nature Computational Science, 2, 433\u2013442.","journal-title":"Nature Computational Science"},{"issue":"5","key":"6955_CR25","doi-asserted-by":"publisher","first-page":"792","DOI":"10.1109\/TEVC.2017.2683489","volume":"21","author":"Q Chen","year":"2017","unstructured":"Chen, Q., Zhang, M., & Xue, B. (2017). Feature selection to improve generalization of genetic programming for high-dimensional symbolic regression. IEEE Transactions on Evolutionary Computation, 21(5), 792\u2013806. https:\/\/doi.org\/10.1109\/TEVC.2017.2683489","journal-title":"IEEE Transactions on Evolutionary Computation"},{"key":"6955_CR26","first-page":"507","volume":"3","author":"DM Chickering","year":"2002","unstructured":"Chickering, D. M. (2002). Optimal structure identification with greedy search. Journal of Machine Learning Research, 3, 507\u2013554.","journal-title":"Journal of Machine Learning Research"},{"key":"6955_CR27","doi-asserted-by":"publisher","first-page":"129","DOI":"10.1613\/jair.295","volume":"4","author":"DA Cohn","year":"1996","unstructured":"Cohn, D. A., Ghahramani, Z., & Jordan, M. I. (1996). Active learning with statistical models. Journal of Artificial Intelligence Research, 4, 129\u2013145.","journal-title":"Journal of Artificial Intelligence Research"},{"key":"6955_CR28","first-page":"3921","volume":"15","author":"D Colombo","year":"2014","unstructured":"Colombo, D., & Maathuis, M. H. (2014). Order-independent constraint-based causal structure learning. Journal of Machine Learning Research, 15, 3921\u20133962.","journal-title":"Journal of Machine Learning Research"},{"key":"6955_CR29","doi-asserted-by":"publisher","unstructured":"Cornelio, C., Dash, S., Austel, V., Josephson, T.R., Goncalves, J., Clarkson, K.L., Megiddo, N., Khadir, B.E., & Horesh, L. (2023). Combining data and theory for derivable scientific discovery with AI-Descartes. Nature Communications 14. https:\/\/doi.org\/10.1038\/s41467-023-37236-y","DOI":"10.1038\/s41467-023-37236-y"},{"key":"6955_CR30","unstructured":"Cranmer, M. D. (2023). Interpretable machine learning for science with PySR and SymbolicRegression.jl. CoRR abs\/2305.01582. arXiv:2305.01582"},{"key":"6955_CR31","unstructured":"Cranmer, M.D., Sanchez-Gonzalez, A., Battaglia, P.W., Xu, R., Cranmer, K., Spergel, D.N., & Ho, S. (2020). Discovering symbolic models from deep learning with inductive biases. In Advances in Neural Information Processing Systems 33. https:\/\/dl.acm.org\/doi\/10.5555\/3495724.3497186"},{"key":"6955_CR32","doi-asserted-by":"publisher","unstructured":"Crochepierre, L., Boudjeloud-Assala, L., & Barbesant, V. (2022). Interactive reinforcement learning for symbolic regression from multi-format human-preference feedbacks. In: Raedt, L. D. (Ed.) International joint conference on artificial intelligence, pp. 5900\u20135903. https:\/\/doi.org\/10.24963\/IJCAI.2022\/849","DOI":"10.24963\/IJCAI.2022\/849"},{"key":"6955_CR33","doi-asserted-by":"publisher","unstructured":"Du, Y., Guo, X., Shehu, A., & Zhao, L. (2022). Interpretable molecular graph generation via monotonic constraints, pp. 73\u201381. https:\/\/doi.org\/10.1137\/1.9781611977172.9","DOI":"10.1137\/1.9781611977172.9"},{"key":"6955_CR34","doi-asserted-by":"publisher","first-page":"138","DOI":"10.1038\/s41524-020-00406-3","volume":"6","author":"A Dunn","year":"2020","unstructured":"Dunn, A., Wang, Q., Ganose, A., Dopp, D., & Jain, A. (2020). Benchmarking materials property prediction methods: The matbench test set and automatminer reference algorithm. npj Computational Materials, 6, 138.","journal-title":"npj Computational Materials"},{"key":"6955_CR35","doi-asserted-by":"crossref","unstructured":"D\u017eeroski, S., & Petrovski, I. (1994). Discovering dynamics with genetic programming. In Proceedings of the seventh European conference on machine learning, pp. 347\u2013350. Springer, Berlin.","DOI":"10.1007\/3-540-57868-4_70"},{"key":"6955_CR36","doi-asserted-by":"crossref","unstructured":"D\u017eeroski, S., & Todorovski, L. (1993). Discovering dynamics. In Proceedings of the tenth international conference on machine learning, pp. 97\u2013103. Morgan Kaufmann, Amherst, MA, USA.","DOI":"10.1016\/B978-1-55860-307-3.50019-8"},{"key":"6955_CR37","doi-asserted-by":"publisher","first-page":"1008","DOI":"10.1038\/d41586-025-00075-6","volume":"637","author":"M Eisenstein","year":"2025","unstructured":"Eisenstein, M. (2025). Self-driving laboratories, advanced immunotherapies and five more technologies to watch in 2025. Nature, 637, 1008\u20131011. https:\/\/doi.org\/10.1038\/d41586-025-00075-6","journal-title":"Nature"},{"key":"6955_CR38","unstructured":"Ellis, K., Morales, L., Sable-Meyer, M., Solar-Lezama, A., & Tenenbaum, J.B. (2018). Library learning for neurally-guided Bayesian program induction. In Advances in Neural Information Processing Systems, 31 (NeurIPS 2018), 31."},{"issue":"10","key":"6955_CR39","doi-asserted-by":"publisher","first-page":"7112","DOI":"10.1109\/TPAMI.2021.3095381","volume":"44","author":"A Elnaggar","year":"2022","unstructured":"Elnaggar, A., Heinzinger, M., Dallago, C., Rehawi, G., Wang, Y., Jones, L., Gibbs, T., Feher, T., Angerer, C., Steinegger, M., Bhowmik, D., & Rost, B. (2022). ProtTrans: Toward understanding the language of life through self-supervised learning. IEEE Transactions on Pattern Analysis and Machine Intelligence, 44(10), 7112\u20137127. https:\/\/doi.org\/10.1109\/TPAMI.2021.3095381","journal-title":"IEEE Transactions on Pattern Analysis and Machine Intelligence"},{"key":"6955_CR40","doi-asserted-by":"publisher","DOI":"10.1088\/2632-2153\/ad4a1e","volume":"5","author":"C Fox","year":"2024","unstructured":"Fox, C., Tran, N. D., Nacion, F. N., Sharlin, S., & Josephson, T. R. (2024). Incorporating background knowledge in symbolic regression using a computer algebra system. Machine Learning: Science and Technology, 5, Article 025057. https:\/\/doi.org\/10.1088\/2632-2153\/ad4a1e","journal-title":"Machine Learning: Science and Technology"},{"key":"6955_CR41","doi-asserted-by":"publisher","unstructured":"Franca, F. O., & Kronberger, G. (2023). Reducing overparameterization of symbolic regression models with equality saturation. In Proceedings of the 2023 genetic and evolutionary computation conference (GECCO), pp. 1064\u20131072. https:\/\/doi.org\/10.1145\/3583131.3590346","DOI":"10.1145\/3583131.3590346"},{"issue":"1","key":"6955_CR42","doi-asserted-by":"publisher","first-page":"29","DOI":"10.1023\/A:1005277613752","volume":"121","author":"D Freedman","year":"1999","unstructured":"Freedman, D., & Humphreys, P. (1999). Are there algorithms that discover causal structure? Synthese, 121(1), 29\u201354.","journal-title":"Synthese"},{"key":"6955_CR43","unstructured":"Gandhi, K., Li, M.Y., Goodyear, L., Li, L., Bhaskar, A., Zaman, M., & Goodman, N.D. (2025). BoxingGym: Benchmarking progress in automated experimental design and model discovery. Project page with environments: https:\/\/github.com\/kanishkg\/boxing-gym"},{"key":"6955_CR44","doi-asserted-by":"crossref","unstructured":"Ganzert, S., Guttmann, J., Steinmann, D., & Kramer, S. (2010). Equation discovery for model identification in respiratory mechanics of the mechanically ventilated human lung. In Proceedings of the 13th international conference on discovery science (DS 2010), pp. 296\u2013310. Springer, Berlin.","DOI":"10.1007\/978-3-642-16184-1_21"},{"key":"6955_CR45","doi-asserted-by":"crossref","unstructured":"Gao, M. L., Williams, J. P., & Kutz, J. N. (2025). Sparse identification of nonlinear dynamics and Koopman operators with Shallow Recurrent Decoder Networks. https:\/\/arxiv.org\/abs\/2501.13329.","DOI":"10.1073\/pnas.2508144123"},{"issue":"22","key":"6955_CR46","doi-asserted-by":"publisher","first-page":"6125","DOI":"10.1016\/j.cell.2024.09.022","volume":"187","author":"S Gao","year":"2024","unstructured":"Gao, S., Fang, A., Huang, Y., Giunchiglia, V., Noori, A., Schwarz, J. R., Ektefaie, Y., Kondic, J., & Zitnik, M. (2024). Empowering biomedical discovery with AI agents. Cell, 187(22), 6125\u20136151.","journal-title":"Cell"},{"issue":"5","key":"6955_CR47","doi-asserted-by":"publisher","first-page":"0268439","DOI":"10.1371\/journal.pone.0268439","volume":"17","author":"A Garcon","year":"2022","unstructured":"Garcon, A., Vexler, J., Budker, D., & Kramer, S. (2022). Deep neural networks to recover unknown physical parameters from oscillating time series. PLoS ONE, 17(5), 0268439.","journal-title":"PLoS ONE"},{"key":"6955_CR48","doi-asserted-by":"crossref","unstructured":"Grayeli, A., Sehgal, A., Costilla-Reyes, O., Cranmer, M.D., & Chaudhuri, S. (2024). Symbolic regression with a learned concept library. In Advances in neural information processing systems (NeurIPS).","DOI":"10.52202\/079017-1419"},{"key":"6955_CR49","doi-asserted-by":"publisher","first-page":"6971","DOI":"10.1126\/sciadv.aav6971","volume":"6","author":"R Guimer\u00e0","year":"2020","unstructured":"Guimer\u00e0, R., Reichardt, I., Aguilar-Mogas, A., Massucci, F. A., Miranda, M., Pallar\u00e8s, J., & Sales-Pardo, M. (2020). A Bayesian machine scientist to aid in the solution of challenging scientific problems. Science Advances, 6, 6971.","journal-title":"Science Advances"},{"key":"6955_CR50","doi-asserted-by":"publisher","first-page":"15202","DOI":"10.1038\/s41598-025-99785-0","volume":"15","author":"R Haddad","year":"2025","unstructured":"Haddad, R., Litsa, E. E., Liu, Z., Burkhardt, D., & Bhisetti, G. (2025). Targeted molecular generation with latent reinforcement learning. Scientific Reports, 15, 15202. https:\/\/doi.org\/10.1038\/s41598-025-99785-0","journal-title":"Scientific Reports"},{"key":"6955_CR51","doi-asserted-by":"publisher","unstructured":"Haider, C., Franca, F.O., Kronberger, G., & Burlacu, B. (2022). Comparing optimistic and pessimistic constraint evaluation in shape-constrained symbolic regression. In Proceedings of the 2022 genetic and evolutionary computation conference (GECCO), pp. 938\u2013945. https:\/\/doi.org\/10.1145\/3512290.3528714","DOI":"10.1145\/3512290.3528714"},{"key":"6955_CR52","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2022.109855","volume":"132","author":"C Haider","year":"2023","unstructured":"Haider, C., Franca, F. O., Burlacu, B., & Kronberger, G. (2023). Shape-constrained multi-objective genetic programming for symbolic regression. Applied Soft Computing, 132, Article 109855. https:\/\/doi.org\/10.1016\/j.asoc.2022.109855","journal-title":"Applied Soft Computing"},{"key":"6955_CR53","first-page":"689","volume":"21","author":"PO Hoyer","year":"2009","unstructured":"Hoyer, P. O., Janzing, D., Mooij, J. M., Peters, J., & Sch\u00f6lkopf, B. (2009). Nonlinear causal discovery with additive noise models. Advances in Neural Information Processing Systems (NeurIPS), 21, 689\u2013696.","journal-title":"Advances in Neural Information Processing Systems (NeurIPS)"},{"key":"6955_CR54","doi-asserted-by":"crossref","unstructured":"Huang, K.-M., & Zytkow, J. M. (1997). Discovering empirical equations from robot-collected data. In Proceedings of the 10th international symposium on foundations of intelligent systems (ISMIS 1997), pp. 287\u2013297. Springer, Charlotte, North Carolina, USA.","DOI":"10.1007\/3-540-63614-5_28"},{"issue":"10","key":"6955_CR55","doi-asserted-by":"publisher","first-page":"7523","DOI":"10.1007\/s00500-019-04379-4","volume":"24","author":"Z Huang","year":"2020","unstructured":"Huang, Z., Zhong, J., Feng, L., Mei, Y., & Cai, W. (2020). A fast parallel genetic programming framework with adaptively weighted primitives for symbolic regression. Soft Computing, 24(10), 7523\u20137539. https:\/\/doi.org\/10.1007\/s00500-019-04379-4","journal-title":"Soft Computing"},{"key":"6955_CR56","doi-asserted-by":"crossref","unstructured":"Jansen, P. E. A. (2024). DiscoveryWorld: A virtual environment for developing and evaluating automated scientific discovery agents. In Advances in neural information processing systems, vol. 37.","DOI":"10.52202\/079017-0324"},{"key":"6955_CR57","doi-asserted-by":"publisher","unstructured":"Jiang, N., & Xue, Y. (2023). Symbolic regression via control variable genetic programming. In Proceedings of the European conference on machine learning and principles and practice of knowledge discovery in databases (ECML PKDD 2023), pp. 178\u2013195. Springer, Berlin. https:\/\/doi.org\/10.1007\/978-3-031-43421-1_11","DOI":"10.1007\/978-3-031-43421-1_11"},{"key":"6955_CR58","doi-asserted-by":"publisher","unstructured":"Jiang, N., & Xue, Y. (2024). Racing control variable genetic programming for symbolic regression. In Wooldridge, M.J., Dy, J.G., Natarajan, S. (Eds.) Thirty-eighth AAAI conference on artificial intelligence, AAAI 2024, thirty-sixth conference on innovative applications of artificial intelligence, IAAI 2024, fourteenth symposium on educational advances in artificial intelligence, EAAI 2014, February 20\u201327, 2024, Vancouver, Canada, pp. 12901\u201312909. AAAI Press, Washington, DC. https:\/\/doi.org\/10.1609\/AAAI.V38I11.29187","DOI":"10.1609\/AAAI.V38I11.29187"},{"key":"6955_CR59","doi-asserted-by":"publisher","unstructured":"Jin, P., Huang, D., Zhang, R., Hu, X., Nan, Z., Du, Z., Guo, Q., & Chen, Y. (2023). Online symbolic regression with informative query. In: Williams, B., Chen, Y., Neville, J. (Eds.) Thirty-seventh AAAI conference on artificial intelligence, AAAI, pp. 5122\u20135130. AAAI Press, Washington, DC. https:\/\/doi.org\/10.1609\/AAAI.V37I4.25641","DOI":"10.1609\/AAAI.V37I4.25641"},{"key":"6955_CR60","doi-asserted-by":"publisher","unstructured":"Kahlmeyer, P., Fischer, M., & Giesen, J. (2025). Dimension reduction for symbolic regression. In AAAI, pp. 17707\u201317714. https:\/\/doi.org\/10.1609\/aaai.v39i17.33947","DOI":"10.1609\/aaai.v39i17.33947"},{"key":"6955_CR61","unstructured":"Kamienny, P., Lample, G., Lamprier, S., & Virgolin, M. (2023). Deep generative symbolic regression with Monte-Carlo-Tree-Search. In: Krause, A., Brunskill, E., Cho, K., Engelhardt, B., Sabato, S., Scarlett, J. (Eds.) International conference on machine learning, ICML 2023, 23\u201329 July 2023. Proceedings of Machine Learning Research, vol. 202, pp. 15655\u201315668."},{"key":"6955_CR62","doi-asserted-by":"crossref","unstructured":"Kamienny, P., d\u2019Ascoli, S., Lample, G., & Charton, F. (2022). End-to-end symbolic regression with transformers. In S. Koyejo, S. Mohamed, A. Agarwal, D. Belgrave, K. Cho, & A. Oh (Eds.), Advances in neural information processing systems 35: Annual conference on neural information processing systems 2022, NeurIPS 2022, November 28\u2013December 9, 2022. New Orleans: LA, USA.","DOI":"10.52202\/068431-0746"},{"key":"6955_CR63","doi-asserted-by":"publisher","unstructured":"Kim, J. T., Kim, S., & Petersen, B. K. (2020). An interactive visualization platform for deep symbolic regression. In International joint conference on artificial intelligence, IJCAI, pp. 5261\u20135263. https:\/\/doi.org\/10.24963\/IJCAI.2020\/763","DOI":"10.24963\/IJCAI.2020\/763"},{"key":"6955_CR64","first-page":"127","volume-title":"Artificial intelligence in science: Challenges, opportunities and the future of research","author":"R King","year":"2023","unstructured":"King, R., Peter, O., & Courtney, P. (2023). Robot scientists: From Adam to Eve to Genesis. In T. Science & O. Innovation (Eds.), Artificial intelligence in science: Challenges, opportunities and the future of research (pp. 127\u2013138). Paris: OECD Publishing."},{"issue":"5923","key":"6955_CR65","doi-asserted-by":"publisher","first-page":"85","DOI":"10.1126\/science.1165620","volume":"324","author":"RD King","year":"2009","unstructured":"King, R. D., Rowland, J., Oliver, S. G., Young, M., Aubrey, W., Byrne, E., Liakata, M., Markham, M., Pir, P., Soldatova, L. N., Sparkes, A., Whelan, K. E., & Clare, A. (2009). The automation of science. Science, 324(5923), 85\u201389.","journal-title":"Science"},{"issue":"12","key":"6955_CR66","doi-asserted-by":"publisher","first-page":"13211","DOI":"10.1609\/aaai.v38i12.29221","volume":"38","author":"KE Kirchoff","year":"2024","unstructured":"Kirchoff, K. E., Maxfield, T., Tropsha, A., & Gomez, S. M. (2024). SALSA: Semantically-aware latent space autoencoder. Proceedings of the AAAI Conference on Artificial Intelligence, 38(12), 13211\u201313219. https:\/\/doi.org\/10.1609\/aaai.v38i12.29221","journal-title":"Proceedings of the AAAI Conference on Artificial Intelligence"},{"key":"6955_CR67","doi-asserted-by":"crossref","unstructured":"Kitano, H. (2021). Nobel Turing challenge: Creating the engine for scientific discovery. npj Systems Biology and Applications 7(29).","DOI":"10.1038\/s41540-021-00189-3"},{"key":"6955_CR68","first-page":"1","volume":"24","author":"N Kovachki","year":"2023","unstructured":"Kovachki, N., Li, Z., Liu, B., Azizzadenesheli, K., Bhattacharya, K., Stuart, A. M., & Anandkumar, A. (2023). Neural operator: Learning maps between function spaces with applications to pdes. Journal of Machine Learning Research, 24, 1\u201397.","journal-title":"Journal of Machine Learning Research"},{"key":"6955_CR69","doi-asserted-by":"publisher","first-page":"87","DOI":"10.1007\/BF00175355","volume":"4","author":"JR Koza","year":"1994","unstructured":"Koza, J. R. (1994). Genetic programming as a means for programming computers by natural selection. Statistics and Computing, 4, 87\u2013112.","journal-title":"Statistics and Computing"},{"key":"6955_CR70","doi-asserted-by":"publisher","unstructured":"Kubalik, J., Alibekov, E., Zegklitz, J., & Babuska, R. (2016). Hybrid single node genetic programming for symbolic regression. In Transactions on computational collective intelligence XXIV, lecture notes in computer science, pp. 61\u201382. https:\/\/doi.org\/10.1007\/978-3-662-53525-7_4","DOI":"10.1007\/978-3-662-53525-7_4"},{"key":"6955_CR71","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2021.115210","volume":"182","author":"J Kubal\u00edk","year":"2021","unstructured":"Kubal\u00edk, J., Derner, E., & Babuska, R. (2021). Multi-objective symbolic regression for physics-aware dynamic modeling. Expert Systems with Applications, 182, Article 115210. https:\/\/doi.org\/10.1016\/j.eswa.2021.115210","journal-title":"Expert Systems with Applications"},{"key":"6955_CR72","doi-asserted-by":"publisher","first-page":"61481","DOI":"10.1109\/ACCESS.2023.3287397","volume":"11","author":"J Kubal\u00edk","year":"2023","unstructured":"Kubal\u00edk, J., Derner, E., & Babuska, R. (2023). Toward physically plausible data-driven models: A novel neural network approach to symbolic regression. IEEE Access, 11, 61481\u201361501. https:\/\/doi.org\/10.1109\/ACCESS.2023.3287397","journal-title":"IEEE Access"},{"key":"6955_CR73","unstructured":"Lachapelle, S., Brouillard, P., Deleu, T., & Lacoste-Julien, S. (2020). Gradient-based neural DAG learning. In International conference on learning representations (ICLR)."},{"key":"6955_CR74","unstructured":"Langley, P. (1977). BACON: A production system that discovers empirical laws. In Proceedings of the 5th international joint conference on artificial intelligence (IJCAI 1977), p. 344."},{"key":"6955_CR75","doi-asserted-by":"crossref","unstructured":"Langley, P. (2024). Integrated systems for computational scientific discovery. In Proceedings of the thirty-eighth AAAI conference on artificial intelligence (AAAI-24), pp. 22598\u201322606. AAAI Press, Washington, DC.","DOI":"10.1609\/aaai.v38i20.30269"},{"issue":"4","key":"6955_CR76","doi-asserted-by":"publisher","first-page":"72","DOI":"10.1609\/aimag.v42i4.15089","volume":"42","author":"P Langley","year":"2022","unstructured":"Langley, P. (2022). Agents of exploration and discovery. AI Magazine, 42(4), 72\u201382.","journal-title":"AI Magazine"},{"key":"6955_CR77","doi-asserted-by":"publisher","DOI":"10.7551\/mitpress\/6090.001.0001","volume-title":"Scientific discovery: Computational explorations of the creative process","author":"PW Langley","year":"1987","unstructured":"Langley, P. W., Simon, H. A., Bradshaw, G., & Zytkow, J. M. (1987). Scientific discovery: Computational explorations of the creative process. Cambridge, MA: MIT Press."},{"key":"6955_CR78","unstructured":"Levina, E., & Bickel, P.J. (2004). Maximum likelihood estimation of intrinsic dimension. In Advances in neural information processing systems 17, pp. 777\u2013784."},{"key":"6955_CR79","unstructured":"Li, Z., Ji, J., & Zhang, Y. (2021). From Kepler to Newton: Explainable AI for science. arXiv preprint. arXiv:2111.12210"},{"key":"6955_CR80","unstructured":"Li, Z., Kovachki, N.B., Azizzadenesheli, K., Liu, B., Bhattacharya, K., Stuart, A.M., & Anandkumar, A. (2021). Fourier neural operator for parametric partial differential equations. In Proceedings of the 9th international conference on learning representations (ICLR 2021)."},{"key":"6955_CR81","unstructured":"Liu, Z., Wang, Y., Vaidya, S., Ruehle, F., Halverson, J., Solja\u010di\u0107, M., Hou, T. Y., & Tegmark, M. (2024). KAN: Kolmogorov\u2013Arnold networks. arXiv preprint arXiv:2404.19756."},{"key":"6955_CR82","unstructured":"Lu, D., Tan, X., Xu, R., Yao, T., Qu, C., Chu, W., Xu, Y., & Qi, Y. (2025). SCP-116K: A high-quality problem-solution dataset and a generalized pipeline for automated extraction in the higher education science domain. https:\/\/arxiv.org\/abs\/2501.15587"},{"issue":"3","key":"6955_CR83","doi-asserted-by":"publisher","first-page":"218","DOI":"10.1038\/s42256-021-00302-5","volume":"3","author":"L Lu","year":"2021","unstructured":"Lu, L., Jin, P., Pang, G., Zhang, Z., & Karniadakis, G. E. (2021). Learning nonlinear operators via DeepONet based on the universal approximation theorem of operators. Nature Machine Intelligence, 3(3), 218\u2013229.","journal-title":"Nature Machine Intelligence"},{"issue":"1","key":"6955_CR84","doi-asserted-by":"publisher","first-page":"4950","DOI":"10.1038\/s41467-018-07210-0","volume":"9","author":"B Lusch","year":"2018","unstructured":"Lusch, B., Kutz, J. N., & Brunton, S. L. (2018). Deep learning for universal linear embeddings of nonlinear dynamics. Nature Communications, 9(1), 4950.","journal-title":"Nature Communications"},{"key":"6955_CR85","unstructured":"Majumder, B.P., Surana, H., Agarwal, D., Dalvi\u00a0Mishra, B., Meena, A., Prakhar, A., Vora, T., Khot, T., Sabharwal, A., & Clark, P. (2024). DiscoveryBench: Towards data-driven discovery with large language models. Dataset and code available on GitHub (https:\/\/github.com\/allenai\/discoverybench) and HuggingFace."},{"key":"6955_CR86","doi-asserted-by":"crossref","unstructured":"Makke, N., & Chawla, S. (2024). Interpretable scientific discovery with symbolic regression: A reviews. Artificial Intelligence Review, 57(2).","DOI":"10.1007\/s10462-023-10622-0"},{"key":"6955_CR87","doi-asserted-by":"crossref","unstructured":"Merler, M., Haitsiukevich, K., Dainese, N., & Marttinen, P. (2024). In-Context Symbolic Regression: Leveraging Large Language Models for Function Discovery. In: Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 4: Student Research Workshop), pp. 589\u2013606. Association for Computational Linguistics, Bangkok, Thailand. https:\/\/doi.org\/10.18653\/v1\/2024.acl-srw.49 . https:\/\/aclanthology.org\/2024.acl-srw.49 Accessed 2025-03-17","DOI":"10.18653\/v1\/2024.acl-srw.49"},{"key":"6955_CR88","doi-asserted-by":"publisher","unstructured":"Meyerson, E., Nelson, M. J., Bradley, H., Gaier, A., Moradi, A., Hoover, A. K., & Lehman, J. (2023). Language Model Crossover: Variation through Few-Shot Prompting. arXiv. Version Number: 3. https:\/\/doi.org\/10.48550\/ARXIV.2302.12170. https:\/\/arxiv.org\/abs\/2302.12170 Accessed 2025-03-27.","DOI":"10.48550\/ARXIV.2302.12170"},{"issue":"11","key":"6955_CR89","doi-asserted-by":"publisher","first-page":"4563","DOI":"10.1007\/s10994-023-06400-2","volume":"112","author":"S Me\u017enar","year":"2023","unstructured":"Me\u017enar, S., D\u017eeroski, S., & Todorovski, L. (2023). Efficient generator of mathematical expressions for symbolic regression. Machine Learning, 112(11), 4563\u20134596.","journal-title":"Machine Learning"},{"key":"6955_CR90","unstructured":"Mundhenk, T. N., Landajuela, M., Glatt, R., Santiago, C. P., Faissol, D. M., & Petersen, B. K. (2021). Symbolic regression via neural-guided genetic programming population seeding. arXiv preprint arXiv:2111.00053."},{"key":"6955_CR91","doi-asserted-by":"crossref","unstructured":"Musslick, S., Bartlett, L.K., Chandramouli, S.H., Dubova, M., Gobet, F., Griffiths, T.L., Hullman, J., King, R.D., Kutz, J.N., Lucas, C.G., Mahesh, S., Pestilli, F., Sloman, S.J., & Holmes, W.R. (2025). Automating the practice of science: Opportunities, challenges, and implications. PNAS 122(5).","DOI":"10.1073\/pnas.2401238121"},{"key":"6955_CR92","doi-asserted-by":"publisher","unstructured":"Neshatian, K., & Varn, L. (2017). On the existence of feature bundles and their effect on symbolic regression algorithms. In 2017 IEEE symposium series on computational intelligence (SSCI), pp. 2974\u20132981. https:\/\/doi.org\/10.1109\/SSCI.2017.8285419.","DOI":"10.1109\/SSCI.2017.8285419"},{"key":"6955_CR93","doi-asserted-by":"crossref","unstructured":"Ohana, R., McCabe, M., Meyer, L.T., Morel, R., Agocs, F.J., Beneitez, M., Berger, M., Burkhart, B., Dalziel, S.B., Fielding, D.B., Fortunato, D., Goldberg, J.A., Hirashima, K., Jiang, Y.-F., Kerswell, R., Maddu, S., Miller, J.M., Mukhopadhyay, P., Nixon, S.S., Shen, J., Watteaux, R., Blancard, B.R.-S., Rozet, F., Parker, L.H., Cranmer, M., & Ho, S. (2024). The Well: a large-scale collection of diverse physics simulations for machine learning. In: The Thirty-eight Conference on neural information processing systems datasets and benchmarks track. https:\/\/openreview.net\/forum?id=00Sx577BT3","DOI":"10.52202\/079017-1430"},{"key":"6955_CR94","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511803161","volume-title":"Causality: Models, reasoning, and inference","author":"J Pearl","year":"2009","unstructured":"Pearl, J. (2009). Causality: Models, reasoning, and inference (2nd ed.). New York: Cambridge University Press.","edition":"2"},{"issue":"5","key":"6955_CR95","doi-asserted-by":"publisher","first-page":"947","DOI":"10.1111\/rssb.12167","volume":"78","author":"J Peters","year":"2016","unstructured":"Peters, J., B\u00fchlmann, P., & Meinshausen, N. (2016). Causal inference by using invariant prediction: Identification and confidence intervals. Journal of the Royal Statistical Society: Series B, 78(5), 947\u20131012. https:\/\/doi.org\/10.1111\/rssb.12167","journal-title":"Journal of the Royal Statistical Society: Series B"},{"key":"6955_CR96","unstructured":"Petersen, B.K., Larma, M.L., Mundhenk, T.N., Santiago, C.P., Kim, S.K., & Kim, J.T. (2021). Deep symbolic regression: Recovering mathematical expressions from data via risk-seeking policy gradients. In Proceedings of the 9th international conference on learning representations (ICLR 2021)."},{"issue":"1","key":"6955_CR97","doi-asserted-by":"publisher","first-page":"2009","DOI":"10.5555\/2627435.2670315","volume":"15","author":"J Peters","year":"2014","unstructured":"Peters, J., Mooij, J. M., Janzing, D., & Sch\u00f6lkopf, B. (2014). Causal discovery with continuous additive noise models. Journal of Machine Learning Research, 15(1), 2009\u20132053. https:\/\/doi.org\/10.5555\/2627435.2670315","journal-title":"Journal of Machine Learning Research"},{"key":"6955_CR98","doi-asserted-by":"publisher","unstructured":"Prieschl, S., Girardi, D., & Kronberger, G. (2019). Using ontologies to express prior knowledge for genetic programming. In 3rd International cross-domain conference for machine learning and knowledge extraction, pp. 362\u2013376. https:\/\/doi.org\/10.1007\/978-3-030-29726-8_23","DOI":"10.1007\/978-3-030-29726-8_23"},{"issue":"15","key":"6955_CR99","doi-asserted-by":"publisher","first-page":"2016239118","DOI":"10.1073\/pnas.2016239118","volume":"118","author":"A Rives","year":"2021","unstructured":"Rives, A., Meier, J., Sercu, T., Goyal, S., Lin, Z., Liu, J., Guo, D., Ott, M., Zitnick, C. L., Ma, J., & Fergus, R. (2021). Biological structure and function emerge from scaling unsupervised learning to 250 million protein sequences. Proceedings of the National Academy of Sciences, 118(15), 2016239118. https:\/\/doi.org\/10.1073\/pnas.2016239118","journal-title":"Proceedings of the National Academy of Sciences"},{"key":"6955_CR100","unstructured":"Rong, Y., Bian, Y., Xu, T., Xie, W., Wei, Y., Huang, W., & Huang, J. (2020). Self-supervised graph transformer on large-scale molecular data. In Advances in neural information processing systems, vol. 33. https:\/\/proceedings.neurips.cc\/paper\/2020\/hash\/94aef38441efa3380a3bed3faf1f9d5d-Abstract.html."},{"issue":"189","key":"6955_CR101","doi-asserted-by":"publisher","first-page":"20210821","DOI":"10.1098\/rsif.2021.0821","volume":"19","author":"K Roper","year":"2022","unstructured":"Roper, K., Abdel-Rehim, A., Hubbard, S., Carpenter, M., Rzhetsky, A., Soldatova, L. N., & King, R. D. (2022). Testing the reproducibility and robustness of the cancer biology literature by robot. Journal of the Royal Society Interface, 19(189), 20210821. https:\/\/doi.org\/10.1098\/rsif.2021.0821","journal-title":"Journal of the Royal Society Interface"},{"key":"6955_CR102","unstructured":"Sahoo, S., Lampert, C., & Martius, G. (2018). Learning Equations for Extrapolation and Control. In Proceedings of the 35th international conference on machine learning, Stockholm, Sweden, pp. 4442\u20134450. ISSN: 2640-3498. https:\/\/proceedings.mlr.press\/v80\/sahoo18a.html. Accessed 2024-02-19"},{"key":"6955_CR103","doi-asserted-by":"publisher","DOI":"10.1016\/j.asoc.2021.107198","volume":"104","author":"AS Sambo","year":"2021","unstructured":"Sambo, A. S., Azad, R. M. A., Kovalchuk, Y., Indramohan, V. P., & Shah, H. (2021). Evolving simple and accurate symbolic regression models via asynchronous parallel computing. Applied Soft Computing, 104, Article 107198. https:\/\/doi.org\/10.1016\/j.asoc.2021.107198","journal-title":"Applied Soft Computing"},{"key":"6955_CR104","first-page":"1855","volume":"28","author":"M Scanagatta","year":"2015","unstructured":"Scanagatta, M., Campos, C. P., Corani, G., & Zaffalon, M. (2015). Learning bayesian networks with thousands of variables. Advances in Neural Information Processing Systems (NeurIPS), 28, 1855\u20131863.","journal-title":"Advances in Neural Information Processing Systems (NeurIPS)"},{"issue":"5923","key":"6955_CR105","doi-asserted-by":"publisher","first-page":"81","DOI":"10.1126\/science.1165893","volume":"324","author":"M Schmidt","year":"2009","unstructured":"Schmidt, M., & Lipson, H. (2009). Distilling free-form natural laws from experimental data. Science, 324(5923), 81\u201385. https:\/\/doi.org\/10.1126\/science.1165893","journal-title":"Science"},{"key":"6955_CR106","unstructured":"Seifner, P., Cvejoski, K., K\u00f6rner, A., & Sanchez, R. (2025). Zero-shot imputation with foundation inference models for dynamical systems. In Proceedings of the international conference on learning representations (ICLR). To appear. https:\/\/openreview.net\/forum?id=NPSZ7V1CCY"},{"key":"6955_CR107","doi-asserted-by":"publisher","unstructured":"Sharlin, S., & Josephson, T. R. (2024). In context learning and reasoning for symbolic regression with large language models. arXiv. Version Number: 2. https:\/\/doi.org\/10.48550\/ARXIV.2410.17448. https:\/\/arxiv.org\/abs\/2410.17448 Accessed 2025-03-17","DOI":"10.48550\/ARXIV.2410.17448"},{"key":"6955_CR108","first-page":"2003","volume":"7","author":"S Shimizu","year":"2006","unstructured":"Shimizu, S., Hoyer, P. O., Hyv\u00e4rinen, A., & Kerminen, A. (2006). A linear non-Gaussian acyclic model for causal discovery. Journal of Machine Learning Research, 7, 2003\u20132030.","journal-title":"Journal of Machine Learning Research"},{"key":"6955_CR109","doi-asserted-by":"publisher","unstructured":"Shojaee, P., Meidani, K., Farimani, A. B., & Reddy, C. K. (2023). Transformer-based planning for symbolic regression https:\/\/doi.org\/10.48550\/ARXIV.2303.06833. Publisher: arXiv Version Number: 4. Accessed 2023-08-09","DOI":"10.48550\/ARXIV.2303.06833"},{"key":"6955_CR110","doi-asserted-by":"publisher","unstructured":"Shojaee, P., Meidani, K., Gupta, S., Farimani, A.B., & Reddy, C.K. (2024). LLM-SR: Scientific equation discovery via programming with large language models. arXiv. Version Number: 2. https:\/\/doi.org\/10.48550\/ARXIV.2404.18400. https:\/\/arxiv.org\/abs\/2404.18400 Accessed 2024-08-05","DOI":"10.48550\/ARXIV.2404.18400"},{"key":"6955_CR111","unstructured":"Sparkes, A., Aubrey, W., Byrne, E., Clare, A., Khan, M.N., Liakata, M., Markham, M., Rowland, J., Soldatova, L.N., Whelan, K.E., Young, M., & King, R.D. (2021). Towards robot scientists for autonomous scientific discovery. Automated Experimentation 2(1)."},{"key":"6955_CR112","volume-title":"Causation, prediction, and search","author":"P Spirtes","year":"2000","unstructured":"Spirtes, P., Glymour, C., & Scheines, R. (2000). Causation, prediction, and search (2nd ed.). Cambridge, MA: MIT Press.","edition":"2"},{"key":"6955_CR113","doi-asserted-by":"publisher","unstructured":"Sun, C., Shen, S., Tao, W., Xue, D., & Zhou, Z. (2025). Noise-resilient symbolic regression with dynamic gating reinforcement learning, 39, 20690\u201320698. https:\/\/doi.org\/10.1609\/aaai.v39i19.34280","DOI":"10.1609\/aaai.v39i19.34280"},{"key":"6955_CR114","volume-title":"Reinforcement learning: An introduction","author":"R Sutton","year":"2018","unstructured":"Sutton, R., & Barto, A. (2018). Reinforcement learning: An introduction. Cambridge, MA: MIT Press."},{"key":"6955_CR115","doi-asserted-by":"publisher","first-page":"3930","DOI":"10.1038\/s41467-025-59288-y","volume":"16","author":"Y Tian","year":"2025","unstructured":"Tian, Y., Zhou, W., Viscione, M., Dong, H., Kammer, D. S., & Fink, O. (2025). Interactive symbolic regression with co-design mechanism through offline reinforcement learning. Nature Communications, 16, 3930. https:\/\/doi.org\/10.1038\/s41467-025-59288-y","journal-title":"Nature Communications"},{"key":"6955_CR116","unstructured":"Todorovski, L., & D\u017eeroski, S. (1997). Declarative bias in equation discovery. In Proceedings of the fourteenth international conference on machine learning, pp. 376\u2013384. Morgan Kaufmann, San Francisco, CA."},{"key":"6955_CR117","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1016\/j.ecolmodel.2005.10.001","volume":"194","author":"L Todorovski","year":"2006","unstructured":"Todorovski, L., & D\u017eeroski, S. (2006). Integrating knowledge-driven and data-driven approaches to modeling. Ecological Modelling, 194, 3\u201313.","journal-title":"Ecological Modelling"},{"issue":"1\u20133","key":"6955_CR118","doi-asserted-by":"publisher","first-page":"71","DOI":"10.1016\/S0304-3800(98)00135-5","volume":"113","author":"L Todorovski","year":"1998","unstructured":"Todorovski, L., Dzeroski, S., & Kompare, B. (1998). Modelling and prediction of phytoplankton growth with equation discovery. Ecological Modelling, 113(1\u20133), 71\u201381. https:\/\/doi.org\/10.1016\/S0304-3800(98)00135-5","journal-title":"Ecological Modelling"},{"key":"6955_CR119","unstructured":"Udrescu, S.-M., Tan, A., Feng, J., Neto, O., Wu, T., & Tegmark, M. (2020). AI Feynman 2.0: Pareto-optimal symbolic regression exploiting graph modularity. In Advances in neural information processing systems, 33."},{"issue":"16","key":"6955_CR120","doi-asserted-by":"publisher","first-page":"2631","DOI":"10.1126\/sciadv.aay2631","volume":"6","author":"S-M Udrescu","year":"2020","unstructured":"Udrescu, S.-M., & Tegmark, M. (2020). AI Feynman: A physics-inspired method for symbolic regression. Science Advances, 6(16), 2631.","journal-title":"Science Advances"},{"key":"6955_CR121","doi-asserted-by":"publisher","first-page":"91","DOI":"10.1007\/s10710-010-9121-2","volume":"12","author":"NQ Uy","year":"2011","unstructured":"Uy, N. Q., Hoai, N. X., O\u2019Neill, M., McKay, R. I., & Galv\u00e1n-L\u00f3pez, E. (2011). Semantically-based crossover in genetic programming: Application to real-valued symbolic regression. Genetic Programming and Evolvable Machines, 12, 91\u2013119.","journal-title":"Genetic Programming and Evolvable Machines"},{"key":"6955_CR122","doi-asserted-by":"publisher","unstructured":"Valipour, M., You, B., Panju, M., & Ghodsi, A. (2021). SymbolicGPT: A generative transformer model for symbolic regression. arXiv. arXiv:2106.14131 [cs] version: 1. https:\/\/doi.org\/10.48550\/arXiv.2106.14131.","DOI":"10.48550\/arXiv.2106.14131"},{"key":"6955_CR123","first-page":"109","volume":"24","author":"S Villar","year":"2023","unstructured":"Villar, S., Yao, W., Hogg, D. W., Blum-Smith, B., & Dumitrascu, B. (2023). Dimensionless machine learning: Imposing exact units equivariance. Journal of Machine Learning Research, 24, 109.","journal-title":"Journal of Machine Learning Research"},{"key":"6955_CR124","doi-asserted-by":"crossref","unstructured":"Wang, R., Jansen, P., C\u00f4t\u00e9, M.-A., & Ammanabrolu, P. (2022). ScienceWorld: Is your Agent Smarter than a 5th Grader?. https:\/\/arxiv.org\/abs\/2203.07540","DOI":"10.18653\/v1\/2022.emnlp-main.775"},{"issue":"2","key":"6955_CR125","doi-asserted-by":"publisher","first-page":"196","DOI":"10.1007\/s41019-024-00270-x","volume":"10","author":"C Wang","year":"2025","unstructured":"Wang, C., Chen, Q., Xue, B., & Zhang, M. (2025). Improving generalization of genetic programming for high-dimensional symbolic regression with Shapley value based feature selection. Data Science and Engineering, 10(2), 196\u2013211. https:\/\/doi.org\/10.1007\/s41019-024-00270-x","journal-title":"Data Science and Engineering"},{"issue":"9","key":"6955_CR126","doi-asserted-by":"publisher","first-page":"3503","DOI":"10.1021\/acs.jctc.4c00249","volume":"20","author":"D Wang","year":"2024","unstructured":"Wang, D., Wang, Y., Evans, L., & Tiwary, P. (2024). From latent dynamics to meaningful representations. Journal of Chemical Theory and Computation, 20(9), 3503\u20133513. https:\/\/doi.org\/10.1021\/acs.jctc.4c00249","journal-title":"Journal of Chemical Theory and Computation"},{"key":"6955_CR127","unstructured":"Washio, T., & Motoda, H. (1997). Discovering admissible models of complex systems based on scale-types and identity constraints. In Proceedings of the fifteenth international joint conference on artificial intelligence (IJCAI 1997), pp. 810\u2013819."},{"key":"6955_CR128","doi-asserted-by":"publisher","DOI":"10.1142\/6019","volume-title":"Fantastic realities: 49 Mind journeys and a trip to stockholm","author":"F Wilczek","year":"2006","unstructured":"Wilczek, F., & Devine, B. (2006). Fantastic realities: 49 Mind journeys and a trip to stockholm. Singapore: World Scientific."},{"issue":"104","key":"6955_CR130","doi-asserted-by":"publisher","first-page":"20141289","DOI":"10.1098\/rsif.2014.1289","volume":"12","author":"K Williams","year":"2015","unstructured":"Williams, K., Bilsland, E., Sparkes, A., Aubrey, W., Young, M., Soldatova, L. N., De Grave, K., Ramon, J., Clare, M., Sirawaraporn, W., Oliver, S. G., & King, R. D. (2015). Cheaper faster drug development validated by the repositioning of drugs against neglected tropical diseases. Journal of the Royal Society Interface, 12(104), 20141289. https:\/\/doi.org\/10.1098\/rsif.2014.1289","journal-title":"Journal of the Royal Society Interface"},{"key":"6955_CR131","first-page":"1466","volume":"26","author":"J Xiang","year":"2013","unstructured":"Xiang, J., & Kim, S. (2013). A* Lasso for learning a sparse Bayesian network structure for continuous variables. Advances in Neural Information Processing Systems (NeurIPS), 26, 1466\u20131474.","journal-title":"Advances in Neural Information Processing Systems (NeurIPS)"},{"key":"6955_CR132","unstructured":"Xu, Y., Liu, Y., & Sun, H. (2024). Reinforcement symbolic regression machine. In International conference on learning representations, ICLR."},{"issue":"8","key":"6955_CR133","doi-asserted-by":"publisher","first-page":"3370","DOI":"10.1021\/acs.jcim.9b00237","volume":"59","author":"K Yang","year":"2019","unstructured":"Yang, K., Swanson, K., Jin, W., Coley, C. W., Eiden, P., Gao, H., Guzman-Perez, A., Hopper, T., Kelley, B., Mathea, M., Palmer, A., Settels, V., Jaakkola, T. S., Jensen, K. F., & Barzilay, R. (2019). Analyzing learned molecular representations for property prediction. Journal of Chemical Information and Modeling, 59(8), 3370\u20133388. https:\/\/doi.org\/10.1021\/acs.jcim.9b00237","journal-title":"Journal of Chemical Information and Modeling"},{"key":"6955_CR134","unstructured":"Ying, C., Cai, T., Luo, S., Zheng, S., Ke, G., He, D., Shen, Y., & Liu, T.-Y. (2021). Do transformers really perform badly for graph representation? In Advances in neural information processing systems, vol. 34. https:\/\/proceedings.neurips.cc\/paper\/2021\/hash\/f1c1592588411002af340cbaedd6fc33-Abstract.html"},{"key":"6955_CR135","unstructured":"Yu, Y., Chen, J., Gao, T., & Yu, M. (2019). DAG-GNN: DAG structure learning with graph neural networks. In Proceedings of the 36th international conference on machine learning (ICML). Proceedings of Machine Learning Research, pp. 7154\u20137163."},{"key":"6955_CR136","unstructured":"Zhang, K., & Lipson, H. (2024). Aligning ai-driven discovery with human intuition. arXiv:2410.07397 [cs.LG]"},{"issue":"4","key":"6955_CR137","doi-asserted-by":"publisher","first-page":"336","DOI":"10.1093\/nsr\/nwad336.","volume":"11","author":"R Zhang","year":"2023","unstructured":"Zhang, R., Meng, Q., & Ma, Z.-M. (2023). Deciphering and integrating invariants for neural operator learning with various physical mechanisms. National Science Review, 11(4), 336. https:\/\/doi.org\/10.1093\/nsr\/nwad336. Published online: 29 Dec 2023.","journal-title":"National Science Review"},{"key":"6955_CR138","doi-asserted-by":"publisher","first-page":"1011","DOI":"10.1007\/S11229-015-0673-9","volume":"193","author":"J Zhang","year":"2016","unstructured":"Zhang, J., & Spirtes, P. (2016). The three faces of faithfulness. Synthese, 193, 1011\u20131027. https:\/\/doi.org\/10.1007\/S11229-015-0673-9","journal-title":"Synthese"},{"key":"6955_CR139","unstructured":"Zheng, X., Dan, C., Aragam, B., Ravikumar, P., & Xing, E.P. (2020). Learning sparse nonparametric DAGs. In S. Chiappa, R. Calandra, (Eds.) The 23rd international conference on artificial intelligence and statistics, AISTATS 2020, 26\u201328 August 2020, Online [Palermo, Sicily, Italy]. Proceedings of Machine Learning Research, vol. 108, pp. 3414\u20133425."},{"key":"6955_CR140","first-page":"9472","volume":"31","author":"X Zheng","year":"2018","unstructured":"Zheng, X., Aragam, B., Ravikumar, P. K., & Xing, E. P. (2018). DAGs with no tears: Continuous optimization for structure learning. Advances in Neural Information Processing Systems (NeurIPS), 31, 9472\u20139483.","journal-title":"Advances in Neural Information Processing Systems (NeurIPS)"},{"issue":"60","key":"6955_CR141","first-page":"1","volume":"25","author":"Y Zheng","year":"2024","unstructured":"Zheng, Y., Huang, B., Chen, W., Ramsey, J., Gong, M., Cai, R., Shimizu, S., Spirtes, P., & Zhang, K. (2024). Causal-learn: Causal discovery in Python. Journal of Machine Learning Research, 25(60), 1\u20138.","journal-title":"Journal of Machine Learning Research"},{"key":"6955_CR142","doi-asserted-by":"publisher","unstructured":"Zhong, L., Zhong, J., & Lu, C. (2021). A comparative analysis of dimensionality reduction methods for genetic programming to solve high-dimensional symbolic regression problems. In 2021 IEEE international conference on systems, man, and cybernetics (SMC), pp. 476\u2013483. https:\/\/doi.org\/10.1109\/SMC52423.2021.9658595","DOI":"10.1109\/SMC52423.2021.9658595"},{"key":"6955_CR143","doi-asserted-by":"publisher","unstructured":"Zhou, M., Song, H., Ye, W., Wang, W., & Lai, Z. (2025). Parameter estimation of structural dynamics with neural operators enabled surrogate modeling. Mechanical Systems and Signal Processing 237, 112914. https:\/\/doi.org\/10.1016\/j.ymssp.2025.112914","DOI":"10.1016\/j.ymssp.2025.112914"},{"key":"6955_CR144","unstructured":"Zhu, S., Ng, I., & Chen, Z. (2020). Causal discovery with reinforcement learning. In International conference on learning representations (ICLR). https:\/\/openreview.net\/forum?id=S1g2skStPB"},{"key":"6955_CR145","unstructured":"Zytkow, J.M., Zhu, J., & Hussam, A. (1990). Automated discovery in a chemistry laboratory. In Proceedings of the 8th National Conference on artificial intelligence (AAAI 1990), pp. 889\u2013894. AAAI Press\/MIT Press, Washington, DC."}],"container-title":["Machine Learning"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-025-06955-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10994-025-06955-2","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10994-025-06955-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,24]],"date-time":"2026-06-24T06:32:32Z","timestamp":1782282752000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10994-025-06955-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,4,29]]},"references-count":143,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2026,5]]}},"alternative-id":["6955"],"URL":"https:\/\/doi.org\/10.1007\/s10994-025-06955-2","relation":{},"ISSN":["0885-6125","1573-0565"],"issn-type":[{"value":"0885-6125","type":"print"},{"value":"1573-0565","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,4,29]]},"assertion":[{"value":"24 April 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 September 2025","order":2,"name":"revised","label":"Revised","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 December 2025","order":3,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"29 April 2026","order":4,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no Conflict of interest.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"109"}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1016_2F0005_1098_2883_2990046_8 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T10:48:48Z","timestamp":1793270928638,"version":"4.5.0"},"reference-count":30,"publisher":"Elsevier BV","issue":"6","license":[{"start":{"date-parts":[[1983,11,1]],"date-time":"1983-11-01T00:00:00Z","timestamp":436492800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Automatica"],"published-print":{"date-parts":[[1983,11]]},"DOI":"10.1016\/0005-1098(83)90046-8","type":"journal-article","created":{"date-parts":[[2003,3,14]],"date-time":"2003-03-14T08:27:27Z","timestamp":1047630447000},"page":"775-779","source":"Crossref","is-referenced-by-count":1714,"title":["Ironies of automation"],"prefix":"10.1016","volume":"19","author":[{"given":"Lisanne","family":"Bainbridge","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/0005-1098(83)90046-8_BIB1","series-title":"Measurement of Human Resources","first-page":"165","article-title":"The representation of working storage and its use in the organisation of behaviour","author":"Bainbridge","year":"1975"},{"key":"10.1016\/0005-1098(83)90046-8_BIB2","series-title":"Measurement of Human Resources","first-page":"259","article-title":"Mathematical equations or processing routines?","author":"Bainbridge","year":"1981"},{"key":"10.1016\/0005-1098(83)90046-8_BIB3","series-title":"Proc. 6th IFAC Congress","article-title":"Man's role in control systems","author":"Bibby","year":"1975"},{"key":"10.1016\/0005-1098(83)90046-8_BIB4","series-title":"Proc. 17th Ann. Conf. on Manual Control","first-page":"669","article-title":"A model for the control mode man-computer interface","author":"Chafin","year":"1981"},{"key":"10.1016\/0005-1098(83)90046-8_BIB5","doi-asserted-by":"crossref","first-page":"769","DOI":"10.1109\/TSMC.1979.4310128","article-title":"Adaptive allocation of decision making responsibility between human and computer in multitask situations","volume":"SMC-9","author":"Chu","year":"1979","journal-title":"IEEE Trans. Syst., Man & Cybern."},{"key":"10.1016\/0005-1098(83)90046-8_BIB6","doi-asserted-by":"crossref","first-page":"63","DOI":"10.1146\/annurev.ps.30.020179.000431","article-title":"Human memory","volume":"30","author":"Craik","year":"1979","journal-title":"Ann. Rev. Psychol."},{"key":"10.1016\/0005-1098(83)90046-8_BIB7","series-title":"Measurement of Human Resources","first-page":"487","article-title":"The user's role in automated fault detection and system recovery","author":"Dellner","year":"1981"},{"key":"10.1016\/0005-1098(83)90046-8_BIB8","series-title":"Measurement of Human Resources","first-page":"553","article-title":"Training for fault diagnosis in industrial process plant","author":"Duncan","year":"1981"},{"key":"10.1016\/0005-1098(83)90046-8_BIB9","doi-asserted-by":"crossref","first-page":"627","DOI":"10.1080\/00140137508931499","article-title":"A simulator and training technique for diagnosing plant failures from control panels","volume":"18","author":"Duncan","year":"1975","journal-title":"Ergonomics"},{"key":"10.1016\/0005-1098(83)90046-8_BIB10","series-title":"Proc. 1st European Ann. Conf. on Human Decision Making and Manual Control","first-page":"228","article-title":"Current research needs in manual control","author":"Edwards","year":"1981"},{"key":"10.1016\/0005-1098(83)90046-8_BIB11","series-title":"The Human Operator in Process Control","year":"1974"},{"key":"10.1016\/0005-1098(83)90046-8_BIB12","series-title":"Case Studies in Automation Related to Humanization of Work","first-page":"23","article-title":"Human control tasks: A comparative study in different man-machine systems","author":"Ekkers","year":"1979"},{"key":"10.1016\/0005-1098(83)90046-8_BIB13","doi-asserted-by":"crossref","first-page":"153","DOI":"10.1109\/TSMC.1977.4309679","article-title":"Real-time determination of how a human has allocated his attention between control and monitoring tasks","volume":"SMC-7","author":"Enstrom","year":"1977","journal-title":"IEEE Trans. Syst., Man & Cybern."},{"key":"10.1016\/0005-1098(83)90046-8_BIB14","series-title":"NATO Symposium on Human Detection and Diagnosis of System Failures","article-title":"Verbal presentation","author":"Ephrath","year":"1980"},{"key":"10.1016\/0005-1098(83)90046-8_BIB15","series-title":"Measurement of Human Resources","first-page":"143","article-title":"Monitoring vs. man-in-the-loop detection of aircraft control failures","author":"Ephrath","year":"1981"},{"key":"10.1016\/0005-1098(83)90046-8_BIB16","series-title":"Measurement of Human Resources","first-page":"433","article-title":"Discriminative display support for process operators","author":"Goodstein","year":"1981"},{"key":"10.1016\/0005-1098(83)90046-8_BIB17","series-title":"E\/REP\/054\/77","article-title":"Trends in operator-process communication development","author":"Jervis","year":"1977"},{"key":"10.1016\/0005-1098(83)90046-8_BIB18","series-title":"Proc. 1st European Ann. Conf. on Human Decision Making and Manual Control","first-page":"142","article-title":"Problem solving behaviour of pilots in abnormal and emergency situations","author":"Johannsen","year":"1981"},{"key":"10.1016\/0005-1098(83)90046-8_BIB19","series-title":"Proc. 17th Ann. Conf. on Manual Control","first-page":"655","article-title":"Interruption as a test of the user-computer interface","author":"Kreifeldt","year":"1981"},{"key":"10.1016\/0005-1098(83)90046-8_BIB20","series-title":"Selected Papers on Human Factors in the Design and Use of Control Systems","first-page":"174","article-title":"Researches on the measurement of human performance","author":"Mackworth","year":"1950"},{"key":"10.1016\/0005-1098(83)90046-8_BIB21","series-title":"Measurement of Human Resources","first-page":"575","article-title":"A fault-finding training programme for continuous plant operators","author":"Marshall","year":"1981"},{"key":"10.1016\/0005-1098(83)90046-8_BIB22","doi-asserted-by":"crossref","first-page":"493","DOI":"10.1037\/0033-2909.86.3.493","article-title":"Review and conceptual analysis of the employee turnover process","volume":"86","author":"Mobley","year":"1979","journal-title":"Psychol. Bull."},{"key":"10.1016\/0005-1098(83)90046-8_BIB23","series-title":"RISO-M-2192","article-title":"On the structure of knowledge\u2014a morphology of mental models in a man-machine system context","author":"Rasmussen","year":"1979"},{"key":"10.1016\/0005-1098(83)90046-8_BIB24","series-title":"Proc. 1st European Ann. Conf. on Human Decision Making and Manual Control","first-page":"70","article-title":"Coping with complexity","author":"Rasmussen","year":"1981"},{"key":"10.1016\/0005-1098(83)90046-8_BIB25","series-title":"Human Detection and Diagnosis of System Failures","year":"1981"},{"key":"10.1016\/0005-1098(83)90046-8_BIB26","doi-asserted-by":"crossref","first-page":"71","DOI":"10.1145\/356835.356839","article-title":"Human-computer interaction in the control of dynamic systems","volume":"13","author":"Rouse","year":"1981","journal-title":"Computing Surveys"},{"key":"10.1016\/0005-1098(83)90046-8_BIB27","article-title":"A simulator study of the interaction of pilot workload with errors, vigilance, and decisions","author":"Ruffell-Smith","year":"1979","journal-title":"NASA TM-78482"},{"key":"10.1016\/0005-1098(83)90046-8_BIB28","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1016\/0003-6870(72)90003-8","article-title":"Human intervention and full automation in control systems","volume":"3","author":"Sinaiko","year":"1972","journal-title":"Appl. Ergonomics"},{"key":"10.1016\/0005-1098(83)90046-8_BIB29","series-title":"Measurement of Human Resources","first-page":"37","article-title":"Commercial air crew detection of system failures: state of the art and future trends","author":"Thompson","year":"1981"},{"key":"10.1016\/0005-1098(83)90046-8_BIB30","doi-asserted-by":"crossref","first-page":"995","DOI":"10.1080\/00140138008924809","article-title":"Flight-deck automation: promises and problems","volume":"23","author":"Wiener","year":"1980","journal-title":"Ergonomics"}],"container-title":["Automatica"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:0005109883900468?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:0005109883900468?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2019,3,26]],"date-time":"2019-03-26T19:20:08Z","timestamp":1553628008000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/0005109883900468"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[1983,11]]},"references-count":30,"journal-issue":{"issue":"6","published-print":{"date-parts":[[1983,11]]}},"alternative-id":["0005109883900468"],"URL":"https:\/\/doi.org\/10.1016\/0005-1098(83)90046-8","relation":{},"ISSN":["0005-1098"],"issn-type":[{"value":"0005-1098","type":"print"}],"subject":[],"published":{"date-parts":[[1983,11]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1016_2Fj_cortex_2012_12_016 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T00:49:40Z","timestamp":1793148580916,"version":"4.5.0"},"reference-count":16,"publisher":"Elsevier BV","issue":"3","license":[{"start":{"date-parts":[[2013,3,1]],"date-time":"2013-03-01T00:00:00Z","timestamp":1362096000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Cortex"],"published-print":{"date-parts":[[2013,3]]},"DOI":"10.1016\/j.cortex.2012.12.016","type":"journal-article","created":{"date-parts":[[2012,12,25]],"date-time":"2012-12-25T22:49:17Z","timestamp":1356475757000},"page":"609-610","source":"Crossref","is-referenced-by-count":419,"title":["Registered Reports: A new publishing initiative at\u00a0Cortex"],"prefix":"10.1016","volume":"49","author":[{"given":"Christopher D.","family":"Chambers","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"issue":"6","key":"10.1016\/j.cortex.2012.12.016_bib1","doi-asserted-by":"crossref","first-page":"543","DOI":"10.1177\/1745691612459060","article-title":"The rules of game called psychological science","volume":"7","author":"Bakker","year":"2012","journal-title":"Perspectives on Psychological Science"},{"issue":"2","key":"10.1016\/j.cortex.2012.12.016_bib2","doi-asserted-by":"crossref","first-page":"234","DOI":"10.1037\/0021-9010.61.2.234","article-title":"A statistical power analysis of applied psychological research","volume":"61","author":"Chase","year":"1976","journal-title":"Journal of Applied Psychology"},{"issue":"1","key":"10.1016\/j.cortex.2012.12.016_bib3","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.cortex.2012.11.003","article-title":"Nothing endures but change","volume":"49","author":"Della Sala","year":"2013","journal-title":"Cortex"},{"issue":"4","key":"10.1016\/j.cortex.2012.12.016_bib4","doi-asserted-by":"crossref","first-page":"e10068","DOI":"10.1371\/journal.pone.0010068","article-title":"\u201cPositive\u201d results increase down the hierarchy of the sciences","volume":"5","author":"Fanelli","year":"2010","journal-title":"PLOS One"},{"issue":"6","key":"10.1016\/j.cortex.2012.12.016_bib5","doi-asserted-by":"crossref","first-page":"661","DOI":"10.1177\/1745691612462587","article-title":"The long way from \u03b1-error control to validity proper: Problems with a short-sighted false-positive debate","volume":"7","author":"Fiedler","year":"2012","journal-title":"Perspectives on Psychological Science"},{"issue":"8","key":"10.1016\/j.cortex.2012.12.016_bib6","doi-asserted-by":"crossref","first-page":"e124","DOI":"10.1371\/journal.pmed.0020124","article-title":"Why most published research findings are false","volume":"2","author":"Ioannidis","year":"2005","journal-title":"PLOS Medicine"},{"issue":"5","key":"10.1016\/j.cortex.2012.12.016_bib7","doi-asserted-by":"crossref","first-page":"524","DOI":"10.1177\/0956797611430953","article-title":"Measuring the prevalence of questionable research practices with incentives for truth telling","volume":"23","author":"John","year":"2012","journal-title":"Psychological Science"},{"issue":"11","key":"10.1016\/j.cortex.2012.12.016_bib8","doi-asserted-by":"crossref","first-page":"2271","DOI":"10.1080\/17470218.2012.711335","article-title":"A peculiar prevalence of p values just below .05","volume":"65","author":"Masicampo","year":"2012","journal-title":"Quarterly Journal of Experimental Psychology"},{"issue":"6","key":"10.1016\/j.cortex.2012.12.016_bib9","doi-asserted-by":"crossref","first-page":"643","DOI":"10.1177\/1745691612459519","article-title":"The nine circles of scientific hell","volume":"7","author":"Neuroskeptic","year":"2012","journal-title":"Perspectives on Psychological Science"},{"issue":"6","key":"10.1016\/j.cortex.2012.12.016_bib10","doi-asserted-by":"crossref","first-page":"615","DOI":"10.1177\/1745691612459058","article-title":"Scientific Utopia II: Restructuring incentives and practices to promote truth over publishability","volume":"7","author":"Nosek","year":"2012","journal-title":"Perspectives on Psychological Science"},{"issue":"6","key":"10.1016\/j.cortex.2012.12.016_bib11","doi-asserted-by":"crossref","first-page":"528","DOI":"10.1177\/1745691612465253","article-title":"Editors' Introduction to the special section on replicability in psychological science: A crisis of confidence?","volume":"7","author":"Pashler","year":"2012","journal-title":"Perspectives on Psychological Science"},{"issue":"3","key":"10.1016\/j.cortex.2012.12.016_bib12","doi-asserted-by":"crossref","first-page":"638","DOI":"10.1037\/0033-2909.86.3.638","article-title":"The file drawer problem and tolerance for null results","volume":"86","author":"Rosenthal","year":"1979","journal-title":"Psychological Bulletin"},{"issue":"11","key":"10.1016\/j.cortex.2012.12.016_bib13","doi-asserted-by":"crossref","first-page":"1359","DOI":"10.1177\/0956797611417632","article-title":"False-positive psychology: Undisclosed flexibility in data collection and analysis allows presenting anything as significant","volume":"22","author":"Simmons","year":"2011","journal-title":"Psychological Science"},{"issue":"2","key":"10.1016\/j.cortex.2012.12.016_bib14","doi-asserted-by":"crossref","first-page":"207","DOI":"10.1016\/S0895-4356(99)00161-4","article-title":"Publication bias in meta-analysis: Its causes and consequences","volume":"53","author":"Thornton","year":"2000","journal-title":"Journal of Clinical Epidemiology"},{"key":"10.1016\/j.cortex.2012.12.016_bib15","doi-asserted-by":"crossref","first-page":"218","DOI":"10.3389\/fpsyg.2012.00218","article-title":"Replication unreliability in psychology: elusive phenomena or \u201celusive\u201d statistical power?","volume":"3","author":"Tressoldi","year":"2012","journal-title":"Frontiers in Psychology"},{"issue":"5","key":"10.1016\/j.cortex.2012.12.016_bib16","doi-asserted-by":"crossref","first-page":"779","DOI":"10.3758\/BF03194105","article-title":"A practical solution to the pervasive problems of p values","volume":"14","author":"Wagenmakers","year":"2007","journal-title":"Psychonomic Bulletin and Review"}],"container-title":["Cortex"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0010945212003735?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0010945212003735?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2018,10,19]],"date-time":"2018-10-19T17:53:39Z","timestamp":1539971619000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0010945212003735"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,3]]},"references-count":16,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2013,3]]}},"alternative-id":["S0010945212003735"],"URL":"https:\/\/doi.org\/10.1016\/j.cortex.2012.12.016","relation":{},"ISSN":["0010-9452"],"issn-type":[{"value":"0010-9452","type":"print"}],"subject":[],"published":{"date-parts":[[2013,3]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1016_2Fj_future_2014_10_008 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T02:06:54Z","timestamp":1793239614931,"version":"4.5.0"},"reference-count":101,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2015,5,1]],"date-time":"2015-05-01T00:00:00Z","timestamp":1430438400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2015,5,1]],"date-time":"2015-05-01T00:00:00Z","timestamp":1430438400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2016,2,9]],"date-time":"2016-02-09T00:00:00Z","timestamp":1454976000000},"content-version":"am","delay-in-days":284,"URL":"http:\/\/www.elsevier.com\/open-access\/userlicense\/1.0\/"},{"start":{"date-parts":[[2015,5,1]],"date-time":"2015-05-01T00:00:00Z","timestamp":1430438400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-017"},{"start":{"date-parts":[[2015,5,1]],"date-time":"2015-05-01T00:00:00Z","timestamp":1430438400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-037"},{"start":{"date-parts":[[2015,5,1]],"date-time":"2015-05-01T00:00:00Z","timestamp":1430438400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-012"},{"start":{"date-parts":[[2015,5,1]],"date-time":"2015-05-01T00:00:00Z","timestamp":1430438400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2015,5,1]],"date-time":"2015-05-01T00:00:00Z","timestamp":1430438400000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/doi.org\/10.15223\/policy-004"}],"funder":[{"DOI":"10.13039\/100007523","name":"Division of Advanced Cyberinfrastructure","doi-asserted-by":"publisher","award":["0722019"],"award-info":[{"award-number":["0722019"]}],"id":[{"id":"10.13039\/100007523","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100007523","name":"Division of Advanced Cyberinfrastructure","doi-asserted-by":"publisher","award":["1148515"],"award-info":[{"award-number":["1148515"]}],"id":[{"id":"10.13039\/100007523","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Future Generation Computer Systems"],"published-print":{"date-parts":[[2015,5]]},"DOI":"10.1016\/j.future.2014.10.008","type":"journal-article","created":{"date-parts":[[2014,10,29]],"date-time":"2014-10-29T18:44:17Z","timestamp":1414608257000},"page":"17-35","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":766,"special_numbering":"C","title":["Pegasus, a workflow management system for science automation"],"prefix":"10.1016","volume":"46","author":[{"given":"Ewa","family":"Deelman","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Karan","family":"Vahi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Gideon","family":"Juve","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mats","family":"Rynge","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Scott","family":"Callaghan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Philip J.","family":"Maechling","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rajiv","family":"Mayani","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Weiwei","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1720-0928","authenticated-orcid":false,"given":"Rafael","family":"Ferreira da Silva","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5444-7439","authenticated-orcid":false,"given":"Miron","family":"Livny","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kent","family":"Wenger","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.future.2014.10.008_br000005","unstructured":"LIGO Scientific Collaboration. URL: http:\/\/ligo.org."},{"key":"10.1016\/j.future.2014.10.008_br000010","unstructured":"Southern California Earthquake Center. URL: http:\/\/scec.org."},{"key":"10.1016\/j.future.2014.10.008_br000015","unstructured":"National Virtual Observatory. URL: http:\/\/us-vo.org."},{"key":"10.1016\/j.future.2014.10.008_br000020","series-title":"Grid Resource Management","first-page":"99","article-title":"Workflow management in GriPhyN","author":"Deelman","year":"2004"},{"key":"10.1016\/j.future.2014.10.008_br000025","doi-asserted-by":"crossref","unstructured":"V. Nefedova, R. Jacob, I. Foster, Z. Liu, Y. Liu, E. Deelman, G. Mehta, M.-H. Su, K. Vahi, Automating climate science: Large ensemble simulations on the teragrid with the griphyn virtual data system, in: 2nd IEEE International Conference on e-Science and Grid Computing, E-SCIENCE 06, 2006. http:\/\/dx.doi.org\/doi:10.1109\/E-SCIENCE.2006.30.","DOI":"10.1109\/E-SCIENCE.2006.261116"},{"key":"10.1016\/j.future.2014.10.008_br000030","unstructured":"E. Deelman, C. Kesselman, G. Mehta, L. Meshkat, L. Pearlman, K. Blackburn, P. Ehrens, A. Lazzarini, R. Williams, S. Koranda, GriPhyN and LIGO, building a virtual data grid for gravitational wave scientists, in: 11th IEEE International Symposium on High Performance Distributed Computing, HPDC 02, 2002."},{"key":"10.1016\/j.future.2014.10.008_br000035","unstructured":"D. Gunter, E. Deelman, T. Samak, C. Brooks, M. Goode, G. Juve, G. Mehta, P. Moraes, F. Silva, M. Swany, K. Vahi, Online workflow management and performance analysis with stampede, in: Network and Service Management, CNSM, 2011 7th International Conference on, 2011, pp. 1\u201310."},{"issue":"3","key":"10.1016\/j.future.2014.10.008_br000040","doi-asserted-by":"crossref","first-page":"38","DOI":"10.1109\/MCSE.2008.82","article-title":"Provenance: the bridge between experiments and data","volume":"10","author":"Miles","year":"2008","journal-title":"Comput. Sci. Eng."},{"key":"10.1016\/j.future.2014.10.008_br000045","doi-asserted-by":"crossref","unstructured":"D.J. Kuck, R.H. Kuhn, D.A. Padua, B. Leasure, M. Wolfe, Dependence graphs and compiler optimizations, in: 8th ACM SIGPLAN-SIGACT Symposium on Principles of Programming Languages, POPL 81, 1981. http:\/\/dx.doi.org\/10.1145\/567532.567555.","DOI":"10.1145\/567532.567555"},{"issue":"6","key":"10.1016\/j.future.2014.10.008_br000050","doi-asserted-by":"crossref","first-page":"686","DOI":"10.1109\/71.242154","article-title":"On the granularity and clustering of directed acyclic task graphs","volume":"4","author":"Gerasoulis","year":"1993","journal-title":"IEEE Trans. Parallel Distrib. Syst."},{"issue":"4","key":"10.1016\/j.future.2014.10.008_br000055","doi-asserted-by":"crossref","first-page":"406","DOI":"10.1145\/344588.344618","article-title":"Static scheduling algorithms for allocating directed task graphs to multiprocessors","volume":"31","author":"Kwok","year":"1999","journal-title":"ACM Comput. Surv."},{"key":"10.1016\/j.future.2014.10.008_br000060","unstructured":"J. Kurzak, J. Dongarra, Fully dynamic scheduler for numerical computing on multicore processors, LAPACK Working Note 220, 2009."},{"key":"10.1016\/j.future.2014.10.008_br000065","series-title":"Transaction Processing: Concepts and Techniques","author":"Gray","year":"1993"},{"issue":"4","key":"10.1016\/j.future.2014.10.008_br000070","doi-asserted-by":"crossref","first-page":"602","DOI":"10.1145\/319628.319650","article-title":"Query processing in a system for distributed databases (SDD-1)","volume":"6","author":"Bernstein","year":"1981","journal-title":"ACM Trans. Database Syst."},{"key":"10.1016\/j.future.2014.10.008_br000075","series-title":"Introduction to Graph Theory","author":"West","year":"2001"},{"key":"10.1016\/j.future.2014.10.008_br000080","series-title":"Data Structures and Network Algorithms","author":"Tarjan","year":"1998"},{"key":"10.1016\/j.future.2014.10.008_br000085","doi-asserted-by":"crossref","unstructured":"A. Ramakrishnan, G. Singh, H. Zhao, E. Deelman, R. Sakellariou, K. Vahi, K. Blackburn, D. Meyers, M. Samidi, Scheduling data-intensive workflows onto storage-constrained distributed resources, in: IEEE International Symposium on Cluster Computing and the Grid, CCGrid, 2007. http:\/\/dx.doi.org\/10.1109\/CCGRID.2007.101.","DOI":"10.1109\/CCGRID.2007.101"},{"issue":"4","key":"10.1016\/j.future.2014.10.008_br000090","first-page":"249","article-title":"Optimizing workflow data footprint","volume":"15","author":"Singh","year":"2007","journal-title":"Sci. Program."},{"issue":"6","key":"10.1016\/j.future.2014.10.008_br000095","doi-asserted-by":"crossref","first-page":"428","DOI":"10.1016\/j.jcss.2009.11.005","article-title":"Scaling up workflow-based applications","volume":"76","author":"Callaghan","year":"2010","journal-title":"J. Comput. System Sci."},{"key":"10.1016\/j.future.2014.10.008_br000100","series-title":"The Grid: Blueprint for a New Computing Infrastructure","year":"1999"},{"issue":"4","key":"10.1016\/j.future.2014.10.008_br000105","doi-asserted-by":"crossref","first-page":"50","DOI":"10.1145\/1721654.1721672","article-title":"A view of cloud computing","volume":"53","author":"Armbrust","year":"2010","journal-title":"Commun. ACM"},{"key":"10.1016\/j.future.2014.10.008_br000110","unstructured":"M. McLennan, S. Clark, F. McKenna, E. Deelman, M. Rynge, K. Vahi, D. Kearney, C. Song, Bringing scientific workflow to the masses via pegasus and hubzero, in: International Workshop on Science Gateways, 2013."},{"issue":"1","key":"10.1016\/j.future.2014.10.008_br000115","doi-asserted-by":"crossref","first-page":"62","DOI":"10.1109\/MIS.2010.9","article-title":"Wings: intelligent workflow-based design of computational experiments","volume":"26","author":"Gil","year":"2011","journal-title":"IEEE Intell. Syst."},{"key":"10.1016\/j.future.2014.10.008_br000120","doi-asserted-by":"crossref","unstructured":"S. Marru, L. Gunathilake, C. Herath, P. Tangchaisin, M. Pierce, C. Mattmann, R. Singh, T. Gunarathne, E. Chinthaka, R. Gardler, A. Slominski, A. Douma, S. Perera, S. Weerawarana, Apache airavata: a framework for distributed applications and computational workflows, in: Workshop on Gateway Computing Environments, GCE\u201911, 2011. http:\/\/dx.doi.org\/10.1145\/2110486.2110490.","DOI":"10.1145\/2110486.2110490"},{"key":"10.1016\/j.future.2014.10.008_br000125","unstructured":"Steven Cox, GRAYSON Git\/README, URL: https:\/\/github.com\/stevencox\/grayson."},{"issue":"2\u20134","key":"10.1016\/j.future.2014.10.008_br000130","doi-asserted-by":"crossref","first-page":"323","DOI":"10.1002\/cpe.938","article-title":"Distributed computing in practice: the condor experience","volume":"17","author":"Thain","year":"2005","journal-title":"Concurr. Comput.: Pract. Exper."},{"issue":"3","key":"10.1016\/j.future.2014.10.008_br000135","doi-asserted-by":"crossref","first-page":"200","DOI":"10.1177\/109434200101500302","article-title":"The anatomy of the grid: enabling scalable virtual organizations","volume":"15","author":"Foster","year":"2001","journal-title":"Int. J. High Perform. Comput. Appl."},{"key":"10.1016\/j.future.2014.10.008_br000140","unstructured":"Amazon.com, Inc., Elastic Compute Cloud (EC2), URL: http:\/\/aws.amazon.com\/ec2."},{"key":"10.1016\/j.future.2014.10.008_br000145","unstructured":"Pegasus 4.3 user guide, URL: http:\/\/pegasus.isi.edu\/wms\/docs\/4.3\/."},{"issue":"3\u20134","key":"10.1016\/j.future.2014.10.008_br000150","doi-asserted-by":"crossref","first-page":"367","DOI":"10.1007\/s00024-010-0161-6","article-title":"Cybershake: a physics-based seismic hazard model for southern California","volume":"168","author":"Graves","year":"2011","journal-title":"Pure Appl. Geophys."},{"key":"10.1016\/j.future.2014.10.008_br000155","unstructured":"Sax xerces Java parser. URL: http:\/\/xerces.apache.org\/xerces2-j\/."},{"key":"10.1016\/j.future.2014.10.008_br000160","doi-asserted-by":"crossref","unstructured":"K. Czajkowski, I.T. Foster, N.T. Karonis, C. Kesselman, S. Martin, W. Smith, S. Tuecke, A resource management architecture for metacomputing systems, in: Proceedings of the Workshop on Job Scheduling Strategies for Parallel Processing, 1998.","DOI":"10.1007\/BFb0053981"},{"key":"10.1016\/j.future.2014.10.008_br000165","unstructured":"P. Andreetto, S.A. Borgia, A. Dorigo, A. Gianelle, M. Marzolla, M. Mordacchini, M. Sgaravatto, S. Andreozzi, M. Cecchi, V. Ciaschini, T. Ferrari, F. Giacomini, R. Lops, E. Ronchieri, G. Fiorentino, V. Martelli, M. Mezzadri, E. Molinari, F. Prelz, CREAM: a simple, grid-accessible, job management system for local computational resources, in: Conference on Computing in High Energy Physics, CHEP, 2006."},{"key":"10.1016\/j.future.2014.10.008_br000170","unstructured":"Simple Linux utility for resource management. URL: http:\/\/slurm.schedmd.com\/."},{"key":"10.1016\/j.future.2014.10.008_br000175","volume":"5","author":"Bayucan","year":"1999"},{"key":"10.1016\/j.future.2014.10.008_br000180","unstructured":"IBM platform computing template: LSF. URL: http:\/\/www.platform.com\/Products\/platform-lsf."},{"key":"10.1016\/j.future.2014.10.008_br000185","unstructured":"Oracle Grid Engine. URL: http:\/\/www.oracle.com\/us\/products\/tools\/oracle-grid-engine-075549.html."},{"key":"10.1016\/j.future.2014.10.008_br000190","doi-asserted-by":"crossref","unstructured":"D. Weitzel, I. Sfiligoi, B. Bockelman, F. Wuerthwein, D. Fraser, D. Swanson, Accessing opportunistic resources with bosco, in: Computing in High Energy and Nuclear Physics, 2013.","DOI":"10.1088\/1742-6596\/513\/3\/032105"},{"key":"10.1016\/j.future.2014.10.008_br000195","unstructured":"Extreme science and engineering discovery environment (XSEDE). URL: http:\/\/www.xsede.org."},{"key":"10.1016\/j.future.2014.10.008_br000200","unstructured":"European grid infrastructure (EGI). URL: http:\/\/www.egi.eu."},{"key":"10.1016\/j.future.2014.10.008_br000205","doi-asserted-by":"crossref","first-page":"012057","DOI":"10.1088\/1742-6596\/78\/1\/012057","article-title":"The open science grid","volume":"78","author":"Pordes","year":"2007","journal-title":"J. Phys. Conf. Ser."},{"issue":"6","key":"10.1016\/j.future.2014.10.008_br000210","doi-asserted-by":"crossref","first-page":"062044","DOI":"10.1088\/1742-6596\/119\/6\/062044","article-title":"glideinWMS\u2014a generic pilot-based workload management system","volume":"119","author":"Sfiligoi","year":"2008","journal-title":"J. Phys. Conf. Ser."},{"key":"10.1016\/j.future.2014.10.008_br000215","unstructured":"Amazon.com, Inc., Amazon Web Services (AWS), URL: http:\/\/aws.amazon.com."},{"key":"10.1016\/j.future.2014.10.008_br000220","unstructured":"Futuregrid. URL: https:\/\/www.futuregrid.org\/."},{"key":"10.1016\/j.future.2014.10.008_br000225","unstructured":"Openstack. URL: https:\/\/www.openstack.org\/."},{"key":"10.1016\/j.future.2014.10.008_br000230","doi-asserted-by":"crossref","unstructured":"K. Vahi, M. Rynge, G. Juve, R. Mayani, E. Deelman, Rethinking data management for big data scientific workflows, in: IEEE International Conference on Big Data, 2013. http:\/\/dx.doi.org\/10.1109\/BigData.2013.6691724.","DOI":"10.1109\/BigData.2013.6691724"},{"key":"10.1016\/j.future.2014.10.008_br000235","doi-asserted-by":"crossref","unstructured":"M. Rynge, S. Callaghan, E. Deelman, G. Juve, G. Mehta, K. Vahi, P.J. Maechling, Enabling large-scale scientific workflows on petascale resources using MPI master\/worker, in: 1st Conference of the Extreme Science and Engineering Discovery Environment: Bridging from the eXtreme to the Campus and Beyond, XSEDE\u201912, 2012, pp. 49:1\u201349:8. http:\/\/dx.doi.org\/10.1145\/2335755.2335846.","DOI":"10.1145\/2335755.2335846"},{"key":"10.1016\/j.future.2014.10.008_br000240","unstructured":"Kraken. URL: http:\/\/www.nics.tennessee.edu\/computing-resources\/kraken."},{"key":"10.1016\/j.future.2014.10.008_br000245","unstructured":"Blue waters: sustained petascale computing. URL: https:\/\/bluewaters.ncsa.illinois.edu\/."},{"key":"10.1016\/j.future.2014.10.008_br000250","doi-asserted-by":"crossref","unstructured":"M. Rynge, G. Juve, G. Mehta, E. Deelman, G. Berriman, K. Larson, B. Holzman, S. Callaghan, I. Sfiligoi, F. Wurthwein, Experiences using glideinWMS and the corral frontend across cyberinfrastructures, in: E-Science, e-Science, 2011 IEEE 7th International Conference on, 2011, pp. 311\u2013318. http:\/\/dx.doi.org\/10.1109\/eScience.2011.50.","DOI":"10.1109\/eScience.2011.50"},{"issue":"1","key":"10.1016\/j.future.2014.10.008_br000255","first-page":"1","article-title":"iRODS primer: integrated rule-oriented data system","volume":"2","author":"Rajasekar","year":"2010","journal-title":"Synth. Lect. Inform. Concepts Retr. Serv."},{"key":"10.1016\/j.future.2014.10.008_br000260","unstructured":"Amazon simple storage service. URL: http:\/\/aws.amazon.com\/s3\/."},{"key":"10.1016\/j.future.2014.10.008_br000265","doi-asserted-by":"crossref","unstructured":"R. Fielding, J. Gettys, J. Mogul, H. Frystyk, L. Masinter, P. Leach, T. Berners-Lee, Hypertext transfer protocol\u2014http\/1.1, 1999.","DOI":"10.17487\/rfc2616"},{"key":"10.1016\/j.future.2014.10.008_br000270","series-title":"SC\u201905","first-page":"54","article-title":"The globus striped GridFTP framework and server","author":"Allcock","year":"2005"},{"key":"10.1016\/j.future.2014.10.008_br000275","unstructured":"A. Lana, B. Paolo, Storage resource manager version 2.2: design, implementation, and testing experience, in: Proceedings of International Conference on Computing in High Energy and Nuclear Physics, CHEP 07."},{"key":"10.1016\/j.future.2014.10.008_br000280","doi-asserted-by":"crossref","unstructured":"T. Ylonen, C. Lonvick, Rfc 4254\u2014the secure shell (SSH) connection protocol, 2006.","DOI":"10.17487\/rfc4254"},{"key":"10.1016\/j.future.2014.10.008_br000285","unstructured":"J.-S. V\u00f6ckler, G. Mehta, Y. Zhao, E. Deelman, M. Wilde, Kickstarting remote applications, in: 2nd International Workshop on Grid Computing Environments, 2006."},{"key":"10.1016\/j.future.2014.10.008_br000290","unstructured":"Gnu make. URL: http:\/\/www.gnu.org\/software\/make."},{"issue":"3","key":"10.1016\/j.future.2014.10.008_br000295","doi-asserted-by":"crossref","first-page":"260","DOI":"10.1109\/71.993206","article-title":"Performance-effective and low-complexity task scheduling for heterogeneous computing","volume":"13","author":"Topcuoglu","year":"2002","journal-title":"IEEE Trans. Parallel Distrib. Syst."},{"key":"10.1016\/j.future.2014.10.008_br000300","series-title":"Workflows for e-Science","year":"2007"},{"key":"10.1016\/j.future.2014.10.008_br000305","doi-asserted-by":"crossref","unstructured":"W. Chen, E. Deelman, Workflow overhead analysis and optimizations, in: 6th Workshop on Workflows in Support of Large-scale Science, WORKS\u201911, 2011. http:\/\/dx.doi.org\/10.1145\/2110497.2110500.","DOI":"10.1145\/2110497.2110500"},{"issue":"3","key":"10.1016\/j.future.2014.10.008_br000310","doi-asserted-by":"crossref","first-page":"682","DOI":"10.1016\/j.future.2012.08.015","article-title":"Characterizing and profiling scientific workflows","volume":"29","author":"Juve","year":"2013","journal-title":"Future Gener. Comput. Syst."},{"key":"10.1016\/j.future.2014.10.008_br000315","doi-asserted-by":"crossref","unstructured":"R. Ferreira da Silva, G. Juve, E. Deelman, T. Glatard, F. Desprez, D. Thain, B. Tovar, M. Livny, Toward fine-grained online task characteristics estimation in scientific workflows, in: Proceedings of the 8th Workshop on Workflows in Support of Large-Scale Science, WORKS\u201913, 2013, pp. 58\u201367. http:\/\/dx.doi.org\/10.1145\/2534248.2534254.","DOI":"10.1145\/2534248.2534254"},{"key":"10.1016\/j.future.2014.10.008_br000320","doi-asserted-by":"crossref","unstructured":"G. Singh, M.-H. Su, K. Vahi, E. Deelman, B. Berriman, J. Good, D.S. Katz, G. Mehta, Workflow task clustering for best effort systems with pegasus, in: 15th ACM Mardi Gras Conference, 2008. http:\/\/dx.doi.org\/10.1145\/1341811.1341822.","DOI":"10.1145\/1341811.1341822"},{"key":"10.1016\/j.future.2014.10.008_br000325","doi-asserted-by":"crossref","unstructured":"W. Chen, R. Ferreira da Silva, E. Deelman, R. Sakellariou, Balanced task clustering in scientific workflows, in: 2013 IEEE 9th International Conference on eScience, eScience\u201913, 2013, pp. 188\u2013195. http:\/\/dx.doi.org\/10.1109\/eScience.2013.40.","DOI":"10.1109\/eScience.2013.40"},{"key":"10.1016\/j.future.2014.10.008_br000330","doi-asserted-by":"crossref","first-page":"69","DOI":"10.1016\/j.future.2014.09.014","article-title":"Using imbalance metrics to optimize task clustering in scientific workflow executions","volume":"46","author":"Chen","year":"2015","journal-title":"Future Gener. Comput. Syst."},{"key":"10.1016\/j.future.2014.10.008_br000335","doi-asserted-by":"crossref","unstructured":"A. Chervenak, R. Schuler, C. Kesselman, S. Koranda, B. Moe, Wide area data replication for scientific collaborations, in: Proceedings of the 6th IEEE\/ACM International Workshop on Grid Computing, GRID\u201905, 2005, pp. 1\u20138. http:\/\/dx.doi.org\/10.1109\/GRID.2005.1542717.","DOI":"10.1109\/GRID.2005.1542717"},{"issue":"3\u20134","key":"10.1016\/j.future.2014.10.008_br000340","doi-asserted-by":"crossref","first-page":"171","DOI":"10.1007\/s10723-005-9010-8","article-title":"A taxonomy of workflow management systems for grid computing","volume":"3","author":"Yu","year":"2005","journal-title":"J. Grid Comput."},{"issue":"2","key":"10.1016\/j.future.2014.10.008_br000345","first-page":"73","article-title":"Montage: a grid portal and software toolkit for science-grade astronomical image mosaicking","volume":"4","author":"Jacob","year":"2009","journal-title":"Int. J. Comput. Sci. Eng."},{"issue":"2","key":"10.1016\/j.future.2014.10.008_br000350","doi-asserted-by":"crossref","first-page":"024033","DOI":"10.1103\/PhysRevD.87.024033","article-title":"Searching for gravitational waves from binary coalescence","volume":"87","author":"Babak","year":"2013","journal-title":"Phys. Rev. D"},{"key":"10.1016\/j.future.2014.10.008_br000355","doi-asserted-by":"crossref","unstructured":"A. Ramakrishnan, G. Singh, H. Zhao, E. Deelman, R. Sakellariou, K. Vahi, K. Blackburn, D. Meyers, M. Samidi, Scheduling data-intensive workflows onto storage-constrained distributed resources, in: 7th IEEE International Symposium on Cluster Computing and the Grid, 2007. http:\/\/dx.doi.org\/10.1109\/CCGRID.2007.101.","DOI":"10.1109\/CCGRID.2007.101"},{"key":"10.1016\/j.future.2014.10.008_br000360","unstructured":"Y. Huang, S. Service, V. Ramensky, C. Schmitt, N. Tran, A. Jasinska, J. Wasserscheid, N. Juretic, B. Pasaniuc, R. Wilson, W. Warren, G. Weinstock, K. Dewar, N. Freimer, A varying-depth sequencing strategy and variant-calling framework for complex pedigrees (in preparation)."},{"key":"10.1016\/j.future.2014.10.008_br000365","unstructured":"GridLab Resource Management System. URL: http:\/\/www.gridlab.org\/WorkPackages\/wp-9."},{"key":"10.1016\/j.future.2014.10.008_br000370","unstructured":"M. Rynge, G. Juve, J. Kinney, J. Good, B. Berriman, A. Merrihew, E. Deelman, Producing an infrared multiwavelength galactic plane atlas using montage, pegasus and Amazon Web services, in: 23rd Annual Astronomical Data Analysis Software and Systems, ADASS, Conference, 2013."},{"issue":"3","key":"10.1016\/j.future.2014.10.008_br000375","doi-asserted-by":"crossref","first-page":"381","DOI":"10.1007\/s10723-013-9265-4","article-title":"A case study into using common real-time workflow monitoring infrastructure for scientific workflows","volume":"11","author":"Vahi","year":"2013","journal-title":"J. Grid Comput."},{"key":"10.1016\/j.future.2014.10.008_br000380","doi-asserted-by":"crossref","unstructured":"S. Miles, E. Deelman, P. Groth, K. Vahi, G. Mehta, L. Moreau, Connecting scientific data to scientific experiments with provenance, in: e-Science and Grid Computing, IEEE International Conference on, 2007, pp. 179\u2013186. http:\/\/dx.doi.org\/10.1109\/E-SCIENCE.2007.22.","DOI":"10.1109\/E-SCIENCE.2007.22"},{"key":"10.1016\/j.future.2014.10.008_br000385","doi-asserted-by":"crossref","unstructured":"L. Moreau, B. Clifford, J. Freire, J. Futrelle, Y. Gil, P.T. Groth, N. Kwasnikowska, S. Miles, P. Missier, J. Myers, B. Plale, Y. Simmhan, E.G. Stephan, J.V. den Bussche, The open provenance model core specification (v1.1), 2011, pp. 743\u2013756.","DOI":"10.1016\/j.future.2010.07.005"},{"key":"10.1016\/j.future.2014.10.008_br000390","unstructured":"A. Pavlo, P. Couvares, R. Gietzel, A. Karp, I.D. Alderman, M. Livny, C. Bacon, The NMI build & test laboratory: continuous integration framework for distributed computing software, in: 20th Conference on Large Installation System Administration, LISA, 2006."},{"key":"10.1016\/j.future.2014.10.008_br000395","unstructured":"Atlassian, Continuous integration & build server\u2014bamboo. URL: https:\/\/www.atlassian.com\/software\/bamboo."},{"issue":"5","key":"10.1016\/j.future.2014.10.008_br000400","doi-asserted-by":"crossref","first-page":"528","DOI":"10.1016\/j.future.2008.06.012","article-title":"Workflows and e-science: an overview of workflow system features and capabilities","volume":"25","author":"Deelman","year":"2009","journal-title":"Future Gener. Comput. Syst."},{"issue":"W1","key":"10.1016\/j.future.2014.10.008_br000405","doi-asserted-by":"crossref","first-page":"W557","DOI":"10.1093\/nar\/gkt328","article-title":"The taverna workflow suite: designing and executing workflows of Web Services on the desktop, web or in the cloud","volume":"41","author":"Wolstencroft","year":"2013","journal-title":"Nucleic Acids Res."},{"issue":"8","key":"10.1016\/j.future.2014.10.008_br000410","doi-asserted-by":"crossref","DOI":"10.1186\/gb-2010-11-8-r86","article-title":"Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences","volume":"11","author":"Goecks","year":"2010","journal-title":"Genome Biol."},{"issue":"1","key":"10.1016\/j.future.2014.10.008_br000415","doi-asserted-by":"crossref","first-page":"77","DOI":"10.1186\/1471-2105-13-77","article-title":"Tavaxy: integrating taverna and galaxy workflows with cloud computing support","volume":"13","author":"Abouelhoda","year":"2012","journal-title":"BMC Bioinformatics"},{"key":"10.1016\/j.future.2014.10.008_br000420","doi-asserted-by":"crossref","unstructured":"E. Caron, F. Desprez, F. Lombard, J.-M. Nicod, L. Philippe, M. Quinson, F. Suter, A scalable approach to network enabled servers (research note), in: Proceedings of the 8th International Euro-Par Conference on Parallel Processing, 2002.","DOI":"10.1007\/3-540-45706-2_128"},{"key":"10.1016\/j.future.2014.10.008_br000425","doi-asserted-by":"crossref","unstructured":"T. Fahringer, R. Prodan, R. Duan, F. Nerieri, S. Podlipnig, J. Qin, M. Siddiqui, H.-L. Truong, A. Villazon, M. Wieczorek, Askalon: a grid application development and computing environment, in: Proceedings of the 6th IEEE\/ACM International Workshop on Grid Computing, 2005. http:\/\/dx.doi.org\/10.1109\/GRID.2005.1542733.","DOI":"10.1109\/GRID.2005.1542733"},{"issue":"3","key":"10.1016\/j.future.2014.10.008_br000430","doi-asserted-by":"crossref","first-page":"347","DOI":"10.1177\/1094342008096067","article-title":"Flexible and efficient workflow deployment of data-intensive applications on grids with MOTEUR","volume":"22","author":"Glatard","year":"2008","journal-title":"Int. J. High Perform. Comput. Appl. (IJHPCA)"},{"key":"10.1016\/j.future.2014.10.008_br000435","doi-asserted-by":"crossref","unstructured":"I. Altintas, C. Berkley, E. Jaeger, M. Jones, B. Ludascher, S. Mock, Kepler: an extensible system for design and execution of scientific workflows, in: Scientific and Statistical Database Management, 2004. Proceedings. 16th International Conference on, 2004. http:\/\/dx.doi.org\/10.1109\/SSDM.2004.1311241.","DOI":"10.1109\/SSDM.2004.1311241"},{"key":"10.1016\/j.future.2014.10.008_br000440","series-title":"Workflows for e-Science","first-page":"320","article-title":"The Triana workflow environment: architecture and applications","author":"Taylor","year":"2007"},{"key":"10.1016\/j.future.2014.10.008_br000445","series-title":"Parallel and Distributed Processing Symposium, International","article-title":"High performance parametric modeling with nimrod\/G: Killer application for the global grid?","author":"Abramson","year":"2000"},{"key":"10.1016\/j.future.2014.10.008_br000450","doi-asserted-by":"crossref","first-page":"960","DOI":"10.1109\/TPDS.2010.177","article-title":"Parameter exploration in science and engineering using many-task computing","volume":"22","author":"Abramson","year":"2011","journal-title":"IEEE Trans. Parallel Distrib. Syst. Many-Task Comput."},{"key":"10.1016\/j.future.2014.10.008_br000455","series-title":"Proceedings of the 2008 ACM\/IEEE Conference on Supercomputing","first-page":"24","article-title":"Nimrod\/K: towards massively parallel dynamic grid workflows","author":"Abramson","year":"2008"},{"key":"10.1016\/j.future.2014.10.008_br000460","doi-asserted-by":"crossref","unstructured":"M. Albrecht, P. Donnelly, P. Bui, D. Thain, Makeflow: a portable abstraction for data intensive computing on clusters, clouds, and grids, in: Proceedings of the 1st ACM SIGMOD Workshop on Scalable Workflow Execution Engines and Technologies, SWEET, 2012.","DOI":"10.1145\/2443416.2443417"},{"key":"10.1016\/j.future.2014.10.008_br000465","unstructured":"Apache Tez, http:\/\/tez.apache.org. URL: http:\/\/tez.apache.org."},{"key":"10.1016\/j.future.2014.10.008_br000470","unstructured":"Apache Ozzie, http:\/\/oozie.apache.org. URL: http:\/\/oozie.apache.org."},{"key":"10.1016\/j.future.2014.10.008_br000475","unstructured":"D. Brown, E. Deelman, Looking for gravitational waves: A computing perspective, in: International Science Grid This Week, 2011."},{"key":"10.1016\/j.future.2014.10.008_br000480","unstructured":"L.S. Collaboration, \u201cblind injection\u201d stress-tests ligo and virgo\u2019s search for gravitational waves! 2011. URL: http:\/\/www.ligo.org\/news.php."},{"key":"10.1016\/j.future.2014.10.008_br000485","unstructured":"The kepler mission, 2011. URL: http:\/\/kepler.nasa.gov."},{"key":"10.1016\/j.future.2014.10.008_br000490","doi-asserted-by":"crossref","unstructured":"G.B. Berriman, E. Deelman, G. Juve, M. Regelson, P. Plavchan, The application of cloud computing to astronomy: a study of cost and performance, in: Proceedings of the e-Science in Astronomy Conference, Brisbane, Australia, 2010.","DOI":"10.1109\/eScienceW.2010.10"},{"key":"10.1016\/j.future.2014.10.008_br000495","doi-asserted-by":"crossref","first-page":"e3197","DOI":"10.1371\/journal.pone.0003197","article-title":"High-throughput, kingdom-wide prediction and annotation of bacterial non-coding RNAs","volume":"3","author":"Livny","year":"2008","journal-title":"PLoS One"},{"key":"10.1016\/j.future.2014.10.008_br000500","unstructured":"V. Marx, UCLA team sequences cell line, puts open source software framework into production newsletter: bioinform, 2010. URL: http:\/\/www.genomeweb.com\/print\/933003."},{"key":"10.1016\/j.future.2014.10.008_br000505","unstructured":"Seqware, 2010. URL: http:\/\/sourceforge.net\/apps\/mediawiki\/seqware."}],"container-title":["Future Generation Computer Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0167739X14002015?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0167739X14002015?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,9,29]],"date-time":"2025-09-29T03:39:04Z","timestamp":1759117144000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0167739X14002015"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,5]]},"references-count":101,"alternative-id":["S0167739X14002015"],"URL":"https:\/\/doi.org\/10.1016\/j.future.2014.10.008","relation":{},"ISSN":["0167-739X"],"issn-type":[{"value":"0167-739X","type":"print"}],"subject":[],"published":{"date-parts":[[2015,5]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Pegasus, a workflow management system for science automation","name":"articletitle","label":"Article Title"},{"value":"Future Generation Computer Systems","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.future.2014.10.008","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"Copyright \u00a9 2014 Elsevier B.V. All rights reserved.","name":"copyright","label":"Copyright"}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1016_2Fj_patter_2021_100322 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T02:08:20Z","timestamp":1793239700207,"version":"4.5.0"},"reference-count":235,"publisher":"Elsevier BV","issue":"9","license":[{"start":{"date-parts":[[2021,9,1]],"date-time":"2021-09-01T00:00:00Z","timestamp":1630454400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2021,9,1]],"date-time":"2021-09-01T00:00:00Z","timestamp":1630454400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"},{"start":{"date-parts":[[2021,7,8]],"date-time":"2021-07-08T00:00:00Z","timestamp":1625702400000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["cell.com","elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Patterns"],"published-print":{"date-parts":[[2021,9]]},"DOI":"10.1016\/j.patter.2021.100322","type":"journal-article","created":{"date-parts":[[2021,9,10]],"date-time":"2021-09-10T11:27:47Z","timestamp":1631273267000},"page":"100322","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":121,"title":["The role of metadata in reproducible computational research"],"prefix":"10.1016","volume":"2","author":[{"given":"Jeremy","family":"Leipzig","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Daniel","family":"N\u00fcst","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Charles Tapley","family":"Hoyt","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Karthik","family":"Ram","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jane","family":"Greenberg","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.patter.2021.100322_bib1","doi-asserted-by":"crossref","first-page":"957","DOI":"10.1136\/amiajnl-2014-002974","article-title":"The National Institutes of Health\u2019s big data to knowledge (BD2K) initiative: capitalizing on biomedical big data","volume":"21","author":"Margolis","year":"2014","journal-title":"J.\u00a0Am. Med. Inform. Assoc."},{"key":"10.1016\/j.patter.2021.100322_bib2","article-title":"Recommendations to enhance rigor and reproducibility in biomedical research","volume":"9","author":"Brito","year":"2020","journal-title":"Gigascience"},{"key":"10.1016\/j.patter.2021.100322_bib3","series-title":"Committee on Reproducibility and Replicability in Science. Reproducibility and Replicability in Science","year":"2019"},{"key":"10.1016\/j.patter.2021.100322_bib4","doi-asserted-by":"crossref","first-page":"160018","DOI":"10.1038\/sdata.2016.18","article-title":"The FAIR guiding principles for scientific data management and stewardship","volume":"3","author":"Wilkinson","year":"2016","journal-title":"Sci. Data"},{"key":"10.1016\/j.patter.2021.100322_bib5","series-title":"Awesome Reproducible Research","author":"Leipzig","year":"2019"},{"key":"10.1016\/j.patter.2021.100322_bib6","doi-asserted-by":"crossref","first-page":"385","DOI":"10.1093\/biostatistics\/kxq028","article-title":"An invitation to reproducible computational research","volume":"11","author":"Donoho","year":"2010","journal-title":"Biostatistics"},{"key":"10.1016\/j.patter.2021.100322_bib7","doi-asserted-by":"crossref","first-page":"172","DOI":"10.1002\/asi.24226","article-title":"Data objects and documenting scientific processes: an analysis of data events in biodiversity data papers","volume":"71","author":"Li","year":"2020","journal-title":"J.\u00a0Assoc. Inf. Sci. Technol."},{"key":"10.1016\/j.patter.2021.100322_bib8","first-page":"4","article-title":"Setting the Default to Reproducible","volume":"46","author":"Stodden","year":"2013","journal-title":"Computat. Sci. Res."},{"key":"10.1016\/j.patter.2021.100322_bib9","article-title":"Showing your working: a guide to reproducible neuroimaging analyses","author":"Whitaker","year":"2016","journal-title":"Figshare"},{"key":"10.1016\/j.patter.2021.100322_bib10","article-title":"Terminologies for reproducible research","author":"Barba","year":"2018","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2021.100322_bib11","series-title":"New Changes to Badging Terminology","year":"2020"},{"key":"10.1016\/j.patter.2021.100322_bib12","doi-asserted-by":"crossref","first-page":"452","DOI":"10.1038\/533452a","article-title":"1,500 scientists lift the lid on reproducibility","volume":"533","author":"Baker","year":"2016","journal-title":"Nat. News"},{"key":"10.1016\/j.patter.2021.100322_bib13","doi-asserted-by":"crossref","first-page":"401","DOI":"10.1038\/467401b","article-title":"Disclose all data in publications","volume":"467","author":"Baggerly","year":"2010","journal-title":"Nature"},{"key":"10.1016\/j.patter.2021.100322_bib14","doi-asserted-by":"crossref","first-page":"531","DOI":"10.1038\/483531a","article-title":"Drug development: raise standards for preclinical cancer research","volume":"483","author":"Begley","year":"2012","journal-title":"Nature"},{"key":"10.1016\/j.patter.2021.100322_bib15","doi-asserted-by":"crossref","first-page":"149","DOI":"10.1038\/ng.295","article-title":"Repeatability of published microarray gene expression analyses","volume":"41","author":"Ioannidis","year":"2009","journal-title":"Nat. Genet."},{"key":"10.1016\/j.patter.2021.100322_bib16","doi-asserted-by":"crossref","first-page":"200","DOI":"10.1124\/jpet.114.219170","article-title":"Common misconceptions about data analysis and statistics","volume":"351","author":"Motulsky","year":"2014","journal-title":"J.\u00a0Pharmacol. Exp. Ther."},{"key":"10.1016\/j.patter.2021.100322_bib17","doi-asserted-by":"crossref","first-page":"e124","DOI":"10.1371\/journal.pmed.0020124","article-title":"Why most published research findings are false","volume":"2","author":"Ioannidis","year":"2005","journal-title":"PLoS Med."},{"key":"10.1016\/j.patter.2021.100322_bib18","doi-asserted-by":"crossref","first-page":"2628","DOI":"10.1073\/pnas.1708272114","article-title":"Opinion: Is science really facing a reproducibility crisis, and do we need it to?","volume":"115","author":"Fanelli","year":"2018","journal-title":"Proc. Natl. Acad. Sci. U S A"},{"key":"10.1016\/j.patter.2021.100322_bib26","doi-asserted-by":"crossref","first-page":"229","DOI":"10.1177\/2515245920918872","article-title":"Analysis of open data and computational reproducibility in registered reports in psychology","volume":"3","author":"Obels","year":"2020","journal-title":"Advances in Methods and Practices in Psychological Science"},{"key":"10.1016\/j.patter.2021.100322_bib19","first-page":"229","article-title":"The truth wears off","volume":"13","author":"Lehrer","year":"2010","journal-title":"New Yorker"},{"key":"10.1016\/j.patter.2021.100322_bib20","series-title":"International Conference on Dublin Core and Metadata Applications","first-page":"140","article-title":"Metadata capital in a data repository","author":"Greenberg","year":"2013"},{"key":"10.1016\/j.patter.2021.100322_bib21","first-page":"279","article-title":"Metadata for big data: a preliminary investigation of metadata quality issues in research data repositories","volume":"34","author":"Rousidis","year":"2014","journal-title":"Inf. Serv. Use"},{"key":"10.1016\/j.patter.2021.100322_bib22","doi-asserted-by":"crossref","first-page":"1523","DOI":"10.1002\/asi.23294","article-title":"Big data, bigger dilemmas: a critical review","volume":"66","author":"Ekbia","year":"2015","journal-title":"J.\u00a0Assn Inf. Sci. Tec"},{"key":"10.1016\/j.patter.2021.100322_bib23","author":"Warden"},{"key":"10.1016\/j.patter.2021.100322_bib24","series-title":"Proceedings of the 36th International Conference on Machine Learning","first-page":"725","article-title":"Unreproducible research is reproducible","volume":"97","author":"Bouthillier","year":"2019"},{"key":"10.1016\/j.patter.2021.100322_bib25","series-title":"Machine Learning Systems Workshop at NIPS","article-title":"Automatically tracking metadata and provenance of machine learning experiments","author":"Schelter","year":"2017"},{"key":"10.1016\/j.patter.2021.100322_bib27","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1186\/s41073-020-0091-5","article-title":"Reproducible and transparent research practices in published neurology research","volume":"5","author":"Rauh","year":"2020","journal-title":"Res. Integr. Peer Rev."},{"key":"10.1016\/j.patter.2021.100322_bib28","series-title":"Proceedings of the First International Workshop on Practical Reproducible Evaluation of Computer Systems; P-RECS\u201918","first-page":"1","article-title":"Enabling the verification of computational results: an empirical evaluation of computational reproducibility","author":"Stodden","year":"2018"},{"key":"10.1016\/j.patter.2021.100322_bib29","doi-asserted-by":"crossref","first-page":"190030","DOI":"10.1038\/sdata.2019.30","article-title":"Assessing data availability and research reproducibility in hydrology and water resources","volume":"6","author":"Stagge","year":"2019","journal-title":"Sci. Data"},{"key":"10.1016\/j.patter.2021.100322_bib30","doi-asserted-by":"crossref","first-page":"e5072","DOI":"10.7717\/peerj.5072","article-title":"Reproducible research and GIScience: an evaluation using AGILE conference papers","volume":"6","author":"N\u00fcst","year":"2018","journal-title":"PeerJ"},{"key":"10.1016\/j.patter.2021.100322_bib31","doi-asserted-by":"crossref","first-page":"e1003285","DOI":"10.1371\/journal.pcbi.1003285","article-title":"Ten simple rules for reproducible computational research","volume":"9","author":"Sandve","year":"2013","journal-title":"PLoS Comput. Biol."},{"key":"10.1016\/j.patter.2021.100322_bib32","series-title":"Measuring Reproducibility in Computer Systems Research","author":"Collberg","year":"2014"},{"key":"10.1016\/j.patter.2021.100322_bib33","doi-asserted-by":"crossref","first-page":"30","DOI":"10.1186\/s13742-016-0135-4","article-title":"Tools and techniques for computational reproducibility","volume":"5","author":"Piccolo","year":"2016","journal-title":"Gigascience"},{"key":"10.1016\/j.patter.2021.100322_bib34","article-title":"Reproducible research is still a challenge","author":"FitzJohn","year":"2014","journal-title":"rOpenSci"},{"key":"10.1016\/j.patter.2021.100322_bib35","author":"Ball"},{"key":"10.1016\/j.patter.2021.100322_bib36","series-title":"RDA Metadata Standards Directory Working Group: Final Report","author":"Ball","year":"2016"},{"key":"10.1016\/j.patter.2021.100322_bib37","series-title":"Understanding Metadata: What is Metadata, and What is it For?: A Primer","author":"Riley","year":"2017"},{"key":"10.1016\/j.patter.2021.100322_bib38","series-title":"Metadata","author":"Qin","year":"2016"},{"key":"10.1016\/j.patter.2021.100322_bib39","doi-asserted-by":"crossref","first-page":"365","DOI":"10.1038\/ng1201-365","article-title":"Minimum information about a microarray experiment (MIAME)-toward standards for microarray data","volume":"29","author":"Brazma","year":"2001","journal-title":"Nat. Genet."},{"key":"10.1016\/j.patter.2021.100322_bib40","doi-asserted-by":"crossref","first-page":"25","DOI":"10.1038\/75556","article-title":"Gene ontology: tool for the unification of biology","volume":"25","author":"Ashburner","year":"2000","journal-title":"Nat. Genet."},{"key":"10.1016\/j.patter.2021.100322_bib41","doi-asserted-by":"crossref","first-page":"63","DOI":"10.1016\/0076-6879(90)83007-V","article-title":"Rapid and sensitive sequence comparison with FASTP and FASTA","volume":"183","author":"Pearson","year":"1990","journal-title":"Methods Enzymol."},{"key":"10.1016\/j.patter.2021.100322_bib42","doi-asserted-by":"crossref","first-page":"180118","DOI":"10.1038\/sdata.2018.118","article-title":"A design framework and exemplar metrics for FAIRness","volume":"5","author":"Wilkinson","year":"2018","journal-title":"Sci. Data"},{"key":"10.1016\/j.patter.2021.100322_bib43","first-page":"1586","article-title":"Digital object identifier (DOI\u00ae) system","volume":"3","author":"Paskin","year":"2010","journal-title":"Encyclopedia Libr. Inf. Sci."},{"key":"10.1016\/j.patter.2021.100322_bib44","doi-asserted-by":"crossref","first-page":"358","DOI":"10.1038\/s41587-019-0080-8","article-title":"FAIRsharing as a community approach to standards, repositories and policies","volume":"37","author":"Sansone","year":"2019","journal-title":"Nat. Biotechnol."},{"key":"10.1016\/j.patter.2021.100322_bib45","doi-asserted-by":"crossref","first-page":"218","DOI":"10.2218\/ijdc.v11i1.399","article-title":"Metadata and reproducibility: a case study of gravitational wave research data management","volume":"11","author":"Qin","year":"2016","journal-title":"Int. J. Digital Curation"},{"key":"10.1016\/j.patter.2021.100322_bib46","series-title":"Proceedings of the 2nd International Workshop on Linked Science","article-title":"From workflows to research objects: an architecture for preserving the semantics of science","author":"Page","year":"2012"},{"key":"10.1016\/j.patter.2021.100322_bib47","first-page":"29","article-title":"CRISP-DM: Towards a standard process model for data mining","author":"Wirth","year":"2000"},{"key":"10.1016\/j.patter.2021.100322_bib48","doi-asserted-by":"crossref","first-page":"e15","DOI":"10.5334\/jors.ax","article-title":"Data management lifecycle and software lifecycle management in the context of conducting science","volume":"2","author":"Lenhardt","year":"2014","journal-title":"J.\u00a0Open Res. Softw."},{"key":"10.1016\/j.patter.2021.100322_bib56","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1016\/j.ecoinf.2005.08.004","article-title":"Meta-information concepts for ecological data management","volume":"1","author":"Michener","year":"2006","journal-title":"Ecol. Inform."},{"key":"10.1016\/j.patter.2021.100322_bib211","doi-asserted-by":"crossref","first-page":"345","DOI":"10.1148\/radiographics.12.2.1561424","article-title":"Introduction to the ACR-NEMA DICOM standard","volume":"12","author":"Bidgood","year":"1992","journal-title":"Radiographics"},{"key":"10.1016\/j.patter.2021.100322_bib212","doi-asserted-by":"crossref","first-page":"e102623","DOI":"10.1371\/journal.pone.0102623","article-title":"The GBIF integrated publishing toolkit: facilitating the efficient publishing of biodiversity data on the internet","volume":"9","author":"Robertson","year":"2014","journal-title":"PLoS One"},{"key":"10.1016\/j.patter.2021.100322_bib213","doi-asserted-by":"crossref","first-page":"277","DOI":"10.1107\/S1600576715021871","article-title":"Specification of the crystallographic information file format, version 2.0","volume":"49","author":"Bernstein","year":"2016","journal-title":"J.\u00a0Appl. Crystallogr."},{"key":"10.1016\/j.patter.2021.100322_bib214","doi-asserted-by":"crossref","first-page":"1344","DOI":"10.1021\/je034088i","article-title":"ThermoML an XML-based approach for storage and exchange of experimental and critically evaluated thermophysical and thermochemical property data. 2. Uncertainties","volume":"48","author":"Chirico","year":"2003","journal-title":"J.\u00a0Chem. Eng. Data"},{"key":"10.1016\/j.patter.2021.100322_bib215","doi-asserted-by":"crossref","first-page":"740","DOI":"10.1177\/0037549703040939","article-title":"An Overview of CellML 1.1, a biological model description language","volume":"79","author":"Cuellar","year":"2003","journal-title":"Simulation"},{"key":"10.1016\/j.patter.2021.100322_bib63","doi-asserted-by":"crossref","first-page":"giz165","DOI":"10.1093\/gigascience\/giz165","article-title":"The data tags suite (DATS) model for discovering data access and use requirements","volume":"9","author":"Alter","year":"2020","journal-title":"Gigascience"},{"key":"10.1016\/j.patter.2021.100322_bib216","doi-asserted-by":"crossref","first-page":"57","DOI":"10.1186\/s13059-015-0622-4","article-title":"Coordinated International action to accelerate genome-to-phenome with FAANG, the functional annotation of animal genomes project","volume":"16","author":"Andersson","year":"2015","journal-title":"Genome Biol."},{"key":"10.1016\/j.patter.2021.100322_bib118","doi-asserted-by":"crossref","first-page":"D38","DOI":"10.1093\/nar\/gkv1116","article-title":"And data services registry: a community effort to document bioinformatics resources","volume":"44","author":"Ison","year":"2016","journal-title":"Nucleic Acids Res."},{"key":"10.1016\/j.patter.2021.100322_bib219","author":"Dumbill"},{"key":"10.1016\/j.patter.2021.100322_bib220","series-title":"Proceedings of the 8th International Conference on Knowledge Capture","first-page":"32","article-title":"OntoSoft: capturing scientific software metadata","author":"Gil","year":"2015"},{"key":"10.1016\/j.patter.2021.100322_bib72","doi-asserted-by":"crossref","first-page":"1325","DOI":"10.1093\/bioinformatics\/btt113","article-title":"EDAM: An ontology of bioinformatics operations, types of data and identifiers, topics and formats","volume":"29","author":"Ison","year":"2013","journal-title":"Bioinformatics"},{"key":"10.1016\/j.patter.2021.100322_bib221","doi-asserted-by":"crossref","first-page":"25","DOI":"10.1186\/2041-1480-5-25","article-title":"The software ontology (SWO): a resource for reproducibility in biomedical data analysis, curation and digital preservation","volume":"5","author":"Malone","year":"2014","journal-title":"J.\u00a0Biomed. Semantics"},{"key":"10.1016\/j.patter.2021.100322_bib222","doi-asserted-by":"crossref","first-page":"53","DOI":"10.1186\/s13326-016-0100-2","article-title":"The ontology of biological and clinical statistics (OBCS) for standardized and reproducible statistical analysis","volume":"7","author":"Zheng","year":"2016","journal-title":"J.\u00a0Biomed. Semantics"},{"key":"10.1016\/j.patter.2021.100322_bib224","first-page":"105","article-title":"Linked SDMX data","volume":"6","author":"Capadisli","year":"2015","journal-title":"Semantic Web"},{"key":"10.1016\/j.patter.2021.100322_bib225","doi-asserted-by":"crossref","DOI":"10.29173\/iq127","article-title":"DDI as a Common Format for Export and Import for Statistical Packages","volume":"39","author":"Hoyle","year":"2016","journal-title":"IASSIST Quarterly"},{"key":"10.1016\/j.patter.2021.100322_bib226","series-title":"Proceedings of the 11th International Conference on Semantic Systems; SEMANTICS \u201915","first-page":"169","article-title":"MEX Vocabulary: A lightweight Interchange format for machine learning experiments","author":"Esteves","year":"2015"},{"key":"10.1016\/j.patter.2021.100322_bib227","article-title":"ML-schema: exposing the semantics of machine learning with schemas and ontologies","author":"Publio","year":"2018","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2021.100322_bib115","series-title":"Common Workflow Language, v1.0","author":"Peter","year":"2016"},{"key":"10.1016\/j.patter.2021.100322_bib229","doi-asserted-by":"crossref","first-page":"354","DOI":"10.1016\/j.future.2015.12.017","article-title":"Reproducibility of execution environments in computational science using semantics and clouds","volume":"67","author":"Santana-Perez","year":"2017","journal-title":"Future Gener. Comput. Syst."},{"key":"10.1016\/j.patter.2021.100322_bib230","article-title":"Open Provenance Model (OPM) OWL Specification","author":"Ding","year":"2010"},{"key":"10.1016\/j.patter.2021.100322_bib231","series-title":"PROV-O: The PROV Ontology","author":"Lebo","year":"2013"},{"key":"10.1016\/j.patter.2021.100322_bib127","doi-asserted-by":"crossref","DOI":"10.1093\/gigascience\/giz095","article-title":"Sharing interoperable workflow provenance: a review of best practices and their practical application in CWLProv","volume":"8","author":"Khan","year":"2019","journal-title":"Gigascience"},{"key":"10.1016\/j.patter.2021.100322_bib232","unstructured":"Cao, Y., Jones, C., Cuevas-Vicentt\u00edn, V., Jones, M.B., Lud\u00e4scher, B., McPhillips, T., Missier, P., Schwalm, C., Slaughter, P., Vieglais, D., et al. (2016). ProvONE: extending PROV to support the DataONE scientific community."},{"key":"10.1016\/j.patter.2021.100322_bib233","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1186\/2041-1480-4-37","article-title":"PAV ontology: provenance, authoring and versioning","volume":"4","author":"Ciccarese","year":"2013","journal-title":"J.\u00a0Biomed. Semantics"},{"key":"10.1016\/j.patter.2021.100322_bib180","article-title":"The BagIt File Packaging Format (V1.0)","author":"Kunze","year":"2018","journal-title":"RFC Editor"},{"key":"10.1016\/j.patter.2021.100322_bib235","doi-asserted-by":"crossref","first-page":"e3000099","DOI":"10.1371\/journal.pbio.3000099","article-title":"Enabling precision medicine via standard communication of HTS provenance, analysis, and results","volume":"16","author":"Alterovitz","year":"2018","journal-title":"PLoS Biol."},{"key":"10.1016\/j.patter.2021.100322_bib236","first-page":"132","article-title":"Dublin core metadata for resource discovery","volume":"2413","author":"Weibel","year":"1998","journal-title":"Internet Eng. Task Force RFC"},{"key":"10.1016\/j.patter.2021.100322_bib237","doi-asserted-by":"crossref","first-page":"99","DOI":"10.6087\/kcse.2014.1.99","article-title":"Journal article tag suite 1.0: National information standards organization standard of journal extensible markup language","volume":"1","author":"Huh","year":"2014","journal-title":"Sci. Ed."},{"key":"10.1016\/j.patter.2021.100322_bib238","doi-asserted-by":"crossref","first-page":"102","DOI":"10.1080\/00987913.2001.10764686","article-title":"ONIX (online information exchange)","volume":"27","author":"Needleman","year":"2001","journal-title":"Serials Rev."},{"key":"10.1016\/j.patter.2021.100322_bib239","first-page":"265","article-title":"Medical subject headings (MeSH)","volume":"88","author":"Lipscomb","year":"2000","journal-title":"Bull. Med. Libr. Assoc."},{"key":"10.1016\/j.patter.2021.100322_bib240","series-title":"Library of Congress Subject Headings: Principles and Application","author":"Chan","year":"1995"},{"key":"10.1016\/j.patter.2021.100322_bib241","doi-asserted-by":"crossref","first-page":"28","DOI":"10.1186\/2041-1480-5-28","article-title":"Micropublications: a semantic model for claims, evidence, arguments and annotations in biomedical communications","volume":"5","author":"Clark","year":"2014","journal-title":"J.\u00a0Biomed. Semantics"},{"key":"10.1016\/j.patter.2021.100322_bib242","doi-asserted-by":"crossref","first-page":"1188","DOI":"10.1016\/j.drudis.2012.05.016","article-title":"Open PHACTS: Semantic interoperability for drug discovery","volume":"17","author":"Williams","year":"2012","journal-title":"Drug Discov. Today"},{"key":"10.1016\/j.patter.2021.100322_bib142","doi-asserted-by":"crossref","first-page":"193","DOI":"10.1016\/j.drudis.2013.12.011","article-title":"Recent advances in modeling languages for pathway maps and computable biological networks","volume":"19","author":"Slater","year":"2014","journal-title":"Drug Discov. Today"},{"key":"10.1016\/j.patter.2021.100322_bib243","doi-asserted-by":"crossref","first-page":"739","DOI":"10.1016\/j.jbi.2008.04.010","article-title":"The SWAN biomedical discourse ontology","volume":"41","author":"Ciccarese","year":"2008","journal-title":"J.\u00a0Biomed. Inform."},{"key":"10.1016\/j.patter.2021.100322_bib244","series-title":"Semantic Web Technologies and Legal Scholarly Publishing","first-page":"121","article-title":"The semantic publishing and referencing ontologies","author":"Peroni","year":"2014"},{"key":"10.1016\/j.patter.2021.100322_bib245","series-title":"Semantic Web","first-page":"1","volume":"0","author":"Gangemi","year":"2017"},{"key":"10.1016\/j.patter.2021.100322_bib49","doi-asserted-by":"crossref","first-page":"588","DOI":"10.1038\/s41592-021-01169-5","article-title":"Diversity in Immunogenomics: the value and the challenge","volume":"18","author":"Peng","year":"2021","journal-title":"Nat. Methods"},{"key":"10.1016\/j.patter.2021.100322_bib50","doi-asserted-by":"crossref","first-page":"1251","DOI":"10.1038\/nbt1346","article-title":"The OBO Foundry: coordinated evolution of ontologies to support biomedical data integration","volume":"25","author":"Smith","year":"2007","journal-title":"Nat. Biotechnol."},{"key":"10.1016\/j.patter.2021.100322_bib51","doi-asserted-by":"crossref","first-page":"1133","DOI":"10.1016\/j.crad.2005.07.003","article-title":"DICOM demystified: a review of digital file formats and their use in radiological practice","volume":"60","author":"Graham","year":"2005","journal-title":"Clin. Radiol."},{"key":"10.1016\/j.patter.2021.100322_bib52","article-title":"Working with the DICOM and NIfTI data standards in R","volume":"44","author":"Whitcher","year":"2011","journal-title":"J.\u00a0Stat. Softw."},{"key":"10.1016\/j.patter.2021.100322_bib53","first-page":"280","article-title":"Quality of DICOM header information for image categorization","volume":"4685","author":"Gueld","year":"2002"},{"key":"10.1016\/j.patter.2021.100322_bib54","doi-asserted-by":"crossref","first-page":"1323","DOI":"10.1016\/j.mri.2012.05.001","article-title":"3D slicer as an image computing platform for the quantitative imaging network","volume":"30","author":"Fedorov","year":"2012","journal-title":"Magn. Reson. Imaging"},{"key":"10.1016\/j.patter.2021.100322_bib55","doi-asserted-by":"crossref","first-page":"e87","DOI":"10.1158\/0008-5472.CAN-17-0336","article-title":"Dcmqi: An open source library for standardized communication of quantitative image analysis results using DICOM","volume":"77","author":"Herz","year":"2017","journal-title":"Cancer Res."},{"key":"10.1016\/j.patter.2021.100322_bib206","unstructured":"Halpern, B., Frazier, M., Potapenko, J., Casey, K., Koenig, K., Longo, C., Lowndes, J.S., Rockwood, C.R., Setig, E., Selkoe, K., et al. (2015). Cumulative Human Impacts: raw stressor data (2008 and 2013). KNB, 10.5063\/F1S180FS."},{"key":"10.1016\/j.patter.2021.100322_bib57","doi-asserted-by":"crossref","first-page":"D866","DOI":"10.1093\/nar\/gkm815","article-title":"Many microbe microarrays database: uniformly normalized affymetrix compendia with structured experimental metadata","volume":"36","author":"Faith","year":"2008","journal-title":"Nucleic Acids Res."},{"key":"10.1016\/j.patter.2021.100322_bib58","doi-asserted-by":"crossref","first-page":"e184","DOI":"10.1371\/journal.pmed.0050184","article-title":"Key issues in conducting a meta-analysis of gene expression microarray datasets","volume":"5","author":"Ramasamy","year":"2008","journal-title":"Plos Med."},{"key":"10.1016\/j.patter.2021.100322_bib59","doi-asserted-by":"crossref","first-page":"2354","DOI":"10.1093\/bioinformatics\/btq415","article-title":"ISA software suite: supporting standards-compliant experimental annotation and enabling curation at the community level","volume":"26","author":"Rocca-Serra","year":"2010","journal-title":"Bioinformatics"},{"key":"10.1016\/j.patter.2021.100322_bib60","series-title":"Proceedings of the 25th International Conference on World Wide Web","first-page":"263","article-title":"Foundations of JSON schema","author":"Pezoa","year":"2016"},{"key":"10.1016\/j.patter.2021.100322_bib61","first-page":"173","article-title":"Five stars of linked data vocabulary use","volume":"5","author":"Janowicz","year":"2014","journal-title":"Semantic Web"},{"key":"10.1016\/j.patter.2021.100322_bib62","series-title":"The World Wide Web Conference","first-page":"1365","article-title":"Google dataset search: building a search engine for datasets in an open web ecosystem","author":"Brickley","year":"2019"},{"key":"10.1016\/j.patter.2021.100322_bib64","doi-asserted-by":"crossref","first-page":"127","DOI":"10.1126\/science.aaw2825","article-title":"Certify reproducibility with confidential data","volume":"365","author":"P\u00e9rignon","year":"2019","journal-title":"Science"},{"key":"10.1016\/j.patter.2021.100322_bib65","doi-asserted-by":"crossref","first-page":"102","DOI":"10.1177\/0002716217742610","article-title":"Research infrastructure for the safe analysis of sensitive data","volume":"675","author":"Foster","year":"2018","journal-title":"Ann. Am. Acad. Pol. Soc. Sci."},{"key":"10.1016\/j.patter.2021.100322_bib66","author":"Jaradeh"},{"key":"10.1016\/j.patter.2021.100322_bib67","doi-asserted-by":"crossref","first-page":"106","DOI":"10.1186\/s13059-021-02332-z","article-title":"Improving the completeness of public metadata accompanying omics studies","volume":"22","author":"Rajesh","year":"2021","journal-title":"Genome Biol."},{"key":"10.1016\/j.patter.2021.100322_bib68","doi-asserted-by":"crossref","first-page":"260","DOI":"10.1111\/nph.16544","article-title":"Enabling reusability of plant phenomic datasets with MIAPPE 1.1","volume":"227","author":"Papoutsoglou","year":"2020","journal-title":"New Phytol."},{"key":"10.1016\/j.patter.2021.100322_bib69","author":"Oberkampf"},{"key":"10.1016\/j.patter.2021.100322_bib70","doi-asserted-by":"crossref","first-page":"180117","DOI":"10.1038\/sdata.2018.117","article-title":"Sustainable data and metadata management at the BD2K-LINCS data coordination and integration center","volume":"5","author":"Stathias","year":"2018","journal-title":"Sci. Data"},{"key":"10.1016\/j.patter.2021.100322_bib207","doi-asserted-by":"crossref","first-page":"550","DOI":"10.1186\/s13059-014-0550-8","article-title":"Moderated estimation of fold change and dispersion for RNA-seq data with DESeq2","volume":"15","author":"Love","year":"2014","journal-title":"Genome Biol."},{"key":"10.1016\/j.patter.2021.100322_bib71","doi-asserted-by":"crossref","first-page":"342","DOI":"10.1038\/nbt.3780","article-title":"Reproducibility of computational workflows is automated using continuous analysis","volume":"35","author":"Beaulieu-Jones","year":"2017","journal-title":"Nat. Biotechnol."},{"key":"10.1016\/j.patter.2021.100322_bib73","doi-asserted-by":"crossref","first-page":"656","DOI":"10.1093\/bioinformatics\/bty646","article-title":"Automated workflow composition in mass spectrometry-based proteomics","volume":"35","author":"Palmblad","year":"2019","journal-title":"Bioinformatics"},{"key":"10.1016\/j.patter.2021.100322_bib74","doi-asserted-by":"crossref","DOI":"10.12688\/f1000research.12974.1","article-title":"Using Bio.tools to generate and annotate workbench tool descriptions","volume":"6","author":"Hillion","year":"2017","journal-title":"F1000Res."},{"key":"10.1016\/j.patter.2021.100322_bib75","doi-asserted-by":"crossref","first-page":"e7223","DOI":"10.7717\/peerj.7223","article-title":"BioShake: A haskell EDSL for bioinformatics workflows","volume":"7","author":"Bed\u0151","year":"2019","journal-title":"PeerJ"},{"key":"10.1016\/j.patter.2021.100322_bib76","series-title":"Bioinformatics Open Source Conference","article-title":"Portable workflow and tool descriptions with the CWL","author":"Amstutz","year":"2015"},{"key":"10.1016\/j.patter.2021.100322_bib77","doi-asserted-by":"crossref","DOI":"10.1093\/gigascience\/giaa152","article-title":"Tool recommender system in Galaxy using deep learning","volume":"10","author":"Kumar","year":"2021","journal-title":"Gigascience"},{"key":"10.1016\/j.patter.2021.100322_bib78","article-title":"CodeMeta: an exchange schema for software metadata","author":"Jones","year":"2016","journal-title":"KNB Data Repository"},{"key":"10.1016\/j.patter.2021.100322_bib208","unstructured":"Price, M.H. (2020). Baydem (Github)."},{"key":"10.1016\/j.patter.2021.100322_bib79","doi-asserted-by":"crossref","first-page":"e86","DOI":"10.7717\/peerj-cs.86","article-title":"Software citation principles","volume":"2","author":"Smith","year":"2016","journal-title":"Peerj Comput. Sci."},{"key":"10.1016\/j.patter.2021.100322_bib80","article-title":"GitHub repositories with links to academic papers: open access, traceability, and evolution","author":"Wattanakriengkrai","year":"2020","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2021.100322_bib81","doi-asserted-by":"crossref","first-page":"198","DOI":"10.3389\/fbioe.2018.00198","article-title":"GitHub statistics as a measure of the impact of open-source bioinformatics software","volume":"6","author":"Dozmorov","year":"2018","journal-title":"Front. Bioeng. Biotechnol."},{"key":"10.1016\/j.patter.2021.100322_bib82","series-title":"2019 IEEE\/ACM 16th International Conference on Mining Software Repositories (MSR)","first-page":"507","article-title":"A large-scale study about quality and reproducibility of Jupyter notebooks","author":"Pimentel","year":"2019"},{"key":"10.1016\/j.patter.2021.100322_bib83","doi-asserted-by":"crossref","first-page":"71","DOI":"10.1145\/2723872.2723882","article-title":"An introduction to docker for reproducible research","volume":"49","author":"Boettiger","year":"2015","journal-title":"Oper. Syst. Rev."},{"key":"10.1016\/j.patter.2021.100322_bib84","doi-asserted-by":"crossref","first-page":"475","DOI":"10.1038\/s41592-018-0046-7","article-title":"Bioconda: sustainable and comprehensive software distribution for the life sciences","volume":"15","author":"Gr\u00fcning","year":"2018","journal-title":"Nat. Methods"},{"key":"10.1016\/j.patter.2021.100322_bib85","doi-asserted-by":"crossref","first-page":"799","DOI":"10.1038\/nature05874","article-title":"Identification and analysis of functional elements in 1% of the human genome by the ENCODE pilot project","volume":"447","author":"Birney","year":"2007","journal-title":"Nature"},{"key":"10.1016\/j.patter.2021.100322_bib86","doi-asserted-by":"crossref","first-page":"508","DOI":"10.1016\/j.cels.2019.08.007","article-title":"Building containerized workflows using the BioDepot-workflow-builder","volume":"9","author":"Hung","year":"2019","journal-title":"Cell Syst."},{"key":"10.1016\/j.patter.2021.100322_bib87","doi-asserted-by":"crossref","first-page":"1443","DOI":"10.12688\/f1000research.7536.1","article-title":"BioShaDock: a community driven bioinformatics shared docker-based tools registry","volume":"4","author":"Moreews","year":"2015","journal-title":"F1000Res."},{"key":"10.1016\/j.patter.2021.100322_bib88","doi-asserted-by":"crossref","first-page":"47","DOI":"10.1186\/s13742-015-0087-0","article-title":"Bioboxes: standardised containers for Interchangeable bioinformatics software","volume":"4","author":"Belmann","year":"2015","journal-title":"Gigascience"},{"key":"10.1016\/j.patter.2021.100322_bib89","doi-asserted-by":"crossref","first-page":"2580","DOI":"10.1093\/bioinformatics\/btx192","article-title":"BioContainers: an open-source and community-driven framework for software standardization","volume":"33","author":"da Veiga Leprevost","year":"2017","journal-title":"Bioinformatics"},{"key":"10.1016\/j.patter.2021.100322_bib90","series-title":"Proceedings of the 10th Working Conference on Mining Software Repositories; MSR \u201913","first-page":"207","article-title":"Mining source code repositories at massive scale using language modeling","author":"Allamanis","year":"2013"},{"key":"10.1016\/j.patter.2021.100322_bib91","doi-asserted-by":"crossref","first-page":"R80","DOI":"10.1186\/gb-2004-5-10-r80","article-title":"Bioconductor: open software development for computational biology and bioinformatics","volume":"5","author":"Gentleman","year":"2004","journal-title":"Genome Biol."},{"key":"10.1016\/j.patter.2021.100322_bib92","first-page":"2825","article-title":"Scikit-learn: machine learning in Python","volume":"12","author":"Pedregosa","year":"2011","journal-title":"J.\u00a0Mach. Learn. Res."},{"key":"10.1016\/j.patter.2021.100322_bib93","article-title":"A realistic guide to making data available alongside code to improve reproducibility","author":"Tierney","year":"2020","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2021.100322_bib94","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1038\/s41467-021-22381-z","article-title":"Go get data (GGD) is a framework that facilitates reproducible access to genomic data","volume":"12","author":"Cormier","year":"2021","journal-title":"Nat. Commun."},{"key":"10.1016\/j.patter.2021.100322_bib95","unstructured":"Open Container Initiative https:\/\/www.opencontainers.org\/."},{"key":"10.1016\/j.patter.2021.100322_bib96","doi-asserted-by":"crossref","first-page":"125","DOI":"10.2218\/ijdc.v12i2.509","article-title":"A framework for the preservation of a docker container","volume":"12","author":"Emsley","year":"2018","journal-title":"Int. J. Digit. Curation"},{"key":"10.1016\/j.patter.2021.100322_bib97","first-page":"225","article-title":"Preserving containers \u2013 requirements and a todo-list","author":"Rechert","year":"2016","journal-title":"Digital libraries: knowledge, information, and data in an open access society"},{"key":"10.1016\/j.patter.2021.100322_bib98","series-title":"Ga4Gh\/Dockstore: 1.0","author":"Yuen","year":"2016"},{"key":"10.1016\/j.patter.2021.100322_bib99","series-title":"Compstat","first-page":"575","article-title":"Sweave: dynamic generation of statistical reports using literate data analysis","author":"Leisch","year":"2002"},{"key":"10.1016\/j.patter.2021.100322_bib100","first-page":"20","article-title":"Knitr: a comprehensive tool for reproducible research in R","volume":"1","author":"Xie","year":"2014","journal-title":"Implement Reprod. Res."},{"key":"10.1016\/j.patter.2021.100322_bib101","series-title":"RStudio: Integrated Development for Rl","year":"2015"},{"key":"10.1016\/j.patter.2021.100322_bib102","series-title":"Positioning and Power in Academic Publishing: Players, Agents and Agendas","first-page":"87","article-title":"Jupyter Notebooks\u2014A Publishing Format for Reproducible Computational Workflows","author":"Kluyver","year":"2016"},{"key":"10.1016\/j.patter.2021.100322_bib103","doi-asserted-by":"crossref","first-page":"151","DOI":"10.1038\/515151a","article-title":"Interactive notebooks: sharing the code","volume":"515","author":"Shen","year":"2014","journal-title":"Nature"},{"key":"10.1016\/j.patter.2021.100322_bib104","doi-asserted-by":"crossref","first-page":"375","DOI":"10.1080\/713827180","article-title":"Data preparation for data mining","volume":"17","author":"Zhang","year":"2003","journal-title":"Appl. Artif. Intell."},{"key":"10.1016\/j.patter.2021.100322_bib105","doi-asserted-by":"crossref","first-page":"252","DOI":"10.1152\/jn.00137.2016","article-title":"Neurophysiological analytics for all! Free open-source software tools for documenting, analyzing, visualizing, and sharing using electronic notebooks","volume":"116","author":"Rosenberg","year":"2016","journal-title":"J.\u00a0Neurophysiol."},{"key":"10.1016\/j.patter.2021.100322_bib106","first-page":"120","article-title":"Binder 2.0-reproducible, interactive, sharable environments for science at scale","volume":"113","author":"Bussonnier","year":"2018"},{"key":"10.1016\/j.patter.2021.100322_bib107","series-title":"Rticles: Article Formats for R Markdown","author":"Allaire","year":"2016"},{"key":"10.1016\/j.patter.2021.100322_bib108","article-title":"Improving reproducibility in machine learning research (A report from the NeurIPS 2019 reproducibility program)","author":"Pineau","year":"2020","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2021.100322_bib209","doi-asserted-by":"crossref","first-page":"70","DOI":"10.1038\/s41597-020-0409-7","article-title":"Semantic concept schema of the linear mixed model of experimental observations","volume":"7","author":"\u0106wiek-Kupczy\u0144ska","year":"2020","journal-title":"Sci. Data"},{"key":"10.1016\/j.patter.2021.100322_bib109","first-page":"39","article-title":"Accelerating the machine learning lifecycle with MLflow","volume":"41","author":"Zaharia","year":"2018","journal-title":"IEEE Data Eng. Bull."},{"key":"10.1016\/j.patter.2021.100322_bib110","first-page":"530","article-title":"A review of bioinformatic pipeline frameworks","volume":"18","author":"Leipzig","year":"2016","journal-title":"Brief. Bioinform."},{"key":"10.1016\/j.patter.2021.100322_bib111","doi-asserted-by":"crossref","first-page":"R86","DOI":"10.1186\/gb-2010-11-8-r86","article-title":"Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences","volume":"11","author":"Goecks","year":"2010","journal-title":"Genome Biol."},{"key":"10.1016\/j.patter.2021.100322_bib112","series-title":"Proceedings of the 16th International Conference on Scientific and Statistical Database Management, 2004","first-page":"423","article-title":"Kepler: an extensible system for design and execution of scientific workflows","author":"Altintas","year":"2004"},{"key":"10.1016\/j.patter.2021.100322_bib113","doi-asserted-by":"crossref","first-page":"26","DOI":"10.1145\/1656274.1656280","article-title":"Knime - the Konstanz information miner: version 2.0 and beyond","volume":"11","author":"Berthold","year":"2009","journal-title":"SIGKDD Explor. Newsl."},{"key":"10.1016\/j.patter.2021.100322_bib114","doi-asserted-by":"crossref","first-page":"W729","DOI":"10.1093\/nar\/gkl320","article-title":"Taverna: a tool for building and running workflows of services","volume":"34","author":"Hull","year":"2006","journal-title":"Nucleic Acids Res."},{"key":"10.1016\/j.patter.2021.100322_bib116","first-page":"154","article-title":"Rabix: an open-source workflow executor supporting recomputability and interoperability of workflow DescriptionS","volume":"22","author":"Kaushik","year":"2016","journal-title":"Pac. Symp. Biocomput."},{"key":"10.1016\/j.patter.2021.100322_bib117","doi-asserted-by":"crossref","first-page":"314","DOI":"10.1038\/nbt.3772","article-title":"Toil enables reproducible, open source, big biomedical data analyses","volume":"35","author":"Vivian","year":"2017","journal-title":"Nat. Biotechnol."},{"key":"10.1016\/j.patter.2021.100322_bib210","series-title":"Bioinformatics Open Source Conference","first-page":"2017","article-title":"CWL viewer: the common workflow language viewer","author":"Robinson","year":"2017"},{"key":"10.1016\/j.patter.2021.100322_bib119","doi-asserted-by":"crossref","first-page":"434","DOI":"10.1016\/j.neuron.2016.04.030","article-title":"A simple step toward improving reproducibility through rigor and transparency of experimental methods","volume":"90","author":"Bandrowski","year":"2016","journal-title":"Neuron"},{"key":"10.1016\/j.patter.2021.100322_bib120","first-page":"1","article-title":"A survey on collecting, managing, and analyzing provenance from scripts","author":"Pimentel","year":"2019","journal-title":"ACM Comput. Surv."},{"key":"10.1016\/j.patter.2021.100322_bib121","series-title":"6th USENIX Workshop on the Theory and Practice of Provenance","article-title":"RDataTracker: collecting provenance in an interactive scripting environment","author":"Lerner","year":"2014"},{"key":"10.1016\/j.patter.2021.100322_bib122","series-title":"8th USENIX Workshop on the Theory and Practice of Provenance","article-title":"Scaling SPADE to \u201cBig Provenance\u201d","author":"Gehani","year":"2016"},{"key":"10.1016\/j.patter.2021.100322_bib123","series-title":"Provenance and Annotation of Data Processes, IPAW 2010","first-page":"236","article-title":"StarFlow: a script-centric data analysis environment","author":"Angelino","year":"2010"},{"key":"10.1016\/j.patter.2021.100322_bib124","article-title":"YesWorkflow: a user-oriented, language-independent tool for recovering workflow information from scripts","author":"McPhillips","year":"2015","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2021.100322_bib125","doi-asserted-by":"crossref","first-page":"18","DOI":"10.1109\/MCSE.2012.76","article-title":"Making computations and publications reproducible with VisTrails","volume":"14","author":"Freire","year":"2012","journal-title":"Comput. Sci. Eng."},{"key":"10.1016\/j.patter.2021.100322_bib126","doi-asserted-by":"crossref","first-page":"271","DOI":"10.1016\/j.future.2017.01.008","article-title":"Abstract, link, publish, exploit: an end to end framework for workflow sharing","volume":"75","author":"Garijo","year":"2017","journal-title":"Future Gener. Comput. Syst."},{"key":"10.1016\/j.patter.2021.100322_bib128","article-title":"Opening the publication process with executable research compendia","volume":"23","author":"N\u00fcst","year":"2017","journal-title":"D-Lib Mag."},{"key":"10.1016\/j.patter.2021.100322_bib129","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/3331158","article-title":"Creating interactive scientific publications using bindings","volume":"3","author":"Konkol","year":"2019","journal-title":"Proc. ACM Hum.-Comput. Interact."},{"key":"10.1016\/j.patter.2021.100322_bib130","doi-asserted-by":"crossref","first-page":"599","DOI":"10.1016\/j.future.2011.08.004","article-title":"Why linked data is not enough for scientists","volume":"29","author":"Bechhofer","year":"2013","journal-title":"Future Gener. Comput. Syst."},{"key":"10.1016\/j.patter.2021.100322_bib132","article-title":"Application profiles: mixing and matching metadata schemas","author":"Heery","year":"2000","journal-title":"Ariadne"},{"key":"10.1016\/j.patter.2021.100322_bib133","doi-asserted-by":"crossref","first-page":"i601","DOI":"10.1093\/bioinformatics\/btu471","article-title":"Extracting patterns of database and software usage from the bioinformatics literature","volume":"30","author":"Duck","year":"2014","journal-title":"Bioinformatics"},{"key":"10.1016\/j.patter.2021.100322_bib134","doi-asserted-by":"crossref","first-page":"359","DOI":"10.1186\/1471-2105-9-359","article-title":"Methodology capture: Discriminating between the \u201cbest\u201d and the rest of community practice","volume":"9","author":"Eales","year":"2008","journal-title":"BMC Bioinformatics"},{"key":"10.1016\/j.patter.2021.100322_bib135","article-title":"Towards an ontology-based recommender system for relevant bioinformatics workflows","author":"Halioui","year":"2016","journal-title":"bioRxiv"},{"key":"10.1016\/j.patter.2021.100322_bib136","doi-asserted-by":"crossref","first-page":"10","DOI":"10.1016\/j.ijmedinf.2018.10.009","article-title":"ProvCaRe: characterizing scientific reproducibility of biomedical research studies using semantic provenance metadata","volume":"121","author":"Sahoo","year":"2019","journal-title":"Int. J. Med. Inform."},{"key":"10.1016\/j.patter.2021.100322_bib137","doi-asserted-by":"crossref","first-page":"319","DOI":"10.1007\/164_2019_290","article-title":"Publishers\u2019 responsibilities in promoting data quality and reproducibility","volume":"257","author":"Hrynaszkiewicz","year":"2020","journal-title":"Handb. Exp. Pharmacol."},{"key":"10.1016\/j.patter.2021.100322_bib138","article-title":"Code Execution in Peer Review","author":"N\u00fcst","year":"2021","journal-title":"Open Sci. Framework"},{"key":"10.1016\/j.patter.2021.100322_bib139","author":"Evanko"},{"key":"10.1016\/j.patter.2021.100322_bib140","doi-asserted-by":"crossref","first-page":"485","DOI":"10.1038\/nature10836","article-title":"The Case for open computer programs","volume":"482","author":"Ince","year":"2012","journal-title":"Nature"},{"key":"10.1016\/j.patter.2021.100322_bib141","doi-asserted-by":"crossref","first-page":"253","DOI":"10.12688\/f1000research.51738.2","article-title":"CODECHECK: an open science initiative for the independent execution of computations underlying research articles during peer review to improve reproducibility","volume":"10","author":"N\u00fcst","year":"2021","journal-title":"F1000Res."},{"key":"10.1016\/j.patter.2021.100322_bib143","first-page":"20190021","article-title":"The systems biology markup language (SBML): language specification for level 3 version 2 core release 2","volume":"16","author":"Hucka","year":"2019","journal-title":"J.\u00a0Integr. Bioinform."},{"key":"10.1016\/j.patter.2021.100322_bib144","doi-asserted-by":"crossref","first-page":"735","DOI":"10.1038\/nbt.1558","article-title":"The systems biology graphical notation","volume":"27","author":"Le Nov\u00e8re","year":"2009","journal-title":"Nat. Biotechnol."},{"key":"10.1016\/j.patter.2021.100322_bib145","doi-asserted-by":"crossref","first-page":"935","DOI":"10.1038\/nbt.1666","article-title":"The BioPAX community standard for pathway data sharing","volume":"28","author":"Demir","year":"2010","journal-title":"Nat. Biotechnol."},{"key":"10.1016\/j.patter.2021.100322_bib146","doi-asserted-by":"crossref","first-page":"D330","DOI":"10.1093\/nar\/gky1055","article-title":"The gene ontology resource: 20 Years and still GOing strong","volume":"47","year":"2019","journal-title":"Nucleic Acids Res."},{"key":"10.1016\/j.patter.2021.100322_bib147","doi-asserted-by":"crossref","first-page":"685","DOI":"10.1093\/nar\/gkq1039","article-title":"Pathway commons, a web resource for biological pathway data","volume":"39","author":"Cerami","year":"2011","journal-title":"Nucleic Acids Res."},{"key":"10.1016\/j.patter.2021.100322_bib148","doi-asserted-by":"crossref","first-page":"D649","DOI":"10.1093\/nar\/gkx1132","article-title":"The reactome pathway Knowledgebase","volume":"46","author":"Fabregat","year":"2018","journal-title":"Nucleic Acids Res."},{"key":"10.1016\/j.patter.2021.100322_bib149","doi-asserted-by":"crossref","first-page":"D353","DOI":"10.1093\/nar\/gkw1092","article-title":"KEGG: new perspectives on genomes, pathways, diseases and drugs","volume":"45","author":"Kanehisa","year":"2017","journal-title":"Nucleic Acids Res."},{"key":"10.1016\/j.patter.2021.100322_bib150","doi-asserted-by":"crossref","first-page":"D548","DOI":"10.1093\/nar\/gkv1048","article-title":"SIGNOR: a database of causal relationships between biological entities","volume":"44","author":"Perfetto","year":"2016","journal-title":"Nucleic Acids Res."},{"key":"10.1016\/j.patter.2021.100322_bib151","doi-asserted-by":"crossref","first-page":"D661","DOI":"10.1093\/nar\/gkx1064","article-title":"WikiPathways: a multifaceted pathway database bridging metabolomics to other omics research","volume":"46","author":"Slenter","year":"2018","journal-title":"Nucleic Acids Res."},{"key":"10.1016\/j.patter.2021.100322_bib152","doi-asserted-by":"crossref","first-page":"baz068","DOI":"10.1093\/database\/baz068","article-title":"Re-curation and rational enrichment of knowledge graphs in biological Expression Language","volume":"2019","author":"Hoyt","year":"2019","journal-title":"Database"},{"key":"10.1016\/j.patter.2021.100322_bib153","doi-asserted-by":"crossref","first-page":"baw136","DOI":"10.1093\/database\/baw136","article-title":"The BEL information extraction workflow (BELIEF): evaluation in the BioCreative V BEL and IAT track","volume":"2016","author":"Madan","year":"2016","journal-title":"Database"},{"key":"10.1016\/j.patter.2021.100322_bib154","series-title":"Proceedings of the 2008 Conference on Semantics in Text Processing","first-page":"343","article-title":"Deep semantic analysis of text","author":"Allen","year":"2008"},{"key":"10.1016\/j.patter.2021.100322_bib155","series-title":"Natural Language Processing and Knowledge Representation: Language for Knowledge and Knowledge for Language","first-page":"77","article-title":"Issues in the representation of real texts: the design of Krisp","author":"McDonald","year":"2000"},{"key":"10.1016\/j.patter.2021.100322_bib156","doi-asserted-by":"crossref","first-page":"bay098","DOI":"10.1093\/database\/bay098","article-title":"Large-scale Automated machine reading discovers new cancer-driving mechanisms","volume":"2018","author":"Valenzuela-Esc\u00e1rcega","year":"2018","journal-title":"Database"},{"key":"10.1016\/j.patter.2021.100322_bib157","doi-asserted-by":"crossref","first-page":"954","DOI":"10.15252\/msb.20177651","article-title":"From word models to executable models of signaling networks using automated assembly","volume":"13","author":"Gyori","year":"2017","journal-title":"Mol. Syst. Biol."},{"key":"10.1016\/j.patter.2021.100322_bib158","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1186\/s12859-018-2211-5","article-title":"FamPlex: a resource for entity recognition and relationship resolution of human protein families and complexes in biomedical text mining","volume":"19","author":"Bachman","year":"2018","journal-title":"BMC Bioinformatics"},{"key":"10.1016\/j.patter.2021.100322_bib159","series-title":"Introducing eLife\u2019s First Computationally Reproducible Article","author":"Maciocci","year":"2019"},{"key":"10.1016\/j.patter.2021.100322_bib161","author":"Tsang"},{"key":"10.1016\/j.patter.2021.100322_bib162","author":"Guizzardi"},{"key":"10.1016\/j.patter.2021.100322_bib163","doi-asserted-by":"crossref","first-page":"17","DOI":"10.1300\/J104v40n03_02","article-title":"Understanding metadata and metadata schemes","volume":"40","author":"Greenberg","year":"2005","journal-title":"Cataloging classification Q."},{"key":"10.1016\/j.patter.2021.100322_bib164","doi-asserted-by":"crossref","first-page":"170124","DOI":"10.1038\/sdata.2017.124","article-title":"The public cancer radiology imaging collections of the cancer imaging archive","volume":"4","author":"Prior","year":"2017","journal-title":"Sci. Data"},{"key":"10.1016\/j.patter.2021.100322_bib165","series-title":"2015 37th Annual International Conference of the IEEE Engineering in Medicine and Biology Society","first-page":"2551","article-title":"An automatic method for the enrichment of DICOM metadata using biomedical ontologies","author":"P\u00e9rez","year":"2015"},{"key":"10.1016\/j.patter.2021.100322_bib166","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1148\/radiol.15150144","article-title":"DOIs for DICOM raw images: enabling science reproducibility","volume":"275","author":"Bourne","year":"2015","journal-title":"Radiology"},{"key":"10.1016\/j.patter.2021.100322_bib167","doi-asserted-by":"crossref","first-page":"2078","DOI":"10.1093\/bioinformatics\/btp352","article-title":"The sequence alignment\/map format and SAMtools","volume":"25","author":"Li","year":"2009","journal-title":"Bioinformatics"},{"key":"10.1016\/j.patter.2021.100322_bib168","doi-asserted-by":"crossref","first-page":"2236","DOI":"10.1093\/bioinformatics\/btw214","article-title":"DisGeNET-RDF: Harnessing the innovative power of the semantic web to explore the genetic basis of diseases","volume":"32","author":"Queralt-Rosinach","year":"2016","journal-title":"Bioinformatics"},{"key":"10.1016\/j.patter.2021.100322_bib169","series-title":"International Encyclopedia of Geography: People, the Earth, Environment and Technology","first-page":"1","article-title":"Geospatial Semantic Web","volume":"284","author":"Janowicz","year":"2016"},{"key":"10.1016\/j.patter.2021.100322_bib170","series-title":"The Semantic Web","first-page":"722","article-title":"DBpedia: A nucleus for a web of open data","author":"Auer","year":"2007"},{"key":"10.1016\/j.patter.2021.100322_bib171","first-page":"401","article-title":"Bio2RDF release 3: a larger connected network of linked data for the life sciences","volume":"1272","author":"Dumontier","year":"2014"},{"key":"10.1016\/j.patter.2021.100322_bib172","article-title":"Machine learning with biomedical ontologies","author":"Kulmanov","year":"2020","journal-title":"bioRxiv"},{"key":"10.1016\/j.patter.2021.100322_bib173","article-title":"The KEEN universe: an ecosystem for knowledge graph embeddings with a focus on reproducibility and transferability","author":"Ali","year":"2020","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2021.100322_bib174","doi-asserted-by":"crossref","first-page":"207","DOI":"10.1186\/gb-2010-11-5-207","article-title":"The case for cloud computing in genome Informatics","volume":"11","author":"Stein","year":"2010","journal-title":"Genome Biol."},{"key":"10.1016\/j.patter.2021.100322_bib175","article-title":"Towards the Preservation of Scientific Workflows","author":"De Roure","year":"2011"},{"key":"10.1016\/j.patter.2021.100322_bib176","doi-asserted-by":"crossref","first-page":"D1172","DOI":"10.1093\/nar\/gky1061","article-title":"BioSamples database: an updated sample metadata hub","volume":"47","author":"Courtot","year":"2019","journal-title":"Nucleic Acids Res."},{"key":"10.1016\/j.patter.2021.100322_bib177","doi-asserted-by":"crossref","first-page":"59","DOI":"10.1186\/s13742-015-0092-3","article-title":"Enhanced reproducibility of SADI web service workflows with Galaxy and Docker","volume":"4","author":"Aranguren","year":"2015","journal-title":"Gigascience"},{"key":"10.1016\/j.patter.2021.100322_bib178","doi-asserted-by":"crossref","first-page":"465","DOI":"10.1002\/wcms.1127","article-title":"Cheminformatics and the semantic web: adding value with linked data and enhanced provenance","volume":"3","author":"Frey","year":"2013","journal-title":"Wiley Interdiscip. Rev. Comput. Mol. Sci."},{"key":"10.1016\/j.patter.2021.100322_bib179","doi-asserted-by":"crossref","first-page":"136","DOI":"10.5731\/pdajpst.2016.006734","article-title":"Biocompute objects-A step towards evaluation and validation of biomedical scientific computations","volume":"71","author":"Simonyan","year":"2017","journal-title":"PDA J. Pharm. Sci. Technol."},{"key":"10.1016\/j.patter.2021.100322_bib181","series-title":"Proceedings of the 2016 International Conference on Management of Data; SIGMOD \u201916","first-page":"2085","article-title":"ReproZip: computational reproducibility with ease","author":"Chirigati","year":"2016"},{"key":"10.1016\/j.patter.2021.100322_bib182","doi-asserted-by":"crossref","first-page":"e1007664","DOI":"10.1371\/journal.pcbi.1007664","article-title":"Reference sequence checksums for provenance identification in RNA-seq","volume":"16","author":"Love","year":"2020","journal-title":"PLoS Comput. Biol."},{"key":"10.1016\/j.patter.2021.100322_bib183","first-page":"193","article-title":"Big metadata, smart metadata, and metadata capital: toward greater synergy between data science and metadata","volume":"2","author":"Greenberg","year":"2017","journal-title":"J.\u00a0Data Inf. Sci."},{"key":"10.1016\/j.patter.2021.100322_bib184","series-title":"Proceedings of the Conference on Artificial Intelligence for Data Discovery and Reuse","article-title":"Artificial intelligence for data discovery and reuse demands healthy data ecosystem and community efforts","author":"Wang","year":"2019"},{"key":"10.1016\/j.patter.2021.100322_bib185","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1002\/meet.2014.14505101155","article-title":"Examining data sharing and data reuse in the DataONE environment","volume":"51","author":"Murillo","year":"2014","journal-title":"Proc. Am. Soc. Inf. Sci. Technol."},{"key":"10.1016\/j.patter.2021.100322_bib186","doi-asserted-by":"crossref","first-page":"2914","DOI":"10.1093\/bioinformatics\/btx334","article-title":"MetaSRA: Normalized human sample-specific metadata for the sequence read archive","volume":"33","author":"Bernstein","year":"2017","journal-title":"Bioinformatics"},{"key":"10.1016\/j.patter.2021.100322_bib187","unstructured":"DUO: the Data Use Ontology (Github)."},{"key":"10.1016\/j.patter.2021.100322_bib188","doi-asserted-by":"crossref","first-page":"13","DOI":"10.1109\/MCSE.2012.38","article-title":"Reproducible research for scientific computing: tools and strategies for changing the culture","volume":"14","author":"LeVeque","year":"2012","journal-title":"Comput. Sci. Eng."},{"key":"10.1016\/j.patter.2021.100322_bib189","article-title":"Case studies and challenges in reproducibility in the computational sciences","author":"Arabas","year":"2014","journal-title":"arXiv"},{"key":"10.1016\/j.patter.2021.100322_bib190","doi-asserted-by":"crossref","first-page":"01620001","DOI":"10.1061\/(ASCE)WR.1943-5452.0001368","article-title":"Reproducible results policy","volume":"147","author":"Rosenberg","year":"2021","journal-title":"J.\u00a0Water Resour. Plan. Manag."},{"key":"10.1016\/j.patter.2021.100322_bib191","article-title":"Guerrilla Badges for Reproducible Geospatial Data Science","author":"N\u00fcst","year":"2019","journal-title":"Earth ArXiv"},{"key":"10.1016\/j.patter.2021.100322_bib193","doi-asserted-by":"crossref","first-page":"1605","DOI":"10.12688\/f1000research.16493.1","article-title":"The evaluation of scholarship in academic promotion and tenure processes: past, present, and future","volume":"7","author":"Schimanski","year":"2018","journal-title":"F1000Res."},{"key":"10.1016\/j.patter.2021.100322_bib194","doi-asserted-by":"crossref","first-page":"1257","DOI":"10.12688\/f1000research.26932.1","article-title":"Recognizing the value of software: a software citation guide","volume":"9","author":"Katz","year":"2020","journal-title":"F1000Res."},{"key":"10.1016\/j.patter.2021.100322_bib195","doi-asserted-by":"crossref","first-page":"e147","DOI":"10.7717\/peerj-cs.147","article-title":"Journal of open source software (JOSS): design and first-year review","volume":"4","author":"Smith","year":"2018","journal-title":"Peerj Comput. Sci."},{"key":"10.1016\/j.patter.2021.100322_bib196","doi-asserted-by":"crossref","first-page":"417","DOI":"10.1016\/j.cels.2019.09.011","article-title":"FAIRshake: toolkit to evaluate the FAIRness of research digital resources","volume":"9","author":"Clarke","year":"2019","journal-title":"Cell Syst."},{"key":"10.1016\/j.patter.2021.100322_bib197","doi-asserted-by":"crossref","first-page":"24","DOI":"10.1186\/1741-7015-8-24","article-title":"Transparent and accurate reporting increases reliability, utility, and impact of Your research: reporting guidelines and the EQUATOR network","volume":"8","author":"Simera","year":"2010","journal-title":"BMC Med."},{"key":"10.1016\/j.patter.2021.100322_bib198","doi-asserted-by":"crossref","first-page":"18","DOI":"10.1186\/1741-7015-8-18","article-title":"CONSORT 2010 statement: updated guidelines for reporting parallel group randomised trials","volume":"8","author":"Schulz","year":"2010","journal-title":"BMC Med."},{"key":"10.1016\/j.patter.2021.100322_bib199","doi-asserted-by":"crossref","first-page":"e1007128","DOI":"10.1371\/journal.pcbi.1007128","article-title":"Open collaborative writing with Manubot","volume":"15","author":"Himmelstein","year":"2019","journal-title":"PLoS Comput. Biol."},{"key":"10.1016\/j.patter.2021.100322_bib200","doi-asserted-by":"crossref","first-page":"295","DOI":"10.12688\/f1000research.23224.2","article-title":"An environment for sustainable research software in Germany and beyond: current state, open challenges, and call for action","volume":"9","author":"Anzt","year":"2021","journal-title":"F1000Res."},{"key":"10.1016\/j.patter.2021.100322_bib201","doi-asserted-by":"crossref","first-page":"2959","DOI":"10.21105\/joss.02959","article-title":"The targets R package: a dynamic make-like function-oriented pipeline toolkit for reproducibility and high-performance computing","volume":"6","author":"Landau","year":"2021","journal-title":"J.\u00a0Open Source Softw."},{"key":"10.1016\/j.patter.2021.100322_bib202","article-title":"The Rold of Metadata in Statistics","author":"Dippo","year":"2000","journal-title":"U.S. Bureau of Labor Statistics"},{"key":"10.1016\/j.patter.2021.100322_bib203","doi-asserted-by":"crossref","first-page":"1393","DOI":"10.1038\/s41467-019-09406-4","article-title":"Systematic benchmarking of omics computational tools","volume":"10","author":"Mangul","year":"2019","journal-title":"Nat. Commun."}],"container-title":["Patterns"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2666389921001707?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S2666389921001707?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,11,12]],"date-time":"2025-11-12T23:45:51Z","timestamp":1762991151000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S2666389921001707"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,9]]},"references-count":235,"journal-issue":{"issue":"9","published-print":{"date-parts":[[2021,9]]}},"alternative-id":["S2666389921001707"],"URL":"https:\/\/doi.org\/10.1016\/j.patter.2021.100322","relation":{},"ISSN":["2666-3899"],"issn-type":[{"value":"2666-3899","type":"print"}],"subject":[],"published":{"date-parts":[[2021,9]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"The role of metadata in reproducible computational research","name":"articletitle","label":"Article Title"},{"value":"Patterns","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.patter.2021.100322","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2021 The Author(s).","name":"copyright","label":"Copyright"}],"article-number":"100322"}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1016_2Fj_websem_2015_04_001 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,24]],"date-time":"2026-10-24T04:02:02Z","timestamp":1792814522428,"version":"4.5.0"},"reference-count":99,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2015,12,1]],"date-time":"2015-12-01T00:00:00Z","timestamp":1448928000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2015,4,13]],"date-time":"2015-04-13T00:00:00Z","timestamp":1428883200000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Journal of Web Semantics"],"published-print":{"date-parts":[[2015,12]]},"DOI":"10.1016\/j.websem.2015.04.001","type":"journal-article","created":{"date-parts":[[2015,4,20]],"date-time":"2015-04-20T17:34:39Z","timestamp":1429551279000},"page":"235-257","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":100,"special_numbering":"P4","title":["The rationale of PROV"],"prefix":"10.1016","volume":"35","author":[{"given":"Luc","family":"Moreau","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0183-6910","authenticated-orcid":false,"given":"Paul","family":"Groth","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"James","family":"Cheney","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8042-7359","authenticated-orcid":false,"given":"Timothy","family":"Lebo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9956-4217","authenticated-orcid":false,"given":"Simon","family":"Miles","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"10.1016\/j.websem.2015.04.001_br000005","unstructured":"K. Belhajjame, R. B\u2019Far, J. Cheney, S. Coppens, S. Cresswell, Y. Gil, P. Groth, G. Klyne, T. Lebo, J. McCusker, S. Miles, J. Myers, S. Sahoo, C. Tilmes, L. Moreau, and P. Missier (Eds.), PROV-DM: The PROV Data Model, W3C Recommendation REC-prov-dm-20130430, World Wide Web Consortium (Oct. 2013). URL http:\/\/www.w3.org\/TR\/2013\/REC-prov-dm-20130430\/."},{"issue":"1","key":"10.1016\/j.websem.2015.04.001_br000010","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1145\/1057977.1057978","article-title":"Lineage retrieval for scientific data processing: a survey","volume":"37","author":"Bose","year":"2005","journal-title":"ACM Comput. Surv."},{"issue":"3","key":"10.1016\/j.websem.2015.04.001_br000015","doi-asserted-by":"crossref","first-page":"31","DOI":"10.1145\/1084805.1084812","article-title":"A survey of data provenance in e-science","volume":"34","author":"Simmhan","year":"2005","journal-title":"SIGMOD Record"},{"issue":"4","key":"10.1016\/j.websem.2015.04.001_br000020","doi-asserted-by":"crossref","first-page":"379","DOI":"10.1561\/1900000006","article-title":"Provenance in databases: Why, how, and where","volume":"1","author":"Cheney","year":"2009","journal-title":"Found. Trends Databases"},{"issue":"2\u20133","key":"10.1016\/j.websem.2015.04.001_br000025","doi-asserted-by":"crossref","first-page":"99","DOI":"10.1561\/1800000010","article-title":"The foundations for provenance on the web","volume":"2","author":"Moreau","year":"2010","journal-title":"Found. Trends Web Sci."},{"issue":"5","key":"10.1016\/j.websem.2015.04.001_br000030","doi-asserted-by":"crossref","first-page":"409","DOI":"10.1002\/cpe.1233","article-title":"The first provenance challenge","volume":"20","author":"Moreau","year":"2008","journal-title":"Concurrency, Pract. Exp."},{"issue":"6","key":"10.1016\/j.websem.2015.04.001_br000035","doi-asserted-by":"crossref","first-page":"737","DOI":"10.1016\/j.future.2010.11.020","article-title":"Special issue: the third provenance challenge on using the open provenance model for interoperability (editorial)","volume":"27","author":"Simmhan","year":"2011","journal-title":"Future Gener. Comput. Syst."},{"issue":"6","key":"10.1016\/j.websem.2015.04.001_br000040","doi-asserted-by":"crossref","first-page":"743","DOI":"10.1016\/j.future.2010.07.005","article-title":"The open provenance model core specification (v1.1)","volume":"27","author":"Moreau","year":"2011","journal-title":"Future Gener. Comput. Syst."},{"key":"10.1016\/j.websem.2015.04.001_br000045","unstructured":"Y. Gil, J. Cheney, P. Groth, O. Hartig, S. Miles, L. Moreau, P. Pinheiro da Silva, Provenance XG Final Report, Tech. rep., World Wide Web Consortium (2010). URL http:\/\/www.w3.org\/2005\/Incubator\/prov\/XGR-prov-20101214\/."},{"key":"10.1016\/j.websem.2015.04.001_br000050","unstructured":"P. Groth, and L. Moreau (Eds.), PROV-Overview. An Overview of the PROV Family of Documents, W3C Working Group Note NOTE-prov-overview-20130430, World Wide Web Consortium (Apr. 2013). URL http:\/\/www.w3.org\/TR\/2013\/NOTE-prov-overview-20130430\/."},{"key":"10.1016\/j.websem.2015.04.001_br000055","unstructured":"K. Belhajjame, H. Deus, D. Garijo, G. Klyne, P. Missier, S. Soiland-Reyes, S. Zednik, Y. Gil, and S. M. (Eds.), Prov model primer, W3C Working Group Note NOTE-prov-primer-20130430, World Wide Web Consortium (Apr. 2013). URL http:\/\/www.w3.org\/TR\/prov-primer\/."},{"key":"10.1016\/j.websem.2015.04.001_br000060","unstructured":"J. Cheney, S. Soiland-Reyes, L. Moreau, and P. Missier (Eds.), PROV-N: The Provenance Notation, W3C Recommendation REC-prov-n-20130430, World Wide Web Consortium (Oct. 2013). URL http:\/\/www.w3.org\/TR\/2013\/REC-prov-n-20130430\/."},{"key":"10.1016\/j.websem.2015.04.001_br000065","unstructured":"T.D. Nies, J. Cheney, P. Missier, and L. Moreau (Eds.), Constraints of the PROV Data Model, W3C Recommendation REC-prov-constraints-20130430, World Wide Web Consortium (Oct. 2013). URL http:\/\/www.w3.org\/TR\/2013\/REC-prov-constraints-20130430\/."},{"key":"10.1016\/j.websem.2015.04.001_br000070","unstructured":"K. Behajjame, J. Cheney, D. Corsar, D. Garijo, S. Soiland-Reyes, S. Zednik, J. Zhao, T. Lebo, S. Sahoo, and D. McGuinness (Eds.), PROV-O: The PROV Ontology, W3C Recommendation REC-prov-o-20130430, World Wide Web Consortium (Oct. 2013). URL http:\/\/www.w3.org\/TR\/2013\/REC-prov-o-20130430\/."},{"key":"10.1016\/j.websem.2015.04.001_br000075","unstructured":"L. Moreau, H. Hua, C. Tilmes, and S. Zednik (Eds.), PROV-XML: The PROV XML Schema, W3C Working Group Note NOTE-prov-xml-20130430, World Wide Web Consortium (Apr. 2013). URL http:\/\/www.w3.org\/TR\/2013\/NOTE-prov-xml-20130430\/."},{"key":"10.1016\/j.websem.2015.04.001_br000080","unstructured":"S. Miles, C.M. Trim, M. Panzer, D. Garijo, and K. Eckert (Eds.), Dublin Core to PROV Mapping, W3C Working Group Note NOTE-prov-dc-20130430, World Wide Web Consortium (Apr. 2013). URL http:\/\/www.w3.org\/TR\/2013\/NOTE-prov-dc-20130430\/."},{"key":"10.1016\/j.websem.2015.04.001_br000085","unstructured":"J. Cheney, Semantics of the PROV Data Model, W3C Working Group Note NOTE-prov-sem-20130430, World Wide Web Consortium (Apr. 2013). URL http:\/\/www.w3.org\/TR\/2013\/NOTE-prov-sem-20130430\/."},{"key":"10.1016\/j.websem.2015.04.001_br000090","unstructured":"L. Moreau, T. Lebo, Linking across provenance bundles, W3C Working Group Note NOTE-prov-sem-20130430, World Wide Web Consortium (Apr. 2013). URL http:\/\/www.w3.org\/TR\/2013\/NOTE-prov-links-20130430\/."},{"key":"10.1016\/j.websem.2015.04.001_br000095","unstructured":"P. Missier, L. Moreau, J. Cheney, T. Lebo, S. Soiland-Reyes, T. De Nies, and S. C (Eds.), Prov-dictionary: Modeling provenance for dictionary data structures, W3C Working Group Note NOTE-prov-dictionary-20130430, World Wide Web Consortium (Apr. 2013). URL http:\/\/www.w3.org\/TR\/2013\/NOTE-prov-dictionary-20130430\/."},{"key":"10.1016\/j.websem.2015.04.001_br000100","unstructured":"L. Moreau, O. Hartig, Y. Simmhan, J. Myers, T. Lebo, K. Belhajjame, S. Miles, G. Klyne, and P. Groth (Eds.), PROV-AQ: Provenance Access and Query, W3C Working Group Note NOTE-prov-aq-20130430, World Wide Web Consortium (Apr. 2013). URL http:\/\/www.w3.org\/TR\/2013\/NOTE-prov-aq-20130430\/."},{"key":"10.1016\/j.websem.2015.04.001_br000105","unstructured":"T.D. Huynh, P. Groth, and S. Z. (Eds.), Prov implementation report, W3C Working Group Note NOTE-prov-overview-20130430, World Wide Web Consortium (April 2013). URL http:\/\/www.w3.org\/TR\/2013\/NOTE-prov-implementations-20130430\/."},{"key":"10.1016\/j.websem.2015.04.001_br000110","doi-asserted-by":"crossref","unstructured":"L. Moreau, P. Groth, Provenance: An Introduction to PROV, Morgan and Claypool, 2013. http:\/\/dx.doi.org\/10.2200\/S00528ED1V01Y201308WBE007.","DOI":"10.2200\/S00528ED1V01Y201308WBE007"},{"issue":"1","key":"10.1016\/j.websem.2015.04.001_br000115","doi-asserted-by":"crossref","first-page":"7","DOI":"10.1016\/j.websem.2003.07.001","article-title":"From SHIQ and RDF to OWL: The making of a web ontology language","volume":"1","author":"Horrocks","year":"2003","journal-title":"Web Semant. Sci. Serv. Agents WWW"},{"issue":"0","key":"10.1016\/j.websem.2015.04.001_br000120","doi-asserted-by":"crossref","first-page":"35","DOI":"10.1016\/j.websem.2013.05.001","article-title":"Key choices in the design of simple knowledge organization system (skos)","volume":"20","author":"Baker","year":"2013","journal-title":"Web Semant. Sci. Serv. Agents WWW"},{"issue":"1","key":"10.1016\/j.websem.2015.04.001_br000125","doi-asserted-by":"crossref","first-page":"39","DOI":"10.2218\/ijdc.v7i1.213","article-title":"Requirements for provenance on the web","volume":"7","author":"Groth","year":"2012","journal-title":"Int. J. Digital Curation"},{"key":"10.1016\/j.websem.2015.04.001_br000130","unstructured":"Y.R. Wang, S.E. Madnick, A polygen model for heterogeneous database systems: The source tagging perspective, in: Proceedings of the 16th International Conference on Very Large Data Bases (VLDB\u201990), Morgan Kaufmann, 1990, pp. 519\u2013538. URL http:\/\/web.mit.edu\/tdqm\/www\/tdqmpub\/polygenmodelAug90.pdf."},{"key":"10.1016\/j.websem.2015.04.001_br000135","doi-asserted-by":"crossref","unstructured":"P. Buneman, J. Cheney, W.-C. Tan, S. Vansummeren, Curated databases, in: PODS, 2008, pp. 1\u201312. http:\/\/dx.doi.org\/10.1007\/978-3-642-04346-8_2.","DOI":"10.1007\/978-3-642-04346-8_2"},{"issue":"2","key":"10.1016\/j.websem.2015.04.001_br000140","doi-asserted-by":"crossref","first-page":"179","DOI":"10.1145\/357775.357777","article-title":"Tracing the lineage of view data in a warehousing environment","volume":"25","author":"Cui","year":"2000","journal-title":"ACM Trans. Database Syst."},{"key":"10.1016\/j.websem.2015.04.001_br000145","doi-asserted-by":"crossref","unstructured":"P. Buneman, S. Khanna, W. Tan, Why and where: A characterization of data provenance, in: ICDT, no. 1973 in LNCS, 2001, pp. 316\u2013330. http:\/\/dx.doi.org\/10.1007\/3-540-44503-X_20.","DOI":"10.1007\/3-540-44503-X_20"},{"key":"10.1016\/j.websem.2015.04.001_br000150","doi-asserted-by":"crossref","unstructured":"T.J. Green, G. Karvounarakis, V. Tannen, Provenance semirings, in: PODS, 2007, pp. 31\u201340. http:\/\/dx.doi.org\/10.1145\/1265530.1265535.","DOI":"10.1145\/1265530.1265535"},{"key":"10.1016\/j.websem.2015.04.001_br000155","doi-asserted-by":"crossref","unstructured":"F. Geerts, G. Karvounarakis, V. Christophides, I. Fundulaki, Algebraic structures for capturing the provenance of SPARQL queries, in: Joint 2013 EDBT\/ICDT Conferences, ICDT \u201913 Proceedings, Genoa, Italy, March 18\u201322, 2013, pp. 153\u2013164. http:\/\/dx.doi.org\/10.1145\/2448496.2448516.","DOI":"10.1145\/2448496.2448516"},{"key":"10.1016\/j.websem.2015.04.001_br000160","doi-asserted-by":"crossref","unstructured":"P. Buneman, A.P. Chapman, J. Cheney, Provenance management in curated databases, in: Proceedings of the 2006 SIGMOD Conference on Management of Data (SIGMOD 2006), Chicago, IL, 2006, pp. 539\u2013550. http:\/\/dx.doi.org\/10.1145\/1142473.1142534.","DOI":"10.1145\/1142473.1142534"},{"key":"10.1016\/j.websem.2015.04.001_br000165","doi-asserted-by":"crossref","first-page":"2","DOI":"10.1145\/974750.974752","article-title":"Archiving scientific data","volume":"29","author":"Buneman","year":"2004","journal-title":"ACM Trans. Database Syst."},{"key":"10.1016\/j.websem.2015.04.001_br000170","doi-asserted-by":"crossref","unstructured":"S.B. Davidson, J. Freire, Provenance and scientific workflows: challenges and opportunities, in: Proceedings of ACM Special Interest Group on Management of Data (SIGMOD 2008), 2008, pp. 1345\u20131350. http:\/\/dx.doi.org\/10.1145\/1376616.1376772.","DOI":"10.1145\/1376616.1376772"},{"issue":"1","key":"10.1016\/j.websem.2015.04.001_br000175","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1007\/s10723-006-9055-3","article-title":"The requirements of recording and using provenance in e-science experiments","volume":"5","author":"Miles","year":"2007","journal-title":"J. Grid Comput."},{"issue":"5","key":"10.1016\/j.websem.2015.04.001_br000180","article-title":"Automatic capture and efficient storage of e-science experiment provenance","volume":"20","author":"Barga","year":"2008","journal-title":"Concurrency, Pract. Exp."},{"issue":"5","key":"10.1016\/j.websem.2015.04.001_br000185","doi-asserted-by":"crossref","first-page":"497","DOI":"10.1002\/cpe.1232","article-title":"Addressing the provenance challenge using zoom","volume":"20","author":"Cohen-Boulakia","year":"2008","journal-title":"Concurrency, Pract. Exp."},{"issue":"2","key":"10.1016\/j.websem.2015.04.001_br000190","doi-asserted-by":"crossref","DOI":"10.4018\/jwsr.2008040101","article-title":"Karma2: Provenance management for data driven workflows","volume":"5","author":"Simmhan","year":"2008","journal-title":"Int. J. Web Serv. Res."},{"key":"10.1016\/j.websem.2015.04.001_br000195","doi-asserted-by":"crossref","unstructured":"S. Bowers, T.M. McPhillips, B. Ludaescher, Provenance in collection-oriented scientific workflows, Concurrency and Computation: Practice and Experience 20\u00a0(5). http:\/\/dx.doi.org\/10.1002\/cpe.1226.","DOI":"10.1002\/cpe.1226"},{"key":"10.1016\/j.websem.2015.04.001_br000200","doi-asserted-by":"crossref","unstructured":"D. Garijo, O. Corcho, Y. Gil, Detecting common scientific workflow fragments using templates and execution provenance, in: Proceedings of the Seventh International Conference on Knowledge Capture, K-CAP \u201913, 2013, pp. 33\u201340. http:\/\/dx.doi.org\/10.1145\/2479832.2479848.","DOI":"10.1145\/2479832.2479848"},{"issue":"5","key":"10.1016\/j.websem.2015.04.001_br000205","doi-asserted-by":"crossref","first-page":"463","DOI":"10.1002\/cpe.1231","article-title":"Mining taverna\u2019s semantic web of provenance","volume":"20","author":"Zhao","year":"2008","journal-title":"Concurrency, Pract. Exp."},{"key":"10.1016\/j.websem.2015.04.001_br000210","unstructured":"P.P. da\u00a0Silva, L. Salayandia, A. Gates, Wdo-it! A Tool for Building Scientific Workflows from Ontologies, Tech. Rep. UTEP-CS-07-XX, University of Texas at El Paso (2007). URL http:\/\/digitalcommons.utep.edu\/cs_techrep\/201."},{"key":"10.1016\/j.websem.2015.04.001_br000215","series-title":"Proceedings of the International Provenance and Annotation Workshop 2006 (IPAW\u20192006)","first-page":"10","article-title":"Managing rapidly-evolving scientific workflows","author":"Freire","year":"2006"},{"issue":"3","key":"10.1016\/j.websem.2015.04.001_br000220","doi-asserted-by":"crossref","first-page":"38","DOI":"10.1109\/MCSE.2008.82","article-title":"Provenance: The bridge between experiments and data","volume":"10","author":"Miles","year":"2008","journal-title":"Comput. Sci. Eng."},{"key":"10.1016\/j.websem.2015.04.001_br000225","unstructured":"P. Missier, S. Dey, K. Belhajjame, V. Cuevas-Vicentt\u00edn, B. Ludaescher, D-prov: Extending the prov provenance model with workflow structure, in: Proceedings of the 5th USENIX Workshop on the Theory and Practice of Provenance (TaPP\u201913), 2013, pp. 9:1\u20139:7. URL https:\/\/www.usenix.org\/conference\/tapp13\/technical-sessions\/presentation\/missier."},{"key":"10.1016\/j.websem.2015.04.001_br000230","series-title":"Provenance and Annotation of Data\u2014International Provenance and Annotation Workshop","volume":"Vol. 4145","year":"2006"},{"key":"10.1016\/j.websem.2015.04.001_br000235","unstructured":"The First Provenance Challenge Web page. URL http:\/\/twiki.ipaw.info\/bin\/view\/Challenge\/FirstProvenanceChallenge (last accessed Nov 2013)."},{"key":"10.1016\/j.websem.2015.04.001_br000240","unstructured":"The Second Provenance Challenge Web page. URL http:\/\/twiki.ipaw.info\/bin\/view\/Challenge\/SecondProvenanceChallenge (last accessed Nov 2013)."},{"key":"10.1016\/j.websem.2015.04.001_br000245","unstructured":"Technical Summary of the Second Provenance Challenge Workshop. URL http:\/\/twiki.ipaw.info\/bin\/view\/Challenge\/SecondWorkshopMinutes (last accessed Nov 2013)."},{"key":"10.1016\/j.websem.2015.04.001_br000250","unstructured":"The Third Provenance Challenge Web page. URL http:\/\/twiki.ipaw.info\/bin\/view\/Challenge\/ThirdProvenanceChallenge (last accessed Nov 2013)."},{"issue":"4","key":"10.1016\/j.websem.2015.04.001_br000255","doi-asserted-by":"crossref","first-page":"381","DOI":"10.1016\/j.is.2005.02.003","article-title":"A proof markup language for semantic web services","volume":"31","author":"da~Silva","year":"2006","journal-title":"Inf. Syst."},{"key":"10.1016\/j.websem.2015.04.001_br000260","unstructured":"D.L. McGuinness, L. Ding, P.P. da\u00a0Silva, C. Chang, Pml 2: A modular explanation interlingua, in: Proceedings of the AAAI 2007 Workshop on Explanation-aware Computing, 2007, pp. 49\u201355. URL https:\/\/www.aaai.org\/Papers\/Workshops\/2007\/WS-07-06\/WS07-06-008.pdf."},{"key":"10.1016\/j.websem.2015.04.001_br000265","unstructured":"S.S. Sahoo, A. Sheth, Provenir ontology: Towards a framework for escience provenance management, in: Microsoft eScience Workshop, 2009. URL http:\/\/knoesis.wright.edu\/library\/resource.php?id=741."},{"issue":"5","key":"10.1016\/j.websem.2015.04.001_br000270","doi-asserted-by":"crossref","first-page":"739","DOI":"10.1016\/j.jbi.2008.04.010","article-title":"The swan biomedical discourse ontology","volume":"41","author":"Ciccarese","year":"2008","journal-title":"J. Biomed. Inform."},{"issue":"1","key":"10.1016\/j.websem.2015.04.001_br000275","doi-asserted-by":"crossref","first-page":"37","DOI":"10.1186\/2041-1480-4-37","article-title":"Pav ontology: provenance, authoring and versioning","volume":"4","author":"Ciccarese","year":"2013","journal-title":"J. Biomed. Semant."},{"key":"10.1016\/j.websem.2015.04.001_br000280","unstructured":"S. Sahoo, Towards desiderata for provenance ontologies in biomedicine, in: ICBO, vol. 833, 2011. URL http:\/\/ceur-ws.org\/Vol-833\/paper45.pdf."},{"issue":"21","key":"10.1016\/j.websem.2015.04.001_br000285","article-title":"The power of the principle of provenance","volume":"1","author":"Bearman","year":"1985","journal-title":"Archivaria"},{"key":"10.1016\/j.websem.2015.04.001_br000290","series-title":"IPAW","first-page":"78","article-title":"Publishing and consuming provenance metadata on the web of linked data","volume":"vol. 6378","author":"Hartig","year":"2010"},{"key":"10.1016\/j.websem.2015.04.001_br000295","unstructured":"J. Zhao, Guide to the Open Provenance Model Vocabulary (2010). URL http:\/\/open-biomed.sourceforge.net\/opmv\/opmv-guide.html."},{"key":"10.1016\/j.websem.2015.04.001_br000300","unstructured":"K. Alexander, M. Hausenblas, Describing linked datasets\u2014on the design and usage of void, the vocabulary of interlinked datasets, in: In Linked Data on the Web Workshop (LDOW 09), in conjunction with 18th International World Wide Web Conference (WWW 09), 2009. URL http:\/\/richard.cyganiak.de\/2008\/papers\/void-ldow2009.pdf."},{"key":"10.1016\/j.websem.2015.04.001_br000305","series-title":"The Semantic Web (ISWC\u201914)","first-page":"245","article-title":"Adoption of the linked data best practices in different topical domains","volume":"vol. 8796","author":"Schmachtenberg","year":"2014"},{"key":"10.1016\/j.websem.2015.04.001_br000310","unstructured":"D. Wood, M. Lanthaler, R. Cyganiak, RDF 1.1 concepts and abstract syntax, W3C recommendation, W3C (Feb. 2014). URL http:\/\/www.w3.org\/TR\/2014\/REC-rdf11-concepts-20140225\/."},{"key":"10.1016\/j.websem.2015.04.001_br000315","doi-asserted-by":"crossref","unstructured":"J.J. Carroll, C. Bizer, P. Hayes, P. Stickler, Named graphs, provenance and trust, in: Proceedings of the 14th International Conference on World Wide Web (WWW\u201905), ACM, 2005, pp. 613\u2013622. http:\/\/dx.doi.org\/10.1145\/1060745.1060835.","DOI":"10.1145\/1060745.1060835"},{"key":"10.1016\/j.websem.2015.04.001_br000320","unstructured":"O. Hartig, Provenance information in the web of data, in: C.\u00a0Bizer, T.\u00a0Heath, T.\u00a0Berners-Lee, K.\u00a0Idehen (Eds.), LDOW, CEUR Workshop Proceedings, Vol. 538, CEUR-WS.org, 2009. URL http:\/\/ceur-ws.org\/Vol-538\/ldow2009_paper18.pdf."},{"key":"10.1016\/j.websem.2015.04.001_br000325","unstructured":"H. Van\u00a0de Sompel, M.L. Nelson, R. Sanderson, L.L. Balakireva, S. Ainsworth, H. Shankar, Memento: Time travel for the web, arXiv preprint. URL http:\/\/arxiv.org\/abs\/0911.1112."},{"issue":"1","key":"10.1016\/j.websem.2015.04.001_br000330","doi-asserted-by":"crossref","first-page":"163","DOI":"10.2218\/ijdc.v7i1.223","article-title":"Opening up climate research: A linked data approach to publishing data provenance","volume":"7","author":"Shaon","year":"2012","journal-title":"International Journal of Digital Curation"},{"key":"10.1016\/j.websem.2015.04.001_br000335","unstructured":"K. Eckert, Provenance and Annotations for Linked Data, in: DCMI International Conference on Dublin Core and Metadata Applications (DC-2013), 2013. URL http:\/\/dcevents.dublincore.org\/IntConf\/dc-2013\/paper\/download\/154\/71."},{"key":"10.1016\/j.websem.2015.04.001_br000340","doi-asserted-by":"crossref","unstructured":"G. Flouris, I. Fundulaki, P. Pediaditis, Y. Theoharis, V. Christophides, Coloring RDF triples to capture provenance, in: Proceedings of the 8th International Semantic Web Conference (ISWC\u201909), Springer-Verlag, 2009, pp. 196\u2013212. http:\/\/dx.doi.org\/10.1007\/978-3-642-04930-9_13.","DOI":"10.1007\/978-3-642-04930-9_13"},{"key":"10.1016\/j.websem.2015.04.001_br000345","doi-asserted-by":"crossref","unstructured":"G. Karvounarakis, I. Fundulaki, V. Christophides, Provenance for linked data, in: In Search of Elegance in the Theory and Practice of Computation, Vol. 8000 of LNCS, Springer-Verlag, 2013, pp. 366\u2013381. http:\/\/dx.doi.org\/10.1007\/978-3-642-41660-6_19.","DOI":"10.1007\/978-3-642-41660-6_19"},{"key":"10.1016\/j.websem.2015.04.001_br000350","doi-asserted-by":"crossref","unstructured":"H. Halpin, J. Cheney, Dynamic provenance for SPARQL updates, in: The Semantic Web (ISWC\u201914), LNCS, vol. 8796, Springer-Verlag, 2014, pp. 425\u2013440. http:\/\/dx.doi.org\/10.1007\/978-3-319-11964-9_27.","DOI":"10.1007\/978-3-319-11964-9_27"},{"key":"10.1016\/j.websem.2015.04.001_br000355","doi-asserted-by":"crossref","unstructured":"M. Wylot, P. Cudre-Mauroux, P. Groth, TripleProv: Efficient processing of lineage queries in a native RDF store, in: 23rd International Conference on World Wide Web (WWW\u201914), ACM, New York, NY, USA, 2014, pp. 455\u2013466. http:\/\/dx.doi.org\/10.1145\/2566486.2568014.","DOI":"10.1145\/2566486.2568014"},{"key":"10.1016\/j.websem.2015.04.001_br000360","doi-asserted-by":"crossref","unstructured":"T. Heath, C. Bizer, Linked Data: Evolving the Web into a Global Data Space, Morgan and Claypool, 2011. http:\/\/dx.doi.org\/10.2200\/S00334ED1V01Y201102WBE001.","DOI":"10.2200\/S00334ED1V01Y201102WBE001"},{"key":"10.1016\/j.websem.2015.04.001_br000365","unstructured":"A. Schultz, A. Matteini, R. Isele, P.N. Mendes, C. Bizer, C. Becker, LDIF\u2014A Framework for Large-Scale Linked Data Integration, in: 21st International World Wide Web Conference (WWW2012), Developers Track, 2012. URL http:\/\/www2012.org\/proceedings\/nocompanion\/DevTrack_017.pdf."},{"key":"10.1016\/j.websem.2015.04.001_br000370","unstructured":"J. Domingue, D. Fensel, J.A. Hendler, R. Studer, Perspectives workshop: Semantic web reflections and future directions, in: Perspectives Workshop: Semantic Web Reflections and Future Directions, no. 09271 in Dagstuhl Seminar Proceedings, Dagstuhl, Germany, 2010. URL http:\/\/drops.dagstuhl.de\/opus\/volltexte\/2010\/2533."},{"key":"10.1016\/j.websem.2015.04.001_br000375","unstructured":"S. Sahoo, P. Groth, O. Hartig, S. Miles, S. Coppens, J. Myers, Y. Gil, L. Moreau, M. Panzer, D. Garijo, Provenance Vocabulary Mappings, Tech. rep., World Wide Web Consortium (2010). URL http:\/\/www.w3.org\/2005\/Incubator\/prov\/wiki\/Provenance_Vocabulary_Mappings."},{"key":"10.1016\/j.websem.2015.04.001_br000380","unstructured":"T. Lebo, Review of prov-xg\u2019s Provenance Vocabulary Mappings. URL http:\/\/inference-web.org\/wiki\/Review_of_prov-xg\u2019s_Provenance_Vocabulary_Mappings (last accessed Feb 2014)."},{"key":"10.1016\/j.websem.2015.04.001_br000385","unstructured":"T.D. Huynh, M. Jewell, A.S. Keshavarz, D. Michaelides, H. Yang, L. Moreau, The PROV-JSON Serialization. A JSON Representation for the PROV Data Model, Tech. rep., University of Southampton (2013). URL http:\/\/provenance.ecs.soton.ac.uk\/prov-json\/."},{"issue":"4","key":"10.1016\/j.websem.2015.04.001_br000390","first-page":"44","article-title":"Provenance in scientific workflow systems","volume":"30","author":"Davidson","year":"2007","journal-title":"Bull. Tech. Committee Data Eng."},{"key":"10.1016\/j.websem.2015.04.001_br000395","doi-asserted-by":"crossref","unstructured":"T. McPhillips, S. Bowers, B. Ludaescher, Collection-oriented scientific workflows for integrating and analyzing biological data, in: In 3rd Intl. Workshop On Data Integration in the Life Sciences (DILS), LNCS, Springer-Verlag, 2006, pp. 248\u2013263. http:\/\/dx.doi.org\/10.1007\/11799511_23.","DOI":"10.1007\/11799511_23"},{"key":"10.1016\/j.websem.2015.04.001_br000400","unstructured":"P. Groth, S. Miles, P. Missier, L. Moreau, A proposal for handling collections in the open provenance model (Jun. 2009). URL http:\/\/mailman.ecs.soton.ac.uk\/pipermail\/provenance-challenge-ipaw-info\/2009-June\/000120.html."},{"issue":"6","key":"10.1016\/j.websem.2015.04.001_br000405","doi-asserted-by":"crossref","first-page":"1195","DOI":"10.1007\/s10664-012-9199-7","article-title":"Software bertillonage - determining the provenance of software development artifacts","volume":"18","author":"Davies","year":"2013","journal-title":"Empir. Softw. Eng."},{"key":"10.1016\/j.websem.2015.04.001_br000410","unstructured":"J. Gray, L. Bounegru, and L. Chambers (Eds.), Data Journalism Handbook 1.0 BETA, O\u2019Reilly Media, 2012. URL http:\/\/datajournalismhandbook.org\/1.0\/en\/."},{"issue":"2","key":"10.1016\/j.websem.2015.04.001_br000415","doi-asserted-by":"crossref","DOI":"10.1017\/S0269888900008122","article-title":"Intelligent Agents: Theory and Practice","volume":"10","author":"Wooldridge","year":"1995","journal-title":"Knowl. Eng. Rev."},{"key":"10.1016\/j.websem.2015.04.001_br000420","doi-asserted-by":"crossref","first-page":"202","DOI":"10.1016\/j.websem.2011.03.001","article-title":"Provenance-based reproducibility in the semantic web, Web Semantics: Science","volume":"9","author":"Moreau","year":"2011","journal-title":"Services and Agents on the World Wide Web"},{"key":"10.1016\/j.websem.2015.04.001_br000425","doi-asserted-by":"crossref","DOI":"10.1145\/2734116","article-title":"A formal account of the open provenance model","author":"Kwasnikowska","year":"2015","journal-title":"ACM Trans. Web."},{"key":"10.1016\/j.websem.2015.04.001_br000430","unstructured":"W.E. Johnson, Logic: Part iii (1924). URL http:\/\/www.ditext.com\/johnson\/intro-3.html."},{"key":"10.1016\/j.websem.2015.04.001_br000435","unstructured":"Provenance Interchange Working Group Charter. URL http:\/\/www.w3.org\/2011\/01\/prov-wg-charter (last accessed Nov 2013)."},{"issue":"2","key":"10.1016\/j.websem.2015.04.001_br000440","first-page":"34","article-title":"Logical versus analogical or symbolic versus connectionist or neat versus scruffy","volume":"12","author":"Minsky","year":"1991","journal-title":"AI Mag."},{"key":"10.1016\/j.websem.2015.04.001_br000445","unstructured":"S. Abiteboul, R. Hull, V. Vianu, Foundations of Databases, Addison Wesley, 1995. URL http:\/\/webdam.inria.fr\/Alice\/."},{"issue":"1","key":"10.1016\/j.websem.2015.04.001_br000450","doi-asserted-by":"crossref","first-page":"89","DOI":"10.1016\/j.tcs.2004.10.033","article-title":"Data exchange: semantics and query answering","volume":"336","author":"Fagin","year":"2005","journal-title":"Theoret. Comput. Sci."},{"key":"10.1016\/j.websem.2015.04.001_br000455","series-title":"Logics for databases and information systems","first-page":"307","article-title":"Logical approaches to incomplete information: a survey","author":"van~der Meyden","year":"1998"},{"issue":"2","key":"10.1016\/j.websem.2015.04.001_br000460","first-page":"84","article-title":"Principles of provenance (Dagstuhl Seminar 12091)","volume":"2","author":"Cheney","year":"2012","journal-title":"Dagstuhl Reports"},{"key":"10.1016\/j.websem.2015.04.001_br000465","unstructured":"B. Motik, B.C. Grau, I. Horrocks, Z. Wu, A. Fokoue, C. Lutz, OWL 2 Web Ontology Language\u2014Profiles, Tech. rep., W3C (2009). URL http:\/\/www.w3.org\/TR\/2009\/REC-owl2-profiles-20091027\/."},{"key":"10.1016\/j.websem.2015.04.001_br000470","unstructured":"L. Dodds, I. Davis, Linked Data Patterns: A pattern catalogue for modelling, publishing, and consuming Linked Data, Published online (2012). URL http:\/\/patterns.dataincubator.org\/book\/."},{"key":"10.1016\/j.websem.2015.04.001_br000475","unstructured":"F. Manola, and E. Miller (Eds.), RDF Primer, W3C Recommendation, World Wide Web Consortium, 2004. URL http:\/\/www.w3.org\/TR\/rdf-primer\/."},{"key":"10.1016\/j.websem.2015.04.001_br000480","doi-asserted-by":"crossref","unstructured":"J. Cheney, Causality and the semantics of provenance, in: DCM, 2010, pp. 63\u201374. http:\/\/dx.doi.org\/10.4204\/EPTCS.26.6.","DOI":"10.4204\/EPTCS.26.6"},{"key":"10.1016\/j.websem.2015.04.001_br000485","doi-asserted-by":"crossref","unstructured":"J. Cheney, A formal framework for provenance security, in: Proceedings of the 24th IEEE Computer Security Foundations Symposium (CSF), IEEE, 2011, pp. 281\u2013293. http:\/\/dx.doi.org\/10.1109\/CSF.2011.26.","DOI":"10.1109\/CSF.2011.26"},{"key":"10.1016\/j.websem.2015.04.001_br000490","doi-asserted-by":"crossref","unstructured":"G. Karvounarakis, Z.G. Ives, V. Tannen, Querying data provenance, in: International Conference on Management of Data (SIGMOD\u201910), 2010, pp. 951\u2013962. http:\/\/dx.doi.org\/10.1145\/1807167.1807269.","DOI":"10.1145\/1807167.1807269"},{"issue":"3","key":"10.1016\/j.websem.2015.04.001_br000495","first-page":"196","article-title":"Querying provenance information in distributed environments, I","volume":"18","author":"Zhao","year":"2011","journal-title":"J. Comput. Appl."}],"container-title":["Journal of Web Semantics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1570826815000177?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S1570826815000177?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2018,11,15]],"date-time":"2018-11-15T02:26:12Z","timestamp":1542248772000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S1570826815000177"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,12]]},"references-count":99,"alternative-id":["S1570826815000177"],"URL":"https:\/\/doi.org\/10.1016\/j.websem.2015.04.001","relation":{},"ISSN":["1570-8268"],"issn-type":[{"value":"1570-8268","type":"print"}],"subject":[],"published":{"date-parts":[[2015,12]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"The rationale of PROV","name":"articletitle","label":"Article Title"},{"value":"Journal of Web Semantics","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.websem.2015.04.001","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"Copyright \u00a9 2015 The Authors. Published by Elsevier B.V.","name":"copyright","label":"Copyright"}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1038_2Fnbt_3820 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T23:17:18Z","timestamp":1793315838608,"version":"4.5.0"},"reference-count":22,"publisher":"Springer Science and Business Media LLC","issue":"4","license":[{"start":{"date-parts":[[2017,4,1]],"date-time":"2017-04-01T00:00:00Z","timestamp":1491004800000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/www.springer.com\/tdm"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat Biotechnol"],"published-print":{"date-parts":[[2017,4]]},"DOI":"10.1038\/nbt.3820","type":"journal-article","created":{"date-parts":[[2017,4,11]],"date-time":"2017-04-11T11:26:35Z","timestamp":1491909995000},"page":"316-319","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":4433,"title":["Nextflow enables reproducible computational workflows"],"prefix":"10.1038","volume":"35","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-5431-190X","authenticated-orcid":false,"given":"Paolo","family":"Di Tommaso","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Maria","family":"Chatzou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Evan W","family":"Floden","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Pablo Prieto","family":"Barja","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Emilio","family":"Palumbo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Cedric","family":"Notredame","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2017,4,11]]},"reference":[{"key":"BFnbt3820_CR1","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1038\/530027a","volume":"530","author":"DB Allison","year":"2016","unstructured":"Allison, D.B., Brown, A.W., George, B.J. & Kaiser, K.A. Nature 530, 27\u201329 (2016).","journal-title":"Nature"},{"key":"BFnbt3820_CR2","doi-asserted-by":"crossref","unstructured":"Editorial. Nat. Methods 12, 1099 (2015).","DOI":"10.1038\/nmeth.3686"},{"key":"BFnbt3820_CR3","doi-asserted-by":"crossref","unstructured":"Editorial. Nat. Biotechnol. 33, 319 (2015).","DOI":"10.1038\/nbt.3202"},{"key":"BFnbt3820_CR4","doi-asserted-by":"publisher","first-page":"30","DOI":"10.1186\/s13742-016-0135-4","volume":"5","author":"SR Piccolo","year":"2016","unstructured":"Piccolo, S.R. & Frampton, M.B. Gigascience 5, 30 (2016).","journal-title":"Gigascience"},{"key":"BFnbt3820_CR5","doi-asserted-by":"publisher","unstructured":"Masca, N.G.D. et al. eLife \n http:\/\/dx.doi.org\/10.7554\/eLife.05519\n \n (2015).","DOI":"10.7554\/eLife.05519"},{"key":"BFnbt3820_CR6","doi-asserted-by":"publisher","first-page":"13","DOI":"10.1109\/MCSE.2012.38","volume":"14","author":"RJ LeVeque","year":"2012","unstructured":"LeVeque, R.J., Mitchell, I.M. & Stodden, V. Comput. Sci. Eng. 14, 13\u201317 (2012).","journal-title":"Comput. Sci. Eng."},{"key":"BFnbt3820_CR7","doi-asserted-by":"publisher","first-page":"e80278","DOI":"10.1371\/journal.pone.0080278","volume":"8","author":"D Garijo","year":"2013","unstructured":"Garijo, D. et al. PLoS One 8, e80278 (2013).","journal-title":"PLoS One"},{"key":"BFnbt3820_CR8","doi-asserted-by":"publisher","first-page":"996","DOI":"10.1038\/nbt.2740","volume":"31","author":"N Loman","year":"2013","unstructured":"Loman, N. & Watson, M. Nat. Biotechnol. 31, 996\u2013998 (2013).","journal-title":"Nat. Biotechnol."},{"key":"BFnbt3820_CR9","doi-asserted-by":"publisher","first-page":"525","DOI":"10.1038\/nbt.3519","volume":"34","author":"NL Bray","year":"2016","unstructured":"Bray, N.L., Pimentel, H., Melsted, P. & Pachter, L. Nat. Biotechnol. 34, 525\u2013527 (2016).","journal-title":"Nat. Biotechnol."},{"issue":"1","key":"BFnbt3820_CR10","doi-asserted-by":"publisher","first-page":"W29","DOI":"10.1093\/nar\/gkw292","volume":"44","author":"S Steinbiss","year":"2016","unstructured":"Steinbiss, S. et al. Nucleic Acids Res. 44 1, W29\u201334 (2016).","journal-title":"Nucleic Acids Res."},{"key":"BFnbt3820_CR11","doi-asserted-by":"publisher","first-page":"1525","DOI":"10.1093\/bioinformatics\/bts167","volume":"28","author":"SP Sadedin","year":"2012","unstructured":"Sadedin, S.P., Pope, B. & Oshlack, A. Bioinformatics 28, 1525\u20131526 (2012).","journal-title":"Bioinformatics"},{"key":"BFnbt3820_CR12","doi-asserted-by":"publisher","first-page":"47","DOI":"10.1186\/s13742-015-0087-0","volume":"4","author":"P Belmann","year":"2015","unstructured":"Belmann, P. et al. Gigascience 4, 47 (2015).","journal-title":"Gigascience"},{"key":"BFnbt3820_CR13","doi-asserted-by":"publisher","first-page":"1443","DOI":"10.12688\/f1000research.7536.1","volume":"4","author":"F Moreews","year":"2015","unstructured":"Moreews, F. et al. F1000Res 4, 1443 (2015).","journal-title":"F1000Res"},{"key":"BFnbt3820_CR14","doi-asserted-by":"publisher","first-page":"2396","DOI":"10.1093\/bioinformatics\/btw120","volume":"32","author":"A Hosny","year":"2016","unstructured":"Hosny, A., Vera-Licona, P., Laubenbacher, R. & Favre, T. Bioinformatics 32, 2396\u20132398 (2016).","journal-title":"Bioinformatics"},{"key":"BFnbt3820_CR15","doi-asserted-by":"publisher","first-page":"127","DOI":"10.1038\/538127a","volume":"538","author":"J Perkel","year":"2016","unstructured":"Perkel, J. Nature 538, 127\u2013128 (2016).","journal-title":"Nature"},{"key":"BFnbt3820_CR16","doi-asserted-by":"publisher","first-page":"2520","DOI":"10.1093\/bioinformatics\/bts480","volume":"28","author":"J K\u00f6ster","year":"2012","unstructured":"K\u00f6ster, J. & Rahmann, S. Bioinformatics 28, 2520\u20132522 (2012).","journal-title":"Bioinformatics"},{"key":"BFnbt3820_CR17","doi-asserted-by":"publisher","first-page":"e1273","DOI":"10.7717\/peerj.1273","volume":"3","author":"P Di Tommaso","year":"2015","unstructured":"Di Tommaso, P. et al. PeerJ 3, e1273 (2015).","journal-title":"PeerJ"},{"key":"BFnbt3820_CR18","doi-asserted-by":"publisher","first-page":"R86","DOI":"10.1186\/gb-2010-11-8-r86","volume":"11","author":"J Goecks","year":"2010","unstructured":"Goecks, J., Nekrutenko, A. & Taylor, J. Genome Biol. 11, R86 (2010).","journal-title":"Genome Biol."},{"key":"BFnbt3820_CR19","doi-asserted-by":"publisher","first-page":"403","DOI":"10.1186\/gb4161","volume":"15","author":"D Blankenberg","year":"2014","unstructured":"Blankenberg, D. et al. Genome Biol. 15, 403 (2014).","journal-title":"Genome Biol."},{"key":"BFnbt3820_CR20","volume-title":"bioRxiv","author":"J Vivian","year":"2016","unstructured":"Vivian, J. et al. Preprint at bioRxiv \n http:\/\/biorxiv.org\/content\/early\/2016\/07\/07\/062497\n \n (2016)."},{"key":"BFnbt3820_CR21","doi-asserted-by":"publisher","first-page":"2688","DOI":"10.1093\/bioinformatics\/btl446","volume":"22","author":"A Stamatakis","year":"2006","unstructured":"Stamatakis, A. Bioinformatics 22, 2688\u20132690 (2006).","journal-title":"Bioinformatics"},{"key":"BFnbt3820_CR22","doi-asserted-by":"publisher","first-page":"257","DOI":"10.1038\/nrg.2016.10","volume":"17","author":"SA Byron","year":"2016","unstructured":"Byron, S.A., Van Keuren-Jensen, K.R., Engelthaler, D.M., Carpten, J.D. & Craig, D.W. Nat. Rev. Genet. 17, 257\u2013271 (2016).","journal-title":"Nat. Rev. Genet."}],"container-title":["Nature Biotechnology"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/www.nature.com\/articles\/nbt.3820.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/www.nature.com\/articles\/nbt.3820","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/www.nature.com\/doifinder\/10.1038\/nbt.3820","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"},{"URL":"http:\/\/www.nature.com\/articles\/nbt.3820.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,5,18]],"date-time":"2023-05-18T14:35:56Z","timestamp":1684420556000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/nbt.3820"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,4]]},"references-count":22,"journal-issue":{"issue":"4","published-print":{"date-parts":[[2017,4]]}},"alternative-id":["BFnbt3820"],"URL":"https:\/\/doi.org\/10.1038\/nbt.3820","relation":{},"ISSN":["1087-0156","1546-1696"],"issn-type":[{"value":"1087-0156","type":"print"},{"value":"1546-1696","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017,4]]},"assertion":[{"value":"11 April 2017","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing financial interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1038_2Fs41562_016_0021 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T13:08:50Z","timestamp":1793279330052,"version":"4.5.0"},"reference-count":87,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2017,1,10]],"date-time":"2017-01-10T00:00:00Z","timestamp":1484006400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2017,1,10]],"date-time":"2017-01-10T00:00:00Z","timestamp":1484006400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat Hum Behav"],"abstract":"<jats:title>Abstract<\/jats:title>\n <jats:p>Improving the reliability and efficiency of scientific research will increase the credibility of the published scientific literature and accelerate discovery. Here we argue for the adoption of measures to optimize key elements of the scientific process: methods, reporting and dissemination, reproducibility, evaluation and incentives. There is some evidence from both simulations and empirical studies supporting the likely effectiveness of these measures, but their broad adoption by researchers, institutions, funders and journals will require iterative evaluation and improvement. We discuss the goals of these measures, and how they can be implemented, in the hope that this will facilitate action toward improving the transparency, reproducibility and efficiency of scientific research.<\/jats:p>","DOI":"10.1038\/s41562-016-0021","type":"journal-article","created":{"date-parts":[[2017,1,10]],"date-time":"2017-01-10T11:42:05Z","timestamp":1484048525000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":2471,"title":["A manifesto for reproducible science"],"prefix":"10.1038","volume":"1","author":[{"given":"Marcus R.","family":"Munaf\u00f2","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Brian A.","family":"Nosek","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dorothy V. M.","family":"Bishop","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Katherine S.","family":"Button","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Christopher D.","family":"Chambers","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Nathalie","family":"Percie du Sert","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Uri","family":"Simonsohn","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Eric-Jan","family":"Wagenmakers","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jennifer J.","family":"Ware","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"John P. A.","family":"Ioannidis","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2017,1,10]]},"reference":[{"issue":"8","key":"BFs415620160021_CR1","doi-asserted-by":"publisher","first-page":"e124","DOI":"10.1371\/journal.pmed.0020124","volume":"2","author":"John P. A. Ioannidis","year":"2005","unstructured":"Ioannidis, J. P. A. Why most published research findings are false. PLoS Med. 2, e124 (2005).","journal-title":"PLoS Medicine"},{"key":"BFs415620160021_CR2","doi-asserted-by":"publisher","first-page":"365","DOI":"10.1038\/nrn3475","volume":"14","author":"KS Button","year":"2013","unstructured":"Button, K. S. et al. Power failure: why small sample size undermines the reliability of neuroscience. Nat. Rev. Neurosci. 14, 365\u2013376 (2013).","journal-title":"Nat. Rev. Neurosci."},{"issue":"4","key":"BFs415620160021_CR3","doi-asserted-by":"publisher","first-page":"e10068","DOI":"10.1371\/journal.pone.0010068","volume":"5","author":"Daniele Fanelli","year":"2010","unstructured":"Fanelli, D. \u201cPositive\u201d results increase down the Hierarchy of the Sciences. PloS ONE 5, e10068 (2010).","journal-title":"PLoS ONE"},{"key":"BFs415620160021_CR4","doi-asserted-by":"publisher","first-page":"524","DOI":"10.1177\/0956797611430953","volume":"23","author":"LK John","year":"2012","unstructured":"John, L. K., Loewenstein, G. & Prelec, D. Measuring the prevalence of questionable research practices with incentives for truth telling. Psychol. Sci. 23, 524\u2013532 (2012).","journal-title":"Psychol. Sci."},{"key":"BFs415620160021_CR5","doi-asserted-by":"publisher","first-page":"537","DOI":"10.1177\/1745691612460688","volume":"7","author":"MC Makel","year":"2012","unstructured":"Makel, M. C., Plucker, J. A. & Hegarty, B. Replications in psychology research: how often do they really occur? Perspect. Psychol. Sci. 7, 537\u2013542 (2012).","journal-title":"Perspect. Psychol. Sci."},{"key":"BFs415620160021_CR6","doi-asserted-by":"publisher","first-page":"726","DOI":"10.1037\/0003-066X.61.7.726","volume":"61","author":"JM Wicherts","year":"2006","unstructured":"Wicherts, J. M., Borsboom, D., Kats, J. & Molenaar, D. The poor availability of psychological research data for reanalysis. Am. Psychol. 61, 726\u2013728 (2006).","journal-title":"Am. Psychol."},{"key":"BFs415620160021_CR7","doi-asserted-by":"publisher","first-page":"196","DOI":"10.1207\/s15327957pspr0203_4","volume":"2","author":"NL Kerr","year":"1998","unstructured":"Kerr, N. L. HARKing: hypothesizing after the results are known. Pers. Soc. Psychol. Rev. 2, 196\u2013217 (1998).","journal-title":"Pers. Soc. Psychol. Rev."},{"key":"BFs415620160021_CR8","doi-asserted-by":"publisher","first-page":"176","DOI":"10.1016\/S0140-6736(13)62297-7","volume":"383","author":"R Al-Shahi Salman","year":"2014","unstructured":"Al-Shahi Salman, R. et al. Increasing value and reducing waste in biomedical research regulation and management. Lancet 383, 176\u2013185 (2014).","journal-title":"Lancet"},{"key":"BFs415620160021_CR9","doi-asserted-by":"publisher","first-page":"116","DOI":"10.1161\/CIRCRESAHA.114.303819","volume":"116","author":"CG Begley","year":"2015","unstructured":"Begley, C. G. & Ioannidis, J. P. Reproducibility in science: improving the standard for basic and preclinical research. Circ. Res. 116, 116\u2013126 (2015).","journal-title":"Circ. Res."},{"key":"BFs415620160021_CR10","doi-asserted-by":"publisher","first-page":"156","DOI":"10.1016\/S0140-6736(13)62229-1","volume":"383","author":"I Chalmers","year":"2014","unstructured":"Chalmers, I. et al. How to increase value and reduce waste when research priorities are set. Lancet 383, 156\u2013165 (2014).","journal-title":"Lancet"},{"key":"BFs415620160021_CR11","doi-asserted-by":"publisher","first-page":"257","DOI":"10.1016\/S0140-6736(13)62296-5","volume":"383","author":"AW Chan","year":"2014","unstructured":"Chan, A. W. et al. Increasing value and reducing waste: addressing inaccessible research. Lancet 383, 257\u2013266 (2014).","journal-title":"Lancet"},{"key":"BFs415620160021_CR12","doi-asserted-by":"publisher","first-page":"267","DOI":"10.1016\/S0140-6736(13)62228-X","volume":"383","author":"P Glasziou","year":"2014","unstructured":"Glasziou, P. et al. Reducing waste from incomplete or unusable reports of biomedical research. Lancet 383, 267\u2013276 (2014).","journal-title":"Lancet"},{"key":"BFs415620160021_CR13","doi-asserted-by":"publisher","first-page":"166","DOI":"10.1016\/S0140-6736(13)62227-8","volume":"383","author":"JP Ioannidis","year":"2014","unstructured":"Ioannidis, J. P. et al. Increasing value and reducing waste in research design, conduct, and analysis. Lancet 383, 166\u2013175 (2014).","journal-title":"Lancet"},{"key":"BFs415620160021_CR14","doi-asserted-by":"publisher","first-page":"101","DOI":"10.1016\/S0140-6736(13)62329-6","volume":"383","author":"MR Macleod","year":"2014","unstructured":"Macleod, M. R. et al. Biomedical research: increasing value, reducing waste. Lancet 383, 101\u2013104 (2014).","journal-title":"Lancet"},{"key":"BFs415620160021_CR15","doi-asserted-by":"publisher","first-page":"452","DOI":"10.1038\/533452a","volume":"533","author":"M Baker","year":"2016","unstructured":"Baker, M. 1,500 scientists lift the lid on reproducibility. Nature 533, 452\u2013454 (2016).","journal-title":"Nature"},{"key":"BFs415620160021_CR16","doi-asserted-by":"crossref","unstructured":"Ioannidis, J. P., Fanelli, D., Dunne, D. D. & Goodman, S. N. Meta-research: evaluation and improvement of research methods and practices. PLoS Biol. 13, e1002264 (2015).","DOI":"10.1371\/journal.pbio.1002264"},{"key":"BFs415620160021_CR17","doi-asserted-by":"publisher","first-page":"514","DOI":"10.1097\/01.ede.0000135915.94799.00","volume":"15","author":"N Paneth","year":"2004","unstructured":"Paneth, N. Assessing the contributions of John Snow to epidemiology: 150 years after removal of the broad street pump handle. Epidemiology 15, 514\u2013516 (2004).","journal-title":"Epidemiology"},{"key":"BFs415620160021_CR18","doi-asserted-by":"publisher","first-page":"1065","DOI":"10.1001\/archneur.1986.00520100069017","volume":"43","author":"EA Berker","year":"1986","unstructured":"Berker, E. A., Berker, A. H. & Smith, A. Translation of Broca's 1865 report. Localization of speech in the third left frontal convolution. Arch. Neurol. 43, 1065\u20131072 (1986).","journal-title":"Arch. Neurol"},{"key":"BFs415620160021_CR19","doi-asserted-by":"publisher","first-page":"358","DOI":"10.1126\/science.189.4200.358","volume":"189","author":"N Wade","year":"1975","unstructured":"Wade, N. Discovery of pulsars: a graduate student's story. Science 189, 358\u2013364 (1975).","journal-title":"Science"},{"key":"BFs415620160021_CR20","doi-asserted-by":"publisher","first-page":"175","DOI":"10.1037\/1089-2680.2.2.175","volume":"2","author":"RS Nickerson","year":"1998","unstructured":"Nickerson, R. S. Confirmation bias: a ubiquitous phenomenon in many guises. Rev. Gen. Psychol. 2, 175\u2013220 (1998).","journal-title":"Rev. Gen. Psychol."},{"key":"BFs415620160021_CR21","volume-title":"The Hunt for Vulcan...and How Albert Einstein Destroyed a Planet, Discovered Relativity, and Deciphered the University","author":"T Levenson","year":"2015","unstructured":"Levenson, T. The Hunt for Vulcan...and How Albert Einstein Destroyed a Planet, Discovered Relativity, and Deciphered the University (Random House, 2015)."},{"key":"BFs415620160021_CR22","volume-title":"Experimenter Effects in Behavioral Research","author":"R Rosenthal","year":"1966","unstructured":"Rosenthal, R. Experimenter Effects in Behavioral Research (Appleton-Century-Crofts, 1966)."},{"key":"BFs415620160021_CR23","doi-asserted-by":"publisher","first-page":"188","DOI":"10.1016\/j.actpsy.2014.02.001","volume":"148","author":"AD de Groot","year":"2014","unstructured":"de Groot, A. D. The meaning of \u201csignificance\u201d for different types of research [translated and annotated by Eric-Jan Wagenmakers, Denny Borsboom, Josine Verhagen, Rogier Kievit, Marjan Bakker, Angelique Cramer, Dora Matzke, Don Mellenbergh, and Han L. J. van der Maas]. Acta Psychol. 148, 188\u2013194 (2014).","journal-title":"Acta Psychol."},{"key":"BFs415620160021_CR24","doi-asserted-by":"crossref","unstructured":"Heininga, V. E., Oldehinkel, A. J., Veenstra, R. & Nederhof, E. I just ran a thousand analyses: benefits of multiple testing in understanding equivocal evidence on gene-environment interactions. PloS ONE 10, e0125383 (2015).","DOI":"10.1371\/journal.pone.0125383"},{"key":"BFs415620160021_CR25","doi-asserted-by":"publisher","first-page":"1046","DOI":"10.1016\/j.jclinepi.2015.05.029","volume":"68","author":"CJ Patel","year":"2015","unstructured":"Patel, C. J., Burford, B. & Ioannidis, J. P. Assessment of vibration of effects due to model specification can demonstrate the instability of observational associations. J. Clin. Epidemiol. 68, 1046\u20131058 (2015).","journal-title":"J. Clin. Epidemiol."},{"key":"BFs415620160021_CR26","doi-asserted-by":"publisher","first-page":"289","DOI":"10.1016\/j.neuroimage.2012.07.004","volume":"63","author":"J Carp","year":"2012","unstructured":"Carp, J. The secret lives of experiments: methods reporting in the fMRI literature. Neuroimage 63, 289\u2013300 (2012).","journal-title":"Neuroimage"},{"key":"BFs415620160021_CR27","doi-asserted-by":"crossref","unstructured":"Carp, J. On the plurality of (methodological) worlds: estimating the analytic flexibility of FMRI experiments. Front. Neurosci. 6, 149 (2012).","DOI":"10.3389\/fnins.2012.00149"},{"key":"BFs415620160021_CR28","doi-asserted-by":"publisher","first-page":"534","DOI":"10.1037\/a0033242","volume":"143","author":"U Simonsohn","year":"2014","unstructured":"Simonsohn, U., Nelson, L. D. & Simmons, J. P. P-curve: a key to the file-drawer. J. Exp. Psychol. Gen. 143, 534\u2013547 (2014).","journal-title":"J. Exp. Psychol. Gen."},{"key":"BFs415620160021_CR29","doi-asserted-by":"publisher","first-page":"182","DOI":"10.1038\/526182a","volume":"526","author":"R Nuzzo","year":"2015","unstructured":"Nuzzo, R. Fooling ourselves. Nature 526, 182\u2013185 (2015).","journal-title":"Nature"},{"key":"BFs415620160021_CR30","doi-asserted-by":"publisher","first-page":"187","DOI":"10.1038\/526187a","volume":"526","author":"R MacCoun","year":"2015","unstructured":"MacCoun, R. & Perlmutter, S. Blind analysis: hide results to seek the truth. Nature 526, 187\u2013189 (2015).","journal-title":"Nature"},{"key":"BFs415620160021_CR31","doi-asserted-by":"publisher","first-page":"337","DOI":"10.1007\/s10654-016-0149-3","volume":"31","author":"S Greenland","year":"2016","unstructured":"Greenland, S. et al. Statistical tests, P values, confidence intervals, and power: a guide to misinterpretations. Eur. J. Epidemiol. 31, 337\u2013350 (2016).","journal-title":"Eur. J. Epidemiol."},{"key":"BFs415620160021_CR32","doi-asserted-by":"publisher","first-page":"226","DOI":"10.1136\/bmj.322.7280.226","volume":"322","author":"JA Sterne","year":"2001","unstructured":"Sterne, J. A. & Davey Smith, G. Sifting the evidence\u2014what's wrong with significance tests? BMJ 322, 226\u2013231 (2001).","journal-title":"BMJ"},{"key":"BFs415620160021_CR33","doi-asserted-by":"publisher","first-page":"645","DOI":"10.2466\/pms.106.2.645-649","volume":"106","author":"A Brand","year":"2008","unstructured":"Brand, A., Bradley, M. T., Best, L. A. & Stoica, G. Accuracy of effect size estimates from published psychological research. Percept. Motor Skill. 106, 645\u2013649 (2008).","journal-title":"Percept. Motor Skill"},{"key":"BFs415620160021_CR34","doi-asserted-by":"publisher","first-page":"1037","DOI":"10.1080\/17470218.2014.885986","volume":"67","author":"I Vankov","year":"2014","unstructured":"Vankov, I., Bowers, J. & Munaf\u00f2, M. R. On the persistence of low power in psychological science. Q. J. Exp. Psychol. 67, 1037\u20131040 (2014).","journal-title":"Q. J. Exp. Psychol."},{"key":"BFs415620160021_CR35","doi-asserted-by":"publisher","first-page":"309","DOI":"10.1037\/0033-2909.105.2.309","volume":"105","author":"P Sedlmeier","year":"1989","unstructured":"Sedlmeier, P. & Gigerenzer, G. Do studies of statistical power have an effect on the power of studies? Psychol. Bull. 105, 309\u2013316 (1989).","journal-title":"Psychol. Bull."},{"key":"BFs415620160021_CR36","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1037\/h0045186","volume":"65","author":"J Cohen","year":"1962","unstructured":"Cohen, J. The statistical power of abnormal-social psychological research: a review. J. Abnorm. Soc. Psychol. 65, 145\u2013153 (1962).","journal-title":"J. Abnorm. Soc. Psychol."},{"key":"BFs415620160021_CR37","doi-asserted-by":"publisher","first-page":"815","DOI":"10.1111\/j.1360-0443.2007.01822.x","volume":"102","author":"JF Etter","year":"2007","unstructured":"Etter, J. F., Burri, M. & Stapleton, J. The impact of pharmaceutical company funding on results of randomized trials of nicotine replacement therapy for smoking cessation: a meta-analysis. Addiction 102, 815\u2013822 (2007).","journal-title":"Addiction"},{"key":"BFs415620160021_CR38","doi-asserted-by":"publisher","first-page":"831","DOI":"10.1016\/j.jclinepi.2008.09.015","volume":"62","author":"JF Etter","year":"2009","unstructured":"Etter, J. F. & Stapleton, J. Citations to trials of nicotine replacement therapy were biased toward positive results and high-impact-factor journals. J. Clin. Epidemiol. 62, 831\u2013837 (2009).","journal-title":"J. Clin. Epidemiol."},{"key":"BFs415620160021_CR39","doi-asserted-by":"publisher","first-page":"740","DOI":"10.1016\/j.jclinepi.2012.01.008","volume":"65","author":"OA Panagiotou","year":"2012","unstructured":"Panagiotou, O. A. & Ioannidis, J. P. Primary study authors of significant studies are more likely to believe that a strong association exists in a heterogeneous meta-analysis compared with methodologists. J. Clin. Epidemiol. 65, 740\u2013747 (2012).","journal-title":"J. Clin. Epidemiol."},{"key":"BFs415620160021_CR40","doi-asserted-by":"publisher","first-page":"615","DOI":"10.1177\/1745691612459058","volume":"7","author":"BA Nosek","year":"2012","unstructured":"Nosek, B. A., Spies, J. R. & Motyl, M. Scientific utopia: II. Restructuring incentives and practices to promote truth over publishability. Perspect. Psychol. Sci. 7, 615\u2013631 (2012).","journal-title":"Perspect. Psychol. Sci."},{"key":"BFs415620160021_CR41","doi-asserted-by":"publisher","first-page":"471","DOI":"10.1111\/j.1747-4949.2009.00386.x","volume":"4","author":"PMW Bath","year":"2009","unstructured":"Bath, P. M. W., Macleod, M. R. & Green, A. R. Emulating multicentre clinical stroke trials: a new paradigm for studying novel interventions in experimental models of stroke. Int. J. Stroke 4, 471\u2013479 (2009).","journal-title":"Int. J. Stroke"},{"key":"BFs415620160021_CR42","doi-asserted-by":"publisher","first-page":"1754","DOI":"10.1161\/STROKEAHA.113.000734","volume":"44","author":"U Dirnagl","year":"2013","unstructured":"Dirnagl, U. et al. A concerted appeal for international cooperation in preclinical stroke research. Stroke 44, 1754\u20131760 (2013).","journal-title":"Stroke"},{"key":"BFs415620160021_CR43","doi-asserted-by":"publisher","first-page":"843","DOI":"10.1161\/STROKEAHA.114.007560","volume":"46","author":"X Milidonis","year":"2015","unstructured":"Milidonis, X., Marshall, I., Macleod, M. R. & Sena, E. S. Magnetic resonance imaging in experimental stroke and comparison with histology systematic review and meta-analysis. Stroke 46, 843\u2013851 (2015).","journal-title":"Stroke"},{"key":"BFs415620160021_CR44","doi-asserted-by":"publisher","first-page":"142","DOI":"10.1027\/1864-9335\/a000178","volume":"45","author":"RA Klein","year":"2014","unstructured":"Klein, R. A. et al. Investigating variation in replicability: a \u201cmany labs\u201d replication project. Soc. Psychol. 45, 142\u2013152 (2014).","journal-title":"Soc. Psychol."},{"key":"BFs415620160021_CR45","doi-asserted-by":"publisher","first-page":"68","DOI":"10.1016\/j.jesp.2015.10.012","volume":"67","author":"CR Ebersole","year":"2016","unstructured":"Ebersole, C. R. et al. Many Labs 3: evaluating participant pool quality across the academic semester via replication. J. Exp. Soc. Psychol. 67, 68\u201382 (2016).","journal-title":"J. Exp. Soc. Psychol."},{"key":"BFs415620160021_CR46","doi-asserted-by":"crossref","unstructured":"Lenzer, J., Hoffman, J. R., Furberg, C. D. & Ioannidis, J. P. A. Ensuring the integrity of clinical practice guidelines: a tool for protecting patients. BMJ 347, f5535 (2013).","DOI":"10.1136\/bmj.f5535"},{"key":"BFs415620160021_CR47","first-page":"30","volume":"54","author":"TD Sterling","year":"1959","unstructured":"Sterling, T. D. Publication decisions and their possible effects on inferences drawn from tests of significance\u2014or vice versa. J. Am. Stat. Assoc. 54, 30\u201334 (1959).","journal-title":"J. Am. Stat. Assoc."},{"key":"BFs415620160021_CR48","doi-asserted-by":"publisher","first-page":"638","DOI":"10.1037\/0033-2909.86.3.638","volume":"86","author":"R Rosenthal","year":"1979","unstructured":"Rosenthal, R. File drawer problem and tolerance for null results. Psychol. Bull. 86, 638\u2013641 (1979).","journal-title":"Psychol. Bull."},{"key":"BFs415620160021_CR49","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1037\/h0076157","volume":"82","author":"TD Sterling","year":"1975","unstructured":"Sterling, T. D. Consequence of prejudice against the null hypothesis. Psychol. Bull. 82, 1\u201320 (1975).","journal-title":"Psychol. Bull."},{"key":"BFs415620160021_CR50","doi-asserted-by":"publisher","first-page":"1502","DOI":"10.1126\/science.1255484","volume":"345","author":"A Franco","year":"2014","unstructured":"Franco, A., Malhotra, N. & Simonovits, G. Publication bias in the social sciences: unlocking the file drawer. Science 345, 1502\u20131505 (2014).","journal-title":"Science"},{"key":"BFs415620160021_CR51","doi-asserted-by":"publisher","first-page":"1359","DOI":"10.1177\/0956797611417632","volume":"22","author":"JP Simmons","year":"2011","unstructured":"Simmons, J. P., Nelson, L. D. & Simonsohn, U. False-positive psychology: undisclosed flexibility in data collection and analysis allows presenting anything as significant. Psychol. Sci. 22, 1359\u20131366 (2011).","journal-title":"Psychol. Sci."},{"key":"BFs415620160021_CR52","doi-asserted-by":"publisher","first-page":"609","DOI":"10.1016\/j.cortex.2012.12.016","volume":"49","author":"CD Chambers","year":"2013","unstructured":"Chambers, C. D. Registered Reports: a new publishing initiative at Cortex. Cortex 49, 609\u2013610 (2013).","journal-title":"Cortex"},{"key":"BFs415620160021_CR53","doi-asserted-by":"publisher","first-page":"137","DOI":"10.1027\/1864-9335\/a000192","volume":"45","author":"BA Nosek","year":"2014","unstructured":"Nosek, B. A. & Lakens, D. Registered Reports: a method to increase the credibility of published results. Soc. Psychol. 45, 137\u2013141 (2014).","journal-title":"Soc. Psychol."},{"key":"BFs415620160021_CR54","doi-asserted-by":"publisher","first-page":"1422","DOI":"10.1126\/science.aab2374","volume":"348","author":"BA Nosek","year":"2015","unstructured":"Nosek, B. A. et al. Promoting an open research culture. Science 348, 1422\u20131425 (2015).","journal-title":"Science"},{"key":"BFs415620160021_CR55","doi-asserted-by":"publisher","first-page":"637","DOI":"10.1001\/jama.1996.03540080059030","volume":"276","author":"C Begg","year":"1996","unstructured":"Begg, C. et al. Improving the quality of reporting of randomized controlled trials: the CONSORT statement. JAMA 276, 637\u2013639 (1996).","journal-title":"JAMA"},{"key":"BFs415620160021_CR56","doi-asserted-by":"publisher","first-page":"122","DOI":"10.1001\/jama.1994.03520020048013","volume":"272","author":"D Moher","year":"1994","unstructured":"Moher, D., Dulberg, C. S. & Wells, G. A. Statistical power, sample size, and their reporting in randomized controlled trials. JAMA 272, 122\u2013124 (1994).","journal-title":"JAMA"},{"key":"BFs415620160021_CR57","doi-asserted-by":"crossref","unstructured":"Schulz, K. F., Altman, D. G., Moher, D. & Group, C. CONSORT 2010 statement: updated guidelines for reporting parallel group randomised trials. BMJ 340, c332 (2010).","DOI":"10.1136\/bmj.c332"},{"key":"BFs415620160021_CR58","doi-asserted-by":"publisher","first-page":"595","DOI":"10.1177\/1049731513498118","volume":"23","author":"S Grant","year":"2013","unstructured":"Grant, S. et al. Developing a reporting guideline for social and psychological intervention trials. Res. Social Work Prac. 23, 595\u2013602 (2013).","journal-title":"Res. Social Work Prac."},{"key":"BFs415620160021_CR59","doi-asserted-by":"crossref","unstructured":"Liberati, A. et al. The PRISMA statement for reporting systematic reviews and meta-analyses of studies that evaluate health care interventions: explanation and elaboration. PLoS Med. 6, e1000100 (2009).","DOI":"10.1371\/journal.pmed.1000100"},{"key":"BFs415620160021_CR60","doi-asserted-by":"publisher","first-page":"g7647","DOI":"10.1136\/bmj.g7647","volume":"349","author":"L Shamseer","year":"2016","unstructured":"Shamseer, L. et al. Preferred reporting items for systematic review and meta-analysis protocols (PRISMA-P) 2015: elaboration and explanation. BMJ 349, g7647 (2015); erratum 354, i4086 (2016).","journal-title":"BMJ"},{"key":"BFs415620160021_CR61","doi-asserted-by":"publisher","first-page":"2","DOI":"10.1016\/j.jesp.2016.03.004","volume":"67","author":"A van \u2018t Veer","year":"2016","unstructured":"van \u2018t Veer, A. & Giner-Sorolla, R. Pre-registration in social psychology: a discussion and suggested template. J. Exp. Soc. Psychol. 67, 2\u201312 (2016).","journal-title":"J. Exp. Soc. Psychol."},{"key":"BFs415620160021_CR62","doi-asserted-by":"publisher","first-page":"8","DOI":"10.1177\/1948550615598377","volume":"7","author":"A Franco","year":"2016","unstructured":"Franco, A., Malhotra, N. & Simonovits, G. Underreporting in psychology experiments: evidence from a study registry. Soc. Psychol. Per. Sci. 7, 8\u201312 (2016).","journal-title":"Soc. Psychol. Per. Sci."},{"key":"BFs415620160021_CR63","doi-asserted-by":"crossref","unstructured":"Alsheikh-Ali, A. A., Qureshi, W., Al-Mallah, M. H. & Ioannidis, J. P. Public availability of published research data in high-impact journals. PloS ONE 6, e24357 (2011).","DOI":"10.1371\/journal.pone.0024357"},{"key":"BFs415620160021_CR64","doi-asserted-by":"crossref","unstructured":"Iqbal, S. A., Wallach, J. D., Khoury, M. J., Schully, S. D. & Ioannidis, J. P. Reproducible research practices and transparency across the biomedical literature. PLoS Biol. 14, e1002333 (2016).","DOI":"10.1371\/journal.pbio.1002333"},{"issue":"6290","key":"BFs415620160021_CR65","doi-asserted-by":"publisher","first-page":"1147","DOI":"10.1126\/science.aag2359","volume":"352","author":"Marcia McNutt","year":"2016","unstructured":"McNutt, M. Taking up TOP. Science 352, 1147 (2016).","journal-title":"Science"},{"key":"BFs415620160021_CR66","doi-asserted-by":"publisher","first-page":"93","DOI":"10.1038\/nature12786","volume":"506","author":"IU Park","year":"2014","unstructured":"Park, I. U., Peacey, M. W. & Munaf\u00f2, M. R. Modelling the effects of subjective and objective decision making in scientific peer review. Nature 506, 93\u201396 (2014).","journal-title":"Nature"},{"key":"BFs415620160021_CR67","doi-asserted-by":"crossref","unstructured":"Button, K. S., Bal, L., Clark, A. G. & Shipley, T. Preventing the ends from justifying the means: withholding results to address publication bias in peer-review. BMC Psychol. 4, 59 (2016).","DOI":"10.1186\/s40359-016-0167-7"},{"key":"BFs415620160021_CR68","doi-asserted-by":"publisher","first-page":"899","DOI":"10.1126\/science.aaf9133","volume":"352","author":"JM Berg","year":"2016","unstructured":"Berg, J. M. et al. Preprints for the life sciences. Science 352, 899\u2013901 (2016).","journal-title":"Science"},{"key":"BFs415620160021_CR69","doi-asserted-by":"publisher","first-page":"217","DOI":"10.1080\/1047840X.2012.692215","volume":"23","author":"BA Nosek","year":"2012","unstructured":"Nosek, B. A. & Bar-Anan, T. Scientific utopia: I. Opening scientific communication. Psychol. Inq. 23, 217\u2013243 (2012).","journal-title":"Psychol. Inq."},{"key":"BFs415620160021_CR70","doi-asserted-by":"publisher","first-page":"47","DOI":"10.1192\/bjp.176.1.47","volume":"176","author":"E Walsh","year":"2000","unstructured":"Walsh, E., Rooney, M., Appleby, L. & Wilkinson, G. Open peer review: a randomised trial. Brit. J. Psychiat. 176, 47\u201351 (2000).","journal-title":"Brit. J. Psychiat."},{"key":"BFs415620160021_CR71","doi-asserted-by":"crossref","unstructured":"Smaldino, P. E. & McElreath, R. The natural selection of bad science. R. Soc. Open Sci. 3, 160384 (2016).","DOI":"10.1098\/rsos.160384"},{"key":"BFs415620160021_CR72","doi-asserted-by":"crossref","unstructured":"Higginson, A. D. & Munaf\u00f2, M. Current incentives for scientists lead to underpowered studies with erroneous conclusions. PLoS Biol. 14, e2000995 (2016).","DOI":"10.1371\/journal.pbio.2000995"},{"issue":"10","key":"BFs415620160021_CR73","doi-asserted-by":"publisher","first-page":"e1001747","DOI":"10.1371\/journal.pmed.1001747","volume":"11","author":"John P. A. Ioannidis","year":"2014","unstructured":"Ioannidis, J. P. How to make more published research true. PLoS Med. 11, e1001747 (2014).","journal-title":"PLoS Medicine"},{"key":"BFs415620160021_CR74","doi-asserted-by":"publisher","first-page":"7900","DOI":"10.1073\/pnas.1602413113","volume":"113","author":"A Eklund","year":"2016","unstructured":"Eklund, A., Nichols, T. E. & Knutsson, H. Cluster failure: why fMRI inferences for spatial extent have inflated false-positive rates. Proc. Natl Acad. Sci. USA 113, 7900\u20137905 (2016).","journal-title":"Proc. Natl Acad. Sci. USA"},{"issue":"5","key":"BFs415620160021_CR75","doi-asserted-by":"publisher","first-page":"e1002456","DOI":"10.1371\/journal.pbio.1002456","volume":"14","author":"Mallory C. Kidwell","year":"2016","unstructured":"Kidwell, M. C. et al. Badges to acknowledge open practices: a simple, low-cost, effective method for increasing transparency. PLoS Biol. 14, e1002456 (2016).","journal-title":"PLOS Biology"},{"key":"BFs415620160021_CR76","doi-asserted-by":"publisher","first-page":"871","DOI":"10.1038\/nbt.3004","volume":"32","author":"M Munaf\u00f2","year":"2014","unstructured":"Munaf\u00f2, M. et al. Scientific rigor and the art of motorcycle maintenance. Nat. Biotechnol. 32, 871\u2013873 (2014).","journal-title":"Nat. Biotechnol."},{"issue":"6","key":"BFs415620160021_CR77","doi-asserted-by":"publisher","first-page":"e1004961","DOI":"10.1371\/journal.pcbi.1004961","volume":"12","author":"Robert E. Kass","year":"2016","unstructured":"Kass, R. E. et al. Ten simple rules for effective statistical practice. PLoS Comput. Biol. 12, e1004961 (2016).","journal-title":"PLOS Computational Biology"},{"key":"BFs415620160021_CR78","first-page":"55","volume":"66","author":"M Schweinsberg","year":"2016","unstructured":"Schweinsberg, M. et al. The pipeline project: pre-publication independent replications of a single laboratory's research pipeline. J. Exp. Psychol. Gen. 66, 55\u201367 (2016).","journal-title":"J. Exp. Psychol. Gen."},{"issue":"jun25 2","key":"BFs415620160021_CR79","doi-asserted-by":"publisher","first-page":"g3804","DOI":"10.1136\/bmj.g3804","volume":"348","author":"A. Stevens","year":"2014","unstructured":"Stevens, A. et al. Relation of completeness of reporting of health research to journals' endorsement of reporting guidelines: systematic review. BMJ 348, g3804 (2014).","journal-title":"BMJ"},{"issue":"11","key":"BFs415620160021_CR80","doi-asserted-by":"publisher","first-page":"e7824","DOI":"10.1371\/journal.pone.0007824","volume":"4","author":"Carol Kilkenny","year":"2009","unstructured":"Kilkenny, C. et al. Survey of the quality of experimental design, statistical analysis and reporting of research using animals. PloS ONE 4, e7824 (2009).","journal-title":"PLoS ONE"},{"key":"BFs415620160021_CR81","doi-asserted-by":"crossref","unstructured":"Baker, D., Lidster, K., Sottomayor, A. & Amor, S. Two years later: journals are not yet enforcing the ARRIVE guidelines on reporting standards for pre-clinical animal studies. PLoS Biol. 12, e1001756 (2014).","DOI":"10.1371\/journal.pbio.1001756"},{"key":"BFs415620160021_CR82","doi-asserted-by":"crossref","unstructured":"Gulin, J. E., Rocco, D. M. & Garcia-Bournissen, F. Quality of reporting and adherence to ARRIVE guidelines in animal studies for Chagas disease preclinical drug research: a systematic review. PLoS Negl. Trop. Dis. 9, e0004194 (2015).","DOI":"10.1371\/journal.pntd.0004194"},{"issue":"5","key":"BFs415620160021_CR83","doi-asserted-by":"publisher","first-page":"e0154657","DOI":"10.1371\/journal.pone.0154657","volume":"11","author":"Yali Liu","year":"2016","unstructured":"Liu, Y. et al. Adherence to ARRIVE guidelines in Chinese journal reports on neoplasms in animals. PloS ONE 11, e0154657 (2016).","journal-title":"PLOS ONE"},{"key":"BFs415620160021_CR84","doi-asserted-by":"crossref","unstructured":"Gotzsche, P. C. & Ioannidis, J. P. Content area experts as authors: helpful or harmful for systematic reviews and meta-analyses? BMJ 345, e7031 (2012).","DOI":"10.1136\/bmj.e7031"},{"issue":"1","key":"BFs415620160021_CR85","doi-asserted-by":"publisher","first-page":"150547","DOI":"10.1098\/rsos.150547","volume":"3","author":"Richard D. Morey","year":"2016","unstructured":"Morey, R. D. et al. The Peer Reviewers' Openness Initiative: incentivizing open research practices through peer review. R. Soc. Open Sci. 3, 150547 (2016).","journal-title":"Royal Society Open Science"},{"key":"BFs415620160021_CR86","doi-asserted-by":"publisher","unstructured":"Simmons, J. P., Nelson, L. D. & Simonsohn, U. A 21 word solution. Preprint at http:\/\/dx.doi.org\/10.2139\/ssrn.2160588(2012).","DOI":"10.2139\/ssrn.2160588(2012)"},{"key":"BFs415620160021_CR87","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1177\/0956797613512465","volume":"25","author":"E Eich","year":"2014","unstructured":"Eich, E. Business not as usual. Psychol. Sci. 25, 3\u20136 (2014).","journal-title":"Psychol. Sci."}],"container-title":["Nature Human Behaviour"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41562-016-0021.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41562-016-0021","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41562-016-0021.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,20]],"date-time":"2024-03-20T10:08:29Z","timestamp":1710929309000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41562-016-0021"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2017,1,10]]},"references-count":87,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2017,1]]}},"alternative-id":["BFs415620160021"],"URL":"https:\/\/doi.org\/10.1038\/s41562-016-0021","relation":{"has-review":[{"id-type":"doi","id":"10.3410\/f.727189988.793527393","asserted-by":"object"}]},"ISSN":["2397-3374"],"issn-type":[{"value":"2397-3374","type":"electronic"}],"subject":[],"published":{"date-parts":[[2017,1,10]]},"assertion":[{"value":"10 January 2017","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"M.R.M, together with C.D.C and D.V.M.B., has received funding from the BBSRC (grant number BB\/N019660\/1) to convene a workshop on advanced methods for reproducible science, and is chair of the CHDI Foundation Independent Statistical Standing Committee. B.A.N. is executive director of the non-profit Center for Open Science with a mission to increase openness, integrity and reproducibility of research. N.P.d.S. leads the NC3Rs programme of work on experimental design, which developed the ARRIVE guidelines and Experimental Design Assistant. J.J.W. is director, experimental design, at CHDI Management\/CHDI Foundation, a non-profit biomedical research organization exclusively dedicated to developing therapeutics for Huntington's disease. The other authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"0021"}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1038_2Fs41562_021_01142_4 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T23:07:27Z","timestamp":1793228847305,"version":"4.5.0"},"reference-count":39,"publisher":"Springer Science and Business Media LLC","issue":"8","license":[{"start":{"date-parts":[[2021,6,24]],"date-time":"2021-06-24T00:00:00Z","timestamp":1624492800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2021,6,24]],"date-time":"2021-06-24T00:00:00Z","timestamp":1624492800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"funder":[{"DOI":"10.13039\/100000913","name":"James S. McDonnell Foundation","doi-asserted-by":"publisher","award":["220020498"],"award-info":[{"award-number":["220020498"]}],"id":[{"id":"10.13039\/100000913","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Arnold Ventures, no reference number"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat Hum Behav"],"DOI":"10.1038\/s41562-021-01142-4","type":"journal-article","created":{"date-parts":[[2021,6,24]],"date-time":"2021-06-24T12:02:42Z","timestamp":1624536162000},"page":"990-997","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":185,"title":["Initial evidence of research quality of registered reports compared with the standard publishing model"],"prefix":"10.1038","volume":"5","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1227-7042","authenticated-orcid":false,"given":"Courtney K.","family":"Soderberg","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4959-5143","authenticated-orcid":false,"given":"Timothy M.","family":"Errington","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8829-137X","authenticated-orcid":false,"given":"Sarah R.","family":"Schiavone","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Julia","family":"Bottesini","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0237-6146","authenticated-orcid":false,"given":"Felix Singleton","family":"Thorn","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3933-9752","authenticated-orcid":false,"given":"Simine","family":"Vazire","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5529-6422","authenticated-orcid":false,"given":"Kevin M.","family":"Esterling","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6797-5476","authenticated-orcid":false,"given":"Brian A.","family":"Nosek","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,6,24]]},"reference":[{"key":"1142_CR1","doi-asserted-by":"publisher","first-page":"187","DOI":"10.1038\/d41586-019-02674-6","volume":"573","author":"C Chambers","year":"2019","unstructured":"Chambers, C. What\u2019s next for registered reports? Nature 573, 187\u2013189 (2019).","journal-title":"Nature"},{"key":"1142_CR2","doi-asserted-by":"publisher","first-page":"23","DOI":"10.1111\/j.1740-9713.2019.01299.x","volume":"16","author":"C Chambers","year":"2019","unstructured":"Chambers, C. The registered reports revolution. Lessons in cultural reform. Significance 16, 23\u201327 (2019).","journal-title":"Significance"},{"key":"1142_CR3","doi-asserted-by":"publisher","first-page":"137","DOI":"10.1027\/1864-9335\/a000192","volume":"45","author":"BA Nosek","year":"2014","unstructured":"Nosek, B. A. & Lakens, D. Registered reports: a method to increase the credibility of published results. Soc. Psychol. 45, 137\u2013141 (2014).","journal-title":"Soc. Psychol."},{"key":"1142_CR4","doi-asserted-by":"publisher","first-page":"615","DOI":"10.1177\/1745691612459058","volume":"7","author":"BA Nosek","year":"2012","unstructured":"Nosek, B. A., Spies, J. R. & Motyl, M. Scientific utopia: II. restructuring incentives and practices to promote truth over publishability. Perspect. Psychol. Sci. 7, 615\u2013631 (2012).","journal-title":"Perspect. Psychol. Sci."},{"key":"1142_CR5","doi-asserted-by":"publisher","first-page":"178","DOI":"10.1177\/014107680609900414","volume":"99","author":"R Smith","year":"2006","unstructured":"Smith, R. Peer review: a flawed process at the heart of science and journals. J. R. Soc. Med. 99, 178\u2013182 (2006).","journal-title":"J. R. Soc. Med."},{"key":"1142_CR6","doi-asserted-by":"publisher","first-page":"891","DOI":"10.1007\/s11192-011-0494-7","volume":"90","author":"D Fanelli","year":"2012","unstructured":"Fanelli, D. Negative results are disappearing from most disciplines and countries. Scientometrics 90, 891\u2013904 (2012).","journal-title":"Scientometrics"},{"key":"1142_CR7","doi-asserted-by":"publisher","first-page":"e10068","DOI":"10.1371\/journal.pone.0010068","volume":"5","author":"D Fanelli","year":"2010","unstructured":"Fanelli, D. \u2018Positive\u2019 results increase down the hierarchy of the sciences. PLoS ONE 5, e10068 (2010).","journal-title":"PLoS ONE"},{"key":"1142_CR8","doi-asserted-by":"publisher","first-page":"1502","DOI":"10.1126\/science.1255484","volume":"345","author":"A Franco","year":"2014","unstructured":"Franco, A., Malhotra, N. & Simonovits, G. Publication bias in the social sciences: unlocking the file drawer. Science 345, 1502\u20131505 (2014).","journal-title":"Science"},{"key":"1142_CR9","doi-asserted-by":"publisher","first-page":"1385","DOI":"10.1001\/jama.1990.03440100097014","volume":"263","author":"K Dickersin","year":"1990","unstructured":"Dickersin, K. The existence of publication bias and risk factors for its occurrence. JAMA 263, 1385\u20131389 (1990).","journal-title":"JAMA"},{"key":"1142_CR10","doi-asserted-by":"publisher","first-page":"161","DOI":"10.1007\/BF01173636","volume":"1","author":"MJ Mahoney","year":"1977","unstructured":"Mahoney, M. J. Publication prejudices: an experimental study of confirmatory bias in the peer review system. Cogn. Ther. Res. 1, 161\u2013175 (1977).","journal-title":"Cogn. Ther. Res."},{"key":"1142_CR11","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1037\/h0076157","volume":"82","author":"AG Greenwald","year":"1975","unstructured":"Greenwald, A. G. Consequences of prejudice against the null hypothesis. Psychol. Bull. 82, 1\u201320 (1975).","journal-title":"Psychol. Bull."},{"key":"1142_CR12","first-page":"30","volume":"54","author":"TD Sterling","year":"1959","unstructured":"Sterling, T. D. Publication decisions and their possible effects on inferences drawn from tests of significance\u2014or vice versa. J. Am. Stat. Assoc. 54, 30\u201334 (1959).","journal-title":"J. Am. Stat. Assoc."},{"key":"1142_CR13","doi-asserted-by":"publisher","first-page":"537","DOI":"10.1177\/1745691612460688","volume":"7","author":"MC Makel","year":"2012","unstructured":"Makel, M. C., Plucker, J. A. & Hegarty, B. Replications in psychology research: How often do they really occur? Perspect. Psychol. Sci. 7, 537\u2013542 (2012).","journal-title":"Perspect. Psychol. Sci."},{"key":"1142_CR14","doi-asserted-by":"publisher","first-page":"90","DOI":"10.1037\/a0015108","volume":"13","author":"S Schmidt","year":"2009","unstructured":"Schmidt, S. Shall we really do it again? The powerful concept of replication is neglected in the social sciences. Rev. Gen. Psychol. 13, 90\u2013100 (2009).","journal-title":"Rev. Gen. Psychol."},{"key":"1142_CR15","doi-asserted-by":"publisher","first-page":"304","DOI":"10.3102\/0013189X14545513","volume":"43","author":"MC Makel","year":"2014","unstructured":"Makel, M. C. & Plucker, J. A. Facts are more important than novelty. Educ. Res. 43, 304\u2013316 (2014).","journal-title":"Educ. Res."},{"key":"1142_CR16","doi-asserted-by":"publisher","first-page":"551","DOI":"10.1037\/a0029487","volume":"17","author":"U Schimmack","year":"2012","unstructured":"Schimmack, U. The ironic effect of significant results on the credibility of multiple-study articles. Psychol. Methods 17, 551\u2013566 (2012).","journal-title":"Psychol. Methods"},{"key":"1142_CR17","doi-asserted-by":"publisher","first-page":"562","DOI":"10.1177\/1745691612457576","volume":"7","author":"R Giner-Sorolla","year":"2012","unstructured":"Giner-Sorolla, R. Science or art? How aesthetic standards grease the way through the publication bottleneck but undermine science. Perspect. Psychol. Sci. 7, 562\u2013571 (2012).","journal-title":"Perspect. Psychol. Sci."},{"key":"1142_CR18","doi-asserted-by":"publisher","first-page":"531","DOI":"10.1038\/483531a","volume":"483","author":"CG Begley","year":"2012","unstructured":"Begley, C. G. & Ellis, L. M. Raise standards for preclinical cancer research. Nature 483, 531\u2013533 (2012).","journal-title":"Nature"},{"key":"1142_CR19","doi-asserted-by":"publisher","first-page":"712","DOI":"10.1038\/nrd3439-c1","volume":"10","author":"F Prinz","year":"2011","unstructured":"Prinz, F., Schlange, T. & Asadullah, K. Believe it or not: how much can we rely on published data on potential drug targets? Nat. Rev. Drug Discov. 10, 712\u2013712 (2011).","journal-title":"Nat. Rev. Drug Discov."},{"key":"1142_CR20","doi-asserted-by":"publisher","first-page":"1433","DOI":"10.1126\/science.aaf0918","volume":"351","author":"CF Camerer","year":"2016","unstructured":"Camerer, C. F. et al. Evaluating replicability of laboratory experiments in economics. Science 351, 1433\u20131436 (2016).","journal-title":"Science"},{"key":"1142_CR21","doi-asserted-by":"publisher","first-page":"637","DOI":"10.1038\/s41562-018-0399-z","volume":"2","author":"CF Camerer","year":"2018","unstructured":"Camerer, C. F. et al. Evaluating the replicability of social science experiments in Nature and Science between 2010 and 2015. Nat. Hum. Behav. 2, 637\u2013644 (2018).","journal-title":"Nat. Hum. Behav."},{"key":"1142_CR22","doi-asserted-by":"publisher","first-page":"443","DOI":"10.1177\/2515245918810225","volume":"1","author":"RA Klein","year":"2018","unstructured":"Klein, R. A. et al. Many Labs 2: investigating variation in replicability across samples and settings. Adv. Methods Pract. Psychol. Sci. 1, 443\u2013490 (2018).","journal-title":"Adv. Methods Pract. Psychol. Sci."},{"key":"1142_CR23","doi-asserted-by":"publisher","first-page":"142","DOI":"10.1027\/1864-9335\/a000178","volume":"45","author":"RA Klein","year":"2014","unstructured":"Klein, R. A. et al. Investigating variation in replicability: a \u2018many labs\u2019 replication project. Soc. Psychol. 45, 142\u2013152 (2014).","journal-title":"Soc. Psychol."},{"key":"1142_CR24","doi-asserted-by":"crossref","unstructured":"Open Science Collaboration. Estimating the reproducibility of psychological science. Science 349, aac4716 (2015).","DOI":"10.1126\/science.aac4716"},{"key":"1142_CR25","doi-asserted-by":"publisher","first-page":"68","DOI":"10.1016\/j.jesp.2015.10.012","volume":"67","author":"CR Ebersole","year":"2016","unstructured":"Ebersole, C. R. et al. Many Labs 3: evaluating participant pool quality across the academic semester via replication. J. Exp. Soc. Psychol. 67, 68\u201382 (2016).","journal-title":"J. Exp. Soc. Psychol."},{"key":"1142_CR26","doi-asserted-by":"publisher","first-page":"e3000246","DOI":"10.1371\/journal.pbio.3000246","volume":"17","author":"C Allen","year":"2019","unstructured":"Allen, C. & Mehler, D. M. A. Open science challenges, benefits and tips in early career and beyond. PLoS Biol. 17, e3000246 (2019).","journal-title":"PLoS Biol."},{"key":"1142_CR27","doi-asserted-by":"crossref","unstructured":"Scheel, A. M., Schijen, M. & Lakens, D. An excess of positive results: comparing the standard psychology literature with registered reports. Preprint at PsyArXiv https:\/\/osf.io\/p6e9c (2020).","DOI":"10.31234\/osf.io\/p6e9c"},{"key":"1142_CR28","doi-asserted-by":"crossref","unstructured":"Hummer, L. T., Singleton Thorn, F., Nosek, B. A. & Errington, T. M. Evaluating registered reports: a naturalistic comparative study of article impact. Preprint at OSF https:\/\/osf.io\/5y8w7 (2017).","DOI":"10.31219\/osf.io\/5y8w7"},{"key":"1142_CR29","first-page":"323","volume":"30","author":"A Cropley","year":"2018","unstructured":"Cropley, A. Research as artisanship versus research as generation of novelty: the march to nowhere. Creat. Res. J 30, 323\u2013328 (2018).","journal-title":"Creat. Res. J"},{"key":"1142_CR30","doi-asserted-by":"publisher","first-page":"153","DOI":"10.1016\/j.jesp.2016.02.003","volume":"66","author":"RF Baumeister","year":"2016","unstructured":"Baumeister, R. F. Charting the future of social psychology on stormy seas: winners, losers, and recommendations. J. Exp. Soc. Psychol. 66, 153\u2013158 (2016).","journal-title":"J. Exp. Soc. Psychol."},{"key":"1142_CR31","doi-asserted-by":"publisher","first-page":"518","DOI":"10.1038\/d41586-020-02142-6","volume":"583","author":"BA Nosek","year":"2020","unstructured":"Nosek, B. A. & Errington, T. M. The best time to argue about what a replication means? Before you do it. Nature 583, 518\u2013520 (2020).","journal-title":"Nature"},{"key":"1142_CR32","first-page":"189","volume":"5","author":"A Gelman","year":"2012","unstructured":"Gelman, A., Hill, J. & Yajima, M. Why we (usually) don\u2019t have to worry about multiple comparisons. J. Res. Educ. Eff. 5, 189\u2013211 (2012).","journal-title":"J. Res. Educ. Eff."},{"key":"1142_CR33","unstructured":"Epskamp, S. & Nuijten, M. B. statcheck: extract statistics from articles and recompute P values. R package version 1.3.1 (2018)."},{"key":"1142_CR34","doi-asserted-by":"publisher","first-page":"793","DOI":"10.1038\/s41562-018-0444-y","volume":"2","author":"TE Hardwicke","year":"2018","unstructured":"Hardwicke, T. E. & Ioannidis, J. P. A. Mapping the universe of registered reports. Nat. Hum. Behav. 2, 793\u2013796 (2018).","journal-title":"Nat. Hum. Behav."},{"key":"1142_CR35","doi-asserted-by":"publisher","first-page":"791","DOI":"10.1038\/s41562-018-0449-6","volume":"2","author":"CD Chambers","year":"2018","unstructured":"Chambers, C. D. & Mellor, D. T. Protocol transparency is vital for registered reports. Nat. Hum. Behav. 2, 791\u2013792 (2018).","journal-title":"Nat. Hum. Behav."},{"key":"1142_CR36","doi-asserted-by":"publisher","first-page":"524","DOI":"10.1177\/0956797611430953","volume":"23","author":"LK John","year":"2012","unstructured":"John, L. K., Loewenstein, G. & Prelec, D. Measuring the prevalence of questionable research practices with incentives for truth telling. Psychol. Sci. 23, 524\u2013532 (2012).","journal-title":"Psychol. Sci."},{"key":"1142_CR37","doi-asserted-by":"publisher","first-page":"1359","DOI":"10.1177\/0956797611417632","volume":"22","author":"JP Simmons","year":"2011","unstructured":"Simmons, J. P., Nelson, L. D. & Simonsohn, U. False-positive psychology: undisclosed flexibility in data collection and analysis allows presenting anything as significant. Psychol. Sci. 22, 1359\u20131366 (2011).","journal-title":"Psychol. Sci."},{"key":"1142_CR38","doi-asserted-by":"crossref","unstructured":"Nuijten, M. B., van Assen, M. A. L. M., Hartgerink, C. H. J., Epskamp, S. & Wicherts, J. M. The validity of the tool \u2018statcheck\u2019 in discovering statistical reporting inconsistencies. Preprint at PsyArXiv https:\/\/osf.io\/tcxaj (2017).","DOI":"10.31234\/osf.io\/tcxaj"},{"key":"1142_CR39","unstructured":"Stan Development Team. Stan Modeling Language Users Guide and Reference Manual (2020)."}],"container-title":["Nature Human Behaviour"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41562-021-01142-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41562-021-01142-4","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41562-021-01142-4.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,12,2]],"date-time":"2022-12-02T19:47:38Z","timestamp":1670010458000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41562-021-01142-4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,6,24]]},"references-count":39,"journal-issue":{"issue":"8","published-online":{"date-parts":[[2021,8]]}},"alternative-id":["1142"],"URL":"https:\/\/doi.org\/10.1038\/s41562-021-01142-4","relation":{"has-preprint":[{"id-type":"doi","id":"10.31222\/osf.io\/7x9vy","asserted-by":"object"}]},"ISSN":["2397-3374"],"issn-type":[{"value":"2397-3374","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,6,24]]},"assertion":[{"value":"7 January 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 May 2021","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"24 June 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"T.M.E., C.K.S. and B.A.N. are employees of the nonprofit Center for Open Science (COS), which has a mission to increase openness, integrity and reproducibility of research. COS offers support to journals, editors and researchers in adopting and conducting RRs. The remaining authors declare no conflicts of interest.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1038_2Fs41562_021_01193_7 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T07:11:40Z","timestamp":1793171500107,"version":"4.5.0"},"reference-count":130,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2021,11,15]],"date-time":"2021-11-15T00:00:00Z","timestamp":1636934400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"},{"start":{"date-parts":[[2021,11,15]],"date-time":"2021-11-15T00:00:00Z","timestamp":1636934400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springernature.com\/gp\/researchers\/text-and-data-mining"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat Hum Behav"],"DOI":"10.1038\/s41562-021-01193-7","type":"journal-article","created":{"date-parts":[[2021,11,15]],"date-time":"2021-11-15T12:02:52Z","timestamp":1636977772000},"page":"29-42","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":335,"title":["The past, present and future of Registered Reports"],"prefix":"10.1038","volume":"6","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6058-4114","authenticated-orcid":false,"given":"Christopher D.","family":"Chambers","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1463-9396","authenticated-orcid":false,"given":"Loukia","family":"Tzavella","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2021,11,15]]},"reference":[{"key":"1193_CR1","doi-asserted-by":"publisher","unstructured":"Vazire, S. Implications of the credibility revolution for productivity, creativity, and progress: Perspect. Psychol. Sci. https:\/\/doi.org\/10.1177\/1745691617751884 (2018).","DOI":"10.1177\/1745691617751884"},{"key":"1193_CR2","doi-asserted-by":"publisher","first-page":"0021","DOI":"10.1038\/s41562-016-0021","volume":"1","author":"MR Munaf\u00f2","year":"2017","unstructured":"Munaf\u00f2, M. R. et al. A manifesto for reproducible science. Nat. Hum. Behav. 1, 0021 (2017).","journal-title":"Nat. Hum. Behav."},{"key":"1193_CR3","unstructured":"Reproducibility and Reliability of Biomedical Research: Improving Research Practice (The Academy of Medical Sciences, 2015); https:\/\/acmedsci.ac.uk\/file-download\/38189-56531416e2949.pdf"},{"key":"1193_CR4","doi-asserted-by":"publisher","first-page":"e10068","DOI":"10.1371\/journal.pone.0010068","volume":"5","author":"D Fanelli","year":"2010","unstructured":"Fanelli, D. \u201cPositive\u201d results increase down the hierarchy of the sciences. PLoS ONE 5, e10068 (2010).","journal-title":"PLoS ONE"},{"key":"1193_CR5","doi-asserted-by":"publisher","first-page":"1502","DOI":"10.1126\/science.1255484","volume":"345","author":"A Franco","year":"2014","unstructured":"Franco, A., Malhotra, N. & Simonovits, G. Publication bias in the social sciences: unlocking the file drawer. Science 345, 1502\u20131505 (2014).","journal-title":"Science"},{"key":"1193_CR6","doi-asserted-by":"publisher","unstructured":"Harrison, J. S., Banks, G. C., Pollack, J. M., O\u2019Boyle, E. H. & Short, J. Publication bias in strategic management research. J. Manag. https:\/\/doi.org\/10.1177\/0149206314535438 (2014).","DOI":"10.1177\/0149206314535438"},{"key":"1193_CR7","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1017\/S1464793101005875","volume":"77","author":"MD Jennions","year":"2002","unstructured":"Jennions, M. D. & M\u00f8ller, A. P. Publication bias in ecology and evolution: an empirical assessment using the \u2018trim and fill\u2019 method. Biol. Rev. 77, 211\u2013222 (2002).","journal-title":"Biol. Rev."},{"key":"1193_CR8","doi-asserted-by":"publisher","first-page":"196","DOI":"10.1207\/s15327957pspr0203_4","volume":"2","author":"NL Kerr","year":"1998","unstructured":"Kerr, N. L. HARKing: hypothesizing after the results are known. Personal. Soc. Psychol. Rev. 2, 196\u2013217 (1998).","journal-title":"Personal. Soc. Psychol. Rev."},{"key":"1193_CR9","doi-asserted-by":"publisher","first-page":"e0149144","DOI":"10.1371\/journal.pone.0149144","volume":"11","author":"SB Bruns","year":"2016","unstructured":"Bruns, S. B. & Ioannidis, J. P. A. p-Curve and p-Hacking in observational research. PLoS ONE 11, e0149144 (2016).","journal-title":"PLoS ONE"},{"key":"1193_CR10","doi-asserted-by":"publisher","first-page":"119","DOI":"10.2308\/bria-52183","volume":"31","author":"MJ Khan","year":"2018","unstructured":"Khan, M. J. & Tr\u00f8nnes, P. C. p-Hacking in experimental audit research. Behav. Res. Account. 31, 119\u2013131 (2018).","journal-title":"Behav. Res. Account."},{"key":"1193_CR11","doi-asserted-by":"publisher","first-page":"e1002190","DOI":"10.1371\/journal.pbio.1002190","volume":"13","author":"L Holman","year":"2015","unstructured":"Holman, L., Head, M. L., Lanfear, R. & Jennions, M. D. Evidence of experimental bias in the life sciences: why we need blind data recording. PLoS Biol. 13, e1002190 (2015).","journal-title":"PLoS Biol."},{"key":"1193_CR12","doi-asserted-by":"publisher","first-page":"45","DOI":"10.1177\/1948550615612150","volume":"7","author":"K Fiedler","year":"2016","unstructured":"Fiedler, K. & Schwarz, N. Questionable research practices revisited. Soc. Psychol. Personal. Sci. 7, 45\u201352 (2016).","journal-title":"Soc. Psychol. Personal. Sci."},{"key":"1193_CR13","doi-asserted-by":"publisher","unstructured":"Rabelo, A. L. A. et al. Questionable research practices among Brazilian psychological researchers: results from a replication study and an international comparison. Int. J. Psychol. https:\/\/doi.org\/10.1002\/ijop.12632 (2019).","DOI":"10.1002\/ijop.12632"},{"key":"1193_CR14","doi-asserted-by":"publisher","first-page":"e0200303","DOI":"10.1371\/journal.pone.0200303","volume":"13","author":"H Fraser","year":"2018","unstructured":"Fraser, H., Parker, T., Nakagawa, S., Barnett, A. & Fidler, F. Questionable research practices in ecology and evolution. PLoS ONE 13, e0200303 (2018).","journal-title":"PLoS ONE"},{"key":"1193_CR15","doi-asserted-by":"publisher","first-page":"524","DOI":"10.1177\/0956797611430953","volume":"23","author":"LK John","year":"2012","unstructured":"John, L. K., Loewenstein, G. & Prelec, D. Measuring the prevalence of questionable research practices with incentives for truth telling. Psychol. Sci. 23, 524\u2013532 (2012).","journal-title":"Psychol. Sci."},{"key":"1193_CR16","doi-asserted-by":"publisher","first-page":"365","DOI":"10.1038\/nrn3475","volume":"14","author":"KS Button","year":"2013","unstructured":"Button, K. S. et al. Power failure: why small sample size undermines the reliability of neuroscience. Nat. Rev. Neurosci. 14, 365\u2013376 (2013).","journal-title":"Nat. Rev. Neurosci."},{"key":"1193_CR17","doi-asserted-by":"publisher","first-page":"726","DOI":"10.1037\/0003-066X.61.7.726","volume":"61","author":"JM Wicherts","year":"2006","unstructured":"Wicherts, J. M., Borsboom, D., Kats, J. & Molenaar, D. The poor availability of psychological research data for reanalysis. Am. Psychol. 61, 726\u2013728 (2006).","journal-title":"Am. Psychol."},{"key":"1193_CR18","doi-asserted-by":"publisher","first-page":"62","DOI":"10.1016\/j.respol.2018.07.019","volume":"48","author":"F Mueller-Langer","year":"2019","unstructured":"Mueller-Langer, F., Fecher, B., Harhoff, D. & Wagner, G. G. Replication studies in economics\u2014how many and which papers are chosen for replication, and why? Res. Policy 48, 62\u201383 (2019).","journal-title":"Res. Policy"},{"key":"1193_CR19","doi-asserted-by":"publisher","first-page":"537","DOI":"10.1177\/1745691612460688","volume":"7","author":"MC Makel","year":"2012","unstructured":"Makel, M. C., Plucker, J. A. & Hegarty, B. Replications in psychology research: how often do they really occur? Perspect. Psychol. Sci. 7, 537\u2013542 (2012).","journal-title":"Perspect. Psychol. Sci."},{"key":"1193_CR20","doi-asserted-by":"publisher","first-page":"1433","DOI":"10.1126\/science.aaf0918","volume":"351","author":"CF Camerer","year":"2016","unstructured":"Camerer, C. F. et al. Evaluating replicability of laboratory experiments in economics. Science 351, 1433\u20131436 (2016).","journal-title":"Science"},{"key":"1193_CR21","unstructured":"Davis, R. J. et al. Reproducibility project: cancer biology. eLife https:\/\/elifesciences.org\/collections\/9b1e83d1\/reproducibility-project-cancer-biology (2014)."},{"key":"1193_CR22","doi-asserted-by":"publisher","first-page":"aac4716","DOI":"10.1126\/science.aac4716","volume":"349","author":"Open Science Collaboration.","year":"2015","unstructured":"Open Science Collaboration. Estimating the reproducibility of psychological science. Science 349, aac4716 (2015).","journal-title":"Science"},{"key":"1193_CR23","doi-asserted-by":"publisher","first-page":"615","DOI":"10.1177\/1745691612459058","volume":"7","author":"BA Nosek","year":"2012","unstructured":"Nosek, B. A., Spies, J. R. & Motyl, M. Scientific utopia: II. Restructuring incentives and practices to promote truth over publishability. Perspect. Psychol. Sci. 7, 615\u2013631 (2012).","journal-title":"Perspect. Psychol. Sci."},{"key":"1193_CR24","first-page":"10","volume":"37","author":"RP Feynman","year":"1974","unstructured":"Feynman, R. P. Cargo cult science. Eng. Sci. 37, 10\u201313 (1974).","journal-title":"Eng. Sci."},{"key":"1193_CR25","unstructured":"Johnson, J. A. Are research psychologists more like detectives or lawyers? Psychol. Today http:\/\/www.psychologytoday.com\/blog\/cui-bono\/201307\/are-research-psychologists-more-detectives-or-lawyers-0 (2013)."},{"key":"1193_CR26","unstructured":"Bem, D. J. In The Compleat Academic: A Practical Guide for the Beginning Social Scientist (eds Zanna, M. P. & Darley, J. M.) Ch. 8 (Lawerence Erlbaum, 1987)."},{"key":"1193_CR27","unstructured":"Bem, D. J. In The Compleat Academic: A Career Guide 2nd edn (eds Darley, J. M. et al.) Ch. 10 (American Psychological Association, 2003)."},{"key":"1193_CR28","unstructured":"Fiske, S. T. In The Sage Handbook of Methods in Social Psychology (eds Sansone, C. et al.) Ch. 4 (SAGE, 2003)."},{"key":"1193_CR29","doi-asserted-by":"publisher","first-page":"e50527","DOI":"10.7554\/eLife.50527","volume":"8","author":"JR Sanes","year":"2019","unstructured":"Sanes, J. R. Tell me a story. eLife 8, e50527 (2019).","journal-title":"eLife"},{"key":"1193_CR30","doi-asserted-by":"publisher","first-page":"543","DOI":"10.1177\/1745691612459060","volume":"7","author":"M Bakker","year":"2012","unstructured":"Bakker, M., van Dijk, A. & Wicherts, J. M. The rules of the game called psychological science. Perspect. Psychol. Sci. 7, 543\u2013554 (2012).","journal-title":"Perspect. Psychol. Sci."},{"key":"1193_CR31","doi-asserted-by":"publisher","first-page":"448","DOI":"10.1177\/1745691618767883","volume":"13","author":"JA Grand","year":"2018","unstructured":"Grand, J. A., Rogelberg, S. G., Banks, G. C., Landis, R. S. & Tonidandel, S. From outcome to process focus: fostering a more robust psychological science through Registered Reports and results\u2014blind reviewing. Perspect. Psychol. Sci. 13, 448\u2013456 (2018).","journal-title":"Perspect. Psychol. Sci."},{"key":"1193_CR32","unstructured":"Eich, E. PSCI Initiatives for 2013. https:\/\/groups.google.com\/group\/openscienceframework\/attach\/8e518ad385b642e5\/PSCI%20Initiatives%20for%202013%20%2820121008%29.docx?part=0.1 (2012)."},{"key":"1193_CR33","unstructured":"Chambers, C. Changing the culture of scientific publishing from within. NeuroChambers (8 October 2021); https:\/\/neurochambers.blogspot.com\/2012\/10\/changing-culture-of-scientific.html"},{"key":"1193_CR34","unstructured":"Simons, D. J. Registered Replication Reports\u2014Stay Tuned! Daniel Simons Blog (13 May 2013); http:\/\/blog.dansimons.com\/2013\/05\/registered-replication-reports-stay.html"},{"key":"1193_CR35","doi-asserted-by":"publisher","first-page":"59","DOI":"10.1027\/1864-9335\/a000143","volume":"44","author":"BA Nosek","year":"2013","unstructured":"Nosek, B. A. & Lakens, D. Call for proposals: special issue of social psychology on \u201creplications of important results in social psychology\u201d. Soc. Psychol. 44, 59\u201360 (2013).","journal-title":"Soc. Psychol."},{"key":"1193_CR36","doi-asserted-by":"publisher","first-page":"609","DOI":"10.1016\/j.cortex.2012.12.016","volume":"49","author":"CD Chambers","year":"2013","unstructured":"Chambers, C. D. Registered Reports: a new publishing initiative at Cortex. Cortex 49, 609\u2013610 (2013).","journal-title":"Cortex"},{"key":"1193_CR37","first-page":"470","volume":"13","author":"CS Peirce","year":"1878","unstructured":"Peirce, C. S. Illustrations of the logic of science VI: deduction, induction, and hypothesis. Pop. Sci. Monthly 13, 470\u2013482 (1878).","journal-title":"Pop. Sci. Monthly"},{"key":"1193_CR38","doi-asserted-by":"publisher","first-page":"418","DOI":"10.1177\/1745691618771357","volume":"13","author":"E-J Wagenmakers","year":"2018","unstructured":"Wagenmakers, E.-J., Dutilh, G. & Sarafoglou, A. The creativity\u2013verification cycle in psychological science: new methods to combat old Iidols. Perspect. Psychol. Sci. 13, 418\u2013427 (2018).","journal-title":"Perspect. Psychol. Sci."},{"key":"1193_CR39","unstructured":"Rosenthal, R. Experimenter Effects in Behavioral Research (Appleton-Century-Crofts, 1966)."},{"key":"1193_CR40","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1177\/014662168901300101","volume":"13","author":"DJ Weiss","year":"1989","unstructured":"Weiss, D. J. An experiment in publication: advance publication review. Appl. Psychol. Meas. 13, 1\u20137 (1989).","journal-title":"Appl. Psychol. Meas."},{"key":"1193_CR41","doi-asserted-by":"publisher","first-page":"635","DOI":"10.1037\/0003-066X.43.8.635","volume":"43","author":"J Kupfersmid","year":"1988","unstructured":"Kupfersmid, J. Improving what is published: a model in search of an editor. Am. Psychol. 43, 635\u2013642 (1988).","journal-title":"Am. Psychol."},{"key":"1193_CR42","doi-asserted-by":"publisher","first-page":"656","DOI":"10.1136\/bmj.295.6599.656","volume":"295","author":"RG Newcombe","year":"1987","unstructured":"Newcombe, R. G. Towards a reduction in publication bias. Br. Med J. Clin. Res. Ed. 295, 656\u2013659 (1987).","journal-title":"Br. Med J. Clin. Res. Ed."},{"key":"1193_CR43","doi-asserted-by":"publisher","first-page":"161","DOI":"10.1007\/BF01173636","volume":"1","author":"MJ Mahoney","year":"1977","unstructured":"Mahoney, M. J. Publication prejudices: an experimental study of confirmatory bias in the peer review system. Cogn. Ther. Res. 1, 161\u2013175 (1977).","journal-title":"Cogn. Ther. Res."},{"key":"1193_CR44","doi-asserted-by":"publisher","first-page":"16","DOI":"10.1080\/00031305.1970.10478884","volume":"24","author":"GW Walster","year":"1970","unstructured":"Walster, G. W. & Cleary, T. A. A proposal for a new editorial policy in the social sciences. Am. Stat. 24, 16\u201319 (1970).","journal-title":"Am. Stat."},{"key":"1193_CR45","doi-asserted-by":"publisher","first-page":"e6232","DOI":"10.7717\/peerj.6232","volume":"7","author":"R Wiseman","year":"2019","unstructured":"Wiseman, R., Watt, C. & Kornbrot, D. Registered Reports: an early example and analysis. PeerJ 7, e6232 (2019).","journal-title":"PeerJ"},{"key":"1193_CR46","doi-asserted-by":"publisher","first-page":"2456","DOI":"10.1016\/S0140-6736(15)01131-9","volume":"386","author":"The Editors of the Lancet.","year":"2015","unstructured":"The Editors of the Lancet. Protocol review at The Lancet: 1997\u20132015. Lancet 386, 2456\u20132457 (2015).","journal-title":"Lancet"},{"key":"1193_CR47","doi-asserted-by":"publisher","first-page":"357","DOI":"10.1016\/j.cortex.2018.12.015","volume":"113","author":"L Maizey","year":"2019","unstructured":"Maizey, L. & Tzavella, L. Barriers and solutions for early career researchers in tackling the reproducibility crisis in cognitive neuroscience. Cortex 113, 357\u2013359 (2019).","journal-title":"Cortex"},{"key":"1193_CR48","doi-asserted-by":"publisher","first-page":"e3000246","DOI":"10.1371\/journal.pbio.3000246","volume":"17","author":"C Allen","year":"2019","unstructured":"Allen, C. & Mehler, D. M. A. Open science challenges, benefits and tips in early career and beyond. PLoS Biol. 17, e3000246 (2019).","journal-title":"PLoS Biol."},{"key":"1193_CR49","first-page":"1","volume":"4","author":"AM Scheel","year":"2021","unstructured":"Scheel, A. M., Schijen, M. & Lakens, D. An excess of positive results: Comparing the standard Psychology literature with Registered Reports. Adv. Meth. Pract. Psychol. Sci. 4, 1\u201312 (2021).","journal-title":"Adv. Meth. Pract. Psychol. Sci."},{"key":"1193_CR50","doi-asserted-by":"publisher","first-page":"e26828","DOI":"10.1371\/journal.pone.0026828","volume":"6","author":"JM Wicherts","year":"2011","unstructured":"Wicherts, J. M., Bakker, M. & Molenaar, D. Willingness to share research data is related to the strength of the evidence and the quality of reporting of statistical results. PLoS ONE 6, e26828 (2011).","journal-title":"PLoS ONE"},{"key":"1193_CR51","doi-asserted-by":"publisher","unstructured":"Obels, P., Lakens, D., Coles, N. A., Gottfried, J. & Green, S. A. Analysis of open data and computational reproducibility in Registered Reports in psychology. Preprint at PsyArXiv https:\/\/doi.org\/10.31234\/osf.io\/fk8vh (2019).","DOI":"10.31234\/osf.io\/fk8vh"},{"key":"1193_CR52","doi-asserted-by":"publisher","first-page":"180448","DOI":"10.1098\/rsos.180448","volume":"5","author":"TE Hardwicke","year":"2018","unstructured":"Hardwicke, T. E. et al. Data availability, reusability, and analytic reproducibility: evaluating the impact of a mandatory open data policy at the journal Cognition. R. Soc. Open Sci. 5, 180448 (2018).","journal-title":"R. Soc. Open Sci."},{"key":"1193_CR53","doi-asserted-by":"publisher","first-page":"296","DOI":"10.1016\/j.jclinepi.2012.09.015","volume":"66","author":"A-S Jannot","year":"2013","unstructured":"Jannot, A.-S., Agoritsas, T., Gayet-Ageron, A. & Perneger, T. V. Citation bias favoring statistically significant studies was present in medical research. J. Clin. Epidemiol. 66, 296\u2013301 (2013).","journal-title":"J. Clin. Epidemiol."},{"key":"1193_CR54","doi-asserted-by":"publisher","DOI":"10.1186\/s13063-016-1595-7","volume":"17","author":"BS Misemer","year":"2016","unstructured":"Misemer, B. S., Platts-Mills, T. F. & Jones, C. W. Citation bias favoring positive clinical trials of thrombolytics for acute ischemic stroke: a cross-sectional analysis. Trials 17, 473 (2016).","journal-title":"Trials"},{"key":"1193_CR55","doi-asserted-by":"publisher","first-page":"291","DOI":"10.3389\/fnhum.2013.00291","volume":"7","author":"B Brembs","year":"2013","unstructured":"Brembs, B., Button, K. & Munaf\u00f2, M. Deep impact: unintended consequences of journal rank. Front. Hum. Neurosci. 7, 291 (2013).","journal-title":"Front. Hum. Neurosci."},{"key":"1193_CR56","doi-asserted-by":"publisher","first-page":"17028","DOI":"10.1073\/pnas.1212247109","volume":"109","author":"FC Fang","year":"2012","unstructured":"Fang, F. C., Steen, R. G. & Casadevall, A. Misconduct accounts for the majority of retracted scientific publications. Proc. Natl Acad. Sci. USA 109, 17028\u201317033 (2012).","journal-title":"Proc. Natl Acad. Sci. USA"},{"key":"1193_CR57","doi-asserted-by":"publisher","first-page":"2140","DOI":"10.1002\/asi.22731","volume":"63","author":"GA Lozano","year":"2012","unstructured":"Lozano, G. A., Larivi\u00e8re, V. & Gingras, Y. The weakening relationship between the impact factor and papers\u2019 citations in the digital age. J. Am. Soc. Inf. Sci. Technol. 63, 2140\u20132145 (2012).","journal-title":"J. Am. Soc. Inf. Sci. Technol."},{"key":"1193_CR58","doi-asserted-by":"publisher","first-page":"498","DOI":"10.1136\/bmj.314.7079.497","volume":"314","author":"PO Seglen","year":"1997","unstructured":"Seglen, P. O. Why the impact factor of journals should not be used for evaluating research. BMJ 314, 498\u2013502 (1997).","journal-title":"BMJ"},{"key":"1193_CR59","doi-asserted-by":"publisher","unstructured":"Hummer, L., Thorn, F. S., Nosek, B. A. & Errington, T. Evaluating Registered Reports: a naturalistic comparative study of article impact. Preprint at OSF https:\/\/doi.org\/10.31219\/osf.io\/5y8w7 (2017).","DOI":"10.31219\/osf.io\/5y8w7"},{"key":"1193_CR60","doi-asserted-by":"publisher","unstructured":"Soderberg, C. K. et al. Initial evidence of research quality of Registered Reports compared with the standard publishing model. Nat. Hum. Behav. https:\/\/doi.org\/10.1038\/s41562-021-01142-4 (2021).","DOI":"10.1038\/s41562-021-01142-4"},{"key":"1193_CR61","doi-asserted-by":"publisher","DOI":"10.1186\/s40359-016-0167-7","volume":"4","author":"KS Button","year":"2016","unstructured":"Button, K. S., Bal, L., Clark, A. & Shipley, T. Preventing the ends from justifying the means: withholding results to address publication bias in peer-review. BMC Psychol. 4, 59 (2016).","journal-title":"BMC Psychol."},{"key":"1193_CR62","unstructured":"Srivastava, S. A Pottery Barn rule for scientific journals. The Hardest Science (27 September 2012); https:\/\/thehardestscience.com\/2012\/09\/27\/a-pottery-barn-rule-for-scientific-journals\/"},{"key":"1193_CR63","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1177\/2167702616673363","volume":"5","author":"SO Lilienfeld","year":"2017","unstructured":"Lilienfeld, S. O. Clinical psychological science: then and now. Clin. Psychol. Sci. 5, 3\u201313 (2017).","journal-title":"Clin. Psychol. Sci."},{"key":"1193_CR64","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1016\/j.jrp.2017.03.002","volume":"68","author":"RE Lucas","year":"2017","unstructured":"Lucas, R. E. & Donnellan, M. B. Enhancing transparency and openness at the Journal of Research in Personality. J. Res. Personal. 68, 1\u20134 (2017).","journal-title":"J. Res. Personal."},{"key":"1193_CR65","unstructured":"Anonymous. Preregistered direct replications: a new article type in psychological science. APS Obs. https:\/\/www.psychologicalscience.org\/observer\/preregistered-direct-replications-a-new-article-type-in-psychological-science (2017)."},{"key":"1193_CR66","unstructured":"Replication Studies (Royal Society Open Science, 2021); https:\/\/royalsocietypublishing.org\/rsos\/replication-studies"},{"key":"1193_CR67","unstructured":"Reproducibility and Transparency Collection (The Royal Society, 2021); https:\/\/royalsocietypublishing.org\/topic\/special-collections\/rsos-reproducibility"},{"key":"1193_CR68","unstructured":"Murray, H. Transparency meets transparency. F1000 Blogs (12 October 2017); https:\/\/blog.f1000.com\/2017\/10\/12\/transparency-meets-transparency\/"},{"key":"1193_CR69","doi-asserted-by":"publisher","first-page":"a1001","DOI":"10.15626\/MP2017.1001","volume":"1","author":"R Carlsson","year":"2017","unstructured":"Carlsson, R. et al. Inaugural editorial of Meta-Psychology. Meta-Psychol. 1, a1001 (2017).","journal-title":"Meta-Psychol."},{"key":"1193_CR70","doi-asserted-by":"publisher","first-page":"568","DOI":"10.1016\/j.tins.2019.07.003","volume":"42","author":"A Kiyonaga","year":"2019","unstructured":"Kiyonaga, A. & Scimeca, J. M. Practical considerations for navigating Registered Reports. Trends Neurosci. 42, 568\u2013572 (2019).","journal-title":"Trends Neurosci."},{"key":"1193_CR71","doi-asserted-by":"publisher","unstructured":"Guest, O. & Martin, A. E. How computational modeling can force theory building in psychological science. Perspect. Psychol. Sci. https:\/\/doi.org\/10.1177\/1745691620970585 (2021).","DOI":"10.1177\/1745691620970585"},{"key":"1193_CR72","doi-asserted-by":"publisher","first-page":"128","DOI":"10.3758\/s13423-017-1230-y","volume":"25","author":"FD Sch\u00f6nbrodt","year":"2018","unstructured":"Sch\u00f6nbrodt, F. D. & Wagenmakers, E.-J. Bayes factor design analysis: planning for compelling evidence. Psychon. Bull. Rev. 25, 128\u2013142 (2018).","journal-title":"Psychon. Bull. Rev."},{"key":"1193_CR73","doi-asserted-by":"publisher","first-page":"1422","DOI":"10.1126\/science.aab2374","volume":"348","author":"BA Nosek","year":"2015","unstructured":"Nosek, B. A. et al. Promoting an open research culture. Science 348, 1422\u20131425 (2015).","journal-title":"Science"},{"key":"1193_CR74","unstructured":"For Authors (BMJ Open Science, 2021); https:\/\/openscience.bmj.com\/pages\/authors\/"},{"key":"1193_CR75","unstructured":"Exploratory Reports at IRSP: Guidelines for Authors (International Review of Social Psychology, 2021); http:\/\/www.rips-irsp.com\/about\/exploratory-reports\/"},{"key":"1193_CR76","doi-asserted-by":"publisher","first-page":"A1","DOI":"10.1016\/j.cortex.2017.07.014","volume":"96","author":"RD McIntosh","year":"2017","unstructured":"McIntosh, R. D. Exploratory reports: a new article type for Cortex. Cortex 96, A1\u2013A4 (2017).","journal-title":"Cortex"},{"key":"1193_CR77","doi-asserted-by":"publisher","first-page":"793","DOI":"10.1038\/s41562-018-0444-y","volume":"2","author":"TE Hardwicke","year":"2018","unstructured":"Hardwicke, T. E. & Ioannidis, J. P. A. Mapping the universe of Registered Reports. Nat. Hum. Behav. 2, 793\u2013796 (2018).","journal-title":"Nat. Hum. Behav."},{"key":"1193_CR78","doi-asserted-by":"publisher","first-page":"791","DOI":"10.1038\/s41562-018-0449-6","volume":"2","author":"CD Chambers","year":"2018","unstructured":"Chambers, C. D. & Mellor, D. T. Protocol transparency is vital for Registered Reports. Nat. Hum. Behav. 2, 791\u2013792 (2018).","journal-title":"Nat. Hum. Behav."},{"key":"1193_CR79","unstructured":"Center for Open Science: Template Reviewer and Author Guidelines (Open Science Framework, 2018); https:\/\/osf.io\/8mpji\/"},{"key":"1193_CR80","unstructured":"OSF Registries (OSF, 2021); https:\/\/osf.io\/registries\/discover?provider=OSF&type=Registered%20Report%20Protocol%20Preregistration"},{"key":"1193_CR81","doi-asserted-by":"publisher","unstructured":"Tiokhin, L., Morgan, T. & Yan, M. Competition for priority and the cultural evolution of research strategies. Preprint at MetaArXiv https:\/\/doi.org\/10.31222\/osf.io\/x4t7q (2020).","DOI":"10.31222\/osf.io\/x4t7q"},{"key":"1193_CR82","unstructured":"Chambers, C. Calling all scientists: rapid evaluation of COVID19-related Registered Reports at Royal Society Open Science. NeuroChambers (16 March 2020); http:\/\/neurochambers.blogspot.com\/2020\/03\/calling-all-scientists-rapid-evaluation.html"},{"key":"1193_CR83","doi-asserted-by":"publisher","first-page":"200705","DOI":"10.1098\/rsos.200705","volume":"7","author":"T Zhou","year":"2020","unstructured":"Zhou, T., Nguyen, T. T., Zhong, J. & Liu, J. A COVID-19 descriptive study of life after lockdown in Wuhan, China. R. Soc. Open Sci. 7, 200705 (2020).","journal-title":"R. Soc. Open Sci."},{"key":"1193_CR84","doi-asserted-by":"publisher","first-page":"200458","DOI":"10.1098\/rsos.200458","volume":"7","author":"N Weinstein","year":"2020","unstructured":"Weinstein, N. & Nguyen, T.-V. Motivation and preference in isolation: a test of their different influences on responses to self-isolation during the COVID-19 outbreak. R. Soc. Open Sci. 7, 200458 (2020).","journal-title":"R. Soc. Open Sci."},{"key":"1193_CR85","doi-asserted-by":"publisher","first-page":"200636","DOI":"10.1098\/rsos.200636","volume":"7","author":"KA Khan","year":"2020","unstructured":"Khan, K. A. & Cheung, P. Presence of mismatches between diagnostic PCR assays and coronavirus SARS-CoV-2 genome. R. Soc. Open Sci. 7, 200636 (2020).","journal-title":"R. Soc. Open Sci."},{"key":"1193_CR86","doi-asserted-by":"publisher","first-page":"200880","DOI":"10.1098\/rsos.200880","volume":"7","author":"M Riello","year":"2020","unstructured":"Riello, M., Purgato, M., Bove, C., MacTaggart, D. & Rusconi, E. Prevalence of post-traumatic symptomatology and anxiety among residential nursing and care home workers following the first COVID-19 outbreak in Northern Italy. R. Soc. Open Sci. 7, 200880 (2020).","journal-title":"R. Soc. Open Sci."},{"key":"1193_CR87","doi-asserted-by":"publisher","first-page":"200589","DOI":"10.1098\/rsos.200589","volume":"8","author":"A Lieberoth","year":"2021","unstructured":"Lieberoth, A. et al. Stress and worry in the 2020 coronavirus pandemic: relationships to trust and compliance with preventive measures across 48 countries in the COVIDiSTRESS global survey. R. Soc. Open Sci. 8, 200589 (2021).","journal-title":"R. Soc. Open Sci."},{"key":"1193_CR88","doi-asserted-by":"publisher","first-page":"200793","DOI":"10.1098\/rsos.200793","volume":"7","author":"F Yonemitsu","year":"2020","unstructured":"Yonemitsu, F. et al. Warning \u2018don\u2019t spread\u2019 versus \u2018don\u2019t be a spreader\u2019 to prevent the COVID-19 pandemic. R. Soc. Open Sci. 7, 200793 (2020).","journal-title":"R. Soc. Open Sci."},{"key":"1193_CR89","unstructured":"PLoS ONE Editors. PLoS ONE partners with the Children\u2019s Tumor Foundation to trial Registered Reports. EveryONE: The PLoS ONE blog (26 September 2017); https:\/\/blogs.plos.org\/everyone\/2017\/09\/26\/registered-reports-with-ctf\/"},{"key":"1193_CR90","doi-asserted-by":"publisher","first-page":"773","DOI":"10.1093\/ntr\/ntx081","volume":"19","author":"MR Munaf\u00f2","year":"2017","unstructured":"Munaf\u00f2, M. R. Improving the efficiency of grant and journal peer review: Registered Reports funding. Nicotine Tob. Res. 19, 773\u2013773 (2017).","journal-title":"Nicotine Tob. Res."},{"key":"1193_CR91","doi-asserted-by":"publisher","unstructured":"Lakens, D. & DeBruine, L. Improving transparency, falsifiability, and rigour by making hypothesis tests machine readable. Preprint at PsyArXiv https:\/\/doi.org\/10.31234\/osf.io\/5xcda (2020).","DOI":"10.31234\/osf.io\/5xcda"},{"key":"1193_CR92","unstructured":"Mellor, D. & DeHaven, A. Templates of OSF Registration Forms (OSF, 2016); https:\/\/osf.io\/zab38\/"},{"key":"1193_CR93","doi-asserted-by":"publisher","first-page":"1832","DOI":"10.3389\/fpsyg.2016.01832","volume":"7","author":"JM Wicherts","year":"2016","unstructured":"Wicherts, J. M. et al. Degrees of freedom in planning, running, analyzing, and reporting psychological studies: a checklist to avoid p-hacking. Front. Psychol. 7, 1832 (2016).","journal-title":"Front. Psychol."},{"key":"1193_CR94","doi-asserted-by":"publisher","first-page":"977","DOI":"10.1001\/jama.2009.1242","volume":"302","author":"S Mathieu","year":"2009","unstructured":"Mathieu, S., Boutron, I., Moher, D., Altman, D. G. & Ravaud, P. Comparison of registered and published primary outcomes in randomized controlled trials. JAMA 302, 977\u2013984 (2009).","journal-title":"JAMA"},{"key":"1193_CR95","doi-asserted-by":"publisher","DOI":"10.1186\/s13063-018-2825-y","volume":"19","author":"AD Gopal","year":"2018","unstructured":"Gopal, A. D. et al. Adherence to the International Committee of Medical Journal Editors\u2019 (ICMJE) prospective registration policy and implications for outcome integrity: a cross-sectional analysis of trials published in high-impact specialty society journals. Trials 19, 448 (2018).","journal-title":"Trials"},{"key":"1193_CR96","unstructured":"Goldacre, B. et al. Tracking switched outcomes in clinical trials. COMPare http:\/\/compare-trials.org (2016)."},{"key":"1193_CR97","doi-asserted-by":"publisher","first-page":"80","DOI":"10.12688\/f1000research.6312.1","volume":"4","author":"SV Ramagopalan","year":"2015","unstructured":"Ramagopalan, S. V. et al. Funding source and primary outcome changes in clinical trials registered on ClinicalTrials.gov are associated with the reporting of a statistically significant primary outcome: a cross-sectional study. F1000Research 4, 80 (2015).","journal-title":"F1000Research"},{"key":"1193_CR98","doi-asserted-by":"publisher","first-page":"k3218","DOI":"10.1136\/bmj.k3218","volume":"362","author":"B Goldacre","year":"2018","unstructured":"Goldacre, B. et al. Compliance with requirement to report results on the EU Clinical Trials Register: cohort study and web resource. BMJ 362, k3218 (2018).","journal-title":"BMJ"},{"key":"1193_CR99","doi-asserted-by":"publisher","first-page":"i637","DOI":"10.1136\/bmj.i637","volume":"352","author":"R Chen","year":"2016","unstructured":"Chen, R. et al. Publication and reporting of clinical trial results: cross sectional analysis across academic medical centers. BMJ 352, i637 (2016).","journal-title":"BMJ"},{"key":"1193_CR100","unstructured":"The BMC Medicine Team. BMC Medicine becomes the first medical journal to accept Registered Reports. Research in Progress Blog (24 August 2017); http:\/\/blogs.biomedcentral.com\/bmcblog\/2017\/08\/24\/bmc-medicine-becomes-the-first-medical-journal-to-accept-registered-reports\/"},{"key":"1193_CR101","unstructured":"Panel Criteria and Working Methods (Research Excellence Framework, 2019); https:\/\/www.ref.ac.uk\/media\/1084\/ref-2019_02-panel-criteria-and-working-methods.pdf"},{"key":"1193_CR102","doi-asserted-by":"publisher","first-page":"183","DOI":"10.1038\/d41586-019-03750-7","volume":"576","author":"M Munaf\u00f2","year":"2019","unstructured":"Munaf\u00f2, M. Raising research quality will require collective action. Nature 576, 183\u2013183 (2019).","journal-title":"Nature"},{"key":"1193_CR103","unstructured":"UCL Statement on Transparency in Research (University College London, 2019); https:\/\/www.ucl.ac.uk\/research\/sites\/research\/files\/ucl_statement_on_transparency_in_research_november_20191.pdf"},{"key":"1193_CR104","doi-asserted-by":"publisher","first-page":"239821281984416","DOI":"10.1177\/2398212819844167","volume":"3","author":"GA Rousselet","year":"2019","unstructured":"Rousselet, G. A., Hazell, G., Cooke, A. & Dalley, J. W. Promoting and supporting credibility in neuroscience. Brain Neurosci. Adv. 3, 2398212819844167 (2019).","journal-title":"Brain Neurosci. Adv."},{"key":"1193_CR105","unstructured":"British Psychological Society. We\u2019re offering Registered Reports across all eleven of our academic journals. BPS News (13 July 2018); https:\/\/www.bps.org.uk\/news-and-policy\/were-offering-registered-reports-across-all-eleven-our-academic-journals"},{"key":"1193_CR106","unstructured":"Stiftelsen Dam. Krav om forh\u00e5ndregistrering av studier finansiert av Stiftelsen Dam. Stiftelsen Dam (9 November 2018); https:\/\/dam.no\/krav-om-forhandsregistrering-av-studier-finansiert-av-stiftelsen-dam\/"},{"key":"1193_CR107","unstructured":"Accelerating Research on Consciousness (Templeton World Charity Foundation, 2021); https:\/\/www.templetonworldcharity.org\/our-priorities\/accelerating-research-consciousness"},{"key":"1193_CR108","doi-asserted-by":"publisher","first-page":"160935","DOI":"10.1098\/rsos.160935","volume":"4","author":"T Heycke","year":"2017","unstructured":"Heycke, T., Aust, F. & Stahl, C. Subliminal influence on preferences? A test of evaluative conditioning for brief visual conditioned stimuli using auditory unconditioned stimuli. R. Soc. Open Sci. 4, 160935 (2017).","journal-title":"R. Soc. Open Sci."},{"key":"1193_CR109","doi-asserted-by":"publisher","first-page":"6","DOI":"10.1111\/ejn.14251","volume":"49","author":"K Ait Ouares","year":"2019","unstructured":"Ait Ouares, K., Beurrier, C., Canepari, M., Laverne, G. & Kuczewski, N. Opto nongenetics inhibition of neuronal firing. Eur. J. Neurosci. 49, 6\u201326 (2019).","journal-title":"Eur. J. Neurosci."},{"key":"1193_CR110","doi-asserted-by":"publisher","first-page":"A3","DOI":"10.1016\/j.cortex.2014.12.019","volume":"66","author":"J Sassenhagen","year":"2015","unstructured":"Sassenhagen, J. & Bornkessel-Schlesewsky, I. The P600 as a correlate of ventral attention network reorientation. Cortex 66, A3\u2013A20 (2015).","journal-title":"Cortex"},{"key":"1193_CR111","unstructured":"Allinson, M. Royal Society Open Science launches Registered Reports. The Royal Society Blog (27 November 2015); https:\/\/web.archive.org\/web\/20160702062134\/https:\/\/blogs.royalsociety.org\/publishing\/registered-reports\/"},{"key":"1193_CR112","doi-asserted-by":"publisher","first-page":"e23383","DOI":"10.7554\/eLife.23383","volume":"6","author":"BA Nosek","year":"2017","unstructured":"Nosek, B. A. & Errington, T. M. Reproducibility in cancer biology: making sense of replications. eLife 6, e23383 (2017).","journal-title":"eLife"},{"key":"1193_CR113","doi-asserted-by":"publisher","first-page":"171172","DOI":"10.1098\/rsos.171172","volume":"4","author":"W Guo","year":"2017","unstructured":"Guo, W., Del Vecchio, M. & Pogrebna, G. Global network centrality of university rankings. R. Soc. Open Sci. 4, 171172 (2017).","journal-title":"R. Soc. Open Sci."},{"key":"1193_CR114","doi-asserted-by":"publisher","unstructured":"[No authors listed]. Promoting reproducibility with Registered Reports. Nat. Hum. Behav. https:\/\/doi.org\/10.1038\/s41562-016-0034 (2017).","DOI":"10.1038\/s41562-016-0034"},{"key":"1193_CR115","doi-asserted-by":"publisher","first-page":"187","DOI":"10.1038\/526187a","volume":"526","author":"R MacCoun","year":"2015","unstructured":"MacCoun, R. & Perlmutter, S. Blind analysis: hide results to seek the truth. Nature 526, 187\u2013189 (2015).","journal-title":"Nature"},{"key":"1193_CR116","doi-asserted-by":"publisher","first-page":"713","DOI":"10.3758\/s13414-017-1304-y","volume":"79","author":"G Dutilh","year":"2017","unstructured":"Dutilh, G. et al. A test of the diffusion model explanation for the worst performance rule using preregistration and blinding. Atten. Percept. Psychophys. 79, 713\u2013725 (2017).","journal-title":"Atten. Percept. Psychophys."},{"key":"1193_CR117","doi-asserted-by":"publisher","first-page":"781","DOI":"10.3389\/fpsyg.2014.00781","volume":"5","author":"Z Dienes","year":"2014","unstructured":"Dienes, Z. Using Bayes to get the most out of non-significant results. Front. Psychol. 5, 781 (2014).","journal-title":"Front. Psychol."},{"key":"1193_CR118","doi-asserted-by":"publisher","first-page":"355","DOI":"10.1177\/1948550617697177","volume":"8","author":"D Lakens","year":"2017","unstructured":"Lakens, D. Equivalence tests: a practical primer for t tests, correlations, and meta-analyses. Soc. Psychol. Personal. Sci. 8, 355\u2013362 (2017).","journal-title":"Soc. Psychol. Personal. Sci."},{"key":"1193_CR119","unstructured":"Preregistered Research Article Guidelines for Authors (PLoS Biology, 2020); https:\/\/plos-marketing.s3.amazonaws.com\/Marketing\/Biology+Preregistered+Articles+Guidelines+for+Authors.pdf"},{"key":"1193_CR120","unstructured":"Registered Reports: Author and Reviewer Guidelines (Nature Human Behaviour, 2021); https:\/\/media.nature.com\/original\/nature-cms\/uploads\/ckeditor\/attachments\/4127\/RegisteredReportsGuidelines_NatureHumanBehaviour.pdf"},{"key":"1193_CR121","unstructured":"Guidelines for Reviewers (Cortex, 2013); https:\/\/cdn.elsevier.com\/promis_misc\/PROMIS%20pub_idt_CORTEX%20Guidelines_RR_29_04_2013.pdf"},{"key":"1193_CR122","unstructured":"Royal Society Open Science. Registered Reports (The Royal Society, 2021); https:\/\/royalsocietypublishing.org\/rsos\/registered-reports#ReviewerGuideRegRep"},{"key":"1193_CR123","unstructured":"Registered Reports: Resources for Editors (Center for Open Science, 2021); https:\/\/cos.io\/rr\/"},{"key":"1193_CR124","doi-asserted-by":"publisher","first-page":"552","DOI":"10.1136\/jech.2006.059394","volume":"62","author":"M Petticrew","year":"2008","unstructured":"Petticrew, M. et al. Publication bias in qualitative research: what becomes of qualitative research presented at conferences? J. Epidemiol. Community Health 62, 552\u2013554 (2008).","journal-title":"J. Epidemiol. Community Health"},{"key":"1193_CR125","first-page":"785","volume":"36","author":"R Pi\u00f1eiro","year":"2016","unstructured":"Pi\u00f1eiro, R. & Rosenblatt, F. Pre-analysis plans for qualitative research. Rev. Cienc. Pol\u00edtica 36, 785\u2013796 (2016).","journal-title":"Rev. Cienc. Pol\u00edtica"},{"key":"1193_CR126","doi-asserted-by":"publisher","unstructured":"Kern, F. G. & Gleditsch, K. S. Exploring pre-registration and pre-analysis plans for qualitative inference. Preprint at ResearchGate https:\/\/doi.org\/10.13140\/RG.2.2.14428.69769 (2017).","DOI":"10.13140\/RG.2.2.14428.69769"},{"key":"1193_CR127","doi-asserted-by":"publisher","first-page":"229","DOI":"10.1080\/08989621.2019.1580147","volume":"26","author":"TL Haven","year":"2019","unstructured":"Haven, T. L. & Grootel, D. L. V. Preregistering qualitative research. Account. Res. 26, 229\u2013244 (2019).","journal-title":"Account. Res."},{"key":"1193_CR128","unstructured":"Hartman, A., Kern, F. & Mellor, D. Preregistration for Qualitative Research Template (OSF, 2018); https:\/\/osf.io\/j7ghv\/"},{"key":"1193_CR129","doi-asserted-by":"publisher","first-page":"38","DOI":"10.1177\/0741088318804534","volume":"36","author":"AR Mehlenbacher","year":"2019","unstructured":"Mehlenbacher, A. R. Registered Reports: genre evolution and the research article. Writ. Commun. 36, 38\u201367 (2019).","journal-title":"Writ. Commun."},{"key":"1193_CR130","doi-asserted-by":"publisher","unstructured":"DeHaven, A. C. et al. Registered Reports: views from editors, reviewers and authors. Preprint at MetaArXiv https:\/\/doi.org\/10.31222\/osf.io\/ndvek (2019).","DOI":"10.31222\/osf.io\/ndvek"}],"container-title":["Nature Human Behaviour"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41562-021-01193-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41562-021-01193-7","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41562-021-01193-7.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,9,4]],"date-time":"2026-09-04T10:02:47Z","timestamp":1788516167000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41562-021-01193-7"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,11,15]]},"references-count":130,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2022,1]]}},"alternative-id":["1193"],"URL":"https:\/\/doi.org\/10.1038\/s41562-021-01193-7","relation":{},"ISSN":["2397-3374"],"issn-type":[{"value":"2397-3374","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,11,15]]},"assertion":[{"value":"10 February 2020","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"5 August 2021","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"15 November 2021","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"C.D.C. is a member of the Advisory Board of\n Nature Human Behaviour\n , is chair of the RRs committee supported by the Center for Open Science, is a co-founder of\n Peer Community in Registered Reports\n , and currently serves as RR editor at\n BMJ Open Science\n ,\n Cortex\n ,\n European Journal of Neuroscience\n ,\n NeuroImage\n ,\n Neuroimage: Reports\n ,\n PLoS Biology\n and\n Royal Society Open Science\n .","order":1,"name":"Ethics","label":"Competing interests","group":{"name":"EthicsHeading","label":"Ethics"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1038_2Fs41562_024_02024_1 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T08:45:20Z","timestamp":1793263520789,"version":"4.5.0"},"reference-count":74,"publisher":"Springer Science and Business Media LLC","issue":"12","license":[{"start":{"date-parts":[[2024,10,28]],"date-time":"2024-10-28T00:00:00Z","timestamp":1730073600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2024,10,28]],"date-time":"2024-10-28T00:00:00Z","timestamp":1730073600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"name":"Accenture Technology Convergence Fellowship"},{"name":"Toyota Research Institute; MIT Quest for Intelligence; National Research Foundation, Prime Minister's Office, Singapore","id":[{"id":"https:\/\/ror.org\/0425b4h79","id-type":"ROR","asserted-by":"crossref"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat Hum Behav"],"abstract":"<jats:title>Abstract<\/jats:title>\n <jats:p>\n Inspired by the increasing use of artificial intelligence (AI) to augment humans, researchers have studied human\u2013AI systems involving different tasks, systems and populations. Despite such a large body of work, we lack a broad conceptual understanding of when combinations of humans and AI are better than either alone. Here we addressed this question by conducting a preregistered systematic review and meta-analysis of 106 experimental studies reporting 370 effect sizes. We searched an interdisciplinary set of databases (the Association for Computing Machinery Digital Library, the Web of Science and the Association for Information Systems eLibrary) for studies published between 1 January 2020 and 30 June 2023. Each study was required to include an original human-participants experiment that evaluated the performance of humans alone, AI alone and human\u2013AI combinations. First, we found that, on average, human\u2013AI combinations performed significantly worse than the best of humans or AI alone (Hedges\u2019\n <jats:italic>g<\/jats:italic>\n \u2009=\u2009\u22120.23; 95% confidence interval, \u22120.39 to \u22120.07). Second, we found performance losses in tasks that involved making decisions and significantly greater gains in tasks that involved creating content. Finally, when humans outperformed AI alone, we found performance gains in the combination, but when AI outperformed humans alone, we found losses. Limitations of the evidence assessed here include possible publication bias and variations in the study designs analysed. Overall, these findings highlight the heterogeneity of the effects of human\u2013AI collaboration and point to promising avenues for improving human\u2013AI systems.\n <\/jats:p>","DOI":"10.1038\/s41562-024-02024-1","type":"journal-article","created":{"date-parts":[[2024,10,28]],"date-time":"2024-10-28T07:04:59Z","timestamp":1730099099000},"page":"2293-2303","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":650,"title":["When combinations of humans and AI are useful: A systematic review and meta-analysis"],"prefix":"10.1038","volume":"8","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6254-9718","authenticated-orcid":false,"given":"Michelle","family":"Vaccaro","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8467-9123","authenticated-orcid":false,"given":"Abdullah","family":"Almaatouq","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7005-1482","authenticated-orcid":false,"given":"Thomas","family":"Malone","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,10,28]]},"reference":[{"key":"2024_CR1","doi-asserted-by":"crossref","unstructured":"Bohr, A. & Memarzadeh, K. in Artificial Intelligence in Healthcare (eds Bohr, A. & Memarzadeh, K.) 25\u201360 (Elsevier, 2020).","DOI":"10.1016\/B978-0-12-818438-7.00002-2"},{"key":"2024_CR2","doi-asserted-by":"publisher","first-page":"1579","DOI":"10.3390\/electronics11101579","volume":"11","author":"L Nicolescu","year":"2022","unstructured":"Nicolescu, L. & Tudorache, M. T. Human\u2013computer interaction in customer service: the experience with AI chatbots\u2014a systematic literature review. Electronics 11, 1579 (2022).","journal-title":"Electronics"},{"key":"2024_CR3","doi-asserted-by":"publisher","first-page":"390","DOI":"10.1038\/s41586-019-1274-4","volume":"570","author":"B Koepnick","year":"2019","unstructured":"Koepnick, B. et al. De novo protein design by citizen scientists. Nature 570, 390\u2013394 (2019).","journal-title":"Nature"},{"key":"2024_CR4","doi-asserted-by":"crossref","unstructured":"Bansal, G. et al. Does the whole exceed its parts? The effect of AI explanations on complementary team performance. In Proc. 2021 CHI Conference on Human Factors in Computing Systems 81 (Association for Computing Machinery, 2021).","DOI":"10.1145\/3411764.3445717"},{"key":"2024_CR5","doi-asserted-by":"publisher","unstructured":"Bu\u00e7inca, Z., Lin, P., Gajos, K. Z. & Glassman, E. L. Proxy tasks and subjective measures can be misleading in evaluating explainable AI systems. In Proc. 25th International Conference on Intelligent User Interfaces 454\u2013464 (Association for Computing Machinery, 2020); https:\/\/doi.org\/10.1145\/3377325.3377498","DOI":"10.1145\/3377325.3377498"},{"key":"2024_CR6","doi-asserted-by":"publisher","unstructured":"Lai, V., Liu, H. & Tan, C. \u201cWhy is \u2018Chicago\u2019 deceptive?\u201d Towards building model-driven tutorials for humans. In Proc. 2020 CHI Conference on Human Factors in Computing Systems 1\u201313 (Association for Computing Machinery, 2020); https:\/\/doi.org\/10.1145\/3313831.3376873","DOI":"10.1145\/3313831.3376873"},{"key":"2024_CR7","doi-asserted-by":"publisher","unstructured":"Zhang, Y., Liao, Q. V. & Bellamy, R. K. E. Effect of confidence and explanation on accuracy and trust calibration in AI-assisted decision making. In Proc. 2020 Conference on Fairness, Accountability, and Transparency 295\u2013305 (Association for Computing Machinery, 2020); https:\/\/doi.org\/10.1145\/3351095.3372852","DOI":"10.1145\/3351095.3372852"},{"key":"2024_CR8","doi-asserted-by":"publisher","unstructured":"Bansal, G. et al. Updates in human\u2013AI teams: understanding and addressing the performance\/compatibility tradeoff. In Proc. 33rd AAAI Conference on Artificial Intelligence and 31st Innovative Applications of Artificial Intelligence Conference and 9th AAAI Symposium on Educational Advances in Artificial Intelligence 2429\u20132437 (AAAI Press, 2019); https:\/\/doi.org\/10.1609\/aaai.v33i01.33012429","DOI":"10.1609\/aaai.v33i01.33012429"},{"key":"2024_CR9","doi-asserted-by":"publisher","first-page":"104","DOI":"10.1145\/3359338","volume":"62","author":"M Vaccaro","year":"2019","unstructured":"Vaccaro, M. & Waldo, J. The effects of mixing machine learning and human judgment. Commun. ACM 62, 104\u2013110 (2019).","journal-title":"Commun. ACM"},{"key":"2024_CR10","doi-asserted-by":"crossref","unstructured":"Larson, J. R. Jr In Search of Synergy in Small Group Performance (Psychology Press, 2013).","DOI":"10.4324\/9780203848784"},{"key":"2024_CR11","doi-asserted-by":"publisher","first-page":"e2101062118","DOI":"10.1073\/pnas.2101062118","volume":"118","author":"A Almaatouq","year":"2021","unstructured":"Almaatouq, A., Alsobay, M., Yin, M. & Watts, D. J. Task complexity moderates group synergy. Proc. Natl Acad. Sci. USA 118, e2101062118 (2021).","journal-title":"Proc. Natl Acad. Sci. USA"},{"key":"2024_CR12","doi-asserted-by":"publisher","first-page":"100197","DOI":"10.1016\/j.patter.2020.100197","volume":"2","author":"Z-H Bo","year":"2021","unstructured":"Bo, Z.-H. et al. Toward human intervention-free clinical diagnosis of intracranial aneurysm via deep neural network. Patterns 2, 100197 (2021).","journal-title":"Patterns"},{"key":"2024_CR13","doi-asserted-by":"publisher","first-page":"945","DOI":"10.1177\/0018720820983632","volume":"64","author":"MM Boskemper","year":"2022","unstructured":"Boskemper, M. M., Bartlett, M. L. & McCarley, J. S. Measuring the efficiency of automation-aided performance in a simulated baggage screening task. Hum. Factors 64, 945\u2013961 (2022).","journal-title":"Hum. Factors"},{"key":"2024_CR14","first-page":"5286","volume":"36","author":"E Bondi","year":"2022","unstructured":"Bondi, E. et al. Role of human\u2013AI interaction in selective prediction. Proc. AAAI Conf. Artif. Intell. 36, 5286\u20135294 (2022).","journal-title":"Proc. AAAI Conf. Artif. Intell."},{"key":"2024_CR15","doi-asserted-by":"crossref","unstructured":"Schemmer, M., Hemmer, P., Nitsche, M., K\u00fchl, N. & V\u00f6ssing, M. A meta-analysis of the utility of explainable artificial intelligence in human\u2013AI decision-making. In Proc. 2022 AAAI\/ACM Conference on AI, Ethics, and Society 617\u2013626 (Association for Computing Machinery, 2022).","DOI":"10.1145\/3514094.3534128"},{"key":"2024_CR16","first-page":"114","volume":"96","author":"HJ Wilson","year":"2018","unstructured":"Wilson, H. J. & Daugherty, P. R. Collaborative intelligence: humans and AI are joining forces. Harv. Bus. Rev. 96, 114\u2013123 (2018).","journal-title":"Harv. Bus. Rev."},{"key":"2024_CR17","first-page":"11405","volume":"35","author":"G Bansal","year":"2021","unstructured":"Bansal, G., Nushi, B., Kamar, E., Horvitz, E. & Weld, D. S. Is the most accurate AI the best teammate? Optimizing AI for teamwork. Proc. AAAI Conf. Artif. Intell. 35, 11405\u201311414 (2021).","journal-title":"Proc. AAAI Conf. Artif. Intell."},{"key":"2024_CR18","doi-asserted-by":"crossref","unstructured":"Wilder, B., Horvitz, E. & Kamar, E. Learning to complement humans. In Proc. of the 29th International Joint Conference on Artificial Intelligence 1526\u20131533 (Association for Computing Machinery, 2020).","DOI":"10.24963\/ijcai.2020\/212"},{"key":"2024_CR19","first-page":"127","volume":"11","author":"C Rastogi","year":"2023","unstructured":"Rastogi, C., Leqi, L., Holstein, K. & Heidari, H. A taxonomy of human and ML strengths in decision-making to investigate human\u2013ML complementarity. Proc. AAAI Conf. Hum. Comput. Crowdsourc. 11, 127\u2013139 (2023).","journal-title":"Proc. AAAI Conf. Hum. Comput. Crowdsourc."},{"key":"2024_CR20","unstructured":"Mozannar, H. et al. Effective human\u2013AI teams via learned natural language rules and onboarding. Adv. Neural Inf. Process. Syst. 36 (2023)."},{"key":"2024_CR21","doi-asserted-by":"crossref","unstructured":"Cohen, J. Statistical Power Analysis for the Behavioral Sciences (Academic Press, 2013).","DOI":"10.4324\/9780203771587"},{"key":"2024_CR22","doi-asserted-by":"publisher","first-page":"119775","DOI":"10.1016\/j.energy.2021.119775","volume":"221","author":"W Shin","year":"2021","unstructured":"Shin, W., Han, J. & Rhee, W. AI-assistance for predictive maintenance of renewable energy systems. Energy 221, 119775 (2021).","journal-title":"Energy"},{"key":"2024_CR23","doi-asserted-by":"publisher","unstructured":"Noti, G. & Chen, Y. Learning when to advise human decision makers. In Proc. of the 33rd International Joint Conference on Artificial Intelligence 3038\u20133048 (Association for Computing Machinery, 2022); https:\/\/doi.org\/10.24963\/ijcai.2023\/339","DOI":"10.24963\/ijcai.2023\/339"},{"key":"2024_CR24","doi-asserted-by":"publisher","first-page":"370","DOI":"10.1145\/3610219","volume":"7","author":"V Chen","year":"2023","unstructured":"Chen, V., Liao, Q. V., Wortman Vaughan, J. & Bansal, G. Understanding the role of human intuition on reliance in human\u2013AI decision-making with explanations. Proc. ACM Hum. Comput. Interact. 7, 370 (2023).","journal-title":"Proc. ACM Hum. Comput. Interact."},{"key":"2024_CR25","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-022-18751-2","volume":"12","author":"C Reverberi","year":"2022","unstructured":"Reverberi, C. et al. Experimental evidence of effective human\u2013AI collaboration in medical decision-making. Sci. Rep. 12, 14952 (2022).","journal-title":"Sci. Rep."},{"key":"2024_CR26","doi-asserted-by":"publisher","first-page":"408","DOI":"10.1145\/3479552","volume":"5","author":"H Liu","year":"2021","unstructured":"Liu, H., Lai, V. & Tan, C. Understanding the effect of out-of-distribution examples and interactive explanations on human\u2013AI decision making. Proc. ACM Hum. Comput. Interact. 5, 408 (2021).","journal-title":"Proc. ACM Hum. Comput. Interact."},{"key":"2024_CR27","doi-asserted-by":"publisher","unstructured":"Lai, V., Chen, C., Smith-Renner, A., Liao, Q. V. & Tan, C. Towards a science of human\u2013AI decision making: an overview of design space in empirical human-subject studies. In Proc. 2023 ACM Conference on Fairness, Accountability, and Transparency 1369\u20131385 (Association for Computing Machinery, 2023); https:\/\/doi.org\/10.1145\/3593013.3594087","DOI":"10.1145\/3593013.3594087"},{"key":"2024_CR28","doi-asserted-by":"publisher","first-page":"543","DOI":"10.1111\/cgf.14329","volume":"40","author":"F Sperrle","year":"2021","unstructured":"Sperrle, F. et al. A survey of human-centered evaluations in human-centered machine learning. Comput. Graph. Forum 40, 543\u2013568 (2021).","journal-title":"Comput. Graph. Forum"},{"key":"2024_CR29","doi-asserted-by":"publisher","first-page":"2121","DOI":"10.3390\/ijerph18042121","volume":"18","author":"M Maadi","year":"2021","unstructured":"Maadi, M., Akbarzadeh Khorshidi, H. & Aickelin, U. A review on human\u2013AI interaction in machine learning and insights for medical applications. Int. J. Environ. Res. Public Health 18, 2121 (2021).","journal-title":"Int. J. Environ. Res. Public Health"},{"key":"2024_CR30","doi-asserted-by":"publisher","first-page":"991","DOI":"10.1006\/ijhc.1999.0252","volume":"51","author":"LJ Skitka","year":"1999","unstructured":"Skitka, L. J., Mosier, K. L. & Burdick, M. Does automation bias decision-making? Int. J. Hum. Comput. Stud. 51, 991\u20131006 (1999).","journal-title":"Int. J. Hum. Comput. Stud."},{"key":"2024_CR31","doi-asserted-by":"publisher","first-page":"188","DOI":"10.1145\/3449287","volume":"5","author":"Z Bu\u00e7inca","year":"2021","unstructured":"Bu\u00e7inca, Z., Malaya, M. B. & Gajos, K. Z. To trust or to think: cognitive forcing functions can reduce overreliance on AI in AI-assisted decision-making. Proc. ACM Hum. Comput. Interact. 5, 188 (2021).","journal-title":"Proc. ACM Hum. Comput. Interact."},{"key":"2024_CR32","doi-asserted-by":"publisher","first-page":"129","DOI":"10.1145\/3579605","volume":"7","author":"H Vasconcelos","year":"2023","unstructured":"Vasconcelos, H. et al. Explanations can reduce overreliance on AI systems during decision-making. Proc. ACM Hum. Comput. Interact. 7, 129 (2023).","journal-title":"Proc. ACM Hum. Comput. Interact."},{"key":"2024_CR33","doi-asserted-by":"publisher","unstructured":"Donahue, K., Chouldechova, A. & Kenthapadi, K. Human\u2013algorithm collaboration: achieving complementarity and avoiding unfairness. In Proc. 2022 ACM Conference on Fairness, Accountability, and Transparency 1639\u20131656 (Association for Computing Machinery, 2022); https:\/\/doi.org\/10.1145\/3531146.3533221","DOI":"10.1145\/3531146.3533221"},{"key":"2024_CR34","doi-asserted-by":"publisher","unstructured":"Cabrera, A. A., Perer, A. & Hong, J. I. Improving human\u2013AI collaboration with descriptions of AI behavior. Proc. ACM Hum. Comput. Interact. https:\/\/doi.org\/10.1145\/3579612 (2023).","DOI":"10.1145\/3579612"},{"key":"2024_CR35","doi-asserted-by":"publisher","unstructured":"Rastogi, C. et al. Deciding fast and slow: the role of cognitive biases in AI-assisted decision-making. Proc. ACM Hum. Comput. Interact. https:\/\/doi.org\/10.1145\/3512930 (2022).","DOI":"10.1145\/3512930"},{"key":"2024_CR36","doi-asserted-by":"publisher","first-page":"167","DOI":"10.1111\/j.1750-8606.2008.00060.x","volume":"2","author":"LV Hedges","year":"2008","unstructured":"Hedges, L. V. What are effect sizes and why do we need them? Child Dev. Perspect. 2, 167\u2013171 (2008).","journal-title":"Child Dev. Perspect."},{"key":"2024_CR37","doi-asserted-by":"publisher","DOI":"10.1038\/s41398-021-01224-x","volume":"11","author":"M Jacobs","year":"2021","unstructured":"Jacobs, M. et al. How machine-learning recommendations influence clinician treatment selections: the example of antidepressant selection. Transl. Psychiatry 11, 108 (2021).","journal-title":"Transl. Psychiatry"},{"key":"2024_CR38","doi-asserted-by":"publisher","first-page":"1229","DOI":"10.1038\/s41591-020-0942-0","volume":"26","author":"P Tschandl","year":"2020","unstructured":"Tschandl, P. et al. Human\u2013computer collaboration for skin cancer recognition. Nat. Med. 26, 1229\u20131234 (2020).","journal-title":"Nat. Med."},{"key":"2024_CR39","doi-asserted-by":"publisher","first-page":"713","DOI":"10.1287\/isre.2020.0980","volume":"32","author":"E Jussupow","year":"2021","unstructured":"Jussupow, E., Spohrer, K., Heinzl, A. & Gawlitza, J. Augmenting medical diagnosis decisions? An investigation into physicians\u2019 decision-making process with artificial intelligence. Inf. Syst. Res. 32, 713\u2013735 (2021).","journal-title":"Inf. Syst. Res."},{"key":"2024_CR40","doi-asserted-by":"publisher","unstructured":"He, Z., Song, Y., Zhou, S. & Cai, Z. Interaction of thoughts: towards mediating task assignment in human\u2013AI cooperation with a capability-aware shared mental model. In Proc. 2023 CHI Conference on Human Factors in Computing Systems 353 (Association for Computing Machinery, 2023); https:\/\/doi.org\/10.1145\/3544548.3580983","DOI":"10.1145\/3544548.3580983"},{"key":"2024_CR41","doi-asserted-by":"publisher","first-page":"14","DOI":"10.1186\/s41235-022-00364-y","volume":"7","author":"G Liang","year":"2022","unstructured":"Liang, G., Sloane, J. F., Donkin, C. & Newell, B. R. Adapting to the algorithm: how accuracy comparisons promote the use of a decision aid. Cogn. Res. Princ. Implic. 7, 14 (2022).","journal-title":"Cogn. Res. Princ. Implic."},{"key":"2024_CR42","doi-asserted-by":"publisher","unstructured":"Papenmeier, A., Kern, D., Englebienne, G. & Seifert, C. It\u2019s complicated: the relationship between user trust, model accuracy and explanations in AI. ACM Trans. Comput. Hum. Interact. https:\/\/doi.org\/10.1145\/3495013 (2022).","DOI":"10.1145\/3495013"},{"key":"2024_CR43","doi-asserted-by":"crossref","unstructured":"Borenstein, M., Hedges, L. V., Higgins, J. P. & Rothstein, H. R. Introduction to Meta-analysis (John Wiley & Sons, 2021).","DOI":"10.1002\/9781119558378"},{"key":"2024_CR44","doi-asserted-by":"publisher","first-page":"316","DOI":"10.1145\/3610107","volume":"7","author":"L Wilcox","year":"2023","unstructured":"Wilcox, L., Brewer, R. & Diaz, F. AI consent futures: a case study on voice data collection with clinicians. Proc. ACM Hum. Comput. Interact. 7, 316 (2023).","journal-title":"Proc. ACM Hum. Comput. Interact."},{"key":"2024_CR45","doi-asserted-by":"publisher","first-page":"116","DOI":"10.1145\/3579592","volume":"7","author":"E Karinshak","year":"2023","unstructured":"Karinshak, E., Liu, S. X., Park, J. S. & Hancock, J. T. Working with AI to persuade: examining a large language model\u2019s ability to generate pro-vaccination messages. Proc. ACM Hum. Comput. Interact. 7, 116 (2023).","journal-title":"Proc. ACM Hum. Comput. Interact."},{"key":"2024_CR46","doi-asserted-by":"publisher","first-page":"131","DOI":"10.1145\/3611025","volume":"7","author":"V Vimpari","year":"2023","unstructured":"Vimpari, V., Kultima, A., H\u00e4m\u00e4l\u00e4inen, P. & Guckelsberger, C. \u2018An adapt-or-die type of situation\u2019: perception, adoption, and use of text-to-image-generation AI by game industry professionals. Proc. ACM Hum. Comput. Interact. 7, 131\u2013164 (2023).","journal-title":"Proc. ACM Hum. Comput. Interact."},{"key":"2024_CR47","doi-asserted-by":"publisher","unstructured":"Liu, X. B. et al. Experiencing visual saptions: augmented communication with real-time visuals using large language models. In Adjunct Proc. 36th Annual ACM Symposium on User Interface Software and Technology 85 (Association for Computing Machinery, 2023); https:\/\/doi.org\/10.1145\/3586182.3615978","DOI":"10.1145\/3586182.3615978"},{"key":"2024_CR48","doi-asserted-by":"crossref","unstructured":"Jo, E., Epstein, D. A., Jung, H. & Kim, Y.-H. Understanding the benefits and challenges of deploying conversational AI leveraging large language models for public health intervention. In Proc. 2023 CHI Conference on Human Factors in Computing Systems 18 (Association for Computing Machinery, 2023).","DOI":"10.1145\/3544548.3581503"},{"key":"2024_CR49","doi-asserted-by":"crossref","unstructured":"Petridis, S. et al. Anglekindling: supporting journalistic angle ideation with large language models. In Proc. 2023 CHI Conference on Human Factors in Computing Systems 225 (Association for Computing Machinery, 2023).","DOI":"10.1145\/3544548.3580907"},{"key":"2024_CR50","doi-asserted-by":"crossref","unstructured":"Jakesch, M., Bhat, A., Buschek, D., Zalmanson, L. & Naaman, M. Co-writing with opinionated language models affects users\u2019 views. In Proc. 2023 CHI conference on human factors in computing systems 111 (Associaton for Computing Machinery, 2023).","DOI":"10.1145\/3544548.3581196"},{"key":"2024_CR51","doi-asserted-by":"crossref","unstructured":"Mirowski, P., Mathewson, K. W., Pittman, J. & Evans, R. Co-writing screenplays and theatre scripts with language models: evaluation by industry professionals. In Proc. 2023 CHI Conference on Human Factors in Computing Systems 355 (Association for Computing Machinery 2023).","DOI":"10.1145\/3544548.3581225"},{"key":"2024_CR52","doi-asserted-by":"publisher","first-page":"187","DOI":"10.1126\/science.adh2586","volume":"381","author":"S Noy","year":"2023","unstructured":"Noy, S. & Zhang, W. Experimental evidence on the productivity effects of generative artificial intelligence. Science 381, 187\u2013192 (2023).","journal-title":"Science"},{"key":"2024_CR53","unstructured":"Baier, P., DeLallo, D. & Sviokla, J. J. Your organization isn\u2019t designed to work with GenAI. Harvard Business Review (26 February 2024); https:\/\/hbr.org\/2024\/02\/your-organization-isnt-designed-to-work-with-genai"},{"key":"2024_CR54","unstructured":"Campero, A. et al. A test for evaluating performance in human\u2013computer systems. Preprint at https:\/\/arxiv.org\/abs\/2206.12390 (2022)."},{"key":"2024_CR55","unstructured":"Kitchenham, B. Procedures for performing systematic reviews. Technical report No. TR\/SE-0401 (Keele University, 2004); https:\/\/citeseerx.ist.psu.edu\/document?repid=rep1&type=pdf&doi=29890a936639862f45cb9a987dd599dce9759bf5"},{"key":"2024_CR56","doi-asserted-by":"publisher","first-page":"264","DOI":"10.7326\/0003-4819-151-4-200908180-00135","volume":"151","author":"D Moher","year":"2009","unstructured":"Moher, D., Liberati, A., Tetzlaff, J., Altman, D. G. & PRISMA Group Preferred reporting items for systematic reviews and meta-analyses: the PRISMA statement. Ann. Intern. Med. 151, 264\u2013269 (2009).","journal-title":"Ann. Intern. Med."},{"key":"2024_CR57","doi-asserted-by":"publisher","first-page":"e2110013119","DOI":"10.1073\/pnas.2110013119","volume":"119","author":"M Groh","year":"2022","unstructured":"Groh, M., Epstein, Z., Firestone, C. & Picard, R. Deepfake detection by human crowds, machines, and machine-informed crowds. Proc. Natl Acad. Sci. USA 119, e2110013119 (2022).","journal-title":"Proc. Natl Acad. Sci. USA"},{"key":"2024_CR58","doi-asserted-by":"publisher","first-page":"491","DOI":"10.1007\/s42113-022-00157-y","volume":"5","author":"H Tejeda","year":"2022","unstructured":"Tejeda, H., Kumar, A., Smyth, P. & Steyvers, M. AI-assisted decision-making: a cognitive modeling approach to infer latent reliance strategies. Comput. Brain Behav. 5, 491\u2013508 (2022).","journal-title":"Comput. Brain Behav."},{"key":"2024_CR59","unstructured":"Higgins, J. P. T., Li, T. & Deeks, J. J. Choosing effect measures and computing estimates of effect. In Cochrane Handbook for Systematic Reviews of Interventions version 6.5 (eds Higgins, J.P. T. et al.) Ch. 6 (Cochrane, 2024)."},{"key":"2024_CR60","unstructured":"Rohatgi, A. WebPlotDigitizer v.4.5 https:\/\/automeris.io\/WebPlotDigitizer (Automeris, 2020)."},{"key":"2024_CR61","doi-asserted-by":"publisher","first-page":"107","DOI":"10.3102\/10769986006002107","volume":"6","author":"LV Hedges","year":"1981","unstructured":"Hedges, L. V. Distribution theory for Glass\u2019s estimator of effect size and related estimators. J. Educ. Stat. 6, 107\u2013128 (1981).","journal-title":"J. Educ. Stat."},{"key":"2024_CR62","unstructured":"Hedges, L. V. & Olkin, I. Statistical Methods for Meta-analysis (Academic Press, 2014)."},{"key":"2024_CR63","doi-asserted-by":"publisher","first-page":"1274","DOI":"10.3758\/s13428-014-0527-2","volume":"47","author":"W Van den Noortgate","year":"2015","unstructured":"Van den Noortgate, W., L\u00f3pez-L\u00f3pez, J. A., Mar\u00edn-Mart\u00ednez, F. & S\u00e1nchez-Meca, J. Meta-analysis of multiple outcomes: a multilevel approach. Behav. Res. Methods 47, 1274\u20131294 (2015).","journal-title":"Behav. Res. Methods"},{"key":"2024_CR64","doi-asserted-by":"publisher","first-page":"211","DOI":"10.1037\/a0032968","volume":"19","author":"MW-L Cheung","year":"2014","unstructured":"Cheung, M. W.-L. Modeling dependent effect sizes with three-level meta-analyses: a structural equation modeling approach. Psychol. Methods 19, 211\u2013229 (2014).","journal-title":"Psychol. Methods"},{"key":"2024_CR65","doi-asserted-by":"publisher","first-page":"39","DOI":"10.1002\/jrsm.5","volume":"1","author":"LV Hedges","year":"2010","unstructured":"Hedges, L. V., Tipton, E. & Johnson, M. C. Robust variance estimation in meta-regression with dependent effect size estimates. Res. Synth. Methods 1, 39\u201365 (2010).","journal-title":"Res. Synth. Methods"},{"key":"2024_CR66","doi-asserted-by":"publisher","first-page":"1539","DOI":"10.1002\/sim.1186","volume":"21","author":"JP Higgins","year":"2002","unstructured":"Higgins, J. P. & Thompson, S. G. Quantifying heterogeneity in a meta-analysis. Stat. Med. 21, 1539\u20131558 (2002).","journal-title":"Stat. Med."},{"key":"2024_CR67","doi-asserted-by":"publisher","first-page":"112","DOI":"10.1002\/jrsm.11","volume":"1","author":"W Viechtbauer","year":"2010","unstructured":"Viechtbauer, W. & Cheung, M. W.-L. Outlier and influence diagnostics for meta-analysis. Res. Synth. Methods 1, 112\u2013125 (2010).","journal-title":"Res. Synth. Methods"},{"key":"2024_CR68","doi-asserted-by":"publisher","first-page":"1046","DOI":"10.1016\/S0895-4356(01)00377-8","volume":"54","author":"JA Sterne","year":"2001","unstructured":"Sterne, J. A. & Egger, M. Funnel plots for detecting bias in meta-analysis: guidelines on choice of axis. J. Clin. Epidemiol. 54, 1046\u20131055 (2001).","journal-title":"J. Clin. Epidemiol."},{"key":"2024_CR69","doi-asserted-by":"publisher","first-page":"991","DOI":"10.1016\/j.jclinepi.2007.11.010","volume":"61","author":"JL Peters","year":"2008","unstructured":"Peters, J. L., Sutton, A. J., Jones, D. R., Abrams, K. R. & Rushton, L. Contour-enhanced meta-analysis funnel plots help distinguish publication bias from other causes of asymmetry. J. Clin. Epidemiol. 61, 991\u2013996 (2008).","journal-title":"J. Clin. Epidemiol."},{"key":"2024_CR70","doi-asserted-by":"publisher","first-page":"629","DOI":"10.1136\/bmj.315.7109.629","volume":"315","author":"M Egger","year":"1997","unstructured":"Egger, M., Smith, G. D., Schneider, M. & Minder, C. Bias in meta-analysis detected by a simple, graphical test. BMJ 315, 629\u2013634 (1997).","journal-title":"BMJ"},{"key":"2024_CR71","doi-asserted-by":"publisher","first-page":"1088","DOI":"10.2307\/2533446","volume":"50","author":"CB Begg","year":"1994","unstructured":"Begg, C. B. & Mazumdar, M. Operating characteristics of a rank correlation test for publication bias. Biometrics 50, 1088\u20131101 (1994).","journal-title":"Biometrics"},{"key":"2024_CR72","doi-asserted-by":"crossref","unstructured":"Rothstein, H. R., Sutton, A. J. & Borenstein, M. in Publication Bias in Meta-analysis: Prevention, Assessment and Adjustments (eds Rothstein, H. R. et al.) 1\u20137 (John Wiley & Sons, 2005).","DOI":"10.1002\/0470870168.ch1"},{"key":"2024_CR73","doi-asserted-by":"publisher","first-page":"281","DOI":"10.1038\/nmeth.3812","volume":"13","author":"N Altman","year":"2016","unstructured":"Altman, N. & Krzywinski, M. Analyzing outliers: influential or nuisance? Nat. Methods 13, 281\u2013283 (2016).","journal-title":"Nat. Methods"},{"key":"2024_CR74","doi-asserted-by":"publisher","first-page":"1","DOI":"10.18637\/jss.v036.i03","volume":"36","author":"W Viechtbauer","year":"2010","unstructured":"Viechtbauer, W. Conducting meta-analyses in R with the metafor package. J. Stat. Softw. 36, 1\u201348 (2010).","journal-title":"J. Stat. Softw."}],"container-title":["Nature Human Behaviour"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41562-024-02024-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41562-024-02024-1","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41562-024-02024-1.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,12,19]],"date-time":"2024-12-19T11:04:12Z","timestamp":1734606252000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41562-024-02024-1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,10,28]]},"references-count":74,"journal-issue":{"issue":"12","published-online":{"date-parts":[[2024,12]]}},"alternative-id":["2024"],"URL":"https:\/\/doi.org\/10.1038\/s41562-024-02024-1","relation":{"has-preprint":[{"id-type":"doi","id":"10.21203\/rs.3.rs-2787476\/v1","asserted-by":"object"}]},"ISSN":["2397-3374"],"issn-type":[{"value":"2397-3374","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,10,28]]},"assertion":[{"value":"6 April 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"23 September 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"28 October 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1038_2Fs41586_026_10265_5 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T16:12:00Z","timestamp":1793290320514,"version":"4.5.0"},"reference-count":64,"publisher":"Springer Science and Business Media LLC","issue":"8107","license":[{"start":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T00:00:00Z","timestamp":1774396800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T00:00:00Z","timestamp":1774396800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nature"],"published-print":{"date-parts":[[2026,3,26]]},"abstract":"<jats:title>Abstract<\/jats:title>\n <jats:p>\n The automation of science is a long-standing ambition in artificial intelligence (AI) research\n <jats:sup>1,2<\/jats:sup>\n . Although the community has made substantial progress in automating individual components of the scientific process, a system that autonomously navigates the entire research life cycle\u2014from conception to publication\u2014has remained out of reach. Here we present a pipeline for automating the entire scientific process end to end. We present The AI Scientist, which creates research ideas, writes code, runs experiments, plots and analyses data, writes the entire scientific manuscript, and performs its own peer review. Its ideas, execution and presentation are of sufficient quality that the manuscript generated by this AI system passed the first round of peer review for a workshop of a\u00a0top-tier machine learning conference. The workshop had an acceptance rate of 70%. Our system leverages modern foundation models\n <jats:sup>3\u20135<\/jats:sup>\n within a complex agentic system. We evaluate The AI Scientist in two settings: a focused mode using human-provided code templates as an initial scaffold for conducting research on a specific topic and a template-free, open-ended mode that leverages agentic search for wider scientific exploration\n <jats:sup>6,7<\/jats:sup>\n . Both settings produce diverse ideas and automatically test, report on and evaluate them. This achievement demonstrates the growing capacity of AI for making scientific contributions and signifies a potential paradigm shift in how research is conducted. As with any impactful new technology, there could be important risks, including taxing overwhelmed review systems and adding noise to the scientific literature. However, if developed responsibly, such autonomous systems could greatly accelerate scientific discovery.\n <\/jats:p>","DOI":"10.1038\/s41586-026-10265-5","type":"journal-article","created":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T16:02:30Z","timestamp":1774454550000},"page":"914-919","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":136,"title":["Towards end-to-end automation of AI research"],"prefix":"10.1038","volume":"651","author":[{"given":"Chris","family":"Lu","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5564-838X","authenticated-orcid":false,"given":"Cong","family":"Lu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Robert Tjarko","family":"Lange","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yutaro","family":"Yamada","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shengran","family":"Hu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9688-2498","authenticated-orcid":false,"given":"Jakob","family":"Foerster","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8765-8574","authenticated-orcid":false,"given":"David","family":"Ha","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4890-5274","authenticated-orcid":false,"given":"Jeff","family":"Clune","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,3,25]]},"reference":[{"key":"10265_CR1","unstructured":"Lenat, D. B. Automated theory formation in mathematics. In Proc. 5th International Joint Conference on Artificial Intelligence 833\u2013842 (ed. Reddy, R.) (William Kaufmann, 1977)."},{"key":"10265_CR2","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1016\/0004-3702(78)90010-3","volume":"11","author":"BG Buchanan","year":"1978","unstructured":"Buchanan, B. G. & Feigenbaum, E. A. Dendral and meta-dendral: their applications dimension. Artif. Intell. 11, 5\u201324 (1978).","journal-title":"Artif. Intell."},{"key":"10265_CR3","doi-asserted-by":"publisher","unstructured":"OpenAI. GPT-4 technical report. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2303.08774 (2023).","DOI":"10.48550\/arXiv.2303.08774"},{"key":"10265_CR4","unstructured":"The Claude 3 Model Family: Opus, Sonnet, Haiku https:\/\/api.semanticscholar.org\/CorpusID:268232499 (Anthropic, 2024)."},{"key":"10265_CR5","doi-asserted-by":"publisher","unstructured":"Grattafiori, A. et al. The Llama 3 herd of models. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2407.21783 (2024).","DOI":"10.48550\/arXiv.2407.21783"},{"key":"10265_CR6","doi-asserted-by":"publisher","unstructured":"Jiang, Z. et al. AIDE: AI-driven exploration in the space of code. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2502.13138 (2025).","DOI":"10.48550\/arXiv.2502.13138"},{"key":"10265_CR7","unstructured":"Chan, J. S. et al. MLE-bench: evaluating machine learning agents on machine learning engineering. In Proc. 13th International Conference on Learning Representation (ed. Yue, Y. et al.) (ICLR, 2025)."},{"key":"10265_CR8","doi-asserted-by":"publisher","first-page":"43","DOI":"10.1126\/science.1172781","volume":"324","author":"D Waltz","year":"2009","unstructured":"Waltz, D. & Buchanan, B. G. Automating science. Science 324, 43\u201344 (2009).","journal-title":"Science"},{"key":"10265_CR9","doi-asserted-by":"crossref","unstructured":"Langley, P. Integrated systems for computational scientific discovery. In Proc. AAAI Conference on Artifical Intelligence Vol. 38, 22598\u201322606 (eds Wooldridge, M. et al.) (AAAI, 2024).","DOI":"10.1609\/aaai.v38i20.30269"},{"key":"10265_CR10","doi-asserted-by":"crossref","unstructured":"Langley, P., Simon, H. A., Bradshaw, G. L. & Zytkow, J. M. Scientific Discovery: Computational Explorations of the Creative Process (MIT, 1987).","DOI":"10.7551\/mitpress\/6090.001.0001"},{"key":"10265_CR11","doi-asserted-by":"publisher","first-page":"269","DOI":"10.1016\/0004-3702(84)90016-X","volume":"23","author":"DB Lenat","year":"1984","unstructured":"Lenat, D. B. & Brown, J. S. Why AM and EURISKO appear to work. Artif. Intell. 23, 269\u2013294 (1984).","journal-title":"Artif. Intell."},{"key":"10265_CR12","doi-asserted-by":"publisher","DOI":"10.1038\/s41524-022-00765-z","volume":"8","author":"EO Pyzer-Knapp","year":"2022","unstructured":"Pyzer-Knapp, E. O. et al. Accelerating materials discovery using artificial intelligence, high performance computing and robotics. npj Comput. Mater. 8, 84 (2022).","journal-title":"npj Comput. Mater."},{"key":"10265_CR13","doi-asserted-by":"publisher","first-page":"80","DOI":"10.1038\/s41586-023-06735-9","volume":"624","author":"A Merchant","year":"2023","unstructured":"Merchant, A. et al. Scaling deep learning for materials discovery. Nature 624, 80\u201385 (2023).","journal-title":"Nature"},{"key":"10265_CR14","doi-asserted-by":"publisher","first-page":"86","DOI":"10.1038\/s41586-023-06734-w","volume":"624","author":"NJ Szymanski","year":"2023","unstructured":"Szymanski, N. J. et al. An autonomous laboratory for the accelerated synthesis of novel materials. Nature 624, 86\u201391 (2023).","journal-title":"Nature"},{"key":"10265_CR15","doi-asserted-by":"publisher","first-page":"583","DOI":"10.1038\/s41586-021-03819-2","volume":"596","author":"J Jumper","year":"2021","unstructured":"Jumper, J. et al. Highly accurate protein structure prediction with AlphaFold. Nature 596, 583\u2013589 (2021).","journal-title":"Nature"},{"key":"10265_CR16","doi-asserted-by":"publisher","first-page":"850","DOI":"10.1126\/science.ads0018","volume":"387","author":"T Hayes","year":"2025","unstructured":"Hayes, T. et al. Simulating 500 million years of evolution with a language model. Science 387, 850\u2013858 (2025).","journal-title":"Science"},{"key":"10265_CR17","doi-asserted-by":"publisher","first-page":"AIoa2400555","DOI":"10.1056\/AIoa2400555","volume":"2","author":"T Ifargan","year":"2025","unstructured":"Ifargan, T., Hafner, L., Kern, M., Alcalay, O. & Kishony, R. Autonomous LLM-driven research\u2014from data to human-verifiable research papers. NEJM AI 2, AIoa2400555 (2025).","journal-title":"NEJM AI"},{"key":"10265_CR18","doi-asserted-by":"publisher","first-page":"367","DOI":"10.1023\/A:1022866732136","volume":"1","author":"BC Falkenhainer","year":"1986","unstructured":"Falkenhainer, B. C. & Michalski, R. S. Integrating quantitative and qualitative discovery: the ABACUS system. Mach. Learn. 1, 367\u2013401 (1986).","journal-title":"Mach. Learn."},{"key":"10265_CR19","doi-asserted-by":"crossref","unstructured":"Girotra, K., Meincke, L., Terwiesch, C. & Ulrich, K. T. Ideas Are Dimes a Dozen: Large Language Models for Idea Generation in Innovation. Working paper (Cornell Univ., 2023).","DOI":"10.2139\/ssrn.4526071"},{"key":"10265_CR20","doi-asserted-by":"crossref","unstructured":"Lehman, J. et al. in Handbook of Evolutionary Machine Learning (eds Banzhaf, W. et al.) 331\u2013366 (Springer Nature, 2023).","DOI":"10.1007\/978-981-99-3814-8_11"},{"key":"10265_CR21","doi-asserted-by":"publisher","unstructured":"Lu, C., Hu, S. & Clune, J. Automated capability discovery via model self-exploration. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2502.07577 (2025).","DOI":"10.48550\/arXiv.2502.07577"},{"key":"10265_CR22","unstructured":"Faldor, M., Zhang, J., Cully, A. & Clune, J. OMNI-EPIC: open-endedness via models of human notions of interestingness with environments programmed in code. In Proc. 13th International Conference on Learning Representations (eds Yue, Y. et al.) (ICLR, 2025)."},{"key":"10265_CR23","unstructured":"Hu, S., Lu, C. & Clune, J. Automated design of agentic systems. In Proc. 13th International Conference on Learning Representations (eds Yue, Y. et al.) (ICLR, 2025)."},{"key":"10265_CR24","doi-asserted-by":"crossref","unstructured":"Baek, J., Jauhar, S. K., Cucerzan, S. & Hwang, S. J. ResearchAgent: Iterative research idea generation over scientific literature with large language models. In Proc. 2025 Conference of the Nations of the Americas Chapter of the Association for Computational Linguistics: Human Language Technologies (eds Chiruzzo, L. et al.) 6709\u20136738 (ACL, 2025).","DOI":"10.18653\/v1\/2025.naacl-long.342"},{"key":"10265_CR25","doi-asserted-by":"crossref","unstructured":"Wang, Y. et al. Autosurvey: large language models can automatically write surveys. In Proc. Advances in Neural Information Processing Systems Vol. 37 (eds Globerson, A. et al.) 115119\u2013115145 (Curran Associates, 2024).","DOI":"10.52202\/079017-3655"},{"key":"10265_CR26","unstructured":"Huang, Q., Vora, J., Liang, P. & Leskovec, J. MLAgentBench: evaluating language agents on machine learning experimentation. In Proc. 41st International Conference on Machine Learning Vol. 235 (eds Salakhutdinov, R. et al.) 20271\u201320309 (PMLR, 2024)."},{"key":"10265_CR27","doi-asserted-by":"crossref","unstructured":"Lu, C. et al. Discovering preference optimization algorithms with and for large language models. In Proc. Advances in Neural Information Processing Systems Vol. 37 (eds Globerson, A. et al.) 86528\u201386573 (Curran Associates, 2024).","DOI":"10.52202\/079017-2748"},{"key":"10265_CR28","unstructured":"Ma, Y. J. et al. Eureka: human-level reward design via coding large language models. In Proc. 12th International Conference on Learning Representations (eds Kim, B. et al.) (ICLR, 2024)."},{"key":"10265_CR29","doi-asserted-by":"crossref","unstructured":"Zhang, J., Hu, S., Lu, C., Lange, R. & Clune, J. Darwin G\u00f6del machine: open-ended evolution of self-improving agents. In Proc. 14th International. Conference Learning Representation (ed. Vondrick, C. et al.) (ICLR, 2026).","DOI":"10.70777\/si.v2i3.15063"},{"key":"10265_CR30","doi-asserted-by":"publisher","unstructured":"Mouret, J.-B. & Clune, J. Illuminating search spaces by mapping elites. Preprint at https:\/\/doi.org\/10.48550\/arXiv.1504.04909 (2015).","DOI":"10.48550\/arXiv.1504.04909"},{"key":"10265_CR31","doi-asserted-by":"publisher","first-page":"145","DOI":"10.5195\/jmla.2018.280","volume":"106","author":"S Fricke","year":"2018","unstructured":"Fricke, S. Semantic Scholar. J. Med. Libr. Assoc. 106, 145 (2018).","journal-title":"J. Med. Libr. Assoc."},{"key":"10265_CR32","doi-asserted-by":"crossref","unstructured":"Schick, T. et al. Toolformer: language models can teach themselves to use tools. In Proc. Advances in Neural Information Processing Systems Vol. 36 (eds Oh, A. et al.) 68539\u201368551 (Curran Associates, 2023).","DOI":"10.52202\/075280-2997"},{"key":"10265_CR33","unstructured":"Gonz\u00e1lez-M\u00e1rquez, R. & Kobak, D. Learning representations of learning representations. In Proc. Data-centric Machine Learning Research (DMLR) Workshop at ICLR (ICLR, 2024)."},{"key":"10265_CR34","unstructured":"Beygelzimer, A., Dauphin, Y., Liang, P. & Vaughan, J. W. The NeurIPS 2021 consistency experiment. NeurIPS Blog https:\/\/blog.neurips.cc\/2021\/12\/08\/the-neurips-2021-consistency-experiment (2021)."},{"key":"10265_CR35","doi-asserted-by":"crossref","unstructured":"Efron, B. & Tibshirani, R. J. An Introduction to the Bootstrap (Chapman & Hall CRC, 1993).","DOI":"10.1007\/978-1-4899-4541-9"},{"key":"10265_CR36","unstructured":"Lehmann, E. L. Testing Statistical Hypotheses 1st edn (Wiley, 1959)."},{"key":"10265_CR37","doi-asserted-by":"publisher","unstructured":"Maslej, N. et al. Artificial intelligence index report 2024. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2405.19522 (2024).","DOI":"10.48550\/arXiv.2405.19522"},{"key":"10265_CR38","unstructured":"I Can\u2019t Believe it\u2019s Not Better: Challenges in Applied Deep Learning. ICLR Workshop at ICLR 2025 http:\/\/openreview.net\/group?id=ICLR.cc\/2025\/Workshop\/ICBINB (OpenReview, 2025)."},{"key":"10265_CR39","unstructured":"13th Annual International Conference on Learning Representations (ICLR) 2025 Fact Sheet https:\/\/media.iclr.cc\/Conferences\/ICLR2025\/ICLR2025_Fact_Sheet.pdf (ICLR, 2025)."},{"key":"10265_CR40","unstructured":"Kwa, T. et al. Measuring AI ability to complete long tasks. METR Blog https:\/\/metr.org\/blog\/2025-03-19-measuring-ai-ability-to-complete-long-tasks\/ (2025)."},{"key":"10265_CR41","doi-asserted-by":"crossref","unstructured":"Nguyen, A., Yosinski, J. & Clune, J. Deep neural networks are easily fooled: high confidence predictions for unrecognizable images. In Proc. IEEE Conference on Computer Vision and Pattern Recognition (eds Grauman, K. et al.) 427\u2013436 (IEEE, 2015).","DOI":"10.1109\/CVPR.2015.7298640"},{"key":"10265_CR42","unstructured":"Szegedy, C. et al. Intriguing properties of neural networks. In Proc. 2nd International Conference on Learning Representations (eds Bengio, Y. & LeCun, Y.) (ICLR, 2014)."},{"key":"10265_CR43","doi-asserted-by":"crossref","unstructured":"Maynez, J., Narayan, S., Bohnet, B. & McDonald, R. On faithfulness and factuality in abstractive summarization. In Proc. 58th Annual Meeting of the Association for Computational Linguistics (eds Jurafsky, D. et al.) 1906\u20131919 (ACL, 2020).","DOI":"10.18653\/v1\/2020.acl-main.173"},{"key":"10265_CR44","unstructured":"GPT-5 System Card https:\/\/openai.com\/index\/gpt-5-system-card\/ (OpenAI, 2025)."},{"key":"10265_CR45","doi-asserted-by":"publisher","DOI":"10.1145\/3703155","volume":"43","author":"L Huang","year":"2025","unstructured":"Huang, L. et al. A survey on hallucination in large language models: principles, taxonomy, challenges, and open questions. ACM Trans. Inf. Syst. 43, 42 (2025).","journal-title":"ACM Trans. Inf. Syst."},{"key":"10265_CR46","doi-asserted-by":"publisher","first-page":"570","DOI":"10.1038\/s41586-023-06792-0","volume":"624","author":"DA Boiko","year":"2023","unstructured":"Boiko, D. A., MacKnight, R., Kline, B. & Gomes, G. Autonomous chemical research with large language models. Nature 624, 570\u2013578 (2023).","journal-title":"Nature"},{"key":"10265_CR47","doi-asserted-by":"publisher","unstructured":"Bommasani, R. et al. On the opportunities and risks of foundation models. Preprint at https:\/\/doi.org\/10.48550\/arXiv.2108.07258 (2021).","DOI":"10.48550\/arXiv.2108.07258"},{"key":"10265_CR48","doi-asserted-by":"crossref","unstructured":"Ecoffet, A., Clune, J. & Lehman, J. Open questions in creating safe open-ended AI: tensions between control and creativity. In Proc. Artificial Life Conference Vol. 32 (eds Bongard, J. et al.) 27\u201335 (MIT Press, 2020).","DOI":"10.1162\/isal_a_00323"},{"key":"10265_CR49","doi-asserted-by":"publisher","first-page":"186345","DOI":"10.1007\/s11704-024-40231-1","volume":"18","author":"L Wang","year":"2024","unstructured":"Wang, L. et al. A survey on large language model based autonomous agents. Front. Comput. Sci. 18, 186345 (2024).","journal-title":"Front. Comput. Sci."},{"key":"10265_CR50","unstructured":"Brown, T. et al. Language models are few-shot learners. In Proc. Advances in Neural Information Processing Systems Vol. 33 (eds Larochelle, H. et al.) 1877\u20131901 (Curran Associates, 2020)."},{"key":"10265_CR51","doi-asserted-by":"crossref","unstructured":"Shinn, N., Cassano, F., Gopinath, A., Narasimhan, K. & Yao, S. Reflexion: language agents with verbal reinforcement learning. In Proc. Advances in Neural Information Processing Systems, Vol. 36 (eds Oh, A. et al.) 8634\u20138652 (Curran Associates, 2024).","DOI":"10.52202\/075280-0377"},{"key":"10265_CR52","unstructured":"Gauthier, P. aider. GitHub https:\/\/github.com\/paul-gauthier\/aider (2024)."},{"key":"10265_CR53","unstructured":"Vaswani, A. et al. Attention is all you need. In Advances in Neural Information Processing Systems (eds Guyon, I. et al.) Vol. 30, 5998\u20136008 (Curran Associates, 2017)."},{"key":"10265_CR54","unstructured":"Stanley, K. O., Lehman, J. & Soros, L. Open-endedness: the last grand challenge you\u2019ve never heard of. O\u2019Reilly Radar https:\/\/www.oreilly.com\/radar\/open-endedness-the-last-grand-challenge-youve-never-heard-of\/ (2017)."},{"key":"10265_CR55","unstructured":"Wolf, T. et al. Transformers: state-of-the-art natural language processing. In Proc.2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations (eds Liu, Q. & Schlangen, D.) 38\u201345 (ACL, 2020)."},{"key":"10265_CR56","unstructured":"El-Kishky, A. OpenAI o1 System Card https:\/\/api.semanticscholar.org\/CorpusID:272648256 (OpenAI, 2024)."},{"key":"10265_CR57","unstructured":"Introducing OpenAI o3 and o4-mini https:\/\/openai.com\/index\/introducing-o3-and-o4-mini\/ (OpenAI, 2025)."},{"key":"10265_CR58","unstructured":"Karpathy, A. NanoGPT. GitHub https:\/\/github.com\/karpathy\/nanoGPT (2022)."},{"key":"10265_CR59","unstructured":"Karpathy, A. The unreasonable effectiveness of recurrent neural networks. GitHub https:\/\/karpathy.github.io\/2015\/05\/21\/rnn-effectiveness\/ (2015)."},{"key":"10265_CR60","unstructured":"Hutter, M. The Hutter Prize http:\/\/prize.hutter1.net (2006)."},{"key":"10265_CR61","unstructured":"Mahoney, M. About the Test Data http:\/\/mattmahoney.net\/dc\/textdata.html (2011)."},{"key":"10265_CR62","doi-asserted-by":"publisher","first-page":"109306","DOI":"10.1016\/j.dib.2023.109306","volume":"49","author":"PK Mensah","year":"2023","unstructured":"Mensah, P. K. et al. CCMT: dataset for crop pest and disease detection. Data Brief 49, 109306 (2023).","journal-title":"Data Brief"},{"key":"10265_CR63","unstructured":"Sagawa, S., Koh, P. W., Hashimoto, T. & Liang, P. Waterbirds. GitHub https:\/\/github.com\/kohpangwei\/group_DRO (2019)."},{"key":"10265_CR64","doi-asserted-by":"crossref","unstructured":"Liu, Z., Luo, P., Wang, X. & Tang, X. Deep learning face attributes in the wild. In Proc. 2015 IEEE International Conference on Computer Vision (eds Ikeuchi, K. et al.) 3730\u20133738 (IEEE, 2015).","DOI":"10.1109\/ICCV.2015.425"}],"container-title":["Nature"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41586-026-10265-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41586-026-10265-5","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41586-026-10265-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,3,25]],"date-time":"2026-03-25T16:03:01Z","timestamp":1774454581000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41586-026-10265-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,3,25]]},"references-count":64,"journal-issue":{"issue":"8107","published-print":{"date-parts":[[2026,3,26]]}},"alternative-id":["10265"],"URL":"https:\/\/doi.org\/10.1038\/s41586-026-10265-5","relation":{},"ISSN":["0028-0836","1476-4687"],"issn-type":[{"value":"0028-0836","type":"print"},{"value":"1476-4687","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,3,25]]},"assertion":[{"value":"8 July 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 February 2026","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"25 March 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The principal investigator, J.C., has affiliations with the Vector Institute and Google DeepMind. This project has Vector Institute affiliations but is not connected to Google DeepMind. Chris Lu, Cong Lu, R.T.L., Y.Y., S.H.\u00a0and D.H. are employees of or consultants for Sakana AI, a machine learning research company involved in the design of The AI Scientist. The principal investigator is not financially compensated by Sakana AI. These arrangements have been reviewed and approved by the University of British Columbia in accordance with its conflict of interest policies. The other authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1038_2Fs41586_026_10644_y +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T16:11:59Z","timestamp":1793290319964,"version":"4.5.0"},"reference-count":37,"publisher":"Springer Science and Business Media LLC","issue":"8122","license":[{"start":{"date-parts":[[2026,5,19]],"date-time":"2026-05-19T00:00:00Z","timestamp":1779148800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2026,7,1]],"date-time":"2026-07-01T00:00:00Z","timestamp":1782864000000},"content-version":"vor","delay-in-days":43,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nature"],"published-print":{"date-parts":[[2026,7,9]]},"abstract":"<jats:title>Abstract<\/jats:title>\n <jats:p>\n Scientific discovery is driven by scientists generating hypotheses for complex problems that undergo rigorous experimental validation. To augment this process, we introduce Co-Scientist, a multi-agent artificial intelligence (AI) system built on Gemini for structured scientific thinking and hypothesis generation. Co-Scientist aims to help scientists discover new original knowledge. Conditioned on their research objectives and previous scientific evidence, it formulates demonstrably novel research hypotheses for experimental verification. The system\u2019s design involves agents continuously generating, critiquing and refining hypotheses accelerated by scaling test-time compute. Key contributions include (1) a multi-agent architecture with an asynchronous task execution framework for flexible compute scaling, and (2) a tournament evolution process for self-improving hypotheses generation. Automated evaluations show continued benefits of test-time compute scaling, improving hypothesis quality over time. Although this is a general-purpose system, we focus the validation in three biomedical applications: drug repurposing; novel-target discovery\n <jats:sup>1<\/jats:sup>\n ; and explaining mechanisms of antimicrobial resistance\n <jats:sup>2<\/jats:sup>\n . Specifically, Co-Scientist helped to identify new\u00a0drug-repurposing candidates and synergistic combination therapies for acute myeloid leukaemia that were validated through in vitro experiments. These real-world validations demonstrate the potential of Co-Scientist to accelerate scientific discovery and usher in an era of AI-empowered scientists.\n <\/jats:p>","DOI":"10.1038\/s41586-026-10644-y","type":"journal-article","created":{"date-parts":[[2026,5,19]],"date-time":"2026-05-19T17:02:08Z","timestamp":1779210128000},"page":"487-496","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":106,"title":["Accelerating scientific discovery with Co-Scientist"],"prefix":"10.1038","volume":"655","author":[{"ORCID":"https:\/\/orcid.org\/0009-0000-6348-5592","authenticated-orcid":false,"given":"Juraj","family":"Gottweis","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2232-0390","authenticated-orcid":false,"given":"Wei-Hung","family":"Weng","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7549-9702","authenticated-orcid":false,"given":"Alexander","family":"Daryin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9191-7938","authenticated-orcid":false,"given":"Tao","family":"Tu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Petar","family":"Sirkovic","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Artiom","family":"Myaskovsky","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Grzegorz","family":"Glowaty","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Felix","family":"Weissenberger","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alessio","family":"Orlandi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dan","family":"Popovici","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4720-8787","authenticated-orcid":false,"given":"Anil","family":"Palepu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4422-1361","authenticated-orcid":false,"given":"Keran","family":"Rong","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8107-6730","authenticated-orcid":false,"given":"Ryutaro","family":"Tanno","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1427-0469","authenticated-orcid":false,"given":"Khaled","family":"Saab","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fan","family":"Zhang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jacob","family":"Blum","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4824-6689","authenticated-orcid":false,"given":"Andrew","family":"Carroll","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kavita","family":"Kulkarni","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1624-0220","authenticated-orcid":false,"given":"Nenad","family":"Toma\u0161ev","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dina","family":"Zverinski","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ivor","family":"Rendulic","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Elahe","family":"Vedadi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-1720-0964","authenticated-orcid":false,"given":"Florian","family":"Hasler","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Luka","family":"Rimanic","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marina","family":"Boia","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-6696-3842","authenticated-orcid":false,"given":"Ivan","family":"Budiselic","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1360-771X","authenticated-orcid":false,"given":"Ben","family":"Feinstein","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mathias","family":"Bellaiche","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tom","family":"Sheffer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jan","family":"Freyberg","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6522-138X","authenticated-orcid":false,"given":"Jeremy","family":"Ratcliff","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8578-3216","authenticated-orcid":false,"given":"Ottavia","family":"Bertolli","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Katherine","family":"Chou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Avinatan","family":"Hassidim","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Burak","family":"Gokturk","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4866-1698","authenticated-orcid":false,"given":"Amin","family":"Vahdat","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yuan","family":"Guan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Vikram","family":"Dhillon","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3720-8051","authenticated-orcid":false,"given":"Eeshit Dhaval","family":"Vaishnav","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7132-2662","authenticated-orcid":false,"given":"Byron","family":"Lee","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2716-9669","authenticated-orcid":false,"given":"Tiago R. D.","family":"Costa","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6439-5262","authenticated-orcid":false,"given":"Jos\u00e9 R.","family":"Penad\u00e9s","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6191-7697","authenticated-orcid":false,"given":"Gary","family":"Peltz","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3960-6002","authenticated-orcid":false,"given":"Yossi","family":"Matias","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"James","family":"Manyika","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2812-9917","authenticated-orcid":false,"given":"Demis","family":"Hassabis","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yunhan","family":"Xu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7466-7997","authenticated-orcid":false,"given":"Pushmeet","family":"Kohli","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3408-986X","authenticated-orcid":false,"given":"Annalisa","family":"Pawlosky","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-4958-5976","authenticated-orcid":false,"given":"Alan","family":"Karthikesalingam","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7849-2074","authenticated-orcid":false,"given":"Vivek","family":"Natarajan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2026,5,19]]},"reference":[{"key":"10644_CR1","doi-asserted-by":"crossref","unstructured":"Guan, Y. et al. AI-assisted drug re-purposing for human liver fibrosis. Adv. Sci. 12, e08751 (2025).","DOI":"10.1002\/advs.202508751"},{"key":"10644_CR2","doi-asserted-by":"publisher","first-page":"6654","DOI":"10.1016\/j.cell.2025.08.018","volume":"188","author":"JR Penad\u00e9s","year":"2025","unstructured":"Penad\u00e9s, J. R. et al. AI mirrors experimental science to uncover a novel mechanism of gene transfer crucial to bacterial evolution. Cell 188, 6654\u20136665 (2025).","journal-title":"Cell"},{"key":"10644_CR3","doi-asserted-by":"publisher","first-page":"816","DOI":"10.1126\/science.1225829","volume":"337","author":"M Jinek","year":"2012","unstructured":"Jinek, M. et al. A programmable dual-RNA-guided DNA endonuclease in adaptive bacterial immunity. Science 337, 816\u2013821 (2012).","journal-title":"Science"},{"key":"10644_CR4","doi-asserted-by":"publisher","first-page":"2554","DOI":"10.1073\/pnas.79.8.2554","volume":"79","author":"JJ Hopfield","year":"1982","unstructured":"Hopfield, J. J. Neural networks and physical systems with emergent collective computational abilities. Proc. Natl Acad. Sci. USA 79, 2554\u20132558 (1982).","journal-title":"Proc. Natl Acad. Sci. USA"},{"key":"10644_CR5","unstructured":"Hinton, G. E. & Sejnowski, T. J. in Parallel Distributed Processing: Explorations in the Microstructure of Cognition Vol. 1 (eds Feldman, J. A. et al.) 282\u2013317 (MIT Press, 1986)."},{"key":"10644_CR6","doi-asserted-by":"crossref","unstructured":"Guo, D. et al. DeepSeek-R1 incentivizes reasoning in LLMs through reinforcement learning. Nature 645, 633\u2013638 (2025).","DOI":"10.1038\/s41586-025-09422-z"},{"key":"10644_CR7","unstructured":"Jaech, A. et al. OpenAI O1 system card. Preprint at https:\/\/arxiv.org\/abs\/2412.16720 (2024)."},{"key":"10644_CR8","unstructured":"Gemini Team, Google. Gemini 1.5: unlocking multimodal understanding across millions of tokens of context. Preprint at https:\/\/arxiv.org\/abs\/2403.05530 (2024)."},{"key":"10644_CR9","unstructured":"Wiesinger, J., Marlow, P. & Vuskovic, V. Agents. kaggle https:\/\/www.kaggle.com\/whitepaper-agents (2024)."},{"key":"10644_CR10","unstructured":"Hinton, G. E., Vinyals, O. & Dean, J. Distilling the knowledge in a neural network. Preprint at https:\/\/arxiv.org\/abs\/1503.02531 (2015)."},{"key":"10644_CR11","unstructured":"Gemma Team, Google. Gemma: open models based on gemini research and technology. Preprint at https:\/\/arxiv.org\/abs\/2403.08295 (2024)."},{"key":"10644_CR12","doi-asserted-by":"publisher","unstructured":"Leslie, D. et al. \u2018Frontier AI,\u2019 power, and the public interest: who benefits, who decides? Harvard Data Sci. Rev. https:\/\/doi.org\/10.1162\/99608f92.4a42495c (2024).","DOI":"10.1162\/99608f92.4a42495c"},{"key":"10644_CR13","doi-asserted-by":"publisher","first-page":"45767","DOI":"10.52202\/079017-1455","volume":"37","author":"L Chen","year":"2024","unstructured":"Chen, L. et al. Are more LLM calls all you need? Towards the scaling properties of compound AI systems. Adv. Neural Inform. Process. Syst. 37, 45767\u201345790 (2024).","journal-title":"Adv. Neural Inform. Process. Syst."},{"key":"10644_CR14","unstructured":"Comanici, G. et al. Gemini 2.5: pushing the frontier with advanced reasoning, multimodality, long context, and next generation agentic capabilities. Preprint at https:\/\/arxiv.org\/abs\/2507.06261 (2025)."},{"key":"10644_CR15","unstructured":"Gower, B. Scientific Method: a Historical and Philosophical Introduction (Routledge, 2012)."},{"key":"10644_CR16","unstructured":"Snell, C., Lee, J., Xu, K. & Kumar, A. Scaling LLM test-time compute optimally can be more effective than scaling parameters for reasoning. In International Conference on Learning Representations (ICLR) (2025)."},{"key":"10644_CR17","doi-asserted-by":"publisher","first-page":"885","DOI":"10.1126\/science.aay2400","volume":"365","author":"N Brown","year":"2019","unstructured":"Brown, N. & Sandholm, T. Superhuman AI for multiplayer poker. Science 365, 885\u2013890 (2019).","journal-title":"Science"},{"key":"10644_CR18","doi-asserted-by":"publisher","first-page":"484","DOI":"10.1038\/nature16961","volume":"529","author":"D Silver","year":"2016","unstructured":"Silver, D. et al. Mastering the game of Go with deep neural networks and tree search. Nature 529, 484\u2013489 (2016).","journal-title":"Nature"},{"key":"10644_CR19","doi-asserted-by":"publisher","first-page":"833","DOI":"10.1038\/d41573-020-00059-3","volume":"19","author":"MS Ringel","year":"2020","unstructured":"Ringel, M. S., Scannell, J. W., Baedeker, M. & Schulze, U. Breaking Eroom\u2019s law. Nat. Rev. Drug Discov. 19, 833\u2013834 (2020).","journal-title":"Nat. Rev. Drug Discov."},{"key":"10644_CR20","doi-asserted-by":"publisher","first-page":"41","DOI":"10.1038\/nrd.2018.168","volume":"18","author":"S Pushpakom","year":"2019","unstructured":"Pushpakom, S. et al. Drug repurposing: progress, challenges and recommendations. Nat. Rev. Drug Discov. 18, 41\u201358 (2019).","journal-title":"Nat. Rev. Drug Discov."},{"key":"10644_CR21","doi-asserted-by":"publisher","first-page":"6636","DOI":"10.1016\/j.cell.2025.08.019","volume":"188","author":"L He","year":"2025","unstructured":"He, L. et al. Chimeric infective particles expand species boundaries in phage-inducible chromosomal island mobilization. Cell 188, 6636\u20136653 (2025).","journal-title":"Cell"},{"key":"10644_CR22","doi-asserted-by":"publisher","first-page":"1136","DOI":"10.1056\/NEJMra1406184","volume":"373","author":"H D\u00f6hner","year":"2015","unstructured":"D\u00f6hner, H., Weisdorf, D. J. & Bloomfield, C. D. Acute myeloid leukemia. N. Engl. J. Med. 373, 1136\u20131152 (2015).","journal-title":"N. Engl. J. Med."},{"key":"10644_CR23","doi-asserted-by":"publisher","first-page":"53","DOI":"10.1038\/s41392-024-01757-9","volume":"9","author":"Q Guo","year":"2024","unstructured":"Guo, Q. et al. NF-\u03baB in biology and targeted therapy: new insights and translational implications. Signal Transduc. Target. Ther. 9, 53 (2024).","journal-title":"Signal Transduc. Target. Ther."},{"key":"10644_CR24","doi-asserted-by":"publisher","first-page":"142","DOI":"10.1016\/j.clml.2018.12.009","volume":"19","author":"A Maiti","year":"2019","unstructured":"Maiti, A. et al. Phase II trial of MEK inhibitor binimetinib (MEK162) in RAS-mutant acute myeloid leukemia. Clin. Lymphoma Myeloma Leuk. 19, 142\u2013148 (2019).","journal-title":"Clin. Lymphoma Myeloma Leuk."},{"key":"10644_CR25","doi-asserted-by":"publisher","first-page":"1764","DOI":"10.1038\/s41375-024-02321-8","volume":"38","author":"C Philippe","year":"2024","unstructured":"Philippe, C. et al. Pivotal role of the endoplasmic reticulum stress-related XBP1s\/miR-22\/SIRT1 axis in acute myeloid leukemia apoptosis and response to chemotherapy. Leukemia 38, 1764\u20131776 (2024).","journal-title":"Leukemia"},{"key":"10644_CR26","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-021-26410-9","volume":"12","author":"Y Guan","year":"2021","unstructured":"Guan, Y. et al. A human multi-lineage hepatic organoid model for liver fibrosis. Nat. Commun. 12, 6138 (2021).","journal-title":"Nat. Commun."},{"key":"10644_CR27","doi-asserted-by":"crossref","unstructured":"Guan, Y. et al. Live-cell imaging of human liver fibrosis using hepatic micro-organoids. JCI Insight 10, e187099 (2024).","DOI":"10.1172\/jci.insight.187099"},{"key":"10644_CR28","doi-asserted-by":"publisher","unstructured":"Jones, N. OpenAI\u2019s \u2018deep research\u2019 tool: is it useful for scientists? Nature https:\/\/doi.org\/10.1038\/d41586-025-00377-9 (2025).","DOI":"10.1038\/d41586-025-00377-9"},{"key":"10644_CR29","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-022-35724-1","volume":"14","author":"S Megat","year":"2023","unstructured":"Megat, S. et al. Integrative genetic analysis illuminates ALS heritability and identifies risk genes. Nat. Commun. 14, 342 (2023).","journal-title":"Nat. Commun."},{"key":"10644_CR30","unstructured":"Elo, A. E. The Rating of Chess Players, Past and Present (Ishi Press, 2008)."},{"key":"10644_CR31","unstructured":"LeCun, Y. A theoretical framework for back-propagation. In Proc. 1988 Connectionist Models Summer School Vol. 1 (eds Touretzky, D. et al.) 21\u201328 (Morgan Kaufmann, 1988)."},{"key":"10644_CR32","doi-asserted-by":"crossref","unstructured":"Hamilton, A. H., Kalenkova, A. & Roughan, M. The impact of intransitivity on the Elo rating system. PLoS One 20, e0338261 (2025).","DOI":"10.1371\/journal.pone.0338261"},{"key":"10644_CR33","doi-asserted-by":"publisher","first-page":"198","DOI":"10.3233\/ICG-2007-30403","volume":"30","author":"R Coulom","year":"2007","unstructured":"Coulom, R. Computing \u2018Elo ratings\u2019 of move patterns in the game of go. ICGA J. 30, 198\u2013208 (2007).","journal-title":"ICGA J."},{"key":"10644_CR34","doi-asserted-by":"publisher","first-page":"1329","DOI":"10.1016\/j.ijforecast.2020.01.006","volume":"36","author":"S Kovalchik","year":"2020","unstructured":"Kovalchik, S. Extension of the Elo rating system to margin of victory. Int. J. Forecast. 36, 1329\u20131341 (2020).","journal-title":"Int. J. Forecast."},{"key":"10644_CR35","unstructured":"Khan, A. et al. Debating with more persuasive LLMs leads to more truthful answers. In Proc. 41st International Conference on Machine Learning Vol. 235 (eds Salakhutdinov, R. et al.) 23662\u201323733 (PMLR, 2024)."},{"key":"10644_CR36","doi-asserted-by":"crossref","unstructured":"Post, M. & Vilar, D. Fast lexically constrained decoding with dynamic beam allocation for neural machine translation. In Proc. 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies Vol. 1 (Long Papers) 1314\u20131324 (Association for Computational Linguistics, 2018).","DOI":"10.18653\/v1\/N18-1119"},{"key":"10644_CR37","unstructured":"Gemini Team, Google. Gemini: a family of highly capable multimodal models. Preprint at https:\/\/arxiv.org\/abs\/2312.11805 (2023)."}],"container-title":["Nature"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41586-026-10644-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41586-026-10644-y","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41586-026-10644-y.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,7,9]],"date-time":"2026-07-09T05:32:18Z","timestamp":1783575138000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41586-026-10644-y"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,5,19]]},"references-count":37,"journal-issue":{"issue":"8122","published-print":{"date-parts":[[2026,7,9]]}},"alternative-id":["10644"],"URL":"https:\/\/doi.org\/10.1038\/s41586-026-10644-y","relation":{},"ISSN":["0028-0836","1476-4687"],"issn-type":[{"value":"0028-0836","type":"print"},{"value":"1476-4687","type":"electronic"}],"subject":[],"published":{"date-parts":[[2026,5,19]]},"assertion":[{"value":"20 March 2025","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 May 2026","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"19 May 2026","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"This study was funded by Alphabet and\/or a subsidiary thereof. Some authors (J.G., W.-H.W., AD, T.T., P.S., A.M., G.G., F.W., A.O., D.P., A. Palepu, K.R., R.T., K.S., F.Z., A.C., K.K., N.T., D.Z., I.R., E.V., F.H., L.R., M. Boia, I.B., B.F., M. Bellaiche, T.S., J.F., J.R., O.B., K.C., A.H., B.G., A.V., Y.M., J.M., D.H., P.K., Y.X., A. Pawlosky, A.K. and V.N.) are employees of Alphabet and may own stock as part of the standard compensation package. E.D.V. and B.L. are employees of Sequome. B.L. is the founder of Dendra Therapeutics.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1038_2Fs41597_022_01143_6 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,23]],"date-time":"2026-10-23T08:28:11Z","timestamp":1792744091401,"version":"4.5.0"},"reference-count":49,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2022,2,21]],"date-time":"2022-02-21T00:00:00Z","timestamp":1645401600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2022,2,21]],"date-time":"2022-02-21T00:00:00Z","timestamp":1645401600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"DOI":"10.13039\/100000879","name":"Alfred P. Sloan Foundation","doi-asserted-by":"publisher","award":["P-2020-13988"],"award-info":[{"award-number":["P-2020-13988"]}],"id":[{"id":"10.13039\/100000879","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Sci Data"],"abstract":"<jats:title>Abstract<\/jats:title>\n <jats:p>This article presents a study on the quality and execution of research code from publicly-available replication datasets at the Harvard Dataverse repository. Research code is typically created by a group of scientists and published together with academic papers to facilitate research transparency and reproducibility. For this study, we define ten questions to address aspects impacting research reproducibility and reuse. First, we retrieve and analyze more than 2000 replication datasets with over 9000 unique R files published from 2010 to 2020. Second, we execute the code in a clean runtime environment to assess its ease of reuse. Common coding errors were identified, and some of them were solved with automatic code cleaning to aid code execution. We find that 74% of R files failed to complete without error in the initial execution, while 56% failed when code cleaning was applied, showing that many errors can be prevented with good coding practices. We also analyze the replication datasets from journals\u2019 collections and discuss the impact of the journal policy strictness on the code re-execution rate. Finally, based on our results, we propose a set of recommendations for code dissemination aimed at researchers, journals, and repositories.<\/jats:p>","DOI":"10.1038\/s41597-022-01143-6","type":"journal-article","created":{"date-parts":[[2022,2,21]],"date-time":"2022-02-21T06:23:53Z","timestamp":1645424633000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":118,"title":["A large-scale study on research code quality and execution"],"prefix":"10.1038","volume":"9","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-1991-0533","authenticated-orcid":false,"given":"Ana","family":"Trisovic","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3758-2406","authenticated-orcid":false,"given":"Matthew K.","family":"Lau","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6876-1306","authenticated-orcid":false,"given":"Thomas","family":"Pasquier","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Merc\u00e8","family":"Crosas","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2022,2,21]]},"reference":[{"key":"1143_CR1","doi-asserted-by":"publisher","unstructured":"Digital Science et al. The State of Open Data Report 2019. Digital Science https:\/\/doi.org\/10.6084\/m9.figshare.9980783.v2 (2019).","DOI":"10.6084\/m9.figshare.9980783.v2"},{"key":"1143_CR2","unstructured":"National Academies of Sciences, Engineering, and Medicine and others. Reproducibility and replicability in science (National Academies Press, 2019)."},{"key":"1143_CR3","doi-asserted-by":"publisher","first-page":"452","DOI":"10.1038\/533452a","volume":"533","author":"M Baker","year":"2016","unstructured":"Baker, M. 1,500 scientists lift the lid on reproducibility. Nature News 533, 452 (2016).","journal-title":"Nature News"},{"key":"1143_CR4","doi-asserted-by":"publisher","first-page":"97","DOI":"10.1093\/comjnl\/27.2.97","volume":"27","author":"DE Knuth","year":"1984","unstructured":"Knuth, D. E. Literate programming. The Computer Journal 27, 97\u2013111 (1984).","journal-title":"The Computer Journal"},{"key":"1143_CR5","doi-asserted-by":"publisher","first-page":"111","DOI":"10.1109\/MCSE.2018.042781334","volume":"20","author":"T Pasquier","year":"2018","unstructured":"Pasquier, T. et al. Sharing and Preserving Computational Analyses for Posterity with encapsulator. Computing in Science Engineering 20, 111\u2013124 (2018).","journal-title":"Computing in Science Engineering"},{"key":"1143_CR6","doi-asserted-by":"publisher","first-page":"15","DOI":"10.3390\/data6020015","volume":"6","author":"A Trisovic","year":"2021","unstructured":"Trisovic, A., Mika, K., Boyd, C., Feger, S. & Crosas, M. Repository approaches to improving the quality of shared data and code. Data 6, 15 (2021).","journal-title":"Data"},{"key":"1143_CR7","unstructured":"Muenchen, R. A. The popularity of data analysis software. Preprint at http:\/\/r4stats.com\/popularity (2012)."},{"key":"1143_CR8","unstructured":"Kaggle ML & DS Survey. The most comprehensive dataset available on the state of ML and data science https:\/\/www.kaggle.com\/kaggle\/kaggle-survey-2018 (2018)."},{"key":"1143_CR9","doi-asserted-by":"publisher","DOI":"10.7910\/DVN\/UZLXSZ","author":"A Trisovic","year":"2021","unstructured":"Trisovic, A. Replication Data for: A large-scale study on research code quality and execution. Harvard Dataverse https:\/\/doi.org\/10.7910\/DVN\/UZLXSZ (2021)."},{"key":"1143_CR10","doi-asserted-by":"crossref","unstructured":"Baumer, B., Cetinkaya-Rundel, M., Bray, A., Loi, L. & Horton, N. J. R markdown: Integrating a reproducible analysis tool into introductory statistics. Preprint at https:\/\/arxiv.org\/abs\/1402.1894 (2014).","DOI":"10.5070\/T581020118"},{"key":"1143_CR11","unstructured":"W3C Working Group. Best practices for publishing linked data http:\/\/www.w3.org\/TR\/ld-bp\/ (2014)."},{"key":"1143_CR12","unstructured":"Martin, R. C. Clean code: a handbook of agile software craftsmanship (Pearson Education, 2009)."},{"key":"1143_CR13","doi-asserted-by":"crossref","unstructured":"Sandve, G. K., Nekrutenko, A., Taylor, J. & Hovig, E. Ten Simple Rules for Reproducible Computational Research. PLoS Comput Biol 9 (Public Library of Science, 2013).","DOI":"10.1371\/journal.pcbi.1003285"},{"key":"1143_CR14","unstructured":"McConnell, S. Code complete (Pearson Education, 2004)."},{"key":"1143_CR15","unstructured":"Thomas, D. & Hunt, A. The Pragmatic Programmer: your journey to mastery (Addison-Wesley Professional, 2019)."},{"key":"1143_CR16","unstructured":"Hyde, R. Write great code: Thinking low-level, writing high-level, 2 (No Starch Press, 2020)."},{"key":"1143_CR17","doi-asserted-by":"crossref","unstructured":"Davidson, S. B. & Freire, J. Provenance and scientific workflows: challenges and opportunities. Proceedings of the 2008 ACM SIGMOD international conference on Management of data, 1345\u20131350 (2008).","DOI":"10.1145\/1376616.1376772"},{"key":"1143_CR18","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/sdata.2017.114","volume":"4","author":"T Pasquier","year":"2017","unstructured":"Pasquier, T. et al. If these data could talk. Scientific data 4, 1\u20135 (2017).","journal-title":"Scientific data"},{"key":"1143_CR19","doi-asserted-by":"publisher","first-page":"80","DOI":"10.1080\/00031305.2017.1375986","volume":"72","author":"B Marwick","year":"2018","unstructured":"Marwick, B., Boettiger, C. & Mullen, L. Packaging data analytical work reproducibly using R (and friends). The American Statistician 72, 80\u201388 (2018).","journal-title":"The American Statistician"},{"key":"1143_CR20","doi-asserted-by":"crossref","unstructured":"Vines, T. H. et al. Mandated data archiving greatly improves access to research data. The FASEB Journal 27, 1304\u20131308 (2013).","DOI":"10.1096\/fj.12-218164"},{"key":"1143_CR21","doi-asserted-by":"publisher","unstructured":"Crosas, M. et al. Data policies of highly-ranked social science journals. Preprint at https:\/\/doi.org\/10.31235\/osf.io\/9h7ay (2018).","DOI":"10.31235\/osf.io\/9h7ay"},{"key":"1143_CR22","doi-asserted-by":"publisher","DOI":"10.7910\/DVN\/SWV9GJ","author":"A White","year":"2017","unstructured":"White, A. & Trump, K.-S. Replication Data for: Does Inequality Beget Inequality? Experimental tests of the prediction that inequality increases system justification motivation. Harvard Dataverse https:\/\/doi.org\/10.7910\/DVN\/SWV9GJ (2017)."},{"key":"1143_CR23","doi-asserted-by":"publisher","DOI":"10.7910\/DVN\/Z02C8Y","author":"J Polk","year":"2019","unstructured":"Polk, J. Replication Data for: Explaining the Salience of Anti-Elitism and Reducing Political Corruption for Political Parties in Europe with the 2014 Chapel Hill Expert Survey Data. Harvard Dataverse https:\/\/doi.org\/10.7910\/DVN\/Z02C8Y (2019)."},{"key":"1143_CR24","doi-asserted-by":"publisher","DOI":"10.7910\/DVN\/YT45AO","author":"E Helland","year":"2016","unstructured":"Helland, E. & Yoon, J. Replication Data for: Estimating the Effects of the English Rule on Litigation Outcomes. Harvard Dataverse https:\/\/doi.org\/10.7910\/DVN\/YT45AO (2016)."},{"key":"1143_CR25","doi-asserted-by":"crossref","unstructured":"Stodden, V., Seiler, J. & Ma, Z. An empirical analysis of journal policy effectiveness for computational reproducibility. Proceedings of the National Academy of Sciences 115, 2584\u20132589 (National Academy of Sciences, 2018).","DOI":"10.1073\/pnas.1708290115"},{"key":"1143_CR26","doi-asserted-by":"crossref","unstructured":"Chang, A. C. & Li, P. Is economics research replicable? Sixty published papers from thirteen journals say \u201cusually not\u201d. Finance and Economics Discussion Series 2015-083 (2015).","DOI":"10.2139\/ssrn.2669564"},{"key":"1143_CR27","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/sdata.2016.18","volume":"3","author":"MD Wilkinson","year":"2016","unstructured":"Wilkinson, M. D. et al. The FAIR guiding principles for scientific data management and stewardship. Scientific data 3, 1\u20139 (2016).","journal-title":"Scientific data"},{"key":"1143_CR28","doi-asserted-by":"publisher","first-page":"37","DOI":"10.3233\/DS-190026","volume":"3","author":"A-L Lamprecht","year":"2020","unstructured":"Lamprecht, A.-L. et al. Towards fair principles for research software. Data Science 3, 37\u201359 (2020).","journal-title":"Data Science"},{"key":"1143_CR29","doi-asserted-by":"crossref","unstructured":"Jim\u00e9nez, R. C. et al. Four simple recommendations to encourage best practices in research software. F1000Research 6 (2017).","DOI":"10.12688\/f1000research.11407.1"},{"key":"1143_CR30","doi-asserted-by":"publisher","first-page":"260","DOI":"10.5860\/crl.80.2.260","volume":"80","author":"F Sayre","year":"2019","unstructured":"Sayre, F. & Riegelman, A. Replicable services for reproducible research: a model for academic libraries. College & Research Libraries 80, 260 (2019).","journal-title":"College & Research Libraries"},{"key":"1143_CR31","doi-asserted-by":"crossref","unstructured":"Sandve, G. K., Nekrutenko, A., Taylor, J. & Hovig, E. Ten simple rules for reproducible computational research. PLoS Comput Biol 9 (2013).","DOI":"10.1371\/journal.pcbi.1003285"},{"key":"1143_CR32","doi-asserted-by":"publisher","first-page":"113","DOI":"10.1038\/s41567-018-0342-2","volume":"15","author":"X Chen","year":"2019","unstructured":"Chen, X. et al. Open is not enough. Nature Physics 15, 113\u2013119 (2019).","journal-title":"Nature Physics"},{"key":"1143_CR33","doi-asserted-by":"publisher","first-page":"e1006561","DOI":"10.1371\/journal.pcbi.1006561","volume":"14","author":"BD Lee","year":"2018","unstructured":"Lee, B. D. Ten simple rules for documenting scientific software. PLoS Comput Biol 14, e1006561 (2018).","journal-title":"PLoS Comput Biol"},{"key":"1143_CR34","doi-asserted-by":"publisher","first-page":"73","DOI":"10.1109\/MCSE.2018.2881905","volume":"21","author":"LA Barba","year":"2019","unstructured":"Barba, L. A. Praxis of reproducible computational science. Computing in Science & Engineering 21, 73\u201378 (2019).","journal-title":"Computing in Science & Engineering"},{"key":"1143_CR35","unstructured":"Arnold, B. et al. The turing way: A handbook for reproducible data science https:\/\/the-turing-way.netlify.app (2019)."},{"key":"1143_CR36","doi-asserted-by":"publisher","first-page":"437","DOI":"10.32614\/RJ-2020-007","volume":"12","author":"D N\u0171st","year":"2020","unstructured":"N\u0171st, D. et al. The Rockerverse: Packages and Applications for Containerisation with R. The R Journal 12, 437\u2013461 (2020).","journal-title":"The R Journal"},{"key":"1143_CR37","doi-asserted-by":"crossref","unstructured":"Clyburne-Sherin, A., Fei, X. & Green, S. A. Computational reproducibility via containers in psychology. Meta-psychology 3 (2019).","DOI":"10.15626\/MP.2018.892"},{"key":"1143_CR38","doi-asserted-by":"publisher","DOI":"10.1186\/s13059-021-02299-x","volume":"22","author":"B Cheifet","year":"2021","unstructured":"Cheifet, B. Promoting reproducibility with Code Ocean. Genome Biol 22, 65 (2021).","journal-title":"Genome Biol"},{"key":"1143_CR39","doi-asserted-by":"publisher","first-page":"854","DOI":"10.1016\/j.future.2017.12.029","volume":"94","author":"A Brinckman","year":"2019","unstructured":"Brinckman, A. et al. Computing environments for reproducibility: Capturing the \u201cwhole tale\u201d. Future Generation Computer Systems 94, 854\u2013867 (2019).","journal-title":"Future Generation Computer Systems"},{"key":"1143_CR40","doi-asserted-by":"crossref","unstructured":"Jupyter et al. Binder 2.0 - reproducible, interactive, sharable environments for science at scale (2018).","DOI":"10.25080\/Majora-4af1f417-011"},{"key":"1143_CR41","doi-asserted-by":"crossref","unstructured":"Trisovic, A. et al. Advancing Computational Reproducibility in the Dataverse Data Repository Platform. Proceedings of the 3rd International Workshop on Practical Reproducible Evaluation of Computer Systems, 15\u201320 (ACM, 2020).","DOI":"10.1145\/3391800.3398173"},{"key":"1143_CR42","doi-asserted-by":"crossref","unstructured":"Chuah, J., Deeds, M., Malik, T., Choi, Y. & Goodall, J. L. Documenting computing environments for reproducible experiments. PARCO, 756\u2013765 (2019).","DOI":"10.3233\/APC200106"},{"key":"1143_CR43","doi-asserted-by":"publisher","year":"2018","unstructured":"The Software Sustainability Institute. Checklist for a software management plan. Zenodo https:\/\/doi.org\/10.5281\/zenodo.2159713 (2018).","DOI":"10.5281\/zenodo.2159713"},{"key":"1143_CR44","doi-asserted-by":"crossref","unstructured":"Goeva, A., Stoudt, S. & Trisovic, A. Toward reproducible and extensible research: from values to action. Harvard Data Science Review 2 (2020).","DOI":"10.1162\/99608f92.1cc3d72a"},{"key":"1143_CR45","unstructured":"Pineau, J. et al. Improving reproducibility in machine learning research: a report from the NeurIPS 2019 reproducibility program. Journal of Machine Learning Research 22 (2021)."},{"key":"1143_CR46","unstructured":"Bahaidarah, L. et al. Toward reusable science with readable code and reproducibility. Preprint at https:\/\/arxiv.org\/abs\/2109.10387 (2021)."},{"key":"1143_CR47","doi-asserted-by":"publisher","first-page":"408","DOI":"10.1080\/13658816.2018.1508687","volume":"33","author":"M Konkol","year":"2019","unstructured":"Konkol, M., Kray, C. & Pfeiffer, M. Computational reproducibility in geoscientific papers: Insights from a series of studies with geoscientists and a reproduction study. International Journal of Geographical Information Science 33, 408\u2013429 (2019).","journal-title":"International Journal of Geographical Information Science"},{"key":"1143_CR48","unstructured":"Chen, C. Coding BetteR: Assessing and Improving the Reproducibility of R-Based Research With containR. Preprint at https:\/\/dash.harvard.edu\/handle\/1\/38811561 (2018)."},{"key":"1143_CR49","doi-asserted-by":"crossref","unstructured":"Pimentel, J. F., Murta, L., Braganholo, V. & Freire, J. A large-scale study about quality and reproducibility of Jupyter notebooks. IEEE\/ACM 16th International Conference on Mining Software Repositories (MSR), 507\u2013517 (IEEE, 2019).","DOI":"10.1109\/MSR.2019.00077"}],"container-title":["Scientific Data"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41597-022-01143-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41597-022-01143-6","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41597-022-01143-6.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,11,24]],"date-time":"2022-11-24T14:04:19Z","timestamp":1669298659000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41597-022-01143-6"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,2,21]]},"references-count":49,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2022,12]]}},"alternative-id":["1143"],"URL":"https:\/\/doi.org\/10.1038\/s41597-022-01143-6","relation":{"references":[{"id-type":"doi","id":"10.7910\/DVN\/UZLXSZ","asserted-by":"subject"},{"id-type":"doi","id":"10.7910\/DVN\/SWV9GJ","asserted-by":"subject"},{"id-type":"doi","id":"10.7910\/DVN\/Z02C8Y","asserted-by":"subject"},{"id-type":"doi","id":"10.7910\/DVN\/YT45AO","asserted-by":"subject"},{"id-type":"doi","id":"10.5281\/zenodo.2159713","asserted-by":"subject"}]},"ISSN":["2052-4463"],"issn-type":[{"value":"2052-4463","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,2,21]]},"assertion":[{"value":"26 March 2021","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"17 January 2022","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 February 2022","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"60"}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1038_2Fs41598_023_41032_5 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,30]],"date-time":"2026-10-30T00:29:55Z","timestamp":1793320195305,"version":"4.5.0"},"reference-count":45,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2023,9,7]],"date-time":"2023-09-07T00:00:00Z","timestamp":1694044800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2023,9,7]],"date-time":"2023-09-07T00:00:00Z","timestamp":1694044800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Sci Rep"],"abstract":"<jats:title>Abstract<\/jats:title>\n <jats:p>Although chatbots such as ChatGPT can facilitate cost-effective text generation and editing, factually incorrect responses (hallucinations) limit their utility. This study evaluates one particular type of hallucination: fabricated bibliographic citations that do not represent actual scholarly works. We used ChatGPT-3.5 and ChatGPT-4 to produce short literature reviews on 42 multidisciplinary topics, compiling data on the 636 bibliographic citations (references) found in the 84 papers. We then searched multiple databases and websites to determine the prevalence of fabricated citations, to identify errors in the citations to non-fabricated papers, and to evaluate adherence to APA citation format. Within this set of documents, 55% of the GPT-3.5 citations but just 18% of the GPT-4 citations are fabricated. Likewise, 43% of the real (non-fabricated) GPT-3.5 citations but just 24% of the real GPT-4 citations include substantive citation errors. Although GPT-4 is a major improvement over GPT-3.5, problems remain.<\/jats:p>","DOI":"10.1038\/s41598-023-41032-5","type":"journal-article","created":{"date-parts":[[2023,9,7]],"date-time":"2023-09-07T06:04:29Z","timestamp":1694066669000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":485,"title":["Fabrication and errors in the bibliographic citations generated by ChatGPT"],"prefix":"10.1038","volume":"13","author":[{"given":"William H.","family":"Walters","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Esther Isabelle","family":"Wilder","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2023,9,7]]},"reference":[{"key":"41032_CR1","unstructured":"OpenAI. Documentation. https:\/\/platform.openai.com\/docs\/introduction (2023)."},{"key":"41032_CR2","unstructured":"D\u2019Agostino, S. Designing assignments in the ChatGPT era. Inside Higher Ed. https:\/\/www.insidehighered.com\/news\/2023\/01\/31\/chatgpt-sparks-debate-how-design-student-assignments-now (2023)."},{"key":"41032_CR3","unstructured":"DePeau-Wilson, M. Peer-reviewed journal publishes paper written almost entirely by ChatGPT. MedPage Today. https:\/\/www.medpagetoday.com\/special-reports\/exclusives\/102960 (2023)."},{"key":"41032_CR4","doi-asserted-by":"publisher","first-page":"570","DOI":"10.1002\/asi.24750","volume":"74","author":"BD Lund","year":"2023","unstructured":"Lund, B. D. et al. ChatGPT and a new academic reality: Artificial Intelligence-written research papers and the ethics of the large language models in scholarly publishing. J. Assoc. Inf. Sci. Technol. 74, 570\u2013581 (2023).","journal-title":"J. Assoc. Inf. Sci. Technol."},{"key":"41032_CR5","unstructured":"McMurtrie, B. AI and the future of undergraduate writing. Chron. High. Educ. https:\/\/www.chronicle.com\/article\/ai-and-the-future-of-undergraduate-writing (2022)."},{"key":"41032_CR6","unstructured":"Tools such as ChatGPT threaten transparent science. Here are our ground rules for their use. Nature. https:\/\/www.nature.com\/articles\/d41586-023-00191-1 (2023)."},{"key":"41032_CR7","unstructured":"Rigolino, R. E. With ChatGPT, we\u2019re all editors now. Inside Higher Ed. https:\/\/www.insidehighered.com\/views\/2023\/01\/31\/chatgpt-we-must-teach-students-be-editors-opinion (2023)."},{"key":"41032_CR8","doi-asserted-by":"crossref","unstructured":"Stokel-Walker, C. AI bot ChatGPT writes smart essays\u2014Should professors worry? Nature. https:\/\/www.nature.com\/articles\/d41586-022-04397-7 (2022).","DOI":"10.1038\/d41586-022-04397-7"},{"key":"41032_CR9","unstructured":"Terry, O. K. I'm a student. You have no idea how much we're using ChatGPT. Chron. High. Educ. https:\/\/www.chronicle.com\/article\/im-a-student-you-have-no-idea-how-much-were-using-chatgpt (2023)."},{"key":"41032_CR10","unstructured":"Chiang, T. ChatGPT is a blurry jpeg of the Web. New Yorker. https:\/\/www.newyorker.com\/tech\/annals-of-technology\/chatgpt-is-a-blurry-jpeg-of-the-web (2023)."},{"key":"41032_CR11","unstructured":"Heaven, W. D. Chatbots could one day replace search engines; Here\u2019s why that\u2019s a terrible idea. MIT Technol. Rev. https:\/\/www.technologyreview.com\/2022\/03\/29\/1048439\/chatbots-replace-search-engine-terrible-idea\/ (2022)."},{"key":"41032_CR12","doi-asserted-by":"publisher","first-page":"e35179","DOI":"10.7759\/cureus.35179","volume":"15","author":"H Alkaissi","year":"2023","unstructured":"Alkaissi, H. & McFarlane, S. I. Artificial hallucinations in ChatGPT: Implications in scientific writing. Cureus 15, e35179. https:\/\/doi.org\/10.7759\/cureus.35179 (2023).","journal-title":"Cureus"},{"key":"41032_CR13","doi-asserted-by":"publisher","DOI":"10.1101\/2023.03.16.23286914(24Mar","author":"J Gravel","year":"2023","unstructured":"Gravel, J., D\u2019Amours-Gravel, M. & Osmanlliu, E. Learning to fake it: Limited responses and fabricated references provided by ChatGPT for medical questions. Mayo Clin. Proc. Digit. Health https:\/\/doi.org\/10.1101\/2023.03.16.23286914(24Mar (2023).","journal-title":"Mayo Clin. Proc. Digit. Health"},{"key":"41032_CR14","unstructured":"OpenAI. GPT-4 technical report. https:\/\/paperswithcode.com\/paper\/gpt-4-technical-report-1 (2023)."},{"key":"41032_CR15","doi-asserted-by":"publisher","DOI":"10.1177\/08465371231171125","author":"MW Wagner","year":"2023","unstructured":"Wagner, M. W. & Ertl-Wagner, B. B. Accuracy of information and references using ChatGPT-3 for retrieval of clinical radiological information. Can. Assoc. Radiol. J. https:\/\/doi.org\/10.1177\/08465371231171125 (2023).","journal-title":"Can. Assoc. Radiol. J."},{"key":"41032_CR16","doi-asserted-by":"publisher","first-page":"e46885","DOI":"10.2196\/46885","volume":"9","author":"G Eysenbach","year":"2023","unstructured":"Eysenbach, G. The role of ChatGPT, generative language models, and artificial intelligence in medical education: A conversation with ChatGPT and a call for papers. JMIR Med. Educ. 9, e46885. https:\/\/doi.org\/10.2196\/46885 (2023).","journal-title":"JMIR Med. Educ."},{"key":"41032_CR17","unstructured":"Fried, E. I. Using GPT-3 to search for scientific \u201creferences\u201d. https:\/\/eiko-fried.com\/using-gpt-3-to-search-scientific-references\/ (2023)."},{"key":"41032_CR18","doi-asserted-by":"publisher","DOI":"10.1007\/s10439-023-03248-4","author":"A Frosolini","year":"2023","unstructured":"Frosolini, A., Gennaro, P., Cascino, F. & Gabriele, G. In reference to \u201cRole of ChatGPT in public health\u201d, to highlight the AI\u2019s incorrect reference generation. Ann. Biomed. Eng. https:\/\/doi.org\/10.1007\/s10439-023-03248-4 (2023).","journal-title":"Ann. Biomed. Eng."},{"key":"41032_CR19","doi-asserted-by":"publisher","first-page":"ep421","DOI":"10.30935\/cedtech\/13036","volume":"15","author":"M Halaweh","year":"2023","unstructured":"Halaweh, M. ChatGPT in education: Strategies for responsible implementation. Contemp. Educ. Technol. 15, ep421. https:\/\/doi.org\/10.30935\/cedtech\/13036 (2023).","journal-title":"Contemp. Educ. Technol."},{"key":"41032_CR20","doi-asserted-by":"publisher","first-page":"13","DOI":"10.1186\/s40902-023-00381-x","volume":"45","author":"SG Kim","year":"2023","unstructured":"Kim, S. G. Using ChatGPT for language editing in scientific articles. Maxillofac. Plast. Reconstr. Surg. 45, 13. https:\/\/doi.org\/10.1186\/s40902-023-00381-x (2023).","journal-title":"Maxillofac. Plast. Reconstr. Surg."},{"key":"41032_CR21","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1007\/s12195-022-00754-8","volume":"16","author":"MR King","year":"2023","unstructured":"King, M. R. A conversation on artificial intelligence, chatbots, and plagiarism in higher education. Cell. Mol. Bioeng. 16, 1\u20132 (2023).","journal-title":"Cell. Mol. Bioeng."},{"key":"41032_CR22","doi-asserted-by":"publisher","DOI":"10.1016\/j.ajog.2023.04.004","author":"L Sanchez-Ramos","year":"2023","unstructured":"Sanchez-Ramos, L., Lin, L. & Romero, R. Beware of references when using ChatGPT as a source of information to write scientific articles. Am. J. Obstet. Gynecol. https:\/\/doi.org\/10.1016\/j.ajog.2023.04.004 (2023).","journal-title":"Am. J. Obstet. Gynecol."},{"key":"41032_CR23","doi-asserted-by":"publisher","first-page":"4879","DOI":"10.3390\/s23104879","volume":"23","author":"Z Tafferner","year":"2023","unstructured":"Tafferner, Z., Bal\u00e1zs, I., Krammer, O. & G\u00e9czy, O. Can ChatGPT help in electronics research and development? A case study with applied sensors. Sensors (Basel) 23, 4879. https:\/\/doi.org\/10.3390\/s23104879 (2023).","journal-title":"Sensors (Basel)"},{"key":"41032_CR24","unstructured":"Weiser, B. Here\u2019s what happens when your lawyer uses ChatGPT. N.Y. Times Web. https:\/\/www.nytimes.com\/2023\/05\/27\/nyregion\/avianca-airline-lawsuit-chatgpt.html (2023)."},{"key":"41032_CR25","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.4334162","author":"J Willems","year":"2023","unstructured":"Willems, J. ChatGPT at universities\u2014The least of our concerns. SSRN J. https:\/\/doi.org\/10.2139\/ssrn.4334162 (2023).","journal-title":"SSRN J."},{"key":"41032_CR26","doi-asserted-by":"publisher","first-page":"e37432","DOI":"10.7759\/cureus.37432","volume":"15","author":"SA Athaluri","year":"2023","unstructured":"Athaluri, S. A. et al. Exploring the boundaries of reality: Investigating the phenomenon of artificial intelligence hallucination in scientific writing through ChatGPT references. Cureus 15, e37432. https:\/\/doi.org\/10.7759\/cureus.37432 (2023).","journal-title":"Cureus"},{"key":"41032_CR27","doi-asserted-by":"publisher","first-page":"e39238","DOI":"10.7759\/cureus.39238","volume":"15","author":"M Bhattacharyya","year":"2023","unstructured":"Bhattacharyya, M., Miller, V. M., Bhattacharyya, D. & Miller, L. E. High rates of fabricated and inaccurate references in ChatGPT-generated medical content. Cureus 15, e39238. https:\/\/doi.org\/10.7759\/cureus.39238 (2023).","journal-title":"Cureus"},{"key":"41032_CR28","doi-asserted-by":"publisher","DOI":"10.1080\/00330124.2023.2190373","author":"T Day","year":"2023","unstructured":"Day, T. A preliminary investigation of fake peer-reviewed citations and references generated by ChatGPT. Prof. Geogr. https:\/\/doi.org\/10.1080\/00330124.2023.2190373 (2023).","journal-title":"Prof. Geogr."},{"key":"41032_CR29","doi-asserted-by":"publisher","first-page":"e003248","DOI":"10.1136\/rmdopen-2023-003248","volume":"9","author":"AJ Hueber","year":"2023","unstructured":"Hueber, A. J. & Kleyer, A. Quality of citation data using the natural language processing tool ChatGPT in rheumatology: Creation of false references. RMD Open 9, e003248. https:\/\/doi.org\/10.1136\/rmdopen-2023-003248 (2023).","journal-title":"RMD Open"},{"key":"41032_CR30","unstructured":"Atlas, S. Chatbot Prompting: A Guide for Students, Educators, and an AI-Augmented Workforce (S. Atlas, 2023). https:\/\/www.researchgate.net\/publication\/367464129_Chatbot_Prompting_A_guide_for_students_educators_and_an_AI-augmented_workforce."},{"key":"41032_CR31","doi-asserted-by":"publisher","first-page":"1553","DOI":"10.1016\/j.ipm.2006.03.019","volume":"42","author":"J Bar-Ilan","year":"2006","unstructured":"Bar-Ilan, J. An ego-centric citation analysis of the works of Michael O. Rabin based on multiple citation indexes. Inf. Process. Manag. 42, 1553\u20131566 (2006).","journal-title":"Inf. Process. Manag."},{"key":"41032_CR32","first-page":"1537","volume":"89","author":"P Jacs\u00f3","year":"2005","unstructured":"Jacs\u00f3, P. As we may search: Comparison of major features of the Web of Science, Scopus, and Google Scholar citation-based and citation-enhanced databases. Curr. Sci. 89, 1537\u20131547 (2005).","journal-title":"Curr. Sci."},{"key":"41032_CR33","doi-asserted-by":"publisher","first-page":"297","DOI":"10.1108\/14684520610675816","volume":"30","author":"P Jacs\u00f3","year":"2006","unstructured":"Jacs\u00f3, P. Deflated, inflated and phantom citation counts. Online Inf. Rev. 30, 297\u2013309 (2006).","journal-title":"Online Inf. Rev."},{"key":"41032_CR34","unstructured":"Beall, J. Beall's list of potential predatory journals and publishers. https:\/\/beallslist.net\/ (2023)."},{"key":"41032_CR35","doi-asserted-by":"publisher","first-page":"5351","DOI":"10.1007\/s11192-023-04804-4","volume":"128","author":"E Ordu\u00f1a-Malea","year":"2023","unstructured":"Ordu\u00f1a-Malea, E. & Cabezas-Clavijo, \u00c1. ChatGPT and the potential growing of ghost bibliographic references. Scientometrics 128, 5351\u20135355 (2023).","journal-title":"Scientometrics"},{"key":"41032_CR36","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2304.02819","author":"W Liang","year":"2023","unstructured":"Liang, W., Yuksekgonul, M., Mao, Y., Wu, E. & Zou, J. GPT detectors are biased against non-native English writers. Patterns https:\/\/doi.org\/10.48550\/arXiv.2304.02819 (2023).","journal-title":"Patterns"},{"key":"41032_CR37","doi-asserted-by":"publisher","unstructured":"Pegoraro, A., Kumari, K., Fereidooni, H. & Sadeghi, A.-R. To ChatGPT, or not to ChatGPT: That is the question! https:\/\/doi.org\/10.48550\/arXiv.2304.01487 (2023).","DOI":"10.48550\/arXiv.2304.01487"},{"key":"41032_CR38","doi-asserted-by":"crossref","unstructured":"Walters, W. H. The effectiveness of software designed to detect AI-generated writing: A comparison of 16 AI text detectors. https:\/\/www.researchgate.net\/publication\/372767498_The_effectiveness_of_software_designed_to_detect_AI-generated_writing_A_comparison_of_16_AI_text_detectors (2023).","DOI":"10.1515\/opis-2022-0158"},{"key":"41032_CR39","doi-asserted-by":"publisher","unstructured":"Wang, J., Liu, S., Xie, X. & Li, Y. Evaluating AIGC detectors on code content. https:\/\/doi.org\/10.48550\/arXiv.2304.05193 (2023).","DOI":"10.48550\/arXiv.2304.05193"},{"key":"41032_CR40","doi-asserted-by":"publisher","unstructured":"Weber-Wulff, D. et al. Testing of detection tools for AI-generated text. https:\/\/doi.org\/10.48550\/arXiv.2306.15666 (2023).","DOI":"10.48550\/arXiv.2306.15666"},{"key":"41032_CR41","unstructured":"Ouyang, L. et al. Training language models to follow instructions with human feedback, in Advances in Neural Information Processing Systems 35: 36th Conference on Neural Information Processing Systems (NeurIPS 2022) (ed. Koyejo. S. et al.) (Neural Information Processing Systems Foundation, 2022); https:\/\/proceedings.neurips.cc\/paper_files\/paper\/2022\/hash\/b1efde53be364a73914f58805a001731-Abstract-Conference.html."},{"key":"41032_CR42","unstructured":"Ramponi, M. How ChatGPT actually works. AssemblyAI. https:\/\/www.assemblyai.com\/blog\/how-chatgpt-actually-works\/ (2022)."},{"key":"41032_CR43","doi-asserted-by":"publisher","first-page":"363","DOI":"10.1111\/j.1467-7687.2009.00906.x","volume":"13","author":"SAJ Birch","year":"2010","unstructured":"Birch, S. A. J., Akmal, N. & Frampton, K. L. Two-year-olds are vigilant of others\u2019 non-verbal cues to credibility. Dev. Sci. 13, 363\u2013369 (2010).","journal-title":"Dev. Sci."},{"key":"41032_CR44","doi-asserted-by":"publisher","first-page":"346","DOI":"10.1207\/s15327957pspr0104_5","volume":"1","author":"BM DePaulo","year":"1997","unstructured":"DePaulo, B. M., Charlton, K., Cooper, H., Lindsay, J. J. & Muhlenbruck, L. The accuracy-confidence correlation in the detection of deception. Pers. Soc. Psychol. Rev. 1, 346\u2013357 (1997).","journal-title":"Pers. Soc. Psychol. Rev."},{"key":"41032_CR45","doi-asserted-by":"publisher","first-page":"39","DOI":"10.1002\/bdm.460","volume":"17","author":"PC Price","year":"2004","unstructured":"Price, P. C. & Stone, E. R. Intuitive evaluation of likelihood judgment producers: Evidence for a confidence heuristic. J. Behav. Decis. Mak. 17, 39\u201357 (2004).","journal-title":"J. Behav. Decis. Mak."}],"container-title":["Scientific Reports"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s41598-023-41032-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41598-023-41032-5","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s41598-023-41032-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,11,19]],"date-time":"2023-11-19T08:14:09Z","timestamp":1700381649000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s41598-023-41032-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,9,7]]},"references-count":45,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2023,12]]}},"alternative-id":["41032"],"URL":"https:\/\/doi.org\/10.1038\/s41598-023-41032-5","relation":{},"ISSN":["2045-2322"],"issn-type":[{"value":"2045-2322","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,9,7]]},"assertion":[{"value":"11 July 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"21 August 2023","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"7 September 2023","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"The authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"14045"}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1038_2Fs42256_024_00832_8 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T14:30:31Z","timestamp":1793284231973,"version":"4.5.0"},"reference-count":103,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2024,5,8]],"date-time":"2024-05-08T00:00:00Z","timestamp":1715126400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2024,5,8]],"date-time":"2024-05-08T00:00:00Z","timestamp":1715126400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"DOI":"10.13039\/501100001711","name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung","doi-asserted-by":"publisher","award":["180544"],"award-info":[{"award-number":["180544"]}],"id":[{"id":"10.13039\/501100001711","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001711","name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung","doi-asserted-by":"publisher","award":["180544"],"award-info":[{"award-number":["180544"]}],"id":[{"id":"10.13039\/501100001711","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001711","name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung","doi-asserted-by":"publisher","award":["180544"],"award-info":[{"award-number":["180544"]}],"id":[{"id":"10.13039\/501100001711","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["1751471"],"award-info":[{"award-number":["1751471"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["1751471"],"award-info":[{"award-number":["1751471"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Nat Mach Intell"],"abstract":"<jats:title>Abstract<\/jats:title>\n <jats:p>Large language models (LLMs) have shown strong performance in tasks across domains but struggle with chemistry-related problems. These models also lack access to external knowledge sources, limiting their usefulness in scientific applications. We introduce ChemCrow, an LLM chemistry agent designed to accomplish tasks across organic synthesis, drug discovery and materials design. By integrating 18 expert-designed tools and using GPT-4 as the LLM, ChemCrow augments the LLM performance in chemistry, and new capabilities emerge. Our agent autonomously planned and executed the syntheses of an insect repellent and three organocatalysts and guided the discovery of a novel chromophore. Our evaluation, including both LLM and expert assessments, demonstrates ChemCrow\u2019s effectiveness in automating a diverse set of chemical tasks. Our work not only aids expert chemists and lowers barriers for non-experts but also fosters scientific advancement by bridging the gap between experimental and computational chemistry.<\/jats:p>","DOI":"10.1038\/s42256-024-00832-8","type":"journal-article","created":{"date-parts":[[2024,5,8]],"date-time":"2024-05-08T06:03:31Z","timestamp":1715148211000},"page":"525-535","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":840,"title":["Augmenting large language models with chemistry tools"],"prefix":"10.1038","volume":"6","author":[{"given":"Andres","family":"M. Bran","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sam","family":"Cox","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0310-0851","authenticated-orcid":false,"given":"Oliver","family":"Schilter","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Carlo","family":"Baldassari","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6647-3965","authenticated-orcid":false,"given":"Andrew D.","family":"White","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3046-6576","authenticated-orcid":false,"given":"Philippe","family":"Schwaller","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2024,5,8]]},"reference":[{"key":"832_CR1","unstructured":"Devlin, J., Chang, M.-W., Lee, K. & Toutanova, K. Bert: pre-training of deep bidirectional transformers for language understanding. In Proc. Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (eds Burstein, J. et al.) 4171\u20134186 (Association for Computational Linguistics, 2019)."},{"key":"832_CR2","first-page":"1877","volume":"33","author":"T Brown","year":"2020","unstructured":"Brown, T. et al. Language models are few-shot learners. Adv. Neural Inf. Process. Syst. 33, 1877\u20131901 (2020).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"832_CR3","unstructured":"Bommasani, R. et al. On the opportunities and risks of foundation models. Preprint at https:\/\/arxiv.org\/abs\/2108.07258 (2021)."},{"key":"832_CR4","first-page":"1","volume":"24","author":"A Chowdhery","year":"2023","unstructured":"Chowdhery, A. et al. Palm: scaling language modeling with pathways. J. Mach. Learn. Res. 24, 1\u2013113 (2023).","journal-title":"J. Mach. Learn. Res."},{"key":"832_CR5","unstructured":"Bubeck, S. et al. Sparks of artificial general intelligence: early experiments with gpt-4. Preprint at https:\/\/arxiv.org\/abs\/2303.12712 (2023)."},{"key":"832_CR6","unstructured":"Github Copilot. GitHub https:\/\/copilot.github.com (2023)."},{"key":"832_CR7","unstructured":"Li, R. et al. Starcoder: may the source be with you! Trans. Mach. Learn. Res. https:\/\/openreview.net\/pdf?id=KoFOg41haE (2023)."},{"key":"832_CR8","doi-asserted-by":"crossref","unstructured":"Ziegler, A. et al. Productivity assessment of neural code completion. In Proc. 6th ACM SIGPLAN International Symposium on Machine Programming (eds Chaudhuri, S. and Sutton, C.) 21\u201329 (ACM, 2022).","DOI":"10.1145\/3520312.3534864"},{"key":"832_CR9","unstructured":"Vaswani, A. et al. Attention is all you need. In Proc. Advances in Neural Information Processing Systems 30 (eds. Guyon, I. et al.) 5999\u20136009 (Curran Associates, 2017)."},{"key":"832_CR10","unstructured":"Schick, T. et al. Toolformer: language models can teach themselves to use tools. In Proc. Advances in Neural Information Processing Systems 36 (eds. Oh, A. et al.) 68539\u201368551 (Curran Associates, 2023)."},{"key":"832_CR11","doi-asserted-by":"publisher","first-page":"1649","DOI":"10.1021\/acs.jcim.3c00285","volume":"63","author":"CM Castro Nascimento","year":"2023","unstructured":"Castro Nascimento, C. M. & Pimentel, A. S. Do large language models understand chemistry? A conversation with ChatGPT. J. Chem. Inf. Model. 63, 1649\u20131655 (2023).","journal-title":"J. Chem. Inf. Model."},{"key":"832_CR12","unstructured":"OpenAI. GPT-4 technical report. Preprint at https:\/\/arxiv.org\/abs\/2303.08774 (2023)."},{"key":"832_CR13","first-page":"27730","volume":"35","author":"L Ouyang","year":"2022","unstructured":"Ouyang, L. et al. Training language models to follow instructions with human feedback. Adv. Neural Inf. Process. Syst. 35, 27730\u201327744 (2022).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"832_CR14","doi-asserted-by":"publisher","first-page":"368","DOI":"10.1039\/D2DD00087C","volume":"2","author":"AD White","year":"2023","unstructured":"White, A. D. et al. Assessment of chemistry knowledge in large language models that generate code. Digit. Discov. 2, 368\u2013376 (2023).","journal-title":"Digit. Discov."},{"key":"832_CR15","doi-asserted-by":"publisher","first-page":"739","DOI":"10.1021\/ci100384d","volume":"51","author":"DM Lowe","year":"2011","unstructured":"Lowe, D. M., Corbett, P. T., Murray-Rust, P. & Glen, R. C. Chemical name to structure: Opsin, an open source solution. J. Chem. Inf. Model. 51, 739\u2013753 (2011).","journal-title":"J. Chem. Inf. Model."},{"key":"832_CR16","doi-asserted-by":"publisher","first-page":"434","DOI":"10.1021\/acscentsci.7b00064","volume":"3","author":"CW Coley","year":"2017","unstructured":"Coley, C. W., Barzilay, R., Jaakkola, T. S., Green, W. H. & Jensen, K. F. Prediction of organic reaction outcomes using machine learning. ACS Cent. Sci. 3, 434\u2013443 (2017).","journal-title":"ACS Cent. Sci."},{"key":"832_CR17","doi-asserted-by":"publisher","first-page":"370","DOI":"10.1039\/C8SC04228D","volume":"10","author":"CW Coley","year":"2019","unstructured":"Coley, C. W. et al. A graph-convolutional neural network model for the prediction of chemical reactivity. Chem. Sci. 10, 370\u2013377 (2019).","journal-title":"Chem. Sci."},{"key":"832_CR18","doi-asserted-by":"publisher","first-page":"1572","DOI":"10.1021\/acscentsci.9b00576","volume":"5","author":"P Schwaller","year":"2019","unstructured":"Schwaller, P. et al. Molecular transformer: a model for uncertainty-calibrated chemical reaction prediction. ACS Cent. Sci. 5, 1572\u20131583 (2019).","journal-title":"ACS Cent. Sci."},{"key":"832_CR19","doi-asserted-by":"publisher","first-page":"4874","DOI":"10.1038\/s41467-020-18671-7","volume":"11","author":"G Pesciullesi","year":"2020","unstructured":"Pesciullesi, G., Schwaller, P., Laino, T. & Reymond, J.-L. Transfer learning enables the molecular transformer to predict regio-and stereoselective reactions on carbohydrates. Nat. Commun. 11, 4874 (2020).","journal-title":"Nat. Commun."},{"key":"832_CR20","doi-asserted-by":"publisher","first-page":"015022","DOI":"10.1088\/2632-2153\/ac3ffb","volume":"3","author":"R Irwin","year":"2022","unstructured":"Irwin, R., Dimitriadis, S., He, J. & Bjerrum, E. J. Chemformer: a pre-trained transformer for computational chemistry. Mach. Learn. Sci.Technol. 3, 015022 (2022).","journal-title":"Mach. Learn. Sci.Technol."},{"key":"832_CR21","doi-asserted-by":"publisher","first-page":"5904","DOI":"10.1002\/anie.201506101","volume":"55","author":"S Szymkuc","year":"2016","unstructured":"Szymkuc, S. et al. Computer-assisted synthetic planning: the end of the beginning. Angew. Chem. Int. Ed. Engl. 55, 5904\u20135937 (2016).","journal-title":"Angew. Chem. Int. Ed. Engl."},{"key":"832_CR22","doi-asserted-by":"publisher","first-page":"604","DOI":"10.1038\/nature25978","volume":"555","author":"MH Segler","year":"2018","unstructured":"Segler, M. H., Preuss, M. & Waller, M. P. Planning chemical syntheses with deep neural networks and symbolic AI. Nature 555, 604\u2013610 (2018).","journal-title":"Nature"},{"key":"832_CR23","doi-asserted-by":"crossref","unstructured":"Coley, C. W. et al. A robotic platform for flow synthesis of organic compounds informed by AI planning. Science 365 (2019).","DOI":"10.1126\/science.aax1566"},{"key":"832_CR24","doi-asserted-by":"publisher","first-page":"3316","DOI":"10.1039\/C9SC05704H","volume":"11","author":"P Schwaller","year":"2020","unstructured":"Schwaller, P. et al. Predicting retrosynthetic pathways using transformer-based models and a hyper-graph exploration strategy. Chem. Sci. 11, 3316\u20133325 (2020).","journal-title":"Chem. Sci."},{"key":"832_CR25","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s13321-020-00472-1","volume":"12","author":"S Genheden","year":"2020","unstructured":"Genheden, S. et al. AiZynthFinder: a fast, robust and flexible open-source software for retrosynthetic planning. J. Cheminf. 12, 1\u20139 (2020).","journal-title":"J. Cheminf."},{"key":"832_CR26","doi-asserted-by":"publisher","first-page":"1094","DOI":"10.1021\/acs.accounts.0c00714","volume":"54","author":"K Molga","year":"2021","unstructured":"Molga, K., Szymkuc, S. & Grzybowski, B. A. Chemist ex machina: advanced synthesis planning by computers. Acc. Chem. Res. 54, 1094\u20131106 (2021).","journal-title":"Acc. Chem. Res."},{"key":"832_CR27","doi-asserted-by":"publisher","first-page":"e1604","DOI":"10.1002\/wcms.1604","volume":"12","author":"P Schwaller","year":"2022","unstructured":"Schwaller, P. et al. Machine intelligence for chemical reaction space. Wiley Interdiscip. Rev. Comput. Mol. Sci. 12, e1604 (2022).","journal-title":"Wiley Interdiscip. Rev. Comput. Mol. Sci."},{"key":"832_CR28","doi-asserted-by":"publisher","first-page":"80","DOI":"10.3389\/fenvs.2015.00080","volume":"3","author":"A Mayr","year":"2016","unstructured":"Mayr, A., Klambauer, G., Unterthiner, T. & Hochreiter, S. Deeptox: toxicity prediction using deep learning. Front. Environ. Sci. 3, 80 (2016).","journal-title":"Front. Environ. Sci."},{"key":"832_CR29","doi-asserted-by":"publisher","first-page":"3370","DOI":"10.1021\/acs.jcim.9b00237","volume":"59","author":"K Yang","year":"2019","unstructured":"Yang, K. et al. Analyzing learned molecular representations for property prediction. J. Chem. Inf. Model. 59, 3370\u20133388 (2019).","journal-title":"J. Chem. Inf. Model."},{"key":"832_CR30","unstructured":"Chithrananda, S., Grand, G. & Ramsundar, B. Chemberta: large-scale self-supervised pretraining for molecular property prediction. Preprint at https:\/\/arxiv.org\/abs\/2010.09885 (2020)."},{"key":"832_CR31","doi-asserted-by":"publisher","first-page":"5938","DOI":"10.1021\/acs.jcim.2c01073","volume":"62","author":"D van Tilborg","year":"2022","unstructured":"van Tilborg, D., Alenicheva, A. & Grisoni, F. Exposing the limitations of molecular machine learning with activity cliffs. J. Chem. Inf. Model. 62, 5938\u20135951 (2022).","journal-title":"J. Chem. Inf. Model."},{"key":"832_CR32","doi-asserted-by":"publisher","first-page":"161","DOI":"10.1038\/s42256-023-00788-1","volume":"6","author":"KM Jablonka","year":"2024","unstructured":"Jablonka, K. M., Schwaller, P., Ortega-Guerrero, A. & Smit, B. Leveraging large language models for predictive chemistry. Nat. Mach. Intell. 6, 161\u2013169 (2024).","journal-title":"Nat. Mach. Intell."},{"key":"832_CR33","doi-asserted-by":"publisher","first-page":"268","DOI":"10.1021\/acscentsci.7b00572","volume":"4","author":"R G\u00f3mez-Bombarelli","year":"2018","unstructured":"G\u00f3mez-Bombarelli, R. et al. Automatic chemical design using a data-driven continuous representation of molecules. ACS Cent. Sci. 4, 268\u2013276 (2018).","journal-title":"ACS Cent. Sci."},{"key":"832_CR34","doi-asserted-by":"publisher","first-page":"5918","DOI":"10.1021\/acs.jcim.0c00915","volume":"60","author":"T Blaschke","year":"2020","unstructured":"Blaschke, T. et al. Reinvent 2.0: an AI tool for de novo drug design. J. Chem. Inf. Model. 60, 5918\u20135922 (2020).","journal-title":"J. Chem. Inf. Model."},{"key":"832_CR35","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1038\/s41524-021-00495-8","volume":"7","author":"Q Tao","year":"2021","unstructured":"Tao, Q., Xu, P., Li, M. & Lu, W. Machine learning for perovskite materials design and discovery. NPJ Comput. Mater. 7, 1\u201318 (2021).","journal-title":"NPJ Comput. Mater."},{"key":"832_CR36","doi-asserted-by":"publisher","first-page":"1120","DOI":"10.1038\/nmat4717","volume":"15","author":"R G\u00f3mez-Bombarelli","year":"2016","unstructured":"G\u00f3mez-Bombarelli, R. et al. Design of efficient molecular organic light-emitting diodes by a high-throughput virtual screening and experimental approach. Nat. Mater. 15, 1120\u20131127 (2016).","journal-title":"Nat. Mater."},{"key":"832_CR37","doi-asserted-by":"publisher","first-page":"89","DOI":"10.1038\/s41586-021-03213-y","volume":"590","author":"BJ Shields","year":"2021","unstructured":"Shields, B. J. et al. Bayesian reaction optimization as a tool for chemical synthesis. Nature 590, 89\u201396 (2021).","journal-title":"Nature"},{"key":"832_CR38","doi-asserted-by":"publisher","first-page":"19999","DOI":"10.1021\/jacs.2c08592","volume":"144","author":"JAG Torres","year":"2022","unstructured":"Torres, J. A. G. et al. A multi-objective active learning platform and web app for reaction optimization. J. Am. Chem. Soc. 144, 19999\u201320007 (2022).","journal-title":"J. Am. Chem. Soc."},{"key":"832_CR39","unstructured":"Ramos, M. C., Michtavy, S. S., Porosoff, M. D. & White, A. D. Bayesian optimization of catalysts with in-context learning. Preprint at https:\/\/arxiv.org\/abs\/2304.05341 (2023)."},{"key":"832_CR40","doi-asserted-by":"crossref","unstructured":"Marra, G., Giannini, F., Diligenti, M. & Gori, M. Integrating learning and reasoning with deep logic models. In Proc. Machine Learning and Knowledge Discovery in Databases, Part II (eds. Hutter, F. et al.) 517\u2013532 (Springer, 2020).","DOI":"10.1007\/978-3-030-46147-8_31"},{"key":"832_CR41","first-page":"24824","volume":"35","author":"J Wei","year":"2022","unstructured":"Wei, J. et al. Chain-of-thought prompting elicits reasoning in large language models. Adv. Neural Inf. Process. Syst. 35, 24824\u201324837 (2022).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"832_CR42","doi-asserted-by":"crossref","unstructured":"Ho, N., Schmid, L. & Yun, S.-Y. Large language models are reasoning teachers. In Proc. 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers) (eds. Rogers, A. et al.) 14852\u201314882 (ACL, 2023).","DOI":"10.18653\/v1\/2023.acl-long.830"},{"key":"832_CR43","unstructured":"Yao, S. et al. ReAct: synergizing reasoning and acting in language models. In Proc. 11th International Conference on Learning Representations (OpenReview, 2023)."},{"key":"832_CR44","first-page":"15476","volume":"35","author":"E Zelikman","year":"2022","unstructured":"Zelikman, E., Wu, Y., Mu, J. & Goodman, N. Star: bootstrapping reasoning with reasoning. Adv. Neural Inf. Process. Syst. 35, 15476\u201315488 (2022).","journal-title":"Adv. Neural Inf. Process. Syst."},{"key":"832_CR45","doi-asserted-by":"publisher","first-page":"266","DOI":"10.1039\/D2DD00004K","volume":"1","author":"Z-W Zhao","year":"2022","unstructured":"Zhao, Z.-W., del Cueto, M. & Troisi, A. Limitations of machine learning models when predicting compounds with completely new chemistries: possible improvements applied to the discovery of new non-fullerene acceptors. Digit. Discov. 1, 266\u2013276 (2022).","journal-title":"Digit. Discov."},{"key":"832_CR46","doi-asserted-by":"publisher","DOI":"10.1038\/s41467-021-22951-1","volume":"12","author":"AC Vaucher","year":"2021","unstructured":"Vaucher, A. C. et al. Inferring experimental procedures from text-based representations of chemical reactions. Nat. Commun. 12, 2573 (2021).","journal-title":"Nat. Commun."},{"key":"832_CR47","doi-asserted-by":"publisher","first-page":"144","DOI":"10.1038\/s42256-020-00284-w","volume":"3","author":"P Schwaller","year":"2021","unstructured":"Schwaller, P. et al. Mapping the space of chemical reactions using attention-based neural networks. Nat. Mach. Intell. 3, 144\u2013152 (2021).","journal-title":"Nat. Mach. Intell."},{"key":"832_CR48","unstructured":"RXN for Chemistry. rxn4Chemistry. GitHub https:\/\/github.com\/rxn4chemistry\/rxn4chemistry (2020)."},{"key":"832_CR49","doi-asserted-by":"publisher","first-page":"154","DOI":"10.1039\/C9SC04944D","volume":"11","author":"A Thakkar","year":"2020","unstructured":"Thakkar, A., Kogej, T., Reymond, J.-L., Engkvist, O. & Bjerrum, E. J. Datasets and their influence on the development of computer assisted synthesis planning tools in the pharmaceutical domain. Chem. Sci. 11, 154\u2013168 (2020).","journal-title":"Chem. Sci."},{"key":"832_CR50","doi-asserted-by":"publisher","first-page":"8791","DOI":"10.1021\/acs.jmedchem.9b01919","volume":"63","author":"A Thakkar","year":"2020","unstructured":"Thakkar, A., Selmi, N., Reymond, J.-L., Engkvist, O. & Bjerrum, E. J. \u2018Ring breaker\u2019: neural network driven synthesis prediction of the ring system chemical space. J. Med. Chem. 63, 8791\u20138808 (2020).","journal-title":"J. Med. Chem."},{"key":"832_CR51","unstructured":"Yang, Z. et al. Mm-react: prompting ChatGPT for multimodal reasoning and action. Preprint at https:\/\/arxiv.org\/abs\/2303.11381 (2023)."},{"key":"832_CR52","unstructured":"Shen, Y. et al. Hugginggpt: solving AI tasks with chatgpt and its friends in huggingface. Poster at Advances in Neural Information Processing Systems 36 (2023)."},{"key":"832_CR53","unstructured":"Karpas, E. et al. Mrkl systems: a modular, neuro-symbolic architecture that combines large language models, external knowledge sources and discrete reasoning. Preprint at https:\/\/arxiv.org\/abs\/2205.00445 (2022)."},{"key":"832_CR54","doi-asserted-by":"publisher","first-page":"570","DOI":"10.1038\/s41586-023-06792-0","volume":"624","author":"DA Boiko","year":"2023","unstructured":"Boiko, D. A., MacKnight, R., Kline, B. & Gomes, G. Autonomous chemical research with large language models. Nature 624, 570\u2013578 (2023).","journal-title":"Nature"},{"key":"832_CR55","unstructured":"RoboRXN. IBM https:\/\/research.ibm.com\/science\/ibm-roborxn\/ (2021)."},{"key":"832_CR56","doi-asserted-by":"publisher","first-page":"407","DOI":"10.1002\/chem.200390042","volume":"9","author":"A Wittkopp","year":"2003","unstructured":"Wittkopp, A. & Schreiner, P. R. Metal-free, noncovalent catalysis of Diels-Alder reactions by neutral hydrogen bond donors in organic solvents and in water. Chem. Eur. J. 9, 407\u2013414 (2003).","journal-title":"Chem. Eur. J."},{"key":"832_CR57","doi-asserted-by":"publisher","first-page":"217","DOI":"10.1021\/ol017117s","volume":"4","author":"PR Schreiner","year":"2002","unstructured":"Schreiner, P. R. & Wittkopp, A. H-bonding additives act like Lewis acid catalysts. Org. Lett. 4, 217\u2013220 (2002).","journal-title":"Org. Lett."},{"key":"832_CR58","doi-asserted-by":"publisher","first-page":"6576","DOI":"10.1002\/anie.200500227","volume":"44","author":"RP Herrera","year":"2005","unstructured":"Herrera, R. P., Sgarzani, V., Bernardi, L. & Ricci, A. Catalytic enantioselective friedel-crafts alkylation of indoles with nitroalkenes by using a simple thiourea organocatalyst. Angew. Chem. Int. Ed. Engl. 44, 6576\u20136579 (2005).","journal-title":"Angew. Chem. Int. Ed. Engl."},{"key":"832_CR59","doi-asserted-by":"publisher","first-page":"12672","DOI":"10.1021\/ja036972z","volume":"125","author":"T Okino","year":"2003","unstructured":"Okino, T., Hoashi, Y. & Takemoto, Y. Enantioselective Michael reaction of malonates to nitroolefins catalyzed by bifunctional organocatalysts. J. Am. Chem. Soc. 125, 12672\u201312673 (2003).","journal-title":"J. Am. Chem. Soc."},{"key":"832_CR60","unstructured":"Joung, J. F., Han, M., Jeong, M. & Park, S. DB for chromophore. figshare https:\/\/figshare.com\/articles\/dataset\/DB_for_chromophore\/12045567 (2020)."},{"key":"832_CR61","unstructured":"Lowe, D. M. Extraction of Chemical Structures and Reactions from the Literature. PhD thesis, Univ. of Cambridge (2012)."},{"key":"832_CR62","doi-asserted-by":"publisher","first-page":"513","DOI":"10.1039\/C7SC02664A","volume":"9","author":"Z Wu","year":"2018","unstructured":"Wu, Z. et al. Moleculenet: a benchmark for molecular machine learning. Chem. Sci. 9, 513\u2013530 (2018).","journal-title":"Chem. Sci."},{"key":"832_CR63","doi-asserted-by":"crossref","unstructured":"Liu, Y. et al. G-Eval: NLG evaluation using GPT-4 with better human alignment. In Proc. Conference on Empirical Methods in Natural Language Processing (eds. Bouamor, H. et al.) 2511\u20132522 (ACL, 2023).","DOI":"10.18653\/v1\/2023.emnlp-main.153"},{"key":"832_CR64","unstructured":"Eloundou, T., Manning, S., Mishkin, P. & Rock, D. GPTs are GPTs: an early look at the labor market impact potential of large language models. Preprint at https:\/\/arxiv.org\/abs\/2303.10130 (2023)."},{"key":"832_CR65","doi-asserted-by":"publisher","first-page":"e1630","DOI":"10.1002\/wcms.1630","volume":"13","author":"BA Grzybowski","year":"2023","unstructured":"Grzybowski, B. A., Badowski, T., Molga, K. & Szymkuc, S. Network search algorithms and scoring functions for advanced-level computerized synthesis planning. Wiley Interdiscip. Rev. Comput. Mol. Sci. 13, e1630 (2023).","journal-title":"Wiley Interdiscip. Rev. Comput. Mol. Sci."},{"key":"832_CR66","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1039\/D0RE00340A","volume":"6","author":"A Thakkar","year":"2021","unstructured":"Thakkar, A. et al. Artificial intelligence and automation in computer aided synthesis planning. React. Chem. Eng. 6, 27\u201351 (2021).","journal-title":"React. Chem. Eng."},{"key":"832_CR67","doi-asserted-by":"publisher","first-page":"189","DOI":"10.1038\/s42256-022-00465-9","volume":"4","author":"F Urbina","year":"2022","unstructured":"Urbina, F., Lentzos, F., Invernizzi, C. & Ekins, S. Dual use of artificial-intelligence-powered drug discovery. Nat. Mach. Intell. 4, 189\u2013191 (2022).","journal-title":"Nat. Mach. Intell."},{"key":"832_CR68","doi-asserted-by":"publisher","first-page":"607","DOI":"10.1038\/s42256-022-00511-6","volume":"4","author":"F Urbina","year":"2022","unstructured":"Urbina, F., Lentzos, F., Invernizzi, C. & Ekins, S. A teachable moment for dual-use. Nat. Mach. Intell. 4, 607\u2013607 (2022).","journal-title":"Nat. Mach. Intell."},{"key":"832_CR69","unstructured":"Campbell, Q. L., Herington, J. & White, A. D. Censoring chemical data to mitigate dual use risk. Preprint at https:\/\/arxiv.org\/abs\/2304.10510 (2023)."},{"key":"832_CR70","unstructured":"Gao, L., Schulman, J. & Hilton, J. Scaling laws for reward model overoptimization. In Proc. International Conference on Machine Learning (eds Krause, A. et al.) 10835\u201310866 (PMLR, 2023)."},{"key":"832_CR71","unstructured":"Radford, A. et al. Improving language understanding by generative pre-training. OpenAI blog https:\/\/cdn.openai.com\/research-covers\/language-unsupervised\/language_understanding_paper.pdf (2018)."},{"key":"832_CR72","first-page":"1\u201346","volume":"55","author":"B Li","year":"2021","unstructured":"Li, B. et al. Trustworthy AI: from principles to practices. ACM Comput. Surv. 55, 1\u201346 (2021).","journal-title":"ACM Comput. Surv."},{"key":"832_CR73","doi-asserted-by":"publisher","first-page":"79","DOI":"10.1039\/D1DD00009H","volume":"1","author":"GM Hocky","year":"2022","unstructured":"Hocky, G. M. & White, A. D. Natural language processing models that automate programming will transform chemistry research and teaching. Dig. Discov. 1, 79\u201383 (2022).","journal-title":"Dig. Discov."},{"key":"832_CR74","doi-asserted-by":"crossref","unstructured":"Henderson, P. et al. Foundation models and fair use. Preprint at https:\/\/arxiv.org\/abs\/2303.15715 (2023).","DOI":"10.2139\/ssrn.4404340"},{"key":"832_CR75","unstructured":"Askell, A., Brundage, M. & Hadfield, G. The role of cooperation in responsible AI development. Preprint at https:\/\/arxiv.org\/abs\/1907.04534 (2019)."},{"key":"832_CR76","doi-asserted-by":"publisher","first-page":"101649","DOI":"10.1016\/j.techsoc.2021.101649","volume":"66","author":"RD Neufville","year":"2021","unstructured":"Neufville, R. D. & Baum, S. D. Collective action on artificial intelligence: a primer and review. Technol. Soc. 66, 101649 (2021).","journal-title":"Technol. Soc."},{"key":"832_CR77","unstructured":"Touvron, H. et al. Llama: open and efficient foundation language models. Preprint at https:\/\/arxiv.org\/abs\/2302.13971 (2023)."},{"key":"832_CR78","unstructured":"Chiang, W.-L. et al. Vicuna: an open-source chatbot impressing GPT-4 with 90%* ChatGPT quality. LMSYS Org. https:\/\/lmsys.org\/blog\/2023-03-30-vicuna\/ (2023)."},{"key":"832_CR79","unstructured":"Mukherjee, S. et al. Orca: progressive learning from complex explanation traces of GPT-4. Preprint at https:\/\/arxiv.org\/abs\/2306.02707 (2023)."},{"key":"832_CR80","unstructured":"Chase, H. LangChain. GitHub https:\/\/github.com\/hwchase17\/langchain (2022)."},{"key":"832_CR81","doi-asserted-by":"crossref","unstructured":"Press, O. et al. Measuring and narrowing the compositionality gap in language models. In Proc. Association for Computational Linguistics: EMNLP (eds. Bouamor, H. et al.) 5687\u20135711 (ACL, 2023).","DOI":"10.18653\/v1\/2023.findings-emnlp.378"},{"key":"832_CR82","unstructured":"Google search API. SerpApi https:\/\/serpapi.com\/ (2023)."},{"key":"832_CR83","unstructured":"Neelakantan, A. et al. Text and code embeddings by contrastive pre-training. Preprint at https:\/\/arxiv.org\/abs\/2201.10005 (2022)."},{"key":"832_CR84","doi-asserted-by":"publisher","first-page":"535","DOI":"10.1109\/TBDATA.2019.2921572","volume":"7","author":"J Johnson","year":"2019","unstructured":"Johnson, J., Douze, M. & J\u00e9gou, H. Billion-scale similarity search with GPUs. IEEE Trans. Big Data 7, 535\u2013547 (2019).","journal-title":"IEEE Trans. Big Data"},{"key":"832_CR85","unstructured":"ChemSpace https:\/\/chem-space.com\/ (2023)."},{"key":"832_CR86","unstructured":"National Center for Biotechnology Information. PubChem. NIH https:\/\/pubchem.ncbi.nlm.nih.gov\/ (2023)."},{"key":"832_CR87","doi-asserted-by":"publisher","first-page":"95","DOI":"10.1186\/s13321-023-00765-1","volume":"15","author":"J Medina","year":"2023","unstructured":"Medina, J. & White, A. D. Bloom filters for molecules. J. Cheminf. 15, 95 (2023).","journal-title":"J. Cheminf."},{"key":"832_CR88","doi-asserted-by":"publisher","first-page":"6065","DOI":"10.1021\/acs.jcim.0c00675","volume":"60","author":"JJ Irwin","year":"2020","unstructured":"Irwin, J. J. et al. Zinc20\u2014a free ultralarge-scale chemical database for ligand discovery. J. Chem. Inf. Model. 60, 6065\u20136073 (2020).","journal-title":"J. Chem. Inf. Model."},{"key":"832_CR89","unstructured":"Chemical Abstracts Service. CAS registry number. CAS www.cas.org\/content\/cas-registry (2023)."},{"key":"832_CR90","unstructured":"Tanimoto, T. T. An Elementary Mathematical Theory of Classification and Prediction (IBM, 1958)."},{"key":"832_CR91","doi-asserted-by":"publisher","first-page":"742","DOI":"10.1021\/ci100050t","volume":"50","author":"D Rogers","year":"2010","unstructured":"Rogers, D. & Hahn, M. Extended-connectivity fingerprints. J. Chem. Inf. Model. 50, 742\u2013754 (2010).","journal-title":"J. Chem. Inf. Model."},{"key":"832_CR92","unstructured":"White, A. D. Synspace. GitHub https:\/\/github.com\/whitead\/synspace (2023)."},{"key":"832_CR93","doi-asserted-by":"publisher","first-page":"3697","DOI":"10.1039\/D1SC05259D","volume":"13","author":"GP Wellawatte","year":"2022","unstructured":"Wellawatte, G. P., Seshadri, A. & White, A. D. Model agnostic generation of counterfactual explanations for molecules. Chem. Sci. 13, 3697\u20133705 (2022).","journal-title":"Chem. Sci."},{"key":"832_CR94","doi-asserted-by":"publisher","first-page":"3093","DOI":"10.1021\/ci200379p","volume":"51","author":"M Hartenfeller","year":"2011","unstructured":"Hartenfeller, M. et al. A collection of robust organic synthesis reactions for in silico molecule design. J. Chem. Inf. Model. 51, 3093\u20133098 (2011).","journal-title":"J. Chem. Inf. Model."},{"key":"832_CR95","doi-asserted-by":"publisher","first-page":"12152","DOI":"10.1039\/C9CC05122H","volume":"55","author":"Q Yang","year":"2019","unstructured":"Yang, Q. et al. Molecular transformer unifies reaction prediction and retrosynthesis across pharma chemical space. Chem. Commun. 55, 12152\u201312155 (2019).","journal-title":"Chem. Commun."},{"key":"832_CR96","unstructured":"Purchasable Mcule. Mcule https:\/\/purchasable.mcule.com\/ (2023)."},{"key":"832_CR97","unstructured":"RDKit: open-source cheminformatics (RDKit, 2023); www.rdkit.org"},{"key":"832_CR98","unstructured":"Chemical weapons convention, annex on chemicals, b. schedules of chemicals. OPCW www.opcw.org\/chemical-weapons-convention\/annexes\/annex-chemicals\/annex-chemicals (2024)."},{"key":"832_CR99","unstructured":"The Australia Group. Australia Group common control lists: chemical weapons precursors. Department of Foreign Affairs and Trade www.dfat.gov.au\/publications\/minisite\/theaustraliagroupnet\/site\/en\/controllists.html (2023)."},{"key":"832_CR100","unstructured":"Namerxn (NextMove Software, 2023); www.nextmovesoftware.com\/namerxn.html"},{"key":"832_CR101","doi-asserted-by":"publisher","first-page":"2337","DOI":"10.1039\/b602413k","volume":"4","author":"JS Carey","year":"2006","unstructured":"Carey, J. S., Laffan, D., Thomson, C. & Williams, M. T. Analysis of the reactions used for the preparation of drug candidate molecules. Org. Biomol. Chem. 4, 2337\u20132347 (2006).","journal-title":"Org. Biomol. Chem."},{"key":"832_CR102","doi-asserted-by":"publisher","unstructured":"Bran, A. & Cox, S. ur-whitelab\/chemcrow-runs: Zendo release. Zenodo https:\/\/doi.org\/10.5281\/zenodo.10884645 (2024).","DOI":"10.5281\/zenodo.10884645"},{"key":"832_CR103","doi-asserted-by":"publisher","unstructured":"Bran, A., Cox, S., White, A. & Schwaller, P. ur-whitelab\/chemcrow-public: v0.3.24. Zenodo https:\/\/doi.org\/10.5281\/zenodo.10884639 (2024).","DOI":"10.5281\/zenodo.10884639"}],"container-title":["Nature Machine Intelligence"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/s42256-024-00832-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s42256-024-00832-8","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/s42256-024-00832-8.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,5,23]],"date-time":"2024-05-23T19:03:31Z","timestamp":1716491011000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/s42256-024-00832-8"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5,8]]},"references-count":103,"journal-issue":{"issue":"5","published-online":{"date-parts":[[2024,5]]}},"alternative-id":["832"],"URL":"https:\/\/doi.org\/10.1038\/s42256-024-00832-8","relation":{},"ISSN":["2522-5839"],"issn-type":[{"value":"2522-5839","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,5,8]]},"assertion":[{"value":"13 September 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"27 March 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"8 May 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"A.D.W. has served as a paid consultant for evaluating AI model safety at OpenAI. The other authors declare no competing interests.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1038_2Fsdata_2016_18 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T16:09:22Z","timestamp":1793290162215,"version":"4.5.0"},"reference-count":32,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2016,3,15]],"date-time":"2016-03-15T00:00:00Z","timestamp":1458000000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2016,3,15]],"date-time":"2016-03-15T00:00:00Z","timestamp":1458000000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Sci Data"],"abstract":"<jats:title>Abstract<\/jats:title>\n <jats:p>There is an urgent need to improve the infrastructure supporting the reuse of scholarly data. A diverse set of stakeholders\u2014representing academia, industry, funding agencies, and scholarly publishers\u2014have come together to design and jointly endorse a concise and measureable set of principles that we refer to as the FAIR Data Principles. The intent is that these may act as a guideline for those wishing to enhance the reusability of their data holdings. Distinct from peer initiatives that focus on the human scholar, the FAIR Principles put specific emphasis on enhancing the ability of machines to automatically find and use the data, in addition to supporting its reuse by individuals. This Comment is the first formal publication of the FAIR Principles, and includes the rationale behind them, and some exemplar implementations in the community.<\/jats:p>","DOI":"10.1038\/sdata.2016.18","type":"journal-article","created":{"date-parts":[[2016,3,15]],"date-time":"2016-03-15T10:23:08Z","timestamp":1458037388000},"update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":15915,"title":["The FAIR Guiding Principles for scientific data management and stewardship"],"prefix":"10.1038","volume":"3","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6960-357X","authenticated-orcid":false,"given":"Mark D.","family":"Wilkinson","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4727-9435","authenticated-orcid":false,"given":"Michel","family":"Dumontier","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0209-4480","authenticated-orcid":false,"given":"IJsbrand Jan","family":"Aalbersberg","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0179-7384","authenticated-orcid":false,"given":"Gabrielle","family":"Appleton","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8042-4131","authenticated-orcid":false,"given":"Myles","family":"Axton","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2829-6715","authenticated-orcid":false,"given":"Arie","family":"Baak","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4155-5910","authenticated-orcid":false,"given":"Niklas","family":"Blomberg","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0327-638X","authenticated-orcid":false,"given":"Jan-Willem","family":"Boiten","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1164-1351","authenticated-orcid":false,"given":"Luiz Bonino","family":"da Silva Santos","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7618-7292","authenticated-orcid":false,"given":"Philip E.","family":"Bourne","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jildau","family":"Bouwman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8686-0017","authenticated-orcid":false,"given":"Anthony J.","family":"Brookes","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4060-7360","authenticated-orcid":false,"given":"Tim","family":"Clark","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1304-1939","authenticated-orcid":false,"given":"Merc\u00e8","family":"Crosas","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5654-2392","authenticated-orcid":false,"given":"Ingrid","family":"Dillo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8599-7345","authenticated-orcid":false,"given":"Olivier","family":"Dumon","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6444-1436","authenticated-orcid":false,"given":"Scott","family":"Edmunds","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5301-3142","authenticated-orcid":false,"given":"Chris T.","family":"Evelo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4368-8058","authenticated-orcid":false,"given":"Richard","family":"Finkers","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3499-8262","authenticated-orcid":false,"given":"Alejandra","family":"Gonzalez-Beltran","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5711-4872","authenticated-orcid":false,"given":"Alasdair J.G.","family":"Gray","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0183-6910","authenticated-orcid":false,"given":"Paul","family":"Groth","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1219-2137","authenticated-orcid":false,"given":"Carole","family":"Goble","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5212-7052","authenticated-orcid":false,"given":"Jeffrey S.","family":"Grethe","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8641-4930","authenticated-orcid":false,"given":"Jaap","family":"Heringa","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4450-3112","authenticated-orcid":false,"given":"Peter A.C","family":"\u2019t Hoen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6825-9439","authenticated-orcid":false,"given":"Rob","family":"Hooft","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1267-0234","authenticated-orcid":false,"given":"Tobias","family":"Kuhn","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ruben","family":"Kok","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7352-1400","authenticated-orcid":false,"given":"Joost","family":"Kok","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2401-4223","authenticated-orcid":false,"given":"Scott J.","family":"Lusher","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8406-3871","authenticated-orcid":false,"given":"Maryann E.","family":"Martone","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Albert","family":"Mons","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9610-5728","authenticated-orcid":false,"given":"Abel L.","family":"Packer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3165-5344","authenticated-orcid":false,"given":"Bengt","family":"Persson","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9853-5668","authenticated-orcid":false,"given":"Philippe","family":"Rocca-Serra","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8691-772X","authenticated-orcid":false,"given":"Marco","family":"Roos","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rene","family":"van Schaik","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5306-5690","authenticated-orcid":false,"given":"Susanna-Assunta","family":"Sansone","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8888-635X","authenticated-orcid":false,"given":"Erik","family":"Schultes","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7516-6246","authenticated-orcid":false,"given":"Thierry","family":"Sengstag","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1386-0731","authenticated-orcid":false,"given":"Ted","family":"Slater","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"George","family":"Strawn","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0979-3401","authenticated-orcid":false,"given":"Morris A.","family":"Swertz","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7633-1442","authenticated-orcid":false,"given":"Mark","family":"Thompson","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Johan","family":"van der Lei","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1377-9386","authenticated-orcid":false,"given":"Erik","family":"van Mulligen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4836-6568","authenticated-orcid":false,"given":"Jan","family":"Velterop","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9773-4008","authenticated-orcid":false,"given":"Andra","family":"Waagmeester","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Peter","family":"Wittenburg","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1279-5133","authenticated-orcid":false,"given":"Katherine","family":"Wolstencroft","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6935-9028","authenticated-orcid":false,"given":"Jun","family":"Zhao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3934-0072","authenticated-orcid":false,"given":"Barend","family":"Mons","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","published-online":{"date-parts":[[2016,3,15]]},"reference":[{"key":"BFsdata201618_CR1","doi-asserted-by":"publisher","first-page":"e1002295","DOI":"10.1371\/journal.pbio.1002295","volume":"13","author":"DG Roche","year":"2015","unstructured":"Roche, D. G., Kruuk, L. E. B., Lanfear, R. & Binning, S. A. Public Data Archiving in Ecology and Evolution: How Well Are We Doing? PLOS Biol. 13, e1002295 (2015).","journal-title":"PLOS Biol."},{"key":"BFsdata201618_CR2","doi-asserted-by":"crossref","unstructured":"Bechhofer, S. et al. Research Objects: Towards Exchange and Reuse of Digital Knowledge. Nat. Preced. 10.1038\/npre.2010.4626.1 (2010).","DOI":"10.1038\/npre.2010.4626.1"},{"key":"BFsdata201618_CR3","doi-asserted-by":"publisher","first-page":"D36","DOI":"10.1093\/nar\/gks1195","volume":"41","author":"DA Benson","year":"2013","unstructured":"Benson, D. A. et al. GenBank. Nucleic Acids Res. 41, D36\u2013D42 (2013).","journal-title":"Nucleic Acids Res."},{"key":"BFsdata201618_CR4","doi-asserted-by":"publisher","first-page":"980","DOI":"10.1038\/nsb1203-980","volume":"10","author":"H Berman","year":"2003","unstructured":"Berman, H., Henrick, K. & Nakamura, H. Announcing the worldwide Protein Data Bank. Nat. Struct. Biol. 10, 980\u2013980 (2003).","journal-title":"Nat. Struct. Biol."},{"key":"BFsdata201618_CR5","doi-asserted-by":"publisher","first-page":"D204","DOI":"10.1093\/nar\/gku989","volume":"43","author":"The Uniprot Consortium","year":"2015","unstructured":"The Uniprot Consortium. UniProt: a hub for protein information. Nucleic Acids Res. 43, D204\u2013D212 (2015).","journal-title":"Nucleic Acids Res."},{"key":"BFsdata201618_CR6","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1051\/aas:2000332","volume":"143","author":"M Wenger","year":"2000","unstructured":"Wenger, M. et al. The SIMBAD astronomical database-The CDS reference database for astronomical objects. Astron. Astrophys. Suppl. Ser. 143, 9\u201322 (2000).","journal-title":"Astron. Astrophys. Suppl. Ser."},{"issue":"1","key":"BFsdata201618_CR7","first-page":"p2","volume":"17","author":"M Crosas","year":"2011","unstructured":"Crosas, M. \"The Dataverse Network\u00ae: An Open-Source Application for Sharing, Discovering and Preserving Data\". D-Lib Mag 17 (1), p2 (2011).","journal-title":"D-Lib Mag"},{"key":"BFsdata201618_CR8","unstructured":"White, H. C., Carrier, S., Thompson, A., Greenberg, J. & Scherle, R. The Dryad data repository: A Singapore framework metadata architecture in a DSpace environment. Univ. G\u00f6ttingen, p157 (2008)."},{"key":"BFsdata201618_CR9","doi-asserted-by":"publisher","first-page":"279","DOI":"10.2218\/ijdc.v8i1.260","volume":"8","author":"D Lecarpentier","year":"2013","unstructured":"Lecarpentier, D. et al. EUDAT: A New Cross-Disciplinary Data Infrastructure for Science. Int. J. Digit. Curation 8, 279\u2013287 (2013).","journal-title":"Int. J. Digit. Curation"},{"key":"BFsdata201618_CR10","doi-asserted-by":"publisher","first-page":"635","DOI":"10.1093\/biosci\/biv095","volume":"65","author":"ME Martone","year":"2015","unstructured":"Martone, M. E. FORCE11: Building the Future for Research Communications and e-Scholarship. Bioscience 65, 635 (2015).","journal-title":"Bioscience"},{"key":"BFsdata201618_CR11","doi-asserted-by":"crossref","unstructured":"White, E. et al. Nine simple ways to make it easier to (re)use your data. Ideas Ecol. Evol. 6 (2013).","DOI":"10.4033\/iee.2013.6b.6.f"},{"key":"BFsdata201618_CR12","doi-asserted-by":"publisher","first-page":"e1003285","DOI":"10.1371\/journal.pcbi.1003285","volume":"9","author":"GK Sandve","year":"2013","unstructured":"Sandve, G. K., Nekrutenko, A., Taylor, J. & Hovig, E. Ten Simple Rules for Reproducible Computational Research. PLoS Comput. Biol. 9, e1003285 (2013).","journal-title":"PLoS Comput. Biol."},{"key":"BFsdata201618_CR13","volume-title":"D-Lib Magazine","author":"M Altman","year":"2007","unstructured":"Altman, M. & King, G. in D-Lib Magazine 13, no. 3\/4 (2007)."},{"key":"BFsdata201618_CR14","doi-asserted-by":"publisher","first-page":"33","DOI":"10.1186\/s12918-015-0174-y","volume":"9","author":"K Wolstencroft","year":"2015","unstructured":"Wolstencroft, K. et al. SEEK: a systems biology data and model management platform. BMC Syst. Biol. 9, 33 (2015).","journal-title":"BMC Syst. Biol."},{"key":"BFsdata201618_CR15","doi-asserted-by":"publisher","first-page":"468","DOI":"10.1186\/1471-2105-12-468","volume":"12","author":"A Bauch","year":"2011","unstructured":"Bauch, A. et al. openBIS: a flexible framework for managing and analyzing complex data in biology research. BMC Bioinformatics 12, 468 (2011).","journal-title":"BMC Bioinformatics"},{"key":"BFsdata201618_CR16","doi-asserted-by":"publisher","first-page":"121","DOI":"10.1038\/ng.1054","volume":"44","author":"S-A Sansone","year":"2012","unstructured":"Sansone, S.-A. et al. Toward interoperable bioscience data. Nat. Genet. 44, 121\u2013126 (2012).","journal-title":"Nat. Genet."},{"key":"BFsdata201618_CR17","doi-asserted-by":"publisher","first-page":"S4","DOI":"10.1186\/1471-2105-15-S14-S4","volume":"15","author":"A Gonz\u00e1lez-Beltr\u00e1n","year":"2014","unstructured":"Gonz\u00e1lez-Beltr\u00e1n, A., Maguire, E., Sansone, S.-A. & Rocca-Serra, P. linkedISA: semantic representation of ISA-Tab experimental metadata. BMC Bioinformatics 15, S4 (2014).","journal-title":"BMC Bioinformatics"},{"key":"BFsdata201618_CR18","doi-asserted-by":"publisher","first-page":"e0127612","DOI":"10.1371\/journal.pone.0127612","volume":"10","author":"A Gonz\u00e1lez-Beltr\u00e1n","year":"2015","unstructured":"Gonz\u00e1lez-Beltr\u00e1n, A. et al. From Peer-Reviewed to Peer-Reproduced in Scholarly Publishing: The Complementary Roles of Data Models and Workflows in Bioinformatics. PLoS ONE 10, e0127612 (2015).","journal-title":"PLoS ONE"},{"key":"BFsdata201618_CR19","first-page":"1","volume":"7603\/2012","author":"L Harland","year":"2012","unstructured":"Harland, L. Open PHACTS: A Semantic Knowledge Infrastructure for Public and Commercial Drug Discovery Research. Knowl. Eng. Knowl. Manag. Lect. Notes Comput. Sci. 7603\/2012, 1\u20137 (2012).","journal-title":"Knowl. Eng. Knowl. Manag. Lect. Notes Comput. Sci."},{"key":"BFsdata201618_CR20","doi-asserted-by":"publisher","first-page":"12","DOI":"10.1016\/j.websem.2014.03.003","volume":"29","author":"P Groth","year":"2014","unstructured":"Groth, P. et al. API-centric Linked Data integration: The Open PHACTS Discovery Platform case study. Web Semant. Sci. Serv. Agents World Wide Web 29, 12\u201318 (2014).","journal-title":"Web Semant. Sci. Serv. Agents World Wide Web"},{"key":"BFsdata201618_CR21","doi-asserted-by":"publisher","first-page":"235","DOI":"10.1093\/nar\/28.1.235","volume":"28","author":"HM Berman","year":"2000","unstructured":"Berman, H. M. et al. The Protein Data Bank. Nucleic Acids Res. 28, 235\u2013242 (2000).","journal-title":"Nucleic Acids Res."},{"key":"BFsdata201618_CR22","doi-asserted-by":"publisher","first-page":"571","DOI":"10.1016\/S0076-6879(97)77032-0","volume":"277","author":"PE Bourne","year":"1997","unstructured":"Bourne, P. E., Berman, H. M., Watenpaugh, K., Westbrook, J. D. & Fitzgerald, P. M. D. The macromolecular crystallographic information file (mmCIF). Meth. Enzym 277, 571\u2013590 (1997).","journal-title":"Meth. Enzym"},{"key":"BFsdata201618_CR23","doi-asserted-by":"publisher","first-page":"D345","DOI":"10.1093\/nar\/gku1214","volume":"43","author":"PW Rose","year":"2015","unstructured":"Rose, P. W. et al. The RCSB Protein Data Bank: views of structural biology for basic and applied research and education. Nucleic Acids Res. 43, D345\u2013D356 (2015).","journal-title":"Nucleic Acids Res."},{"key":"BFsdata201618_CR24","doi-asserted-by":"publisher","first-page":"D453","DOI":"10.1093\/nar\/gkr811","volume":"40","author":"AR Kinjo","year":"2012","unstructured":"Kinjo, A. R. et al. Protein Data Bank Japan (PDBj): maintaining a structural data archive and resource description framework format. Nucleic Acids Res. 40, D453\u2013D460 (2012).","journal-title":"Nucleic Acids Res."},{"key":"BFsdata201618_CR25","doi-asserted-by":"publisher","first-page":"D285","DOI":"10.1093\/nar\/gkt1180","volume":"42","author":"A Gutmanas","year":"2014","unstructured":"Gutmanas, A. et al. PDBe: Protein Data Bank in Europe. Nucleic Acids Res. 42, D285\u2013D291 (2014).","journal-title":"Nucleic Acids Res."},{"key":"BFsdata201618_CR26","doi-asserted-by":"publisher","first-page":"D204","DOI":"10.1093\/nar\/gku989","volume":"43","author":"UniProt Consortium","year":"2015","unstructured":"UniProt Consortium. UniProt: a hub for protein information. Nucleic Acids Res. 43, D204\u2013D212 (2015).","journal-title":"Nucleic Acids Res."},{"key":"BFsdata201618_CR27","doi-asserted-by":"publisher","first-page":"e1","DOI":"10.7717\/peerj-cs.1","volume":"1","author":"J Starr","year":"2015","unstructured":"Starr, J. et al. Achieving human and machine accessibility of cited data in scholarly publications. PeerJ Comput. Sci. 1, e1 (2015).","journal-title":"PeerJ Comput. Sci."},{"key":"BFsdata201618_CR28","unstructured":"Wilkinson, M., Dumontier, M. & Durbin, P. DataFairPort: The Perl libraries version 0.231 10.5281\/zenodo.33584 (2015)."},{"key":"BFsdata201618_CR29","unstructured":"Data Citation Synthesis Group: Joint Declaration of Data Citation Principles. San Diego CA: FORCE11 https:\/\/www.force11.org\/datacitation (2014)."},{"key":"BFsdata201618_CR30","doi-asserted-by":"publisher","unstructured":"Ohno-machado, L. et al. NIH BD2K bioCADDIE white paper\u2014Data Discovery Index. http:\/\/dx.doi.org\/10.6084\/m9.figshare.1362572 (2015).","DOI":"10.6084\/m9.figshare.1362572"},{"key":"BFsdata201618_CR31","unstructured":"NIH BD2K bioCADDIE WG3 Members. WG3-MetadataSpecifications: NIH BD2K bioCADDIE Data Discovery Index WG3 Metadata Specification v1 doi:10.5281\/zenodo.28019 (2015)."},{"key":"BFsdata201618_CR32","doi-asserted-by":"crossref","first-page":"1148","DOI":"10.1093\/jamia\/ocv048","volume":"22","author":"MA Musen","year":"2015","unstructured":"Musen, M. A. et al. The center for expanded data annotation and retrieval. J. Am. Med. Informatics Assoc. 22, 1148\u20131152 (2015).","journal-title":"J. Am. Med. Informatics Assoc."}],"updated-by":[{"DOI":"10.1038\/s41597-019-0009-6","type":"addendum","label":"Addendum","source":"publisher","updated":{"date-parts":[[2019,3,19]],"date-time":"2019-03-19T00:00:00Z","timestamp":1552953600000}}],"container-title":["Scientific Data"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.nature.com\/articles\/sdata201618.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/sdata201618","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.nature.com\/articles\/sdata201618.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,4]],"date-time":"2023-01-04T07:47:12Z","timestamp":1672818432000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nature.com\/articles\/sdata201618"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,3,15]]},"references-count":32,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2016,12,20]]}},"alternative-id":["BFsdata201618"],"URL":"https:\/\/doi.org\/10.1038\/sdata.2016.18","relation":{"has-review":[{"id-type":"doi","id":"10.3410\/f.726216348.793543848","asserted-by":"object"}],"is-referenced-by":[{"id-type":"doi","id":"10.1186\/s12913-026-14123-y","asserted-by":"object"},{"id-type":"doi","id":"10.1007\/s42452-026-09293-7","asserted-by":"object"},{"id-type":"doi","id":"10.1007\/s13562-026-01064-0","asserted-by":"object"}]},"ISSN":["2052-4463"],"issn-type":[{"value":"2052-4463","type":"electronic"}],"subject":[],"published":{"date-parts":[[2016,3,15]]},"assertion":[{"value":"10 December 2015","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 February 2016","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"15 March 2016","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"M.A. is the <i>Nature Genetics<\/i>\u2019 Editor in Chief; S.A.S. is <i>Scientific Data<\/i>\u2019s Honorary Academic Editor and consultant.","order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"160018"}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1056_2FAIoa2400555 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T20:32:34Z","timestamp":1793305954178,"version":"4.5.0"},"reference-count":51,"publisher":"Massachusetts Medical Society","issue":"1","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["NEJM AI"],"published-print":{"date-parts":[[2025,1,1]]},"DOI":"10.1056\/aioa2400555","type":"journal-article","created":{"date-parts":[[2024,12,3]],"date-time":"2024-12-03T10:00:16Z","timestamp":1733220016000},"source":"Crossref","is-referenced-by-count":37,"title":["Autonomous LLM-Driven Research \u2014 from Data to Human-Verifiable Research Papers"],"prefix":"10.1056","volume":"2","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-2827-7517","authenticated-orcid":false,"given":"Tal","family":"Ifargan","sequence":"first","affiliation":[{"name":"Faculty of Data and Decision Sciences, Technion\u2013Israel Institute of Technology, Haifa, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6380-627X","authenticated-orcid":false,"given":"Lukas","family":"Hafner","sequence":"additional","affiliation":[{"name":"Faculty of Biology, Technion\u2013Israel Institute of Technology, Haifa, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0004-3124-4179","authenticated-orcid":false,"given":"Maor","family":"Kern","sequence":"additional","affiliation":[{"name":"Epsio, Tel Aviv, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-3644-1381","authenticated-orcid":false,"given":"Ori","family":"Alcalay","sequence":"additional","affiliation":[{"name":"Epsio, Tel Aviv, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5013-5072","authenticated-orcid":false,"given":"Roy","family":"Kishony","sequence":"additional","affiliation":[{"name":"Faculty of Biology, Technion\u2013Israel Institute of Technology, Haifa, Israel"},{"name":"Faculty of Computer Science, Technion\u2013Israel Institute of Technology, Haifa, Israel"},{"name":"Faculty of Biomedical Engineering, Technion\u2013Israel Institute of Technology, Haifa, Israel"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"150","reference":[{"key":"e_1_3_5_2_2","unstructured":"Xi Z Chen W Guo X et al. The rise and potential of large language model based agents: a survey. September 14 2023 (http:\/\/arXiv.org\/abs\/2309.07864). Preprint."},{"key":"e_1_3_5_3_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.metrad.2023.100017"},{"key":"e_1_3_5_4_2","unstructured":"Wang L Ma C Feng X et al. A survey on large language model based autonomous agents. August 22 2023 (http:\/\/arXiv.org\/abs\/2308.11432). Preprint."},{"key":"e_1_3_5_5_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.abq1158"},{"key":"e_1_3_5_6_2","doi-asserted-by":"publisher","DOI":"10.1126\/sciadv.adh1850"},{"key":"e_1_3_5_7_2","unstructured":"He-Yueya J Poesia G Wang RE Goodman ND. Solving math word problems by combining language models with symbolic solvers. April 26 2023 (http:\/\/arXiv.org\/abs\/2304.09102). Preprint."},{"key":"e_1_3_5_8_2","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-023-06924-6"},{"key":"e_1_3_5_9_2","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-023-06747-5"},{"key":"e_1_3_5_10_2","unstructured":"Qian C Cong X Liu W et al. Communicative agents for software development. July 16 2023 (http:\/\/arXiv.org\/abs\/2307.07924). Preprint."},{"key":"e_1_3_5_11_2","unstructured":"Dong Y Jiang X Jin Z Li G. Self-collaboration code generation via ChatGPT. April 15 2023 (http:\/\/arXiv.org\/abs\/2304.07590). Preprint."},{"key":"e_1_3_5_12_2","unstructured":"Zhou W Jiang YE Cui P Kannan A. RecurrentGPT: interactive generation of (arbitrarily) long text. May 22 2023 (http:\/\/arXiv.org\/abs\/2305.13304). Preprint."},{"key":"e_1_3_5_13_2","unstructured":"Nair V Schumacher E Tso G Kannan A. DERA: enhancing large language model completions with dialog-enabled resolving agents. March 30 2023 (http:\/\/arXiv.org\/abs\/2303.17071). Preprint."},{"key":"e_1_3_5_14_2","unstructured":"Tu T Palepu A Schaekermann M et al. Towards conversational diagnostic AI. January 11 2024 (http:\/\/arXiv.org\/abs\/2401.05654). Preprint."},{"key":"e_1_3_5_15_2","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-023-06792-0"},{"key":"e_1_3_5_16_2","doi-asserted-by":"publisher","DOI":"10.1056\/AIoa2400196"},{"key":"e_1_3_5_17_2","doi-asserted-by":"publisher","DOI":"10.1038\/s42254-023-00581-4"},{"key":"e_1_3_5_18_2","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-023-03144-w"},{"key":"e_1_3_5_19_2","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-023-00340-6"},{"key":"e_1_3_5_20_2","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-023-00107-z"},{"key":"e_1_3_5_21_2","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-022-03479-w"},{"key":"e_1_3_5_22_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.adg9714"},{"key":"e_1_3_5_23_2","doi-asserted-by":"publisher","DOI":"10.1038\/s41586-024-07146-0"},{"key":"e_1_3_5_24_2","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-023-03266-1"},{"key":"e_1_3_5_25_2","volume-title":"An introduction to scientific research","unstructured":"Wilson EB. An introduction to scientific research. McGraw-Hill Book Company, 1952."},{"key":"e_1_3_5_26_2","unstructured":"Teboul A. Diabetes health indicators dataset. 2021 (https:\/\/www.kaggle.com\/datasets\/alexteboul\/diabetes-health-indicators-dataset)."},{"key":"e_1_3_5_27_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.dib.2023.109521"},{"key":"e_1_3_5_28_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.1004125"},{"key":"e_1_3_5_29_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0289945"},{"key":"e_1_3_5_30_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0257069"},{"key":"e_1_3_5_31_2","unstructured":"Kinney R Anastasiades C Authur R et al. The semantic scholar open data platform. 2023 (http:\/\/arXiv.org\/abs\/2301.10140). Preprint."},{"key":"e_1_3_5_32_2","unstructured":"Ji Z Lee N Frieske R et al. Survey of hallucination in natural language generation. February 8 2022 (http:\/\/arXiv.org\/abs\/2202.03629). Preprint."},{"key":"e_1_3_5_33_2","unstructured":"OpenAI Platform (https:\/\/platform.openai.com\/docs\/api-reference)."},{"key":"e_1_3_5_34_2","unstructured":"Touvron H Martin L Stone K et al. Llama 2: open foundation and fine-tuned chat models. July 18 2023 (http:\/\/arXiv.org\/abs\/2307.09288). Preprint."},{"key":"e_1_3_5_35_2","unstructured":"Rozi\u00e8re B Gehring J Gloeckle F et al. Code Llama: open foundation models for code. August 24 2023 (http:\/\/arXiv.org\/abs\/2308.12950). Preprint."},{"key":"e_1_3_5_36_2","unstructured":"Wu Q Bansal G Zhang J et al. AutoGen: enabling next-gen LLM applications via multi-agent conversation. August 16 2023 (http:\/\/arXiv.org\/abs\/2308.08155). Preprint."},{"key":"e_1_3_5_37_2","unstructured":"Hong S Zhuge M Chen J et al. MetaGPT: meta programming for a multi-agent collaborative framework. August 1 2023 (http:\/\/arXiv.org\/abs\/2308.00352). Preprint."},{"key":"e_1_3_5_38_2","unstructured":"Madaan A Tandon N Gupta P et al. Self-refine: iterative refinement with self-feedback. March 30 2023 (http:\/\/arXiv.org\/abs\/2303.17651). Preprint."},{"key":"e_1_3_5_39_2","unstructured":"Harrison C. LangChain. 2022 (https:\/\/github.com\/langchain-ai\/langchain)."},{"key":"e_1_3_5_40_2","unstructured":"Gravitas S. AutoGPT. 2023 (https:\/\/github.com\/Significant-Gravitas\/AutoGPT)."},{"key":"e_1_3_5_41_2","unstructured":"Peng B Galley M He P et al. Check your facts and try again: improving large language models with external knowledge and automated feedback. February 24 2023 (http:\/\/arXiv.org\/abs\/2302.12813). Preprint."},{"key":"e_1_3_5_42_2","volume-title":"StatPearls","unstructured":"Rolle-Lake L, Robbins E. Behavioral risk factor surveillance system. In: StatPearls. Treasure Island, FL: StatPearls Publishing, 2023 (https:\/\/www.ncbi.nlm.nih.gov\/pubmed\/31971707)."},{"key":"e_1_3_5_43_2","doi-asserted-by":"crossref","unstructured":"Cohan A Feldman S Beltagy I Downey D Weld DS. SPECTER: document-level representation learning using citation-informed transformers. April 15 2020 (http:\/\/arXiv.org\/abs\/2004.07180). Preprint.","DOI":"10.18653\/v1\/2020.acl-main.207"},{"key":"e_1_3_5_44_2","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-030-05318-5_9"},{"key":"e_1_3_5_45_2","unstructured":"Majumder BP Surana H Agarwal D et al. Data-driven discovery with large generative models. February 21 2024 (http:\/\/arXiv.org\/abs\/2402.13610). Preprint."},{"key":"e_1_3_5_46_2","unstructured":"Abdel-Rehim A Zenil H Orhobor O et al. Scientific hypothesis generation by a large language model: laboratory validation in breast cancer treatment. May 20 2024 (http:\/\/arXiv.org\/abs\/2405.12258). Preprint."},{"key":"e_1_3_5_47_2","unstructured":"Wang Q Downey D Ji H Hope T. SciMON: scientific inspiration machines optimized for novelty. May 23 2023 (http:\/\/arXiv.org\/abs\/2305.14259). Preprint."},{"key":"e_1_3_5_48_2","unstructured":"Lu C Lu C Lange RT Foerster J Clune J Ha D. The AI scientist: towards fully automated open-ended scientific discovery. August 12 2024 (http:\/\/arXiv.org\/abs\/2408.06292). Preprint."},{"key":"e_1_3_5_49_2","doi-asserted-by":"publisher","DOI":"10.1038\/nmeth.4120"},{"key":"e_1_3_5_50_2","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-021-01436-7"},{"key":"e_1_3_5_51_2","doi-asserted-by":"publisher","DOI":"10.1002\/asi.24495"},{"key":"e_1_3_5_52_2","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-023-01780-w"}],"container-title":["NEJM AI"],"original-title":[],"language":"en","deposited":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T20:10:48Z","timestamp":1793304648000},"score":1,"resource":{"primary":{"URL":"https:\/\/ai.nejm.org\/doi\/10.1056\/AIoa2400555"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,1,1]]},"references-count":51,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2025,1,1]]}},"alternative-id":["10.1056\/AIoa2400555"],"URL":"https:\/\/doi.org\/10.1056\/aioa2400555","relation":{},"ISSN":["2836-9386"],"issn-type":[{"value":"2836-9386","type":"print"}],"subject":[],"published":{"date-parts":[[2025,1,1]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1073_2Fpnas_1708274114 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T21:07:26Z","timestamp":1793308046439,"version":"4.5.0"},"reference-count":60,"publisher":"National Academy of Sciences","issue":"11","license":[{"start":{"date-parts":[[2018,9,12]],"date-time":"2018-09-12T00:00:00Z","timestamp":1536710400000},"content-version":"vor","delay-in-days":184,"URL":"http:\/\/www.pnas.org\/site\/aboutpnas\/licenses.xhtml"}],"funder":[{"DOI":"10.13039\/100009827","name":"Laura and John Arnold Foundation","doi-asserted-by":"publisher","award":["none"],"award-info":[{"award-number":["none"]}],"id":[{"id":"10.13039\/100009827","id-type":"DOI","asserted-by":"publisher"}]},{"name":"National Institute of Aging","award":["R24AG048124"],"award-info":[{"award-number":["R24AG048124"]}]}],"content-domain":{"domain":["www.pnas.org"],"crossmark-restriction":true},"short-container-title":["Proc. Natl. Acad. Sci. U.S.A."],"published-print":{"date-parts":[[2018,3,13]]},"abstract":"<jats:p>Progress in science relies in part on generating hypotheses with existing observations and testing hypotheses with new observations. This distinction between postdiction and prediction is appreciated conceptually but is not respected in practice. Mistaking generation of postdictions with testing of predictions reduces the credibility of research findings. However, ordinary biases in human reasoning, such as hindsight bias, make it hard to avoid this mistake. An effective solution is to define the research questions and analysis plan before observing the research outcomes\u2014a process called preregistration. Preregistration distinguishes analyses and outcomes that result from predictions from those that result from postdictions. A variety of practical strategies are available to make the best possible use of preregistration in circumstances that fall short of the ideal application, such as when the data are preexisting. Services are now available for preregistration across all disciplines, facilitating a rapid increase in the practice. Widespread adoption of preregistration will increase distinctiveness between hypothesis generation and hypothesis testing and will improve the credibility of research findings.<\/jats:p>","DOI":"10.1073\/pnas.1708274114","type":"journal-article","created":{"date-parts":[[2018,3,12]],"date-time":"2018-03-12T15:42:17Z","timestamp":1520869337000},"page":"2600-2606","update-policy":"https:\/\/doi.org\/10.1073\/pnas.cm10313","source":"Crossref","is-referenced-by-count":1587,"title":["The preregistration revolution"],"prefix":"10.1073","volume":"115","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-6797-5476","authenticated-orcid":false,"given":"Brian A.","family":"Nosek","sequence":"first","affiliation":[{"name":"Center for Open Science, Charlottesville, VA 22903;"},{"name":"Department of Psychology, University of Virginia, Charlottesville, VA 22904"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8607-2579","authenticated-orcid":false,"given":"Charles R.","family":"Ebersole","sequence":"additional","affiliation":[{"name":"Department of Psychology, University of Virginia, Charlottesville, VA 22904"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2241-7259","authenticated-orcid":false,"given":"Alexander C.","family":"DeHaven","sequence":"additional","affiliation":[{"name":"Center for Open Science, Charlottesville, VA 22903;"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3125-5888","authenticated-orcid":false,"given":"David T.","family":"Mellor","sequence":"additional","affiliation":[{"name":"Center for Open Science, Charlottesville, VA 22903;"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"341","published-online":{"date-parts":[[2018,3,12]]},"reference":[{"key":"e_1_3_3_1_2","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1976.10480949"},{"key":"e_1_3_3_2_2","doi-asserted-by":"publisher","DOI":"10.1016\/B978-0-12-438150-6.50018-2"},{"key":"e_1_3_3_3_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.actpsy.2014.02.001"},{"key":"e_1_3_3_4_2","doi-asserted-by":"publisher","DOI":"10.1016\/0039-3681(87)90005-7"},{"key":"e_1_3_3_5_2","first-page":"1","volume-title":"Criticism and the Growth of Knowledge","author":"Kuhn TS","year":"1970","unstructured":"TS Kuhn, Logic of discovery or psychology of research? Criticism and the Growth of Knowledge, eds I Lakatos, A Musgrave (Cambridge Univ Press, Cambridge, UK), pp. 1\u201323 (1970)."},{"key":"e_1_3_3_6_2","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612463078"},{"key":"e_1_3_3_7_2","doi-asserted-by":"publisher","DOI":"10.1111\/brv.12315"},{"key":"e_1_3_3_8_2","doi-asserted-by":"publisher","DOI":"10.1038\/s41562-016-0021"},{"key":"e_1_3_3_9_2","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612459058"},{"key":"e_1_3_3_10_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.aac4716"},{"key":"e_1_3_3_11_2","doi-asserted-by":"publisher","DOI":"10.1093\/ije\/30.5.948"},{"key":"e_1_3_3_12_2","doi-asserted-by":"publisher","DOI":"10.1207\/s15327957pspr0203_4"},{"key":"e_1_3_3_13_2","doi-asserted-by":"publisher","DOI":"10.1016\/0030-5073(75)90002-1"},{"key":"e_1_3_3_14_2","doi-asserted-by":"publisher","DOI":"10.1136\/qhc.12.4.304"},{"key":"e_1_3_3_15_2","volume-title":"The Undoing Project: A Friendship That Changed Our Minds","author":"Lewis M","year":"2016","unstructured":"M Lewis The Undoing Project: A Friendship That Changed Our Minds (W. W. Norton & Company, New York, 2016)."},{"key":"e_1_3_3_16_2","doi-asserted-by":"publisher","DOI":"10.1177\/0146167212452313"},{"key":"e_1_3_3_17_2","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511809477"},{"key":"e_1_3_3_18_2","volume-title":"Thinking, Fast and Slow","author":"Kahneman D","year":"2011","unstructured":"D Kahneman Thinking, Fast and Slow (Farrar, Straus and Giroux, New York, 2011)."},{"key":"e_1_3_3_19_2","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612459060"},{"key":"e_1_3_3_20_2","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612457576"},{"key":"e_1_3_3_21_2","doi-asserted-by":"publisher","DOI":"10.1007\/BF01173636"},{"key":"e_1_3_3_22_2","doi-asserted-by":"publisher","DOI":"10.1016\/0749-5978(91)90010-Q"},{"key":"e_1_3_3_23_2","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.108.3.480"},{"key":"e_1_3_3_24_2","doi-asserted-by":"publisher","DOI":"10.1037\/1089-2680.2.2.175"},{"key":"e_1_3_3_25_2","doi-asserted-by":"publisher","DOI":"10.1111\/j.1751-2409.2011.01037.x"},{"key":"e_1_3_3_26_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jesp.2006.05.011"},{"key":"e_1_3_3_27_2","doi-asserted-by":"publisher","DOI":"10.1198\/000313001300339950"},{"key":"e_1_3_3_28_2","first-page":"661","article-title":"Statistical significance with comments by editors of marketing journals: The historical growth of statistical significance testing in psychology\u2014and its future prospects","volume":"60","author":"Hubbard R","year":"2000","unstructured":"R Hubbard, PA Ryan, Statistical significance with comments by editors of marketing journals: The historical growth of statistical significance testing in psychology\u2014and its future prospects. Educ Psychol Meas 60, 661\u2013681 (2000).","journal-title":"Educ Psychol Meas"},{"key":"e_1_3_3_29_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.tree.2006.12.003"},{"key":"e_1_3_3_30_2","doi-asserted-by":"publisher","DOI":"10.1177\/0956797611417632"},{"key":"e_1_3_3_31_2","doi-asserted-by":"publisher","DOI":"10.1080\/00031305.2016.1154108"},{"key":"e_1_3_3_32_2","doi-asserted-by":"crossref","DOI":"10.1038\/s41562-017-0189-z","article-title":"Redefine statistical significance","author":"Benjamin DJ","year":"2017","unstructured":"DJ Benjamin, , Redefine statistical significance. Nat Hum Behav, 2017).","journal-title":"Nat Hum Behav"},{"key":"e_1_3_3_33_2","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1955.10501294"},{"key":"e_1_3_3_34_2","doi-asserted-by":"publisher","DOI":"10.2307\/3001913"},{"key":"e_1_3_3_35_2","doi-asserted-by":"publisher","DOI":"10.1002\/bimj.200900299"},{"key":"e_1_3_3_36_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.neuroimage.2005.12.062"},{"key":"e_1_3_3_37_2","doi-asserted-by":"publisher","DOI":"10.1511\/2014.111.460"},{"key":"e_1_3_3_38_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.1255484"},{"key":"e_1_3_3_39_2","doi-asserted-by":"publisher","DOI":"10.1037\/h0076157"},{"key":"e_1_3_3_40_2","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.86.3.638"},{"key":"e_1_3_3_41_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.0020124"},{"key":"e_1_3_3_42_2","doi-asserted-by":"publisher","DOI":"10.1177\/0956797611430953"},{"key":"e_1_3_3_43_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0132382"},{"key":"e_1_3_3_44_2","doi-asserted-by":"publisher","DOI":"10.1037\/ccp0000115"},{"key":"e_1_3_3_45_2","doi-asserted-by":"publisher","DOI":"10.1136\/bmj.j917"},{"key":"e_1_3_3_46_2","doi-asserted-by":"publisher","DOI":"10.2307\/2343787"},{"key":"e_1_3_3_47_2","doi-asserted-by":"publisher","DOI":"10.3758\/s13414-017-1304-y"},{"key":"e_1_3_3_48_2","doi-asserted-by":"publisher","DOI":"10.1038\/526187a"},{"key":"e_1_3_3_49_2","doi-asserted-by":"publisher","DOI":"10.1017\/S1049096516000810"},{"key":"e_1_3_3_50_2","doi-asserted-by":"publisher","DOI":"10.1080\/15548627.2015.1100356"},{"key":"e_1_3_3_51_2","doi-asserted-by":"publisher","DOI":"10.1037\/0022-3514.85.2.197"},{"key":"e_1_3_3_52_2","doi-asserted-by":"publisher","DOI":"10.1111\/pere.12053"},{"key":"e_1_3_3_53_2","doi-asserted-by":"publisher","DOI":"10.1093\/beheco\/arx002"},{"key":"e_1_3_3_54_2","doi-asserted-by":"crossref","unstructured":"M Fafchamps J Labonne Using split samples to improve inference about causal effects (National Bureau of Economic Research Cambridge MA). (2016).","DOI":"10.3386\/w21842"},{"key":"e_1_3_3_55_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.146.3642.347"},{"key":"e_1_3_3_56_2","first-page":"65","article-title":"A simple sequentially rejective multiple test procedure","volume":"6","author":"Holm S","year":"1979","unstructured":"S Holm, A simple sequentially rejective multiple test procedure. Scand J Stat 6, 65\u201370 (1979).","journal-title":"Scand J Stat"},{"key":"e_1_3_3_57_2","doi-asserted-by":"publisher","DOI":"10.1525\/jer.2007.2.4.3"},{"key":"e_1_3_3_58_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pbio.1002456"},{"key":"e_1_3_3_59_2","doi-asserted-by":"publisher","DOI":"10.3934\/Neuroscience.2014.1.4"},{"key":"e_1_3_3_60_2","doi-asserted-by":"publisher","DOI":"10.1027\/1864-9335\/a000192"}],"container-title":["Proceedings of the National Academy of Sciences"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/www.pnas.org\/syndication\/doi\/10.1073\/pnas.1708274114","content-type":"unspecified","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/pnas.org\/doi\/pdf\/10.1073\/pnas.1708274114","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,9,1]],"date-time":"2023-09-01T11:33:25Z","timestamp":1693568005000},"score":1,"resource":{"primary":{"URL":"https:\/\/pnas.org\/doi\/full\/10.1073\/pnas.1708274114"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,3,12]]},"references-count":60,"journal-issue":{"issue":"11","published-print":{"date-parts":[[2018,3,13]]}},"alternative-id":["10.1073\/pnas.1708274114"],"URL":"https:\/\/doi.org\/10.1073\/pnas.1708274114","relation":{"has-preprint":[{"id-type":"doi","id":"10.31219\/osf.io\/2dxu5","asserted-by":"object"}]},"ISSN":["0027-8424","1091-6490"],"issn-type":[{"value":"0027-8424","type":"print"},{"value":"1091-6490","type":"electronic"}],"subject":[],"published":{"date-parts":[[2018,3,12]]},"assertion":[{"value":"2018-03-12","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1073_2Fpnas_1708290115 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T18:36:57Z","timestamp":1793299017242,"version":"4.5.0"},"reference-count":25,"publisher":"National Academy of Sciences","issue":"11","license":[{"start":{"date-parts":[[2018,9,12]],"date-time":"2018-09-12T00:00:00Z","timestamp":1536710400000},"content-version":"vor","delay-in-days":184,"URL":"http:\/\/www.pnas.org\/site\/aboutpnas\/licenses.xhtml"}],"funder":[{"DOI":"10.13039\/100000879","name":"Alfred P Sloan Foundation","doi-asserted-by":"crossref","award":["PG004545"],"award-info":[{"award-number":["PG004545"]}],"id":[{"id":"10.13039\/100000879","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["1153384"],"award-info":[{"award-number":["1153384"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["www.pnas.org"],"crossmark-restriction":true},"short-container-title":["Proc. Natl. Acad. Sci. U.S.A."],"published-print":{"date-parts":[[2018,3,13]]},"abstract":"<jats:p>\n A key component of scientific communication is sufficient information for other researchers in the field to reproduce published findings. For computational and data-enabled research, this has often been interpreted to mean making available the raw data from which results were generated, the computer code that generated the findings, and any additional information needed such as workflows and input parameters. Many journals are revising author guidelines to include data and code availability. This work evaluates the effectiveness of journal policy that requires the data and code necessary for reproducibility be made available postpublication by the authors upon request. We assess the effectiveness of such a policy by (\n <jats:italic>i<\/jats:italic>\n ) requesting data and code from authors and (\n <jats:italic>ii<\/jats:italic>\n ) attempting replication of the published findings. We chose a random sample of 204 scientific papers published in the journal\n <jats:italic>Science<\/jats:italic>\n after the implementation of their policy in February 2011. We found that we were able to obtain artifacts from 44% of our sample and were able to reproduce the findings for 26%. We find this policy\u2014author remission of data and code postpublication upon request\u2014an improvement over no policy, but currently insufficient for reproducibility.\n <\/jats:p>","DOI":"10.1073\/pnas.1708290115","type":"journal-article","created":{"date-parts":[[2018,3,12]],"date-time":"2018-03-12T15:44:40Z","timestamp":1520869480000},"page":"2584-2589","update-policy":"https:\/\/doi.org\/10.1073\/pnas.cm10313","source":"Crossref","is-referenced-by-count":302,"title":["An empirical analysis of journal policy effectiveness for computational reproducibility"],"prefix":"10.1073","volume":"115","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-2015-7825","authenticated-orcid":false,"given":"Victoria","family":"Stodden","sequence":"first","affiliation":[{"name":"School of Information Sciences, University of Illinois at Urbana\u2013Champaign, Champaign, IL 61820;"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jennifer","family":"Seiler","sequence":"additional","affiliation":[{"name":"Department of Statistics, Columbia University, New York, NY 10027"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhaokun","family":"Ma","sequence":"additional","affiliation":[{"name":"Department of Statistics, Columbia University, New York, NY 10027"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"341","published-online":{"date-parts":[[2018,3,12]]},"reference":[{"key":"e_1_3_4_1_2","doi-asserted-by":"crossref","first-page":"444","DOI":"10.1017\/S1049096500057607","article-title":"Replication, replication","volume":"28","author":"King G","year":"1995","unstructured":"G King, Replication, replication. PS Polit Sci Polit 28, 444\u2013452 (1995).","journal-title":"PS Polit Sci Polit"},{"key":"e_1_3_4_2_2","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1198\/106186007X178663","article-title":"Statistical analyses and reproducible research","volume":"16","author":"Gentleman R","year":"2007","unstructured":"R Gentleman, DT Lang, Statistical analyses and reproducible research. J Comput Graphical Stat 16, 1\u201323 (2007).","journal-title":"J Comput Graphical Stat"},{"key":"e_1_3_4_3_2","doi-asserted-by":"crossref","first-page":"8","DOI":"10.1109\/MCSE.2009.15","article-title":"Reproducible research in computational harmonic analysis","volume":"11","author":"Donoho DL","year":"2009","unstructured":"DL Donoho, A Maleki, IU Rahman, M Shahram, V Stodden, Reproducible research in computational harmonic analysis. Comput Sci Eng 11, 8\u201318 (2009).","journal-title":"Comput Sci Eng"},{"key":"e_1_3_4_4_2","doi-asserted-by":"crossref","first-page":"1240","DOI":"10.1126\/science.aah6168","article-title":"Enhancing reproducibility for computational methods","volume":"354","author":"Stodden V","year":"2016","unstructured":"V Stodden, , Enhancing reproducibility for computational methods. Science 354, 1240\u20131241 (2016).","journal-title":"Science"},{"key":"e_1_3_4_5_2","unstructured":"; National Academies of Sciences Engineering and Medicine Fostering Integrity in Research . (The National Academies Pres Washington DC. (2017)."},{"key":"e_1_3_4_6_2","doi-asserted-by":"crossref","first-page":"1422","DOI":"10.1126\/science.aab2374","article-title":"Promoting an open research culture","volume":"348","author":"Nosek BA","year":"2015","unstructured":"BA Nosek, , Promoting an open research culture. Science 348, 1422\u20131425 (2015).","journal-title":"Science"},{"key":"e_1_3_4_7_2","doi-asserted-by":"crossref","first-page":"e67111","DOI":"10.1371\/journal.pone.0067111","article-title":"Toward reproducible computational research: An empirical analysis of data and code policy adoption by journals","volume":"8","author":"Stodden V","year":"2013","unstructured":"V Stodden, P Guo, Z Ma, Toward reproducible computational research: An empirical analysis of data and code policy adoption by journals. PLoS One 8, e67111 (2013).","journal-title":"PLoS One"},{"key":"e_1_3_4_8_2","doi-asserted-by":"crossref","first-page":"649","DOI":"10.1126\/science.1203354","article-title":"Making data maximally available","volume":"331","author":"Hanson B","year":"2011","unstructured":"B Hanson, A Sugden, B Alberts, Making data maximally available. Science 331, 649 (2011).","journal-title":"Science"},{"key":"e_1_3_4_9_2","unstructured":"; American Association for the Advancement of Science Science journals: Editorial Policies. Available at www.sciencemag.org\/authors\/science-journals-editorial-policies. Accessed September 11 2011. (2011)."},{"key":"e_1_3_4_10_2","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1002\/meet.14504701240","article-title":"Definitions of dataset in the scientific and technical literature","volume":"47","author":"Renear AH","year":"2010","unstructured":"AH Renear, S Sacchi, KM Wickett, Definitions of dataset in the scientific and technical literature. Proc Am Soc Inf Sci Technol 47, 1\u20134 (2010).","journal-title":"Proc Am Soc Inf Sci Technol"},{"key":"e_1_3_4_11_2","unstructured":"V Stodden JM Borwein DH Bailey \u201cSetting the default to reproducible\u2019 in Computational Science Research. SIAM News . Available at https:\/\/sinews.siam.org\/Aboutthe-Author\/setting-the-default-to-reproducible-in-computational-science-research. Accessed February 16 2018. (2013)."},{"key":"e_1_3_4_12_2","doi-asserted-by":"crossref","first-page":"1226","DOI":"10.1126\/science.1213847","article-title":"Reproducible research in computational science","volume":"334","author":"Peng RD","year":"2011","unstructured":"RD Peng, Reproducible research in computational science. Science 334, 1226\u20131227 (2011).","journal-title":"Science"},{"key":"e_1_3_4_13_2","unstructured":"V Stodden DH Bailey J Borwein Set the default to \u2019open\u2019. Notices of the AMS. (2013)."},{"key":"e_1_3_4_14_2","doi-asserted-by":"crossref","first-page":"35","DOI":"10.1109\/MCSE.2009.19","article-title":"The legal framework for reproducible scientific research: Licensing and copyright","volume":"11","author":"Stodden V","year":"2009","unstructured":"V Stodden, The legal framework for reproducible scientific research: Licensing and copyright. Comput Sci Eng 11, 35\u201340 (2009).","journal-title":"Comput Sci Eng"},{"key":"e_1_3_4_15_2","doi-asserted-by":"crossref","first-page":"149","DOI":"10.1038\/ng.295","article-title":"Repeatability of published microarray gene expression analyses","volume":"41","author":"Ioannidis JPA","year":"2008","unstructured":"JPA Ioannidis, , Repeatability of published microarray gene expression analyses. Nat Genet 41, 149\u2013155 (2008).","journal-title":"Nat Genet"},{"key":"e_1_3_4_16_2","doi-asserted-by":"crossref","first-page":"4925","DOI":"10.1111\/j.1365-294X.2012.05754.x","article-title":"Recommendations for utilizing and reporting population genetic analyses: The reproducibility of genetic clustering using the program structure","volume":"21","author":"Gilbert KJ","year":"2012","unstructured":"KJ Gilbert, , Recommendations for utilizing and reporting population genetic analyses: The reproducibility of genetic clustering using the program structure. Mol Ecol 21, 4925\u20134930 (2012).","journal-title":"Mol Ecol"},{"key":"e_1_3_4_17_2","first-page":"1","article-title":"Is economics research replicable? Sixty published papers from thirteen journals say \u201cusually not\u201d","volume":"83","author":"Chang AC","year":"2015","unstructured":"AC Chang, L Phillip, Is economics research replicable? Sixty published papers from thirteen journals say \u201cusually not\u201d. Finance Econ Discussion Ser 83, 1\u201326 (2015).","journal-title":"Finance Econ Discussion Ser"},{"key":"e_1_3_4_18_2","unstructured":"M Fuentes Reproducible research in JASA. AMSTAT News. Available at https:\/\/magazine.amstat.org\/blog\/2016\/07\/01\/jasa-reproducible16\/. Accessed February 16 2018. (2016)."},{"key":"e_1_3_4_19_2","doi-asserted-by":"crossref","first-page":"112","DOI":"10.1017\/CBO9781107590205","volume-title":"Privacy, Big Data, and the Public Good","author":"Lane J","year":"2014","unstructured":"J Lane, V Stodden, S Bender, H Nissenbaum, Enabling reproducibility in big data research: Balancing confidentiality and scientific transparency. Privacy, Big Data, and the Public Good (Cambridge Univ Press, New York), pp. 112\u2013132 (2014)."},{"key":"e_1_3_4_20_2","doi-asserted-by":"crossref","first-page":"594","DOI":"10.1111\/jlme.12302","article-title":"Certificates of confidentiality: Protecting human subject research data in law and practice","volume":"43","author":"Wolf LE","year":"2015","unstructured":"LE Wolf, , Certificates of confidentiality: Protecting human subject research data in law and practice. J L Med Ethics 43, 594\u2013609 (2015).","journal-title":"J L Med Ethics"},{"key":"e_1_3_4_21_2","doi-asserted-by":"crossref","first-page":"393","DOI":"10.1093\/aje\/kwx065","article-title":"Reproducing epidemiologic research and ensuring transparency","volume":"186","author":"Coughlin S","year":"2017","unstructured":"S Coughlin, Reproducing epidemiologic research and ensuring transparency. Am J Epidemiol 186, 393\u2013394 (2017).","journal-title":"Am J Epidemiol"},{"key":"e_1_3_4_22_2","doi-asserted-by":"crossref","first-page":"513","DOI":"10.1016\/B978-0-44-451875-0.00019-1","article-title":"Perturbation methods for protecting numerical data: Evolution and evaluation","volume":"28","author":"Muralidhar K","year":"2012","unstructured":"K Muralidhar, R Sarathy, Perturbation methods for protecting numerical data: Evolution and evaluation. Handbook of Statistics 28, 513\u2013531 (2012).","journal-title":"Handbook of Statistics"},{"key":"e_1_3_4_23_2","doi-asserted-by":"crossref","first-page":"296","DOI":"10.1017\/CBO9781107590205","volume-title":"Privacy, Big Data, and the Public Good","author":"Lane J","year":"2014","unstructured":"J Lane, V Stodden, S Bender, H Nissenbaum, Differential privacy: A cryptographic approach to private data analysis. Privacy, Big Data, and the Public Good (Cambridge Univ Press, New York), pp. 296\u2013320 (2014)."},{"key":"e_1_3_4_24_2","doi-asserted-by":"crossref","DOI":"10.1145\/2602649.2602651","article-title":"A primer on provenance","volume":"12","author":"Carata L","year":"2014","unstructured":"L Carata, , A primer on provenance. ACM Queue 12 (2014).","journal-title":"ACM Queue"},{"key":"e_1_3_4_25_2","article-title":"Computing Environments for Reproducibility: Capturing the \u201cWhole Tale.\u201d","author":"Brinckman A","year":"2018","unstructured":"A Brinckman, , Computing Environments for Reproducibility: Capturing the \u201cWhole Tale.\u201d. Future Gener Comp Sy, February 10, 2018).","journal-title":"Future Gener Comp Sy"}],"container-title":["Proceedings of the National Academy of Sciences"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/www.pnas.org\/syndication\/doi\/10.1073\/pnas.1708290115","content-type":"unspecified","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/pnas.org\/doi\/pdf\/10.1073\/pnas.1708290115","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,6,7]],"date-time":"2022-06-07T12:56:04Z","timestamp":1654606564000},"score":1,"resource":{"primary":{"URL":"https:\/\/pnas.org\/doi\/full\/10.1073\/pnas.1708290115"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,3,12]]},"references-count":25,"journal-issue":{"issue":"11","published-print":{"date-parts":[[2018,3,13]]}},"alternative-id":["10.1073\/pnas.1708290115"],"URL":"https:\/\/doi.org\/10.1073\/pnas.1708290115","relation":{},"ISSN":["0027-8424","1091-6490"],"issn-type":[{"value":"0027-8424","type":"print"},{"value":"1091-6490","type":"electronic"}],"subject":[],"published":{"date-parts":[[2018,3,12]]},"assertion":[{"value":"2018-03-12","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1088_2F1742_6596_2F46_2F1_2F067 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,20]],"date-time":"2026-10-20T16:22:36Z","timestamp":1792513356076,"version":"4.5.0"},"reference-count":11,"publisher":"IOP Publishing","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["J. Phys.: Conf. Ser."],"published-print":{"date-parts":[[2006,9,1]]},"DOI":"10.1088\/1742-6596\/46\/1\/067","type":"journal-article","created":{"date-parts":[[2006,9,21]],"date-time":"2006-09-21T03:27:44Z","timestamp":1158809264000},"page":"494-499","source":"Crossref","is-referenced-by-count":269,"title":["Berkeley lab checkpoint\/restart (BLCR) for Linux clusters"],"prefix":"10.1088","volume":"46","author":[{"given":"Paul H","family":"Hargrove","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jason C","family":"Duell","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"266","published-online":{"date-parts":[[2006,9,20]]},"reference":[{"key":"1","author":""},{"key":"2","author":"Geist A","year":"2001"},{"key":"3","first-page":"546","volume":"16","author":"Bode B","year":"2005"},{"key":"4","author":"Duell J","year":"2002"},{"key":"5","author":"Roman E","year":"2003"},{"key":"6","author":"Zhong H","year":"2002"},{"key":"7","author":""},{"key":"8","author":"Osman S","year":"2002"},{"key":"9","author":"Duell J","year":"2002"},{"key":"10","author":"Sankaran S","year":"2003"},{"key":"11","doi-asserted-by":"publisher","DOI":"10.1177\/1094342005056139"}],"container-title":["Journal of Physics: Conference Series"],"original-title":[],"link":[{"URL":"http:\/\/stacks.iop.org\/1742-6596\/46\/i=1\/a=067\/pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,4,11]],"date-time":"2020-04-11T01:31:19Z","timestamp":1586568679000},"score":1,"resource":{"primary":{"URL":"https:\/\/iopscience.iop.org\/article\/10.1088\/1742-6596\/46\/1\/067"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2006,9,1]]},"references-count":11,"URL":"https:\/\/doi.org\/10.1088\/1742-6596\/46\/1\/067","relation":{},"ISSN":["1742-6588","1742-6596"],"issn-type":[{"value":"1742-6588","type":"print"},{"value":"1742-6596","type":"electronic"}],"subject":[],"published":{"date-parts":[[2006,9,1]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1093_2Fbioinformatics_2Fbts480 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T00:49:42Z","timestamp":1793148582666,"version":"4.5.0"},"reference-count":15,"publisher":"Oxford University Press (OUP)","issue":"19","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2012,10,1]]},"abstract":"<jats:title>Abstract<\/jats:title>\n <jats:p>Summary: Snakemake is a workflow engine that provides a readable Python-based workflow definition language and a powerful execution environment that scales from single-core workstations to compute clusters without modifying the workflow. It is the first system to support the use of automatically inferred multiple named wildcards (or variables) in input and output filenames.<\/jats:p>\n <jats:p>Availability: \u00a0http:\/\/snakemake.googlecode.com.<\/jats:p>\n <jats:p>Contact: \u00a0johannes.koester@uni-due.de<\/jats:p>","DOI":"10.1093\/bioinformatics\/bts480","type":"journal-article","created":{"date-parts":[[2012,8,20]],"date-time":"2012-08-20T21:12:25Z","timestamp":1345497145000},"page":"2520-2522","source":"Crossref","is-referenced-by-count":2762,"title":["Snakemake\u2014a scalable bioinformatics workflow engine"],"prefix":"10.1093","volume":"28","author":[{"given":"Johannes","family":"K\u00f6ster","sequence":"first","affiliation":[{"name":"1 Genome Informatics, Institute of Human Genetics, University of Duisburg-Essen and 2Paediatric Oncology, University Childrens Hospital, 45147 Essen, Germany"},{"name":"1 Genome Informatics, Institute of Human Genetics, University of Duisburg-Essen and 2Paediatric Oncology, University Childrens Hospital, 45147 Essen, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sven","family":"Rahmann","sequence":"additional","affiliation":[{"name":"1 Genome Informatics, Institute of Human Genetics, University of Duisburg-Essen and 2Paediatric Oncology, University Childrens Hospital, 45147 Essen, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2012,8,20]]},"reference":[{"key":"2023012513143574900_bts480-B1","doi-asserted-by":"crossref","first-page":"1203","DOI":"10.1002\/1097-024X(200009)30:11<1203::AID-SPE338>3.0.CO;2-N","article-title":"An open graph visualization system and its applications to software engineering","volume":"30","author":"Gansner","year":"2000","journal-title":"Software Pract Exper"},{"key":"2023012513143574900_bts480-B2","doi-asserted-by":"crossref","first-page":"R86","DOI":"10.1186\/gb-2010-11-8-r86","article-title":"Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences","volume":"11","author":"Goecks","year":"2010","journal-title":"Genome Biol."},{"key":"2023012513143574900_bts480-B3","doi-asserted-by":"crossref","first-page":"2778","DOI":"10.1093\/bioinformatics\/btq524","article-title":"Ruffus: a lightweight Python library for computational pipelines","volume":"26","author":"Goodstadt","year":"2010","journal-title":"Bioinformatics"},{"key":"2023012513143574900_bts480-B4","doi-asserted-by":"crossref","first-page":"7","DOI":"10.1038\/nmeth.1809","article-title":"GeneProf: analysis of high-throughput sequencing experiments","volume":"9","author":"Halbritter","year":"2011","journal-title":"Nat. Methods"},{"key":"2023012513143574900_bts480-B5","doi-asserted-by":"crossref","first-page":"1904","DOI":"10.1101\/gr.1363103","article-title":"Biopipe: a flexible framework for protocol-based bioinformatics analysis","volume":"13","author":"Hoon","year":"2003","journal-title":"Genome Res."},{"key":"2023012513143574900_bts480-B6","doi-asserted-by":"crossref","first-page":"90","DOI":"10.1109\/MCSE.2007.55","article-title":"Matplotlib: a 2D graphics environment","volume":"9","author":"Hunter","year":"2007","journal-title":"Comput. Sci. Eng."},{"key":"2023012513143574900_bts480-B7","doi-asserted-by":"crossref","first-page":"1754","DOI":"10.1093\/bioinformatics\/btp324","article-title":"Fast and accurate short read alignment with Burrows\u2013Wheeler transform","volume":"25","author":"Li","year":"2009","journal-title":"Bioinformatics"},{"key":"2023012513143574900_bts480-B8","doi-asserted-by":"crossref","first-page":"2078","DOI":"10.1093\/bioinformatics\/btp352","article-title":"The sequence Alignment\/Map format and SAMtools","volume":"25","author":"Li","year":"2009","journal-title":"Bioinformatics"},{"key":"2023012513143574900_bts480-B9","doi-asserted-by":"crossref","first-page":"685","DOI":"10.1038\/nrg2841","article-title":"Advances in understanding cancer genomes through second-generation sequencing","volume":"11","author":"Meyerson","year":"2010","journal-title":"Nat. Rev. Genet."},{"key":"2023012513143574900_bts480-B10","doi-asserted-by":"crossref","first-page":"3045","DOI":"10.1093\/bioinformatics\/bth361","article-title":"Taverna: a tool for the composition and enactment of bioinformatics workflows","volume":"20","author":"Oinn","year":"2004","journal-title":"Bioinformatics"},{"key":"2023012513143574900_bts480-B11","doi-asserted-by":"crossref","first-page":"1525","DOI":"10.1093\/bioinformatics\/bts167","article-title":"Bpipe: a tool for running and managing bioinformatics pipelines","volume":"28","author":"Sadedin","year":"2012","journal-title":"Bioinformatics"},{"key":"2023012513143574900_bts480-B12","doi-asserted-by":"crossref","first-page":"40","DOI":"10.1186\/1471-2105-5-40","article-title":"Pegasys: software for executing and integrating analyses of biological sequences","volume":"5","author":"Shah","year":"2004","journal-title":"BMC Bioinformatics"},{"key":"2023012513143574900_bts480-B13","volume-title":"GNU Make\u2014A Program for Directing Recompilation","author":"Stallman","year":"1991"},{"key":"2023012513143574900_bts480-B14","first-page":"356","article-title":"Pwrake: a parallel and distributed flexible workflow management tool for wide-area data intensive computing","volume-title":"HPDC\u201910, ACM","author":"Tanaka","year":"2010"},{"key":"2023012513143574900_bts480-B15","article-title":"Design and Implementation of {GXP} Make \u2013 A Workflow System Based on Make","author":"Taura","year":"2010","journal-title":"In IEEE International Conference on eScience"}],"container-title":["Bioinformatics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article-pdf\/28\/19\/2520\/48879301\/bioinformatics_28_19_2520.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article-pdf\/28\/19\/2520\/48879301\/bioinformatics_28_19_2520.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,25]],"date-time":"2023-01-25T14:16:22Z","timestamp":1674656182000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/bioinformatics\/article\/28\/19\/2520\/290322"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012,8,20]]},"references-count":15,"journal-issue":{"issue":"19","published-print":{"date-parts":[[2012,10,1]]}},"URL":"https:\/\/doi.org\/10.1093\/bioinformatics\/bts480","relation":{},"ISSN":["1367-4811","1367-4803"],"issn-type":[{"value":"1367-4811","type":"electronic"},{"value":"1367-4803","type":"print"}],"subject":[],"published-other":{"date-parts":[[2012,10,1]]},"published":{"date-parts":[[2012,8,20]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1098_2Frsta_2020_0069 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,24]],"date-time":"2026-09-24T14:25:10Z","timestamp":1790259910591,"version":"4.1.0"},"reference-count":122,"publisher":"The Royal Society","issue":"2197","license":[{"start":{"date-parts":[[2021,3,29]],"date-time":"2021-03-29T00:00:00Z","timestamp":1616976000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/royalsociety.org\/journals\/ethics-policies\/data-sharing-mining\/"}],"funder":[{"DOI":"10.13039\/100000144","name":"Division of Computer and Network Systems","doi-asserted-by":"crossref","award":["1646305"],"award-info":[{"award-number":["1646305"]}],"id":[{"id":"10.13039\/100000144","id-type":"DOI","asserted-by":"crossref"}]},{"name":"Office of Cyberinfrastructure","award":["1839010"],"award-info":[{"award-number":["1839010"]}]}],"content-domain":{"domain":["royalsocietypublishing.org"],"crossmark-restriction":true},"short-container-title":["Phil. Trans. R. Soc. A."],"published-print":{"date-parts":[[2021,5,17]]},"abstract":"<jats:p>\n We carry out efforts to reproduce computational results for seven published articles and identify barriers to computational reproducibility. We then derive three principles to guide the practice and dissemination of reproducible computational research: (i) Provide transparency regarding how computational results are produced; (ii) When writing and releasing research software, aim for ease of (re-)executability; (iii) Make any code upon which the results rely as deterministic as possible. We then exemplify these three principles with 12 specific guidelines for their implementation in practice. We illustrate the three principles of reproducible research with a series of vignettes from our experimental reproducibility work. We define a novel\n <jats:italic>Reproduction Package<\/jats:italic>\n , a formalism that specifies a structured way to share computational research artifacts that implements the guidelines generated from our reproduction efforts to allow others to build, reproduce and extend computational science. We make our reproduction efforts in this paper publicly available as exemplar\n <jats:italic>Reproduction Packages<\/jats:italic>\n .\n <\/jats:p>\n <jats:p>\n This article is part of the theme issue \u2018Reliability and reproducibility in computational science: implementing verification, validation and uncertainty quantification\n <jats:italic>in silico<\/jats:italic>\n \u2019.\n <\/jats:p>","DOI":"10.1098\/rsta.2020.0069","type":"journal-article","created":{"date-parts":[[2021,3,29]],"date-time":"2021-03-29T03:05:47Z","timestamp":1616987147000},"update-policy":"https:\/\/doi.org\/10.1098\/crossmark-policy","source":"Crossref","is-referenced-by-count":29,"title":["Learning from reproducing computational results: introducing three principles and the\n <i>Reproduction Package<\/i>"],"prefix":"10.1098","volume":"379","author":[{"given":"M. S.","family":"Krafczyk","sequence":"first","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Urbana, IL, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"A.","family":"Shi","sequence":"additional","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Urbana, IL, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"A.","family":"Bhaskar","sequence":"additional","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Urbana, IL, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"D.","family":"Marinov","sequence":"additional","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Urbana, IL, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2015-7825","authenticated-orcid":true,"given":"V.","family":"Stodden","sequence":"additional","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Urbana, IL, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"175","published-online":{"date-parts":[[2021,3,29]]},"reference":[{"key":"e_1_3_6_2_2","unstructured":"National Academies of Sciences Engineering and Medicine Reproducibility and Replicability in Science . Washington DC: The National Academies Press 2019."},{"key":"e_1_3_6_3_2","unstructured":"Claerbout J. 1994 Hypertext documents about reproducible research. Technical Report. Stanford University."},{"key":"e_1_3_6_4_2","doi-asserted-by":"crossref","unstructured":"Buckheit JB Donoho DL. 1995 Wavelab and reproducible research. In Wavelets and statistics (eds A Antoniadis G Oppenheim). Springer.","DOI":"10.1007\/978-1-4612-2544-7_5"},{"key":"e_1_3_6_5_2","doi-asserted-by":"publisher","DOI":"10.1017\/S1049096500057607"},{"key":"e_1_3_6_6_2","doi-asserted-by":"publisher","DOI":"10.1109\/5992.881708"},{"key":"e_1_3_6_7_2","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2009.15"},{"key":"e_1_3_6_8_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.aab2374"},{"key":"e_1_3_6_9_2","unstructured":"Barba LA. 2018 Terminologies for reproducible research. ArXiv e-prints ."},{"key":"e_1_3_6_10_2","doi-asserted-by":"publisher","DOI":"10.1038\/467753a"},{"key":"e_1_3_6_11_2","doi-asserted-by":"crossref","unstructured":"Stodden V Krafczyk MS Bhaskar A. 2018 Enabling the verification of computational results: an empirical evaluation of computational reproducibility. In Proc. First International Workshop on Practical Reproducible Evaluation of Computer Systems ser. P-RECS\u201918 pp. 3:1\u20133:5. New York NY USA: ACM. [Online]. Available: http:\/\/doi.acm.org\/10.1145\/3214239.3214242.","DOI":"10.1145\/3214239.3214242"},{"key":"e_1_3_6_12_2","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1708290115"},{"key":"e_1_3_6_13_2","unstructured":"Treister E Haber E. 2016 A fast marching algorithm for the factored eikonal equation. J. Comput. Phys. 324 210\u2013225. (doi:10.1016\/j.jcp.2016.08.012)"},{"key":"e_1_3_6_14_2","unstructured":"lei Lin X Lu X Ng MK Sun H-W. 2016 A fast accurate approximation method with multigrid solver for two-dimensional fractional sub-diffusion equation. J. Comput. Phys. 323 204\u2013218. (doi:10.1016\/j.jcp.2016.07.031)"},{"key":"e_1_3_6_15_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jcp.2016.10.022"},{"key":"e_1_3_6_16_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jcp.2016.11.009"},{"key":"e_1_3_6_17_2","unstructured":"Zhao W Yong W-A. 2017 Single-node second-order boundary schemes for the lattice Boltzmann method. J. Comput. Phys. 329 1\u201315. (doi:10.1016\/j.jcp.2016.10.049)"},{"key":"e_1_3_6_18_2","unstructured":"Madadi-Kandjani E Passalacqua A. 2015 An extended quadrature-based moment method with log-normal kernel density functions. Chem. Eng. Sci. 131 323\u2013339. (doi:10.1016\/j.ces.2015.04.005)"},{"key":"e_1_3_6_19_2","unstructured":"Hernandez M Rubio M. 2004 A modification of Newton\u2019s method for nondifferentiable equations. J. Comput. Appl. Math. 164\u2013165 409\u2013417. proceedings of the 10th International Congress on Computational and Applied Mathematics. [Online]. Available: http:\/\/www.sciencedirect.com\/science\/article\/pii\/S0377042703006502. (doi:10.1016\/S0377-0427(03)00650-2)"},{"key":"e_1_3_6_20_2","unstructured":"Claerbout JF Karrenbach M. 1992 Electronic documents give reproducible research a new meaning. In SEG Technical Program Expanded Abstracts 1992 . Society of Exploration Geophysicists pp. 601\u2013604. (doi:10.1190\/1.1822162)"},{"key":"e_1_3_6_21_2","unstructured":"Gentleman R Lang D. 2004 Statistical analyses and reproducible research. Bioconductor Project Working Papers no. 2."},{"key":"e_1_3_6_22_2","volume-title":"Statistical Methods for Environmental Epidemiology With R: A Case Study in Air Pollution and Health","author":"Peng RD","year":"2008","unstructured":"Peng RD, Dominici F. 2008 Statistical Methods for Environmental Epidemiology With R: A Case Study in Air Pollution and Health. Springer Publishing Company."},{"key":"e_1_3_6_23_2","unstructured":"Barba LA. 2012 Reproducibility pi manifesto. Dec. [Online]. Available: https:\/\/figshare.com\/articles\/presentation\/Reproducibility_PI_Manifesto\/104539\/1."},{"key":"e_1_3_6_24_2","unstructured":"Reproducibility in science: a guide to enhancing reproducibility in scientific results and writing. http:\/\/ropensci.github.io\/reproducibility-guide\/ 2015."},{"key":"e_1_3_6_25_2","first-page":"20","article-title":"Standing on the shoulders of giants by managing scientific experiments like software","volume":"41","author":"Jimenez I","year":"2016","unstructured":"Jimenez I, Sevilla M, Watkins N, Maltzahn C, Lofstead J, Mohror K, Arpaci-Dusseau R, Arpaci-Dusseau A. 2016 Standing on the shoulders of giants by managing scientific experiments like software. The USENIX Magazine 41, 20\u201326.","journal-title":"The USENIX Magazine"},{"key":"e_1_3_6_26_2","unstructured":"Perkel J. 2017 Techblog: My digital toolbox: Lorena barba. Apr. [Online]. Available: http:\/\/blogs.nature.com\/naturejobs\/2017\/04\/17\/techblog-my-digital-toolbox-lorena-barba\/."},{"key":"e_1_3_6_27_2","unstructured":"N\u00fcst D Boettiger C Marwick B. 2018 How to read a research compendium. CoRR vol. abs\/1806.09525. [Online]. Available: http:\/\/arxiv.org\/abs\/1806.09525."},{"key":"e_1_3_6_28_2","doi-asserted-by":"crossref","unstructured":"Chard K et al. 2019 Implementing computational reproducibility in the whole tale environment. In Proceedings of the 2nd International Workshop on Practical Reproducible Evaluation of Computer Systems ser. P-RECS 2019 pp. 17\u201322. New York NY USA: Association for Computing Machinery. [Online]. Available: https:\/\/doi.org\/10.1145\/3322790.3330594.","DOI":"10.1145\/3322790.3330594"},{"key":"e_1_3_6_29_2","unstructured":"Lu Y-H Thiruvathukal GK. 2019 Reproducibility. In Software Engineering for Machine Learning . Zenodo Jun. [Online]. Available: https:\/\/doi.org\/10.5281\/zenodo.3243385."},{"key":"e_1_3_6_30_2","doi-asserted-by":"crossref","unstructured":"Stodden V. 2020 Beyond open data: a model for linking digital artifacts to enable reproducibility of scientific claims. In Proceedings of the 3rd International Workshop on Practical Reproducible Evaluation of Computer Systems ser. P-RECS\u201920 pp. 9\u201314. New York NY USA: Association for Computing Machinery. [Online]. Available: https:\/\/doi.org\/10.1145\/3391800.3398172.","DOI":"10.1145\/3391800.3398172"},{"key":"e_1_3_6_31_2","doi-asserted-by":"publisher","DOI":"10.1162\/99608f92.25982dcf"},{"key":"e_1_3_6_32_2","doi-asserted-by":"publisher","DOI":"10.1016\/0039-3681(84)90005-0"},{"key":"e_1_3_6_33_2","volume-title":"Changing Order: Replication and Induction in Scientific Practice","author":"Collins H","year":"1985","unstructured":"Collins H. 1985 Changing Order: Replication and Induction in Scientific Practice. Sage Publications."},{"key":"e_1_3_6_34_2","doi-asserted-by":"publisher","DOI":"10.1215\/00182702-23-1-143"},{"key":"e_1_3_6_35_2","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2010.113"},{"key":"e_1_3_6_36_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.1213847"},{"key":"e_1_3_6_37_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.aah6168"},{"key":"e_1_3_6_38_2","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612465253"},{"key":"e_1_3_6_39_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0005996"},{"key":"e_1_3_6_40_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pbio.1001609"},{"key":"e_1_3_6_41_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.0020124"},{"key":"e_1_3_6_42_2","doi-asserted-by":"publisher","DOI":"10.1001\/jama.294.2.218"},{"key":"e_1_3_6_43_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pbio.1001636"},{"key":"e_1_3_6_44_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0010271"},{"key":"e_1_3_6_45_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0026828"},{"key":"e_1_3_6_46_2","doi-asserted-by":"publisher","DOI":"10.1145\/2743015"},{"key":"e_1_3_6_47_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0024357"},{"key":"e_1_3_6_48_2","first-page":"587","article-title":"Replication in empirical economics: the journal of money, credit and banking project","volume":"76","author":"Dewald WG","year":"1986","unstructured":"Dewald WG, Thursby JG, Anderson RG. 1986 Replication in empirical economics: the journal of money, credit and banking project. Am. Econ. Rev. 76, 587\u2013603.","journal-title":"Am. Econ. Rev."},{"key":"e_1_3_6_49_2","doi-asserted-by":"publisher","DOI":"10.1093\/aje\/kwj093"},{"key":"e_1_3_6_50_2","doi-asserted-by":"publisher","DOI":"10.1038\/ng.295"},{"key":"e_1_3_6_51_2","doi-asserted-by":"publisher","DOI":"10.1038\/nrd3439"},{"key":"e_1_3_6_52_2","doi-asserted-by":"publisher","DOI":"10.1038\/nrd3439-c1"},{"key":"e_1_3_6_53_2","doi-asserted-by":"publisher","DOI":"10.1038\/483531a"},{"key":"e_1_3_6_54_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.neuroimage.2012.07.004"},{"key":"e_1_3_6_55_2","unstructured":"Chang AC Li P. 2015 Is economics research replicable? Sixty published papers from thirteen journals say usually not. Finance and Economics Discussion Series 2015-083 pp. 1\u201326 Washington DC: Board of Governors of the Federal Reserve System (doi:10.17016\/FEDS.2015.083)"},{"key":"e_1_3_6_56_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.aab2069"},{"key":"e_1_3_6_57_2","doi-asserted-by":"publisher","DOI":"10.1145\/2812803"},{"key":"e_1_3_6_58_2","doi-asserted-by":"publisher","DOI":"10.1038\/nrd.2016.184"},{"key":"e_1_3_6_59_2","doi-asserted-by":"publisher","DOI":"10.1136\/bmjopen-2016-011784"},{"key":"e_1_3_6_60_2","doi-asserted-by":"publisher","DOI":"10.1145\/2723872.2723882"},{"key":"e_1_3_6_61_2","volume-title":"Object Oriented Design: With Applications","author":"Booch G","year":"1991","unstructured":"Booch G. 1991 Object Oriented Design: With Applications. Benjamin\/Cummings Publishing."},{"key":"e_1_3_6_62_2","doi-asserted-by":"crossref","unstructured":"Hilton M Nelson N Tunnell T Marinov D Dig D. 2017 Trade-offs in continuous integration: assurance security and flexibility. In Proc. 2017 11th Joint Meeting on Foundations of Software Engineering ser. ESEC\/FSE 2017 pp. 197\u2013207. New York NY USA: ACM.","DOI":"10.1145\/3106237.3106270"},{"key":"e_1_3_6_63_2","doi-asserted-by":"crossref","unstructured":"Hilton M Tunnell T Huang K Marinov D Dig D. 2016 Usage costs and benefits of continuous integration in open-source projects. In Proc. 31st IEEE\/ACM International Conference on Automated Software Engineering ser. ASE 2016 pp. 426\u2013437. New York NY USA: ACM.","DOI":"10.1145\/2970276.2970358"},{"key":"e_1_3_6_64_2","doi-asserted-by":"crossref","unstructured":"Krafczyk M Shi A Bhaskar A Marinov D Stodden V. 2019 Scientific tests and continuous integration strategies to enhance reproducibility in the scientific software context. In Proc. 2nd International Workshop on Practical Reproducible Evaluation of Computer Systems ser. P-RECS 2019 pp. 23\u201328. New York NY USA: Association for Computing Machinery.","DOI":"10.1145\/3322790.3330595"},{"key":"e_1_3_6_65_2","doi-asserted-by":"publisher","DOI":"10.1038\/nbt.3780"},{"key":"e_1_3_6_66_2","doi-asserted-by":"publisher","DOI":"10.1109\/ACCESS.2018.2822835"},{"key":"e_1_3_6_67_2","doi-asserted-by":"publisher","DOI":"10.3389\/fnins.2018.00727"},{"key":"e_1_3_6_68_2","doi-asserted-by":"crossref","unstructured":"Jimenez I Arpaci-Dusseau A Arpaci-Dusseau R Lofstead J Maltzahn C Mohror K Ricci R. 2017 Popperci: automated reproducibility validation. In 2017 IEEE Conference on Computer Communications Workshops (INFOCOM WKSHPS) pp. 450\u2013455.","DOI":"10.1109\/INFCOMW.2017.8116506"},{"key":"e_1_3_6_69_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jcp.2016.08.017"},{"key":"e_1_3_6_70_2","first-page":"9","article-title":"The fenics project version 1.5","volume":"3","author":"Aln\u00e6s MS","year":"2015","unstructured":"Aln\u00e6s MS et al. 2015 The fenics project version 1.5. Arch. Numer. Softw. 3, 9\u201323.","journal-title":"Arch. Numer. Softw."},{"key":"e_1_3_6_71_2","unstructured":"Jasak H Jemcov A Kingdom U. 2007 Openfoam: A c++ library for complex physics simulations. In International Workshop on Coupled Methods in Numerical Dynamics IUC pp. 1\u201320."},{"key":"e_1_3_6_72_2","doi-asserted-by":"crossref","unstructured":"Arteaga A Fuhrer O Hoefler T. 2014 Designing bit-reproducible portable high-performance applications. In 2014 IEEE 28th International Parallel and Distributed Processing Symposium May pp. 1235\u20131244.","DOI":"10.1109\/IPDPS.2014.127"},{"key":"e_1_3_6_73_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pcbi.1005412"},{"key":"e_1_3_6_74_2","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2009.19"},{"key":"e_1_3_6_75_2","article-title":"Docker: Lightweight Linux containers for consistent development and deployment","author":"Merkel D","year":"2014","unstructured":"Merkel D. 2014 Docker: Lightweight Linux containers for consistent development and deployment. Linux J. (https:\/\/www.linuxjournal.com\/content\/docker-lightweight-linux-containers-consistent-development-and-deployment)","journal-title":"Linux J."},{"key":"e_1_3_6_76_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0177459"},{"key":"e_1_3_6_77_2","doi-asserted-by":"crossref","unstructured":"Gamblin T LeGendre M Collette MR Lee GL Moody A de Supinski BR Futral S. 2015 The Spack package manager: bringing order to HPC software chaos. In Proc. International Conference for High Performance Computing Networking Storage and Analysis ser. SC\u201915 pp. 40:1\u201340:12. New York NY USA: ACM.","DOI":"10.1145\/2807591.2807623"},{"key":"e_1_3_6_78_2","first-page":"54","article-title":"Twelve ways to fool the masses when giving performance results on parallel computers","volume":"4","author":"Bailey D","year":"1991","unstructured":"Bailey D. 1991 Twelve ways to fool the masses when giving performance results on parallel computers. Supercomput. Rev. 4,54\u201355.","journal-title":"Supercomput. Rev."},{"key":"e_1_3_6_79_2","unstructured":"Claerbout J Karrenbach M. 1993 How to use cake with interactive documents. In in SEP-77 427\u2013444 Stanford Exploration Project ."},{"key":"e_1_3_6_80_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jcp.2016.06.053"},{"key":"e_1_3_6_81_2","doi-asserted-by":"publisher","DOI":"10.1002\/jae.709"},{"key":"e_1_3_6_82_2","unstructured":"Zaks S. 2020 What is a Compute Capsule? (accessed October 3 2020). [Online]. Available: https:\/\/help.codeocean.com\/en\/articles\/1204225-what-is-a-compute-capsule."},{"key":"e_1_3_6_83_2","unstructured":"Chard K et al. 2016 I\u2019ll take that to go: Big data bags and minimal identifiers for exchange of large complex datasets. In 2016 IEEE International Conference on Big Data (Big Data) Washington DC 5\u20138 December 2016 pp. 319\u2013328. New York NY: IEEE. (doi:10.1109\/BigData.2016.7840618)"},{"key":"e_1_3_6_84_2","first-page":"108","article-title":"Reproducibility of data-oriented experiments in e-science (Dagstuhl seminar 16041)","volume":"6","author":"Freire J","year":"2016","unstructured":"Freire J, Fuhr N, Rauber A. 2016 Reproducibility of data-oriented experiments in e-science (Dagstuhl seminar 16041). Dagstuhl Rep. 6, 108\u2013159.","journal-title":"Dagstuhl Rep."},{"key":"e_1_3_6_85_2","unstructured":"Aminzadeh F Jean B Kunz T S. of Exploration Geophysicists and E. A. of Exploration Geophysicists. 1997 3-D salt and overthrust models ser. SEG\/EAGE 3-D modeling series; 1. [Tulsa Oklahoma]: Society of Exploration Geophysicists."},{"key":"e_1_3_6_86_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.aml.2006.10.008"},{"key":"e_1_3_6_87_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jcp.2016.10.009"},{"key":"e_1_3_6_88_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.jcp.2016.09.039"},{"key":"e_1_3_6_89_2","doi-asserted-by":"crossref","unstructured":"Chirigati F Rampin R Shasha D Freire J. 2016 Reprozip: Computational reproducibility with ease. In Proc. 2016 International Conference on Management of Data ser. SIGMOD\u201916 pp. 2085\u20132088. New York NY USA: ACM.","DOI":"10.1145\/2882903.2899401"},{"key":"e_1_3_6_90_2","doi-asserted-by":"publisher","DOI":"10.1038\/nbt.3820"},{"key":"e_1_3_6_91_2","first-page":"2016","article-title":"update,\u201d Nucleic Acids Research, vol","volume":"3","year":"2016","unstructured":"2016 update,\u201d Nucleic Acids Research, vol. 44, no. W1, pp. W3\u2013W10, 2016.","journal-title":"44, no. W1, pp."},{"key":"e_1_3_6_92_2","doi-asserted-by":"crossref","unstructured":"Cranmer K Heinrich L. 2017 Yadage and Packtivity\u2014analysis preservation using parametrized workflows. ArXiv e-prints .","DOI":"10.1088\/1742-6596\/898\/10\/102019"},{"key":"e_1_3_6_93_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2014.10.008"},{"key":"e_1_3_6_94_2","unstructured":"Hunold S Tr\u00e4ff JL. 2013 On the state and importance of reproducible experimental research in parallel computing. CoRR vol. abs\/1308.3648. [Online]. Available: http:\/\/arxiv.org\/abs\/1308.3648."},{"key":"e_1_3_6_95_2","unstructured":"Fursin G Lokhmotov A Savenko D Upton E. 2018 A collective knowledge workflow for collaborative research into multi-objective autotuning and machine learning techniques. CoRR vol. abs\/1801.08024."},{"key":"e_1_3_6_96_2","doi-asserted-by":"publisher","DOI":"10.1177\/1094342019839124"},{"key":"e_1_3_6_97_2","unstructured":"Monajemi H Donoho DL Stodden V. 2016 Making massive computational experiments painless. In 2016 IEEE International Conference on Big Data (Big Data) Washington DC 5\u20138 December 2016 pp. 2368\u20132373. New York NY: IEEE. (doi:10.1109\/BigData.2016.7840870)"},{"key":"e_1_3_6_98_2","unstructured":"Kluyver T et al. and J. development team. 2016 Jupyter notebooks \u2013 a publishing format for reproducible computational workflows. In Positioning and Power in Academic Publishing: Players Agents and Agendas (eds F Loizides B Scmidt) pp. 87\u201390. IOS Press."},{"key":"e_1_3_6_99_2","doi-asserted-by":"publisher","DOI":"10.1007\/JHEP04(2011)038"},{"key":"e_1_3_6_100_2","unstructured":"Ludaescher B Chard K Gaffney N Jones MB Nabrzyski J Stodden V Turk M. 2016 Capturing the \u2018whole tale\u2019 of computational research: Reproducibility in computing environments. Arxiv: CoRR ."},{"key":"e_1_3_6_101_2","doi-asserted-by":"publisher","DOI":"10.1198\/106186007X178663"},{"key":"e_1_3_6_102_2","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2015.18"},{"key":"e_1_3_6_103_2","unstructured":"Stodden V Wu X Sochat V. 2018 Aim: an abstraction for improving machine learning prediction. In 2018 IEEE Data Science Workshop (DSW) Lausanne Switzerland 4\u20136 June 2018 pp. 1\u20135. New York NY: IEEE. (doi:10.1109\/DSW.2018.8439914)"},{"key":"e_1_3_6_104_2","unstructured":"Stodden V Bailey D Borwein J LeVeque R Rider W Stein W. 2013 Setting the default to reproducible: reproducibility in computational and experimental mathematics. SIAM News pp. 1\u20134. (https:\/\/sinews.siam.org\/Details-Page\/setting-the-default-to-reproducible-in-computational-science-research)"},{"key":"e_1_3_6_105_2","unstructured":"Bailey D Borwein J Stodden V. 2013 Set the default to \u2018open\u2019. Not. AMS Accepted March."},{"key":"e_1_3_6_106_2","doi-asserted-by":"publisher","DOI":"10.5334\/jors.184"},{"key":"e_1_3_6_107_2","doi-asserted-by":"publisher","DOI":"10.1145\/3188721"},{"key":"e_1_3_6_108_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pcbi.1003285"},{"key":"e_1_3_6_109_2","unstructured":"Crick T Ishtiaq S Hall BA. 2015 Towards \u2018reproducibility-as-a-service\u2019. CoRR vol. abs\/1503.02388. [Online]. Available: http:\/\/arxiv.org\/abs\/1503.02388."},{"key":"e_1_3_6_110_2","doi-asserted-by":"crossref","first-page":"1","DOI":"10.5334\/jors.bt","article-title":"Community recommendations for sustainable scientific software","volume":"3","author":"Downs RR","year":"2015","unstructured":"Downs RR, Lenhardt W, Robinson E, Davis E, Weber N. 2015 Community recommendations for sustainable scientific software. J. Open Res. Softw. 3,1\u20135.","journal-title":"J. Open Res. Softw."},{"key":"e_1_3_6_111_2","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2018.2881905"},{"key":"e_1_3_6_112_2","doi-asserted-by":"publisher","DOI":"10.5334\/jors.ay"},{"key":"e_1_3_6_113_2","unstructured":"Donoho DL Stodden V. 2015 Reproducible research in the mathematical sciences. In The Princeton Companion to Applied Mathematics (eds NJ Higham MR Dennis P Glendinning PA Martin F Santosa J Tanner) pp. 916\u2013925. Princeton NJ USA: Princeton University Press."},{"key":"e_1_3_6_114_2","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511809163"},{"key":"e_1_3_6_115_2","doi-asserted-by":"crossref","unstructured":"Luo Q Hariri F Eloussi L Marinov D. 2014 An empirical analysis of flaky tests. In Proceedings of the 22nd ACM SIGSOFT International Symposium on Foundations of Software Engineering ser. FSE 2014 pp. 643\u2013653. New York NY USA: Association for Computing Machinery. [Online]. Available: https:\/\/doi.org\/10.1145\/2635868.2635920.","DOI":"10.1145\/2635868.2635920"},{"key":"e_1_3_6_116_2","unstructured":"Lam W Oei R Shi A Marinov D Xie T. 2019 iDFlakies: a framework for detecting and partially classifying flaky tests. In 2019 12th IEEE Conference on Software Testing Validation and Verification (ICST) Xi\u2019an China 22\u201327 April 2019 pp. 312\u2013322. New York NY: IEEE. (doi: 10.1109\/ICST.2019.00038)"},{"key":"e_1_3_6_117_2","unstructured":"Gambi A Bell J Zeller A. 2018 Practical test dependency detection. In 2018 IEEE 11th International Conference on Software Testing Verification and Validation (ICST) V\u00e4ster\u00e5s Sweden 9\u201313 April 2018 pp. 1\u201311. New York NY: IEEE. (doi:10.1109\/ICST.2018.00011)"},{"key":"e_1_3_6_118_2","doi-asserted-by":"publisher","DOI":"10.1145\/280324.280335"},{"key":"e_1_3_6_119_2","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.46.6.745.11941"},{"key":"e_1_3_6_120_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0080278"},{"key":"e_1_3_6_121_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0067111"},{"key":"e_1_3_6_122_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0007078"},{"key":"e_1_3_6_123_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.354.6308.142"}],"container-title":["Philosophical Transactions of the Royal Society A: Mathematical, Physical and Engineering Sciences"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/royalsocietypublishing.org\/doi\/pdf\/10.1098\/rsta.2020.0069","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/royalsocietypublishing.org\/doi\/full-xml\/10.1098\/rsta.2020.0069","content-type":"application\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/royalsocietypublishing.org\/doi\/pdf\/10.1098\/rsta.2020.0069","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,12]],"date-time":"2026-05-12T21:25:45Z","timestamp":1778621145000},"score":1,"resource":{"primary":{"URL":"http:\/\/royalsocietypublishing.org\/rsta\/article\/111842"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,3,29]]},"references-count":122,"journal-issue":{"issue":"2197","published-print":{"date-parts":[[2021,5,17]]}},"alternative-id":["10.1098\/rsta.2020.0069"],"URL":"https:\/\/doi.org\/10.1098\/rsta.2020.0069","relation":{},"ISSN":["1364-503X","1471-2962"],"issn-type":[{"value":"1364-503X","type":"print"},{"value":"1471-2962","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,3,29]]},"assertion":[{"value":"2021-01-20","order":1,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2021-03-29","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}],"article-number":"rsta.2020.0069"}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1109_2F2_84898 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,6]],"date-time":"2026-10-06T16:37:57Z","timestamp":1791304677394,"version":"4.3.1"},"reference-count":15,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"9","license":[{"start":{"date-parts":[[1991,9,1]],"date-time":"1991-09-01T00:00:00Z","timestamp":683683200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Computer"],"published-print":{"date-parts":[[1991,9]]},"DOI":"10.1109\/2.84898","type":"journal-article","created":{"date-parts":[[2002,8,24]],"date-time":"2002-08-24T20:00:39Z","timestamp":1030219239000},"page":"39-48","source":"Crossref","is-referenced-by-count":221,"title":["High-availability computer systems"],"prefix":"10.1109","volume":"24","author":[{"given":"J.","family":"Gray","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"D.P.","family":"Siewiorek","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref10","volume":"1","year":"1985","journal-title":"Resilient Computing Systems"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1145\/356725.356729"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.1985.231893"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1147\/rd.281.0002"},{"key":"ref14","doi-asserted-by":"crossref","first-page":"1135","DOI":"10.1109\/TSE.1987.232855","article-title":"on the reliability of the ibm mvs\/xa operating system","volume":"se 13","author":"mourad","year":"1987","journal-title":"IEEE Transactions on Software Engineering"},{"key":"ref15","first-page":"22","author":"bartlett","year":"1981","journal-title":"Eighth Sigops"},{"key":"ref4","first-page":"8","article-title":"Powering Computer-Controlled Systems: AC or DC?","volume":"11","author":"tullis","year":"1984","journal-title":"Telesis"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1109\/24.58719"},{"key":"ref6","author":"siewiorek","year":"1992","journal-title":"Reliable Computer Systems Design and Evaluation"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1109\/FTCSH.1995.532603"},{"key":"ref8","volume":"1 2","author":"pradhan","year":"1986","journal-title":"Fault Tolerant Computing Theory and Techniques"},{"key":"ref7","author":"johnson","year":"1989","journal-title":"Design and Analysis of Fault Tolerant Digital Systems"},{"key":"ref2","author":"watanabe","year":"1986","journal-title":"Survey on Computer Security"},{"key":"ref1","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-7091-8871-2","author":"avizienis","year":"1987","journal-title":"Dependable Computing and Fault-Tolerant Systems"},{"key":"ref9","author":"tanenbaum","year":"1989","journal-title":"Operating Systems Design and Implementation"}],"container-title":["Computer"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx1\/2\/2779\/00084898.pdf?arnumber=84898","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,11,29]],"date-time":"2021-11-29T20:04:35Z","timestamp":1638216275000},"score":1,"resource":{"primary":{"URL":"http:\/\/ieeexplore.ieee.org\/document\/84898\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[1991,9]]},"references-count":15,"journal-issue":{"issue":"9"},"URL":"https:\/\/doi.org\/10.1109\/2.84898","relation":{},"ISSN":["0018-9162"],"issn-type":[{"value":"0018-9162","type":"print"}],"subject":[],"published":{"date-parts":[[1991,9]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1109_2F3468_844354 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T14:25:49Z","timestamp":1793283949005,"version":"4.5.0"},"reference-count":83,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"3","license":[{"start":{"date-parts":[[2000,5,1]],"date-time":"2000-05-01T00:00:00Z","timestamp":957139200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Trans. Syst., Man, Cybern. A"],"published-print":{"date-parts":[[2000,5]]},"DOI":"10.1109\/3468.844354","type":"journal-article","created":{"date-parts":[[2002,8,24]],"date-time":"2002-08-24T15:45:59Z","timestamp":1030203959000},"page":"286-297","source":"Crossref","is-referenced-by-count":3428,"title":["A model for types and levels of human interaction with automation"],"prefix":"10.1109","volume":"30","author":[{"given":"R.","family":"Parasuraman","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"T.B.","family":"Sheridan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"C.D.","family":"Wickens","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref73","article-title":"rtca","year":"1995","journal-title":"Report of the RTCA Board of Director s Select Committee on Free Flight"},{"key":"ref72","first-page":"42","article-title":"aviation week and space technology","year":"1998","journal-title":"Answers to the Gridlock"},{"key":"ref71","author":"merlo","year":"1999","journal-title":"Effects of reliability on cue effectiveness and display signalling"},{"key":"ref70","doi-asserted-by":"publisher","DOI":"10.1080\/001401397188224"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1177\/106480469800600305"},{"key":"ref77","author":"pew","year":"1998","journal-title":"Modeling Human and Organizational Behavior Application to Military Simulations"},{"key":"ref39","first-page":"84","article-title":"the effect of adaptive air traffic control (atc) decision aiding on controller mental workload","author":"hilburn","year":"1997","journal-title":"Human-Automation Interaction Research and Practice"},{"key":"ref74","first-page":"63","article-title":"free flight with airborne separation assurance","author":"van gent","year":"1998","journal-title":"Proc Int Conf Human Computer Interaction in Aeronautics"},{"key":"ref75","author":"fitts","year":"1951","journal-title":"Human Engineering for an Effective Air Navigation and Traffic Control System"},{"key":"ref38","doi-asserted-by":"publisher","DOI":"10.1016\/0301-0511(95)05161-9"},{"key":"ref78","doi-asserted-by":"publisher","DOI":"10.1177\/154193120004401301"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1518\/001872098779480406"},{"key":"ref33","first-page":"627","article-title":"an adaptive human-machine system","author":"hancock","year":"1985","journal-title":"Proc 15th Annual IEEE Conf Syst Man Cybern"},{"key":"ref32","author":"han","year":"0"},{"key":"ref31","first-page":"324","article-title":"cockpit data link technology and flight crew communications procedures","author":"logdson","year":"1995","journal-title":"Proc 8th Int Symp Aviation Psychology"},{"key":"ref30","doi-asserted-by":"crossref","first-page":"189","DOI":"10.1177\/154193129904300313","article-title":"supporting informed consent in human-machine collaboration: the role of conflict type, time pressure, display design, and trust","author":"olson","year":"1999","journal-title":"Proc Human Factors and Ergonomics Society 43rd Annual Meeting"},{"key":"ref37","first-page":"154","article-title":"situation-adaptive autonomy: trading control of authority in human-machine systems","author":"inagaki","year":"1999","journal-title":"Automation Technology and Human Performance Current Research and Trends"},{"key":"ref36","author":"barnes","year":"1985","journal-title":"The Intelligent Assistant Concept for Electronic Warfare Systems"},{"key":"ref35","doi-asserted-by":"crossref","first-page":"431","DOI":"10.1177\/001872088803000405","article-title":"adaptive aiding for human\/computer control","volume":"30","author":"rouse","year":"1988","journal-title":"Human Factors"},{"key":"ref34","author":"parasuraman","year":"1992","journal-title":"Theory and Design of Adaptive Automation in Aviation Systems"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1177\/154193129904300105"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1177\/154193129003401922"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1518\/001872099779656752"},{"key":"ref28","doi-asserted-by":"crossref","first-page":"94","DOI":"10.1177\/001872089403600106","article-title":"design of a cooperative problem-solving system for en-route flight planning: an empirical evaluation","volume":"36","author":"layton","year":"1994","journal-title":"Human Factors"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4757-9244-7_30"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1016\/0951-8320(90)90013-D"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1162\/evco.1999.7.1.45"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1145\/78973.78974"},{"key":"ref29","first-page":"76","article-title":"automatic gcas: you can't fly any lower","author":"scott","year":"1999","journal-title":"Aviation Week and Space Technology"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1080\/00140139208967392"},{"key":"ref67","doi-asserted-by":"crossref","first-page":"184","DOI":"10.1177\/154193129904300312","article-title":"trust as a construct for evaluation of automated aids: past and present theory and research","author":"masalonis","year":"1999","journal-title":"Proc Human Factors and Ergonomics Society 43rd Annual Meeting"},{"key":"ref68","author":"swets","year":"1982","journal-title":"Evaluation of Diagnostic Systems Methods from Signal Detection Theory"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1016\/0005-1098(83)90046-8"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.2514\/3.21637"},{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1080\/00140138008924809"},{"key":"ref20","author":"hart","year":"1979","journal-title":"Cockpit Display of Traffic Information Airline Pilots Opinions about Content Symbology and Format"},{"key":"ref22","first-page":"1944","article-title":"human factors in process control","author":"moray","year":"1997","journal-title":"Handbook of Human Actors and Ergonomics"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1177\/154193129804200113"},{"key":"ref24","author":"mundra","year":"1989","journal-title":"A New Automation Aid to Air Traffic Controllers for Improving Airport Capacity"},{"key":"ref23","doi-asserted-by":"crossref","first-page":"513","DOI":"10.1177\/001872089203400502","article-title":"graphical displays: implications for divided attention, focused attention, and problem solving","volume":"34","author":"bennett","year":"1992","journal-title":"Human Factors"},{"key":"ref26","author":"shortliffe","year":"1976","journal-title":"Computer-Based Medical Consultation Mycin"},{"key":"ref25","doi-asserted-by":"crossref","first-page":"395","DOI":"10.1177\/001872088803000403","article-title":"the role of human factors in expert systems design and acceptance","volume":"30","author":"madni","year":"1988","journal-title":"Human Factors"},{"key":"ref50","author":"wiener","year":"0"},{"key":"ref51","doi-asserted-by":"crossref","first-page":"221","DOI":"10.1177\/001872089303500203","article-title":"modeling strategic behavior in human-automation interaction: why \"aid\" can (and should) go unused","volume":"35","author":"kirlik","year":"1993","journal-title":"Human Factors"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1037\/0278-7393.26.2.395"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1207\/s15327108ijap0301_1"},{"key":"ref57","article-title":"complacency: is the term useful for air safety?","author":"wiener","year":"1981","journal-title":"Proc 26th Corporate Aviation Safety Seminar"},{"key":"ref56","author":"sarter","year":"1995","journal-title":"Strong silent and out-of-the-loop Properties of advanced (cockpit) automation and their impact on human-automation interaction"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1518\/001872095779064555"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1080\/001401399185595"},{"key":"ref53","first-page":"165","article-title":"level of automation effects on telerobot performance and human operator situation awareness and subjective workload","author":"kaber","year":"1999","journal-title":"Automation Technology and Human Performance Current Research and Trends"},{"key":"ref52","author":"landauer","year":"1995","journal-title":"The Trouble With Computers"},{"key":"ref10","author":"wickens","year":"1998","journal-title":"The Future of Air Traffic Control Human Operators and Automation"},{"key":"ref40","first-page":"129","article-title":"adaptive automation of a dynamic control task based on workload assessment through a secondary monitoring task","author":"kaber","year":"1999","journal-title":"Automation Technology and Human Performance Current Research and Trends"},{"key":"ref11","doi-asserted-by":"crossref","DOI":"10.21236\/ADA057655","author":"sheridan","year":"1978","journal-title":"Human and Computer Control of Undersea Teleoperators"},{"key":"ref12","doi-asserted-by":"crossref","first-page":"124","DOI":"10.1177\/154193128903300227","article-title":"a general model of mixed-initiative human-machine systems","author":"riley","year":"1989","journal-title":"Proc 33rd Annual Human Factors Society Conf"},{"key":"ref13","author":"baddeley","year":"1996","journal-title":"Working Memory"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1037\/10037-000"},{"key":"ref15","author":"wickens","year":"1999","journal-title":"Engineering Psychology and Human Performance"},{"key":"ref82","doi-asserted-by":"publisher","DOI":"10.1080\/001401300409125"},{"key":"ref16","author":"gibson","year":"1979","journal-title":"The Ecological Approach to Visual Perception"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.1207\/s15327108ijap0902_3"},{"key":"ref17","author":"wickens","year":"1998","journal-title":"An Introduction to Human Factors Engineering"},{"key":"ref18","article-title":"rumination on automation","author":"sheridan","year":"1998","journal-title":"Proc JFAC-MMS Conf"},{"key":"ref83","author":"norman","year":"1998","journal-title":"The Invisible Computer"},{"key":"ref19","first-page":"365","author":"lee","year":"1996","journal-title":"Maritime automation in &#x201C Automation and Human Performance Theory and Applications&#x201D"},{"key":"ref80","article-title":"empirical and analytic studies of human\/automation dynamics in airspace management for free flight","author":"corker","year":"1997","journal-title":"Proc 10th Int CEAS Conf Free Flight"},{"key":"ref4","doi-asserted-by":"crossref","first-page":"695","DOI":"10.1177\/001872088702900609","article-title":"human-computer monitoring","volume":"29","author":"parasuraman","year":"1987","journal-title":"Human Factors"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/4547.4551"},{"key":"ref6","author":"parasuraman","year":"1996","journal-title":"Automation and Human performance theory and applications"},{"key":"ref5","author":"sheridan","year":"1992","journal-title":"Telerobotics Automation and Supervisory Control"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1518\/001872097778543886"},{"key":"ref7","first-page":"1","article-title":"decomposing automation: apparent simplicity, real complexity","author":"woods","year":"1996","journal-title":"Automation and Human performance theory and applications"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1109\/21.156574"},{"key":"ref9","author":"billings","year":"1997","journal-title":"Aviation Automation The Search for a Human-Centered Approach"},{"key":"ref46","first-page":"1926","article-title":"automation surprises","author":"sarter","year":"1997","journal-title":"Handbook of Human Factors and Ergonomics 2nd ed"},{"key":"ref45","first-page":"67","article-title":"designing for human-agent interaction","volume":"summer","author":"lewis","year":"1998","journal-title":"Artif Intell Mag"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-662-02933-6_7"},{"key":"ref47","article-title":"designing for situation awareness and trust in automation","author":"wickens","year":"1994","journal-title":"Proc IFAC Conf Integrated Systems Engineering"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1518\/001872096778827279"},{"key":"ref41","doi-asserted-by":"publisher","DOI":"10.1037\/1076-898X.6.1.44"},{"key":"ref44","first-page":"37","article-title":"theoretical perspectives on adaptive automation","author":"scerbo","year":"1996","journal-title":"Automation and Human performance theory and applications"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1016\/0003-6870(95)00054-2"}],"container-title":["IEEE Transactions on Systems, Man, and Cybernetics - Part A: Systems and Humans"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx5\/3468\/18307\/00844354.pdf?arnumber=844354","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,11,29]],"date-time":"2021-11-29T15:06:58Z","timestamp":1638198418000},"score":1,"resource":{"primary":{"URL":"http:\/\/ieeexplore.ieee.org\/document\/844354\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2000,5]]},"references-count":83,"journal-issue":{"issue":"3"},"URL":"https:\/\/doi.org\/10.1109\/3468.844354","relation":{},"ISSN":["1083-4427"],"issn-type":[{"value":"1083-4427","type":"print"}],"subject":[],"published":{"date-parts":[[2000,5]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1109_2FMS_2016_60 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T10:17:52Z","timestamp":1793269072453,"version":"4.5.0"},"reference-count":11,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"3","license":[{"start":{"date-parts":[[2016,5,1]],"date-time":"2016-05-01T00:00:00Z","timestamp":1462060800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/ieeexplore.ieee.org\/Xplorehelp\/downloads\/license-information\/IEEE.html"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IEEE Softw."],"published-print":{"date-parts":[[2016,5]]},"DOI":"10.1109\/ms.2016.60","type":"journal-article","created":{"date-parts":[[2016,3,18]],"date-time":"2016-03-18T19:11:04Z","timestamp":1458328264000},"page":"35-41","source":"Crossref","is-referenced-by-count":276,"title":["Chaos Engineering"],"prefix":"10.1109","volume":"33","author":[{"given":"Ali","family":"Basiri","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Niosha","family":"Behnam","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ruud","family":"de Rooij","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lorin","family":"Hochstein","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Luke","family":"Kosewski","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Justin","family":"Reynolds","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Casey","family":"Rosenthal","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref4","doi-asserted-by":"crossref","DOI":"10.1145\/2367376.2371297","article-title":"Resilience Engineering: Learning to Embrace Failure","volume":"10","author":"robbins","year":"2012","journal-title":"ACM Queue"},{"key":"ref3","article-title":"FIT: Failure Injection Testing","author":"andrus","year":"2014","journal-title":"The Netflix Tech Blog"},{"key":"ref10","article-title":"Too Big to Test: Breaking a Production Brokerage Platform without Causing Financial Devastation","author":"parrish","year":"0","journal-title":"presentation at Velocity 2015 Conf"},{"key":"ref6","article-title":"Facebook Turned Off Entire Data Center to Test Resiliency","author":"sverdlik","year":"2014","journal-title":"Data Center Knowledge"},{"key":"ref11","author":"newman","year":"2015","journal-title":"Building Microservices"},{"key":"ref5","article-title":"Inside Azure Search: Chaos Engineering","author":"nakama","year":"2015"},{"key":"ref8","article-title":"SPS: The Pulse of Netflix Streaming","author":"fisher-ogden","year":"2015","journal-title":"The Netflix Tech Blog"},{"key":"ref7","author":"shadish","year":"2001","journal-title":"Experimental and Quasi-Experimental Designs for Generalized Causal Inference"},{"key":"ref2","article-title":"Chaos Monkey Released into the Wild","author":"bennett","year":"2012","journal-title":"The Netflix Tech Blog"},{"key":"ref9","article-title":"Simple Testing Can Prevent Most Critical Failures: An Analysis of Production Failures in Distributed Data-Intensive Systems","author":"yuan","year":"0","journal-title":"Proc USENIX Symp on Operating System Design and Implementation"},{"key":"ref1","author":"gray","year":"1985","journal-title":"Why Do Computers Stop and What Can Be Done about It?"}],"container-title":["IEEE Software"],"original-title":[],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx7\/52\/7458753\/07436642.pdf?arnumber=7436642","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,1,12]],"date-time":"2022-01-12T11:34:08Z","timestamp":1641987248000},"score":1,"resource":{"primary":{"URL":"https:\/\/ieeexplore.ieee.org\/document\/7436642\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,5]]},"references-count":11,"journal-issue":{"issue":"3"},"URL":"https:\/\/doi.org\/10.1109\/ms.2016.60","relation":{},"ISSN":["0740-7459","1937-4194"],"issn-type":[{"value":"0740-7459","type":"print"},{"value":"1937-4194","type":"electronic"}],"subject":[],"published":{"date-parts":[[2016,5]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1126_2Fscience_1213847 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T22:35:34Z","timestamp":1793313334328,"version":"4.5.0"},"reference-count":13,"publisher":"American Association for the Advancement of Science (AAAS)","issue":"6060","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Science"],"published-print":{"date-parts":[[2011,12,2]]},"abstract":"<jats:p>Computational science has led to exciting new developments, but the nature of the work has exposed limitations in our ability to evaluate published findings. Reproducibility has the potential to serve as a minimum standard for judging scientific claims when full independent replication of a study is not possible.<\/jats:p>","DOI":"10.1126\/science.1213847","type":"journal-article","created":{"date-parts":[[2011,12,2]],"date-time":"2011-12-02T20:35:45Z","timestamp":1322858145000},"page":"1226-1227","source":"Crossref","is-referenced-by-count":1197,"title":["Reproducible Research in Computational Science"],"prefix":"10.1126","volume":"334","author":[{"given":"Roger D.","family":"Peng","sequence":"first","affiliation":[{"name":"Department of Biostatistics, Johns Hopkins Bloomberg School of Public Health, Baltimore MD 21205, USA."}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"221","reference":[{"key":"e_1_3_1_2_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.1203354"},{"key":"e_1_3_1_3_2","doi-asserted-by":"publisher","DOI":"10.1093\/aje\/kwj093"},{"key":"e_1_3_1_4_2","unstructured":"J. P. Madrid D. Macchetto Bull. AAS http:\/\/arxiv.org\/abs\/0901.4552 (2009)."},{"key":"e_1_3_1_5_2","doi-asserted-by":"publisher","DOI":"10.1109\/5992.881708"},{"key":"e_1_3_1_6_2","doi-asserted-by":"publisher","DOI":"10.7326\/0003-4819-146-6-200703200-00154"},{"key":"e_1_3_1_7_2","doi-asserted-by":"publisher","DOI":"10.2307\/420301"},{"key":"e_1_3_1_8_2","unstructured":"R. Gentleman Stat. Appl. Genet. Mol. Biol. 4 article 2 (2005)."},{"key":"e_1_3_1_9_2","first-page":"8","volume":"12","author":"Yale Law School Roundtable on Data and Code Sharing","year":"2010","unstructured":"Yale Law School Roundtable on Data and Code Sharing, Comput. Sci. Eng. 12, 8 (2010).","journal-title":"Comput. Sci. Eng."},{"key":"e_1_3_1_10_2","doi-asserted-by":"publisher","DOI":"10.1038\/ng.295"},{"key":"e_1_3_1_11_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.1179653"},{"key":"e_1_3_1_12_2","doi-asserted-by":"publisher","DOI":"10.1093\/biostatistics\/kxp014"},{"key":"e_1_3_1_13_2","doi-asserted-by":"publisher","DOI":"10.1111\/j.1540-5982.2008.00509.x"},{"key":"e_1_3_1_14_2","doi-asserted-by":"publisher","DOI":"10.1038\/467753a"}],"container-title":["Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.science.org\/doi\/pdf\/10.1126\/science.1213847","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,10]],"date-time":"2024-01-10T14:15:23Z","timestamp":1704896123000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.science.org\/doi\/10.1126\/science.1213847"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2011,12,2]]},"references-count":13,"journal-issue":{"issue":"6060","published-print":{"date-parts":[[2011,12,2]]}},"alternative-id":["10.1126\/science.1213847"],"URL":"https:\/\/doi.org\/10.1126\/science.1213847","relation":{},"ISSN":["0036-8075","1095-9203"],"issn-type":[{"value":"0036-8075","type":"print"},{"value":"1095-9203","type":"electronic"}],"subject":[],"published":{"date-parts":[[2011,12,2]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1126_2Fscience_aab2374 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T19:40:48Z","timestamp":1793302848221,"version":"4.5.0"},"reference-count":14,"publisher":"American Association for the Advancement of Science (AAAS)","issue":"6242","license":[{"start":{"date-parts":[[2016,6,26]],"date-time":"2016-06-26T00:00:00Z","timestamp":1466899200000},"content-version":"vor","delay-in-days":366,"URL":"http:\/\/www.sciencemag.org\/about\/science-licenses-journal-article-reuse"}],"funder":[{"DOI":"10.13039\/100009827","name":"Laura and John Arnold Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100009827","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Science"],"published-print":{"date-parts":[[2015,6,26]]},"abstract":"<jats:p>Author guidelines for journals could help to promote transparency, openness, and reproducibility<\/jats:p>","DOI":"10.1126\/science.aab2374","type":"journal-article","created":{"date-parts":[[2015,6,25]],"date-time":"2015-06-25T14:15:05Z","timestamp":1435241705000},"page":"1422-1425","source":"Crossref","is-referenced-by-count":2093,"title":["Promoting an open research culture"],"prefix":"10.1126","volume":"348","author":[{"given":"B. A.","family":"Nosek","sequence":"first","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"G.","family":"Alter","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"G. C.","family":"Banks","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"D.","family":"Borsboom","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"S. D.","family":"Bowman","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"S. J.","family":"Breckler","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"S.","family":"Buck","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"C. D.","family":"Chambers","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"G.","family":"Chin","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"G.","family":"Christensen","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"M.","family":"Contestabile","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"A.","family":"Dafoe","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"E.","family":"Eich","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"J.","family":"Freese","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"R.","family":"Glennerster","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"D.","family":"Goroff","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"D. P.","family":"Green","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"B.","family":"Hesse","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"M.","family":"Humphreys","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"J.","family":"Ishiyama","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"D.","family":"Karlan","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"A.","family":"Kraut","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"A.","family":"Lupia","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"P.","family":"Mabry","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"T.","family":"Madon","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"N.","family":"Malhotra","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"E.","family":"Mayo-Wilson","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"M.","family":"McNutt","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"E.","family":"Miguel","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"E. Levy","family":"Paluck","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"U.","family":"Simonsohn","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"C.","family":"Soderberg","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"B. A.","family":"Spellman","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"J.","family":"Turitto","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"G.","family":"VandenBos","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"S.","family":"Vazire","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"E. J.","family":"Wagenmakers","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"R.","family":"Wilson","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"T.","family":"Yarkoni","sequence":"additional","affiliation":[{"name":"Affiliations for the authors, all of whom are members of the TOP Guidelines Committee, are given in the supplementary materials."}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"221","reference":[{"key":"e_1_3_3_2_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.1250475"},{"key":"e_1_3_3_3_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.1245317"},{"key":"e_1_3_3_4_2","doi-asserted-by":"publisher","DOI":"10.1525\/jer.2007.2.4.3"},{"key":"e_1_3_3_5_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.tics.2014.02.010"},{"key":"e_1_3_3_6_2","doi-asserted-by":"publisher","DOI":"10.1177\/0956797611430953"},{"key":"e_1_3_3_7_2","doi-asserted-by":"crossref","unstructured":"O'Boyle E. H. Banks G. C. Gonzalez-Mule E. J. Manage. 10.1177\/0149206314527133 (2014).10.1177\/0149206314527133","DOI":"10.1055\/s-0034-1379309"},{"key":"e_1_3_3_8_2","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612459058"},{"key":"e_1_3_3_9_2","doi-asserted-by":"publisher","DOI":"10.1002\/per.1919"},{"key":"e_1_3_3_10_2","doi-asserted-by":"publisher","DOI":"10.1177\/0956797611417632"},{"key":"e_1_3_3_11_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.1255484"},{"key":"e_1_3_3_12_2","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.86.3.638"},{"key":"e_1_3_3_13_2","doi-asserted-by":"publisher","DOI":"10.1177\/0956797613512465"},{"key":"e_1_3_3_14_2","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612463078"},{"key":"e_1_3_3_15_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.cortex.2012.12.016"}],"container-title":["Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.science.org\/doi\/pdf\/10.1126\/science.aab2374","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,15]],"date-time":"2024-01-15T11:30:31Z","timestamp":1705318231000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.science.org\/doi\/10.1126\/science.aab2374"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,6,26]]},"references-count":14,"journal-issue":{"issue":"6242","published-print":{"date-parts":[[2015,6,26]]}},"alternative-id":["10.1126\/science.aab2374"],"URL":"https:\/\/doi.org\/10.1126\/science.aab2374","relation":{"has-preprint":[{"id-type":"doi","id":"10.31219\/osf.io\/vj54c","asserted-by":"object"}]},"ISSN":["0036-8075","1095-9203"],"issn-type":[{"value":"0036-8075","type":"print"},{"value":"1095-9203","type":"electronic"}],"subject":[],"published":{"date-parts":[[2015,6,26]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1126_2Fscience_aac4716 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T12:40:49Z","timestamp":1793277649741,"version":"4.5.0"},"reference-count":41,"publisher":"American Association for the Advancement of Science (AAAS)","issue":"6251","license":[{"start":{"date-parts":[[2016,8,28]],"date-time":"2016-08-28T00:00:00Z","timestamp":1472342400000},"content-version":"vor","delay-in-days":366,"URL":"http:\/\/www.sciencemag.org\/about\/science-licenses-journal-article-reuse"}],"funder":[{"DOI":"10.13039\/100009827","name":"Laura and John Arnold Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100009827","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Center for Open Science"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Science"],"published-print":{"date-parts":[[2015,8,28]]},"abstract":"<jats:title>Empirically analyzing empirical evidence<\/jats:title>\n <jats:p>\n One of the central goals in any scientific endeavor is to understand causality. Experiments that seek to demonstrate a cause\/effect relation most often manipulate the postulated causal factor. Aarts\n <jats:italic>et al.<\/jats:italic>\n describe the replication of 100 experiments reported in papers published in 2008 in three high-ranking psychology journals. Assessing whether the replication and the original experiment yielded the same result according to several criteria, they find that about one-third to one-half of the original findings were also observed in the replication study.\n <\/jats:p>\n <jats:p>\n <jats:italic>Science<\/jats:italic>\n , this issue\n <jats:related-article xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" ext-link-type=\"doi\" related-article-type=\"in-this-issue\" xlink:href=\"10.1126\/science.aac4716\">10.1126\/science.aac4716<\/jats:related-article>\n <\/jats:p>","DOI":"10.1126\/science.aac4716","type":"journal-article","created":{"date-parts":[[2015,8,27]],"date-time":"2015-08-27T14:28:21Z","timestamp":1440685701000},"source":"Crossref","is-referenced-by-count":5689,"title":["Estimating the reproducibility of psychological science"],"prefix":"10.1126","volume":"349","author":[{"name":"Open Science Collaboration","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"221","reference":[{"key":"e_1_3_2_2_2","doi-asserted-by":"publisher","DOI":"10.1086\/288197"},{"key":"e_1_3_2_3_2","doi-asserted-by":"publisher","DOI":"10.1086\/286983"},{"key":"e_1_3_2_4_2","doi-asserted-by":"crossref","unstructured":"I. Lakatos in Criticism and the Growth of Knowledge I. Lakatos A. Musgrave Eds. (Cambridge Univ. Press London 1970) pp. 170\u2013196.","DOI":"10.1017\/CBO9781139171434"},{"key":"e_1_3_2_5_2","doi-asserted-by":"publisher","DOI":"10.1207\/s15327965pli0102_1"},{"key":"e_1_3_2_6_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.146.3642.347"},{"key":"e_1_3_2_7_2","unstructured":"W. C. Salmon in Introduction to the Philosophy of Science M. H. Salmon Ed. (Hackett Publishing Company Indianapolis 1999) pp. 7\u201341."},{"key":"e_1_3_2_8_2","doi-asserted-by":"publisher","DOI":"10.1027\/1864-9335\/a000192"},{"key":"e_1_3_2_9_2","doi-asserted-by":"publisher","DOI":"10.1037\/a0015108"},{"key":"e_1_3_2_10_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.0020124"},{"key":"e_1_3_2_11_2","doi-asserted-by":"publisher","DOI":"10.1038\/483531a"},{"key":"e_1_3_2_12_2","doi-asserted-by":"publisher","DOI":"10.1038\/nrd3439-c1"},{"key":"e_1_3_2_13_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.1250475"},{"key":"e_1_3_2_14_2","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612465253"},{"key":"e_1_3_2_15_2","doi-asserted-by":"publisher","DOI":"10.1038\/nrn3475"},{"key":"e_1_3_2_16_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0010068"},{"key":"e_1_3_2_17_2","doi-asserted-by":"publisher","DOI":"10.1037\/h0076157"},{"key":"e_1_3_2_18_2","doi-asserted-by":"publisher","DOI":"10.1037\/a0015468"},{"key":"e_1_3_2_19_2","doi-asserted-by":"publisher","DOI":"10.1016\/j.tics.2014.02.010"},{"key":"e_1_3_2_20_2","doi-asserted-by":"publisher","DOI":"10.1177\/0956797611430953"},{"key":"e_1_3_2_21_2","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612459058"},{"key":"e_1_3_2_22_2","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.86.3.638"},{"key":"e_1_3_2_23_2","doi-asserted-by":"publisher","DOI":"10.1111\/j.1745-6924.2009.01151.x"},{"key":"e_1_3_2_24_2","doi-asserted-by":"publisher","DOI":"10.1177\/0956797611417632"},{"key":"e_1_3_2_25_2","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612462588"},{"key":"e_1_3_2_26_2","unstructured":"Open Science Collaboration in Implementing Reproducible Computational Research (A Volume in The R Series) V. Stodden F. Leisch R. Peng Eds. (Taylor & Francis New York 2014) pp. 299\u2013323."},{"key":"e_1_3_2_27_2","doi-asserted-by":"publisher","DOI":"10.1017\/S0305004100009580"},{"key":"e_1_3_2_28_2","doi-asserted-by":"publisher","DOI":"10.18637\/jss.v036.i03"},{"key":"e_1_3_2_29_2","doi-asserted-by":"publisher","DOI":"10.1177\/1745691614529796"},{"key":"e_1_3_2_30_2","doi-asserted-by":"publisher","DOI":"10.1177\/0956797614567341"},{"key":"e_1_3_2_31_2","doi-asserted-by":"publisher","DOI":"10.3389\/fpsyg.2013.00863"},{"key":"e_1_3_2_32_2","unstructured":"J. Lehrer The truth wears off: Is there something wrong with the scientific method? The New Yorker 52\u201357 (2010)."},{"key":"e_1_3_2_33_2","doi-asserted-by":"publisher","DOI":"10.1027\/1864-9335\/a000178"},{"key":"e_1_3_2_34_2","doi-asserted-by":"publisher","DOI":"10.1037\/h0045186"},{"key":"e_1_3_2_35_2","first-page":"30","article-title":"Publication decisions and their possible effects on inferences drawn from tests of significance\u2014or vice versa","volume":"54","author":"Sterling T. D.","year":"1959","unstructured":"Sterling T. D., Publication decisions and their possible effects on inferences drawn from tests of significance\u2014or vice versa. J. Am. Stat. Assoc. 54, 30\u201334 (1959).","journal-title":"J. Am. Stat. Assoc."},{"key":"e_1_3_2_36_2","doi-asserted-by":"publisher","DOI":"10.7554\/eLife.04333"},{"key":"e_1_3_2_37_2","doi-asserted-by":"publisher","DOI":"10.3389\/fncom.2012.00008"},{"key":"e_1_3_2_38_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.aab2374"},{"key":"e_1_3_2_39_2","doi-asserted-by":"publisher","DOI":"10.1002\/bs.3830080302"},{"key":"e_1_3_2_40_2","doi-asserted-by":"publisher","DOI":"10.1111\/j.1467-9280.2008.02060.x"},{"key":"e_1_3_2_41_2","doi-asserted-by":"publisher","DOI":"10.1037\/a0012833"},{"key":"e_1_3_2_42_2","doi-asserted-by":"publisher","DOI":"10.1177\/0956797613504966"}],"container-title":["Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.science.org\/doi\/pdf\/10.1126\/science.aac4716","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,15]],"date-time":"2024-01-15T11:29:29Z","timestamp":1705318169000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.science.org\/doi\/10.1126\/science.aac4716"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,8,28]]},"references-count":41,"journal-issue":{"issue":"6251","published-print":{"date-parts":[[2015,8,28]]}},"alternative-id":["10.1126\/science.aac4716"],"URL":"https:\/\/doi.org\/10.1126\/science.aac4716","relation":{"has-review":[{"id-type":"doi","id":"10.3410\/f.725745455.793510459","asserted-by":"object"}],"has-preprint":[{"id-type":"doi","id":"10.31219\/osf.io\/447b3","asserted-by":"object"}]},"ISSN":["0036-8075","1095-9203"],"issn-type":[{"value":"0036-8075","type":"print"},{"value":"1095-9203","type":"electronic"}],"subject":[],"published":{"date-parts":[[2015,8,28]]},"article-number":"aac4716"}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1126_2Fscience_aah6168 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T16:09:40Z","timestamp":1793290180270,"version":"4.5.0"},"reference-count":16,"publisher":"American Association for the Advancement of Science (AAAS)","issue":"6317","license":[{"start":{"date-parts":[[2017,12,9]],"date-time":"2017-12-09T00:00:00Z","timestamp":1512777600000},"content-version":"vor","delay-in-days":365,"URL":"http:\/\/www.sciencemag.org\/about\/science-licenses-journal-article-reuse"}],"funder":[{"DOI":"10.13039\/100009827","name":"Laura and John Arnold Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100009827","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Science"],"published-print":{"date-parts":[[2016,12,9]]},"abstract":"<jats:p>Data, code, and workflows should be available and cited<\/jats:p>","DOI":"10.1126\/science.aah6168","type":"journal-article","created":{"date-parts":[[2016,12,8]],"date-time":"2016-12-08T14:10:19Z","timestamp":1481206219000},"page":"1240-1241","source":"Crossref","is-referenced-by-count":345,"title":["Enhancing reproducibility for computational methods"],"prefix":"10.1126","volume":"354","author":[{"given":"Victoria","family":"Stodden","sequence":"first","affiliation":[{"name":"University of Illinois at Urbana-Champaign, Champaign, IL 61801, USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marcia","family":"McNutt","sequence":"additional","affiliation":[{"name":"National Academy of Sciences, Washington, DC 20418, USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"David H.","family":"Bailey","sequence":"additional","affiliation":[{"name":"University of California, Davis, CA 95616, USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ewa","family":"Deelman","sequence":"additional","affiliation":[{"name":"University of Southern California, Los Angeles, CA 90007, USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yolanda","family":"Gil","sequence":"additional","affiliation":[{"name":"University of Southern California, Los Angeles, CA 90007, USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Brooks","family":"Hanson","sequence":"additional","affiliation":[{"name":"American Geophysical Union, Washington, DC 20009, USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Michael A.","family":"Heroux","sequence":"additional","affiliation":[{"name":"Sandia National Laboratories, Avon, MN 56310, USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"John P.A.","family":"Ioannidis","sequence":"additional","affiliation":[{"name":"Stanford University, Stanford, CA 94305, USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Michela","family":"Taufer","sequence":"additional","affiliation":[{"name":"University of Delaware, Newark, DE 19716, USA."}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"221","reference":[{"key":"e_1_3_3_2_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.aab2374"},{"key":"e_1_3_3_3_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.aad7048"},{"key":"e_1_3_3_4_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0024357"},{"key":"e_1_3_3_5_2","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2009.15"},{"key":"e_1_3_3_6_2","author":"Stodden V.","year":"2013","unstructured":"Stodden V., IMS Bull. Online, 17 November (2013); http:\/\/bit.ly\/BullIMStat2013.","journal-title":"IMS Bull. Online"},{"key":"e_1_3_3_7_2","first-page":"679","volume":"60","author":"Bailey D. H.","year":"2013","unstructured":"Bailey D. H., Borwein J. M., Stodden V., Notices Amer. Math. Soc. 60 (6), 679 (2013).","journal-title":"Notices Amer. Math. Soc."},{"key":"e_1_3_3_8_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0080278"},{"key":"e_1_3_3_9_2","first-page":"916","volume-title":"The Princeton Companion to Applied Mathematics.","author":"Donoho D.","year":"2016","unstructured":"Donoho D., Stodden V., in The Princeton Companion to Applied Mathematics. , Higham N. J., Ed. (Princeton Univ. Press, Princeton, NJ, 2016), pp. 916\u2013925."},{"key":"e_1_3_3_10_2","doi-asserted-by":"publisher","DOI":"10.1198\/106186007X178663"},{"key":"e_1_3_3_11_2","doi-asserted-by":"publisher","DOI":"10.5334\/jors.ay"},{"key":"e_1_3_3_12_2","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2009.19"},{"key":"e_1_3_3_13_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0067111"},{"key":"e_1_3_3_14_2","doi-asserted-by":"crossref","first-page":"art13","DOI":"10.1145\/2743015","volume":"41","author":"Heroux M.","year":"2015","unstructured":"Heroux M., ACM Trans. Math. Softw. 41(3), art13 (2015).","journal-title":"ACM Trans. Math. Softw."},{"key":"e_1_3_3_15_2","first-page":"205","volume-title":"Reproducibility: Principles, Problems, Practices","author":"Bailey D. H.","year":"2015","unstructured":"Bailey D. H., Borwein J. M., Stodden V., in Reproducibility: Principles, Problems, Practices, , Atmanspacher H. , Maasen S., Eds. (Wiley, New York, 2015), pp. 205\u2013232."},{"key":"e_1_3_3_16_2","author":"Fuentes M.","year":"2016","unstructured":"Fuentes M., AMSTAT News, July 2016; http:\/\/bit.ly\/JASA2gb.","journal-title":"AMSTAT News"},{"key":"e_1_3_3_17_2","volume":"46","author":"LeVeque R. J.","year":"2013","unstructured":"LeVeque R. J., SIAM News 46, April 2013.","journal-title":"SIAM News"}],"container-title":["Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.science.org\/doi\/pdf\/10.1126\/science.aah6168","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,15]],"date-time":"2024-01-15T11:32:29Z","timestamp":1705318349000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.science.org\/doi\/10.1126\/science.aah6168"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,12,9]]},"references-count":16,"journal-issue":{"issue":"6317","published-print":{"date-parts":[[2016,12,9]]}},"alternative-id":["10.1126\/science.aah6168"],"URL":"https:\/\/doi.org\/10.1126\/science.aah6168","relation":{"has-review":[{"id-type":"doi","id":"10.3410\/f.727085595.793529043","asserted-by":"object"}]},"ISSN":["0036-8075","1095-9203"],"issn-type":[{"value":"0036-8075","type":"print"},{"value":"1095-9203","type":"electronic"}],"subject":[],"published":{"date-parts":[[2016,12,9]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1126_2Fscitranslmed_aaf5027 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T00:48:44Z","timestamp":1793148524734,"version":"4.5.0"},"reference-count":38,"publisher":"American Association for the Advancement of Science (AAAS)","issue":"341","license":[{"start":{"date-parts":[[2017,6,1]],"date-time":"2017-06-01T00:00:00Z","timestamp":1496275200000},"content-version":"vor","delay-in-days":365,"URL":"http:\/\/www.sciencemag.org\/about\/science-licenses-journal-article-reuse"}],"funder":[{"DOI":"10.13039\/100009827","name":"John and Laura Arnold Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100009827","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Sci. Transl. Med."],"published-print":{"date-parts":[[2016,6]]},"abstract":"<jats:p>The language and conceptual framework of \u201cresearch reproducibility\u201d are nonstandard and unsettled across the sciences.<\/jats:p>","DOI":"10.1126\/scitranslmed.aaf5027","type":"journal-article","created":{"date-parts":[[2016,6,1]],"date-time":"2016-06-01T21:19:13Z","timestamp":1464815953000},"source":"Crossref","is-referenced-by-count":1011,"title":["What does research reproducibility mean?"],"prefix":"10.1126","volume":"8","author":[{"given":"Steven N.","family":"Goodman","sequence":"first","affiliation":[{"name":"Meta-Research Innovation Center at Stanford (METRICS), Stanford University, Stanford, CA 94305, USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Daniele","family":"Fanelli","sequence":"additional","affiliation":[{"name":"Meta-Research Innovation Center at Stanford (METRICS), Stanford University, Stanford, CA 94305, USA."}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"John P. A.","family":"Ioannidis","sequence":"additional","affiliation":[{"name":"Meta-Research Innovation Center at Stanford (METRICS), Stanford University, Stanford, CA 94305, USA."}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"221","reference":[{"key":"e_1_3_1_2_2","doi-asserted-by":"publisher","DOI":"10.14694\/EdBook_AM.2013.33.466"},{"key":"e_1_3_1_3_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.aac4716"},{"key":"e_1_3_1_4_2","doi-asserted-by":"publisher","DOI":"10.1161\/CIRCRESAHA.114.303819"},{"key":"e_1_3_1_5_2","doi-asserted-by":"crossref","unstructured":"J. Claerbout M. Karrenbach Electronic documents give reproducible research a new meaning in Proceedings of the 62nd Annual International Meeting of the Society of Exploration Geophysics New Orleans USA 25 to 29 October 1992.","DOI":"10.1190\/1.1822162"},{"key":"e_1_3_1_6_2","doi-asserted-by":"publisher","DOI":"10.1093\/aje\/kwj093"},{"key":"e_1_3_1_7_2","doi-asserted-by":"publisher","DOI":"10.1093\/nar\/gkw343"},{"key":"e_1_3_1_8_2","doi-asserted-by":"publisher","DOI":"10.1111\/joes.12032"},{"key":"e_1_3_1_9_2","doi-asserted-by":"publisher","DOI":"10.1001\/jama.2015.292"},{"key":"e_1_3_1_10_2","unstructured":"K. Bollen J. T. Cacioppo R. Kaplan J. Krosnick J. L. Olds Social Behavioral and Economic Sciences Perspectives on Robust and Reliable Science (National Science Foundation Arlington VA 2015)."},{"key":"e_1_3_1_11_2","doi-asserted-by":"publisher","DOI":"10.1038\/505612a"},{"key":"e_1_3_1_12_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0066938"},{"key":"e_1_3_1_13_2","article-title":"The meaning of failed replications: A review and proposal","author":"Clemens M. A.","year":"2015","unstructured":"Clemens M. A., The meaning of failed replications: A review and proposal. J. Econ. Surv. 10.1111\/joes.12139 (2015).","journal-title":"J. Econ. Surv."},{"key":"e_1_3_1_14_2","doi-asserted-by":"publisher","DOI":"10.1007\/BF02042161"},{"key":"e_1_3_1_15_2","doi-asserted-by":"publisher","DOI":"10.1038\/nature15513"},{"key":"e_1_3_1_16_2","doi-asserted-by":"publisher","DOI":"10.1056\/NEJM198812293192601"},{"key":"e_1_3_1_17_2","doi-asserted-by":"publisher","DOI":"10.3945\/ajcn.112.047142"},{"key":"e_1_3_1_18_2","doi-asserted-by":"publisher","DOI":"10.1002\/(SICI)1097-0258(19980430)17:8<841::AID-SIM781>3.0.CO;2-D"},{"key":"e_1_3_1_19_2","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0114614"},{"key":"e_1_3_1_20_2","doi-asserted-by":"publisher","DOI":"10.1126\/science.aad7243"},{"key":"e_1_3_1_21_2","doi-asserted-by":"publisher","DOI":"10.7326\/0003-4819-130-12-199906150-00019"},{"key":"e_1_3_1_22_2","doi-asserted-by":"publisher","DOI":"10.7326\/0003-4819-130-12-199906150-00008"},{"key":"e_1_3_1_23_2","doi-asserted-by":"publisher","DOI":"10.3758\/s13423-011-0088-7"},{"key":"e_1_3_1_24_2","unstructured":"S. N. Goodman J. Gerson Mechanistic Evidence in Evidence-Based Medicine: A Conceptual Framework (Agency for Healthcare Research and Quality Rockville MD 2013); http:\/\/www.ncbi.nlm.nih.gov\/books\/NBK154584\/."},{"key":"e_1_3_1_25_2","doi-asserted-by":"crossref","unstructured":"P. Keating A. Cambrosio Cancer on Trial: Oncology as a New Style of Practice (University of Chicago Press Chicago 2011).","DOI":"10.7208\/chicago\/9780226428932.001.0001"},{"key":"e_1_3_1_26_2","doi-asserted-by":"publisher","DOI":"10.1002\/sim.4780110705"},{"key":"e_1_3_1_27_2","doi-asserted-by":"publisher","DOI":"10.1198\/tas.2011.10129"},{"key":"e_1_3_1_28_2","doi-asserted-by":"publisher","DOI":"10.1198\/000313006X152649"},{"key":"e_1_3_1_29_2","doi-asserted-by":"publisher","DOI":"10.1038\/nmeth.3741"},{"key":"e_1_3_1_30_2","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.86.3.638"},{"key":"e_1_3_1_31_2","doi-asserted-by":"publisher","DOI":"10.1207\/s15327957pspr0203_4"},{"key":"e_1_3_1_32_2","doi-asserted-by":"publisher","DOI":"10.1037\/a0033242"},{"key":"e_1_3_1_33_2","doi-asserted-by":"publisher","DOI":"10.2307\/1942661"},{"key":"e_1_3_1_34_2","doi-asserted-by":"publisher","DOI":"10.7326\/0003-4819-105-3-445"},{"key":"e_1_3_1_35_2","doi-asserted-by":"publisher","DOI":"10.1056\/NEJM199310143291613"},{"key":"e_1_3_1_36_2","doi-asserted-by":"publisher","DOI":"10.1111\/1468-0262.00152"},{"key":"e_1_3_1_37_2","doi-asserted-by":"publisher","DOI":"10.1001\/jama.291.20.2457"},{"key":"e_1_3_1_38_2","doi-asserted-by":"publisher","DOI":"10.1093\/jnci\/djs301"},{"key":"e_1_3_1_39_2","unstructured":"E. E. Leamer Specification Searches: Ad Hoc Inference with Nonexperimental Data (Wiley Hoboken NJ 1978)."}],"container-title":["Science Translational Medicine"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.science.org\/doi\/pdf\/10.1126\/scitranslmed.aaf5027","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,16]],"date-time":"2024-01-16T13:42:07Z","timestamp":1705412527000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.science.org\/doi\/10.1126\/scitranslmed.aaf5027"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,6]]},"references-count":38,"journal-issue":{"issue":"341","published-print":{"date-parts":[[2016,6]]}},"alternative-id":["10.1126\/scitranslmed.aaf5027"],"URL":"https:\/\/doi.org\/10.1126\/scitranslmed.aaf5027","relation":{},"ISSN":["1946-6234","1946-6242"],"issn-type":[{"value":"1946-6234","type":"print"},{"value":"1946-6242","type":"electronic"}],"subject":[],"published":{"date-parts":[[2016,6]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1136_2Famiajnl_2011_000089 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,30]],"date-time":"2026-10-30T00:39:44Z","timestamp":1793320784174,"version":"4.5.0"},"reference-count":75,"publisher":"Oxford University Press (OUP)","issue":"1","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["J Am Med Inform Assoc"],"published-print":{"date-parts":[[2012,1]]},"DOI":"10.1136\/amiajnl-2011-000089","type":"journal-article","created":{"date-parts":[[2011,6,17]],"date-time":"2011-06-17T22:08:15Z","timestamp":1308348495000},"page":"121-127","source":"Crossref","is-referenced-by-count":1081,"title":["Automation bias: a systematic review of frequency, effect mediators, and mitigators"],"prefix":"10.1093","volume":"19","author":[{"given":"Kate","family":"Goddard","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Abdul","family":"Roudsari","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jeremy C","family":"Wyatt","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","published-online":{"date-parts":[[2012,1,1]]},"reference":[{"key":"2015071523040803000_19.1.121.1","doi-asserted-by":"publisher","DOI":"10.1001\/jama.293.10.1223"},{"key":"2015071523040803000_19.1.121.2","doi-asserted-by":"publisher","DOI":"10.1136\/bmj.331.7520.831"},{"key":"2015071523040803000_19.1.121.3","unstructured":"Coiera E Westbrook J Wyatt JC . The Safety and Quality of Decision Support Systems. Yearbook of Medical Informatics 2006. Stuttgart: Schattauer Verlag, 2006."},{"key":"2015071523040803000_19.1.121.4","doi-asserted-by":"publisher","DOI":"10.1006\/ijhc.1999.0252"},{"key":"2015071523040803000_19.1.121.5","unstructured":"Mosier K Skitka LJ . Human decision makers and automated decision aids: made for each other? In: Parasuraman R Mouloua M , eds. Automation and Human Performance: Theory and Applications. Hillsdale, NJ: Lawrence Erlbaum Associates, Inc., 1996:pp. 201\u201320."},{"key":"2015071523040803000_19.1.121.6","doi-asserted-by":"publisher","DOI":"10.1001\/jama.293.10.1197"},{"key":"2015071523040803000_19.1.121.7","doi-asserted-by":"publisher","DOI":"10.1001\/jama.282.19.1851"},{"key":"2015071523040803000_19.1.121.8","doi-asserted-by":"publisher","DOI":"10.1518\/hfes.46.1.50_30392"},{"key":"2015071523040803000_19.1.121.9","doi-asserted-by":"publisher","DOI":"10.1207\/s15327108ijap0302_2"},{"key":"2015071523040803000_19.1.121.10","doi-asserted-by":"crossref","unstructured":"Cummings ML . Automation bias in intelligent time critical decision support systems. In: AIAA 1st Intelligent Systems Technical Conference. AIAA, 2004:6313. [as referenced in Mckibbon: http:\/\/171.67.114.118\/content\/13\/6\/653.abstract].","DOI":"10.2514\/6.2004-6313"},{"key":"2015071523040803000_19.1.121.11","doi-asserted-by":"publisher","DOI":"10.1177\/0018720810376055"},{"key":"2015071523040803000_19.1.121.12","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.1000097"},{"key":"2015071523040803000_19.1.121.13","doi-asserted-by":"crossref","unstructured":"Wang L Jamieson GA Hollands JG . Selecting methods for the analysis of reliance on automation. Proceedings of the 52nd Annual Meeting of the Human Factors and Ergonomics Society. Santa Monica, CA: Human Factors and Ergonomics Society, 2008:287\u201391.","DOI":"10.1177\/154193120805200419"},{"key":"2015071523040803000_19.1.121.14","doi-asserted-by":"publisher","DOI":"10.2307\/2529310"},{"key":"2015071523040803000_19.1.121.15","doi-asserted-by":"publisher","DOI":"10.1001\/jama.282.11.1054"},{"key":"2015071523040803000_19.1.121.16","doi-asserted-by":"publisher","DOI":"10.1016\/j.acra.2004.05.012"},{"key":"2015071523040803000_19.1.121.17","doi-asserted-by":"publisher","DOI":"10.1197\/jamia.M1279"},{"key":"2015071523040803000_19.1.121.18","unstructured":"Berner ES Maisiak RS Heudebert GR . Clinician performance and prominence of diagnoses displayed by a clinical diagnostic decision support system. AMIA Annu Symp Proc 2003:76\u201380."},{"key":"2015071523040803000_19.1.121.19","doi-asserted-by":"publisher","DOI":"10.1177\/0272989X9501500202"},{"key":"2015071523040803000_19.1.121.20","doi-asserted-by":"publisher","DOI":"10.1016\/j.amjmed.2004.06.024"},{"key":"2015071523040803000_19.1.121.21","doi-asserted-by":"publisher","DOI":"10.1016\/j.artmed.2004.07.007"},{"key":"2015071523040803000_19.1.121.22","doi-asserted-by":"crossref","unstructured":"Guerlain S Smith PJ Obradovich J . The Antibody Identification Assistant (AIDA), an Example of a Cooperative Computer Support System. Systems, Man and Cybernetics, 1995. Intelligent Systems for the 21st Century. 1995. 2:1909\u201314.","DOI":"10.1109\/ICSMC.1995.538055"},{"key":"2015071523040803000_19.1.121.23","unstructured":"Wyatt J . Lessons learned from the field trial of ACORN, an expert system to advise on chest pain. In: Barber B Cao D Qin D , eds. Proc. Sixth World Conference on Medical Informatics, Singapore. Amsterdam: North Holland, 1989:111\u201315."},{"key":"2015071523040803000_19.1.121.24","doi-asserted-by":"publisher","DOI":"10.1197\/jamia.M1717"},{"key":"2015071523040803000_19.1.121.25","doi-asserted-by":"publisher","DOI":"10.1197\/jamia.M2087"},{"key":"2015071523040803000_19.1.121.26","doi-asserted-by":"publisher","DOI":"10.1177\/0272989X09333125"},{"key":"2015071523040803000_19.1.121.27","unstructured":"Sowan AK Gaffoor M Soeken K . A comparison of medication administrations errors using CPOE orders vs. handwritten orders for pediatric continuous drug infusions. AMIA Annu Symp Proc 2006:1105."},{"key":"2015071523040803000_19.1.121.28","doi-asserted-by":"publisher","DOI":"10.1001\/jama.283.21.2816"},{"key":"2015071523040803000_19.1.121.29","doi-asserted-by":"crossref","unstructured":"Guerlain S . Factors influencing the cooperative problem-solving of people and computers. Proceedings of the Human Factors and Ergonomics Society 37th Annual Meeting. Santa Monica, CA: Human Factors and Ergonomics Society, 1993:387\u201391.","DOI":"10.1177\/154193129303700428"},{"key":"2015071523040803000_19.1.121.30","doi-asserted-by":"publisher","DOI":"10.1016\/j.intcom.2005.09.007"},{"key":"2015071523040803000_19.1.121.31","doi-asserted-by":"publisher","DOI":"10.1177\/0272989X0002000311"},{"key":"2015071523040803000_19.1.121.32","doi-asserted-by":"publisher","DOI":"10.1016\/j.chb.2004.03.011"},{"key":"2015071523040803000_19.1.121.33","doi-asserted-by":"publisher","DOI":"10.1016\/j.acra.2006.04.010"},{"key":"2015071523040803000_19.1.121.34","doi-asserted-by":"publisher","DOI":"10.1056\/NEJMoa066099"},{"key":"2015071523040803000_19.1.121.35","doi-asserted-by":"publisher","DOI":"10.1007\/s11604-007-0224-5"},{"key":"2015071523040803000_19.1.121.36","doi-asserted-by":"publisher","DOI":"10.1148\/radiol.2213010308"},{"key":"2015071523040803000_19.1.121.37","doi-asserted-by":"publisher","DOI":"10.1148\/radiol.2331030432"},{"key":"2015071523040803000_19.1.121.38","doi-asserted-by":"publisher","DOI":"10.1148\/radiol.2311030429"},{"key":"2015071523040803000_19.1.121.39","doi-asserted-by":"publisher","DOI":"10.1148\/radiol.2453062161"},{"key":"2015071523040803000_19.1.121.40","doi-asserted-by":"publisher","DOI":"10.1046\/j.1440-1673.2003.01173.x"},{"key":"2015071523040803000_19.1.121.41","doi-asserted-by":"publisher","DOI":"10.14219\/jada.archive.2002.0303"},{"key":"2015071523040803000_19.1.121.42","doi-asserted-by":"publisher","DOI":"10.1016\/S0720-048X(01)00291-1"},{"key":"2015071523040803000_19.1.121.43","doi-asserted-by":"publisher","DOI":"10.1148\/radiology.199.3.8638015"},{"key":"2015071523040803000_19.1.121.44","doi-asserted-by":"publisher","DOI":"10.1080\/00140130601002658"},{"key":"2015071523040803000_19.1.121.45","doi-asserted-by":"publisher","DOI":"10.1006\/ijhc.1999.0349"},{"key":"2015071523040803000_19.1.121.46","doi-asserted-by":"publisher","DOI":"10.1207\/s15327108ijap0801_3"},{"key":"2015071523040803000_19.1.121.47","unstructured":"Bailey NR Scerbo MW . The Effects of Operator Trust, Complacency Potential, and Task Complexity on Monitoring a Highly Reliable Automated System. Dissertation Abstracts International: Section B: The Sciences and Engineering. US: ProQuest Information & Learning, 2005."},{"key":"2015071523040803000_19.1.121.48","doi-asserted-by":"publisher","DOI":"10.1080\/00140130701318913"},{"key":"2015071523040803000_19.1.121.49","doi-asserted-by":"publisher","DOI":"10.1037\/1076-898X.6.1.44"},{"key":"2015071523040803000_19.1.121.50","doi-asserted-by":"crossref","unstructured":"Wickens C Colcombe A . Dual-task performance consequences of imperfect alerting associated with a cockpit display of traffic information. Hum Factors 2007;49:5839\u201350.","DOI":"10.1518\/001872007X230217"},{"key":"2015071523040803000_19.1.121.51","doi-asserted-by":"crossref","unstructured":"Madhavan P Wiegmann DA . Effects of information source, pedigree, and reliability on operator interaction with decision support systems. Hum Factors 2007;49:5773\u201385.","DOI":"10.1518\/001872007X230154"},{"key":"2015071523040803000_19.1.121.52","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijhcs.2008.06.001"},{"key":"2015071523040803000_19.1.121.53","doi-asserted-by":"crossref","unstructured":"Marten K Seyfarth T Auer F . Computer-assisted detection of pulmonary nodules: performance evaluation of an expert knowledge-based detection system in consensus reading with experienced and inexperienced chest radiologists. Eur Radiol 2004;14:1930\u20138.","DOI":"10.1007\/s00330-004-2389-y"},{"key":"2015071523040803000_19.1.121.54","doi-asserted-by":"publisher","DOI":"10.1097\/RCT.0b013e31815b3ed0"},{"key":"2015071523040803000_19.1.121.55","doi-asserted-by":"crossref","unstructured":"Sarter NB Schroeder B . Supporting decision making and action selection under time pressure and uncertainty: the case of in-flight icing. Hum Factors 2001;43:4573\u201383.","DOI":"10.1518\/001872001775870403"},{"key":"2015071523040803000_19.1.121.56","doi-asserted-by":"crossref","unstructured":"McGuirl JM Sarter NB . Supporting trust calibration and the effective use of decision aids by presenting dynamic system confidence information. Hum Factors 2006;48:4656\u201365.","DOI":"10.1518\/001872006779166334"},{"key":"2015071523040803000_19.1.121.57","doi-asserted-by":"publisher","DOI":"10.1080\/00140139208967392"},{"key":"2015071523040803000_19.1.121.58","doi-asserted-by":"publisher","DOI":"10.1080\/00140139608964474"},{"key":"2015071523040803000_19.1.121.59","unstructured":"Prinzel LJ Freeman FG Prinzel HD . Individual differences in complacency and monitoring for automation failures. Individual Differences Research 2005;3:27\u201349."},{"key":"2015071523040803000_19.1.121.60","doi-asserted-by":"publisher","DOI":"10.1518\/001872001775898269"},{"key":"2015071523040803000_19.1.121.61","doi-asserted-by":"publisher","DOI":"10.1023\/B:GRUP.0000021840.85686.57"},{"key":"2015071523040803000_19.1.121.62","doi-asserted-by":"publisher","DOI":"10.1016\/S1071-5819(03)00038-7"},{"key":"2015071523040803000_19.1.121.63","doi-asserted-by":"publisher","DOI":"10.1518\/0018720024494847"},{"key":"2015071523040803000_19.1.121.64","doi-asserted-by":"crossref","unstructured":"Dieckmann A Dippold K Dietrich H . Compensatory versus non-compensatory models for predicting consumer preferences. Judgm Decis Mak 2009;4:200\u201313.","DOI":"10.1017\/S193029750000173X"},{"key":"2015071523040803000_19.1.121.65","doi-asserted-by":"crossref","unstructured":"Burdick MD Skitka LJ Mosier KL . The ameliorating effects of accountability on automation bias. Proceedings of the 3rd Symposium on Human Interaction with Complex Systems. Dayton, OH: IEEE Computer Society, 1996:142.","DOI":"10.1109\/HUICS.1996.549504"},{"key":"2015071523040803000_19.1.121.66","doi-asserted-by":"crossref","unstructured":"Grubb PL Warm JS Dember WN . Effects of Multiple-Signal Discrimination on Vigilance Performance and Perceived Workload. Proceedings of the Human Factors and Ergonomics Society 39th Annual Meeting. Santa Monica, CA: Human Factors and Ergonomics Society, 1995:1360\u20134.","DOI":"10.1177\/154193129503902101"},{"key":"2015071523040803000_19.1.121.67","doi-asserted-by":"publisher","DOI":"10.1518\/001872006778606822"},{"key":"2015071523040803000_19.1.121.68","doi-asserted-by":"publisher","DOI":"10.1080\/00140130310001629748"},{"key":"2015071523040803000_19.1.121.69","doi-asserted-by":"publisher","DOI":"10.1207\/S15327108IJAP1101_1"},{"key":"2015071523040803000_19.1.121.70","doi-asserted-by":"crossref","unstructured":"Masalonis AJ . Effects of training operators on situation-specific automation reliability. IEEE International Conference on Systems, Man, and Cybernetics. Washington DC: IEEE Computer Society Press, 2003;2:1595\u20139.","DOI":"10.1109\/ICSMC.2003.1244640"},{"key":"2015071523040803000_19.1.121.71","doi-asserted-by":"publisher","DOI":"10.1518\/001872096778827279"},{"key":"2015071523040803000_19.1.121.72","doi-asserted-by":"publisher","DOI":"10.1207\/s15327108ijap0301_1"},{"key":"2015071523040803000_19.1.121.73","doi-asserted-by":"publisher","DOI":"10.1006\/ijhc.1996.0081"},{"key":"2015071523040803000_19.1.121.74","doi-asserted-by":"publisher","DOI":"10.1136\/jamia.1999.0060104"},{"key":"2015071523040803000_19.1.121.75","unstructured":"Campbell EM Sittig DF Guappone KP . \u201cOverdependence on technology: an unintended adverse consequence of computerized provider order entry\u201d. AMIA Ann Symp Proc 2007;11:94\u20138."}],"container-title":["Journal of the American Medical Informatics Association"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/academic.oup.com\/jamia\/article-pdf\/19\/1\/121\/5911703\/19-1-121.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,3,6]],"date-time":"2025-03-06T04:24:34Z","timestamp":1741235074000},"score":1,"resource":{"primary":{"URL":"https:\/\/academic.oup.com\/jamia\/article-lookup\/doi\/10.1136\/amiajnl-2011-000089"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012,1]]},"references-count":75,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2012,1,1]]},"published-print":{"date-parts":[[2012,1]]}},"alternative-id":["10.1136\/amiajnl-2011-000089"],"URL":"https:\/\/doi.org\/10.1136\/amiajnl-2011-000089","relation":{},"ISSN":["1067-5027","1527-974X"],"issn-type":[{"value":"1067-5027","type":"print"},{"value":"1527-974X","type":"electronic"}],"subject":[],"published":{"date-parts":[[2012,1]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F2160718_2160734 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,8,18]],"date-time":"2026-08-18T01:41:34Z","timestamp":1787017294815,"version":"build-2736575974"},"reference-count":5,"publisher":"Association for Computing Machinery (ACM)","issue":"5","license":[{"start":{"date-parts":[[2012,5,1]],"date-time":"2012-05-01T00:00:00Z","timestamp":1335830400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["Commun. ACM"],"published-print":{"date-parts":[[2012,5]]},"abstract":"<jats:p>Messages may be retried. Idempotence means that's OK.<\/jats:p>","DOI":"10.1145\/2160718.2160734","type":"journal-article","created":{"date-parts":[[2012,4,24]],"date-time":"2012-04-24T14:41:10Z","timestamp":1335278470000},"page":"56-65","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":10,"title":["Idempotence is not a medical condition"],"prefix":"10.1145","volume":"55","author":[{"given":"Pat","family":"Helland","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2012,5]]},"reference":[{"key":"e_1_2_1_1_1","unstructured":"IBM. WebSphere MQ; http:\/\/www-01.ibm.com\/software\/integration\/wmq\/. IBM. WebSphere MQ; http:\/\/www-01.ibm.com\/software\/integration\/wmq\/."},{"key":"e_1_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.5555\/572404"},{"key":"e_1_2_1_3_1","volume-title":"Network Working Group","author":"World Wide Web Consortium","year":"1999","unstructured":"World Wide Web Consortium , Network Working Group . 1999 . Hypertext Transfer Protocol---HTTP 1.1; http:\/\/www.w3.org\/Protocols\/rfc2616\/rfc2616.html. World Wide Web Consortium, Network Working Group. 1999. Hypertext Transfer Protocol---HTTP1.1; http:\/\/www.w3.org\/Protocols\/rfc2616\/rfc2616.html."},{"key":"e_1_2_1_4_1","unstructured":"Wolter R. An introduction to SQL Server Service Broker (2005); http:\/\/msdn.microsoft.com\/en-us\/library\/ms345108(v=sql.90).aspx. Wolter R. An introduction to SQL Server Service Broker (2005); http:\/\/msdn.microsoft.com\/en-us\/library\/ms345108(v=sql.90).aspx."},{"key":"e_1_2_1_5_1","unstructured":"Transmission Control Protocol; http:\/\/www.ietf.org\/rfc\/rfc793.txt Transmission Control Protocol; http:\/\/www.ietf.org\/rfc\/rfc793.txt"}],"container-title":["Communications of the ACM"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2160718.2160734","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2160718.2160734","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T05:54:51Z","timestamp":1750226091000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2160718.2160734"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012,5]]},"references-count":5,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2012,5]]}},"alternative-id":["10.1145\/2160718.2160734"],"URL":"https:\/\/doi.org\/10.1145\/2160718.2160734","relation":{},"ISSN":["0001-0782","1557-7317"],"issn-type":[{"value":"0001-0782","type":"print"},{"value":"1557-7317","type":"electronic"}],"subject":[],"published":{"date-parts":[[2012,5]]},"assertion":[{"value":"2012-05-01","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F2408776_2408794 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T23:09:48Z","timestamp":1793315388006,"version":"4.5.0"},"reference-count":12,"publisher":"Association for Computing Machinery (ACM)","issue":"2","license":[{"start":{"date-parts":[[2013,2,1]],"date-time":"2013-02-01T00:00:00Z","timestamp":1359676800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["Commun. ACM"],"published-print":{"date-parts":[[2013,2]]},"abstract":"<jats:p>Software techniques that tolerate latency variability are vital to building responsive large-scale Web services.<\/jats:p>","DOI":"10.1145\/2408776.2408794","type":"journal-article","created":{"date-parts":[[2013,1,29]],"date-time":"2013-01-29T11:20:55Z","timestamp":1359458455000},"page":"74-80","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1608,"title":["The tail at scale"],"prefix":"10.1145","volume":"56","author":[{"given":"Jeffrey","family":"Dean","sequence":"first","affiliation":[{"name":"Google Inc., Mountain View, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Luiz Andr\u00e9","family":"Barroso","sequence":"additional","affiliation":[{"name":"Google Inc., Mountain View, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2013,2]]},"reference":[{"key":"e_1_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2007.443"},{"key":"e_1_2_1_2_1","series-title":"Synthesis Series on Computer Architecture","doi-asserted-by":"crossref","DOI":"10.1007\/978-3-031-01722-3","volume-title":"The Datacenter as a Computer: An Introduction to the Design of Warehouse-scale Machines","author":"Barroso L.A.","year":"2009","unstructured":"Barroso , L.A. and H\u00f6elzle , U . The Datacenter as a Computer: An Introduction to the Design of Warehouse-scale Machines . Synthesis Series on Computer Architecture , Morgan & Claypool Publishers , May 2009 . Barroso, L.A. and H\u00f6elzle, U. The Datacenter as a Computer: An Introduction to the Design of Warehouse-scale Machines. Synthesis Series on Computer Architecture, Morgan & Claypool Publishers, May 2009."},{"key":"e_1_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/108844.108874"},{"key":"e_1_2_1_4_1","first-page":"218","volume":"205","author":"Chang F.","year":"2006","unstructured":"Chang F. , Dean J. , Ghemawat , S. , Hsieh , W.C. , Wallach , D.A. , Burrows , M. , Chandra , T. , Fikes , A. , and Gruber , R.E. BigTable : A distributed storage system for structured data. In Proceedings of the Seventh Symposium on Operating Systems Design and Implementation (Seattle, Nov.). USENIX Association , Berkeley CA , 2006 , 205 -- 218 . Chang F., Dean J., Ghemawat, S., Hsieh, W.C., Wallach, D.A., Burrows, M., Chandra, T., Fikes, A., and Gruber, R.E. BigTable: A distributed storage system for structured data. In Proceedings of the Seventh Symposium on Operating Systems Design and Implementation (Seattle, Nov.). USENIX Association, Berkeley CA, 2006, 205--218.","journal-title":"Berkeley CA"},{"key":"e_1_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1109\/IISWC.2009.5306782"},{"key":"e_1_2_1_6_1","first-page":"40","volume":"27","author":"DeWitt D.J.","year":"1992","unstructured":"DeWitt , D.J. , Naughton , J.F. , Schneider , D.A. , and Seshadri , S. Practical skew handling in parallel joins. In Proceedings of the 18th International Conference on Very Large Data Bases, Li-Yan Yuan , Ed. (Vancouver, BC, Aug . 24--27). Morgan Kaufmann Publishers, Inc. , San Francisco , 1992 , 27 -- 40 . DeWitt, D.J., Naughton, J.F., Schneider, D.A., and Seshadri, S. Practical skew handling in parallel joins. In Proceedings of the 18th International Conference on Very Large Data Bases, Li-Yan Yuan, Ed. (Vancouver, BC, Aug. 24--27). Morgan Kaufmann Publishers, Inc., San Francisco, 1992, 27--40.","journal-title":"San Francisco"},{"key":"e_1_2_1_7_1","unstructured":"Google Inc. Project Glass; http:\/\/g.co\/projectglass Google Inc. Project Glass; http:\/\/g.co\/projectglass"},{"key":"e_1_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/279227.279229"},{"key":"e_1_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1024393.1024399"},{"key":"e_1_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/71.963420"},{"key":"e_1_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1109\/MM.2010.61"},{"key":"e_1_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/383059.383071"}],"container-title":["Communications of the ACM"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2408776.2408794","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2408776.2408794","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T16:13:58Z","timestamp":1750263238000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2408776.2408794"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,2]]},"references-count":12,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2013,2]]}},"alternative-id":["10.1145\/2408776.2408794"],"URL":"https:\/\/doi.org\/10.1145\/2408776.2408794","relation":{},"ISSN":["0001-0782","1557-7317"],"issn-type":[{"value":"0001-0782","type":"print"},{"value":"1557-7317","type":"electronic"}],"subject":[],"published":{"date-parts":[[2013,2]]},"assertion":[{"value":"2013-02-01","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F2452376_2452478 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T02:06:54Z","timestamp":1793239614291,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":16,"publisher":"ACM","license":[{"start":{"date-parts":[[2013,3,18]],"date-time":"2013-03-18T00:00:00Z","timestamp":1363564800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2013,3,18]]},"DOI":"10.1145\/2452376.2452478","type":"proceedings-article","created":{"date-parts":[[2013,3,25]],"date-time":"2013-03-25T14:14:26Z","timestamp":1364220866000},"page":"773-776","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":165,"title":["The W3C PROV family of specifications for modelling provenance metadata"],"prefix":"10.1145","author":[{"given":"Paolo","family":"Missier","sequence":"first","affiliation":[{"name":"Newcastle University, UK"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Khalid","family":"Belhajjame","sequence":"additional","affiliation":[{"name":"University of Manchester, UK"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"James","family":"Cheney","sequence":"additional","affiliation":[{"name":"University of Edinburgh, UK"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2013,3,18]]},"reference":[{"key":"e_1_3_2_1_1_1","first-page":"1151","volume-title":"Proceedings of the 32nd international conference on Very large data bases, VLDB '06","author":"Agrawal P.","year":"2006","unstructured":"P. Agrawal , O. Benjelloun , : a system for data, uncertainty, and lineage . In Proceedings of the 32nd international conference on Very large data bases, VLDB '06 , pages 1151 -- 1154 . VLDB Endowment , 2006 . P. Agrawal, O. Benjelloun, et al. Trio: a system for data, uncertainty, and lineage. In Proceedings of the 32nd international conference on Very large data bases, VLDB '06, pages 1151--1154. VLDB Endowment, 2006."},{"key":"e_1_3_2_1_2_1","first-page":"1","volume-title":"Proceedings of Sepublica 2012","author":"Belhajjame K.","year":"2012","unstructured":"K. Belhajjame Workflow-centric research objects: First class citizens in scholarly discourse . In Proceedings of Sepublica 2012 , pages 1 -- 12 , Hersonissos , 2012 . K. Belhajjame et al. Workflow-centric research objects: First class citizens in scholarly discourse. In Proceedings of Sepublica 2012, pages 1--12, Hersonissos, 2012."},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.5555\/645504.656274"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-35173-0_3"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1561\/1900000006"},{"issue":"2","key":"e_1_3_2_1_6_1","first-page":"84","article-title":"Principles of Provenance (Dagstuhl Seminar 12091)","volume":"2","author":"Cheney J.","year":"2012","unstructured":"J. Cheney , A. Finkelstein , B. Ludaescher , and S. Vansummeren . Principles of Provenance (Dagstuhl Seminar 12091) . Dagstuhl Reports , 2 ( 2 ): 84 -- 113 , 2012 . J. Cheney, A. Finkelstein, B. Ludaescher, and S. Vansummeren. Principles of Provenance (Dagstuhl Seminar 12091). Dagstuhl Reports, 2(2):84--113, 2012.","journal-title":"Dagstuhl Reports"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1066157.1066296"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/SC.Companion.2012.27"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2008.06.012"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-34222-6_13"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1O16\/j.tcs.2004.10.033"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-34222-6"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/1807167.1807269"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-34222-6_6"},{"key":"e_1_3_2_1_15_1","volume-title":"Procs. SSDBM 2010","volume":"6187","author":"Missier P.","year":"2010","unstructured":"P. Missier , S. Soiland-Reyes , S. Owen , In Procs. SSDBM 2010 , volume 6187 of Lecture Notes in Computer Science, pages 471--481, Heidelberg, Germany , 2010 . Springer. P. Missier, S. Soiland-Reyes, S. Owen, et al. Taverna, reloaded. In Procs. SSDBM 2010, volume 6187 of Lecture Notes in Computer Science, pages 471--481, Heidelberg, Germany, 2010. Springer."},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-642-34222-6_3"}],"event":{"name":"EDBT\/ICDT '13: Joint 2013 EDBT\/ICDT Conferences","location":"Genoa Italy","acronym":"EDBT\/ICDT '13"},"container-title":["Proceedings of the 16th International Conference on Extending Database Technology"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2452376.2452478","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2452376.2452478","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T08:18:25Z","timestamp":1750234705000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2452376.2452478"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2013,3,18]]},"references-count":16,"alternative-id":["10.1145\/2452376.2452478","10.1145\/2452376"],"URL":"https:\/\/doi.org\/10.1145\/2452376.2452478","relation":{},"subject":[],"published":{"date-parts":[[2013,3,18]]},"assertion":[{"value":"2013-03-18","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F2670979_2670986 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T10:17:50Z","timestamp":1793269070194,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":56,"publisher":"ACM","license":[{"start":{"date-parts":[[2014,11,3]],"date-time":"2014-11-03T00:00:00Z","timestamp":1414972800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2014,11,3]]},"DOI":"10.1145\/2670979.2670986","type":"proceedings-article","created":{"date-parts":[[2014,11,7]],"date-time":"2014-11-07T12:10:54Z","timestamp":1415362254000},"page":"1-14","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":137,"title":["What Bugs Live in the Cloud? A Study of 3000+ Issues in Cloud Systems"],"prefix":"10.1145","author":[{"given":"Haryadi S.","family":"Gunawi","sequence":"first","affiliation":[{"name":"University of Chicago"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mingzhe","family":"Hao","sequence":"additional","affiliation":[{"name":"University of Chicago"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tanakorn","family":"Leesatapornwongsa","sequence":"additional","affiliation":[{"name":"University of Chicago"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tiratat","family":"Patana-anake","sequence":"additional","affiliation":[{"name":"University of Chicago"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Thanh","family":"Do","sequence":"additional","affiliation":[{"name":"University of Wisconsin--Madison"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jeffry","family":"Adityatama","sequence":"additional","affiliation":[{"name":"Surya University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kurnia J.","family":"Eliazar","sequence":"additional","affiliation":[{"name":"Surya University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Agung","family":"Laksono","sequence":"additional","affiliation":[{"name":"Surya University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jeffrey F.","family":"Lukman","sequence":"additional","affiliation":[{"name":"Surya University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Vincentius","family":"Martin","sequence":"additional","affiliation":[{"name":"Surya University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Anang D.","family":"Satria","sequence":"additional","affiliation":[{"name":"Surya University"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2014,11,3]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Apache Cassandra Project. http:\/\/cassandra.apache.org. Apache Cassandra Project. http:\/\/cassandra.apache.org."},{"key":"e_1_3_2_1_2_1","unstructured":"Apache Flume Project. http:\/\/flume.apache.org. Apache Flume Project. http:\/\/flume.apache.org."},{"key":"e_1_3_2_1_3_1","unstructured":"Apache Hadoop Project. http:\/\/hadoop.apache.org. Apache Hadoop Project. http:\/\/hadoop.apache.org."},{"key":"e_1_3_2_1_4_1","unstructured":"Apache HBase Project. http:\/\/hadoop.apache.org\/hbase. Apache HBase Project. http:\/\/hadoop.apache.org\/hbase."},{"key":"e_1_3_2_1_5_1","unstructured":"Apache ZooKeeper Project. http:\/\/zookeeper.apache.org. Apache ZooKeeper Project. http:\/\/zookeeper.apache.org."},{"key":"e_1_3_2_1_6_1","unstructured":"HDFS Architecture. http:\/\/hadoop.apache.org\/docs\/current\/hadoop-project-dist\/hadoop-hdfs\/HdfsDesign.html. HDFS Architecture. http:\/\/hadoop.apache.org\/docs\/current\/hadoop-project-dist\/hadoop-hdfs\/HdfsDesign.html."},{"key":"e_1_3_2_1_7_1","unstructured":"The 10 Biggest Cloud Outages Of 2013. http:\/\/www.crn.com\/slide-shows\/cloud\/240165024\/the-10-biggest-cloud-outages-of-2013.htm. The 10 Biggest Cloud Outages Of 2013. http:\/\/www.crn.com\/slide-shows\/cloud\/240165024\/the-10-biggest-cloud-outages-of-2013.htm."},{"key":"e_1_3_2_1_8_1","unstructured":"The Apache Software Foundation. http:\/\/www.apache.org\/. The Apache Software Foundation. http:\/\/www.apache.org\/."},{"key":"e_1_3_2_1_9_1","unstructured":"The Cloud Bug Study (CBS) Project. http:\/\/ucare.cs.uchicago.edu\/projects\/cbs. The Cloud Bug Study (CBS) Project. http:\/\/ucare.cs.uchicago.edu\/projects\/cbs."},{"key":"e_1_3_2_1_10_1","volume-title":"Attariyan and Jason Flinn. Automating Configuration Troubleshooting with Dynamic Information Flow Analysis. In OSDI '10","author":"Mona"},{"key":"e_1_3_2_1_11_1","volume-title":"Arpaci-Dusseau. An Analysis of Data Corruption in the Storage Stack. In FAST '08","author":"Bairavasundaram Lakshmi N."},{"key":"e_1_3_2_1_12_1","unstructured":"Cory Bennett and Ariel Tseitlin. Chaos Monkey Released Into The Wild. http:\/\/techblog.netflix.com 2012. Cory Bennett and Ariel Tseitlin. Chaos Monkey Released Into The Wild. http:\/\/techblog.netflix.com 2012."},{"key":"e_1_3_2_1_13_1","volume-title":"OSDI '06","author":"Burrows Mike"},{"key":"e_1_3_2_1_14_1","volume-title":"Robert Gruber. Bigtable: A Distributed Storage System for Structured Data. In OSDI '06","author":"Chang Fay"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/502034.502042"},{"key":"e_1_3_2_1_16_1","volume-title":"Marco Serafini. Practical Hardening of Crash-Tolerant Systems. In USENIX ATC '12","author":"Correia Miguel"},{"key":"e_1_3_2_1_17_1","unstructured":"Jeffrey Dean. Underneath the Covers at Google: Current Systems and Future Directions. In Google I\/O '08. Jeffrey Dean. Underneath the Covers at Google: Current Systems and Future Directions. In Google I\/O '08."},{"key":"e_1_3_2_1_18_1","volume-title":"Dean and Sanjay Ghemawat. MapReduce: Simplified Data Processing on Large Clusters. In OSDI '04","author":"Jeffrey"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/1294261.1294281"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2523616.2523627"},{"key":"e_1_3_2_1_21_1","volume-title":"Arpaci-Dusseau. HARDFS: Hardening HDFS with Selective and Lightweight Versioning. In FAST '13","author":"Do Thanh"},{"key":"e_1_3_2_1_22_1","volume-title":"NSDI '13","author":"Fan Bin"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/945445.945450"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/2451116.2451142"},{"key":"e_1_3_2_1_25_1","volume-title":"Dhruba Borthakur. Fate and Destini: A Framework for Cloud Recovery Testing. In NSDI '11","author":"Gunawi Haryadi S."},{"key":"e_1_3_2_1_26_1","volume-title":"FAST '08","author":"Gunawi Haryadi S."},{"key":"e_1_3_2_1_27_1","volume-title":"Li Zhuang. Nectar: Automatic Management of Data and Computation in Datacenters. In OSDI '10","author":"Gunda Pradeep Kumar"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2043556.2043582"},{"key":"e_1_3_2_1_29_1","volume-title":"HotOS XIV","author":"Guo Zhenyu","year":"2013"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/2038916.2038934"},{"key":"e_1_3_2_1_31_1","volume-title":"Ion Stoica. Mesos: A Platform for Fine-Grained Resource Sharing in the Data Center. In NSDI '11","author":"Hindman Benjamin"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2670979.2670992"},{"key":"e_1_3_2_1_33_1","volume-title":"Gunawi. SAMC: Semantic-Aware Model Checking for Fast Discovery of Deep Bugs in Cloud Systems. In OSDI '14","author":"Leesatapornwongsa Tanakorn"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2043556.2043583"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/2670979.2671004"},{"key":"e_1_3_2_1_36_1","volume-title":"February","author":"Lipcon Todd","year":"2011"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/2043556.2043593"},{"key":"e_1_3_2_1_38_1","volume-title":"Shan Lu. A Study of Linux File System Evolution. In FAST '13","author":"Lu Lanyue"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/1346281.1346323"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/2517349.2522738"},{"key":"e_1_3_2_1_41_1","volume-title":"Yutaka Suzue. Flat Datacenter Storage. In OSDI '12","author":"Nightingale Edmund B."},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2011.6100053"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/2367376.2371297"},{"key":"e_1_3_2_1_44_1","volume-title":"John Ousterhout. Log-structured Memory for DRAM-based Storage. In FAST '14","author":"Rumble Stephen M."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/1555349.1555372"},{"key":"e_1_3_2_1_46_1","volume-title":"Anees Shaikh. Performance Isolation and Fairness for Multi-Tenant Cloud Storage. In OSDI '12","author":"Shue David"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/2517349.2522731"},{"key":"e_1_3_2_1_48_1","volume-title":"Eric Baldeschwieler. Apache Hadoop YARN: Yet Another Resource Negotiator. In SoCC '13","author":"Vavilapalli Vinod Kumar"},{"key":"e_1_3_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/2391229.2391243"},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/1352592.1352612"},{"key":"e_1_3_2_1_51_1","volume-title":"Scott Mahlke. Gadara: Dynamic Deadlock Avoidance for Multithreaded Programs. In OSDI '08","author":"Wang Yin"},{"key":"e_1_3_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/2517349.2522727"},{"key":"e_1_3_2_1_53_1","volume-title":"Lidong Zhou. MODIST: Transparent Model Checking of Unmodified Distributed Systems. In NSDI '09","author":"Yang Junfeng"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/2043556.2043572"},{"key":"e_1_3_2_1_55_1","volume-title":"Michael Stumm. Simple Testing Can Prevent Most Critical Failures: An Analysis of Production Failures in Distributed Dataintensive Systems. In OSDI '14","author":"Yuan Ding"},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/2517349.2522729"}],"event":{"name":"SOCC '14: ACM Symposium on Cloud Computing","location":"Seattle WA USA","acronym":"SOCC '14","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGOPS ACM Special Interest Group on Operating Systems"]},"container-title":["Proceedings of the ACM Symposium on Cloud Computing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2670979.2670986","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2670979.2670986","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T02:12:18Z","timestamp":1750212738000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2670979.2670986"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014,11,3]]},"references-count":56,"alternative-id":["10.1145\/2670979.2670986","10.1145\/2670979"],"URL":"https:\/\/doi.org\/10.1145\/2670979.2670986","relation":{},"subject":[],"published":{"date-parts":[[2014,11,3]]},"assertion":[{"value":"2014-11-03","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F2723872_2723882 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T00:34:09Z","timestamp":1793147649370,"version":"4.5.0"},"reference-count":28,"publisher":"Association for Computing Machinery (ACM)","issue":"1","license":[{"start":{"date-parts":[[2015,1,20]],"date-time":"2015-01-20T00:00:00Z","timestamp":1421712000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["SIGOPS Oper. Syst. Rev."],"published-print":{"date-parts":[[2015,1,20]]},"abstract":"<jats:p>As computational work becomes more and more integral to many aspects of scientific research, computational reproducibility has become an issue of increasing importance to computer systems researchers and domain scientists alike. Though computational reproducibility seems more straight forward than replicating physical experiments, the complex and rapidly changing nature of computer environments makes being able to reproduce and extend such work a serious challenge. In this paper, I explore common reasons that code developed for one research project cannot be successfully executed or extended by subsequent researchers. I review current approaches to these issues, including virtual machines and workflow systems, and their limitations. I then examine how the popular emerging technology Docker combines several areas from systems research - such as operating system virtualization, cross-platform portability, modular re-usable elements, versioning, and a 'DevOps' philosophy, to address these challenges. I illustrate this with several examples of Docker use with a focus on the R statistical environment.<\/jats:p>","DOI":"10.1145\/2723872.2723882","type":"journal-article","created":{"date-parts":[[2015,1,28]],"date-time":"2015-01-28T14:05:51Z","timestamp":1422453951000},"page":"71-79","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":876,"title":["An introduction to Docker for reproducible research"],"prefix":"10.1145","volume":"49","author":[{"given":"Carl","family":"Boettiger","sequence":"first","affiliation":[{"name":"Center for Stock Assessment Research, Santa Cruz, CA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2015,1,20]]},"reference":[{"key":"e_1_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.5555\/998688.1007097"},{"key":"e_1_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1038\/467753a"},{"key":"e_1_2_1_3_1","volume-title":"Proceedings of the 13th Python in Science Conference (SciPy 2014","author":"Clark D.","year":"2014"},{"key":"e_1_2_1_4_1","unstructured":"Collberg C. et al. 2014. Measuring Reproducibility in Computer Systems Research. Collberg C. et al. 2014. Measuring Reproducibility in Computer Systems Research."},{"key":"e_1_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1038\/nbt1110-1181"},{"key":"e_1_2_1_6_1","volume-title":"Toward Replayable Research in Networking and Systems. Archive '10, the nSF workshop on archiving experiments to raise scientific standards","author":"Eide E.","year":"2010"},{"key":"e_1_2_1_7_1","unstructured":"FitzJohn R. et al. 2014. Reproducible research is still a challenge. http:\/\/ropensci.org\/blog\/2014\/06\/09\/reproducibility\/. FitzJohn R. et al. 2014. Reproducible research is still a challenge. http:\/\/ropensci.org\/blog\/2014\/06\/09\/reproducibility\/."},{"key":"e_1_2_1_8_1","doi-asserted-by":"crossref","unstructured":"Garijo D. et al. 2013. Quantifying reproducibility in computational biology: The case of the tuberculosis drugome. {PLoS} {ONE}. 8 11 (Nov. 2013) e80278. Garijo D. et al. 2013. Quantifying reproducibility in computational biology: The case of the tuberculosis drugome. {PLoS} {ONE}. 8 11 (Nov. 2013) e80278.","DOI":"10.1371\/journal.pone.0080278"},{"key":"e_1_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2007.421"},{"key":"e_1_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1365-294X.2012.05754.x"},{"key":"e_1_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2506164.2506175"},{"key":"e_1_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2012.62"},{"key":"e_1_2_1_13_1","doi-asserted-by":"crossref","unstructured":"Hull D. et al. 2006. Taverna: a tool for building and running workflows of services. Nucleic Acids Research. 34 Web Server (Jul. 2006) W729--W732. Hull D. et al. 2006. Taverna: a tool for building and running workflows of services. Nucleic Acids Research. 34 Web Server (Jul. 2006) W729--W732.","DOI":"10.1093\/nar\/gkl320"},{"key":"e_1_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1038\/nature10836"},{"key":"e_1_2_1_15_1","doi-asserted-by":"crossref","unstructured":"Joppa L.N. et al. 2013. Troubling Trends in Scientific Software Use. Science (New York N.Y.). 340 6134 (May 2013) 814--815. Joppa L.N. et al. 2013. Troubling Trends in Scientific Software Use. Science (New York N.Y.). 340 6134 (May 2013) 814--815.","DOI":"10.1126\/science.1231535"},{"key":"e_1_2_1_16_1","unstructured":"Lapp Hilmar 2014. Reproducibility \/ repeatability big- Think (with tweets) @hlapp. Storify. http:\/\/storify.com\/hlapp\/reproducibility-repeatability-bigthink. Lapp Hilmar 2014. Reproducibility \/ repeatability big- Think (with tweets) @hlapp. Storify. http:\/\/storify.com\/hlapp\/reproducibility-repeatability-bigthink."},{"key":"e_1_2_1_17_1","volume-title":"Sweave: Dynamic Generation of Statistical Reports Using Literate Data Analysis","author":"Leisch F.","year":"2002"},{"key":"e_1_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1038\/467775a"},{"key":"e_1_2_1_19_1","first-page":"7391","article-title":"Must try harder","volume":"483","author":"Nature Editors","year":"2012","journal-title":"Nature."},{"key":"e_1_2_1_20_1","unstructured":"Ooms J. 2013. Possible directions for improving dependency versioning in r. arXiv.org. http:\/\/arxiv.org\/abs\/1303. 2140v2. Ooms J. 2013. Possible directions for improving dependency versioning in r. arXiv.org. http:\/\/arxiv.org\/abs\/1303. 2140v2."},{"key":"e_1_2_1_21_1","unstructured":"Ooms J. 2014. The openCPU system: Towards a universal interface for scientific computing through separation of concerns. arXiv.org. http:\/\/arxiv.org\/abs\/1406.4806. Ooms J. 2014. The openCPU system: Towards a universal interface for scientific computing through separation of concerns. arXiv.org. http:\/\/arxiv.org\/abs\/1406.4806."},{"key":"e_1_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1126\/science.1213847"},{"key":"e_1_2_1_23_1","doi-asserted-by":"crossref","unstructured":"Stodden V. 2010. The scientific method in practice: Reproducibility in the computational sciences. SSRN Journal. (2010). Stodden V. 2010. The scientific method in practice: Reproducibility in the computational sciences. SSRN Journal. (2010).","DOI":"10.2139\/ssrn.1550193"},{"key":"e_1_2_1_24_1","unstructured":"Stodden V. et al. 2013. Setting the Default to Reproducible. (2013) 1--19. Stodden V. et al. 2013. Setting the Default to Reproducible. (2013) 1--19."},{"key":"e_1_2_1_25_1","unstructured":"The Economist 2013. How science goes wrong. The Economist. http:\/\/www.economist.com\/news\/leaders\/21588069-scientific-research-has-changed-world-now-itneeds-change-itself-how-science-goes-wrong. The Economist 2013. How science goes wrong. The Economist. http:\/\/www.economist.com\/news\/leaders\/21588069-scientific-research-has-changed-world-now-itneeds-change-itself-how-science-goes-wrong."},{"key":"e_1_2_1_26_1","unstructured":"Xie Y. 2013. Dynamic documents with R and knitr. Chapman; Hall\/CRC. Xie Y. 2013. Dynamic documents with R and knitr. Chapman; Hall\/CRC."},{"key":"e_1_2_1_27_1","unstructured":"2014. Examining reproducibility in computer science. http:\/\/cs.brown.edu\/~sk\/Memos\/Examining- Reproducibility\/. 2014. Examining reproducibility in computer science. http:\/\/cs.brown.edu\/~sk\/Memos\/Examining- Reproducibility\/."},{"key":"e_1_2_1_28_1","unstructured":"2012. Mick Watson on Twitter: @ewanbirney @pathogenomenick @ctitusbrown you can't install an image for every pipeline you want... https:\/\/twitter.com\/BioMickWatson\/status\/265037994526928896. 2012. Mick Watson on Twitter: @ewanbirney @pathogenomenick @ctitusbrown you can't install an image for every pipeline you want... https:\/\/twitter.com\/BioMickWatson\/status\/265037994526928896."}],"container-title":["ACM SIGOPS Operating Systems Review"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2723872.2723882","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2723872.2723882","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T06:16:43Z","timestamp":1750227403000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2723872.2723882"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,1,20]]},"references-count":28,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2015,1,20]]}},"alternative-id":["10.1145\/2723872.2723882"],"URL":"https:\/\/doi.org\/10.1145\/2723872.2723882","relation":{},"ISSN":["0163-5980"],"issn-type":[{"value":"0163-5980","type":"print"}],"subject":[],"published":{"date-parts":[[2015,1,20]]},"assertion":[{"value":"2015-01-20","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F2882903_2899401 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T00:49:40Z","timestamp":1793148580515,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":26,"publisher":"ACM","license":[{"start":{"date-parts":[[2017,6,26]],"date-time":"2017-06-26T00:00:00Z","timestamp":1498435200000},"content-version":"vor","delay-in-days":365,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["CNS-1229185, CI-EN 1405927"],"award-info":[{"award-number":["CNS-1229185, CI-EN 1405927"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000879","name":"Alfred P. Sloan Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000879","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000936","name":"Gordon and Betty Moore Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100000936","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2016,6,26]]},"DOI":"10.1145\/2882903.2899401","type":"proceedings-article","created":{"date-parts":[[2016,6,16]],"date-time":"2016-06-16T15:01:52Z","timestamp":1466089312000},"page":"2085-2088","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":115,"title":["ReproZip"],"prefix":"10.1145","author":[{"given":"Fernando","family":"Chirigati","sequence":"first","affiliation":[{"name":"New York University, Brooklyn, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"R\u00e9mi","family":"Rampin","sequence":"additional","affiliation":[{"name":"New York University, Brooklyn, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dennis","family":"Shasha","sequence":"additional","affiliation":[{"name":"New York University, New York, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Juliana","family":"Freire","sequence":"additional","affiliation":[{"name":"New York University, Brooklyn, NY, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2016,6,26]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1038\/483531a"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/2034863.2034873"},{"key":"e_1_3_2_1_3_1","unstructured":"Chef. https:\/\/www.chef.io\/."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2463676.2465269"},{"key":"e_1_3_2_1_5_1","unstructured":"C. Collberg T. Proebsting and A. M. Warren. Repeatability and Benefaction in Computer Systems Research. Technical Report TR 14-04 University of Arizona 2015."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/1376616.1376772"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2012.41"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.5555\/2685048.2685090"},{"key":"e_1_3_2_1_9_1","unstructured":"Docker. https:\/\/www.docker.com\/."},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2009.15"},{"key":"e_1_3_2_1_11_1","volume-title":"The Architecture of Open Source Applications","author":"Freire J.","year":"2011","unstructured":"J. Freire, D. Koop, E. Santos, C. Scheidegger, C. T. Silva, and H. T. Vo. The Architecture of Open Source Applications, chapter VisTrails. Lulu Publishing, 2011."},{"key":"e_1_3_2_1_12_1","unstructured":"GenePattern. http:\/\/www.broadinstitute.org\/cancer\/software\/genepattern\/."},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2012.36"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.5555\/2342875.2342882"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2618137.2618138"},{"key":"e_1_3_2_1_16_1","unstructured":"Madagascar. http:\/\/www.ahay.org\/wiki\/Main_Page."},{"key":"e_1_3_2_1_17_1","unstructured":"MongoDB. http:\/\/www.mongodb.org\/."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-16462-5_6"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1038\/526182a"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.5555\/2482949.2482952"},{"key":"e_1_3_2_1_21_1","unstructured":"Puppet. http:\/\/puppetlabs.com\/."},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-319-13326-3_4"},{"key":"e_1_3_2_1_23_1","unstructured":"SystemTap. http:\/\/sourceware.org\/systemtap\/."},{"key":"e_1_3_2_1_24_1","volume-title":"Data Sharing by Scientists: Practices and Perceptions. PLoS ONE, 6(6)","author":"Tenopir C.","year":"2011","unstructured":"C. Tenopir, S. Allard, K. Douglass, A. U. Aydinoglu, L. Wu, E. Read, M. Manoff, and M. Frame. Data Sharing by Scientists: Practices and Perceptions. PLoS ONE, 6(6), 2011."},{"key":"e_1_3_2_1_25_1","unstructured":"Vagrant. https:\/\/www.vagrantup.com\/."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2009.932122"}],"event":{"name":"SIGMOD\/PODS'16: International Conference on Management of Data","location":"San Francisco California USA","acronym":"SIGMOD\/PODS'16","sponsor":["SIGMOD ACM Special Interest Group on Management of Data"]},"container-title":["Proceedings of the 2016 International Conference on Management of Data"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2882903.2899401","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2882903.2899401","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2882903.2899401","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,18]],"date-time":"2025-11-18T09:18:13Z","timestamp":1763457493000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2882903.2899401"}},"subtitle":["Computational Reproducibility With Ease"],"short-title":[],"issued":{"date-parts":[[2016,6,26]]},"references-count":26,"alternative-id":["10.1145\/2882903.2899401","10.1145\/2882903"],"URL":"https:\/\/doi.org\/10.1145\/2882903.2899401","relation":{},"subject":[],"published":{"date-parts":[[2016,6,26]]},"assertion":[{"value":"2016-06-26","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F2987550_2987568 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,21]],"date-time":"2026-09-21T11:12:15Z","timestamp":1789989135239,"version":"4.0.1"},"publisher-location":"New York, NY, USA","reference-count":56,"publisher":"ACM","license":[{"start":{"date-parts":[[2016,10,5]],"date-time":"2016-10-05T00:00:00Z","timestamp":1475625600000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["CNS-1117567, CNS-1452712"],"award-info":[{"award-number":["CNS-1117567, CNS-1452712"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2016,10,5]]},"DOI":"10.1145\/2987550.2987568","type":"proceedings-article","created":{"date-parts":[[2016,9,21]],"date-time":"2016-09-21T08:42:53Z","timestamp":1474447373000},"page":"401-414","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":40,"title":["Principled workflow-centric tracing of distributed systems"],"prefix":"10.1145","author":[{"given":"Raja R.","family":"Sambasivan","sequence":"first","affiliation":[{"name":"Carnegie Mellon University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ilari","family":"Shafer","sequence":"additional","affiliation":[{"name":"Microsoft"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jonathan","family":"Mace","sequence":"additional","affiliation":[{"name":"Brown University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Benjamin H.","family":"Sigelman","sequence":"additional","affiliation":[{"name":"LightStep"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rodrigo","family":"Fonseca","sequence":"additional","affiliation":[{"name":"Brown University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Gregory R.","family":"Ganger","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2016,10,5]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","unstructured":"M. Abd-El-Malek W. V. Courtright II C. Cranor G. R. Ganger J. Hendricks A. J. Klosterman M. Mesnier M. Prasad B. Salmon R. R. Sambasivan S. Sinnamohideen J. Strunk E. Thereska M. Wachs and J. Wylie. Ursa Minor: versatile cluster-based storage. FAST' 05.","DOI":"10.5555\/1251028.1251033"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/945445.945454"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","unstructured":"P. S. Almeida C. Baquero and V. Fonte. Interval tree clocks: a logical clock for dynamic systems. OPODIS' 08. 10.1007\/978-3-540-92221-6_18","DOI":"10.1007\/978-3-540-92221-6_18"},{"key":"e_1_3_2_1_4_1","unstructured":"Apache HTrace. http:\/\/htrace.incubator.apache.org\/."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","unstructured":"P. Barham A. Donnelly R. Isaacs and R. Mortier. Using Magpie for request extraction and workload modelling. OSDI' 04.","DOI":"10.5555\/1251254.1251272"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","unstructured":"M. Burrows. The Chubby lock service for loosely-coupled distributed systems. OSDI' 06.","DOI":"10.5555\/1267308.1267332"},{"key":"e_1_3_2_1_7_1","unstructured":"Cassandra tracing. https:\/\/issues.apache.org\/jira\/browse\/CASSANDRA-10392."},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","unstructured":"A. Chanda A. L. Cox and W. Zwaenepoel. Whodunit: transactional profiling for multi-tier applications. EuroSys' 07. 10.1145\/1272996.1273001","DOI":"10.1145\/1272996.1273001"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","unstructured":"M. Y. Chen A. Accardi E. Kiciman J. Lloyd D. Patterson A. Fox and E. Brewer. Path-based failure and evolution management. NSDI' 04.","DOI":"10.5555\/1251175.1251198"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","unstructured":"D. R. Cheriton and D. Skeen. Understanding the limitations of causally and totally ordered communication. SOSP' 93. 10.1145\/168619.168623","DOI":"10.1145\/168619.168623"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","unstructured":"A. Cheung A. Solar-Lezama and S. Madden. Partial replay of long-running applications. FSE' 11. 10.1145\/2025113.2025135","DOI":"10.1145\/2025113.2025135"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","unstructured":"M. Chow D. Meisner J. Flinn D. Peek and T. F. Wenisch. The mystery machine: end-to-end performance analysis of large-scale internet services. OSDI' 14.","DOI":"10.5555\/2685048.2685066"},{"key":"e_1_3_2_1_13_1","unstructured":"Compuware dynaTrace PurePath. http:\/\/www.compuware.com."},{"key":"e_1_3_2_1_14_1","unstructured":"Distributed tracing workshop meeting October 2015. https:\/\/groups.google.com\/forum\/#!forum\/distributed-tracing."},{"key":"e_1_3_2_1_15_1","unstructured":"Distributed tracing workshop meeting July 2016. https:\/\/groups.google.com\/forum\/#!forum\/distributed-tracing."},{"key":"e_1_3_2_1_16_1","unstructured":"Performance instrumentation for Android apps. https:\/\/code.facebook.com\/posts\/747457662026706\/performance-instrumentation-for-android-apps\/."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","unstructured":"R. Fonseca P. Dutta P. Levis and I. Stoica. Quanto: tracking energy in networked embedded systems. OSDI' 08.","DOI":"10.5555\/1855741.1855764"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","unstructured":"R. Fonseca M. J. Freedman and G. Porter. Experiences with tracing causality in networked services. INM\/WREN' 10.","DOI":"10.5555\/1863133.1863143"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","unstructured":"R. Fonseca G. Porter R. H. Katz S. Shenker and I. Stoica. X-Trace: a pervasive network tracing framework. NSDI' 07.","DOI":"10.5555\/1973430.1973450"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1002\/cpe.v22:6"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","unstructured":"S. Ghemawat H. Gobioff and S.-T Leung. The Google file system. SOSP' 03. 10.1145\/945445.945450","DOI":"10.1145\/945445.945450"},{"key":"e_1_3_2_1_22_1","unstructured":"Google Census. https:\/\/github.com\/grpc\/grpc\/tree\/master\/src\/core\/ext\/census."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","unstructured":"Z. Guo S. McDirmid M. Yang L. Zhuang P. Zhang Y. Luo T. Bergan M. Musuvathi Z. Zhang and L. Zhou. Failure Recovery: when the cure is worse than the disease. HotOS' 13.","DOI":"10.5555\/2490483.2490491"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","unstructured":"E. Hilsdale and J. Hugunin. Advice weaving in AspectJ. AOSD' 04. 10.1145\/976270.976276","DOI":"10.1145\/976270.976276"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","unstructured":"J. Huang C. Zhang J. Dolby J. Huang C. Zhang and J. Dolby. CLAP: recording local executions to reproduce concurrency failures. PLDI' 13. 10.1145\/2491956.2462167","DOI":"10.1145\/2491956.2462167"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2807591.2807634"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","unstructured":"S. P. Kavulya S. Daniels K. Joshi M. Hultunen R. Gandhi and P. Narasimhan. Draco: statistical diagnosis of chronic problems in distributed systems. DSN' 12.","DOI":"10.5555\/2354410.2355155"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","unstructured":"V. P. Kemerlis G. Portokalidis K. Jee and A. D. Keromytis. libdft: practical dynamic data flow tracking for commodity systems. VEE' 12. 10.1145\/2151024.2151042","DOI":"10.1145\/2151024.2151042"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","unstructured":"C. E. Killian J. W. Anderson R. Braud R. Jhala and A. M. Vahdat. Mace: language support for building distributed systems. PLDI' 07. 10.1145\/1250734.1250755","DOI":"10.1145\/1250734.1250755"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","unstructured":"E. Koskinen and J. Jannotti. BorderPatrol: isolating events for black-box tracing. EuroSys' 08. 10.1145\/1352592.1352613","DOI":"10.1145\/1352592.1352613"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/359545.359563"},{"key":"e_1_3_2_1_32_1","unstructured":"LightStep. http:\/\/lightstep.com\/."},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.5555\/2789770.2789811"},{"key":"e_1_3_2_1_34_1","unstructured":"J. Mace R. Roelke and R. Fonseca. Pivot Tracing: dynamic causal monitoring for distributed systems."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","unstructured":"G. Mann M. Sandler D. Krushevskaja S. Guha and E. Even-dar. Modeling the parallel execution of black-box services. HotCloud' 11.","DOI":"10.5555\/2170444.2170464"},{"key":"e_1_3_2_1_36_1","volume-title":"The Ganglia distributed monitoring system: design, implementation, and experience. Parallel Computing, 30(7)","author":"Massie M. L.","year":"2004","unstructured":"M. L. Massie, B. N. Chun, and D. E. Culler. The Ganglia distributed monitoring system: design, implementation, and experience. Parallel Computing, 30(7), 2004."},{"key":"e_1_3_2_1_37_1","unstructured":"M. Mesnier M. Wachs R. R. Sambasivan J. Lopez J. Hendricks G. R. Ganger and D. O'Hallaron. \/\/Trace: Parallel trace replay with approximate causal events."},{"key":"e_1_3_2_1_38_1","unstructured":"OpenTracingwebsite. http:\/\/opentracing.io\/."},{"key":"e_1_3_2_1_39_1","unstructured":"Personal communication with Google engineers (Sambasivan) 2011."},{"key":"e_1_3_2_1_40_1","unstructured":"Personal conversations with Google engineers (Sigelman) 2003-2012."},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","unstructured":"P. Reynolds C. Killian J. L. Wiener J. C. Mogul M. Shah and A. Vahdat. Pip: detecting the unexpected in distributed systems. NSDI' 06.","DOI":"10.5555\/1267680.1267689"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/1135777.1135830"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2013.233"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","unstructured":"R. R. Sambasivan A. X. Zheng M. De Rosa E. Krevat S. Whitman M. Stroucken W Wang L. Xu and G. R. Ganger. Diagnosing performance changes by comparing request flows. NSDI' 11.","DOI":"10.5555\/1972457.1972463"},{"key":"e_1_3_2_1_45_1","volume-title":"Dapper, a large-scale distributed systems tracing infrastructure. dapper-2010-1","author":"Sigelman B. H.","year":"2010","unstructured":"B. H. Sigelman, L. A. Barroso, M. Burrows, P. Stephenson, M. Plakal, D. Beaver, S. Jaspan, and C. Shanbhag. Dapper, a large-scale distributed systems tracing infrastructure. dapper-2010-1. 2010."},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","unstructured":"B. C. Tak C. Tang C. Zhang S. Govindan B. Urgaonkar and R. N. Chang. vPath: precise discovery of request processing paths from black-box observations of thread and network activities. USENIX' 09.","DOI":"10.5555\/1855807.1855826"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","unstructured":"N. R. Tallent J. Mellor-Crummey M. Franco R. Landrum and L. Adhianto. Scalable fine-grained call path tracing. ISC' 11. 10.1145\/1995896.1995908","DOI":"10.1145\/1995896.1995908"},{"key":"e_1_3_2_1_48_1","doi-asserted-by":"publisher","unstructured":"J. Tan S. P. Kavulya R. Gandhi and P. Narasimhan. Visual log-based causal tracing for performance debugging of mapreduce systems. ICDCS' 10. 10.1109\/ICDCS.2010.63","DOI":"10.1109\/ICDCS.2010.63"},{"key":"e_1_3_2_1_49_1","unstructured":"The strace system call tracer. http:\/\/sourceforge.net\/projects\/strace\/."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","unstructured":"E. Thereska and G. R. Ganger. Ironmodel: robust performance models in the wild. SIGMETRICS' 08. 10.1145\/1375457.1375486","DOI":"10.1145\/1375457.1375486"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","unstructured":"E. Thereska B. Salmon J. Strunk M. Wachs M. Abd-El-Malek J. Lopez and G. R. Ganger. Stardust: tracking activity in a distributed storage system. SIGMETRICS' 06\/Performance' 06. 10.1145\/1140277.1140280","DOI":"10.1145\/1140277.1140280"},{"key":"e_1_3_2_1_52_1","unstructured":"Tracelytics. http:\/\/www.tracelytics.com."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","unstructured":"M. Wachs L. Xu A. Kanevsky and G. R. Ganger. Exertion-based billing for cloud storage access. HotCloud' 11.","DOI":"10.5555\/2170444.2170451"},{"key":"e_1_3_2_1_54_1","unstructured":"Introduction to Hotpatching. https:\/\/technet.microsoft.com\/en-us\/library\/cc781109(v=ws.10).aspx."},{"key":"e_1_3_2_1_55_1","doi-asserted-by":"publisher","unstructured":"W. Xu L. Huang A. Fox D. Patterson and M. Jordan. Detecting large-scale system problems by mining console logs. SOSP' 09. 10.1145\/1629575.1629587","DOI":"10.1145\/1629575.1629587"},{"key":"e_1_3_2_1_56_1","unstructured":"Zipkin. http:\/\/zipkin.io\/."}],"event":{"name":"SoCC '16: ACM Symposium on Cloud Computing","location":"Santa Clara CA USA","acronym":"SoCC '16","sponsor":["SIGMOD ACM Special Interest Group on Management of Data","SIGOPS ACM Special Interest Group on Operating Systems"]},"container-title":["Proceedings of the Seventh ACM Symposium on Cloud Computing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2987550.2987568","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2987550.2987568","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2987550.2987568","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,11,18]],"date-time":"2025-11-18T09:48:25Z","timestamp":1763459305000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2987550.2987568"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,10,5]]},"references-count":56,"alternative-id":["10.1145\/2987550.2987568","10.1145\/2987550"],"URL":"https:\/\/doi.org\/10.1145\/2987550.2987568","relation":{},"subject":[],"published":{"date-parts":[[2016,10,5]]},"assertion":[{"value":"2016-10-05","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F3290605_3300233 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T10:56:27Z","timestamp":1793271387846,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":45,"publisher":"ACM","license":[{"start":{"date-parts":[[2019,5,2]],"date-time":"2019-05-02T00:00:00Z","timestamp":1556755200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2019,5,2]]},"DOI":"10.1145\/3290605.3300233","type":"proceedings-article","created":{"date-parts":[[2019,4,29]],"date-time":"2019-04-29T13:04:32Z","timestamp":1556543072000},"page":"1-13","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":2207,"title":["Guidelines for Human-AI Interaction"],"prefix":"10.1145","author":[{"given":"Saleema","family":"Amershi","sequence":"first","affiliation":[{"name":"Microsoft, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dan","family":"Weld","sequence":"additional","affiliation":[{"name":"University of Washington & Microsoft Research, Seattle, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mihaela","family":"Vorvoreanu","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Adam","family":"Fourney","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Besmira","family":"Nushi","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Penny","family":"Collisson","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jina","family":"Suh","sequence":"additional","affiliation":[{"name":"Microsoft, Remond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shamsi","family":"Iqbal","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Paul N.","family":"Bennett","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kori","family":"Inkpen","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jaime","family":"Teevan","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ruth","family":"Kikin-Gil","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Eric","family":"Horvitz","sequence":"additional","affiliation":[{"name":"Microsoft, Redmond, WA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2019,5,2]]},"reference":[{"key":"e_1_3_2_2_1_1","volume-title":"Retrieved","year":"2018","unstructured":"Alexa. 2018. Top sites in the United States. Retrieved July, 2018 from https:\/\/www.alexa.com\/topsites\/countries\/US"},{"key":"e_1_3_2_2_2_1","volume-title":"Medium. Retrieved","author":"Baxter Kathy","year":"2017","unstructured":"Kathy Baxter. 2017. How to Meet User Expectations for Artificial Intelligence. Medium. Retrieved September, 2018 from https:\/\/medium.com\/salesforce-ux\/how-to-meet-userexpectations-for-artificial-intelligence-a51d3c82af6"},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1207\/S15327051HCI16234_05"},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/2166966.2166996"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1093\/iwc\/iwv046"},{"key":"e_1_3_2_2_6_1","volume-title":"Retrieved","year":"2018","unstructured":"comScore. 2018. Latest rankings. Retrieved July, 2018 from https: \/\/www.comscore.com\/Insights\/Rankings"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/2909824.3020236"},{"key":"e_1_3_2_2_8_1","volume-title":"Damn You Auto Correct! Retrieved","author":"Media Defy","year":"2018","unstructured":"Defy Media. 2015. Damn You Auto Correct! Retrieved September, 2018 from http:\/\/www.damnyouautocorrect.com\/"},{"key":"e_1_3_2_2_9_1","volume-title":"On the Importance of Spatial Perception for the Design of Adaptive User Interfaces. In 2016 IEEE 10th International Conference on Self-Adaptive and Self-Organizing Systems (SASO). 70--79","author":"Deuschel T.","unstructured":"T. Deuschel and T. Scully. 2016. On the Importance of Spatial Perception for the Design of Adaptive User Interfaces. In 2016 IEEE 10th International Conference on Self-Adaptive and Self-Organizing Systems (SASO). 70--79."},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1145\/571985.572003"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/985692.985704"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/1133265.1133306"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","unstructured":"Krzysztof Z Gajos Katherine Everitt Desney S Tan Mary Czerwinski and Daniel S Weld. 2008. Predictability and accuracy in adaptive user interfaces. In CHI. ACM 1271--1274. 10.1145\/1357054.1357252","DOI":"10.1145\/1357054.1357252"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/358916.358995"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.apergo.2015.11.016"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1016\/S0953-5438(99)00006-5"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/302979.303030"},{"key":"e_1_3_2_2_18_1","volume-title":"Proc. UAI '98","author":"Horvitz Eric","year":"1998","unstructured":"Eric Horvitz, Jack Breese, David Heckerman, David Hovel, and Koos Rommelse. 1998. The Lumi\u00e8re Project: Bayesian User Modeling for Inferring the Goals and Needs of Software Users. In Proc. UAI '98. Morgan Kaufmann Publishers Inc., San Francisco, CA, USA, 256--265."},{"key":"e_1_3_2_2_19_1","volume-title":"Retrieved","author":"IDC Corp","year":"2018","unstructured":"IDC Corporate USA. 2018. IDC: The premier global market intelligence firm. Retrieved September, 2018 from https:\/\/www.idc.com\/"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.5555\/772072.772094"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/2678025.2701399"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2600428.2609588"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/1620545.1620576"},{"key":"e_1_3_2_2_24_1","volume-title":"Google Design. Retrieved","author":"Lovejoy Josh","year":"2018","unstructured":"Josh Lovejoy. 2018. The UX of AI. Google Design. Retrieved September, 2018 from https:\/\/design.google\/library\/ux-ai\/"},{"key":"e_1_3_2_2_25_1","volume-title":"Medium. Retrieved","author":"Lovejoy Josh","year":"2017","unstructured":"Josh Lovejoy and Jess Holbrook. 2017. Human-Centered Machine Learning. 7 steps to stay focused on the user when designing with ML. Medium. Retrieved September, 2018 from https:\/\/medium.com\/google-design\/human-centered-machinelearning-a770d10562cd"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858288"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1093\/jssam\/smw014"},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1093\/obo\/9780199756810-0126"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173580"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/142750.142834"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/97243.97281"},{"key":"e_1_3_2_2_32_1","volume-title":"The psychology of everyday things","author":"Norman D.A.","unstructured":"D.A. Norman. 1988. The psychology of everyday things. Basic Books, New York."},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/176789.176796"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1177\/1525822X10379795"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174214"},{"key":"e_1_3_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173677"},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025453.3025557"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_2_39_1","volume-title":"Retrieved","author":"Schwab Katharine","year":"2017","unstructured":"Katharine Schwab. 2017. 10 Principles For Design In The Age Of AI. Retrieved September, 2018 from https:\/\/www.fastcompany.com\/3067632\/10-principles-for-designin-the-age-of-ai"},{"key":"e_1_3_2_2_40_1","volume-title":"Retrieved","year":"2018","unstructured":"SimilarWeb. 2018. Top websites ranking. Retrieved July, 2018 from https:\/\/www.similarweb.com\/top-websites\/united-states"},{"key":"e_1_3_2_2_41_1","volume-title":"Retrieved","author":"Stewart Jack","year":"2018","unstructured":"Jack Stewart. 2018. Why Tesla's Autopilot Can't See a Stopped Firetruck. Retrieved September, 2018 from https: \/\/www.wired.com\/story\/tesla-autopilot-why-crash-radar\/"},{"key":"e_1_3_2_2_42_1","unstructured":"Erica Virtue. 2017. Designing with AI."},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10551-014-2527-7"},{"key":"e_1_3_2_2_44_1","volume-title":"Intelligible Artificial Intelligence. arXiv preprint arXiv:1803.04263","author":"Weld Daniel S","year":"2018","unstructured":"Daniel S Weld and Gagan Bansal. 2018. Intelligible Artificial Intelligence. arXiv preprint arXiv:1803.04263 (2018)."},{"key":"e_1_3_2_2_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/2750858.2804269"}],"event":{"name":"CHI '19: CHI Conference on Human Factors in Computing Systems","location":"Glasgow Scotland Uk","acronym":"CHI '19","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2019 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3290605.3300233","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3290605.3300233","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,9,21]],"date-time":"2026-09-21T14:07:46Z","timestamp":1789999666000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3290605.3300233"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,5,2]]},"references-count":45,"alternative-id":["10.1145\/3290605.3300233","10.1145\/3290605"],"URL":"https:\/\/doi.org\/10.1145\/3290605.3300233","relation":{},"subject":[],"published":{"date-parts":[[2019,5,2]]},"assertion":[{"value":"2019-05-02","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F3411764_3445717 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T10:18:09Z","timestamp":1793269089043,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":83,"publisher":"ACM","license":[{"start":{"date-parts":[[2022,5,7]],"date-time":"2022-05-07T00:00:00Z","timestamp":1651881600000},"content-version":"vor","delay-in-days":366,"URL":"http:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100006112","name":"Microsoft Research","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100006112","id-type":"DOI","asserted-by":"publisher"}]},{"name":"the University of Washington WRF\/Cable Professorship","id":[{"id":"https:\/\/ror.org\/00cvxb145","id-type":"ROR","asserted-by":"crossref"}]},{"DOI":"10.13039\/100000006","name":"Office of Naval Research","doi-asserted-by":"publisher","award":["N00014-18-1-2193"],"award-info":[{"award-number":["N00014-18-1-2193"]}],"id":[{"id":"10.13039\/100000006","id-type":"DOI","asserted-by":"publisher"}]},{"name":"NSF RAPID grant","award":["2040196"],"award-info":[{"award-number":["2040196"]}]},{"name":"the Allen Institute for Artificial Intelligence (AI2)","id":[{"id":"https:\/\/ror.org\/05w520734","id-type":"ROR","asserted-by":"crossref"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,5,6]]},"DOI":"10.1145\/3411764.3445717","type":"proceedings-article","created":{"date-parts":[[2021,5,7]],"date-time":"2021-05-07T23:50:37Z","timestamp":1620431437000},"page":"1-16","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":637,"title":["Does the Whole Exceed its Parts? The Effect of AI Explanations on Complementary Team Performance"],"prefix":"10.1145","author":[{"given":"Gagan","family":"Bansal","sequence":"first","affiliation":[{"name":"University of Washington, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tongshuang","family":"Wu","sequence":"additional","affiliation":[{"name":"University of Washington, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Joyce","family":"Zhou","sequence":"additional","affiliation":[{"name":"Paul G. Allen School of Computer Science & Engineering University of Washington, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Raymond","family":"Fok","sequence":"additional","affiliation":[{"name":"Paul G. Allen School of Computer Science & Engineering University of Washington, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Besmira","family":"Nushi","sequence":"additional","affiliation":[{"name":"Microsoft Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ece","family":"Kamar","sequence":"additional","affiliation":[{"name":"Microsoft Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marco Tulio","family":"Ribeiro","sequence":"additional","affiliation":[{"name":"Microsoft Research, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Daniel","family":"Weld","sequence":"additional","affiliation":[{"name":"Paul G. Allen School of Computer Science & Engineering University of Washington, United States"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2021,5,7]]},"reference":[{"key":"e_1_3_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.5555\/647985.743843"},{"key":"e_1_3_2_2_2_1","unstructured":"Julia Angwin Jeff Larson Surya Mattu and Lauren Kirchner. 2016. Machine bias: There\u2019s software across the country to predict future criminals and it\u2019s biased against blacks."},{"key":"e_1_3_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2018.00402"},{"key":"e_1_3_2_2_4_1","unstructured":"Gagan Bansal Besmira Nushi Ece Kamar Eric Horvitz and Daniel\u00a0S. Weld. 2020. Optimizing AI for Teamwork. arxiv:2004.13102\u00a0[cs.AI]"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v7i1.5285"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33012429"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1972.10481216"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0109264"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3375624"},{"key":"e_1_3_2_2_10_1","volume-title":"Explaining Recommendations: Satisfaction vs. Promotion. , 13\u201318\u00a0pages.","author":"Bilgic Mustafa","year":"2005","unstructured":"Mustafa Bilgic. 2005. Explaining Recommendations: Satisfaction vs. Promotion. , 13\u201318\u00a0pages."},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173951"},{"key":"e_1_3_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377325.3377498"},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICHI.2015.26"},{"key":"e_1_3_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359206"},{"key":"e_1_3_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2783258.2788613"},{"key":"e_1_3_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1128"},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3172944.3172983"},{"key":"e_1_3_2_2_18_1","volume-title":"Clinical cognition and diagnostic error: applications of a dual process model of reasoning. Advances in health sciences education 14, 1","author":"Croskerry Pat","year":"2009","unstructured":"Pat Croskerry. 2009. Clinical cognition and diagnostic error: applications of a dual process model of reasoning. Advances in health sciences education 14, 1 (2009), 27\u201335."},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1177\/1555343411432338"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1177\/0146167205282152"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3302265"},{"key":"e_1_3_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173718"},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376316"},{"key":"e_1_3_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D18-1537"},{"key":"e_1_3_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P16-2043"},{"key":"e_1_3_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2699751"},{"key":"e_1_3_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359152"},{"key":"e_1_3_2_2_28_1","volume-title":"Proceedings of the 34th International Conference on Machine Learning -","volume":"1330","author":"Guo Chuan","year":"2017","unstructured":"Chuan Guo, Geoff Pleiss, Yu Sun, and Kilian\u00a0Q. Weinberger. 2017. On Calibration of Modern Neural Networks. In Proceedings of the 34th International Conference on Machine Learning - Volume 70(ICML\u201917). JMLR.org, Sydney, NSW, Australia, 1321\u20131330."},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3022198.3026338"},{"key":"e_1_3_2_2_31_1","doi-asserted-by":"publisher","unstructured":"Ruining He and Julian McAuley. 2016. Ups and Downs: Modeling the Visual Evolution of Fashion Trends with One-Class Collaborative Filtering. In Proceedings of the 25th International Conference on World Wide Web (Montr\u00e9al Qu\u00e9bec Canada) (WWW \u201916). International World Wide Web Conferences Steering Committee Republic and Canton of Geneva CHE 507\u2013517. https:\/\/doi.org\/10.1145\/2872427.2883037","DOI":"10.1145\/2872427.2883037"},{"key":"e_1_3_2_2_32_1","volume-title":"5th International Conference on Learning Representations, ICLR","author":"Hendrycks Dan","year":"2017","unstructured":"Dan Hendrycks and Kevin Gimpel. 2017. A Baseline for Detecting Misclassified and Out-of-Distribution Examples in Neural Networks. In 5th International Conference on Learning Representations, ICLR 2017. OpenReview.net, Toulon, France, 1\u201312. https:\/\/openreview.net\/forum?id=Hkg4TI9xl"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2736277.2741102"},{"key":"e_1_3_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/302979.303030"},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1007\/s11257-006-9026-1"},{"key":"e_1_3_2_2_36_1","volume-title":"Proceedings of the ACM on Human-Computer Interaction 3, CSCW(2019)","author":"Jhaver Shagun","year":"2019","unstructured":"Shagun Jhaver, Amy Bruckman, and Eric Gilbert. 2019. Does transparency in moderation really matter? User behavior after content removal explanations on reddit. Proceedings of the ACM on Human-Computer Interaction 3, CSCW(2019), 1\u201327."},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1177\/0963721413481473"},{"key":"e_1_3_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376219"},{"key":"e_1_3_2_2_39_1","volume-title":"imagination, and confidence in judgment.Psychological bulletin 110, 3","author":"Koehler J","year":"1991","unstructured":"Derek\u00a0J Koehler. 1991. Explanation, imagination, and confidence in judgment.Psychological bulletin 110, 3 (1991), 499."},{"key":"e_1_3_2_2_40_1","volume-title":"Proceedings of the 34th International Conference on Machine Learning -","volume":"1894","author":"Koh Pang\u00a0Wei","year":"2017","unstructured":"Pang\u00a0Wei Koh and Percy Liang. 2017. Understanding Black-Box Predictions via Influence Functions. In Proceedings of the 34th International Conference on Machine Learning - Volume 70(ICML\u201917). JMLR.org, Sydney, NSW, Australia, 1885\u20131894."},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300717"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376873"},{"key":"e_1_3_2_2_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287590"},{"key":"e_1_3_2_2_44_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v31i1.10821"},{"key":"e_1_3_2_2_45_1","unstructured":"Himabindu Lakkaraju Ece Kamar Rich Caruana and Jure Leskovec. 2017. Interpretable & Explorable Approximations of Black Box Models. arxiv:1707.01154\u00a0[cs.AI]"},{"key":"e_1_3_2_2_46_1","doi-asserted-by":"publisher","DOI":"10.1518\/hfes.46.1.50_30392"},{"key":"e_1_3_2_2_47_1","doi-asserted-by":"publisher","DOI":"10.1177\/0261927X14535916"},{"key":"e_1_3_2_2_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/1518701.1519023"},{"key":"e_1_3_2_2_49_1","volume-title":"5th International Conference on Learning Representations, ICLR 2017, Conference Track Proceedings. OpenReview.net","author":"Lin Zhouhan","year":"2017","unstructured":"Zhouhan Lin, Minwei Feng, C\u00edcero\u00a0Nogueira dos Santos, Mo Yu, Bing Xiang, Bowen Zhou, and Yoshua Bengio. 2017. A Structured Self-Attentive Sentence Embedding. In 5th International Conference on Learning Representations, ICLR 2017, Conference Track Proceedings. OpenReview.net, Toulon, France, 1\u201315. https:\/\/openreview.net\/forum?id=BJC_jUqxe"},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3236386.3241340"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N16-1104"},{"key":"e_1_3_2_2_52_1","unstructured":"Yinhan Liu Myle Ott Naman Goyal Jingfei Du Mandar Joshi Danqi Chen Omer Levy Mike Lewis Luke Zettlemoyer and Veselin Stoyanov. 2019. RoBERTa: A Robustly Optimized BERT Pretraining Approach. arxiv:1907.11692\u00a0[cs.CL]"},{"key":"e_1_3_2_2_53_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41551-018-0304-0"},{"key":"e_1_3_2_2_54_1","doi-asserted-by":"publisher","DOI":"10.1097\/01.NUMA.0000394953.94337.b4"},{"key":"e_1_3_2_2_55_1","volume-title":"Proceedings of the 32nd International Conference on Neural Information Processing Systems","author":"Madras David","year":"2018","unstructured":"David Madras, Toniann Pitassi, and Richard Zemel. 2018. Predict Responsibly: Improving Fairness and Accuracy by Learning to Defer. In Proceedings of the 32nd International Conference on Neural Information Processing Systems(Montr\u00e9al, Canada) (NIPS\u201918). Curran Associates Inc., Red Hook, NY, USA, 6150\u20136160."},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDM.2012.110"},{"key":"e_1_3_2_2_57_1","volume-title":"Explanation in artificial intelligence: Insights from the social sciences. Artificial Intelligence 267 (February","author":"Miller T.","year":"2018","unstructured":"T. Miller. 2018. Explanation in artificial intelligence: Insights from the social sciences. Artificial Intelligence 267 (February 2018), 1\u201338."},{"key":"e_1_3_2_2_58_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287596"},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.5555\/3524938.3525594"},{"key":"e_1_3_2_2_60_1","doi-asserted-by":"publisher","DOI":"10.1518\/155534309X474460"},{"key":"e_1_3_2_2_61_1","unstructured":"Menaka Narayanan Emily Chen Jeffrey He Been Kim Sam Gershman and Finale Doshi-Velez. 2018. How do Humans Understand Explanations from Machine Learning Systems? An Evaluation of the Human-Interpretability of Explanation. arxiv:1802.00682\u00a0[cs.AI]"},{"key":"e_1_3_2_2_62_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1"},{"key":"e_1_3_2_2_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359204"},{"key":"e_1_3_2_2_64_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1244"},{"key":"e_1_3_2_2_65_1","unstructured":"Forough Poursabzi-Sangdeh Daniel\u00a0G. Goldstein Jake\u00a0M. Hofman Jennifer\u00a0Wortman Vaughan and Hanna Wallach. 2019. Manipulating and Measuring Model Interpretability. arxiv:1802.07810\u00a0[cs.AI]"},{"key":"e_1_3_2_2_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_2_67_1","unstructured":"Philipp Schmidt and Felix Biessmann. 2019. Quantifying Interpretability and Trust in Machine Learning Systems. arxiv:1901.08558\u00a0[cs.LG]"},{"key":"e_1_3_2_2_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376783"},{"key":"e_1_3_2_2_69_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ijhcs.2009.03.004"},{"key":"e_1_3_2_2_70_1","unstructured":"Sarah Tan Julius Adebayo Kori Inkpen and Ece Kamar. 2018. Investigating Human + Machine Complementarity for Recidivism Predictions. arxiv:1808.09123\u00a0[cs.LG]"},{"key":"e_1_3_2_2_71_1","volume-title":"LSAT prep book study guide: quick study & practice test questions for the Law School Admissions council\u2019s (LSAC) Law school admission test. Mometrix Test Preparation","author":"Prep\u00a0Books Team LSAT","unstructured":"LSAT Prep\u00a0Books Team. 2017. LSAT prep book study guide: quick study & practice test questions for the Law School Admissions council\u2019s (LSAC) Law school admission test. Mometrix Test Preparation, Beaumont, TX."},{"key":"e_1_3_2_2_72_1","unstructured":"Kush\u00a0R. Varshney Prashant Khanduri Pranay Sharma Shan Zhang and Pramod\u00a0K. Varshney. 2018. Why Interpretability in Machine Learning? An Answer Using Distributed Detection and Data Fusion Theory. arxiv:1806.09710\u00a0[stat.ML]"},{"key":"e_1_3_2_2_73_1","unstructured":"J. von Neumann and O. Morgenstern. 1947. Theory of games and economic behavior. Princeton University Press Princeton New Jersey."},{"key":"e_1_3_2_2_74_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300831"},{"key":"e_1_3_2_2_75_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D16-1058"},{"key":"e_1_3_2_2_76_1","unstructured":"Hilde J.\u00a0P. Weerts Werner van Ipenburg and Mykola Pechenizkiy. 2019. A Human-Grounded Evaluation of SHAP for Alert Processing. arxiv:1907.03324\u00a0[cs.LG]"},{"key":"e_1_3_2_2_77_1","doi-asserted-by":"publisher","DOI":"10.1145\/3282486"},{"key":"e_1_3_2_2_78_1","doi-asserted-by":"publisher","DOI":"10.5555\/2946645.3007032"},{"key":"e_1_3_2_2_79_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2020"},{"key":"e_1_3_2_2_80_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377325.3377480"},{"key":"e_1_3_2_2_81_1","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3302277"},{"key":"e_1_3_2_2_82_1","volume-title":"ReClor: A Reading Comprehension Dataset Requiring Logical Reasoning. In 8th International Conference on Learning Representations, ICLR","author":"Yu Weihao","year":"2020","unstructured":"Weihao Yu, Zihang Jiang, Yanfei Dong, and Jiashi Feng. 2020. ReClor: A Reading Comprehension Dataset Requiring Logical Reasoning. In 8th International Conference on Learning Representations, ICLR 2020. OpenReview.net, Addis Ababa, Ethiopia, 1\u201326. https:\/\/openreview.net\/forum?id=HJgJtT4tvB"},{"key":"e_1_3_2_2_83_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372852"}],"event":{"name":"CHI '21: CHI Conference on Human Factors in Computing Systems","location":"Yokohama Japan","acronym":"CHI '21","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3411764.3445717","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3411764.3445717","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3411764.3445717","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,9,21]],"date-time":"2026-09-21T14:17:48Z","timestamp":1790000268000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3411764.3445717"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,5,6]]},"references-count":83,"alternative-id":["10.1145\/3411764.3445717","10.1145\/3411764"],"URL":"https:\/\/doi.org\/10.1145\/3411764.3445717","relation":{},"subject":[],"published":{"date-parts":[[2021,5,6]]},"assertion":[{"value":"2021-05-07","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F3449287 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T10:18:07Z","timestamp":1793269087301,"version":"4.5.0"},"reference-count":67,"publisher":"Association for Computing Machinery (ACM)","issue":"CSCW1","license":[{"start":{"date-parts":[[2021,4,13]],"date-time":"2021-04-13T00:00:00Z","timestamp":1618272000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["Proc. ACM Hum.-Comput. Interact."],"published-print":{"date-parts":[[2021,4,13]]},"abstract":"<jats:p>People supported by AI-powered decision support tools frequently overrely on the AI: they accept an AI's suggestion even when that suggestion is wrong. Adding explanations to the AI decisions does not appear to reduce the overreliance and some studies suggest that it might even increase it. Informed by the dual-process theory of cognition, we posit that people rarely engage analytically with each individual AI recommendation and explanation, and instead develop general heuristics about whether and when to follow the AI suggestions. Building on prior research on medical decision-making, we designed three cognitive forcing interventions to compel people to engage more thoughtfully with the AI-generated explanations. We conducted an experiment (N=199), in which we compared our three cognitive forcing designs to two simple explainable AI approaches and to a no-AI baseline. The results demonstrate that cognitive forcing significantly reduced overreliance compared to the simple explainable AI approaches. However, there was a trade-off: people assigned the least favorable subjective ratings to the designs that reduced the overreliance the most. To audit our work for intervention-generated inequalities, we investigated whether our interventions benefited equally people with different levels of Need for Cognition (i.e., motivation to engage in effortful mental activities). Our results show that, on average, cognitive forcing interventions benefited participants higher in Need for Cognition more. Our research suggests that human cognitive motivation moderates the effectiveness of explainable AI solutions.<\/jats:p>","DOI":"10.1145\/3449287","type":"journal-article","created":{"date-parts":[[2021,4,22]],"date-time":"2021-04-22T13:51:09Z","timestamp":1619099469000},"page":"1-21","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1095,"title":["To Trust or to Think"],"prefix":"10.1145","volume":"5","author":[{"given":"Zana","family":"Bu\u00e7inca","sequence":"first","affiliation":[{"name":"Harvard University, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Maja Barbara","family":"Malaya","sequence":"additional","affiliation":[{"name":"Institute of Applied Computer Science, Lodz, Poland"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Krzysztof Z.","family":"Gajos","sequence":"additional","affiliation":[{"name":"Harvard University, Cambridge, MA, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2021,4,22]]},"reference":[{"key":"e_1_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v7i1.5285"},{"key":"e_1_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445717"},{"key":"e_1_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jml.2012.11.001"},{"key":"e_1_2_1_4_1","volume-title":"Overconfidence as a cause of diagnostic error in medicine. The American journal of medicine","author":"Berner Eta S","year":"2008","unstructured":"Eta S Berner and Mark L Graber. 2008. Overconfidence as a cause of diagnostic error in medicine. The American journal of medicine, Vol. 121, 5 (2008), S2--S23."},{"key":"e_1_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2020"},{"key":"e_1_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1046\/j.1365-2753.2001.00284.x"},{"key":"e_1_2_1_7_1","volume-title":"Proceedings of the 25th International Conference on Intelligent User Interfaces (IUI '20)","author":"Buccinca Zana","unstructured":"Zana Buccinca, Phoebe Lin, Krzysztof Z. Gajos, and Elena L. Glassman. 2020. Proxy Tasks and Subjective Measures Can Be Misleading in Evaluating Explainable AI Systems. In Proceedings of the 25th International Conference on Intelligent User Interfaces (IUI '20). ACM, New York, NY, USA."},{"key":"e_1_2_1_8_1","volume-title":"Conference on fairness, accountability and transparency. 77--91","author":"Buolamwini Joy","year":"2018","unstructured":"Joy Buolamwini and Timnit Gebru. 2018. Gender shades: Intersectional accuracy disparities in commercial gender classification. In Conference on fairness, accountability and transparency. 77--91."},{"key":"e_1_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICHI.2015.26"},{"key":"e_1_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1037\/0022-3514.42.1.116"},{"key":"e_1_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1207\/s15327752jpa4803_13"},{"key":"e_1_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/634067.634293"},{"key":"e_1_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.sbspro.2014.03.227"},{"key":"e_1_2_1_14_1","volume-title":"An exploratory cognitive DSS for strategic decision making. Decision support systems","author":"Chen Jim Q","year":"2003","unstructured":"Jim Q Chen and Sang M Lee. 2003. An exploratory cognitive DSS for strategic decision making. Decision support systems, Vol. 36, 2 (2003), 147--160."},{"key":"e_1_2_1_15_1","volume-title":"Cognitive bias research in forensic science: a systematic review. Forensic science international","author":"Cooper Glinda S","year":"2019","unstructured":"Glinda S Cooper and Vanessa Meterko. 2019. Cognitive bias research in forensic science: a systematic review. Forensic science international, Vol. 297 (2019), 35--46."},{"key":"e_1_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1067\/mem.2003.22"},{"key":"e_1_2_1_17_1","volume-title":"2003 b. The importance of cognitive errors in diagnosis and strategies to minimize them","author":"Croskerry Pat","year":"2003","unstructured":"Pat Croskerry. 2003 b. The importance of cognitive errors in diagnosis and strategies to minimize them. Academic medicine, Vol. 78, 8 (2003), 775--780."},{"key":"e_1_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1821936116"},{"key":"e_1_2_1_19_1","volume-title":"Biased: Uncovering the hidden prejudice that shapes what we see, think, and do","author":"Eberhardt Jennifer L","year":"2020","unstructured":"Jennifer L Eberhardt. 2020. Biased: Uncovering the hidden prejudice that shapes what we see, think, and do. Penguin Books."},{"key":"e_1_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1097\/ACM.0b013e31820824cd"},{"key":"e_1_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.5555\/2882794.2882801"},{"key":"e_1_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3025171.3025192"},{"key":"e_1_2_1_23_1","volume-title":"et almbox","author":"Garg Neelansh","year":"2018","unstructured":"Neelansh Garg, Apuroop Sethupathy, Rudraksh Tuwani, Rakhi Nk, Shubham Dokania, Arvind Iyer, Ayushi Gupta, Shubhra Agrawal, Navjot Singh, Shubham Shukla, et almbox. 2018. FlavorDB: a database of flavor molecules. Nucleic acids research, Vol. 46, D1 (2018), D1210-D1216."},{"key":"e_1_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3432934"},{"key":"e_1_2_1_25_1","volume-title":"Asta Sorensen, Nancy Lenfestey, Elizabeth Tant, Kerm Henriksen, Kenneth LaBresh, and Hardeep Singh.","author":"Graber Mark L","year":"2012","unstructured":"Mark L Graber, Stephanie Kissam, Velma L Payne, Ashley ND Meyer, Asta Sorensen, Nancy Lenfestey, Elizabeth Tant, Kerm Henriksen, Kenneth LaBresh, and Hardeep Singh. 2012. Cognitive interventions to reduce diagnostic error: a narrative review. BMJ quality & safety, Vol. 21, 7 (2012), 535--557."},{"key":"e_1_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359152"},{"key":"e_1_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1016\/S1057-7408(08)80038-1"},{"key":"e_1_2_1_28_1","first-page":"65","article-title":"A simple sequentially rejective multiple test procedure","volume":"6","author":"Holm Sture","year":"1979","unstructured":"Sture Holm. 1979. A simple sequentially rejective multiple test procedure. Scandinavian Journal of Statistics, Vol. 6, 2 (1979), 65--70.","journal-title":"Scandinavian Journal of Statistics"},{"key":"e_1_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/TVCG.2011.175"},{"key":"e_1_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1038\/s41398-021-01224-x"},{"key":"e_1_2_1_31_1","volume-title":"Proceedings of the 32nd International Conference on Neural Information Processing Systems. 5546--5557","author":"Jiang Heinrich","year":"2018","unstructured":"Heinrich Jiang, Been Kim, Melody Y Guan, and Maya Gupta. 2018. To trust or not to trust a classifier. In Proceedings of the 32nd International Conference on Neural Information Processing Systems. 5546--5557."},{"key":"e_1_2_1_32_1","volume-title":"fast and slow","author":"Kahneman Daniel","unstructured":"Daniel Kahneman. 2011. Thinking, fast and slow. Macmillan."},{"key":"e_1_2_1_33_1","volume-title":"Representativeness revisited: Attribute substitution in intuitive judgment.","author":"Kahneman Daniel","unstructured":"Daniel Kahneman and Shane Frederick. 2002. Representativeness revisited: Attribute substitution in intuitive judgment. In Representativeness revisited: Attribute substitution in intuitive judgment. New York. Cambridge University Press., 49--81."},{"key":"e_1_2_1_34_1","volume-title":"Paul Slovic, and Amos Tversky.","author":"Kahneman Daniel","year":"1982","unstructured":"Daniel Kahneman, Stewart Paul Slovic, Paul Slovic, and Amos Tversky. 1982. Judgment under uncertainty: Heuristics and biases. Cambridge university press."},{"key":"e_1_2_1_35_1","unstructured":"Ece Kamar. 2016. Directions in Hybrid Intelligence: Complementing AI Systems with Human Intelligence.. In IJCAI. 4070--4073."},{"key":"e_1_2_1_36_1","volume-title":"Proceedings of the 11th International Conference on Autonomous Agents and Multiagent Systems-Volume 1. International Foundation for Autonomous Agents and Multiagent Systems, 467--474","author":"Kamar Ece","year":"2012","unstructured":"Ece Kamar, Severin Hacker, and Eric Horvitz. 2012. Combining human and machine intelligence in large-scale crowdsourcing. In Proceedings of the 11th International Conference on Autonomous Agents and Multiagent Systems-Volume 1. International Foundation for Autonomous Agents and Multiagent Systems, 467--474."},{"key":"e_1_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.5465\/amj.2009.41331431"},{"key":"e_1_2_1_38_1","volume-title":"Mental labour. Nature human behaviour","author":"Kool Wouter","year":"2018","unstructured":"Wouter Kool and Matthew Botvinick. 2018. Mental labour. Nature human behaviour, Vol. 2, 12 (2018), 899--908."},{"key":"e_1_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2013.6645235"},{"key":"e_1_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376873"},{"key":"e_1_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287590"},{"key":"e_1_2_1_42_1","volume-title":"Dual-process cognitive interventions to enhance diagnostic reasoning: a systematic review. BMJ quality & safety","author":"Lambe Kathryn Ann","year":"2016","unstructured":"Kathryn Ann Lambe, Gary O'Reilly, Brendan D Kelly, and Sarah Curristan. 2016. Dual-process cognitive interventions to enhance diagnostic reasoning: a systematic review. BMJ quality & safety, Vol. 25, 10 (2016), 808--820."},{"key":"e_1_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1177\/0146167291176013"},{"key":"e_1_2_1_44_1","volume-title":"Trust in automation: Designing for appropriate reliance. Human factors","author":"Lee John D","year":"2004","unstructured":"John D Lee and Katrina A See. 2004. Trust in automation: Designing for appropriate reliance. Human factors, Vol. 46, 1 (2004), 50--80."},{"key":"e_1_2_1_45_1","volume-title":"The effects of online reviews on purchasing intention: The moderating role of need for cognition. Social Behavior and Personality: an international journal","author":"Lin Chin-Lung","year":"2011","unstructured":"Chin-Lung Lin, Sheng-Hsien Lee, and Der-Juinn Horng. 2011. The effects of online reviews on purchasing intention: The moderating role of need for cognition. Social Behavior and Personality: an international journal, Vol. 39, 1 (2011), 71--81."},{"key":"e_1_2_1_46_1","volume-title":"Advances in Neural Information Processing Systems 30. Curran Associates","author":"Lundberg Scott M","unstructured":"Scott M Lundberg and Su-In Lee. 2017. A Unified Approach to Interpreting Model Predictions. In Advances in Neural Information Processing Systems 30. Curran Associates, Inc., 4765--4774."},{"key":"e_1_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3302313"},{"key":"e_1_2_1_48_1","doi-asserted-by":"publisher","DOI":"10.1145\/3340631.3394844"},{"key":"e_1_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1097\/ACM.0b013e3181405a76"},{"key":"e_1_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359204"},{"key":"e_1_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-540-74788-8_15"},{"key":"e_1_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4612-4964-1_1"},{"key":"e_1_2_1_53_1","volume-title":"Individual differences in the calibration of trust in automation. Human factors","author":"Pop Vlad L","year":"2015","unstructured":"Vlad L Pop, Alex Shrewsbury, and Francis T Durso. 2015. Individual differences in the calibration of trust in automation. Human factors, Vol. 57, 4 (2015), 545--556."},{"key":"e_1_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372873"},{"key":"e_1_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.2310\/8000.2013.130860"},{"key":"e_1_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1080\/00913367.2005.10639202"},{"key":"e_1_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1439-037X.2004.00120.x"},{"key":"e_1_2_1_58_1","volume-title":"Understanding differences in web usage: The role of need for cognition and the five factor model of personality. Social Behavior and Personality: an international journal","author":"Tuten Tracy L","year":"2001","unstructured":"Tracy L Tuten and Michael Bosnjak. 2001. Understanding differences in web usage: The role of need for cognition and the five factor model of personality. Social Behavior and Personality: an international journal, Vol. 29, 4 (2001), 391--398."},{"key":"e_1_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359338"},{"key":"e_1_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1093\/jamia\/ocy052"},{"key":"e_1_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.0021-9029.2007.00149.x"},{"key":"e_1_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3241365"},{"key":"e_1_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1016\/0010-0277(74)90017-1"},{"key":"e_1_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1037\/xge0000014"},{"key":"e_1_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1207\/S15327027HC1504_01"},{"key":"e_1_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300509"},{"key":"e_1_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372852"}],"container-title":["Proceedings of the ACM on Human-Computer Interaction"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3449287","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3449287","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T17:28:22Z","timestamp":1750181302000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3449287"}},"subtitle":["Cognitive Forcing Functions Can Reduce Overreliance on AI in AI-assisted Decision-making"],"short-title":[],"issued":{"date-parts":[[2021,4,13]]},"references-count":67,"journal-issue":{"issue":"CSCW1","published-print":{"date-parts":[[2021,4,13]]}},"alternative-id":["10.1145\/3449287"],"URL":"https:\/\/doi.org\/10.1145\/3449287","relation":{},"ISSN":["2573-0142"],"issn-type":[{"value":"2573-0142","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,4,13]]},"assertion":[{"value":"2021-04-22","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F3485510 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,9,29]],"date-time":"2026-09-29T00:29:59Z","timestamp":1790641799326,"version":"4.1.0"},"reference-count":39,"publisher":"Association for Computing Machinery (ACM)","issue":"OOPSLA","license":[{"start":{"date-parts":[[2021,10,15]],"date-time":"2021-10-15T00:00:00Z","timestamp":1634256000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["Proc. ACM Program. Lang."],"published-print":{"date-parts":[[2021,10,20]]},"abstract":"<jats:p>Serverless, or Functions-as-a-Service (FaaS), is an increasingly popular paradigm for application development, as it provides implicit elastic scaling and load based billing. However, the weak execution guarantees and intrinsic compute-storage separation of FaaS create serious challenges when developing applications that require persistent state, reliable progress, or synchronization. This has motivated a new generation of serverless frameworks that provide stateful abstractions. For instance, Azure's Durable Functions (DF) programming model enhances FaaS with actors, workflows, and critical sections. As a programming model, DF is interesting because it combines task and actor parallelism, which makes it suitable for a wide range of serverless applications. We describe DF both informally, using examples, and formally, using an idealized high-level model based on the untyped lambda calculus. Next, we demystify how the DF runtime can (1) execute in a distributed unreliable serverless environment with compute-storage separation, yet still conform to the fault-free high-level model, and (2) persist execution progress without requiring checkpointing support by the language runtime. To this end we define two progressively more complex execution models, which contain the compute-storage separation and the record-replay, and prove that they are equivalent to the high-level model.<\/jats:p>","DOI":"10.1145\/3485510","type":"journal-article","created":{"date-parts":[[2021,10,15]],"date-time":"2021-10-15T15:18:28Z","timestamp":1634311108000},"page":"1-27","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":58,"title":["Durable functions: semantics for stateful serverless"],"prefix":"10.1145","volume":"5","author":[{"given":"Sebastian","family":"Burckhardt","sequence":"first","affiliation":[{"name":"Microsoft Research, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chris","family":"Gillum","sequence":"additional","affiliation":[{"name":"Microsoft Azure, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"David","family":"Justo","sequence":"additional","affiliation":[{"name":"Microsoft Azure, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Konstantinos","family":"Kallas","sequence":"additional","affiliation":[{"name":"University of Pennsylvania, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Connor","family":"McMahon","sequence":"additional","affiliation":[{"name":"Microsoft Azure, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Christopher S.","family":"Meiklejohn","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2021,10,15]]},"reference":[{"key":"e_1_2_2_1_1","unstructured":"2020. AWS Step Functions. https:\/\/docs.aws.amazon.com\/step-functions\/latest\/dg\/welcome.html 2020. AWS Step Functions. https:\/\/docs.aws.amazon.com\/step-functions\/latest\/dg\/welcome.html"},{"key":"e_1_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/258949.258967"},{"key":"e_1_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1145\/3133850.3133855"},{"key":"e_1_2_2_4_1","unstructured":"Jonas Bon\u00e9r. 2020. Towards Stateful Serverless. https:\/\/www.youtube.com\/watch?v=DVTf5WQlgB8 Jonas Bon\u00e9r. 2020. Towards Stateful Serverless. https:\/\/www.youtube.com\/watch?v=DVTf5WQlgB8"},{"key":"e_1_2_2_5_1","volume-title":"Meiklejohn","author":"Burckhardt Sebastian","year":"2021"},{"key":"e_1_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2038916.2038932"},{"key":"e_1_2_2_7_1","volume-title":"Apache flink: Stream and batch processing in a single engine. Bulletin of the IEEE Computer Society Technical Committee on Data Engineering, 36, 4","author":"Carbone Paris","year":"2015"},{"key":"e_1_2_2_8_1","unstructured":"2021. Note on usage of yield for async\/await.. https:\/\/twitter.com\/cgillum\/status\/1378128701317992451 Accessed: 2021-04-13 2021. Note on usage of yield for async\/await.. https:\/\/twitter.com\/cgillum\/status\/1378128701317992451 Accessed: 2021-04-13"},{"key":"e_1_2_2_9_1","unstructured":"2020. Using Durable Objects Cloudflare Docs. https:\/\/developers.cloudflare.com\/workers\/learning\/using-durable-objects 2020. Using Durable Objects Cloudflare Docs. https:\/\/developers.cloudflare.com\/workers\/learning\/using-durable-objects"},{"key":"e_1_2_2_10_1","unstructured":"2020. Workers Durable Objects Beta: A New Approach to Stateful Serverless. https:\/\/blog.cloudflare.com\/introducing-workers-durable-objects\/ 2020. Workers Durable Objects Beta: A New Approach to Stateful Serverless. https:\/\/blog.cloudflare.com\/introducing-workers-durable-objects\/"},{"key":"e_1_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/1327452.1327492"},{"key":"e_1_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/199448.199484"},{"key":"e_1_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.5555\/3358807.3358848"},{"key":"e_1_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.5555\/3154630.3154660"},{"key":"e_1_2_2_15_1","doi-asserted-by":"publisher","DOI":"10.1109\/ucc-companion.2018.00049"},{"key":"e_1_2_2_16_1","unstructured":"github. [n. d.]. Durable Task Framework. https:\/\/github.com\/Azure\/durabletask github. [n. d.]. Durable Task Framework. https:\/\/github.com\/Azure\/durabletask"},{"key":"e_1_2_2_17_1","unstructured":"github. 2021. DurableTask.MSSQL Backend. https:\/\/github.com\/microsoft\/durabletask-mssql github. 2021. DurableTask.MSSQL Backend. https:\/\/github.com\/microsoft\/durabletask-mssql"},{"key":"e_1_2_2_18_1","unstructured":"github. 2021. DurableTask.Netherite Backend. https:\/\/github.com\/microsoft\/durabletask-netherite github. 2021. DurableTask.Netherite Backend. https:\/\/github.com\/microsoft\/durabletask-netherite"},{"key":"e_1_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.14778\/3377369.3377370"},{"key":"e_1_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2414639.2414641"},{"key":"e_1_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.14778\/3476249.3476273"},{"key":"e_1_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3360575"},{"key":"e_1_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3127479.3128601"},{"key":"e_1_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3401025.3401731"},{"key":"e_1_2_2_25_1","unstructured":"2020. What are Durable Functions? - Microsoft Azure. https:\/\/docs.microsoft.com\/en-us\/azure\/azure-functions\/durable\/durable-functions-overview 2020. What are Durable Functions? - Microsoft Azure. https:\/\/docs.microsoft.com\/en-us\/azure\/azure-functions\/durable\/durable-functions-overview"},{"key":"e_1_2_2_26_1","unstructured":"2020. Azure Logic Apps Service. https:\/\/azure.microsoft.com\/en-us\/services\/logic-apps\/ 2020. Azure Logic Apps Service. https:\/\/azure.microsoft.com\/en-us\/services\/logic-apps\/"},{"key":"e_1_2_2_27_1","unstructured":"2021. Azure SQL. https:\/\/azure.microsoft.com\/en-us\/services\/azure-sql\/ Accessed: 2021-04-13 2021. Azure SQL. https:\/\/azure.microsoft.com\/en-us\/services\/azure-sql\/ Accessed: 2021-04-13"},{"key":"e_1_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-662-44202-9_13"},{"key":"e_1_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/232629.232644"},{"key":"e_1_2_2_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3318464.3389758"},{"key":"e_1_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/3318464.3380609"},{"key":"e_1_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.5555\/509043"},{"key":"e_1_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2480359.2429100"},{"key":"e_1_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.5555\/767132.767139"},{"key":"e_1_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3342195.3387535"},{"key":"e_1_2_2_36_1","doi-asserted-by":"publisher","DOI":"10.14778\/3407790.3407836"},{"key":"e_1_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.5555\/2228298.2228301"},{"key":"e_1_2_2_38_1","doi-asserted-by":"publisher","DOI":"10.5555\/3488766.3488833"},{"key":"e_1_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/3419111.3421277"}],"container-title":["Proceedings of the ACM on Programming Languages"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3485510","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3485510","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:18:40Z","timestamp":1750177120000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3485510"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,10,15]]},"references-count":39,"journal-issue":{"issue":"OOPSLA","published-print":{"date-parts":[[2021,10,20]]}},"alternative-id":["10.1145\/3485510"],"URL":"https:\/\/doi.org\/10.1145\/3485510","relation":{},"ISSN":["2475-1421"],"issn-type":[{"value":"2475-1421","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,10,15]]},"assertion":[{"value":"2021-10-15","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F3486897 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,23]],"date-time":"2026-10-23T20:29:55Z","timestamp":1792787395421,"version":"4.5.0"},"reference-count":34,"publisher":"Association for Computing Machinery (ACM)","issue":"6","license":[{"start":{"date-parts":[[2022,5,20]],"date-time":"2022-05-20T00:00:00Z","timestamp":1653004800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"name":"ELIXIR the research infrastructure for life-science data","award":["2018-CWL"],"award-info":[{"award-number":["2018-CWL"]}]},{"DOI":"10.13039\/100010662","name":"H2020 Excellent Science","doi-asserted-by":"publisher","award":["823830,675728,824087,739563,730976,676559,653477"],"award-info":[{"award-number":["823830,675728,824087,739563,730976,676559,653477"]}],"id":[{"id":"10.13039\/100010662","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["Commun. ACM"],"published-print":{"date-parts":[[2022,6]]},"abstract":"<jats:p>Standardizing computational reuse and portability with the Common Workflow Language.<\/jats:p>","DOI":"10.1145\/3486897","type":"journal-article","created":{"date-parts":[[2022,5,20]],"date-time":"2022-05-20T10:56:52Z","timestamp":1653044212000},"page":"54-63","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":199,"title":["Methods included"],"prefix":"10.1145","volume":"65","author":[{"given":"Michael R.","family":"Crusoe","sequence":"first","affiliation":[{"name":"VU Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sanne","family":"Abeln","sequence":"additional","affiliation":[{"name":"VU Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alexandru","family":"Iosup","sequence":"additional","affiliation":[{"name":"VU Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Peter","family":"Amstutz","sequence":"additional","affiliation":[{"name":"Curii Corporation, Sommerville, MA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"John","family":"Chilton","sequence":"additional","affiliation":[{"name":"Pennsylvania State University, State College, PA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Neboj\u0161a","family":"Tijani\u0107","sequence":"additional","affiliation":[{"name":"Seven Bridges Genomics Inc., Charlestown, MA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Herv\u00e9","family":"M\u00e9nager","sequence":"additional","affiliation":[{"name":"Universit\u00e9 de Paris, Paris, France"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Stian","family":"Soiland-Reyes","sequence":"additional","affiliation":[{"name":"University of Manchester, U.K and University of Amsterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Bogdan","family":"Gavrilovi\u0107","sequence":"additional","affiliation":[{"name":"Seven Bridges Genomics Inc., Charlestown, MA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Carole","family":"Goble","sequence":"additional","affiliation":[{"name":"The University of Manchester, Manchester, U.K"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"The CWL","family":"Community","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,5,20]]},"reference":[{"key":"e_1_2_2_1_1","doi-asserted-by":"publisher","DOI":"10.1093\/nar\/gky379"},{"key":"e_1_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3307681.3325400"},{"key":"e_1_2_2_3_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.websem.2015.01.003"},{"key":"e_1_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/1656274.1656280"},{"key":"e_1_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/TETC.2020.3019202"},{"key":"e_1_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-84628-757-2_22"},{"key":"e_1_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1007\/s13222-012-0100-z"},{"key":"e_1_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1186\/s12859-016-0978-9"},{"key":"e_1_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2014.10.008"},{"key":"e_1_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2723872.2723875"},{"key":"e_1_2_2_12_1","doi-asserted-by":"publisher","DOI":"10.1093\/gigascience\/giz109"},{"key":"e_1_2_2_13_1","volume-title":"OGC Earth observations applications pilot: Terradue engineering report. OGC Public Engineering Report OGC 20-042","author":"Gon\u00e7alves P.","unstructured":"Gon\u00e7alves , P. OGC Earth observations applications pilot: Terradue engineering report. OGC Public Engineering Report OGC 20-042 . Open Geospatial Consortium . http:\/\/docs.opengeospatial.org\/per\/20-042.html. Gon\u00e7alves, P. OGC Earth observations applications pilot: Terradue engineering report. OGC Public Engineering Report OGC 20-042. Open Geospatial Consortium. http:\/\/docs.opengeospatial.org\/per\/20-042.html."},{"key":"e_1_2_2_14_1","doi-asserted-by":"publisher","DOI":"10.1353\/lib.2017.0018"},{"key":"e_1_2_2_15_1","volume-title":"Bioinformatics Open Source Conference (July","author":"Guarracino A.","year":"2020","unstructured":"Guarracino , A. et al. COVID-19 PubSeq: Public SARS-CoV-2 sequence resource . Bioinformatics Open Source Conference (July 2020 ), https:\/\/sched.co\/coLw. Guarracino, A. et al. COVID-19 PubSeq: Public SARS-CoV-2 sequence resource. Bioinformatics Open Source Conference (July 2020), https:\/\/sched.co\/coLw."},{"key":"e_1_2_2_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/IEEESTD.2020.9094416"},{"key":"e_1_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3186266"},{"key":"e_1_2_2_18_1","volume-title":"TR-19-01: A cloud-agnostic framework for geo-distributed data-intensive applications. RENCI","author":"Jiang F.","year":"2019","unstructured":"Jiang , F. Castillo , C. , and Ahalt , S . TR-19-01: A cloud-agnostic framework for geo-distributed data-intensive applications. RENCI , University of North Carolina at Chapel Hill , ( 2019 ), https:\/\/renci.org\/technical-reports\/tr-19-01\/. Jiang, F. Castillo, C., and Ahalt, S. TR-19-01: A cloud-agnostic framework for geo-distributed data-intensive applications. RENCI, University of North Carolina at Chapel Hill, (2019), https:\/\/renci.org\/technical-reports\/tr-19-01\/."},{"key":"e_1_2_2_19_1","volume-title":"PIVOT: Cost-aware scheduling of data-intensive applications in a cloud-agnostic system. RENCI","author":"Jiang F.","year":"2019","unstructured":"Jiang , F. , Ferriter , K. , and Castillo , C . PIVOT: Cost-aware scheduling of data-intensive applications in a cloud-agnostic system. RENCI , University of North Carolina at Chapel Hill , ( 2019 ). https:\/\/renci.org\/technical-reports\/tr-19-02\/. Jiang, F., Ferriter, K., and Castillo, C. PIVOT: Cost-aware scheduling of data-intensive applications in a cloud-agnostic system. RENCI, University of North Carolina at Chapel Hill, (2019). https:\/\/renci.org\/technical-reports\/tr-19-02\/."},{"key":"e_1_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1093\/gigascience\/giz095"},{"key":"e_1_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1093\/gigascience\/giz084"},{"key":"e_1_2_2_22_1","doi-asserted-by":"publisher","DOI":"10.17487\/RFC8493"},{"key":"e_1_2_2_23_1","doi-asserted-by":"crossref","DOI":"10.62973\/20-045","volume-title":"OGC Earth observation applications pilot: CRIM engineering report","author":"Landry T.","year":"2020","unstructured":"Landry , T. OGC Earth observation applications pilot: CRIM engineering report . Open Geospatial Consortium Public Engineering Report 20-045 ( 2020 ), http:\/\/docs.opengeospatial.org\/per\/20-045.html Landry, T. OGC Earth observation applications pilot: CRIM engineering report. Open Geospatial Consortium Public Engineering Report 20-045 (2020), http:\/\/docs.opengeospatial.org\/per\/20-045.html"},{"key":"e_1_2_2_24_1","doi-asserted-by":"publisher","DOI":"10.1158\/0008-5472.can-17-0387"},{"key":"e_1_2_2_25_1","doi-asserted-by":"publisher","DOI":"10.1007\/s12275-011-1213-z"},{"key":"e_1_2_2_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/2452376.2452478"},{"key":"e_1_2_2_27_1","doi-asserted-by":"publisher","DOI":"10.1093\/nar\/gkz1035"},{"key":"e_1_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1109\/MCSE.2019.2906593.00000"},{"key":"e_1_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-019-02619-z"},{"key":"e_1_2_2_30_1","volume-title":"Issue 7","year":"1979","unstructured":"POSIX.1-2008 : IEEE Std 1003.1\u2122-2008 and The Open Group Technical Standard Base Specifications , Issue 7 . IEEE and The Open Group , https:\/\/pubs.opengroup.org\/onlinepubs\/96999 1979 9.2008edition\/. POSIX.1-2008: IEEE Std 1003.1\u2122-2008 and The Open Group Technical Standard Base Specifications, Issue 7. IEEE and The Open Group, https:\/\/pubs.opengroup.org\/onlinepubs\/9699919799.2008edition\/."},{"key":"e_1_2_2_31_1","doi-asserted-by":"publisher","DOI":"10.1186\/2047-217X-2-15"},{"key":"e_1_2_2_32_1","doi-asserted-by":"crossref","DOI":"10.62973\/20-073","volume-title":"OGC Earth observation applications pilot: Summary engineering report","author":"Simonis I.","year":"2020","unstructured":"Simonis , I. OGC Earth observation applications pilot: Summary engineering report . Open Geospatial Consortium Public Engineering Report OGC 20-073 ( 2020 ), https:\/\/docs.ogc.org\/per\/20-073.html. Simonis, I. OGC Earth observation applications pilot: Summary engineering report. Open Geospatial Consortium Public Engineering Report OGC 20-073 (2020), https:\/\/docs.ogc.org\/per\/20-073.html."},{"key":"e_1_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1186\/1471-2105-11-S12-S1"},{"key":"e_1_2_2_34_1","doi-asserted-by":"publisher","DOI":"10.4233\/uuid:9e9fa82e-06c1-4d0d-9e20-5620259a6c65"},{"key":"e_1_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1038\/nbt.3772"}],"container-title":["Communications of the ACM"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3486897","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3486897","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T16:18:47Z","timestamp":1750177127000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3486897"}},"subtitle":["standardizing computational reuse and portability with the Common Workflow Language"],"short-title":[],"issued":{"date-parts":[[2022,5,20]]},"references-count":34,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2022,6]]}},"alternative-id":["10.1145\/3486897"],"URL":"https:\/\/doi.org\/10.1145\/3486897","relation":{},"ISSN":["0001-0782","1557-7317"],"issn-type":[{"value":"0001-0782","type":"print"},{"value":"1557-7317","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,5,20]]},"assertion":[{"value":"2022-05-20","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F3571730 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T16:15:00Z","timestamp":1793290500254,"version":"4.5.0"},"reference-count":170,"publisher":"Association for Computing Machinery (ACM)","issue":"12","license":[{"start":{"date-parts":[[2023,3,3]],"date-time":"2023-03-03T00:00:00Z","timestamp":1677801600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["ACM Comput. Surv."],"published-print":{"date-parts":[[2023,12,31]]},"abstract":"<jats:p>Natural Language Generation (NLG) has improved exponentially in recent years thanks to the development of sequence-to-sequence deep learning technologies such as Transformer-based language models. This advancement has led to more fluent and coherent NLG, leading to improved development in downstream tasks such as abstractive summarization, dialogue generation, and data-to-text generation. However, it is also apparent that deep learning based generation is prone to hallucinate unintended text, which degrades the system performance and fails to meet user expectations in many real-world scenarios. To address this issue, many studies have been presented in measuring and mitigating hallucinated texts, but these have never been reviewed in a comprehensive manner before.<\/jats:p><jats:p>In this survey, we thus provide a broad overview of the research progress and challenges in the hallucination problem in NLG. The survey is organized into two parts: (1) a general overview of metrics, mitigation methods, and future directions, and (2) an overview of task-specific research progress on hallucinations in the following downstream tasks, namely abstractive summarization, dialogue generation, generative question answering, data-to-text generation, and machine translation. This survey serves to facilitate collaborative efforts among researchers in tackling the challenge of hallucinated texts in NLG.<\/jats:p><jats:p\/>","DOI":"10.1145\/3571730","type":"journal-article","created":{"date-parts":[[2022,11,17]],"date-time":"2022-11-17T15:05:37Z","timestamp":1668697537000},"page":"1-38","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":4810,"title":["Survey of Hallucination in Natural Language Generation"],"prefix":"10.1145","volume":"55","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0206-7861","authenticated-orcid":false,"given":"Ziwei","family":"Ji","sequence":"first","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1144-1103","authenticated-orcid":false,"given":"Nayeon","family":"Lee","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3921-3519","authenticated-orcid":false,"given":"Rita","family":"Frieske","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5396-950X","authenticated-orcid":false,"given":"Tiezheng","family":"Yu","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-5746-9545","authenticated-orcid":false,"given":"Dan","family":"Su","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2205-2429","authenticated-orcid":false,"given":"Yan","family":"Xu","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6423-539X","authenticated-orcid":false,"given":"Etsuko","family":"Ishii","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2968-0033","authenticated-orcid":false,"given":"Ye Jin","family":"Bang","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8672-715X","authenticated-orcid":false,"given":"Andrea","family":"Madotto","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0628-7132","authenticated-orcid":false,"given":"Pascale","family":"Fung","sequence":"additional","affiliation":[{"name":"Hong Kong University of Science and Technology, Hong Kong"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,3,3]]},"reference":[{"key":"e_1_3_2_2_2","article-title":"Flamingo: A visual language model for few-shot learning","author":"Alayrac Jean-Baptiste","year":"2022","unstructured":"Jean-Baptiste Alayrac, Jeff Donahue, Pauline Luc, Antoine Miech, Iain Barr, Yana Hasson, Karel Lenc, et\u00a0al. 2022. Flamingo: A visual language model for few-shot learning. arXiv preprint arXiv:2204.14198 (2022).","journal-title":"arXiv preprint arXiv:2204.14198"},{"key":"e_1_3_2_3_2","first-page":"6078","volume-title":"Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing","author":"Aralikatte Rahul","year":"2021","unstructured":"Rahul Aralikatte, Shashi Narayan, Joshua Maynez, Sascha Rothe, and Ryan McDonald. 2021. Focus attention: Promoting faithfulness and diversity in summarization. In Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing. 6078\u20136095."},{"key":"e_1_3_2_4_2","first-page":"83","volume-title":"Proceedings of the 4th IEEE International Conference on Automatic Face and Gesture Recognition","author":"Baker S.","year":"2000","unstructured":"S. Baker and T. Kanade. 2000. Hallucinating faces. In Proceedings of the 4th IEEE International Conference on Automatic Face and Gesture Recognition. 83\u201388."},{"key":"e_1_3_2_5_2","volume-title":"Proceedings of the Annual Meeting of the Association for Computational Linguistics","author":"Balakrishnan Anusha","year":"2019","unstructured":"Anusha Balakrishnan, Jinfeng Rao, Kartikeya Upasani, Michael White, and Rajen Subba. 2019. Constrained decoding for neural NLG from compositional representations in task-oriented dialogue. In Proceedings of the Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_2_6_2","volume-title":"Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing","author":"Baly Ramy","year":"2018","unstructured":"Ramy Baly, Georgi Karadzhov, Dimitar Alexandrov, James Glass, and Preslav Nakov. 2018. Predicting factuality of reporting and bias of news media sources. In Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing."},{"key":"e_1_3_2_7_2","article-title":"Longformer: The long-document transformer","author":"Beltagy Iz","year":"2020","unstructured":"Iz Beltagy, Matthew E. Peters, and Arman Cohan. 2020. Longformer: The long-document transformer. arXiv:2004.05150 (2020).","journal-title":"arXiv:2004.05150"},{"key":"e_1_3_2_8_2","article-title":"Scheduled sampling for sequence prediction with recurrent neural networks","author":"Bengio Samy","year":"2015","unstructured":"Samy Bengio, Oriol Vinyals, Navdeep Jaitly, and Noam Shazeer. 2015. Scheduled sampling for sequence prediction with recurrent neural networks. In Proceedings of the 28th International Conference on Neural Information Processing Systems (NIPS\u201915). 1171\u20131179.","journal-title":"Proceedings of the 28th International Conference on Neural Information Processing Systems (NIPS\u201915)."},{"key":"e_1_3_2_9_2","first-page":"4164","volume-title":"Proceedings of the 2021 Conference of North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921)","author":"Beyer Anne","year":"2021","unstructured":"Anne Beyer, Sharid Lo\u00e1iciga, and David Schlangen. 2021. Is incoherence surprising? Targeted evaluation of coherence prediction from language models. In Proceedings of the 2021 Conference of North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921). 4164\u20134173."},{"key":"e_1_3_2_10_2","first-page":"2521","volume-title":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing","author":"Bi Bin","year":"2019","unstructured":"Bin Bi, Chen Wu, Ming Yan, Wei Wang, Jiangnan Xia, and Chenliang Li. 2019. Incorporating external knowledge into machine reading for generative question answering. In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing. 2521\u20132530."},{"key":"e_1_3_2_11_2","volume-title":"Proceedings of the 2022 IEEE\/CVF Winter Conference on Applications of Computer Vision","author":"Biten Ali Furkan","year":"2022","unstructured":"Ali Furkan Biten, Llu\u00eds G\u00f3mez, and Dimosthenis Karatzas. 2022. Let there be a clock on the beach: Reducing object hallucination in image captioning. In Proceedings of the 2022 IEEE\/CVF Winter Conference on Applications of Computer Vision. IEEE, Los Alamitos, CA."},{"key":"e_1_3_2_12_2","doi-asserted-by":"crossref","DOI":"10.1007\/978-1-4419-1223-7","volume-title":"A Dictionary of Hallucinations","author":"Blom Jan Dirk","year":"2010","unstructured":"Jan Dirk Blom. 2010. A Dictionary of Hallucinations. Springer."},{"key":"e_1_3_2_13_2","first-page":"7236","volume-title":"Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing","author":"Briakou Eleftheria","year":"2021","unstructured":"Eleftheria Briakou and Marine Carpuat. 2021. Beyond noise: Mitigating the impact of fine-grained semantic divergences on neural machine translation. In Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing. 7236\u20137249."},{"key":"e_1_3_2_14_2","first-page":"1877","volume-title":"Advances in Neural Information Processing Systems","author":"Brown Tom","year":"2020","unstructured":"Tom Brown, Benjamin Mann, Nick Ryder, Melanie Subbiah, Jared D. Kaplan, Prafulla Dhariwal, Arvind Neelakantan, et\u00a0al. 2020. Language models are few-shot learners. In Advances in Neural Information Processing Systems33. Curran Associates, 1877\u20131901."},{"key":"e_1_3_2_15_2","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing","author":"Cao Meng","year":"2020","unstructured":"Meng Cao, Yue Dong, Jiapeng Wu, and Jackie Chi Kit Cheung. 2020. Factual error correction for abstractive summarization models. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing."},{"key":"e_1_3_2_16_2","first-page":"6633","volume-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","author":"Cao Shuyang","year":"2021","unstructured":"Shuyang Cao and Lu Wang. 2021. CLIFF: Contrastive learning for improving faithfulness and factuality in abstractive summarization. In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing. 6633\u20136649."},{"key":"e_1_3_2_17_2","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","author":"Cao Ziqiang","year":"2018","unstructured":"Ziqiang Cao, Furu Wei, Wenjie Li, and Sujian Li. 2018. Faithful to the original: Fact aware neural abstractive summarization. In Proceedings of the AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_18_2","first-page":"2633","volume-title":"Proceedings of the 30th USENIX Security Symposium","author":"Carlini Nicholas","year":"2021","unstructured":"Nicholas Carlini, Florian Tramer, Eric Wallace, Matthew Jagielski, Ariel Herbert-Voss, Katherine Lee, Adam Roberts, et\u00a0al. 2021. Extracting training data from large language models. In Proceedings of the 30th USENIX Security Symposium. 2633\u20132650."},{"key":"e_1_3_2_19_2","first-page":"5935","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921)","author":"Chen Sihao","year":"2021","unstructured":"Sihao Chen, Fan Zhang, Kazoo Sone, and Dan Roth. 2021. Improving faithfulness in abstractive summarization with contrast candidate generation and selection. In Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921). 5935\u20135941."},{"key":"e_1_3_2_20_2","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2020","author":"Chen Zhiyu","year":"2020","unstructured":"Zhiyu Chen, Wenhu Chen, Hanwen Zha, Xiyou Zhou, Yunkai Zhang, Sairam Sundaresan, and William Yang Wang. 2020. Logic2Text: High-fidelity natural language generation from logical forms. In Findings of the Association for Computational Linguistics: EMNLP 2020. Association for Computational Linguistics, 2096\u20132111."},{"key":"e_1_3_2_21_2","volume-title":"arXiv preprint arXiv:2009.09796","author":"Crawshaw Michael","year":"2020","unstructured":"Michael Crawshaw. 2020. Multi-task learning with deep neural networks: A survey. arXiv preprint arXiv:2009.09796 (2020)."},{"key":"e_1_3_2_22_2","first-page":"1406","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Cui Leyang","year":"2020","unstructured":"Leyang Cui, Yu Wu, Shujie Liu, Yue Zhang, and Ming Zhou. 2020. MuTual: A dataset for multi-turn dialogue reasoning. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics. 1406\u20131416."},{"key":"e_1_3_2_23_2","doi-asserted-by":"crossref","first-page":"4884","DOI":"10.18653\/v1\/P19-1483","volume-title":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","author":"Dhingra Bhuwan","year":"2019","unstructured":"Bhuwan Dhingra, Manaal Faruqui, Ankur Parikh, Ming-Wei Chang, Dipanjan Das, and William Cohen. 2019. Handling divergent reference texts when evaluating table-to-text generation. In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics. 4884\u20134895."},{"key":"e_1_3_2_24_2","doi-asserted-by":"crossref","first-page":"187","DOI":"10.1007\/978-3-030-29135-8_7","volume-title":"The NeurIPS\u201918 Competition","author":"Dinan Emily","year":"2020","unstructured":"Emily Dinan, Varvara Logacheva, Valentin Malykh, Alexander Miller, Kurt Shuster, Jack Urbanek, Douwe Kiela, et\u00a0al. 2020. The second conversational intelligence challenge (ConvAI2). In The NeurIPS\u201918 Competition. Springer, 187\u2013208."},{"key":"e_1_3_2_25_2","volume-title":"Proceedings of the International Conference on Learning Representations","author":"Dinan Emily","year":"2018","unstructured":"Emily Dinan, Stephen Roller, Kurt Shuster, Angela Fan, Michael Auli, and Jason Weston. 2018. Wizard of Wikipedia: Knowledge-powered conversational agents. In Proceedings of the International Conference on Learning Representations."},{"key":"e_1_3_2_26_2","first-page":"9320","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing","author":"Dong Yue","year":"2020","unstructured":"Yue Dong, Shuohang Wang, Zhe Gan, Yu Cheng, Jackie Chi Kit Cheung, and Jingjing Liu. 2020. Multi-fact correction in abstractive text summarization. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing. 9320\u20139331."},{"key":"e_1_3_2_27_2","first-page":"5055","volume-title":"Proceedings of the 58th Annual Meeting of the ACL","author":"Durmus Esin","year":"2020","unstructured":"Esin Durmus, He He, and Mona Diab. 2020. FEQA: A question answering evaluation framework for faithfulness assessment in abstractive summarization. In Proceedings of the 58th Annual Meeting of the ACL. 5055\u20135070."},{"key":"e_1_3_2_28_2","doi-asserted-by":"crossref","first-page":"421","DOI":"10.18653\/v1\/W19-8652","volume-title":"Proceedings of the 12th International Conference on Natural Language Generation","author":"Du\u0161ek Ond\u0159ej","year":"2019","unstructured":"Ond\u0159ej Du\u0161ek, David M. Howcroft, and Verena Rieser. 2019. Semantic noise matters for neural natural language generation. In Proceedings of the 12th International Conference on Natural Language Generation. 421\u2013426."},{"key":"e_1_3_2_29_2","doi-asserted-by":"crossref","first-page":"131","DOI":"10.18653\/v1\/2020.inlg-1.19","volume-title":"Proceedings of the 13th International Conference on Natural Language Generation","author":"Du\u0161ek Ond\u0159ej","year":"2020","unstructured":"Ond\u0159ej Du\u0161ek and Zden\u011bk Kasner. 2020. Evaluating semantic accuracy of data-to-text generation with natural language inference. In Proceedings of the 13th International Conference on Natural Language Generation. 131\u2013137."},{"key":"e_1_3_2_30_2","first-page":"3806","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","author":"Dziri Nouha","year":"2019","unstructured":"Nouha Dziri, Ehsan Kamalloo, Kory Mathewson, and Osmar Zaiane. 2019. Evaluating coherence in dialogue systems using entailment. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 3806\u20133812."},{"key":"e_1_3_2_31_2","doi-asserted-by":"crossref","first-page":"2197","DOI":"10.18653\/v1\/2021.emnlp-main.168","volume-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","author":"Dziri Nouha","year":"2021","unstructured":"Nouha Dziri, Andrea Madotto, Osmar R. Zaiane, and Avishek Joey Bose. 2021. Neural path hunter: Reducing hallucination in dialogue systems via path grounding. In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing. 2197\u20132214."},{"key":"e_1_3_2_32_2","article-title":"Evaluating groundedness in dialogue systems: The BEGIN benchmark","author":"Dziri Nouha","year":"2021","unstructured":"Nouha Dziri, Hannah Rashkin, Tal Linzen, and David Reitter. 2021. Evaluating groundedness in dialogue systems: The BEGIN benchmark. In Findings of the Association for Computational Linguistics. Association for Computational Linguistics, 1\u201312.","journal-title":"Findings of the Association for Computational Linguistics."},{"key":"e_1_3_2_33_2","doi-asserted-by":"crossref","first-page":"2214","DOI":"10.18653\/v1\/P19-1213","volume-title":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","author":"Falke Tobias","year":"2019","unstructured":"Tobias Falke, Leonardo F. R. Ribeiro, Prasetya Ajie Utama, Ido Dagan, and Iryna Gurevych. 2019. Ranking generated summaries by correctness: An interesting but challenging application for natural language inference. In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics. 2214\u20132220."},{"key":"e_1_3_2_34_2","first-page":"4186","volume-title":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing","author":"Fan Angela","year":"2019","unstructured":"Angela Fan, Claire Gardent, Chlo\u00e9 Braud, and Antoine Bordes. 2019. Using local knowledge graph construction to scale seq2seq models to multi-document inputs. In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing. 4186\u20134196."},{"key":"e_1_3_2_35_2","volume-title":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","author":"Fan Angela","year":"2019","unstructured":"Angela Fan, Yacine Jernite, Ethan Perez, David Grangier, Jason Weston, and Michael Auli. 2019. ELI5: Long form question answering. In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_2_36_2","volume-title":"Proceedings of the 2016 IEEE 12th Image, Video, and Multidimensional Signal Processing Workshop","author":"Fawzi Alhussein","year":"2016","unstructured":"Alhussein Fawzi, Horst Samulowitz, Deepak Turaga, and Pascal Frossard. 2016. Image inpainting through neural networks hallucinations. In Proceedings of the 2016 IEEE 12th Image, Video, and Multidimensional Signal Processing Workshop. IEEE, Los Alamitos, CA."},{"key":"e_1_3_2_37_2","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","author":"Feng Yang","year":"2020","unstructured":"Yang Feng, Wanying Xie, Shuhao Gu, Chenze Shao, Wen Zhang, Zhengxin Yang, and Dong Yu. 2020. Modeling fluency and faithfulness for diverse neural machine translation. In Proceedings of the AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_38_2","first-page":"864","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: Findings","author":"Filippova Katja","year":"2020","unstructured":"Katja Filippova. 2020. Controlled hallucinations: Learning to generate faithfully from noisy data. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: Findings. 864\u2013870."},{"key":"e_1_3_2_39_2","volume-title":"Perception, Hallucination, and Illusion","year":"2009","unstructured":"William Fish. 2009. Perception, Hallucination, and Illusion. Oxford University Press."},{"key":"e_1_3_2_40_2","doi-asserted-by":"crossref","first-page":"478","DOI":"10.18653\/v1\/2021.findings-acl.42","volume-title":"Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021","author":"Gabriel Saadia","year":"2021","unstructured":"Saadia Gabriel, Asli Celikyilmaz, Rahul Jha, Yejin Choi, and Jianfeng Gao. 2021. GO FIGURE: A meta evaluation of factuality in summarization. In Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021. Association for Computational Linguistics, 478\u2013487."},{"key":"e_1_3_2_41_2","first-page":"2","volume-title":"Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics: Tutorial Abstracts","author":"Gao Jianfeng","year":"2018","unstructured":"Jianfeng Gao, Michel Galley, and Lihong Li. 2018. Neural approaches to conversational AI. In Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics: Tutorial Abstracts. 2\u20137."},{"key":"e_1_3_2_42_2","volume-title":"Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics","author":"Gardent Claire","year":"2017","unstructured":"Claire Gardent, Anastasia Shimorina, Shashi Narayan, and Laura Perez-Beltrachini. 2017. Creating training corpora for NLG micro-planning. In Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_2_43_2","first-page":"4453","volume-title":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing","author":"Garg Sarthak","year":"2019","unstructured":"Sarthak Garg, Stephan Peitz, Udhyakumar Nallasamy, and Matthias Paulik. 2019. Jointly learning to align and translate with transformer models. In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing. 4453\u20134462."},{"key":"e_1_3_2_44_2","article-title":"CIDER: Commonsense inference for dialogue explanation and reasoning","author":"Ghosal Deepanway","year":"2021","unstructured":"Deepanway Ghosal, Pengfei Hong, Siqi Shen, Navonil Majumder, Rada Mihalcea, and Soujanya Poria. 2021. CIDER: Commonsense inference for dialogue explanation and reasoning. arXiv:2106.00510 (2021).","journal-title":"arXiv:2106.00510"},{"key":"e_1_3_2_45_2","doi-asserted-by":"crossref","first-page":"355","DOI":"10.1561\/1500000046","article-title":"Credibility in information retrieval","author":"Ginsca Alexandru L.","year":"2015","unstructured":"Alexandru L. Ginsca, Adrian Popescu, and Mihai Lupu. 2015. Credibility in information retrieval. Foundations and Trends in Information Retrieval 9, 5 (2015), 355\u2013475.","journal-title":"Foundations and Trends in Information Retrieval"},{"key":"e_1_3_2_46_2","first-page":"R517\u2013R519","article-title":"Cognitive neuroscience: Acting on numbers","author":"G\u00f6bel Silke M.","year":"2004","unstructured":"Silke M. G\u00f6bel and Matthew F. S. Rushworth. 2004. Cognitive neuroscience: Acting on numbers. Current Biology 14, 13 (2004), R517\u2013R519.","journal-title":"Current Biology"},{"key":"e_1_3_2_47_2","doi-asserted-by":"crossref","first-page":"166","DOI":"10.1145\/3292500.3330955","volume-title":"Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","author":"Goodrich Ben","year":"2019","unstructured":"Ben Goodrich, Vinay Rao, Peter J. Liu, and Mohammad Saleh. 2019. Assessing the factual accuracy of generated text. In Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining. 166\u2013175."},{"key":"e_1_3_2_48_2","first-page":"366","article-title":"Differentiable scheduled sampling for credit assignment","author":"Goyal Kartik","year":"2017","unstructured":"Kartik Goyal, Chris Dyer, and Taylor Berg-Kirkpatrick. 2017. Differentiable scheduled sampling for credit assignment. In Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics.366\u2013371.","journal-title":"Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_2_49_2","first-page":"3592","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: Findings","author":"Goyal Tanya","year":"2020","unstructured":"Tanya Goyal and Greg Durrett. 2020. Evaluating factuality in generation with dependency-level entailment. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: Findings. 3592\u20133603."},{"key":"e_1_3_2_50_2","unstructured":"Beliz Gunel Chenguang Zhu Michael Zeng and Xuedong Huang. 2020. Mind the facts: Knowledge-boosted coherent abstractive text summarization. arXiv:2006.15435 (2020)."},{"key":"e_1_3_2_51_2","article-title":"DialFact: A benchmark for fact-checking in dialogue","author":"Gupta Prakhar","year":"2021","unstructured":"Prakhar Gupta, Chien-Sheng Wu, Wenhao Liu, and Caiming Xiong. 2021. DialFact: A benchmark for fact-checking in dialogue. arXiv preprint arXiv:2110.08222 (2021).","journal-title":"arXiv preprint arXiv:2110.08222"},{"key":"e_1_3_2_52_2","doi-asserted-by":"crossref","first-page":"1535","DOI":"10.18653\/v1\/P17-1141","article-title":"Lexically constrained decoding for sequence generation using grid beam search","author":"Hokamp Chris","year":"2017","unstructured":"Chris Hokamp and Qun Liu. 2017. Lexically constrained decoding for sequence generation using grid beam search. In Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics.1535\u20131546.","journal-title":"Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_2_53_2","volume-title":"Proceedings of the International Conference on Learning Representations","author":"Holtzman Ari","year":"2019","unstructured":"Ari Holtzman, Jan Buys, Li Du, Maxwell Forbes, and Yejin Choi. 2019. The curious case of neural text degeneration. In Proceedings of the International Conference on Learning Representations."},{"key":"e_1_3_2_54_2","volume-title":"Proceedings of the 2nd DialDoc Workshop on Document-Grounded Dialogue and Conversational Question Answering","author":"Honovich Or","year":"2022","unstructured":"Or Honovich, Roee Aharoni, Jonathan Herzig, Hagai Taitelbaum, Doron Kukliansy, Vered Cohen, Thomas Scialom, Idan Szpektor, Avinatan Hassidim, and Yossi Matias. 2022. TRUE: Re-evaluating factual consistency evaluation. In Proceedings of the 2nd DialDoc Workshop on Document-Grounded Dialogue and Conversational Question Answering."},{"key":"e_1_3_2_55_2","doi-asserted-by":"crossref","first-page":"7856","DOI":"10.18653\/v1\/2021.emnlp-main.619","volume-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","author":"Honovich Or","year":"2021","unstructured":"Or Honovich, Leshem Choshen, Roee Aharoni, Ella Neeman, Idan Szpektor, and Omri Abend. 2021. Q \$^2\$ : Evaluating factual consistency in knowledge-grounded dialogues via question generation and question answering. In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing. 7856\u20137870."},{"key":"e_1_3_2_56_2","volume-title":"Proceedings of the Annual Meeting of the Association for Computational Linguistics","author":"Huang Luyang","year":"2020","unstructured":"Luyang Huang, Lingfei Wu, and Lu Wang. 2020. Knowledge graph-augmented abstractive summarization with semantic-driven cloze reward. In Proceedings of the Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_2_57_2","first-page":"Article 21, 32","article-title":"Challenges in building intelligent open-domain dialog systems","author":"Huang Minlie","year":"2020","unstructured":"Minlie Huang, Xiaoyan Zhu, and Jianfeng Gao. 2020. Challenges in building intelligent open-domain dialog systems. ACM Transactions on Information Systems 38, 3 (2020), Article 21, 32 pages.","journal-title":"ACM Transactions on Information Systems"},{"key":"e_1_3_2_58_2","article-title":"The factual inconsistency problem in abstractive text summarization: A survey","author":"Huang Yichong","year":"2021","unstructured":"Yichong Huang, Xiachong Feng, Xiaocheng Feng, and Bing Qin. 2021. The factual inconsistency problem in abstractive text summarization: A survey. arXiv preprint arXiv:2104.14839 (2021).","journal-title":"arXiv preprint arXiv:2104.14839"},{"key":"e_1_3_2_59_2","article-title":"Imitation learning: A survey of learning methods","author":"Hussein Ahmed","year":"2017","unstructured":"Ahmed Hussein, Mohamed Medhat Gaber, Eyad Elyan, and Chrisina Jayne. 2017. Imitation learning: A survey of learning methods. ACM Computing Surveys 50, 2 (2017), Article 21, 35 pages.","journal-title":"ACM Computing Surveys"},{"key":"e_1_3_2_60_2","unstructured":"Ziwei Ji Yan Xu I.-Tsun Cheng Samuel Cahyawijaya Rita Frieske Etsuko Ishii Min Zeng Andrea Madotto and Pascale Fung. 2022. VScript: Controllable script generation with visual presentation. arxiv:2203.00314 (2022)."},{"key":"e_1_3_2_61_2","doi-asserted-by":"crossref","first-page":"888","DOI":"10.18653\/v1\/W18-6478","volume-title":"Proceedings of the 3rd Conference on Machine Translation: Shared Task Papers","author":"Junczys-Dowmunt Marcin","year":"2018","unstructured":"Marcin Junczys-Dowmunt. 2018. Dual conditional cross-entropy filtering of noisy parallel corpora. In Proceedings of the 3rd Conference on Machine Translation: Shared Task Papers. 888\u2013895."},{"key":"e_1_3_2_62_2","doi-asserted-by":"crossref","first-page":"718","DOI":"10.18653\/v1\/2020.acl-main.66","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Kang Daniel","year":"2020","unstructured":"Daniel Kang and Tatsunori B. Hashimoto. 2020. Improved natural language generation via loss truncation. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics. 718\u2013731."},{"key":"e_1_3_2_63_2","first-page":"2234","volume-title":"Proceedings of the 2021 IEEE International Conference on Image Processing","author":"Kayhan Osman Semih","year":"2021","unstructured":"Osman Semih Kayhan, Bart Vredebregt, and Jan C. van Gemert. 2021. Hallucination in object detection\u2014A study in visual part verification. In Proceedings of the 2021 IEEE International Conference on Image Processing. IEEE, Los Alamitos, CA, 2234\u20132238."},{"key":"e_1_3_2_64_2","doi-asserted-by":"crossref","first-page":"421","DOI":"10.18653\/v1\/2021.findings-emnlp.38","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2021","author":"Khashabi Daniel","year":"2021","unstructured":"Daniel Khashabi, Amos Ng, Tushar Khot, Ashish Sabharwal, Hannaneh Hajishirzi, and Chris Callison-Burch. 2021. GooAQ: Open question answering with diverse answer types. In Findings of the Association for Computational Linguistics: EMNLP 2021. Association for Computational Linguistics, 421\u2013433."},{"key":"e_1_3_2_65_2","doi-asserted-by":"crossref","unstructured":"Philipp Koehn and Rebecca Knowles. 2017. Six challenges for neural machine translation. In Proceedings of the 1st Workshop on Neural Machine Translation . 28\u201339.","DOI":"10.18653\/v1\/W17-3204"},{"key":"e_1_3_2_66_2","first-page":"6618","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","author":"Kong Xiang","year":"2019","unstructured":"Xiang Kong, Zhaopeng Tu, Shuming Shi, Eduard Hovy, and Tong Zhang. 2019. Neural machine translation with adequacy-oriented learning. In Proceedings of the AAAI Conference on Artificial Intelligence. 6618\u20136625."},{"key":"e_1_3_2_67_2","first-page":"4940","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921)","author":"Krishna Kalpesh","year":"2021","unstructured":"Kalpesh Krishna, Aurko Roy, and Mohit Iyyer. 2021. Hurdles to progress in long-form question answering. In Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921). 4940\u20134957."},{"key":"e_1_3_2_68_2","first-page":"9332","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing","author":"Kryscinski Wojciech","year":"2020","unstructured":"Wojciech Kryscinski, Bryan McCann, Caiming Xiong, and Richard Socher. 2020. Evaluating the factual consistency of abstractive text summarization. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing. 9332\u20139346."},{"key":"e_1_3_2_69_2","article-title":"SummaC: Re-visiting NLI-based models for inconsistency detection in summarization","author":"Laban Philippe","year":"2022","unstructured":"Philippe Laban, Tobias Schnabel, Paul N. Bennett, and Marti A. Hearst. 2022. SummaC: Re-visiting NLI-based models for inconsistency detection in summarization. Transactions of the Association for Computational Linguistics 10 (2022), 163\u2013177.","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"e_1_3_2_70_2","volume-title":"Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing","author":"Lebret R\u00e9mi","year":"2016","unstructured":"R\u00e9mi Lebret, David Grangier, and Michael Auli. 2016. Neural text generation from structured data with application to the biography domain. In Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing."},{"key":"e_1_3_2_71_2","article-title":"Hallucinations in neural machine translation","author":"Lee Katherine","year":"2019","unstructured":"Katherine Lee, Orhan Firat, Ashish Agarwal, Clara Fannjiang, and David Sussillo. 2019. Hallucinations in neural machine translation. In Proceedings of the International Conference on Learning Representations.","journal-title":"Proceedings of the International Conference on Learning Representations."},{"key":"e_1_3_2_72_2","article-title":"Deduplicating training data makes language models better","author":"Lee Katherine","year":"2021","unstructured":"Katherine Lee, Daphne Ippolito, Andrew Nystrom, Chiyuan Zhang, Douglas Eck, Chris Callison-Burch, and Nicholas Carlini. 2021. Deduplicating training data makes language models better. arXiv preprint arXiv:2107.06499 (2021).","journal-title":"arXiv preprint arXiv:2107.06499"},{"key":"e_1_3_2_73_2","first-page":"36","volume-title":"Proceedings of the 3rd Workshop on Fact Extraction and VERification (FEVER\u201920)","author":"Lee Nayeon","year":"2020","unstructured":"Nayeon Lee, Belinda Z. Li, Sinong Wang, Wen-Tau Yih, Hao Ma, and Madian Khabsa. 2020. Language models as fact checkers? In Proceedings of the 3rd Workshop on Fact Extraction and VERification (FEVER\u201920). 36\u201341."},{"key":"e_1_3_2_74_2","article-title":"Factuality enhanced language models for open-ended text generation","author":"Lee Nayeon","year":"2022","unstructured":"Nayeon Lee, Wei Ping, Peng Xu, Mostofa Patwary, Mohammad Shoeybi, and Bryan Catanzaro. 2022. Factuality enhanced language models for open-ended text generation. arXiv preprint arXiv:2206.04624 (2022).","journal-title":"arXiv preprint arXiv:2206.04624"},{"key":"e_1_3_2_75_2","first-page":"1133","volume-title":"Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing (EMNLP\u201918)","author":"Lee Nayeon","year":"2018","unstructured":"Nayeon Lee, Chien-Sheng Wu, and Pascale Fung. 2018. Improving large-scale fact-checking using decomposable attention models and lexical tagging. In Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing (EMNLP\u201918). 1133\u20131138."},{"key":"e_1_3_2_76_2","doi-asserted-by":"crossref","first-page":"7871","DOI":"10.18653\/v1\/2020.acl-main.703","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Lewis Mike","year":"2020","unstructured":"Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Veselin Stoyanov, and Luke Zettlemoyer. 2020. BART: Denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics. 7871\u20137880."},{"key":"e_1_3_2_77_2","article-title":"Data augmentation approaches in natural language processing: A survey","author":"Li Bohan","year":"2021","unstructured":"Bohan Li, Yutai Hou, and Wanxiang Che. 2021. Data augmentation approaches in natural language processing: A survey. arXiv preprint arXiv:2110.01852 (2021).","journal-title":"arXiv preprint arXiv:2110.01852"},{"key":"e_1_3_2_78_2","first-page":"942","volume-title":"Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing","author":"Li Chenliang","year":"2021","unstructured":"Chenliang Li, Bin Bi, Ming Yan, Wei Wang, and Songfang Huang. 2021. Addressing semantic drift in generative question answering with auxiliary extraction. In Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing. 942\u2013947."},{"key":"e_1_3_2_79_2","first-page":"1430","volume-title":"Proceedings of the 27th International Conference on Computational Linguistics","author":"Li Haoran","year":"2018","unstructured":"Haoran Li, Junnan Zhu, Jiajun Zhang, and Chengqing Zong. 2018. Ensure the correctness of the summary: Incorporate entailment knowledge into abstractive sentence summarization. In Proceedings of the 27th International Conference on Computational Linguistics. 1430\u20131441."},{"key":"e_1_3_2_80_2","volume-title":"Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics","author":"Li Jiwei","year":"2016","unstructured":"Jiwei Li, Michel Galley, Chris Brockett, Georgios Spithourakis, Jianfeng Gao, and Bill Dolan. 2016. A persona-based neural conversation model. In Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_2_81_2","first-page":"4715","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Li Margaret","year":"2020","unstructured":"Margaret Li, Stephen Roller, Ilia Kulikov, Sean Welleck, Y-Lan Boureau, Kyunghyun Cho, and Jason Weston. 2020. Don\u2019t say that! Making inconsistent dialogue unlikely with unlikelihood training. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics. 4715\u20134728."},{"key":"e_1_3_2_82_2","volume-title":"Proceedings of the International Conference on Learning Representations","author":"Li Tian","year":"2020","unstructured":"Tian Li, Ahmad Beirami, Maziar Sanjabi, and Virginia Smith. 2020. Tilted empirical risk minimization. In Proceedings of the International Conference on Learning Representations."},{"key":"e_1_3_2_83_2","first-page":"97","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Li Yangming","year":"2020","unstructured":"Yangming Li, Kaisheng Yao, Libo Qin, Wanxiang Che, Xiaolong Li, and Ting Liu. 2020. Slot-consistent NLG for task-oriented dialogue systems with iterative rectification network. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics. 97\u2013106."},{"key":"e_1_3_2_84_2","article-title":"TruthfulQA: Measuring how models mimic human falsehoods","author":"Lin Stephanie","year":"2021","unstructured":"Stephanie Lin, Jacob Hilton, and Owain Evans. 2021. TruthfulQA: Measuring how models mimic human falsehoods. arXiv preprint arXiv:2109.07958 (2021).","journal-title":"arXiv preprint arXiv:2109.07958"},{"key":"e_1_3_2_85_2","volume-title":"arXiv preprint arXiv:2104.08704","author":"Liu Tianyu","year":"2021","unstructured":"Tianyu Liu, Yizhe Zhang, Chris Brockett, Yi Mao, Zhifang Sui, Weizhu Chen, and Bill Dolan. 2021. A token-level reference-free hallucination detection benchmark for free-form text generation. arXiv preprint arXiv:2104.08704 (2021)."},{"key":"e_1_3_2_86_2","first-page":"13415","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","author":"Liu Tianyu","year":"2021","unstructured":"Tianyu Liu, Xin Zheng, Baobao Chang, and Zhifang Sui. 2021. Towards faithfulness in open domain table-to-text generation from an entity-centric view. In Proceedings of the AAAI Conference on Artificial Intelligence. 13415\u201313423."},{"key":"e_1_3_2_87_2","doi-asserted-by":"crossref","first-page":"7052","DOI":"10.18653\/v1\/2021.emnlp-main.565","volume-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing (EMNLP\u201921)","author":"Longpre Shayne","year":"2021","unstructured":"Shayne Longpre, Kartik Perisetla, Anthony Chen, Nikhil Ramesh, Chris DuBois, and Sameer Singh. 2021. Entity-based knowledge conflicts in question answering. In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing (EMNLP\u201921). 7052\u20137063."},{"key":"e_1_3_2_88_2","first-page":"1468","volume-title":"Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics","author":"Madotto Andrea","year":"2018","unstructured":"Andrea Madotto, Chien-Sheng Wu, and Pascale Fung. 2018. Mem2Seq: Effectively incorporating knowledge bases into end-to-end task-oriented dialog systems. In Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics. 1468\u20131478."},{"key":"e_1_3_2_89_2","doi-asserted-by":"crossref","first-page":"103","DOI":"10.1145\/1871985.1872002","volume-title":"Proceedings of the 2nd International Workshop on Search and Mining User-Generated Contents","author":"Magdy Amr","year":"2010","unstructured":"Amr Magdy and Nayer Wanas. 2010. Web-based statistical fact checking of textual documents. In Proceedings of the 2nd International Workshop on Search and Mining User-Generated Contents. 103\u2013110."},{"key":"e_1_3_2_90_2","volume-title":"Proceedings of Machine Translation Summit XVII: Research Track","author":"Martindale Marianna J.","year":"2019","unstructured":"Marianna J. Martindale, Marine Carpuat, Kevin Duh, and Paul McNamee. 2019. Identifying fluently inadequate output in neural and statistical machine translation. In Proceedings of Machine Translation Summit XVII: Research Track. 233\u2013243."},{"key":"e_1_3_2_91_2","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Maynez Joshua","year":"2020","unstructured":"Joshua Maynez, Shashi Narayan, Bernd Bohnet, and Ryan McDonald. 2020. On faithfulness and factuality in abstractive summarization. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_2_92_2","first-page":"549","volume-title":"Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics","author":"Mesgar Mohsen","year":"2021","unstructured":"Mohsen Mesgar, Edwin Simpson, and Iryna Gurevych. 2021. Improving factual consistency between a response and persona facts. In Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics. 549\u2013562."},{"key":"e_1_3_2_93_2","first-page":"1322","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921)","author":"Mishra Anshuman","year":"2021","unstructured":"Anshuman Mishra, Dhruvesh Patel, Aparna Vijayakumar, Xiang Lorraine Li, Pavan Kapanipathi, and Kartik Talamadupula. 2021. Looking beyond sentence-level natural language inference for question answering and text summarization. In Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921). 1322\u20131336."},{"key":"e_1_3_2_94_2","first-page":"151","volume-title":"Proceedings of the 14th Conference of the Association for Machine Translation in the Americas","author":"M\u00fcller Mathias","year":"2020","unstructured":"Mathias M\u00fcller, Annette Rios, and Rico Sennrich. 2020. Domain robustness in neural machine translation. In Proceedings of the 14th Conference of the Association for Machine Translation in the Americas. 151\u2013164."},{"key":"e_1_3_2_95_2","article-title":"WebGPT: Browser-assisted question-answering with human feedback","author":"Nakano Reiichiro","year":"2021","unstructured":"Reiichiro Nakano, Jacob Hilton, Suchir Balaji, Jeff Wu, Long Ouyang, Christina Kim, Christopher Hesse, et\u00a0al. 2021. WebGPT: Browser-assisted question-answering with human feedback. arXiv preprint arXiv:2112.09332 (2021).","journal-title":"arXiv preprint arXiv:2112.09332"},{"key":"e_1_3_2_96_2","first-page":"2727","volume-title":"Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics","author":"Nan Feng","year":"2021","unstructured":"Feng Nan, Ramesh Nallapati, Zhiguo Wang, Cicero dos Santos, Henghui Zhu, Dejiao Zhang, Kathleen McKeown, and Bing Xiang. 2021. Entity-level factual consistency of abstractive text summarization. In Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics. 2727\u20132733."},{"key":"e_1_3_2_97_2","unstructured":"Tri Nguyen Mir Rosenberg Xia Song Jianfeng Gao Saurabh Tiwary Rangan Majumder and Li Deng. 2016. MS MARCO: A human generated machine reading comprehension dataset. arXiv:1611.09268 (2016)."},{"key":"e_1_3_2_98_2","volume-title":"Proceedings of Conference on Empirical Methods in Natural Language Processing","author":"Nie Feng","year":"2018","unstructured":"Feng Nie, Jinpeng Wang, Jin-Ge Yao, Rong Pan, and Chin-Yew Lin. 2018. Operation-guided neural networks for high fidelity data-to-text generation. In Proceedings of Conference on Empirical Methods in Natural Language Processing."},{"key":"e_1_3_2_99_2","first-page":"2673","volume-title":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","author":"Nie Feng","year":"2019","unstructured":"Feng Nie, Jin-Ge Yao, Jinpeng Wang, Rong Pan, and Chin-Yew Lin. 2019. A simple recipe towards reducing hallucination in neural surface realisation. In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics. 2673\u20132679."},{"key":"e_1_3_2_100_2","first-page":"4812","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921)","author":"Pagnoni Artidoro","year":"2021","unstructured":"Artidoro Pagnoni, Vidhisha Balachandran, and Yulia Tsvetkov. 2021. Understanding factuality in abstractive summarization with FRANK: A benchmark for factuality metrics. In Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921). 4812\u20134829."},{"key":"e_1_3_2_101_2","first-page":"1173","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing","author":"Parikh Ankur","year":"2020","unstructured":"Ankur Parikh, Xuezhi Wang, Sebastian Gehrmann, Manaal Faruqui, Bhuwan Dhingra, Diyi Yang, and Dipanjan Das. 2020. ToTTo: A controlled table-to-text generation dataset. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing. 1173\u20131186."},{"key":"e_1_3_2_102_2","doi-asserted-by":"crossref","first-page":"3205","DOI":"10.18653\/v1\/2021.findings-emnlp.275","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2021","author":"Parthasarathi Prasanna","year":"2021","unstructured":"Prasanna Parthasarathi, Koustuv Sinha, Joelle Pineau, and Adina Williams. 2021. Sometimes we want ungrammatical translations. In Findings of the Association for Computational Linguistics: EMNLP 2021. Association for Computational Linguistics, 3205\u20133227."},{"key":"e_1_3_2_103_2","first-page":"2173","volume-title":"Proceedings of the 25th ACM International Conference on Information and Knowledge Management","author":"Popat Kashyap","year":"2016","unstructured":"Kashyap Popat, Subhabrata Mukherjee, Jannik Str\u00f6tgen, and Gerhard Weikum. 2016. Credibility assessment of textual claims on the web. In Proceedings of the 25th ACM International Conference on Information and Knowledge Management. 2173\u20132178."},{"key":"e_1_3_2_104_2","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","author":"Puduppully Ratish","year":"2019","unstructured":"Ratish Puduppully, Li Dong, and Mirella Lapata. 2019. Data-to-text generation with content selection and planning. In Proceedings of the AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_105_2","doi-asserted-by":"crossref","DOI":"10.1162\/tacl_a_00381","article-title":"Data-to-text generation with macro planning","author":"Puduppully Ratish","year":"2021","unstructured":"Ratish Puduppully and Mirella Lapata. 2021. Data-to-text generation with macro planning. Transactions of the Association for Computational Linguistics 9 (2021), 510\u2013527.","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"e_1_3_2_106_2","first-page":"9","article-title":"Language models are unsupervised multitask learners","author":"Radford Alec","year":"2019","unstructured":"Alec Radford, Jeffrey Wu, Rewon Child, David Luan, Dario Amodei, and Ilya Sutskever. 2019. Language models are unsupervised multitask learners. OpenAI Blog 1, 8 (2019), 9.","journal-title":"OpenAI Blog"},{"key":"e_1_3_2_107_2","first-page":"1","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","author":"Raffel Colin","year":"2020","unstructured":"Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, and Peter J. Liu. 2020. Exploring the limits of transfer learning with a unified text-to-text transformer. Journal of Machine Learning Research 21, 140 (2020), 1\u201367.","journal-title":"Journal of Machine Learning Research"},{"key":"e_1_3_2_108_2","unstructured":"Marc\u2019Aurelio Ranzato Sumit Chopra Michael Auli and Wojciech Zaremba. 2016. Sequence level training with recurrent neural networks. In Proceedings of the 4th International Conference on Learning Representations (ICLR\u201916) ."},{"key":"e_1_3_2_109_2","first-page":"704","volume-title":"Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing","author":"Rashkin Hannah","year":"2021","unstructured":"Hannah Rashkin, David Reitter, Gaurav Singh Tomar, and Dipanjan Das. 2021. Increasing faithfulness in knowledge-grounded dialogue with controllable features. In Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing. 704\u2013718."},{"key":"e_1_3_2_110_2","doi-asserted-by":"crossref","unstructured":"Vikas Raunak Arul Menezes and Marcin Junczys-Dowmunt. 2021. The curious case of hallucinations in neural machine translation. In Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologys (NAACL-HLT\u201921) . 1172\u20131183.","DOI":"10.18653\/v1\/2021.naacl-main.92"},{"key":"e_1_3_2_111_2","doi-asserted-by":"crossref","first-page":"318","DOI":"10.1007\/s10618-021-00801-4","article-title":"Controlling hallucinations at word level in data-to-text generation","author":"Rebuffel Cl\u00e9ment","year":"2022","unstructured":"Cl\u00e9ment Rebuffel, Marco Roberti, Laure Soulier, Geoffrey Scoutheeten, Rossella Cancelliere, and Patrick Gallinari. 2022. Controlling hallucinations at word level in data-to-text generation. Data Mining and Knowledge Discovery 36 (2022), 318\u2013354.","journal-title":"Data Mining and Knowledge Discovery"},{"key":"e_1_3_2_112_2","volume-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","author":"Rebuffel Cl\u00e9ment","year":"2021","unstructured":"Cl\u00e9ment Rebuffel, Thomas Scialom, Laure Soulier, Benjamin Piwowarski, Sylvain Lamprier, Jacopo Staiano, Geoffrey Scoutheeten, and Patrick Gallinari. 2021. Data-QuestEval: A reference-less metric for data-to-text semantic evaluation. In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing."},{"key":"e_1_3_2_113_2","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing","author":"Roberts Adam","year":"2020","unstructured":"Adam Roberts, Colin Raffel, and Noam Shazeer. 2020. How much knowledge can you pack into the parameters of a language model? In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing."},{"key":"e_1_3_2_114_2","volume-title":"Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing","author":"Rohrbach Anna","year":"2018","unstructured":"Anna Rohrbach, Lisa Anne Hendricks, Kaylee Burns, Trevor Darrell, and Kate Saenko. 2018. Object hallucination in image captioning. In Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing."},{"key":"e_1_3_2_115_2","article-title":"Open-domain conversational agents: Current progress, open problems, and future directions","author":"Roller Stephen","year":"2020","unstructured":"Stephen Roller, Y.-Lan Boureau, Jason Weston, Antoine Bordes, Emily Dinan, Angela Fan, David Gunning, et\u00a0al. 2020. Open-domain conversational agents: Current progress, open problems, and future directions. arXiv preprint arXiv:2006.12442 (2020).","journal-title":"arXiv preprint arXiv:2006.12442"},{"key":"e_1_3_2_116_2","first-page":"300","volume-title":"Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics","author":"Roller Stephen","year":"2021","unstructured":"Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, et\u00a0al. 2021. Recipes for building an open-domain chatbot. In Proceedings of the 16th Conference of the European Chapter of the Association for Computational Linguistics. 300\u2013325."},{"key":"e_1_3_2_117_2","article-title":"Rome was built in 1776: A case study on factual correctness in knowledge-grounded response generation","author":"Santhanam Sashank","year":"2021","unstructured":"Sashank Santhanam, Behnam Hedayatnia, Spandana Gella, Aishwarya Padmakumar, Seokhwan Kim, Yang Liu, and Dilek Hakkani-Tur. 2021. Rome was built in 1776: A case study on factual correctness in knowledge-grounded response generation. arXiv preprint arXiv:2110.05456 (2021).","journal-title":"arXiv preprint arXiv:2110.05456"},{"key":"e_1_3_2_118_2","volume-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","author":"Scialom Thomas","year":"2021","unstructured":"Thomas Scialom, Paul-Alexis Dray, Patrick Gallinari, Sylvain Lamprier, Benjamin Piwowarski, Jacopo Staiano, and Alex Wang. 2021. QuestEval: Summarization asks for fact-based evaluation. In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing."},{"key":"e_1_3_2_119_2","doi-asserted-by":"crossref","first-page":"7881","DOI":"10.18653\/v1\/2020.acl-main.704","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Sellam Thibault","year":"2020","unstructured":"Thibault Sellam, Dipanjan Das, and Ankur Parikh. 2020. BLEURT: Learning robust metrics for text generation. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics. 7881\u20137892."},{"key":"e_1_3_2_120_2","article-title":"Hallucination of speech recognition errors with sequence to sequence learning","author":"Serai Prashant","year":"2022","unstructured":"Prashant Serai, Vishal Sunder, and Eric Fosler-Lussier. 2022. Hallucination of speech recognition errors with sequence to sequence learning. IEEE\/ACM Transactions on Audio, Speech, and Language Processing 30 (2022), 890\u2013900.","journal-title":"IEEE\/ACM Transactions on Audio, Speech, and Language Processing"},{"key":"e_1_3_2_121_2","first-page":"1598","volume-title":"Proceedings of the 30th ACM International Conference on Information and Knowledge Management","author":"Shen Lei","year":"2021","unstructured":"Lei Shen, Haolan Zhan, Xin Shen, Hongshen Chen, Xiaofang Zhao, and Xiaodan Zhu. 2021. Identifying untrustworthy samples: Data filtering for open-domain dialogues with Bayesian optimization. In Proceedings of the 30th ACM International Conference on Information and Knowledge Management. 1598\u20131608."},{"key":"e_1_3_2_122_2","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2021","author":"Shuster Kurt","year":"2021","unstructured":"Kurt Shuster, Spencer Poff, Moya Chen, Douwe Kiela, and Jason Weston. 2021. Retrieval augmentation reduces hallucination in conversation. In Findings of the Association for Computational Linguistics: EMNLP 2021. Association for Computational Linguistics, 3784\u20133803."},{"key":"e_1_3_2_123_2","doi-asserted-by":"crossref","first-page":"8878","DOI":"10.1609\/aaai.v34i05.6417","article-title":"Generating persona consistent dialogues by exploiting natural language inference","author":"Song Haoyu","year":"2020","unstructured":"Haoyu Song, Wei-Nan Zhang, Jingwen Hu, and Ting Liu. 2020. Generating persona consistent dialogues by exploiting natural language inference. In Proceedings of the AAAI Conference on Artificial Intelligence.8878\u20138885.","journal-title":"Proceedings of the AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_124_2","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","author":"Song Kaiqiang","year":"2020","unstructured":"Kaiqiang Song, Logan Lebanoff, Qipeng Guo, Xipeng Qiu, Xiangyang Xue, Chen Li, Dong Yu, and Fei Liu. 2020. Joint parsing and generation for abstractive summarization. In Proceedings of the AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_125_2","doi-asserted-by":"crossref","first-page":"7409","DOI":"10.18653\/v1\/2020.acl-main.661","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Sperber Matthias","year":"2020","unstructured":"Matthias Sperber and Matthias Paulik. 2020. Speech translation and the end-to-end promise: Taking stock of where we are. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics. 7409\u20137421."},{"key":"e_1_3_2_126_2","doi-asserted-by":"crossref","unstructured":"Dan Su Xiaoguang Li Jindi Zhang Lifeng Shang Xin Jiang Qun Liu and Pascale Fung. 2022. Read before generate! Faithful long form question answering with machine reading. arXiv:2203.00343 (2022).","DOI":"10.18653\/v1\/2022.findings-acl.61"},{"key":"e_1_3_2_127_2","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Su Hui","year":"2020","unstructured":"Hui Su, Xiaoyu Shen, Sanqiang Zhao, Zhou Xiao, Pengwei Hu, Cheng Niu, and Jie Zhou. 2020. Diversifying dialogue generation with non-conversational text. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_2_128_2","article-title":"Plan-then-generate: Controlled data-to-text generation via planning","author":"Su Yixuan","year":"2021","unstructured":"Yixuan Su, David Vandyke, Sihui Wang, Yimai Fang, and Nigel Collier. 2021. Plan-then-generate: Controlled data-to-text generation via planning. In Findings of the Association for Computational Linguistics: EMNLP 2021. Association for Computational Linguistics, 895\u2013909.","journal-title":"Findings of the Association for Computational Linguistics: EMNLP 2021."},{"key":"e_1_3_2_129_2","volume-title":"Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing","author":"Suadaa Lya Hulliyyatus","year":"2021","unstructured":"Lya Hulliyyatus Suadaa, Hidetaka Kamigaito, Kotaro Funakoshi, Manabu Okumura, and Hiroya Takamura. 2021. Towards table-to-text generation with numerical reasoning. In Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing."},{"key":"e_1_3_2_130_2","volume-title":"Proceedings of the 7th International Conference on Language Resources and Evaluation (LREC\u201910)","author":"Sun Yanli","year":"2010","unstructured":"Yanli Sun. 2010. Mining the correlation between human and automatic evaluation at sentence level. In Proceedings of the 7th International Conference on Language Resources and Evaluation (LREC\u201910)."},{"key":"e_1_3_2_131_2","article-title":"CONFIT: Toward faithful dialogue summarization with linguistically-informed contrastive fine-tuning","author":"Tang Xiangru","year":"2021","unstructured":"Xiangru Tang, Arjun Nair, Borui Wang, Bingyao Wang, Jai Desai, Aaron Wade, Haoran Li, Asli Celikyilmaz, Yashar Mehdad, and Dragomir Radev. 2021. CONFIT: Toward faithful dialogue summarization with linguistically-informed contrastive fine-tuning. arXiv preprint arXiv:2112.08713 (2021).","journal-title":"arXiv preprint arXiv:2112.08713"},{"key":"e_1_3_2_132_2","first-page":"644","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921)","author":"Thawani Avijit","year":"2021","unstructured":"Avijit Thawani, Jay Pujara, Filip Ilievski, and Pedro Szekely. 2021. Representing numbers in NLP: A survey and a vision. In Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921). 644\u2013656."},{"key":"e_1_3_2_133_2","first-page":"809","volume-title":"Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","author":"Thorne James","year":"2018","unstructured":"James Thorne, Andreas Vlachos, Christos Christodoulopoulos, and Arpit Mittal. 2018. FEVER: A large-scale dataset for Fact Extraction and VERification. In Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies. 809\u2013819."},{"key":"e_1_3_2_134_2","first-page":"2944","volume-title":"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing","author":"Thorne James","year":"2019","unstructured":"James Thorne, Andreas Vlachos, Christos Christodoulopoulos, and Arpit Mittal. 2019. Evaluating adversarial attacks against multiple fact verification systems. In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing. 2944\u20132953."},{"key":"e_1_3_2_135_2","unstructured":"Ran Tian Shashi Narayan Thibault Sellam and Ankur P. Parikh. 2020. Sticking to the facts: Confident decoding for faithful data-to-text generation. arXiv:1910.08684 (2020)."},{"key":"e_1_3_2_136_2","volume-title":"Proceedings of the 31st AAAI Conference on Artificial Intelligence","author":"Tu Zhaopeng","year":"2017","unstructured":"Zhaopeng Tu, Yang Liu, Lifeng Shang, Xiaohua Liu, and Hang Li. 2017. Neural machine translation with reconstruction. In Proceedings of the 31st AAAI Conference on Artificial Intelligence."},{"key":"e_1_3_2_137_2","first-page":"76","volume-title":"Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics","author":"Tu Zhaopeng","year":"2016","unstructured":"Zhaopeng Tu, Zhengdong Lu, Yang Liu, Xiaohua Liu, and Hang Li. 2016. Modeling coverage for neural machine translation. In Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics. 76\u201385."},{"key":"e_1_3_2_138_2","article-title":"Survey on reinforcement learning for language processing","author":"Uc-Cetina Victor","year":"2021","unstructured":"Victor Uc-Cetina, Nicolas Navarro-Guerrero, Anabel Martin-Gonzalez, Cornelius Weber, and Stefan Wermter. 2021. Survey on reinforcement learning for language processing. arXiv preprint arXiv:2104.05565 (2021).","journal-title":"arXiv preprint arXiv:2104.05565"},{"key":"e_1_3_2_139_2","first-page":"5998","volume-title":"Advances in Neural Information Processing Systems","author":"Vaswani Ashish","year":"2017","unstructured":"Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, \u0141ukasz Kaiser, and Illia Polosukhin. 2017. Attention is all you need. In Advances in Neural Information Processing Systems. 5998\u20136008."},{"key":"e_1_3_2_140_2","article-title":"A neural conversational model","author":"Vinyals Oriol","year":"2015","unstructured":"Oriol Vinyals and Quoc Le. 2015. A neural conversational model. In Proceedings of the ICML Deep Learning Workshop.","journal-title":"Proceedings of the ICML Deep Learning Workshop."},{"key":"e_1_3_2_141_2","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Wang Alex","year":"2020","unstructured":"Alex Wang, Kyunghyun Cho, and Mike Lewis. 2020. Asking and answering questions to evaluate the factual consistency of summaries. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_2_142_2","unstructured":"Ben Wang and Aran Komatsuzaki. 2021. GPT-J-6B: A 6 Billion Parameter Autoregressive Language Model. Retrieved November 23 2022 from https:\/\/github.com\/kingoflolz\/mesh-transformer-jax."},{"key":"e_1_3_2_143_2","first-page":"3544","volume-title":"Proceedings of the 2020 Annual Conference of the Association for Computational Linguistics","author":"Wang Chaojun","year":"2020","unstructured":"Chaojun Wang and Rico Sennrich. 2020. On exposure bias, hallucination and domain shift in neural machine translation. In Proceedings of the 2020 Annual Conference of the Association for Computational Linguistics. 3544\u20133552."},{"key":"e_1_3_2_144_2","doi-asserted-by":"crossref","first-page":"311","DOI":"10.18653\/v1\/W19-8639","volume-title":"Proceedings of the 12th International Conference on Natural Language Generation","author":"Wang Hongmin","year":"2019","unstructured":"Hongmin Wang. 2019. Revisiting challenges in data-to-text generation with fact grounding. In Proceedings of the 12th International Conference on Natural Language Generation. 311\u2013322."},{"key":"e_1_3_2_145_2","volume-title":"Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021","author":"Wang Peng","year":"2021","unstructured":"Peng Wang, Junyang Lin, An Yang, Chang Zhou, Yichang Zhang, Jingren Zhou, and Hongxia Yang. 2021. Sketch and refine: Towards faithful and informative table-to-text generation. In Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021. Association for Computational Linguistics, 4831\u20134843."},{"key":"e_1_3_2_146_2","volume-title":"Proceedings of the Annual Meeting of the Association for Computational Linguistics","author":"Wang Zhenyi","year":"2020","unstructured":"Zhenyi Wang, Xiaoyang Wang, Bang An, Dong Yu, and Changyou Chen. 2020. Towards faithful neural table-to-text generation with content-matching constraints. In Proceedings of the Annual Meeting of the Association for Computational Linguistics."},{"key":"e_1_3_2_147_2","volume-title":"Proceedings of the International Conference on Learning Representations","author":"Welleck Sean","year":"2019","unstructured":"Sean Welleck, Ilia Kulikov, Stephen Roller, Emily Dinan, Kyunghyun Cho, and Jason Weston. 2019. Neural text generation with unlikelihood training. In Proceedings of the International Conference on Learning Representations."},{"key":"e_1_3_2_148_2","doi-asserted-by":"crossref","first-page":"3731","DOI":"10.18653\/v1\/P19-1363","volume-title":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","author":"Welleck Sean","year":"2019","unstructured":"Sean Welleck, Jason Weston, Arthur Szlam, and Kyunghyun Cho. 2019. Dialogue natural language inference. In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics. 3731\u20133741."},{"key":"e_1_3_2_149_2","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing","author":"Weng Rongxiang","year":"2020","unstructured":"Rongxiang Weng, Heng Yu, Xiangpeng Wei, and Weihua Luo. 2020. Towards enhancing faithfulness for neural machine translation. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing."},{"key":"e_1_3_2_150_2","first-page":"2253","volume-title":"Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing","author":"Wiseman Sam","year":"2017","unstructured":"Sam Wiseman, Stuart Shieber, and Alexander Rush. 2017. Challenges in data-to-document generation. In Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing. 2253\u20132263."},{"key":"e_1_3_2_151_2","article-title":"Global-to-local memory pointer networks for task-oriented dialogue","author":"Wu Chien-Sheng","year":"2019","unstructured":"Chien-Sheng Wu, Richard Socher, and Caiming Xiong. 2019. Global-to-local memory pointer networks for task-oriented dialogue. arXiv preprint arXiv:1901.04713 (2019).","journal-title":"arXiv preprint arXiv:1901.04713"},{"key":"e_1_3_2_152_2","first-page":"14085","volume-title":"Proceedings of the AAAI Conference on Artificial Intelligence","author":"Wu Zeqiu","year":"2021","unstructured":"Zeqiu Wu, Michel Galley, Chris Brockett, Yizhe Zhang, Xiang Gao, Chris Quirk, Rik Koncel-Kedziorski, et\u00a0al. 2021. A controllable model of grounded response generation. In Proceedings of the AAAI Conference on Artificial Intelligence. 14085\u201314093."},{"key":"e_1_3_2_153_2","volume-title":"Proceedings of the Conference of the European Chapter of the Association for Computational Linguistics","author":"Xiao Yijun","year":"2021","unstructured":"Yijun Xiao and William Yang Wang. 2021. On hallucination and predictive uncertainty in conditional language generation. In Proceedings of the Conference of the European Chapter of the Association for Computational Linguistics."},{"key":"e_1_3_2_154_2","article-title":"Beyond goldfish memory: Long-term open-domain conversation","author":"Xu Jing","year":"2021","unstructured":"Jing Xu, Arthur Szlam, and Jason Weston. 2021. Beyond goldfish memory: Long-term open-domain conversation. arXiv preprint arXiv:2107.07567 (2021).","journal-title":"arXiv preprint arXiv:2107.07567"},{"key":"e_1_3_2_155_2","first-page":"2047","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201919)","author":"Xu Weijia","year":"2019","unstructured":"Weijia Xu, Xing Niu, and Marine Carpuat. 2019. Differentiable sampling with flexible reference word order for neural machine translation. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201919). 2047\u20132053."},{"key":"e_1_3_2_156_2","volume-title":"Proceedings of the Joint Conference of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing","author":"Xu Xinnuo","year":"2021","unstructured":"Xinnuo Xu, Ondrej Du\u0161ek, Verena Rieser, and Ioannis Konstas. 2021. AggGen: Ordering and aggregating while generating. In Proceedings of the Joint Conference of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing."},{"key":"e_1_3_2_157_2","volume-title":"Proceedings of the 2nd DialDoc Workshop on Document-Grounded Dialogue and Conversational Question Answering","author":"Xu Yan","year":"2022","unstructured":"Yan Xu, Etsuko Ishii, Samuel Cahyawijaya, Zihan Liu, Genta Indra Winata, Andrea Madotto, Dan Su, and Pascale Fung. 2022. Retrieval-free knowledge-grounded dialogue response generation with adapters. In Proceedings of the 2nd DialDoc Workshop on Document-Grounded Dialogue and Conversational Question Answering."},{"key":"e_1_3_2_158_2","first-page":"2972","volume-title":"Proceedings of the 25th International Joint Conference on Artificial Intelligence","author":"Yin Jun","year":"2016","unstructured":"Jun Yin, Xin Jiang, Zhengdong Lu, Lifeng Shang, Hang Li, and Xiaoming Li. 2016. Neural generative question answering. In Proceedings of the 25th International Joint Conference on Artificial Intelligence. 2972\u20132978."},{"key":"e_1_3_2_159_2","first-page":"5892","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921)","author":"Yu Tiezheng","year":"2021","unstructured":"Tiezheng Yu, Zihan Liu, and Pascale Fung. 2021. AdaptSum: Towards low-resource domain adaptation for abstractive summarization. In Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921). 5892\u20135904."},{"key":"e_1_3_2_160_2","doi-asserted-by":"crossref","first-page":"2502","DOI":"10.1109\/TASLP.2021.3074012","article-title":"D-score: Holistic dialogue evaluation without reference","author":"Zhang Chen","year":"2021","unstructured":"Chen Zhang, Grandee Lee, Luis Fernando D\u2019Haro, and Haizhou Li. 2021. D-score: Holistic dialogue evaluation without reference. IEEE\/ACM Transactions on Audio, Speech, and Language Processing 29 (2021), 2502\u20132516.","journal-title":"IEEE\/ACM Transactions on Audio, Speech, and Language Processing"},{"key":"e_1_3_2_161_2","volume-title":"Proceedings of the 2019 IEEE\/CVF Conference on Computer Vision and Pattern Recognition","author":"Zhang Hongguang","year":"2019","unstructured":"Hongguang Zhang, Jing Zhang, and Piotr Koniusz. 2019. Few-shot learning via saliency-guided hallucination of samples. In Proceedings of the 2019 IEEE\/CVF Conference on Computer Vision and Pattern Recognition. IEEE, Los Alamitos, CA."},{"key":"e_1_3_2_162_2","first-page":"2204","volume-title":"Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics","author":"Zhang Saizheng","year":"2018","unstructured":"Saizheng Zhang, Emily Dinan, Jack Urbanek, Arthur Szlam, Douwe Kiela, and Jason Weston. 2018. Personalizing dialogue agents: I have a dog, do you have pets too? In Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics. 2204\u20132213."},{"key":"e_1_3_2_163_2","volume-title":"Proceedings of the International Conference on Learning Representations","author":"Zhang Tianyi","year":"2019","unstructured":"Tianyi Zhang, Varsha Kishore, Felix Wu, Kilian Q. Weinberger, and Yoav Artzi. 2019. BERTScore: Evaluating text generation with BERT. In Proceedings of the International Conference on Learning Representations."},{"key":"e_1_3_2_164_2","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2020","author":"Zhang Xikun","year":"2020","unstructured":"Xikun Zhang, Deepak Ramachandran, Ian Tenney, Yanai Elazar, and Dan Roth. 2020. Do language embeddings capture scales? In Findings of the Association for Computational Linguistics: EMNLP 2020. Association for Computational Linguistics, 4889\u20134896."},{"key":"e_1_3_2_165_2","doi-asserted-by":"crossref","first-page":"5108","DOI":"10.18653\/v1\/2020.acl-main.458","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Zhang Yuhao","year":"2020","unstructured":"Yuhao Zhang, Derek Merck, Emily Tsai, Christopher D. Manning, and Curtis Langlotz. 2020. Optimizing the factual correctness of a summary: A study of summarizing radiology reports. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics. 5108\u20135120."},{"key":"e_1_3_2_166_2","doi-asserted-by":"crossref","first-page":"2237","DOI":"10.18653\/v1\/2020.findings-emnlp.203","volume-title":"Findings of the Association for Computational Linguistics: EMNLP 2020","author":"Zhao Zheng","year":"2020","unstructured":"Zheng Zhao, Shay B. Cohen, and Bonnie Webber. 2020. Reducing quantity hallucinations in abstractive summarization. In Findings of the Association for Computational Linguistics: EMNLP 2020. Association for Computational Linguistics, 2237\u20132249."},{"key":"e_1_3_2_167_2","first-page":"5905","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921)","author":"Zhong Ming","year":"2021","unstructured":"Ming Zhong, Da Yin, Tao Yu, Ahmad Zaidi, Mutethia Mutuma, Rahul Jha, Ahmed Hassan, et\u00a0al. 2021. QMSum: A new benchmark for query-based multi-domain meeting summarization. In Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921). 5905\u20135921."},{"key":"e_1_3_2_168_2","first-page":"1588","volume-title":"Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201919)","author":"Zhou Chunting","year":"2019","unstructured":"Chunting Zhou, Xuezhe Ma, and Graham Neubig Di Wang. 2019. Density matching for bilingual word embedding. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201919). 1588\u20131598."},{"key":"e_1_3_2_169_2","first-page":"Association for","volume-title":"Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021","author":"Zhou Chunting","year":"2021","unstructured":"Chunting Zhou, Graham Neubig, Jiatao Gu, Mona Diab, Francisco Guzm\u00e1n, Luke Zettlemoyer, and Marjan Ghazvininejad. 2021. Detecting hallucinated content in conditional neural sequence generation. In Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021. Association for Computational Linguistics, 1393\u20131404."},{"key":"e_1_3_2_170_2","volume-title":"Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing","author":"Zhou Kangyan","year":"2018","unstructured":"Kangyan Zhou, Shrimai Prabhumoye, and Alan W. Black. 2018. A dataset for document grounded conversations. In Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing."},{"key":"e_1_3_2_171_2","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921)","author":"Zhu Chenguang","year":"2021","unstructured":"Chenguang Zhu, William Hinthorn, Ruochen Xu, Qingkai Zeng, Michael Zeng, Xuedong Huang, and Meng Jiang. 2021. Enhancing factual consistency of abstractive summarization. In Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL-HLT\u201921). 718\u2013733."}],"container-title":["ACM Computing Surveys"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3571730","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3571730","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T17:49:33Z","timestamp":1750182573000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3571730"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,3,3]]},"references-count":170,"journal-issue":{"issue":"12","published-print":{"date-parts":[[2023,12,31]]}},"alternative-id":["10.1145\/3571730"],"URL":"https:\/\/doi.org\/10.1145\/3571730","relation":{},"ISSN":["0360-0300","1557-7341"],"issn-type":[{"value":"0360-0300","type":"print"},{"value":"1557-7341","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,3,3]]},"assertion":[{"value":"2022-03-11","order":0,"name":"received","label":"Received","group":{"name":"publication_history","label":"Publication History"}},{"value":"2022-11-08","order":1,"name":"accepted","label":"Accepted","group":{"name":"publication_history","label":"Publication History"}},{"value":"2023-03-03","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F3593013_3594087 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,27]],"date-time":"2026-10-27T08:13:54Z","timestamp":1793088834703,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":112,"publisher":"ACM","license":[{"start":{"date-parts":[[2023,6,12]],"date-time":"2023-06-12T00:00:00Z","timestamp":1686528000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,6,12]]},"DOI":"10.1145\/3593013.3594087","type":"proceedings-article","created":{"date-parts":[[2023,6,12]],"date-time":"2023-06-12T10:40:46Z","timestamp":1686566446000},"page":"1369-1385","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":208,"title":["Towards a Science of Human-AI Decision Making: An Overview of Design Space in Empirical Human-Subject Studies"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3165-6136","authenticated-orcid":false,"given":"Vivian","family":"Lai","sequence":"first","affiliation":[{"name":"University of Colorado Boulder, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0000-6101-2150","authenticated-orcid":false,"given":"Chacha","family":"Chen","sequence":"additional","affiliation":[{"name":"University of Chicago, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-6600-267X","authenticated-orcid":false,"given":"Alison","family":"Smith-Renner","sequence":"additional","affiliation":[{"name":"Dataminr Inc., USA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-4543-7196","authenticated-orcid":false,"given":"Q. Vera","family":"Liao","sequence":"additional","affiliation":[{"name":"Microsoft Research, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3981-2116","authenticated-orcid":false,"given":"Chenhao","family":"Tan","sequence":"additional","affiliation":[{"name":"University of Chicago, USA"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2023,6,12]]},"reference":[{"key":"e_1_3_2_1_1_1","volume-title":"Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems. 1\u201314","author":"Abdul Ashraf","unstructured":"Ashraf Abdul, Christian von der Weth, Mohan Kankanhalli, and Brian Y Lim. 2020. COGAM: Measuring and Moderating Cognitive Load in Machine Learning Model Explanations. In Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems. 1\u201314."},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377325.3377519"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1609\/aimag.v35i4.2513"},{"key":"e_1_3_2_1_4_1","volume-title":"Machine bias. ProPublica. See https:\/\/www. propublica. org\/article\/machine-bias-risk-assessments-in-criminal-sentencing","author":"Angwin Julia","year":"2016","unstructured":"Julia Angwin, Jeff Larson, Surya Mattu, and Lauren Kirchner. 2016. Machine bias. ProPublica. See https:\/\/www. propublica. org\/article\/machine-bias-risk-assessments-in-criminal-sentencing (2016)."},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445736"},{"key":"e_1_3_2_1_6_1","first-page":"799","article-title":"System and method for predicting consumer credit risk using income risk based credit score","volume":"8","author":"Annappindi Suresh Kumar","year":"2014","unstructured":"Suresh Kumar Annappindi. 2014. System and method for predicting consumer credit risk using income risk based credit score. US Patent 8,799,150.","journal-title":"US Patent"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1147\/JRD.2019.2942288"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/2838739.2838753"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v7i1.5285"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v33i01.33012429"},{"key":"e_1_3_2_1_11_1","volume-title":"Marco Tulio Ribeiro, and Daniel S. Weld","author":"Bansal Gagan","year":"2020","unstructured":"Gagan Bansal, Tongshuang Wu, Joyce Zhou, Raymond Fok, Besmira Nushi, Ece Kamar, Marco Tulio Ribeiro, and Daniel S. Weld. 2020. Does the Whole Exceed its Parts? The Effect of AI Explanations on Complementary Team Performance. arXiv preprint arXiv:2006.14779 (2020)."},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173951"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.24963\/ijcai.2017\/202"},{"key":"e_1_3_2_1_14_1","volume-title":"The values encoded in machine learning research. arXiv preprint arXiv:2106.15590","author":"Birhane Abeba","year":"2021","unstructured":"Abeba Birhane, Pratyusha Kalluri, Dallas Card, William Agnew, Ravit Dotan, and Michelle Bao. 2021. The values encoded in machine learning research. arXiv preprint arXiv:2106.15590 (2021)."},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377325.3377498"},{"key":"e_1_3_2_1_16_1","volume-title":"Maja Barbara Malaya, and Krzysztof Z Gajos","author":"Bu\u00e7inca Zana","year":"2021","unstructured":"Zana Bu\u00e7inca, Maja Barbara Malaya, and Krzysztof Z Gajos. 2021. To Trust or to Think: Cognitive Forcing Functions Can Reduce Overreliance on AI in AI-assisted Decision-making. arXiv preprint arXiv:2102.09692 (2021)."},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICHI.2015.26"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3302289"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300234"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1609\/icwsm.v14i1.7282"},{"key":"e_1_3_2_1_21_1","volume-title":"Do explanations make VQA models more predictable to a human?arXiv preprint arXiv:1810.12366","author":"Chandrasekaran Arjun","year":"2018","unstructured":"Arjun Chandrasekaran, Viraj Prabhu, Deshraj Yadav, Prithvijit Chattopadhyay, and Devi Parikh. 2018. Do explanations make VQA models more predictable to a human?arXiv preprint arXiv:1810.12366 (2018)."},{"key":"e_1_3_2_1_22_1","volume-title":"Machine explanations and human understanding. arXiv preprint arXiv:2202.04092","author":"Chen Chacha","year":"2022","unstructured":"Chacha Chen, Shi Feng, Amit Sharma, and Chenhao Tan. 2022. Machine explanations and human understanding. arXiv preprint arXiv:2202.04092 (2022)."},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300789"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397481.3450644"},{"key":"e_1_3_2_1_25_1","volume-title":"Construct validity in psychological tests.Psychological bulletin 52, 4","author":"Cronbach Lee J","year":"1955","unstructured":"Lee J Cronbach and Paul E Meehl. 1955. Construct validity in psychological tests.Psychological bulletin 52, 4 (1955), 281."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377325.3377512"},{"key":"e_1_3_2_1_27_1","volume-title":"Amazon scraps secret AI recruiting tool that showed bias against women","author":"Dastin Jeffrey","year":"2016","unstructured":"Jeffrey Dastin. 2016. Amazon scraps secret AI recruiting tool that showed bias against women. Reuters. See https:\/\/www.reuters.com\/article\/us-amazon-com-jobs-automation-insight\/amazon-scraps-secret-ai-recruiting-tool-that-showed-bias-against-women-idUSKCN1MK08G (2016)."},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376638"},{"key":"e_1_3_2_1_29_1","first-page":"114","article-title":"Algorithm aversion: People erroneously avoid algorithms after seeing them err.Journal of Experimental Psychology","volume":"144","author":"Dietvorst Berkeley J","year":"2015","unstructured":"Berkeley J Dietvorst, Joseph P Simmons, and Cade Massey. 2015. Algorithm aversion: People erroneously avoid algorithms after seeing them err.Journal of Experimental Psychology: General 144, 1 (2015), 114.","journal-title":"General"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.2016.2643"},{"key":"e_1_3_2_1_31_1","volume-title":"Artificial intelligence in medicine and cardiac imaging: harnessing big data and advanced computing to provide personalized medical diagnosis and treatment. Current cardiology reports 16, 1","author":"Dilsizian Steven E","year":"2014","unstructured":"Steven E Dilsizian and Eliot L Siegel. 2014. Artificial intelligence in medicine and cardiac imaging: harnessing big data and advanced computing to provide personalized medical diagnosis and treatment. Current cardiology reports 16, 1 (2014), 441."},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3302310"},{"key":"e_1_3_2_1_33_1","volume-title":"The accuracy, fairness, and limits of predicting recidivism. Science advances 4, 1","author":"Dressel Julia","year":"2018","unstructured":"Julia Dressel and Hany Farid. 2018. The accuracy, fairness, and limits of predicting recidivism. Science advances 4, 1 (2018), eaao5580."},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/1943403.1943424"},{"key":"e_1_3_2_1_35_1","volume-title":"What can AI do for me: Evaluating Machine Learning Interpretations in Cooperative Play. arXiv preprint arXiv:1810.09648","author":"Feng Shi","year":"2018","unstructured":"Shi Feng and Jordan Boyd-Graber. 2018. What can AI do for me: Evaluating Machine Learning Interpretations in Cooperative Play. arXiv preprint arXiv:1810.09648 (2018)."},{"key":"e_1_3_2_1_36_1","volume-title":"Carlos Scheidegger, and Dylan Slack.","author":"Friedler Sorelle A","year":"2019","unstructured":"Sorelle A Friedler, Chitradeep Dutta Roy, Carlos Scheidegger, and Dylan Slack. 2019. Assessing the Local Interpretability of Machine Learning Models. arXiv preprint arXiv:1902.03501 (2019)."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/332040.332455"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376316"},{"key":"e_1_3_2_1_39_1","volume-title":"Explainable Active Learning (XAL): An Empirical Study of How Local Explanations Impact Annotator Experience. arXiv preprint arXiv:2001.09219","author":"Ghai Bhavya","year":"2020","unstructured":"Bhavya Ghai, Q Vera Liao, Yunfeng Zhang, Rachel Bellamy, and Klaus Mueller. 2020. Explainable Active Learning (XAL): An Empirical Study of How Local Explanations Impact Annotator Experience. arXiv preprint arXiv:2001.09219 (2020)."},{"key":"e_1_3_2_1_40_1","volume-title":"Jiri Navratil, Prasanna Sattigeri, Kush R Varshney, and Yunfeng Zhang.","author":"Ghosh Soumya","year":"2021","unstructured":"Soumya Ghosh, Q Vera Liao, Karthikeyan Natesan Ramamurthy, Jiri Navratil, Prasanna Sattigeri, Kush R Varshney, and Yunfeng Zhang. 2021. Uncertainty Quantification 360: A Holistic Toolkit for Quantifying and Communicating the Uncertainty of AI. arXiv preprint arXiv:2106.01410 (2021)."},{"key":"e_1_3_2_1_41_1","volume-title":"Human Evaluation of Spoken vs. Visual Explanations for Open-Domain QA. arXiv preprint arXiv:2012.15075","author":"Gonzalez Ana Valeria","year":"2020","unstructured":"Ana Valeria Gonzalez, Gagan Bansal, Angela Fan, Robin Jia, Yashar Mehdad, and Srinivasan Iyer. 2020. Human Evaluation of Spoken vs. Visual Explanations for Open-Domain QA. arXiv preprint arXiv:2012.15075 (2020)."},{"key":"e_1_3_2_1_42_1","volume-title":"The Flaws of Policies Requiring Human Oversight of Government Algorithms. Available at SSRN","author":"Green Ben","year":"2021","unstructured":"Ben Green. 2021. The Flaws of Policies Requiring Human Oversight of Government Algorithms. Available at SSRN (2021)."},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287563"},{"key":"e_1_3_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359152"},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.2139\/ssrn.3465622"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300803"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372831"},{"key":"e_1_3_2_1_48_1","volume-title":"Evaluating Explainable AI: Which Algorithmic Explanations Help Users Predict Model Behavior?arXiv preprint arXiv:2005.01831","author":"Hase Peter","year":"2020","unstructured":"Peter Hase and Mohit Bansal. 2020. Evaluating Explainable AI: Which Algorithmic Explanations Help Users Predict Model Behavior?arXiv preprint arXiv:2005.01831 (2020)."},{"key":"e_1_3_2_1_49_1","volume-title":"Visual analytics in deep learning: An interrogative survey for the next frontiers","author":"Hohman Fred","year":"2018","unstructured":"Fred Hohman, Minsuk Kahng, Robert Pienta, and Duen Horng Chau. 2018. Visual analytics in deep learning: An interrogative survey for the next frontiers. IEEE transactions on visualization and computer graphics 25, 8 (2018), 2674\u20132693."},{"key":"e_1_3_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445385"},{"key":"e_1_3_2_1_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442188.3445923"},{"key":"e_1_3_2_1_52_1","volume-title":"The limits of human predictions of recidivism. Science Advances 6, 7","author":"Jung Jongbin","year":"2020","unstructured":"Jongbin Jung, Sharad Goel, Jennifer Skeem, 2020. The limits of human predictions of recidivism. Science Advances 6, 7 (2020), eaaz0652."},{"key":"e_1_3_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/2858036.2858558"},{"key":"e_1_3_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jbankfin.2010.06.001"},{"key":"e_1_3_2_1_55_1","volume-title":"Impact of a deep learning assistant on the histopathologic classification of liver cancer. NPJ digital medicine 3, 1","author":"Kiani Amirhossein","year":"2020","unstructured":"Amirhossein Kiani, Bora Uyumazturk, Pranav Rajpurkar, Alex Wang, Rebecca Gao, Erik Jones, Yifan Yu, Curtis P Langlotz, Robyn L Ball, Thomas J Montine, 2020. Impact of a deep learning assistant on the histopathologic classification of liver cancer. NPJ digital medicine 3, 1 (2020), 1\u20138."},{"key":"e_1_3_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300641"},{"key":"e_1_3_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/2207676.2207678"},{"key":"e_1_3_2_1_58_1","doi-asserted-by":"publisher","DOI":"10.1109\/VLHCC.2013.6645235"},{"key":"e_1_3_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300717"},{"key":"e_1_3_2_1_60_1","volume-title":"An evaluation of the human-interpretability of explanation. arXiv preprint arXiv:1902.00006","author":"Lage Isaac","year":"2019","unstructured":"Isaac Lage, Emily Chen, Jeffrey He, Menaka Narayanan, Been Kim, Sam Gershman, and Finale Doshi-Velez. 2019. An evaluation of the human-interpretability of explanation. arXiv preprint arXiv:1902.00006 (2019)."},{"key":"e_1_3_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376873"},{"key":"e_1_3_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287590"},{"key":"e_1_3_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939874"},{"key":"e_1_3_2_1_64_1","volume-title":"A quantitative approach to content validity. Personnel psychology 28, 4","author":"Lawshe Charles H","year":"1975","unstructured":"Charles H Lawshe. 1975. A quantitative approach to content validity. Personnel psychology 28, 4 (1975), 563\u2013575."},{"key":"e_1_3_2_1_65_1","volume-title":"Trust in automation: Designing for appropriate reliance. Human factors 46, 1","author":"Lee John D","year":"2004","unstructured":"John D Lee and Katrina A See. 2004. Trust in automation: Designing for appropriate reliance. Human factors 46, 1 (2004), 50\u201380."},{"key":"e_1_3_2_1_66_1","doi-asserted-by":"publisher","DOI":"10.1145\/3415227"},{"key":"e_1_3_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445472"},{"key":"e_1_3_2_1_68_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359284"},{"key":"e_1_3_2_1_69_1","volume-title":"Explanation-Based Human Debugging of NLP Models: A Survey. arXiv preprint arXiv:2104.15135","author":"Lertvittayakumjorn Piyawat","year":"2021","unstructured":"Piyawat Lertvittayakumjorn and Francesca Toni. 2021. Explanation-Based Human Debugging of NLP Models: A Survey. arXiv preprint arXiv:2104.15135 (2021)."},{"key":"e_1_3_2_1_70_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445522"},{"key":"e_1_3_2_1_71_1","doi-asserted-by":"publisher","DOI":"10.1145\/1518701.1519023"},{"key":"e_1_3_2_1_72_1","volume-title":"Understanding the Effect of Out-of-distribution Examples and Interactive Explanations on Human-AI Decision Making. arXiv preprint arXiv:2101.05303","author":"Liu Han","year":"2021","unstructured":"Han Liu, Vivian Lai, and Chenhao Tan. 2021. Understanding the Effect of Out-of-distribution Examples and Interactive Explanations on Human-AI Decision Making. arXiv preprint arXiv:2101.05303 (2021)."},{"key":"e_1_3_2_1_73_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.obhdp.2018.12.005"},{"key":"e_1_3_2_1_74_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445562"},{"key":"e_1_3_2_1_75_1","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372824"},{"key":"e_1_3_2_1_76_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376257"},{"key":"e_1_3_2_1_77_1","volume-title":"When Does Uncertainty Matter?: Understanding the Impact of Predictive Uncertainty in ML Assisted Decision Making. arXiv preprint arXiv:2011.06167","author":"McGrath Sean","year":"2020","unstructured":"Sean McGrath, Parth Mehta, Alexandra Zytek, Isaac Lage, and Himabindu Lakkaraju. 2020. When Does Uncertainty Matter?: Understanding the Impact of Predictive Uncertainty in ML Assisted Decision Making. arXiv preprint arXiv:2011.06167 (2020)."},{"key":"e_1_3_2_1_78_1","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287596"},{"key":"e_1_3_2_1_79_1","volume-title":"How do humans understand explanations from machine learning systems? an evaluation of the human-interpretability of explanation. arXiv preprint arXiv:1802.00682","author":"Narayanan Menaka","year":"2018","unstructured":"Menaka Narayanan, Emily Chen, Jeffrey He, Been Kim, Sam Gershman, and Finale Doshi-Velez. 2018. How do humans understand explanations from machine learning systems? an evaluation of the human-interpretability of explanation. arXiv preprint arXiv:1802.00682 (2018)."},{"key":"e_1_3_2_1_80_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/N18-1097"},{"key":"e_1_3_2_1_81_1","volume-title":"Anchoring Bias Affects Mental Model Formation and User Reliance in Explainable AI Systems. In 26th International Conference on Intelligent User Interfaces. 340\u2013350","author":"Nourani Mahsan","year":"2021","unstructured":"Mahsan Nourani, Chiradeep Roy, Jeremy E Block, Donald R Honeycutt, Tahrima Rahman, Eric Ragan, and Vibhav Gogate. 2021. Anchoring Bias Affects Mental Model Formation and User Reliance in Explainable AI Systems. In 26th International Conference on Intelligent User Interfaces. 340\u2013350."},{"key":"e_1_3_2_1_82_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359204"},{"key":"e_1_3_2_1_83_1","volume-title":"Jennifer Wortman Vaughan, and Hanna Wallach","author":"Poursabzi-Sangdeh Forough","year":"2018","unstructured":"Forough Poursabzi-Sangdeh, Daniel G Goldstein, Jake M Hofman, Jennifer Wortman Vaughan, and Hanna Wallach. 2018. Manipulating and measuring model interpretability. arXiv preprint arXiv:1802.07810 (2018)."},{"key":"e_1_3_2_1_84_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3173677"},{"key":"e_1_3_2_1_85_1","unstructured":"UCI Mahcine Learning Repository. 1994. Adult Data Set."},{"key":"e_1_3_2_1_86_1","doi-asserted-by":"publisher","DOI":"10.1145\/2939672.2939778"},{"key":"e_1_3_2_1_87_1","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v32i1.11491"},{"key":"e_1_3_2_1_88_1","doi-asserted-by":"publisher","DOI":"10.1038\/s42256-021-00307-0"},{"key":"e_1_3_2_1_89_1","doi-asserted-by":"publisher","DOI":"10.1007\/s12369-010-0066-7"},{"key":"e_1_3_2_1_90_1","volume-title":"Predicting recidivism in north carolina","author":"Schmidt Peter","year":"1978","unstructured":"Peter Schmidt and Ann D Witte. 1988. Predicting recidivism in north carolina, 1978 and 1980. Inter-university Consortium for Political and Social Research."},{"key":"e_1_3_2_1_91_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376624"},{"key":"e_1_3_2_1_92_1","volume-title":"Computer Graphics Forum","author":"Sperrle Fabian","unstructured":"Fabian Sperrle, Mennatallah El-Assady, Grace Guo, Rita Borgo, D Horng Chau, Alex Endert, and Daniel Keim. 2021. A Survey of Human-Centered Evaluations in Human-Centered Machine Learning. In Computer Graphics Forum, Vol. 40. Wiley Online Library, 543\u2013567."},{"key":"e_1_3_2_1_93_1","volume-title":"Alex Endert, and Daniel Keim.","author":"Sperrle Fabian","year":"2020","unstructured":"Fabian Sperrle, Mennatallah El-Assady, Grace Guo, Duen Horng Chau, Alex Endert, and Daniel Keim. 2020. Should We Trust (X) AI? Design Dimensions for Structured Experimental Evaluations. arXiv preprint arXiv:2009.06433 (2020)."},{"key":"e_1_3_2_1_94_1","volume-title":"Progressive disclosure: designing for effective transparency. arXiv preprint arXiv:1811.02164","author":"Springer Aaron","year":"2018","unstructured":"Aaron Springer and Steve Whittaker. 2018. Progressive disclosure: designing for effective transparency. arXiv preprint arXiv:1811.02164 (2018)."},{"key":"e_1_3_2_1_95_1","unstructured":"Lending Club Statistics. 2019. Lending Club."},{"key":"e_1_3_2_1_96_1","volume-title":"Advances in Human Factors in Robots and Unmanned Systems","author":"Stowers Kimberly","unstructured":"Kimberly Stowers, Nicholas Kasdaglis, Michael Rupp, Jessie Chen, Daniel Barber, and Michael Barnes. 2017. Insights into human-agent teaming: Intelligent agent transparency and uncertainty. In Advances in Human Factors in Robots and Unmanned Systems. Springer, 149\u2013160."},{"key":"e_1_3_2_1_97_1","unstructured":"Supreme Court of Wisconsin. 2016. State of Wisconsin Plaintiff-Respondent v. Eric L. Loomis Defendant-Appellant."},{"key":"e_1_3_2_1_98_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397481.3450662"},{"key":"e_1_3_2_1_99_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445101"},{"key":"e_1_3_2_1_100_1","volume-title":"Office of Justice Programs","author":"United States Department of Justice.","year":"2014","unstructured":"United States Department of Justice. Office of Justice Programs. Bureau of Justice Statistics.2014. State Court Processing Statistics, 1990-2009: Felony Defendants in Large Urban Counties."},{"key":"e_1_3_2_1_101_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445365"},{"key":"e_1_3_2_1_102_1","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174014"},{"key":"e_1_3_2_1_103_1","doi-asserted-by":"publisher","DOI":"10.1145\/3476068"},{"key":"e_1_3_2_1_104_1","doi-asserted-by":"publisher","DOI":"10.1145\/2647868.2654940"},{"key":"e_1_3_2_1_105_1","doi-asserted-by":"publisher","DOI":"10.1145\/3397481.3450650"},{"key":"e_1_3_2_1_106_1","volume-title":"A human-grounded evaluation of shap for alert processing. arXiv preprint arXiv:1907.03324","author":"Weerts Hilde JP","year":"2019","unstructured":"Hilde JP Weerts, Werner van Ipenburg, and Mykola Pechenizkiy. 2019. A human-grounded evaluation of shap for alert processing. arXiv preprint arXiv:1907.03324 (2019)."},{"key":"e_1_3_2_1_107_1","volume-title":"A Survey of Human-in-the-loop for Machine Learning. arXiv preprint arXiv:2108.00941","author":"Wu Xingjiao","year":"2021","unstructured":"Xingjiao Wu, Luwei Xiao, Yixuan Sun, Junhang Zhang, Tianlong Ma, and Liang He. 2021. A Survey of Human-in-the-loop for Machine Learning. arXiv preprint arXiv:2108.00941 (2021)."},{"key":"e_1_3_2_1_108_1","doi-asserted-by":"publisher","DOI":"10.1145\/3377325.3377480"},{"key":"e_1_3_2_1_109_1","doi-asserted-by":"publisher","DOI":"10.1080\/07350015.2016.1200981"},{"key":"e_1_3_2_1_110_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300509"},{"key":"e_1_3_2_1_111_1","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3302277"},{"key":"e_1_3_2_1_112_1","volume-title":"Effect of Confidence and Explanation on Accuracy and Trust Calibration in AI-Assisted Decision Making. arXiv preprint arXiv:2001.02114","author":"Zhang Yunfeng","year":"2020","unstructured":"Yunfeng Zhang, Q Vera Liao, and Rachel KE Bellamy. 2020. Effect of Confidence and Explanation on Accuracy and Trust Calibration in AI-Assisted Decision Making. arXiv preprint arXiv:2001.02114 (2020)."}],"event":{"name":"FAccT '23: the 2023 ACM Conference on Fairness, Accountability, and Transparency","location":"Chicago IL USA","acronym":"FAccT '23"},"container-title":["2023 ACM Conference on Fairness Accountability and Transparency"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3593013.3594087","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3593013.3594087","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T12:37:19Z","timestamp":1750163839000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3593013.3594087"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,6,12]]},"references-count":112,"alternative-id":["10.1145\/3593013.3594087","10.1145\/3593013"],"URL":"https:\/\/doi.org\/10.1145\/3593013.3594087","relation":{},"subject":[],"published":{"date-parts":[[2023,6,12]]},"assertion":[{"value":"2023-06-12","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F3769733_3769747 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,20]],"date-time":"2026-10-20T07:21:27Z","timestamp":1792480887028,"version":"4.5.0"},"reference-count":20,"publisher":"Association for Computing Machinery (ACM)","issue":"1","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["SIGIR Forum"],"published-print":{"date-parts":[[2025,6]]},"abstract":"<jats:p>\n Recently, Sakana.ai introduced the AI Scientist, a system claiming to automate the entire research lifecycle and conduct research autonomously, a concept we term\n <jats:italic toggle=\"yes\">Artificial Research Intelligence<\/jats:italic>\n (ARI). Achieving ARI would be a major milestone toward Artificial General Intelligence (AGI) and a prerequisite to achieving Super Intelligence. The AI Scientist received much attention in the academic and broader AI community. A thorough evaluation of the AI Scientist, however, had not yet been conducted.\n <jats:sup>1<\/jats:sup>\n We evaluated the AI Scientist and found several critical shortcomings. The system's literature review process is inadequate, relying on simplistic keyword searches rather than profound synthesis, which leads to poor novelty assessments. In our experiments, several generated research ideas were incorrectly classified as novel, including well-established concepts such as micro-batching for stochastic gradient descent (SGD). The AI Scientist also lacks robustness in experiment execution\u2014five out of twelve proposed experiments (42%) failed due to coding errors, and those that did run often produced logically flawed or misleading results. In one case, an experiment designed to optimize energy efficiency reported improvements in accuracy while consuming more computational resources, contradicting its stated goal. Furthermore, the system modifies experimental code minimally, with each iteration adding only 8% more characters on average, suggesting limited adaptability. The generated manuscripts were poorly substantiated, with a median of just five citations per paper\u2014most of which were outdated (only five out of 34 citations were from 2020 or later). Structural errors were frequent, including missing figures, repeated sections, and placeholder text such as \"Conclusions Here\". Hallucinated numerical results were contained in several manuscripts, undermining the reliability of its outputs. Despite its limitations, the AI Scientist represents a significant leap forward in research automation. It produces complete research manuscripts with minimal human intervention, challenging conventional expectations of AI-generated scientific work. Many reviewers or university instructors conducting only a superficial assessment may struggle to distinguish its output from that of human researchers, demonstrating how far AI has progressed in mimicking academic writing and structuring scientific arguments. While the quality of its manuscripts currently aligns with that of an unmotivated undergraduate student rushing to meet a deadline, this level of autonomy in research generation is remarkable. More strikingly, it achieves this at an unprecedented speed and cost efficiency\u2014our analysis indicates that generating a full research paper costs only $6\u2013$15, with just 3.5 hours of human involvement. This is significantly faster than traditional human researchers. Given that AI research automation was nearly nonexistent just a few years ago, the AI Scientist marks a substantial milestone toward Artificial Research Intelligence (ARI), signalling the acceleration of AI-driven scientific discovery. The AI Scientist also illustrates the urgent need for a discussion within the Information Retrieval (IR) and broader scientific communities. Whether and when ARI becomes a reality depends on how the academic and AI communities shape its development and governance. We propose concrete steps, including pilot projects and competitions, and standardized attribution frameworks such as research logs and markup languages.\n <\/jats:p>","DOI":"10.1145\/3769733.3769747","type":"journal-article","created":{"date-parts":[[2025,10,10]],"date-time":"2025-10-10T16:04:53Z","timestamp":1760112293000},"page":"1-20","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":7,"title":["Evaluating Sakana's AI Scientist: Bold Claims, Mixed Results, and a Promising Future?"],"prefix":"10.1145","volume":"59","author":[{"given":"Joeran","family":"Beel","sequence":"first","affiliation":[{"name":"University of Siegen, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Min-Yen","family":"Kan","sequence":"additional","affiliation":[{"name":"National University of Singapore, Singapore"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Moritz","family":"Baumgart","sequence":"additional","affiliation":[{"name":"University of Siegen, Germany"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2025,10,10]]},"reference":[{"key":"e_1_2_1_1_1","volume-title":"International Workshop on Recommender Systems for Sustainability and Social Good (RecSoGood) at the 18th ACM Conference on Recommender Systems (ACM RecSys)","author":"Baumgart Moritz","year":"2024","unstructured":"Moritz Baumgart, Lukas Wegmeth, Tobias Vente, and Joeran Beel. e-fold cross-validation for recommender-system evaluation. In International Workshop on Recommender Systems for Sustainability and Social Good (RecSoGood) at the 18th ACM Conference on Recommender Systems (ACM RecSys), 2024."},{"key":"e_1_2_1_2_1","volume-title":"Our use of ai-tools for writing research papers","author":"Beel Joeran","year":"2024","unstructured":"Joeran Beel. Our use of ai-tools for writing research papers. In Intelligent Systems Group, Blog, 2024."},{"key":"e_1_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.31219\/osf.io\/5ru2g"},{"key":"e_1_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.31219\/osf.io\/exw3j"},{"key":"e_1_2_1_5_1","volume-title":"YouTube","author":"Berman Matthew","year":"2024","unstructured":"Matthew Berman. \"ai scientist\" can discover new science! (self-improving ai = agi). YouTube, 2024. URL https:\/\/www.youtube.com\/watch?v=LTM_3EUwUmE."},{"key":"e_1_2_1_6_1","volume-title":"Parallelizing stochastic gradient descent for least squares regression: mini-batching, averaging, and model misspecification. Journal of machine learning research, 18(223):1\u201342","author":"Jain Prateek","year":"2018","unstructured":"Prateek Jain, Sham M Kakade, Rahul Kidambi, Praneeth Netrapalli, and Aaron Sidford. Parallelizing stochastic gradient descent for least squares regression: mini-batching, averaging, and model misspecification. Journal of machine learning research, 18(223):1\u201342, 2018."},{"key":"e_1_2_1_7_1","volume-title":"Chain of ideas: Revolutionizing research via novel idea development with llm agents. arXiv:2410.13185","author":"Li Long","year":"2024","unstructured":"Long Li, Weiwen Xu, Jiayan Guo, Ruochen Zhao, Xingxuan Li, Yuqian Yuan, Boqiang Zhang, Yuming Jiang, Yifei Xin, Ronghao Dang, Deli Zhao, Yu Rong, Tian Feng, and Lidong Bing. Chain of ideas: Revolutionizing research via novel idea development with llm agents. arXiv:2410.13185, 2024. URL https:\/\/arxiv.org\/abs\/2410.13185."},{"key":"e_1_2_1_8_1","volume-title":"Jakob Foerster, Jeff Clune, and David Ha. The ai scientist: Towards fully automated open-ended scientific discovery","author":"Lu Chris","year":"2024","unstructured":"Chris Lu, Cong Lu, Robert Tjarko Lange, Jakob Foerster, Jeff Clune, and David Ha. The ai scientist: Towards fully automated open-ended scientific discovery, 2024. URL https:\/\/arxiv.org\/abs\/2408.06292."},{"key":"e_1_2_1_9_1","volume-title":"International Conference on Artificial Intelligence and Machine Learning Research (CAIMLR)","author":"Mahlich Christopher","year":"2024","unstructured":"Christopher Mahlich, Tobias Vente, and Joeran Beel. From theory to practice: Implementing and evaluating e-fold cross-validation. In International Conference on Artificial Intelligence and Machine Learning Research (CAIMLR), 2024."},{"key":"e_1_2_1_10_1","volume-title":"A new era of discovery. Google Blog","author":"Manyika James","year":"2024","unstructured":"James Manyika. A new era of discovery. Google Blog, 2024. URL https:\/\/blog.google\/technology\/ai\/a-new-era-of-discovery\/."},{"key":"e_1_2_1_11_1","volume-title":"Scideator: Human-llm scientific idea generation grounded in research-paper facet recombination. arXiv:2409.14634","author":"Radensky Marissa","year":"2025","unstructured":"Marissa Radensky, Simra Shahid, Raymond Fok, Pao Siangliulue, Tom Hope, and Daniel S. Weld. Scideator: Human-llm scientific idea generation grounded in research-paper facet recombination. arXiv:2409.14634, 2025. URL https:\/\/arxiv.org\/abs\/2409.14634."},{"key":"e_1_2_1_12_1","unstructured":"Sakana.ai. Sakana \"ai scientist\". Sakana.ai Website 2024a. URL https:\/\/www.sakanaai.org\/sakana-ai-scientist\/."},{"key":"e_1_2_1_13_1","volume-title":"The ai scientist: Towards fully automated open-ended scientific discovery. Sakan.ai Blog","year":"2024","unstructured":"Sakana.ai. The ai scientist: Towards fully automated open-ended scientific discovery. Sakan.ai Blog, 2024b. URL https:\/\/sakana.ai\/ai-scientist\/."},{"key":"e_1_2_1_14_1","volume-title":"Can llms generate novel research ideas? a large-scale human study with 100+ nlp researchers. arXiv:2409.04109","author":"Si Chenglei","year":"2024","unstructured":"Chenglei Si, Diyi Yang, and Tatsunori Hashimoto. Can llms generate novel research ideas? a large-scale human study with 100+ nlp researchers. arXiv:2409.04109, 2024. URL https:\/\/arxiv.org\/abs\/2409.04109."},{"key":"e_1_2_1_15_1","volume-title":"Two heads are better than one: A multi-agent system has the potential to improve scientific idea generation. arxiv:2410.09403","author":"Su Haoyang","year":"2024","unstructured":"Haoyang Su, Renqi Chen, Shixiang Tang, Xinzhe Zheng, Jingzhe Li, Zhenfei Yin, Wanli Ouyang, and Nanqing Dong. Two heads are better than one: A multi-agent system has the potential to improve scientific idea generation. arxiv:2410.09403, 2024. URL https:\/\/arxiv.org\/abs\/2410.09403."},{"key":"e_1_2_1_16_1","volume-title":"Sakanaai new ai agent researcher stuns the enitre industry! YouTube","author":"AIGRID.","year":"2024","unstructured":"TheAIGRID. Sakanaai new ai agent researcher stuns the enitre industry! YouTube, 2024. URL https:\/\/www.youtube.com\/watch?v=hP-IzCZAZDc."},{"key":"e_1_2_1_17_1","volume-title":"Artificial intelligence, scientific discovery, and product innovation. arXiv:2412.17866","author":"Toner-Rodgers Aidan","year":"2024","unstructured":"Aidan Toner-Rodgers. Artificial intelligence, scientific discovery, and product innovation. arXiv:2412.17866, 2024. URL https:\/\/arxiv.org\/abs\/2412.17866."},{"key":"e_1_2_1_18_1","volume-title":"Ai-driven review systems: Evaluating llms in scalable and bias-aware academic reviews. 2408.10365","author":"Tyser Keith","year":"2024","unstructured":"Keith Tyser, Ben Segev, Gaston Longhitano, Xin-Yu Zhang, Zachary Meeks, Jason Lee, Uday Garg, Nicholas Belsten, Avi Shporer, Madeleine Udell, Dov Te'eni, and Iddo Drori. Ai-driven review systems: Evaluating llms in scalable and bias-aware academic reviews. 2408.10365, 2024. URL https:\/\/arxiv.org\/abs\/2408.10365."},{"key":"e_1_2_1_19_1","volume-title":"Are we there yet? revealing the risks of utilizing large language models in scholarly peer review. arXiv:2412.01708","author":"Ye Rui","year":"2024","unstructured":"Rui Ye, Xianghe Pang, Jingyi Chai, Jiaao Chen, Zhenfei Yin, Zhen Xiang, Xiaowen Dong, Jing Shao, and Siheng Chen. Are we there yet? revealing the risks of utilizing large language models in scholarly peer review. arXiv:2412.01708, 2024. URL https:\/\/arxiv.org\/abs\/2412.01708."},{"key":"e_1_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/3626772.3657848"}],"container-title":["ACM SIGIR Forum"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3769733.3769747","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,10,10]],"date-time":"2025-10-10T16:05:15Z","timestamp":1760112315000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3769733.3769747"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,6]]},"references-count":20,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2025,6]]}},"alternative-id":["10.1145\/3769733.3769747"],"URL":"https:\/\/doi.org\/10.1145\/3769733.3769747","relation":{},"ISSN":["0163-5840"],"issn-type":[{"value":"0163-5840","type":"print"}],"subject":[],"published":{"date-parts":[[2025,6]]},"assertion":[{"value":"2025-10-10","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F3805760_3814924 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,22]],"date-time":"2026-10-22T20:44:29Z","timestamp":1792701869838,"version":"4.5.0"},"publisher-location":"New York, NY, USA","reference-count":42,"publisher":"ACM","license":[{"start":{"date-parts":[[2026,7,5]],"date-time":"2026-07-05T00:00:00Z","timestamp":1783209600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/legalcode"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2026,7,5]]},"DOI":"10.1145\/3805760.3814924","type":"proceedings-article","created":{"date-parts":[[2026,7,5]],"date-time":"2026-07-05T08:46:13Z","timestamp":1783241173000},"page":"332-339","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1,"title":["SWE-Bench+: Enhanced LLM Coding Benchmark"],"prefix":"10.1145","author":[{"ORCID":"https:\/\/orcid.org\/0009-0005-1937-3733","authenticated-orcid":false,"given":"Haoran","family":"Xue","sequence":"first","affiliation":[{"name":"York University, Department of Electrical Engineering and Computer Science, North York, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8856-9685","authenticated-orcid":false,"given":"Reem","family":"Aleithan","sequence":"additional","affiliation":[{"name":"York University, Department of Electrical Engineering and Computer Science, North York, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0007-4357-5719","authenticated-orcid":false,"given":"Nafid","family":"Enan","sequence":"additional","affiliation":[{"name":"York University, Department of Electrical Engineering and Computer Science, North York, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1376-095X","authenticated-orcid":false,"given":"Gias","family":"Uddin","sequence":"additional","affiliation":[{"name":"York University, Department of Electrical Engineering and Computer Science, North York, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0617-2877","authenticated-orcid":false,"given":"Song","family":"Wang","sequence":"additional","affiliation":[{"name":"York University, Department of Electrical Engineering and Computer Science, North York, Canada"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2026,7,5]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Kamel Alrashedy and Ahmed Binjahlan. 2024. Language Models are Better Bug Detector Through Code-Pair Classification. arXiv:2311.07957 [cs.SE] https: \/\/arxiv.org\/abs\/2311.07957"},{"key":"e_1_3_2_1_2_1","unstructured":"Antonis Antoniades Albert \u00d6rwall Kexun Zhang Yuxi Xie Anirudh Goyal and William Wang. 2025. SWE-Search: Enhancing Software Agents with Monte Carlo Tree Search and Iterative Refinement. arXiv:2410.20285 [cs.AI] https: \/\/arxiv.org\/abs\/2410.20285"},{"key":"e_1_3_2_1_3_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10462-022-10371-6"},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2406.01304"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2107.03374"},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00022"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.engappai.2024.109291"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2304.02014"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE55347.2025.00038"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.findings-acl.716"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3639219"},{"key":"e_1_3_2_1_12_1","volume-title":"Jimenez et al","author":"Carlos","year":"2024","unstructured":"Carlos E. Jimenez et al. 2024. SWE-bench: Can Language Models Resolve Real- World GitHub Issues? arXiv:2310.06770 [cs.CL] https:\/\/arxiv.org\/abs\/2310.06770"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.dibe.2024"},{"key":"e_1_3_2_1_14_1","volume-title":"Software Engineering: An Example from ChatGPT and Automated Program Repair. arXiv:2310.08879 [cs.SE] https:\/\/arxiv.org\/abs\/2310.08879","author":"Quanjun Zhang","year":"2024","unstructured":"Quanjun Zhang et al. 2024. A Critical Review of Large Language Model on Software Engineering: An Example from ChatGPT and Automated Program Repair. arXiv:2310.08879 [cs.SE] https:\/\/arxiv.org\/abs\/2310.08879"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/3747588"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/3715765"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2411.01114"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.52202\/079017-1837"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/3650203.3663334"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","unstructured":"Shanchao Liang Spandan Garg and Roshanak Moghaddam. 2025. The SWE- Bench Illusion: When State-of-the-Art LLMs Remember Instead of Reason. doi:10. 48550\/arXiv.2506.12286 10.48550\/arXiv.2506.12286","DOI":"10.48550\/arXiv.2506.12286"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1109\/AITest62860.2024.00019"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/3715727"},{"key":"e_1_3_2_1_23_1","volume-title":"Quantifying contamination in evaluating code generation capabilities of language models. arXiv preprint arXiv:2403.04811","author":"Riddell Martin","year":"2024","unstructured":"Martin Riddell, Ansong Ni, and Arman Cohan. 2024. Quantifying contamination in evaluating code generation capabilities of language models. arXiv preprint arXiv:2403.04811 (2024)."},{"key":"e_1_3_2_1_24_1","unstructured":"Ricardo La Rosa Corey Hulse and Bangdi Liu. 2024. Can Github issues be solved with Tree Of Thoughts? arXiv:2405.13057 [cs.SE] https:\/\/arxiv.org\/abs\/2405. 13057"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/3643769"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2013.11"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/3540250.3549145"},{"key":"e_1_3_2_1_28_1","doi-asserted-by":"publisher","unstructured":"Jiho Shin Reem Aleithan Hadi Hemmati and Song Wang. 2024. Retrieval- Augmented Test Generation: How Far Are We? doi:10.48550\/arXiv.2409.12682 10.48550\/arXiv.2409.12682","DOI":"10.48550\/arXiv.2409.12682"},{"key":"e_1_3_2_1_29_1","unstructured":"5 Team and Aohan Zeng et al. 2025. GLM-4.5: Agentic Reasoning and Coding (ARC) Foundation Models. arXiv:2508.06471 [cs.CL] https:\/\/arxiv.org\/abs\/2508. 06471"},{"key":"e_1_3_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2507.23370"},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2406.19314"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2021"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3715754"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2024.acl-long.301"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/3650212.3680328"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.52202\/079017-1601"},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3623316"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2412"},{"key":"e_1_3_2_1_39_1","volume-title":"Proceedings of the 42nd International Conference on Machine Learning","author":"Yu Zhongming","year":"2025","unstructured":"Zhongming et al. Yu. 2025. OrcaLoca: an LLM agent framework for software issue localization. In Proceedings of the 42nd International Conference on Machine Learning (Vancouver, Canada) (ICML'25). JMLR.org, Article 2939, 21 pages."},{"key":"e_1_3_2_1_40_1","unstructured":"Daoguang Zan Zhirong Huang and Ailun Yu et al. 2024. SWE-bench-java: A GitHub Issue Resolving Benchmark for Java. arXiv:2408.14354 [cs.SE] https: \/\/arxiv.org\/abs\/2408.14354"},{"key":"e_1_3_2_1_41_1","unstructured":"Kexun Zhang Weiran Yao and Zuxin Liu et al. 2024. Diversity Empowers Intelligence: Integrating Expertise of Software Engineering Agents. arXiv:2408.07060 [cs.SE] https:\/\/arxiv.org\/abs\/2408.07060"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1145\/3650212.3680384"}],"event":{"name":"AIware '26: 3rd ACM International Conference on AI-Powered Software","location":"Montreal QC Canada","acronym":"AIware '26","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"]},"container-title":["Proceedings of the 3rd ACM International Conference on AI-Powered Software"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3805760.3814924","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,7,5]],"date-time":"2026-07-05T08:47:57Z","timestamp":1783241277000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3805760.3814924"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2026,7,5]]},"references-count":42,"alternative-id":["10.1145\/3805760.3814924","10.1145\/3805760"],"URL":"https:\/\/doi.org\/10.1145\/3805760.3814924","relation":{},"subject":[],"published":{"date-parts":[[2026,7,5]]},"assertion":[{"value":"2026-07-05","order":3,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1145_2F568522_568525 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,17]],"date-time":"2026-10-17T16:40:30Z","timestamp":1792255230343,"version":"4.4.0"},"reference-count":56,"publisher":"Association for Computing Machinery (ACM)","issue":"3","license":[{"start":{"date-parts":[[2002,9,1]],"date-time":"2002-09-01T00:00:00Z","timestamp":1030838400000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":["ACM Comput. Surv."],"published-print":{"date-parts":[[2002,9]]},"abstract":"<jats:p>\n This survey covers rollback-recovery techniques that do not require special language constructs. In the first part of the survey we classify rollback-recovery protocols into\n <jats:italic>checkpoint-based<\/jats:italic>\n and\n <jats:italic>log-based.<\/jats:italic>\n <jats:italic>Checkpoint-based<\/jats:italic>\n protocols rely solely on checkpointing for system state restoration. Checkpointing can be coordinated, uncoordinated, or communication-induced.\n <jats:italic>Log-based<\/jats:italic>\n protocols combine checkpointing with logging of nondeterministic events, encoded in tuples called\n <jats:italic>determinants<\/jats:italic>\n . Depending on how determinants are logged, log-based protocols can be pessimistic, optimistic, or causal. Throughout the survey, we highlight the research issues that are at the core of rollback-recovery and present the solutions that currently address them. We also compare the performance of different rollback-recovery protocols with respect to a series of desirable properties and discuss the issues that arise in the practical implementations of these protocols.\n <\/jats:p>","DOI":"10.1145\/568522.568525","type":"journal-article","created":{"date-parts":[[2002,10,7]],"date-time":"2002-10-07T09:52:47Z","timestamp":1033984367000},"page":"375-408","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":1112,"title":["A survey of rollback-recovery protocols in message-passing systems"],"prefix":"10.1145","volume":"34","author":[{"given":"E. N. (Mootaz)","family":"Elnozahy","sequence":"first","affiliation":[{"name":"IBM Research, Austin, TX"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lorenzo","family":"Alvisi","sequence":"additional","affiliation":[{"name":"The University of Texas at Austin, Austin, TX"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yi-Min","family":"Wang","sequence":"additional","affiliation":[{"name":"Microsoft Research, Redmond, WA"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"David B.","family":"Johnson","sequence":"additional","affiliation":[{"name":"Rice University, Houston, TX"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2002,9]]},"reference":[{"key":"e_1_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/32.666828"},{"key":"e_1_2_1_3_1","volume-title":"The Twenty Nineth Annual International Symposium on Fault-Tolerant Computing","author":"Alvisi L.","unstructured":"Alvisi , L. , Elnozahy , E. N. , Rao , S. , Husain , S. A. , and Mel , A. D . 1999. An analysis of communication-induced checkpointing. In Digest of Papers, FTCS-29 , The Twenty Nineth Annual International Symposium on Fault-Tolerant Computing ( Madison, Wisconsin), 242--249. Alvisi, L., Elnozahy, E. N., Rao, S., Husain, S. A., and Mel, A. D. 1999. An analysis of communication-induced checkpointing. In Digest of Papers, FTCS-29, The Twenty Nineth Annual International Symposium on Fault-Tolerant Computing (Madison, Wisconsin), 242--249."},{"key":"e_1_2_1_5_1","volume-title":"Proceedings of the Eighth ACM Symposium on Operating Systems Principles, 78--86","author":"Babaoglu O.","unstructured":"Babaoglu , O. and Joy , W . 1981. Converting a swap-based system to do paging in an architecture lacking page-reference bits . In Proceedings of the Eighth ACM Symposium on Operating Systems Principles, 78--86 . 10.1145\/800216.806595 Babaoglu, O. and Joy, W. 1981. Converting a swap-based system to do paging in an architecture lacking page-reference bits. In Proceedings of the Eighth ACM Symposium on Operating Systems Principles, 78--86. 10.1145\/800216.806595"},{"key":"e_1_2_1_6_1","volume-title":"Proceedings, Seventeenth Symposium on Reliable Distributed Systems, 61--67","author":"Baldoni R.","unstructured":"Baldoni , R. , Quaglia , F. , and Ciciani , B . 1998. A VP-accordant checkpointing protocol preventing useless checkpoints . In Proceedings, Seventeenth Symposium on Reliable Distributed Systems, 61--67 . Baldoni, R., Quaglia, F., and Ciciani, B. 1998. A VP-accordant checkpointing protocol preventing useless checkpoints. In Proceedings, Seventeenth Symposium on Reliable Distributed Systems, 61--67."},{"key":"e_1_2_1_7_1","volume-title":"Proceedings of The Fourth Conference on Data Engineering, 285--293","author":"Ban\u00e2tre J. P.","unstructured":"Ban\u00e2tre , J. P. , Ban\u00e2tre , M. , and Muller , G . 1988. Ensuring data security and integrity with a fast stable storage . In Proceedings of The Fourth Conference on Data Engineering, 285--293 . Ban\u00e2tre, J. P., Ban\u00e2tre, M., and Muller, G. 1988. Ensuring data security and integrity with a fast stable storage. In Proceedings of The Fourth Conference on Data Engineering, 285--293."},{"key":"e_1_2_1_8_1","volume-title":"Proceedings of the Eighth ACM Symposium on Operating Systems Principles, 22--29","author":"Bartlett J. F.","year":"1981","unstructured":"Bartlett , J. F. 1981 . A Non Stop Kernel . In Proceedings of the Eighth ACM Symposium on Operating Systems Principles, 22--29 . 10.1145\/800216.806587 Bartlett, J. F. 1981. A Non Stop Kernel. In Proceedings of the Eighth ACM Symposium on Operating Systems Principles, 22--29. 10.1145\/800216.806587"},{"key":"e_1_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1006\/jpdc.1997.1338"},{"key":"e_1_2_1_10_1","volume-title":"Proceedings, Seventh Symposium on Reliable Distributed Systems, 3--12","author":"Bhargava B.","unstructured":"Bhargava , B. and Lian , S. R . 1988. Independent checkpointing and concurrent rollback for recovery---An optimistic approach . In Proceedings, Seventh Symposium on Reliable Distributed Systems, 3--12 . Bhargava, B. and Lian, S. R. 1988. Independent checkpointing and concurrent rollback for recovery---An optimistic approach. In Proceedings, Seventh Symposium on Reliable Distributed Systems, 3--12."},{"key":"e_1_2_1_11_1","volume-title":"Proceedings of the Sixth International Conference on Data Engineering, 182--189","author":"Bhargava B.","unstructured":"Bhargava , B. , Lian , S. R. , and Leu , P. J . 1990. Experimental evaluation of concurrent checkpointing and rollback-recovery algorithms . In Proceedings of the Sixth International Conference on Data Engineering, 182--189 . Bhargava, B., Lian, S. R., and Leu, P. J. 1990. Experimental evaluation of concurrent checkpointing and rollback-recovery algorithms. In Proceedings of the Sixth International Conference on Data Engineering, 182--189."},{"key":"e_1_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/58564.58565"},{"key":"e_1_2_1_13_1","volume-title":"Proceedings of the Fifteenth ACM Symposium on Operating Systems Principles, 1--11","author":"Bressoud T. C.","unstructured":"Bressoud , T. C. and Schneider , F. B . 1995. Hypervisor-based fault tolerance . In Proceedings of the Fifteenth ACM Symposium on Operating Systems Principles, 1--11 . 10.1145\/224056.224058 Bressoud, T. C. and Schneider, F. B. 1995. Hypervisor-based fault tolerance. In Proceedings of the Fifteenth ACM Symposium on Operating Systems Principles, 1--11. 10.1145\/224056.224058"},{"key":"e_1_2_1_14_1","volume-title":"IEEE International Symposium on Reliability, Distributed Software, and Databases, 207--215","author":"Briatico D.","unstructured":"Briatico , D. , Ciuffoletti , A. , and Simoncini , L . 1984. A distributed domino-effect free recovery algorithm . In IEEE International Symposium on Reliability, Distributed Software, and Databases, 207--215 . Briatico, D., Ciuffoletti, A., and Simoncini, L. 1984. A distributed domino-effect free recovery algorithm. In IEEE International Symposium on Reliability, Distributed Software, and Databases, 207--215."},{"key":"e_1_2_1_15_1","doi-asserted-by":"crossref","first-page":"546","DOI":"10.1109\/TC.1972.5009007","article-title":"Rollback and recovery strategies for computer programs","volume":"21","author":"Chandy M.","year":"1972","unstructured":"Chandy , M. and Ramamoorthy , C. V. 1972 . Rollback and recovery strategies for computer programs . IEEE Trans. Comput. 21 , 6, 546 -- 556 . Chandy, M. and Ramamoorthy, C. V. 1972. Rollback and recovery strategies for computer programs. IEEE Trans. Comput. 21, 6, 546--556.","journal-title":"IEEE Trans. Comput."},{"key":"e_1_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/214451.214456"},{"key":"e_1_2_1_17_1","volume-title":"Proceedings, Tenth Symposium on Reliable Distributed Systems, 12--20","author":"Cristian F.","unstructured":"Cristian , F. and Jahanian , F . 1991. A timestamp-based checkpointing protocol for long-lived distributed computations . In Proceedings, Tenth Symposium on Reliable Distributed Systems, 12--20 . Cristian, F. and Jahanian, F. 1991. A timestamp-based checkpointing protocol for long-lived distributed computations. In Proceedings, Tenth Symposium on Reliable Distributed Systems, 12--20."},{"key":"e_1_2_1_18_1","unstructured":"Elnozahy E. N. 1993. Manetho: Fault Tolerance in Distributed Systems using Rollback-Recovery and Process Replication. Ph.D. Thesis Rice University Department of Computer Science. Elnozahy E. N. 1993. Manetho: Fault Tolerance in Distributed Systems using Rollback-Recovery and Process Replication. Ph.D. Thesis Rice University Department of Computer Science."},{"key":"e_1_2_1_19_1","doi-asserted-by":"crossref","unstructured":"Elnozahy E. N. 1998. How safe is probabilistic checkpointing? In Digest of Papers FTCS-28 the Twenty Eight Annual International Symposium on Fault-Tolerant Computing 358--363. Elnozahy E. N. 1998. How safe is probabilistic checkpointing? In Digest of Papers FTCS-28 the Twenty Eight Annual International Symposium on Fault-Tolerant Computing 358--363.","DOI":"10.1109\/FTCS.1998.689486"},{"key":"e_1_2_1_20_1","volume-title":"The Twenty Fourth International Symposium on Fault-Tolerant Computing, 298--307","author":"Elnozahy E. N.","unstructured":"Elnozahy , E. N. and Zwaenepoel , W . 1994. On the use and implementing of message logging. In Digest of Papers, FTCS-24 , The Twenty Fourth International Symposium on Fault-Tolerant Computing, 298--307 . Elnozahy, E. N. and Zwaenepoel, W. 1994. On the use and implementing of message logging. In Digest of Papers, FTCS-24, The Twenty Fourth International Symposium on Fault-Tolerant Computing, 298--307."},{"key":"e_1_2_1_21_1","volume-title":"Proceedings, Eleventh Symposium on Reliable Distributed Systems, 39--47","author":"Elnozahy E. N.","unstructured":"Elnozahy , E. N. , Johnson , D. B. , and Zwaenepoel , W . 1992. The performance of consistent checkpointing . In Proceedings, Eleventh Symposium on Reliable Distributed Systems, 39--47 . Elnozahy, E. N., Johnson, D. B., and Zwaenepoel, W. 1992. The performance of consistent checkpointing. In Proceedings, Eleventh Symposium on Reliable Distributed Systems, 39--47."},{"key":"e_1_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/69215.69226"},{"key":"e_1_2_1_23_1","volume-title":"Usenix Mach Workshop Proceedings, 169--184","author":"Goldberg A.","unstructured":"Goldberg , A. , Gopal , A. , Li , K. , Strom , R. , and Bacon , D . 1990. Transparent recovery of Mach applications . In Usenix Mach Workshop Proceedings, 169--184 . Goldberg, A., Gopal, A., Li, K., Strom, R., and Bacon, D. 1990. Transparent recovery of Mach applications. In Usenix Mach Workshop Proceedings, 169--184."},{"key":"e_1_2_1_24_1","volume-title":"Proceedings of the 11th Workshop on Distributed Algorithms, WDAG'97.","author":"H\u00e9lary J. M.","unstructured":"H\u00e9lary , J. M. , Mostefaoui , A. , and Raynal , M . 1997a. Virtual precedence in asynchronous systems: concepts and applications . In Proceedings of the 11th Workshop on Distributed Algorithms, WDAG'97. H\u00e9lary, J. M., Mostefaoui, A., and Raynal, M. 1997a. Virtual precedence in asynchronous systems: concepts and applications. In Proceedings of the 11th Workshop on Distributed Algorithms, WDAG'97."},{"key":"e_1_2_1_25_1","volume-title":"Proceedings, Sixteenth Symposium on Reliable Distributed Systems, 183--190","author":"H\u00e9lary J. M.","unstructured":"H\u00e9lary , J. M. , Mostefaoui , A. , Netzer , R. H. , and Raynal , M . 1997b. Preventing useless checkpoints in distributed computations . In Proceedings, Sixteenth Symposium on Reliable Distributed Systems, 183--190 . H\u00e9lary, J. M., Mostefaoui, A., Netzer, R. H., and Raynal, M. 1997b. Preventing useless checkpoints in distributed computations. In Proceedings, Sixteenth Symposium on Reliable Distributed Systems, 183--190."},{"key":"e_1_2_1_26_1","doi-asserted-by":"crossref","unstructured":"Huang Y. and Kintala C. 1993. Software implemented fault tolerance: Technologies and experience. In Digest of Papers FTCS-23 the Twenty Third Annual International Symposium on Fault-Tolerant Computing 2--9. Huang Y. and Kintala C. 1993. Software implemented fault tolerance: Technologies and experience. In Digest of Papers FTCS-23 the Twenty Third Annual International Symposium on Fault-Tolerant Computing 2--9.","DOI":"10.1109\/FTCS.1993.627302"},{"key":"e_1_2_1_27_1","volume-title":"-M","author":"Huang Y.","year":"1995","unstructured":"Huang , Y. and Wang , Y . -M . 1995 . Why optimistic message logging has not been used in telecommunication systems. In Digest of Papers, FTCS-25, the Twenty Fifth Annual International Symposium on Fault-Tolerant Computing , 459--463. Huang, Y. and Wang, Y.-M. 1995. Why optimistic message logging has not been used in telecommunication systems. In Digest of Papers, FTCS-25, the Twenty Fifth Annual International Symposium on Fault-Tolerant Computing, 459--463."},{"key":"e_1_2_1_29_1","volume-title":"The Seventeenth Annual International Symposium on Fault-Tolerant Computing, 14--19","author":"Johnson D. B.","unstructured":"Johnson , D. B. and Zwaenepoel , W . 1987. Sender-based message logging. In Digest of Papers, FTCS-17 , The Seventeenth Annual International Symposium on Fault-Tolerant Computing, 14--19 . Johnson, D. B. and Zwaenepoel, W. 1987. Sender-based message logging. In Digest of Papers, FTCS-17, The Seventeenth Annual International Symposium on Fault-Tolerant Computing, 14--19."},{"key":"e_1_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1016\/0196-6774(90)90022-7"},{"key":"e_1_2_1_31_1","volume-title":"Proceedings, The 11th International Conference on Distributed Computing Systems, 454--461","author":"Juang T. T.-Y.","unstructured":"Juang , T. T.-Y. and Venkatesan , S . 1991. Crash recovery with little overhead . In Proceedings, The 11th International Conference on Distributed Computing Systems, 454--461 . Juang, T. T.-Y. and Venkatesan, S. 1991. Crash recovery with little overhead. In Proceedings, The 11th International Conference on Distributed Computing Systems, 454--461."},{"key":"e_1_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.1987.232562"},{"key":"e_1_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1016\/0020-0190(87)90125-6"},{"key":"e_1_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/359545.359563"},{"key":"e_1_2_1_35_1","unstructured":"Lampson B. W. and Sturgis H. E. 1979. Crash recovery in a distributed data storage system. Technical Report Xerox Palo Alto Research Center. Lampson B. W. and Sturgis H. E. 1979. Crash recovery in a distributed data storage system. Technical Report Xerox Palo Alto Research Center."},{"key":"e_1_2_1_36_1","volume-title":"The Twentieth Annual International Symposium on Fault-Tolerant Computing, 74--81","author":"Li C. C.","unstructured":"Li , C. C. and Fuchs , W. K . 1990. CATCH: Compiler-assisted techniques for checkpointing. In Digest of Papers, FTCS-20 , The Twentieth Annual International Symposium on Fault-Tolerant Computing, 74--81 . Li, C. C. and Fuchs, W. K. 1990. CATCH: Compiler-assisted techniques for checkpointing. In Digest of Papers, FTCS-20, The Twentieth Annual International Symposium on Fault-Tolerant Computing, 74--81."},{"key":"e_1_2_1_37_1","volume-title":"Proceedings of the Third International Conference on Architectural Support for Programming Languages and Operating Systems, 78--86","author":"Mellor-Crummey J.","year":"1989","unstructured":"Mellor-Crummey , J. and LeBlanc , T. 1989 . A software instruction counter . In Proceedings of the Third International Conference on Architectural Support for Programming Languages and Operating Systems, 78--86 . 10.1145\/70082.68189 Mellor-Crummey, J. and LeBlanc, T. 1989. A software instruction counter. In Proceedings of the Third International Conference on Architectural Support for Programming Languages and Operating Systems, 78--86. 10.1145\/70082.68189"},{"key":"e_1_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1109\/71.615441"},{"key":"e_1_2_1_39_1","volume-title":"Computer Science: Dependable Computing, EDDC-1, 491--508.","author":"Muller G.","year":"1994","unstructured":"Muller , G. , Hue , M. , and Peyrouz , N . 1994 . Performance of consistent checkpointing in a modular operating system: Results of the FTM experiment. In Lecture Notes in Computer Science: Dependable Computing, EDDC-1, 491--508. Muller, G., Hue, M., and Peyrouz, N. 1994. Performance of consistent checkpointing in a modular operating system: Results of the FTM experiment. In Lecture Notes in Computer Science: Dependable Computing, EDDC-1, 491--508."},{"key":"e_1_2_1_40_1","volume-title":"The Twenty Seventh Annual International Symposium on Fault-Tolerant Computing, 48--57","author":"Nam H.-C.","unstructured":"Nam , H.-C. , Kim , J. , Hong , S. J. , and Lee , S . 1997. Probabilistic checkpointing. In Digest of Papers, FTCS-27 , The Twenty Seventh Annual International Symposium on Fault-Tolerant Computing, 48--57 . Nam, H.-C., Kim, J., Hong, S. J., and Lee, S. 1997. Probabilistic checkpointing. In Digest of Papers, FTCS-27, The Twenty Seventh Annual International Symposium on Fault-Tolerant Computing, 48--57."},{"key":"e_1_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1109\/71.342127"},{"key":"e_1_2_1_44_1","volume-title":"The Twenty Fourth Annual International Symposium on Fault-Tolerant Computing, 288--297","author":"Plank J. S.","unstructured":"Plank , J. S. and Li , K . 1994. Faster checkpointing with N + 1 parity. In Digest of Papers, FTCS-24 , The Twenty Fourth Annual International Symposium on Fault-Tolerant Computing, 288--297 . Plank, J. S. and Li, K. 1994. Faster checkpointing with N + 1 parity. In Digest of Papers, FTCS-24, The Twenty Fourth Annual International Symposium on Fault-Tolerant Computing, 288--297."},{"key":"e_1_2_1_45_1","volume-title":"Technical Report CS-95-302, University of Tennessee at Knoxville.","author":"Plank J. S.","year":"1995","unstructured":"Plank , J. S. , Xu , J. , and Netzer , R. H . 1995 a. Compressed differences: An algorithm for fast incremental checkpointing. Technical Report CS-95-302, University of Tennessee at Knoxville. Plank, J. S., Xu, J., and Netzer, R. H. 1995a. Compressed differences: An algorithm for fast incremental checkpointing. Technical Report CS-95-302, University of Tennessee at Knoxville."},{"key":"e_1_2_1_46_1","volume-title":"Proceedings of the USENIX","author":"Plank J. S.","year":"1995","unstructured":"Plank , J. S. , Beck , M. , Kingsley , G. , and Li , K . 1995b. Libckpt: Transparent checkpointing under UNIX . In Proceedings of the USENIX Winter 1995 Technical Conference, 213--223. Plank, J. S., Beck, M., Kingsley, G., and Li, K. 1995b. Libckpt: Transparent checkpointing under UNIX. In Proceedings of the USENIX Winter 1995 Technical Conference, 213--223."},{"key":"e_1_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1109\/71.539735"},{"key":"e_1_2_1_48_1","doi-asserted-by":"crossref","first-page":"220","DOI":"10.1109\/TSE.1975.6312842","article-title":"System structure for software fault tolerance","volume":"1","author":"Randell B.","year":"1975","unstructured":"Randell , B. 1975 . System structure for software fault tolerance . IEEE Trans. Softw. Engin. 1 , 2, 220 -- 232 . Randell, B. 1975. System structure for software fault tolerance. IEEE Trans. Softw. Engin. 1, 2, 220--232.","journal-title":"IEEE Trans. Softw. Engin."},{"key":"e_1_2_1_49_1","volume-title":"Proceedings, Seventeenth Symposium on Reliable Distributed Systems, 10--18","author":"Rao S.","unstructured":"Rao , S. , Alvisi , L. , and Vin , H. M . 1998. The cost of recovery in message logging protocols . In Proceedings, Seventeenth Symposium on Reliable Distributed Systems, 10--18 . Rao, S., Alvisi, L., and Vin, H. M. 1998. The cost of recovery in message logging protocols. In Proceedings, Seventeenth Symposium on Reliable Distributed Systems, 10--18."},{"key":"e_1_2_1_50_1","volume-title":"Proceedings, Eleventh Symposium on Reliable Distributed Systems, 139--148","author":"Ruffin M.","year":"1992","unstructured":"Ruffin , M. 1992 . KITLOG: A generic logging service . In Proceedings, Eleventh Symposium on Reliable Distributed Systems, 139--148 . Ruffin, M. 1992. KITLOG: A generic logging service. In Proceedings, Eleventh Symposium on Reliable Distributed Systems, 139--148."},{"key":"e_1_2_1_51_1","doi-asserted-by":"crossref","first-page":"183","DOI":"10.1109\/TSE.1980.230469","article-title":"State restoration in systems of communicating processes","volume":"6","author":"Russell D. L.","year":"1980","unstructured":"Russell , D. L. 1980 . State restoration in systems of communicating processes . IEEE Trans. Softw. Engin. 6 , 2, 183 -- 194 . Russell, D. L. 1980. State restoration in systems of communicating processes. IEEE Trans. Softw. Engin. 6, 2, 183--194.","journal-title":"IEEE Trans. Softw. Engin."},{"key":"e_1_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1145\/357369.357371"},{"key":"e_1_2_1_54_1","volume-title":"Proceedings of the 8th Annual ACM Symposium on Principles of Distributed Computing (PODC), 223--238","author":"Sistla A.","unstructured":"Sistla , A. and Welch , J . 1989. Efficient distributed recovery using message logging . In Proceedings of the 8th Annual ACM Symposium on Principles of Distributed Computing (PODC), 223--238 . 10.1145\/72981.72997 Sistla, A. and Welch, J. 1989. Efficient distributed recovery using message logging. In Proceedings of the 8th Annual ACM Symposium on Principles of Distributed Computing (PODC), 223--238. 10.1145\/72981.72997"},{"key":"e_1_2_1_55_1","doi-asserted-by":"publisher","DOI":"10.1109\/12.729794"},{"key":"e_1_2_1_56_1","volume-title":"Proceedings, the Fifteenth Symposium on Reliable Distributed Systems, 66--75","author":"Smith S. W.","unstructured":"Smith , S. W. and Johnson , D. B . 1996. Minimizing timestamp size for completely asynchronous optimistic recovery with minimal rollback . In Proceedings, the Fifteenth Symposium on Reliable Distributed Systems, 66--75 . Smith, S. W. and Johnson, D. B. 1996. Minimizing timestamp size for completely asynchronous optimistic recovery with minimal rollback. In Proceedings, the Fifteenth Symposium on Reliable Distributed Systems, 66--75."},{"key":"e_1_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/3959.3962"},{"key":"e_1_2_1_58_1","volume-title":"Proceedings of the International Conference on Parallel Processing, 32--41","author":"Tamir Y.","unstructured":"Tamir , Y. and Sequin , C. H . 1984. Error recovery in multicomputers using global checkpoints . In Proceedings of the International Conference on Parallel Processing, 32--41 . Tamir, Y. and Sequin, C. H. 1984. Error recovery in multicomputers using global checkpoints. In Proceedings of the International Conference on Parallel Processing, 32--41."},{"key":"e_1_2_1_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/71.127264"},{"key":"e_1_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1109\/12.588059"},{"key":"e_1_2_1_62_1","unstructured":"Wang Y.-M. Chung P. Y. and Fuchs W. K. 1995a. Tight upper bound on useful distributed system checkpoints. Technical Report University of Illinois. Wang Y.-M. Chung P. Y. and Fuchs W. K. 1995a. Tight upper bound on useful distributed system checkpoints. Technical Report University of Illinois."},{"key":"e_1_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1109\/71.382324"}],"container-title":["ACM Computing Surveys"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/568522.568525","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/568522.568525","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T14:43:39Z","timestamp":1750257819000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/568522.568525"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2002,9]]},"references-count":56,"journal-issue":{"issue":"3","published-print":{"date-parts":[[2002,9]]}},"alternative-id":["10.1145\/568522.568525"],"URL":"https:\/\/doi.org\/10.1145\/568522.568525","relation":{},"ISSN":["0360-0300","1557-7341"],"issn-type":[{"value":"0360-0300","type":"print"},{"value":"1557-7341","type":"electronic"}],"subject":[],"published":{"date-parts":[[2002,9]]},"assertion":[{"value":"2002-09-01","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1162_2Fcoli_a_00486 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,27]],"date-time":"2026-10-27T16:58:31Z","timestamp":1793120311306,"version":"4.5.0"},"reference-count":42,"publisher":"MIT Press","issue":"4","license":[{"start":{"date-parts":[[2023,6,20]],"date-time":"2023-06-20T00:00:00Z","timestamp":1687219200000},"content-version":"vor","delay-in-days":170,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc-nd\/4.0\/"}],"content-domain":{"domain":["direct.mit.edu"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2023,12,1]]},"abstract":"<jats:title>Abstract<\/jats:title>\n <jats:p>Large neural models have brought a new challenge to natural language generation (NLG): It has become imperative to ensure the safety and reliability of the output of models that generate freely. To this end, we present an evaluation framework, Attributable to Identified Sources (AIS), stipulating that NLG output pertaining to the external world is to be verified against an independent, provided source. We define AIS and a two-stage annotation pipeline for allowing annotators to evaluate model output according to annotation guidelines. We successfully validate this approach on generation datasets spanning three tasks (two conversational QA datasets, a summarization dataset, and a table-to-text dataset). We provide full annotation guidelines in the appendices and publicly release the annotated data at https:\/\/github.com\/google-research-datasets\/AIS.<\/jats:p>","DOI":"10.1162\/coli_a_00486","type":"journal-article","created":{"date-parts":[[2023,6,20]],"date-time":"2023-06-20T20:19:46Z","timestamp":1687292386000},"page":"777-840","update-policy":"https:\/\/doi.org\/10.1162\/mitpressjournals.corrections.policy","source":"Crossref","is-referenced-by-count":74,"title":["Measuring Attribution in Natural Language Generation Models"],"prefix":"10.1162","volume":"49","author":[{"given":"Hannah","family":"Rashkin","sequence":"first","affiliation":[{"name":"Google DeepMind. hrashkin@google.com"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Vitaly","family":"Nikolaev","sequence":"additional","affiliation":[{"name":"Google DeepMind. vitalyn@google.com"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Matthew","family":"Lamm","sequence":"additional","affiliation":[{"name":"Google DeepMind. mrlamm@google.com"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lora","family":"Aroyo","sequence":"additional","affiliation":[{"name":"Google Research. loraa@google.com"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Michael","family":"Collins","sequence":"additional","affiliation":[{"name":"Google DeepMind. mjcollins@google.com"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Dipanjan","family":"Das","sequence":"additional","affiliation":[{"name":"Google DeepMind. dipanjand@google.com"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Slav","family":"Petrov","sequence":"additional","affiliation":[{"name":"Google DeepMind. slav@google.com"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Gaurav Singh","family":"Tomar","sequence":"additional","affiliation":[{"name":"Google DeepMind. gtomar@google.com"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Iulia","family":"Turc","sequence":"additional","affiliation":[{"name":"Storia AI. iulia@iuliaturc.com"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"David","family":"Reitter","sequence":"additional","affiliation":[{"name":"Google DeepMind. reitter@google.com"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"281","published-online":{"date-parts":[[2023,12,1]]},"reference":[{"key":"2024013020233352600_bib1","article-title":"Towards a human-like open-domain chatbot","author":"Adiwardana","year":"2020","journal-title":"arXiv preprint arXiv:2001.09977"},{"key":"2024013020233352600_bib2","doi-asserted-by":"publisher","first-page":"520","DOI":"10.18653\/v1\/2021.naacl-main.44","article-title":"Open-domain question answering goes conversational via question rewriting","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","author":"Anantha","year":"2021"},{"key":"2024013020233352600_bib3","doi-asserted-by":"crossref","first-page":"183","DOI":"10.18653\/v1\/2020.inlg-1.24","article-title":"Disentangling the properties of human evaluation methods: A classification system to support comparability, meta-evaluation and reproducibility testing","volume-title":"Proceedings of the 13th International Conference on Natural Language Generation","author":"Belz","year":"2020"},{"key":"2024013020233352600_bib4","first-page":"155","article-title":"Implicature, explicature, and truth-theoretic semantics","volume-title":"Mental Representations: The Interface Between Language and Reality","author":"Carston","year":"1988"},{"key":"2024013020233352600_bib5","doi-asserted-by":"publisher","first-page":"2174","DOI":"10.18653\/v1\/D18-1241","article-title":"QuAC: Question answering in context","volume-title":"Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing","author":"Choi","year":"2018"},{"key":"2024013020233352600_bib6","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1162\/tacl_a_00377","article-title":"Decontextualization: Making sentences stand-alone","volume":"9","author":"Choi","year":"2021","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"2024013020233352600_bib7","doi-asserted-by":"publisher","first-page":"1985","DOI":"10.1145\/3397271.3401206","article-title":"Cast-19: A dataset for conversational information seeking","author":"Dalton","year":"2020","journal-title":"Proceedings of SIGIR"},{"key":"2024013020233352600_bib8","article-title":"Wizard of Wikipedia: Knowledge-powered conversational agents","volume-title":"Proceedings of the International Conference on Learning Representations (ICLR)","author":"Dinan","year":"2019"},{"key":"2024013020233352600_bib9","doi-asserted-by":"publisher","first-page":"5055","DOI":"10.18653\/v1\/2020.acl-main.454","article-title":"FEQA: A question answering evaluation framework for faithfulness assessment in abstractive summarization","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Durmus","year":"2020"},{"key":"2024013020233352600_bib10","doi-asserted-by":"publisher","first-page":"1066","DOI":"10.1162\/tacl_a_00506","article-title":"Evaluating attribution in dialogue systems: The BEGIN benchmark","volume":"10","author":"Dziri","year":"2022","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"2024013020233352600_bib11","doi-asserted-by":"publisher","first-page":"96","DOI":"10.18653\/v1\/2021.gem-1.10","article-title":"The GEM benchmark: Natural language generation, its evaluation and metrics","volume-title":"Proceedings of the 1st Workshop on Natural Language Generation, Evaluation, and Metrics (GEM 2021)","author":"Gehrmann","year":"2021"},{"issue":"1-2","key":"2024013020233352600_bib12","doi-asserted-by":"publisher","first-page":"145","DOI":"10.1111\/j.1468-0017.1992.tb00202.x","article-title":"Why the child\u2019s theory of mind really is a theory","volume":"7","author":"Gopnik","year":"1992","journal-title":"Mind & Language"},{"key":"2024013020233352600_bib13","doi-asserted-by":"publisher","first-page":"41","DOI":"10.1163\/9789004368811_003","article-title":"Logic and conversation","volume-title":"Speech Acts","author":"Grice","year":"1975"},{"key":"2024013020233352600_bib14","doi-asserted-by":"publisher","first-page":"3785","DOI":"10.18653\/v1\/2022.acl-long.263","article-title":"DialFact: A benchmark for fact-checking in dialogue","volume-title":"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","author":"Gupta","year":"2022"},{"key":"2024013020233352600_bib15","volume-title":"Harvey Friedman\u2019s Research on the Foundations of Mathematics","author":"Harrington","year":"1985"},{"key":"2024013020233352600_bib16","doi-asserted-by":"publisher","first-page":"7856","DOI":"10.18653\/v1\/2021.emnlp-main.619","article-title":"q2: Evaluating factual consistency in knowledge-grounded dialogues via question generation and question answering","volume-title":"Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing","author":"Honovich","year":"2021"},{"key":"2024013020233352600_bib17","doi-asserted-by":"crossref","first-page":"169","DOI":"10.18653\/v1\/2020.inlg-1.23","article-title":"Twenty years of confusion in human evaluation: NLG needs evaluation sheets and standardised definitions","volume-title":"Proceedings of the 13th International Conference on Natural Language Generation","author":"Howcroft","year":"2020"},{"key":"2024013020233352600_bib18","doi-asserted-by":"publisher","first-page":"452","DOI":"10.1162\/tacl_a_00276","article-title":"Natural questions: A benchmark for question answering research","volume":"7","author":"Kwiatkowski","year":"2019","journal-title":"Transactions of the Association for Computational Linguistics"},{"key":"2024013020233352600_bib19","doi-asserted-by":"publisher","first-page":"1410","DOI":"10.18653\/v1\/2022.acl-long.100","article-title":"Faithful or extractive? On mitigating the faithfulness-abstractiveness trade-off in abstractive summarization","volume-title":"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","author":"Ladhak","year":"2022"},{"key":"2024013020233352600_bib20","doi-asserted-by":"publisher","first-page":"1906","DOI":"10.18653\/v1\/2020.acl-main.173","article-title":"On faithfulness and factuality in abstractive summarization","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Maynez","year":"2020"},{"key":"2024013020233352600_bib21","doi-asserted-by":"publisher","first-page":"681","DOI":"10.18653\/v1\/2020.acl-main.64","article-title":"USR: An unsupervised and reference free evaluation metric for dialog generation","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Mehri","year":"2020"},{"key":"2024013020233352600_bib22","doi-asserted-by":"publisher","first-page":"280","DOI":"10.18653\/v1\/K16-1028","article-title":"Abstractive text summarization using sequence-to-sequence RNNs and beyond","volume-title":"Proceedings of the 20th SIGNLL Conference on Computational Natural Language Learning","author":"Nallapati","year":"2016"},{"key":"2024013020233352600_bib23","doi-asserted-by":"publisher","first-page":"6881","DOI":"10.18653\/v1\/2021.acl-long.536","article-title":"Improving factual consistency of abstractive summarization via question answering","volume-title":"Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)","author":"Nan","year":"2021"},{"issue":"01","key":"2024013020233352600_bib24","doi-asserted-by":"publisher","first-page":"6859","DOI":"10.1609\/aaai.v33i01.33016859","article-title":"Combining fact extraction and verification with neural semantic matching networks","volume":"33","author":"Nie","year":"2019","journal-title":"Proceedings of the AAAI Conference on Artificial Intelligence"},{"key":"2024013020233352600_bib25","doi-asserted-by":"publisher","first-page":"4812","DOI":"10.18653\/v1\/2021.naacl-main.383","article-title":"Understanding factuality in abstractive summarization with FRANK: A benchmark for factuality metrics","volume-title":"Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies","author":"Pagnoni","year":"2021"},{"key":"2024013020233352600_bib26","doi-asserted-by":"publisher","first-page":"1173","DOI":"10.18653\/v1\/2020.emnlp-main.89","article-title":"ToTTo: A controlled table-to-text generation dataset","volume-title":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)","author":"Parikh","year":"2020"},{"key":"2024013020233352600_bib27","doi-asserted-by":"publisher","first-page":"677","DOI":"10.1162\/tacl_a_00293","article-title":"Inherent disagreements in human textual inferences","volume":"7","author":"Pavlick","year":"2019","journal-title":"Transactions of the Association for Computational Linguistics"},{"issue":"1","key":"2024013020233352600_bib28","first-page":"5485","article-title":"Exploring the limits of transfer learning with a unified text-to-text transformer","volume":"21","author":"Raffel","year":"2020","journal-title":"Journal of Machine Learning Research"},{"key":"2024013020233352600_bib29","doi-asserted-by":"publisher","first-page":"704","DOI":"10.18653\/v1\/2021.acl-long.58","article-title":"Increasing faithfulness in knowledge-grounded dialogue with controllable features","volume-title":"Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)","author":"Rashkin","year":"2021"},{"key":"2024013020233352600_bib30","article-title":"Model-agnostic interpretability of machine learning","author":"Ribeiro","year":"2016","journal-title":"arXiv preprint arXiv:1606.05386"},{"key":"2024013020233352600_bib31","article-title":"Rome was built in 1776: A case study on factual correctness in knowledge-grounded response generation","author":"Santhanam","year":"2021","journal-title":"arXiv preprint arXiv:2110.05456"},{"key":"2024013020233352600_bib32","doi-asserted-by":"publisher","first-page":"1073","DOI":"10.18653\/v1\/P17-1099","article-title":"Get to the point: Summarization with pointer-generator networks","volume-title":"Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","author":"See","year":"2017"},{"key":"2024013020233352600_bib33","doi-asserted-by":"publisher","first-page":"2453","DOI":"10.18653\/v1\/2020.acl-main.222","article-title":"The dialogue dodecathlon: Open-domain knowledge and image grounded conversational agents","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Shuster","year":"2020"},{"key":"2024013020233352600_bib34","article-title":"Evidence-based verification for real world information needs","author":"Thorne","year":"2021","journal-title":"arXiv preprint arXiv:2104.00640"},{"key":"2024013020233352600_bib35","first-page":"3346","article-title":"Automated fact checking: Task formulations, methods and future directions","volume-title":"Proceedings of the 27th International Conference on Computational Linguistics","author":"Thorne","year":"2018"},{"key":"2024013020233352600_bib36","doi-asserted-by":"publisher","first-page":"809","DOI":"10.18653\/v1\/N18-1074","article-title":"FEVER: A large-scale dataset for fact extraction and VERification","volume-title":"Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)","author":"Thorne","year":"2018"},{"key":"2024013020233352600_bib37","doi-asserted-by":"publisher","first-page":"5008","DOI":"10.18653\/v1\/2020.acl-main.450","article-title":"Asking and answering questions to evaluate the factual consistency of summaries","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Wang","year":"2020"},{"key":"2024013020233352600_bib38","doi-asserted-by":"publisher","first-page":"3731","DOI":"10.18653\/v1\/P19-1363","article-title":"Dialogue natural language inference","volume-title":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","author":"Welleck","year":"2019"},{"key":"2024013020233352600_bib39","first-page":"607","article-title":"Relevance theory","volume-title":"The Handbook of Pragmatics","author":"Wilson","year":"2004"},{"key":"2024013020233352600_bib40","doi-asserted-by":"publisher","first-page":"2253","DOI":"10.18653\/v1\/D17-1239","article-title":"Challenges in data-to-document generation","volume-title":"Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing","author":"Wiseman","year":"2017"},{"key":"2024013020233352600_bib41","first-page":"17283","article-title":"Big Bird: Transformers for longer sequences","volume-title":"Advances in Neural Information Processing Systems","author":"Zaheer","year":"2020"},{"key":"2024013020233352600_bib42","doi-asserted-by":"publisher","first-page":"6197","DOI":"10.18653\/v1\/2020.acl-main.552","article-title":"Extractive summarization as text matching","volume-title":"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics","author":"Zhong","year":"2020"}],"container-title":["Computational Linguistics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/direct.mit.edu\/coli\/article-pdf\/49\/4\/777\/2269661\/coli_a_00486.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/direct.mit.edu\/coli\/article-pdf\/49\/4\/777\/2269661\/coli_a_00486.pdf","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,1,30]],"date-time":"2024-01-30T20:24:42Z","timestamp":1706646282000},"score":1,"resource":{"primary":{"URL":"https:\/\/direct.mit.edu\/coli\/article\/49\/4\/777\/116438\/Measuring-Attribution-in-Natural-Language"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"references-count":42,"journal-issue":{"issue":"4","published-online":{"date-parts":[[2023,12,1]]},"published-print":{"date-parts":[[2023,12,1]]}},"URL":"https:\/\/doi.org\/10.1162\/coli_a_00486","relation":{},"ISSN":["0891-2017","1530-9312"],"issn-type":[{"value":"0891-2017","type":"print"},{"value":"1530-9312","type":"electronic"}],"subject":[],"published-other":{"date-parts":[[2023]]},"published":{"date-parts":[[2023]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1177_2F0956797611417632 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T16:11:58Z","timestamp":1793290318859,"version":"4.5.0"},"reference-count":11,"publisher":"SAGE Publications","issue":"11","license":[{"start":{"date-parts":[[2011,10,17]],"date-time":"2011-10-17T00:00:00Z","timestamp":1318809600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/journals.sagepub.com\/page\/policies\/text-and-data-mining-license"}],"content-domain":{"domain":["journals.sagepub.com"],"crossmark-restriction":true},"short-container-title":["Psychol Sci"],"published-print":{"date-parts":[[2011,11]]},"abstract":"<jats:p>In this article, we accomplish two things. First, we show that despite empirical psychologists\u2019 nominal endorsement of a low rate of false-positive findings (\u2264 .05), flexibility in data collection, analysis, and reporting dramatically increases actual false-positive rates. In many cases, a researcher is more likely to falsely find evidence that an effect exists than to correctly find evidence that it does not. We present computer simulations and a pair of actual experiments that demonstrate how unacceptably easy it is to accumulate (and report) statistically significant evidence for a false hypothesis. Second, we suggest a simple, low-cost, and straightforwardly effective disclosure-based solution to this problem. The solution involves six concrete requirements for authors and four guidelines for reviewers, all of which impose a minimal burden on the publication process.<\/jats:p>","DOI":"10.1177\/0956797611417632","type":"journal-article","created":{"date-parts":[[2011,10,18]],"date-time":"2011-10-18T01:51:09Z","timestamp":1318902669000},"page":"1359-1366","update-policy":"https:\/\/doi.org\/10.1177\/sage-journals-update-policy","source":"Crossref","is-referenced-by-count":5281,"title":["False-Positive Psychology"],"prefix":"10.1177","volume":"22","author":[{"given":"Joseph P.","family":"Simmons","sequence":"first","affiliation":[{"name":"Wharton School, University of Pennsylvania"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Leif D.","family":"Nelson","sequence":"additional","affiliation":[{"name":"Haas School of Business, University of California, Berkeley"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Uri","family":"Simonsohn","sequence":"additional","affiliation":[{"name":"Wharton School, University of Pennsylvania"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"179","published-online":{"date-parts":[[2011,10,17]]},"reference":[{"key":"bibr1-0956797611417632","doi-asserted-by":"publisher","DOI":"10.1257\/jep.11.1.109"},{"key":"bibr2-0956797611417632","doi-asserted-by":"publisher","DOI":"10.1177\/014616702236869"},{"key":"bibr3-0956797611417632","doi-asserted-by":"publisher","DOI":"10.1037\/0022-3514.44.6.1110"},{"key":"bibr4-0956797611417632","doi-asserted-by":"publisher","DOI":"10.1037\/h0057880"},{"key":"bibr5-0956797611417632","doi-asserted-by":"crossref","unstructured":"Ioannidis J. P. A. (2005). Why most published research findings are false. PLoS Medicine, 2(8), e124. Retrieved from http:\/\/www.plosmedicine.org\/article\/info:doi\/10.1371\/journal.pmed.0020124","DOI":"10.1371\/journal.pmed.0020124"},{"key":"bibr6-0956797611417632","doi-asserted-by":"crossref","unstructured":"John L., Loewenstein G. F., Prelec D. (2011). Measuring the prevalence of questionable research practices with incentives for truth-telling. Manuscript submitted for publication.","DOI":"10.1037\/e632032012-001"},{"key":"bibr7-0956797611417632","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.108.3.480"},{"key":"bibr8-0956797611417632","doi-asserted-by":"publisher","DOI":"10.1093\/biomet\/64.2.191"},{"key":"bibr9-0956797611417632","doi-asserted-by":"crossref","unstructured":"Schooler J. (2011). Unpublished results hide the decline effect. Nature, 470, 437.","DOI":"10.1038\/470437a"},{"key":"bibr10-0956797611417632","doi-asserted-by":"publisher","DOI":"10.1037\/a0022790"},{"key":"bibr11-0956797611417632","doi-asserted-by":"publisher","DOI":"10.1111\/j.1467-6494.1979.tb00202.x"}],"container-title":["Psychological Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/journals.sagepub.com\/doi\/pdf\/10.1177\/0956797611417632","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/journals.sagepub.com\/doi\/full-xml\/10.1177\/0956797611417632","content-type":"application\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/journals.sagepub.com\/doi\/pdf\/10.1177\/0956797611417632","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,1]],"date-time":"2026-05-01T21:18:39Z","timestamp":1777670319000},"score":1,"resource":{"primary":{"URL":"https:\/\/journals.sagepub.com\/doi\/10.1177\/0956797611417632"}},"subtitle":["Undisclosed Flexibility in Data Collection and Analysis Allows Presenting Anything as Significant"],"short-title":[],"issued":{"date-parts":[[2011,10,17]]},"references-count":11,"journal-issue":{"issue":"11","published-print":{"date-parts":[[2011,11]]}},"alternative-id":["10.1177\/0956797611417632"],"URL":"https:\/\/doi.org\/10.1177\/0956797611417632","relation":{"has-review":[{"id-type":"doi","id":"10.3410\/f.717958189.793468208","asserted-by":"object"},{"id-type":"doi","id":"10.3410\/f.717958189.793462486","asserted-by":"object"}]},"ISSN":["0956-7976","1467-9280"],"issn-type":[{"value":"0956-7976","type":"print"},{"value":"1467-9280","type":"electronic"}],"subject":[],"published":{"date-parts":[[2011,10,17]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1177_2F1745691612463078 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T21:06:51Z","timestamp":1793308011359,"version":"4.5.0"},"reference-count":45,"publisher":"SAGE Publications","issue":"6","license":[{"start":{"date-parts":[[2012,11,1]],"date-time":"2012-11-01T00:00:00Z","timestamp":1351728000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/journals.sagepub.com\/page\/policies\/text-and-data-mining-license"}],"content-domain":{"domain":["journals.sagepub.com"],"crossmark-restriction":true},"short-container-title":["Perspect Psychol Sci"],"published-print":{"date-parts":[[2012,11]]},"abstract":"<jats:p>The veracity of substantive research claims hinges on the way experimental data are collected and analyzed. In this article, we discuss an uncomfortable fact that threatens the core of psychology\u2019s academic enterprise: almost without exception, psychologists do not commit themselves to a method of data analysis before they see the actual data. It then becomes tempting to fine tune the analysis to the data in order to obtain a desired result\u2014a procedure that invalidates the interpretation of the common statistical tests. The extent of the fine tuning varies widely across experiments and experimenters but is almost impossible for reviewers and readers to gauge. To remedy the situation, we propose that researchers preregister their studies and indicate in advance the analyses they intend to conduct. Only these analyses deserve the label \u201cconfirmatory,\u201d and only for these analyses are the common statistical tests valid. Other analyses can be carried out but these should be labeled \u201cexploratory.\u201d We illustrate our proposal with a confirmatory replication attempt of a study on extrasensory perception.<\/jats:p>","DOI":"10.1177\/1745691612463078","type":"journal-article","created":{"date-parts":[[2012,11,7]],"date-time":"2012-11-07T18:09:48Z","timestamp":1352311788000},"page":"632-638","update-policy":"https:\/\/doi.org\/10.1177\/sage-journals-update-policy","source":"Crossref","is-referenced-by-count":749,"title":["An Agenda for Purely Confirmatory Research"],"prefix":"10.1177","volume":"7","author":[{"given":"Eric-Jan","family":"Wagenmakers","sequence":"first","affiliation":[{"name":"University of Amsterdam, The Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ruud","family":"Wetzels","sequence":"additional","affiliation":[{"name":"University of Amsterdam, The Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Denny","family":"Borsboom","sequence":"additional","affiliation":[{"name":"University of Amsterdam, The Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Han L. J.","family":"van der Maas","sequence":"additional","affiliation":[{"name":"University of Amsterdam, The Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rogier A.","family":"Kievit","sequence":"additional","affiliation":[{"name":"University of Amsterdam, The Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"179","published-online":{"date-parts":[[2012,11,7]]},"reference":[{"key":"bibr1-1745691612463078","first-page":"311","volume":"29","author":"Anderson D. R.","year":"2001","journal-title":"Wildlife Society Bulletin"},{"key":"bibr2-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612459060"},{"key":"bibr3-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1038\/483531a"},{"key":"bibr4-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1037\/a0021524"},{"key":"bibr5-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1037\/a0024777"},{"key":"bibr6-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1126\/science.335.6076.1558"},{"key":"bibr7-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1177\/1745691611406920"},{"key":"bibr8-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1037\/h0044139"},{"key":"bibr9-1745691612463078","first-page":"10","volume":"37","author":"Feynman R. P.","year":"1974","journal-title":"Engineering & Science"},{"key":"bibr10-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612460686"},{"key":"bibr11-1745691612463078","author":"Galak J.","journal-title":"Journal of Personality and Social Psychology"},{"key":"bibr12-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1037\/a0015251"},{"key":"bibr13-1745691612463078","volume-title":"Bad science","author":"Goldacre B.","year":"2009"},{"key":"bibr14-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612459057"},{"key":"bibr15-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1007\/978-0-387-09612-4"},{"key":"bibr16-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.0020124"},{"key":"bibr17-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1126\/science.334.6060.1225"},{"key":"bibr18-1745691612463078","volume-title":"Theory of probability","author":"Jeffreys H.","year":"1961","edition":"3"},{"key":"bibr19-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1177\/0956797611430953"},{"key":"bibr20-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1080\/01621459.1995.10476572"},{"key":"bibr21-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1207\/s15327957pspr0203_4"},{"key":"bibr22-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1214\/aoms\/1177704038"},{"key":"bibr23-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1038\/nn.2303"},{"key":"bibr24-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.111.3.490"},{"key":"bibr25-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1037\/14716-000"},{"key":"bibr26-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612459519"},{"key":"bibr27-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1037\/1089-2680.2.2.175"},{"key":"bibr28-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612459058"},{"key":"bibr29-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1038\/scibx.2011.416"},{"key":"bibr30-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1038\/nrd3439-c1"},{"key":"bibr31-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0033423"},{"issue":"2","key":"bibr32-1745691612463078","first-page":"9","volume":"25","author":"Roediger H. L.","year":"2012","journal-title":"APS Observer"},{"key":"bibr33-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1177\/1745691612454303"},{"key":"bibr34-1745691612463078","doi-asserted-by":"publisher","DOI":"10.3758\/PBR.16.2.225"},{"key":"bibr35-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1038\/485149a"},{"key":"bibr36-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1198\/000313001300339950"},{"key":"bibr37-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1177\/0956797611417632"},{"key":"bibr38-1745691612463078","first-page":"30","volume":"54","author":"Sterling T. D.","year":"1959","journal-title":"Journal of the American Statistical Association"},{"key":"bibr39-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1080\/00031305.1995.10476125"},{"key":"bibr40-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1111\/j.1745-6924.2009.01125.x"},{"key":"bibr41-1745691612463078","doi-asserted-by":"publisher","DOI":"10.3758\/BF03194105"},{"key":"bibr42-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1037\/a0022790"},{"key":"bibr43-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1177\/1745691611406923"},{"key":"bibr44-1745691612463078","doi-asserted-by":"publisher","DOI":"10.3758\/PBR.16.4.752"},{"key":"bibr45-1745691612463078","doi-asserted-by":"publisher","DOI":"10.1038\/485298a"}],"container-title":["Perspectives on Psychological Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/journals.sagepub.com\/doi\/pdf\/10.1177\/1745691612463078","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/journals.sagepub.com\/doi\/full-xml\/10.1177\/1745691612463078","content-type":"application\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/journals.sagepub.com\/doi\/pdf\/10.1177\/1745691612463078","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,2]],"date-time":"2026-05-02T08:34:28Z","timestamp":1777710868000},"score":1,"resource":{"primary":{"URL":"https:\/\/journals.sagepub.com\/doi\/10.1177\/1745691612463078"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2012,11]]},"references-count":45,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2012,11]]}},"alternative-id":["10.1177\/1745691612463078"],"URL":"https:\/\/doi.org\/10.1177\/1745691612463078","relation":{},"ISSN":["1745-6916","1745-6924"],"issn-type":[{"value":"1745-6916","type":"print"},{"value":"1745-6924","type":"electronic"}],"subject":[],"published":{"date-parts":[[2012,11]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1177_2F17456916231181102 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,24]],"date-time":"2026-10-24T01:19:21Z","timestamp":1792804761006,"version":"4.5.0"},"reference-count":92,"publisher":"SAGE Publications","issue":"5","license":[{"start":{"date-parts":[[2023,7,13]],"date-time":"2023-07-13T00:00:00Z","timestamp":1689206400000},"content-version":"unspecified","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["journals.sagepub.com"],"crossmark-restriction":true},"short-container-title":["Perspect Psychol Sci"],"published-print":{"date-parts":[[2024,9]]},"abstract":"<jats:p>Artificial intelligence (AI) has the potential to improve human decision-making by providing decision recommendations and problem-relevant information to assist human decision-makers. However, the full realization of the potential of human\u2013AI collaboration continues to face several challenges. First, the conditions that support complementarity (i.e., situations in which the performance of a human with AI assistance exceeds the performance of an unassisted human or the AI in isolation) must be understood. This task requires humans to be able to recognize situations in which the AI should be leveraged and to develop new AI systems that can learn to complement the human decision-maker. Second, human mental models of the AI, which contain both expectations of the AI and reliance strategies, must be accurately assessed. Third, the effects of different design choices for human-AI interaction must be understood, including both the timing of AI assistance and the amount of model information that should be presented to the human decision-maker to avoid cognitive overload and ineffective reliance strategies. In response to each of these three challenges, we present an interdisciplinary perspective based on recent empirical and theoretical findings and discuss new research directions.<\/jats:p>","DOI":"10.1177\/17456916231181102","type":"journal-article","created":{"date-parts":[[2023,7,13]],"date-time":"2023-07-13T10:27:45Z","timestamp":1689244065000},"page":"722-734","update-policy":"https:\/\/doi.org\/10.1177\/sage-journals-update-policy","source":"Crossref","is-referenced-by-count":209,"title":["Three Challenges for AI-Assisted Decision-Making"],"prefix":"10.1177","volume":"19","author":[{"given":"Mark","family":"Steyvers","sequence":"first","affiliation":[{"name":"Department of Cognitive Sciences, University of California, Irvine"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Aakriti","family":"Kumar","sequence":"additional","affiliation":[{"name":"Department of Cognitive Sciences, University of California, Irvine"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"179","published-online":{"date-parts":[[2023,7,13]]},"reference":[{"key":"bibr1-17456916231181102","first-page":"8","author":"Abraham H.","year":"2017","journal-title":"Proceedings of the Transportation Research Board 96th Annual Meeting"},{"issue":"3","key":"bibr2-17456916231181102","first-page":"3","volume":"1","author":"Aggarwal I.","year":"2015","journal-title":"Proceedings of Collective Intelligence"},{"key":"bibr3-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1016\/0749-5978(86)90046-4"},{"key":"bibr4-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v35i13.17359"},{"key":"bibr5-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v7i1.5285"},{"key":"bibr6-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445717"},{"key":"bibr7-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1002\/aaai.12085"},{"key":"bibr8-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3461702.3462571"},{"key":"bibr9-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1016\/j.cognition.2018.08.003"},{"key":"bibr10-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1016\/j.obhdp.2006.07.001"},{"key":"bibr11-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3377325.3377498"},{"key":"bibr12-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3449287"},{"key":"bibr13-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1002\/bdm.2155"},{"key":"bibr14-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1007\/s10614-020-10042-0"},{"key":"bibr15-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300234"},{"key":"bibr16-17456916231181102","doi-asserted-by":"publisher","DOI":"10.31234\/osf.io\/7ahdc"},{"key":"bibr17-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2117432119"},{"key":"bibr18-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1177\/0022243719851788"},{"key":"bibr19-17456916231181102","doi-asserted-by":"crossref","unstructured":"Chen C., Feng S., Sharma A., Tan C. (2022). Machine explanations and human understanding. arXiv. https:\/\/doi.org\/10.48550\/arXiv.2202.04092","DOI":"10.1145\/3593013.3593970"},{"key":"bibr20-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1016\/j.chb.2021.107018"},{"key":"bibr21-17456916231181102","volume-title":"The alignment problem: Machine learning and human values","author":"Christian B.","year":"2020"},{"key":"bibr22-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1016\/j.socec.2022.101897"},{"key":"bibr23-17456916231181102","unstructured":"Consul S., Stojcheski J., Felso V., Lieder F. (2022). Leveraging AI for effective to-do list gamification. In Marky K., Gr\u00fcnefeld U., Kosch T. (Eds.), Mensch und Computer 2022 - Workshopband. Gesellschaft f\u00fcr Informatik e.V. https:\/\/doi.org\/10.18420\/muc2022-mci-ws08-241"},{"key":"bibr24-17456916231181102","volume-title":"The nature of explanation","volume":"445","author":"Craik K. J. W.","year":"1952"},{"key":"bibr25-17456916231181102","doi-asserted-by":"publisher","DOI":"10.4324\/9781315095080-17"},{"key":"bibr26-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1287\/deca.2015.0315"},{"key":"bibr27-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v34i03.5645"},{"key":"bibr28-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1037\/xge0000033"},{"key":"bibr29-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.2016.2643"},{"key":"bibr30-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3302265"},{"key":"bibr31-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1007\/s11023-020-09539-2"},{"key":"bibr32-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3490099.3511138"},{"key":"bibr33-17456916231181102","first-page":"251","volume-title":"The Oxford handbook of ethics of AI","author":"Gebru T.","year":"2020"},{"key":"bibr34-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1126\/science.aac6076"},{"key":"bibr35-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1002\/bdm.539"},{"key":"bibr36-17456916231181102","doi-asserted-by":"publisher","DOI":"10.5465\/annals.2018.0057"},{"key":"bibr37-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3359152"},{"key":"bibr38-17456916231181102","doi-asserted-by":"crossref","unstructured":"Grgi\u0107-Hla\u010da N., Castelluccia C., Gummadi K. P. (2022). Taking advice from (dis)similar machines: The impact of human-machine similarity on machine-assisted decision-making. arXiv https:\/\/doi.org\/10.48550\/arXiv.2209.03821","DOI":"10.1609\/hcomp.v10i1.21989"},{"key":"bibr39-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3359280"},{"key":"bibr40-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1016\/j.jmp.2022.102695"},{"key":"bibr41-17456916231181102","unstructured":"Hoffman R. R., Mueller S. T., Klein G., Litman J. (2018). Metrics for explainable AI: Challenges and prospects. arXiv. https:\/\/doi.org\/10.48550\/arXiv.1812.04608"},{"key":"bibr42-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.0403723101"},{"key":"bibr43-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1703695114"},{"key":"bibr44-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1038\/s44159-022-00054-y"},{"key":"bibr45-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1186\/s12916-019-1426-2"},{"key":"bibr46-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3593013.3594111"},{"key":"bibr47-17456916231181102","first-page":"483","volume-title":"Dual-process theories in social psychology","author":"Koriat A.","year":"1999"},{"key":"bibr48-17456916231181102","doi-asserted-by":"crossref","unstructured":"Kumar A., Patel T., Benjamin A. S., Steyvers M. (2021). Explaining algorithm aversion with metacognitive bandits. Proceedings of the Annual Meeting of the Cognitive Science Society, 43. https:\/\/escholarship.org\/uc\/item\/7xc470dt","DOI":"10.1038\/s41539-022-00142-x"},{"key":"bibr49-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3501999"},{"key":"bibr50-17456916231181102","unstructured":"Lai V., Chen C., Liao Q. V., Smith-Renner A., Tan C. (2021). Towards a science of human-AI decision making: A survey of empirical studies. arXiv. https:\/\/doi.org\/10.48550\/arXiv.2112.11471"},{"key":"bibr51-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3287560.3287590"},{"key":"bibr52-17456916231181102","unstructured":"Lakkaraju H., Slack D., Chen Y., Tan C., Singh S. (2022). Rethinking explainability as a dialogue: A practitioner\u2019s perspective. arXiv. https:\/\/doi.org\/10.48550\/arXiv.2202.01875."},{"key":"bibr53-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1518\/hfes.46.1.50_30392"},{"key":"bibr54-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1111\/tops.12086"},{"key":"bibr55-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1186\/s41235-022-00364-y"},{"key":"bibr56-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1038\/s41562-019-0672-9"},{"key":"bibr57-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1017\/S0140525X1900061X"},{"key":"bibr58-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pcbi.1006043"},{"key":"bibr59-17456916231181102","doi-asserted-by":"crossref","unstructured":"Logg J. M. (2017). Theory of machine: When do people rely on algorithms? (Working Paper No. 17-086). Harvard Business School.","DOI":"10.2139\/ssrn.2941774"},{"key":"bibr60-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1016\/j.obhdp.2018.12.005"},{"key":"bibr61-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445562"},{"key":"bibr62-17456916231181102","first-page":"57","volume-title":"Proceedings of the 33rd International Conference on Neural Information Processing Systems","author":"Lubars B.","year":"2019"},{"key":"bibr63-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1016\/j.obhdp.2006.09.002"},{"key":"bibr64-17456916231181102","first-page":"26422","volume":"34","author":"Nguyen G.","year":"2021","journal-title":"Advances in Neural Information Processing Systems"},{"key":"bibr65-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1016\/j.jacr.2017.08.027"},{"key":"bibr66-17456916231181102","first-page":"1","author":"Oulasvirta A.","year":"2022","journal-title":"CHI Conference on Human Factors in Computing Systems"},{"key":"bibr67-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1518\/001872097778543886"},{"key":"bibr68-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3359204"},{"key":"bibr69-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1016\/j.cognition.2021.104810"},{"key":"bibr70-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1037\/xge0000960"},{"key":"bibr71-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445315"},{"key":"bibr72-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3306618.3314244"},{"key":"bibr73-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1038\/s41746-020-00322-2"},{"key":"bibr74-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3512930"},{"key":"bibr75-17456916231181102","volume-title":"Human compatible: Artificial intelligence and the problem of control","author":"Russell S.","year":"2019"},{"key":"bibr76-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1037\/0003-066X.55.1.68"},{"key":"bibr77-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1016\/j.ophtha.2018.11.016"},{"key":"bibr78-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3301275.3302308"},{"key":"bibr79-17456916231181102","unstructured":"Shen Z., Liu J., He Y., Zhang X., Xu R., Yu H., Cui P. (2021). Towards out-of-distribution generalization: A survey. https:\/\/doi.org\/10.48550\/arXiv.2108.13624"},{"key":"bibr80-17456916231181102","volume-title":"Cognition in action","author":"Smyth M. M.","year":"1994"},{"key":"bibr81-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1146\/annurev-psych-010418-103211"},{"key":"bibr82-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2111547119"},{"key":"bibr83-17456916231181102","first-page":"34287","volume":"35","author":"Taesiri M. R.","year":"2022","journal-title":"Advances in Neural Information Processing Systems"},{"key":"bibr84-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1007\/s42113-022-00157-y"},{"key":"bibr85-17456916231181102","volume-title":"Nudge: Improving decisions about health, wealth and happiness","author":"Thaler R.","year":"2008"},{"key":"bibr86-17456916231181102","first-page":"4004","volume":"35","author":"Vodrahalli K.","year":"2022","journal-title":"Advances in Neural Information Processing Systems"},{"key":"bibr87-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3485447.3512240"},{"key":"bibr88-17456916231181102","doi-asserted-by":"crossref","unstructured":"Wilder B., Horvitz E., Kamar E. (2020). Learning to complement humans. arXiv. https:\/\/doi.org\/10.48550\/arXiv.2005.00582","DOI":"10.24963\/ijcai.2020\/212"},{"key":"bibr89-17456916231181102","first-page":"1526","volume-title":"Proceedings of the Twenty-Ninth International Conference on International Joint Conferences on Artificial Intelligence","author":"Wilder B.","year":"2021"},{"key":"bibr90-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300509"},{"key":"bibr91-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3491102.3517791"},{"key":"bibr92-17456916231181102","doi-asserted-by":"publisher","DOI":"10.1145\/3351095.3372852"}],"container-title":["Perspectives on Psychological Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/journals.sagepub.com\/doi\/pdf\/10.1177\/17456916231181102","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/journals.sagepub.com\/doi\/full-xml\/10.1177\/17456916231181102","content-type":"application\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/journals.sagepub.com\/doi\/pdf\/10.1177\/17456916231181102","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,5,2]],"date-time":"2026-05-02T08:44:03Z","timestamp":1777711443000},"score":1,"resource":{"primary":{"URL":"https:\/\/journals.sagepub.com\/doi\/10.1177\/17456916231181102"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,7,13]]},"references-count":92,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2024,9]]}},"alternative-id":["10.1177\/17456916231181102"],"URL":"https:\/\/doi.org\/10.1177\/17456916231181102","relation":{},"ISSN":["1745-6916","1745-6924"],"issn-type":[{"value":"1745-6916","type":"print"},{"value":"1745-6924","type":"electronic"}],"subject":[],"published":{"date-parts":[[2023,7,13]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1177_2F25152459211007467 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T17:44:11Z","timestamp":1793295851721,"version":"4.5.0"},"reference-count":52,"publisher":"SAGE Publications","issue":"2","license":[{"start":{"date-parts":[[2021,4,1]],"date-time":"2021-04-01T00:00:00Z","timestamp":1617235200000},"content-version":"unspecified","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by-nc\/4.0\/"}],"content-domain":{"domain":["journals.sagepub.com"],"crossmark-restriction":true},"short-container-title":["Advances in Methods and Practices in Psychological Science"],"published-print":{"date-parts":[[2021,4]]},"abstract":"<jats:p>Selectively publishing results that support the tested hypotheses (\u201cpositive\u201d results) distorts the available evidence for scientific claims. For the past decade, psychological scientists have been increasingly concerned about the degree of such distortion in their literature. A new publication format has been developed to prevent selective reporting: In Registered Reports (RRs), peer review and the decision to publish take place before results are known. We compared the results in published RRs ( N = 71 as of November 2018) with a random sample of hypothesis-testing studies from the standard literature ( N = 152) in psychology. Analyzing the first hypothesis of each article, we found 96% positive results in standard reports but only 44% positive results in RRs. We discuss possible explanations for this large difference and suggest that a plausible factor is the reduction of publication bias and\/or Type I error inflation in the RR literature.<\/jats:p>","DOI":"10.1177\/25152459211007467","type":"journal-article","created":{"date-parts":[[2021,4,16]],"date-time":"2021-04-16T12:53:03Z","timestamp":1618577583000},"update-policy":"https:\/\/doi.org\/10.1177\/sage-journals-update-policy","source":"Crossref","is-referenced-by-count":263,"title":["An Excess of Positive Results: Comparing the Standard Psychology Literature With Registered Reports"],"prefix":"10.1177","volume":"4","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-6627-0746","authenticated-orcid":false,"given":"Anne M.","family":"Scheel","sequence":"first","affiliation":[{"name":"Human-Technology Interaction Group, Eindhoven University of Technology, Eindhoven, The Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mitchell R. M. J.","family":"Schijen","sequence":"additional","affiliation":[{"name":"Human-Technology Interaction Group, Eindhoven University of Technology, Eindhoven, The Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-0247-239X","authenticated-orcid":false,"given":"Dani\u00ebl","family":"Lakens","sequence":"additional","affiliation":[{"name":"Human-Technology Interaction Group, Eindhoven University of Technology, Eindhoven, The Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"179","published-online":{"date-parts":[[2021,4,16]]},"reference":[{"key":"bibr1-25152459211007467","doi-asserted-by":"crossref","unstructured":"Agnoli F., Wicherts J. M., Veldkamp C. L. S., Albiero P., Cubelli R. (2017). Questionable research practices among italian research psychologists. PLOS ONE, 12(3), Article e0172792. https:\/\/doi.org\/10.1371\/journal.pone.0172792","DOI":"10.1371\/journal.pone.0172792"},{"key":"bibr2-25152459211007467","doi-asserted-by":"crossref","unstructured":"Allen C., Mehler D. M. A. (2019). Open science challenges, benefits and tips in early career and beyond. PLOS Biology, 17(5), Article e3000246. https:\/\/doi.org\/10.1371\/journal.pbio.3000246","DOI":"10.1371\/journal.pbio.3000246"},{"key":"bibr3-25152459211007467","doi-asserted-by":"crossref","unstructured":"Arslan R. C. (2018). How to automatically generate rich codebooks from study metadata. PsyArxiv. https:\/\/doi.org\/10.31234\/osf.io\/5qc6h","DOI":"10.31234\/osf.io\/5qc6h"},{"key":"bibr4-25152459211007467","doi-asserted-by":"crossref","unstructured":"Atkinson D. R., Furlong M. J., Wampold B. E. (1982). Statistical significance, reviewer evaluations, and the scientific process: Is there a (statistically) significant relationship? Journal of Counseling Psychology, 29(2), 189\u2013194. https:\/\/doi.org\/10.1037\/0022-0167.29.2.189","DOI":"10.1037\/\/0022-0167.29.2.189"},{"key":"bibr5-25152459211007467","unstructured":"Aust F., Barth M. (2018). papaja: Create APA manuscripts with R Markdown. https:\/\/github.com\/crsh\/papaja"},{"key":"bibr6-25152459211007467","doi-asserted-by":"crossref","unstructured":"Baumeister R. F. (2016). Charting the future of social psychology on stormy seas: Winners, losers, and recommendations. Journal of Experimental Social Psychology, 66, 153\u2013158. https:\/\/doi.org\/10.1016\/j.jesp.2016.02.003","DOI":"10.1016\/j.jesp.2016.02.003"},{"key":"bibr7-25152459211007467","doi-asserted-by":"crossref","unstructured":"Chambers C. D., Tzavella L. (2020). Registered Reports: Past, present and future. MetaArXiv. https:\/\/doi.org\/10.31222\/osf.io\/43298","DOI":"10.31222\/osf.io\/43298"},{"key":"bibr8-25152459211007467","unstructured":"Chan C.h., Chan G. C., Leeper T. J., Becker J. (2018). Rio: A swiss-army knife for data file i\/o. https:\/\/cran.r-project.org\/web\/packages\/rio\/"},{"issue":"1","key":"bibr9-25152459211007467","first-page":"1","volume":"37","author":"Cleophas R. C.","year":"1999","journal-title":"International Journal of Clinical Pharmacology and Therapeutics"},{"key":"bibr10-25152459211007467","doi-asserted-by":"crossref","unstructured":"Fanelli D. (2010). \u201cPositive\u201d results increase down the hierarchy of the sciences. PLOS ONE, 5(4), Article e10068. https:\/\/doi.org\/10.1371\/journal.pone.0010068","DOI":"10.1371\/journal.pone.0010068"},{"key":"bibr11-25152459211007467","doi-asserted-by":"crossref","unstructured":"Fanelli D. (2012). Negative results are disappearing from most disciplines and countries. Scientometrics, 90(3), 891\u2013904. https:\/\/doi.org\/10.1007\/s11192-011-0494-7","DOI":"10.1007\/s11192-011-0494-7"},{"key":"bibr12-25152459211007467","doi-asserted-by":"crossref","unstructured":"Fiedler K., Schwarz N. (2016). Questionable research practices revisited. Social Psychological and Personality Science, 7(1), 45\u201352. https:\/\/doi.org\/10.1177\/1948550615612150","DOI":"10.1177\/1948550615612150"},{"key":"bibr13-25152459211007467","doi-asserted-by":"crossref","unstructured":"Franco A., Malhotra N., Simonovits G. (2014). Publication bias in the social sciences: Unlocking the file drawer. Science, 345(6203), 1502\u20131505. https:\/\/doi.org\/10.1126\/science.1255484","DOI":"10.1126\/science.1255484"},{"key":"bibr14-25152459211007467","doi-asserted-by":"crossref","unstructured":"Franco A., Malhotra N., Simonovits G. (2016). Underreporting in psychology experiments: Evidence from a study registry. Social Psychological and Personality Science, 7(1), 8\u201312. https:\/\/doi.org\/10.1177\/1948550615598377","DOI":"10.1177\/1948550615598377"},{"key":"bibr15-25152459211007467","doi-asserted-by":"crossref","unstructured":"Fraser H., Parker T., Nakagawa S., Barnett A., Fidler F. (2018). Questionable research practices in ecology and evolution. PLOS ONE, 13(7), Article e0200303. https:\/\/doi.org\/10.1371\/journal.pone.0200303","DOI":"10.1371\/journal.pone.0200303"},{"key":"bibr16-25152459211007467","unstructured":"Goldacre B., Drysdale H., Powell-Smith A., Dale A., Milosevic I., Slade E., Hartley P., Marston C., Mahtani K., Heneghan C. (2016). The COMPare trials project. Compare. http:\/\/compare-trials.org"},{"key":"bibr17-25152459211007467","doi-asserted-by":"publisher","DOI":"10.1037\/h0076157"},{"key":"bibr18-25152459211007467","doi-asserted-by":"crossref","unstructured":"John L. K., Loewenstein G., Prelec D. (2012). Measuring the prevalence of questionable research practices with incentives for truth telling. Psychological Science, 23(5), 524\u2013532. https:\/\/doi.org\/10.1177\/0956797611430953","DOI":"10.1177\/0956797611430953"},{"key":"bibr19-25152459211007467","doi-asserted-by":"crossref","unstructured":"Kaplan R. M., Irvin V. L. (2015). Likelihood of null effects of large NHLBI clinical trials has increased over time. PLOS ONE, 10(8), Article e0132382. https:\/\/doi.org\/10.1371\/journal.pone.0132382","DOI":"10.1371\/journal.pone.0132382"},{"key":"bibr20-25152459211007467","doi-asserted-by":"crossref","unstructured":"Kerr N. L. (1998). HARKing: Hypothesizing after the results are known. Personality and Social Psychology Review, 2(3), 196\u2013217. https:\/\/doi.org\/10.1207\/s15327957pspr0203_4","DOI":"10.1207\/s15327957pspr0203_4"},{"key":"bibr21-25152459211007467","doi-asserted-by":"crossref","unstructured":"Lakens D. (2017). Equivalence tests: A practical primer for t-tests, correlations, and meta-analyses. Social Psychological and Personality Science, 1, 1\u20138. https:\/\/doi.org\/10.1177\/1948550617697177","DOI":"10.1177\/1948550617697177"},{"key":"bibr22-25152459211007467","doi-asserted-by":"crossref","unstructured":"Lakens D. (2019). The value of preregistration for psychological science: A conceptual analysis. Japanese Psychological Review, 62(3), 221\u2013230. https:\/\/doi.org\/10.24602\/sjpr.62.3_221","DOI":"10.31234\/osf.io\/jbh4w"},{"key":"bibr23-25152459211007467","doi-asserted-by":"crossref","unstructured":"Lakens D., Scheel A. M., Isager P. M. (2018). Equivalence testing for psychological research: A tutorial. Advances in Methods and Practices in Psychological Science, 1(2), 259\u2013269. https:\/\/doi.org\/10.1177\/2515245918770963","DOI":"10.1177\/2515245918770963"},{"key":"bibr24-25152459211007467","doi-asserted-by":"crossref","unstructured":"Mahoney M. J. (1977). Publication prejudices: An experimental study of confirmatory bias in the peer review system. Cognitive Therapy and Research, 1(2), 161\u2013175. https:\/\/doi.org\/10.1007\/BF01173636","DOI":"10.1007\/BF01173636"},{"key":"bibr25-25152459211007467","doi-asserted-by":"crossref","unstructured":"Makel M. C., Hodges J., Cook B. G., Plucker J. (2021). Both questionable and open research practices are prevalent in education research. Educational Researcher. Advance online publication. https:\/\/doi.org\/10.3102\/0013189X211001356","DOI":"10.3102\/0013189X211001356"},{"key":"bibr26-25152459211007467","doi-asserted-by":"crossref","unstructured":"Makel M. C., Plucker J. A., Hegarty B. (2012). Replications in psychology research: How often do they really occur? Perspectives on Psychological Science, 7(6), 537\u2013542. https:\/\/doi.org\/10.1177\/1745691612460688","DOI":"10.1177\/1745691612460688"},{"key":"bibr27-25152459211007467","doi-asserted-by":"crossref","unstructured":"Maxwell S. E. (2004). The persistence of underpowered studies in psychological research: Causes, consequences, and remedies. Psychological Methods, 9(2), 147\u2013163. https:\/\/doi.org\/10.1037\/1082-989X.9.2.147","DOI":"10.1037\/1082-989X.9.2.147"},{"key":"bibr28-25152459211007467","unstructured":"Mitchell J. (2014). On the evidentiary emptiness of failed replications. https:\/\/jasonmitchell.fas.harvard.edu\/Papers\/Mitchell_failed_science_2014.pdf"},{"key":"bibr29-25152459211007467","doi-asserted-by":"crossref","unstructured":"Motyl M., Demos A. P., Carsel T. S., Hanson B. E., Melton Z. J., Mueller A. B., Prims J. P., Sun J., Washburn A. N., Wong K. M., Yantis C., Skitka L. J. (2017). The state of social and personality science: Rotten to the core, not so bad, getting better, or getting worse? Journal of Personality and Social Psychology, 113(1), 34\u201358. https:\/\/doi.org\/10.1037\/pspa0000084","DOI":"10.1037\/pspa0000084"},{"key":"bibr30-25152459211007467","doi-asserted-by":"crossref","unstructured":"Mueller-Langer F., Fecher B., Harhoff D., Wagner G. G. (2019). Replication studies in economics: How many and which papers are chosen for replication, and why? Research Policy, 48(1), 62\u201383. https:\/\/doi.org\/10.1016\/j.respol.2018.07.019","DOI":"10.1016\/j.respol.2018.07.019"},{"key":"bibr31-25152459211007467","doi-asserted-by":"crossref","unstructured":"M\u00fcller K. (2017). Here: A simpler way to find your files. https:\/\/CRAN.R-project.org\/package=here","DOI":"10.32614\/CRAN.package.here"},{"key":"bibr32-25152459211007467","doi-asserted-by":"crossref","unstructured":"Open Science Collaboration. (2015). Estimating the reproducibility of psychological science. Science, 349(6251), Article aac4716. https:\/\/doi.org\/10.1126\/science.aac4716","DOI":"10.1126\/science.aac4716"},{"key":"bibr33-25152459211007467","doi-asserted-by":"crossref","unstructured":"Rasmussen N., Lee K., Bero L. (2009). Association of trial registration with the results and conclusions of published trials of new oncology drugs. Trials, 10(1), Article 116. https:\/\/doi.org\/10.1186\/1745-6215-10-116","DOI":"10.1186\/1745-6215-10-116"},{"key":"bibr34-25152459211007467","unstructured":"R Core Team. (2019). R: A language and environment for statistical computing. R Foundation for Statistical Computing. https:\/\/www.R-project.org\/"},{"key":"bibr35-25152459211007467","unstructured":"Registered Replication Reports. (n.d.). Association for Psychological Science - APS. https:\/\/www.psychologicalscience.org\/publications\/replication"},{"key":"bibr36-25152459211007467","volume-title":"WebPlotDigitizer - Web Based Plot Digitizer","author":"Rohatgi A.","year":"2018"},{"key":"bibr37-25152459211007467","doi-asserted-by":"crossref","unstructured":"Rosenthal R. (1979). The file drawer problem and tolerance for null results. Psychological Bulletin, 86(3), 638\u2013641. https:\/\/doi.org\/10.1037\/0033-2909.86.3.638","DOI":"10.1037\/\/0033-2909.86.3.638"},{"key":"bibr38-25152459211007467","volume-title":"RStudio: Integrated development environment for r","author":"RStudio Team","year":"2019"},{"key":"bibr39-25152459211007467","doi-asserted-by":"crossref","unstructured":"Simmons J. P., Nelson L. D., Simonsohn U. (2011). False-positive psychology: Undisclosed flexibility in data collection and analysis allows presenting anything as significant. Psychological Science, 22(11), 1359\u20131366. https:\/\/doi.org\/10.1177\/0956797611417632","DOI":"10.1177\/0956797611417632"},{"key":"bibr40-25152459211007467","doi-asserted-by":"crossref","unstructured":"Simons D. J. (2018). Introducing advances in methods and practices in psychological science. Advances in Methods and Practices in Psychological Science, 1(1), 3\u20136. https:\/\/doi.org\/10.1177\/2515245918757424","DOI":"10.1177\/2515245918757424"},{"key":"bibr41-25152459211007467","doi-asserted-by":"crossref","unstructured":"Simons D. J., Holcombe A. O., Spellman B. A. (2014). An introduction to Registered Replication Reports at Perspectives on Psychological Science. Perspectives on Psy-chological Science, 9(5), 552\u2013555. https:\/\/doi.org\/10.1177\/1745691614543974","DOI":"10.1177\/1745691614543974"},{"key":"bibr42-25152459211007467","doi-asserted-by":"crossref","unstructured":"Sterling T. D. (1959). Publication decisions and their possible effects on inferences drawn from tests of significance or vice versa. Journal of the American Statistical Association, 54(285), 30\u201334. https:\/\/doi.org\/10.1080\/01621459.1959.10501497","DOI":"10.1080\/01621459.1959.10501497"},{"key":"bibr43-25152459211007467","doi-asserted-by":"crossref","unstructured":"Sterling T. D., Rosenbaum W. L., Weinkam J. J. (1995). Publication decisions revisited: The effect of the outcome of statistical tests on the decision to publish and vice versa. The American Statistician, 49(1), 108\u2013112. https:\/\/doi.org\/10.2307\/2684823","DOI":"10.1080\/00031305.1995.10476125"},{"key":"bibr44-25152459211007467","doi-asserted-by":"crossref","unstructured":"Szucs D., Ioannidis J. P. A. (2017). Empirical assessment of published effect sizes and power in the recent cognitive neuroscience and psychology literature. PLOS Biology, 15(3), Article e2000797. https:\/\/doi.org\/10.1371\/journal.pbio.2000797","DOI":"10.1371\/journal.pbio.2000797"},{"key":"bibr45-25152459211007467","doi-asserted-by":"crossref","unstructured":"Wickham H. (2007). Reshaping data with the reshape package. Journal of Statistical Software, 21(12). https:\/\/doi.org\/10.18637\/jss.v021.i12","DOI":"10.18637\/jss.v021.i12"},{"key":"bibr46-25152459211007467","doi-asserted-by":"crossref","unstructured":"Wickham H. (2016). Ggplot2: Elegant graphics for data analysis. Springer-Verlag. https:\/\/ggplot2.tidyverse.org","DOI":"10.1007\/978-3-319-24277-4_9"},{"key":"bibr47-25152459211007467","unstructured":"Wickham H. (2019). Stringr: Simple, consistent wrappers for common string operations. https:\/\/CRAN.R-project.org\/package=stringr"},{"key":"bibr48-25152459211007467","unstructured":"Wickham H., Henry L. (2019). Tidyr: Tidy messy data. https:\/\/CRAN.R-project.org\/package=tidyr"},{"key":"bibr49-25152459211007467","doi-asserted-by":"crossref","unstructured":"Wiseman R., Watt C., Kornbrot D. (2019). Registered reports: An early example and analysis. PeerJ, 7, Article e6232. https:\/\/doi.org\/10.7717\/peerj.6232","DOI":"10.7717\/peerj.6232"},{"key":"bibr50-25152459211007467","volume-title":"Dynamic documents with R and knitr","author":"Xie Y.","year":"2015","edition":"2"},{"key":"bibr51-25152459211007467","doi-asserted-by":"publisher","DOI":"10.1201\/9781315204963"},{"key":"bibr52-25152459211007467","doi-asserted-by":"publisher","DOI":"10.1201\/9781138359444"}],"container-title":["Advances in Methods and Practices in Psychological Science"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/journals.sagepub.com\/doi\/pdf\/10.1177\/25152459211007467","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/journals.sagepub.com\/doi\/full-xml\/10.1177\/25152459211007467","content-type":"application\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/journals.sagepub.com\/doi\/pdf\/10.1177\/25152459211007467","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,28]],"date-time":"2026-04-28T10:10:52Z","timestamp":1777371052000},"score":1,"resource":{"primary":{"URL":"https:\/\/journals.sagepub.com\/doi\/10.1177\/25152459211007467"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,4]]},"references-count":52,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2021,4]]}},"alternative-id":["10.1177\/25152459211007467"],"URL":"https:\/\/doi.org\/10.1177\/25152459211007467","relation":{},"ISSN":["2515-2459","2515-2467"],"issn-type":[{"value":"2515-2459","type":"print"},{"value":"2515-2467","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,4]]},"article-number":"25152459211007467"}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1207_2Fs15327957pspr0203_4 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T19:40:50Z","timestamp":1793302850143,"version":"4.5.0"},"reference-count":69,"publisher":"SAGE Publications","issue":"3","license":[{"start":{"date-parts":[[1998,8,1]],"date-time":"1998-08-01T00:00:00Z","timestamp":901929600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/journals.sagepub.com\/page\/policies\/text-and-data-mining-license"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Pers Soc Psychol Rev"],"published-print":{"date-parts":[[1998,8]]},"abstract":"<jats:p>This article considers a practice in scientific communication termed HARKing (Hypothesizing After the Results are Known). HARKing is defined as presenting a post hoc hypothesis (i.e., one based on or informed by one's results) in one's research report as if it were, in fact, an a priori hypotheses. Several forms of HARKing are identified and survey data are presented that suggests that at least some forms of HARKing are widely practiced and widely seen as inappropriate. I identify several reasons why scientists might HARK. Then I discuss several reasons why scientists ought not to HARK. It is conceded that the question of whether HARKing's costs exceed its benefits is a complex one that ought to be addressed through research, open discussion, and debate. To help stimulate such discussion (and for those such as myself who suspect that HARKing's costs do exceed its benefits), I conclude the article with some suggestions for deterring HARKing.<\/jats:p>","DOI":"10.1207\/s15327957pspr0203_4","type":"journal-article","created":{"date-parts":[[2005,1,12]],"date-time":"2005-01-12T15:03:28Z","timestamp":1105542208000},"page":"196-217","source":"Crossref","is-referenced-by-count":1487,"title":["HARKing: Hypothesizing After the Results are Known"],"prefix":"10.1177","volume":"2","author":[{"given":"Norbert L.","family":"Kerr","sequence":"first","affiliation":[{"name":"Department of Psychology, Michigan State University"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"179","published-online":{"date-parts":[[1998,8,1]]},"reference":[{"key":"atypb1","volume-title":"Task force on statistical inference initial report (draft) [posted on the World Wide Web]","author":"American Psychological Association","year":"1996"},{"key":"atypb2","doi-asserted-by":"publisher","DOI":"10.1037\/0021-9010.73.2.305"},{"key":"atypb3","doi-asserted-by":"publisher","DOI":"10.1037\/0021-9010.66.2.252"},{"key":"atypb4","volume-title":"Thinking and deciding","author":"Baron, J.","year":"1988"},{"key":"atypb5","doi-asserted-by":"publisher","DOI":"10.1037\/0022-3514.51.6.1173"},{"key":"atypb6","first-page":"10","volume":"8","author":"Baumeister, R.","year":"1993","journal-title":"Dialogue: Newsletter of the Society for Personality and Social Psychology"},{"key":"atypb7","first-page":"171","volume-title":"The compleat academic: A practical guide for the beginning social scientist","author":"Berm, D. J.","year":"1987"},{"key":"atypb8","first-page":"453","volume-title":"Research methods in social relations","author":"Bem, D. J.","year":"1991","edition":"6"},{"key":"atypb9","doi-asserted-by":"publisher","DOI":"10.1086\/209293"},{"key":"atypb10","doi-asserted-by":"publisher","DOI":"10.1037\/10162-000"},{"key":"atypb11","first-page":"69","volume-title":"Mathematical and theoretical systems","author":"Cliff, N.","year":"1989"},{"key":"atypb12","volume-title":"Responsible science: Ensuring the integrity of the research process","author":"Committee on Science, Engineering, and Public Policy","year":"1992"},{"key":"atypb13","volume-title":"Research methods","author":"Dane, F. C.","year":"1990"},{"key":"atypb14","doi-asserted-by":"publisher","DOI":"10.1016\/0749-5978(87)90005-7"},{"key":"atypb15","doi-asserted-by":"publisher","DOI":"10.1177\/004839317100100211"},{"key":"atypb16","doi-asserted-by":"publisher","DOI":"10.1146\/annurev.ps.31.020180.001125"},{"key":"atypb17","volume-title":"The Complete Sherlock Holmes (Vol. II)","author":"Doyle, A. C.","year":"1953"},{"key":"atypb18","volume-title":"Bias in human reasoning: Causes and consequences","author":"Evans, J.","year":"1989"},{"key":"atypb19","volume-title":"Against method","author":"Feyerabend, P. K.","year":"1993","edition":"3"},{"key":"atypb20","doi-asserted-by":"publisher","DOI":"10.1037\/0096-1523.1.3.288"},{"key":"atypb21","doi-asserted-by":"publisher","DOI":"10.1037\/0096-1523.3.2.349"},{"key":"atypb22","doi-asserted-by":"publisher","DOI":"10.1016\/0030-5073(79)90018-7"},{"key":"atypb23","doi-asserted-by":"publisher","DOI":"10.1037\/h0038813"},{"key":"atypb24","doi-asserted-by":"publisher","DOI":"10.1037\/h0076157"},{"key":"atypb25","doi-asserted-by":"publisher","DOI":"10.1037\/0033-295X.93.2.216"},{"key":"atypb26","doi-asserted-by":"publisher","DOI":"10.1016\/S0022-5371(81)90323-6"},{"key":"atypb27","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.107.3.311"},{"key":"atypb28","volume-title":"Philosophy of natural science","author":"Hempel, C. G.","year":"1966"},{"key":"atypb29","doi-asserted-by":"publisher","DOI":"10.7208\/chicago\/9780226327785.001.0001"},{"key":"atypb30","first-page":"605","volume":"15","author":"Hoch, S.","year":"1989","journal-title":"Organizational Behavior and Human Decision Processes"},{"key":"atypb31","volume-title":"Probability and evidence","author":"Horwich, P.","year":"1982"},{"key":"atypb32","doi-asserted-by":"publisher","DOI":"10.1093\/bjps\/35.3.245"},{"key":"atypb33","doi-asserted-by":"publisher","DOI":"10.7208\/chicago\/9780226360492.001.0001"},{"key":"atypb34","volume-title":"Bad blood: The Tuskegee syphilis experiment","author":"Jones, J. H.","year":"1993"},{"key":"atypb35","volume-title":"Research methods in social relations","author":"Judd, C.","year":"1991","edition":"6"},{"key":"atypb36","doi-asserted-by":"crossref","unstructured":"Kerr, N. L. & Harris, S. E. (1998).\n HARKing-hypothesizing after the results are known: Views from three disciplines\n . Unpublished manuscript, Michigan State University, East Lansing.","DOI":"10.1207\/s15327957pspr0203_4"},{"key":"atypb37","unstructured":"Kerr, N. L., Garst, J., Harris, S. E. & Sheppard, L. (1998).\n The effect of having a sufficient hypothesis on generation of alternative hypotheses\n . Unpublished manuscript, Michigan State University, East Lansing."},{"key":"atypb38","doi-asserted-by":"publisher","DOI":"10.1016\/S0079-7421(08)60315-1"},{"key":"atypb39","doi-asserted-by":"publisher","DOI":"10.1037\/0278-7393.6.2.107"},{"key":"atypb40","doi-asserted-by":"publisher","DOI":"10.1525\/9780520917804"},{"key":"atypb41","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511621123"},{"key":"atypb42","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.111.3.490"},{"key":"atypb43","volume-title":"Scientist as subject: The psychological imperative","author":"Mahoney, M. J.","year":"1976"},{"key":"atypb44","doi-asserted-by":"publisher","DOI":"10.1016\/S0065-2601(08)60393-7"},{"key":"atypb45","first-page":"558","volume-title":"A century of psychology as science","author":"McGuire, W. J.","year":"1983"},{"key":"atypb46","volume-title":"Social theory and social structure","author":"Merton, R. K.","year":"1968"},{"key":"atypb47","first-page":"11","volume-title":"Annual review of personality and social psychology","author":"Messick, D. M.","year":"1983"},{"key":"atypb48","first-page":"85","volume":"5","author":"Neuliep, J. W.","year":"1990","journal-title":"Journal of Social Behavior and Personality"},{"key":"atypb49","first-page":"21","volume":"8","author":"Neuliep, J. W.","year":"1993","journal-title":"Journal of Social Behavior and Personality"},{"key":"atypb50","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9780511752896.010"},{"key":"atypb51","doi-asserted-by":"publisher","DOI":"10.1126\/science.146.3642.347"},{"key":"atypb52","volume-title":"The logic of scientific discovery","author":"Popper, K.","year":"1959"},{"key":"atypb53","volume-title":"Handbook of qualitative research methodsfor psychology and the social sciences","author":"Richardson, J. T. E.","year":"1996"},{"key":"atypb54","volume-title":"Scripts, plans, goals, and understanding: An inquiry into human knowledge structures","author":"Schank, R. C.","year":"1977"},{"key":"atypb55","volume-title":"Learned optimism","author":"Seligman, M. E. P.","year":"1991"},{"key":"atypb56","doi-asserted-by":"publisher","DOI":"10.3758\/BF03202434"},{"key":"atypb57","doi-asserted-by":"publisher","DOI":"10.1086\/287427"},{"key":"atypb58","doi-asserted-by":"publisher","DOI":"10.1037\/0096-1523.3.4.544"},{"key":"atypb59","doi-asserted-by":"publisher","DOI":"10.1007\/978-1-4612-2748-9"},{"key":"atypb60","first-page":"12","volume":"5","author":"Sternberg, R. J.","year":"1992","journal-title":"The APS Observer"},{"key":"atypb61","doi-asserted-by":"publisher","DOI":"10.1086\/209097"},{"key":"atypb62","volume-title":"On becoming a social psychologist","author":"Tesser, A.","year":"1994"},{"key":"atypb63","doi-asserted-by":"publisher","DOI":"10.1016\/0749-596X(85)90049-X"},{"key":"atypb64","doi-asserted-by":"publisher","DOI":"10.1086\/268196"},{"key":"atypb65","unstructured":"van Andel, P. (1998).\n An anatomy of serendipity\n . Unpublished manuscript, University of Groningen, Netherlands."},{"key":"atypb66","doi-asserted-by":"publisher","DOI":"10.1037\/0022-3514.67.2.233"},{"key":"atypb67","doi-asserted-by":"publisher","DOI":"10.1080\/17470216008416717"},{"key":"atypb68","first-page":"135","volume-title":"New horizons in psychology","author":"Wason, P. C.","year":"1966"},{"key":"atypb69","doi-asserted-by":"publisher","DOI":"10.1177\/0146167292184017"}],"container-title":["Personality and Social Psychology Review"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/journals.sagepub.com\/doi\/pdf\/10.1207\/s15327957pspr0203_4","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/journals.sagepub.com\/doi\/pdf\/10.1207\/s15327957pspr0203_4","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,3,10]],"date-time":"2025-03-10T13:09:04Z","timestamp":1741612144000},"score":1,"resource":{"primary":{"URL":"https:\/\/journals.sagepub.com\/doi\/10.1207\/s15327957pspr0203_4"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[1998,8]]},"references-count":69,"journal-issue":{"issue":"3","published-print":{"date-parts":[[1998,8]]}},"alternative-id":["10.1207\/s15327957pspr0203_4"],"URL":"https:\/\/doi.org\/10.1207\/s15327957pspr0203_4","relation":{},"ISSN":["1088-8683","1532-7957"],"issn-type":[{"value":"1088-8683","type":"print"},{"value":"1532-7957","type":"electronic"}],"subject":[],"published":{"date-parts":[[1998,8]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_12688_2Ff1000research_29032_2 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T14:22:57Z","timestamp":1793283777847,"version":"4.5.0"},"update-to":[{"DOI":"10.12688\/f1000research.29032.1","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2021,4,19]],"date-time":"2021-04-19T00:00:00Z","timestamp":1618790400000}}],"reference-count":40,"publisher":"F1000 Research Ltd","license":[{"start":{"date-parts":[[2021,4,19]],"date-time":"2021-04-19T00:00:00Z","timestamp":1618790400000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100005970","name":"Deutsche Stiftung f\u00fcr Herzforschung","doi-asserted-by":"publisher","award":["SFB876"],"award-info":[{"award-number":["SFB876"]}],"id":[{"id":"10.13039\/501100005970","id-type":"DOI","asserted-by":"publisher"}]},{"name":"United States National Science Foundation Graduate Research Fellowship Program","award":["1745303"],"award-info":[{"award-number":["1745303"]}],"id":[{"id":"https:\/\/ror.org\/021nxhr62","id-type":"ROR","asserted-by":"crossref"}]},{"DOI":"10.13039\/501100003246","name":"Netherlands Organisation for Scientific Research","doi-asserted-by":"crossref","award":["VENI grant 016.Veni.173.076"],"award-info":[{"award-number":["VENI grant 016.Veni.173.076"]}],"id":[{"id":"10.13039\/501100003246","id-type":"DOI","asserted-by":"crossref"}]},{"DOI":"10.13039\/100006785","name":"Google LLC","doi-asserted-by":"crossref","id":[{"id":"10.13039\/100006785","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["f1000research.com"],"crossmark-restriction":false},"short-container-title":["F1000Res"],"abstract":"<ns4:p>Data analysis often entails a multitude of heterogeneous steps, from the application of various command line tools to the usage of scripting languages like R or Python for the generation of plots and tables. It is widely recognized that data analyses should ideally be conducted in a reproducible way.\u00a0Reproducibility enables technical validation and regeneration of results on the original or even new data. However, reproducibility alone is by no means sufficient to deliver an analysis that is of lasting impact (i.e., sustainable) for the field, or even just one research group. We postulate that it is equally important to ensure adaptability and transparency. The former describes the ability to modify the analysis to answer extended or slightly different research questions. The latter describes the ability to understand the analysis in order to judge whether it is not only technically, but methodologically valid.<\/ns4:p>\n <ns4:p>Here, we analyze the properties needed for a data analysis to become reproducible, adaptable, and transparent. We show how the popular workflow management system Snakemake can be used to guarantee this, and how it enables an ergonomic, combined, unified representation of all steps involved in data analysis, ranging from raw data processing, to quality control and fine-grained, interactive exploration and plotting of final results.<\/ns4:p>","DOI":"10.12688\/f1000research.29032.2","type":"journal-article","created":{"date-parts":[[2021,4,19]],"date-time":"2021-04-19T05:30:34Z","timestamp":1618810234000},"page":"33","update-policy":"https:\/\/doi.org\/10.12688\/f1000research.crossmark-policy","source":"Crossref","is-referenced-by-count":1667,"title":["Sustainable data analysis with Snakemake"],"prefix":"10.12688","volume":"10","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-3976-9701","authenticated-orcid":false,"given":"Felix","family":"M\u00f6lder","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4166-4343","authenticated-orcid":false,"given":"Kim Philipp","family":"Jablonski","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-8921-6005","authenticated-orcid":false,"given":"Brice","family":"Letcher","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3683-6208","authenticated-orcid":false,"given":"Michael B.","family":"Hall","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-9114-6421","authenticated-orcid":false,"given":"Christopher H.","family":"Tomkins-Tinch","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4387-3819","authenticated-orcid":false,"given":"Vanessa","family":"Sochat","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jan","family":"Forster","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3594-6213","authenticated-orcid":false,"given":"Soohyun","family":"Lee","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sven O.","family":"Twardziok","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3468-0652","authenticated-orcid":false,"given":"Alexander","family":"Kanitz","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Andreas","family":"Wilm","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Manuel","family":"Holtgrewe","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sven","family":"Rahmann","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Sven","family":"Nahnsen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9818-9320","authenticated-orcid":false,"given":"Johannes","family":"K\u00f6ster","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"2560","published-online":{"date-parts":[[2021,4,19]]},"reference":[{"key":"ref-1","doi-asserted-by":"publisher","first-page":"452-4","DOI":"10.1038\/533452a","article-title":"1,500 scientists lift the lid on reproducibility.","volume":"533","author":"M Baker","year":"2016","journal-title":"Nature."},{"key":"ref-2","doi-asserted-by":"publisher","first-page":"415-6","DOI":"10.1126\/science.1179653","article-title":"Computer science. Accessible reproducible research.","volume":"327","author":"J Mesirov","year":"2010","journal-title":"Science."},{"key":"ref-3","doi-asserted-by":"publisher","first-page":"0021","DOI":"10.1038\/s41562-016-0021","article-title":"A manifesto for reproducible science.","volume":"1","author":"M Munaf\u00f2","year":"2017","journal-title":"Nat Hum Behav."},{"key":"ref-4","doi-asserted-by":"publisher","first-page":"W537-W544","DOI":"10.1093\/nar\/gky379","article-title":"The Galaxy platform for accessible, reproducible and collaborative biomedical analyses: 2018 update.","volume":"46","author":"E Afgan","year":"2018","journal-title":"Nucleic Acids Res."},{"key":"ref-5","article-title":"KNIME: The Konstanz Information Miner.","author":"M Berthold","year":"2007"},{"key":"ref-6","doi-asserted-by":"publisher","first-page":"giaa068","DOI":"10.1093\/gigascience\/giaa068","article-title":"Watchdog 2.0: New developments for reusability, reproducibility, and workflow execution.","volume":"9","author":"M Kluge","year":"2020","journal-title":"GigaScience."},{"key":"ref-7","doi-asserted-by":"publisher","first-page":"3815-3817","DOI":"10.1093\/bioinformatics\/btz133","article-title":"Anduril 2: upgraded large\u2013scale data integration framework.","volume":"35","author":"A Cervera","year":"2019","journal-title":"Bioinformatics."},{"key":"ref-8","article-title":"Balsam: Automated Scheduling and Execution of Dynamic, Data-Intensive HPC Work\ufb02ows.","author":"M Salim","year":"2018","journal-title":"In: Proceedings of the 8th Workshop on Python for High-Performance and Scienti\ufb01c Computing. ACM Press."},{"key":"ref-9","doi-asserted-by":"publisher","first-page":"1-6","DOI":"10.1145\/3183767.3183768","article-title":"HyperLoom: A Platform for Defining and Executing Scientific Pipelines in Distributed Environments.","author":"V Cima","year":"2018","journal-title":"ACM."},{"key":"ref-10","doi-asserted-by":"publisher","first-page":"30","DOI":"10.5334\/jors.161","article-title":"Jug: Software for Parallel Reproducible Computation in Python.","volume":"5","author":"L Coelho","year":"2017","journal-title":"J Open Res Softw."},{"key":"ref-11","doi-asserted-by":"publisher","first-page":"356-359","DOI":"10.1145\/1851476.1851529","article-title":"Pwrake: a parallel and distributed flexible workflow management tool for wide-area data intensive computing.","author":"M Tanaka","year":"2010","journal-title":"Proceedings of the 19th ACM International Symposium on High Performance Distributed Computing -HPDC 2010."},{"key":"ref-12","doi-asserted-by":"publisher","first-page":"2778-9","DOI":"10.1093\/bioinformatics\/btq524","article-title":"Ruffus: a lightweight Python library for computational pipelines.","volume":"26","author":"L Goodstadt","year":"2010","journal-title":"Bioinformatics."},{"key":"ref-13","doi-asserted-by":"publisher","DOI":"10.1093\/gigascience\/giz044","article-title":"SciPipe: A workflow library for agile development of complex and dynamic bioinformatics pipelines.","volume":"8","author":"S Lampa","year":"2019","journal-title":"Gigascience."},{"key":"ref-14","doi-asserted-by":"publisher","first-page":"1-8","DOI":"10.1145\/2616498.2616565","article-title":"Once you SCOOP, no need to fork","author":"Y Hold-Geoffroy","year":"2014","journal-title":"Proceedings of the 2014 Annual Conference on Extreme Science and Engineering Discovery Environment."},{"key":"ref-15","doi-asserted-by":"publisher","first-page":"67-91","DOI":"10.1007\/s10723-013-9272-5","article-title":"ServiceSs: An Interoperable Programming Framework for the Cloud.","volume":"12","author":"F Lordan","year":"2013","journal-title":"J Grid Comput."},{"key":"ref-16","doi-asserted-by":"publisher","first-page":"2572-2574","DOI":"10.1093\/bioinformatics\/btz956","article-title":"Bioinformatics pipeline using JUDI: Just Do It!","volume":"36","author":"S Pal","year":"2020","journal-title":"Bioinformatics."},{"key":"ref-17","doi-asserted-by":"publisher","first-page":"316-319","DOI":"10.1038\/nbt.3820","article-title":"Nextflow enables reproducible computational workflows.","volume":"35","author":"P Di Tommaso","year":"2017","journal-title":"Nat Biotechnol."},{"key":"ref-18","doi-asserted-by":"publisher","first-page":"2520","DOI":"10.1093\/bioinformatics\/bts480","article-title":"Snakemake\u2013a scalable bioinformatics workflow engine.","volume":"28","author":"J K\u00f6ster","year":"2012","journal-title":"Bioinformatics."},{"key":"ref-19","doi-asserted-by":"publisher","first-page":"3286-3288","DOI":"10.1093\/bioinformatics\/btx403","article-title":"BioQueue: a novel pipeline framework to accelerate bioinformatics analysis.","volume":"33","author":"L Yao","year":"2017","journal-title":"Bioinformatics."},{"key":"ref-20","doi-asserted-by":"publisher","first-page":"1525-6","DOI":"10.1093\/bioinformatics\/bts167","article-title":"Bpipe: a tool for running and managing bioinformatics pipelines.","volume":"28","author":"S Sadedin","year":"2012","journal-title":"Bioinformatics."},{"key":"ref-21","doi-asserted-by":"publisher","first-page":"2824","DOI":"10.12688\/f1000research.10335.1","article-title":"Cluster Flow: A user-friendly bioinformatics workflow tool [version 1; peer review: 3 approved].","volume":"5","author":"P Ewels","year":"2016","journal-title":"F1000Res."},{"key":"ref-22","doi-asserted-by":"publisher","first-page":"737","DOI":"10.21105\/joss.00737","article-title":"Cylc: A Work\ufb02ow Engine for Cycling Systems.","volume":"3","author":"H Oliver","year":"2018","journal-title":"J Open Source Softw."},{"key":"ref-23","doi-asserted-by":"publisher","first-page":"10-16","DOI":"10.1093\/bioinformatics\/btu595","article-title":"BigDataScript: a scripting language for data pipelines.","volume":"31","author":"P Cingolani","year":"2015","journal-title":"Bioinformatics."},{"key":"ref-24","doi-asserted-by":"publisher","DOI":"10.1109\/IPDPSW.2017.157","article-title":"The Popper Convention: Making Reproducible Systems Evaluation Practical","author":"I Jimenez","year":"2017","journal-title":"2017 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)."},{"key":"ref-25","article-title":"YAML Ain\u2019t Markup Language YAML Version 1.2.","author":"C Evans","year":"2009"},{"key":"ref-26","doi-asserted-by":"publisher","DOI":"10.6084\/m9.figshare.3115156.v2","article-title":"Common Workflow Language, v1.0","author":"P Amstutz","year":"2016"},{"key":"ref-27","doi-asserted-by":"publisher","DOI":"10.7490\/f1000research.1114634.1","article-title":"Full-stack genomics pipelining with GATK4 +WDL +Cromwell.","volume":"6","author":"K Voss","year":"2017","journal-title":"F1000Res."},{"key":"ref-28","doi-asserted-by":"publisher","first-page":"314-316","DOI":"10.1038\/nbt.3772","article-title":"Toil enables reproducible open source, big biomedical data analyses.","volume":"35","author":"J Vivian","year":"2017","journal-title":"Nat Biotechnol."},{"key":"ref-29","doi-asserted-by":"publisher","first-page":"4424-4426","DOI":"10.1093\/bioinformatics\/btz379","article-title":"Tibanna: software for scalable execution of portable pipelines on the cloud.","volume":"35","author":"S Lee","year":"2019","journal-title":"Bioinformatics."},{"key":"ref-30","doi-asserted-by":"publisher","first-page":"e0177459","DOI":"10.1371\/journal.pone.0177459","article-title":"Singularity: Scientific containers for mobility of compute.","volume":"12","author":"G Kurtzer","year":"2017","journal-title":"PLoS One."},{"key":"ref-31","doi-asserted-by":"crossref","DOI":"10.1002\/9780470165171","article-title":"Automated Defect Prevention: Best Practices in Software Management","author":"D Huizinga","year":"2007"},{"key":"ref-32","article-title":"Readability revisited: the new Dale-Chall readability formula.","author":"J Chall","year":"1995"},{"key":"ref-33","article-title":"Code Styling and its Effects on Code Readability and Interpretation","author":"L Sundkvist","year":"2017"},{"key":"ref-34","doi-asserted-by":"publisher","first-page":"631-635","DOI":"10.1016\/j.cels.2018.03.014","article-title":"Practical Computational Reproducibility in the Life Sciences.","volume":"6","author":"B Gr\u00fcning","year":"2018","journal-title":"Cell Syst."},{"key":"ref-35","article-title":"Data analysis for paper \"Sustainable data analysis with Snakemake\".","author":"J K\u00f6ster,","year":"2020","journal-title":"Zenodo."},{"key":"ref-36","doi-asserted-by":"publisher","first-page":"565-567","DOI":"10.1007\/0-387-23483-7_388","article-title":"SHA Family (Secure Hash Algorithm).","author":"H Handschuh","year":"2005","journal-title":"Encyclopedia of Cryptography and Security. Springer US."},{"key":"ref-37","article-title":"Bitcoin and Cryptocurrency Technologies: A Comprehensive Introduction.","author":"A Narayanan","year":"2016"},{"key":"ref-38","doi-asserted-by":"publisher","first-page":"56-61","DOI":"10.25080\/Majora-92bf1922-00a","article-title":"Data Structures for Statistical Computing in Python.","author":"W McKinney","year":"2010","journal-title":"Proceedings of the 9th Python in Science Conference."},{"key":"ref-39","doi-asserted-by":"publisher","article-title":"pandas-dev\/pandas: Pandas","year":"2020","DOI":"10.5281\/zenodo.3509134"},{"key":"ref-40","doi-asserted-by":"publisher","first-page":"475-476","DOI":"10.1038\/s41592-018-0046-7","article-title":"Bioconda: sustainable and comprehensive software distribution for the life sciences.","volume":"15","author":"B Gr\u00fcning","year":"2018","journal-title":"Nat Methods."}],"updated-by":[{"DOI":"10.12688\/f1000research.29032.3","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2025,9,23]],"date-time":"2025-09-23T00:00:00Z","timestamp":1758585600000}}],"container-title":["F1000Research"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/f1000research.com\/articles\/10-33\/v2\/xml","content-type":"application\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/f1000research.com\/articles\/10-33\/v2\/pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/f1000research.com\/articles\/10-33\/v2\/iparadigms","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,11,2]],"date-time":"2023-11-02T05:08:28Z","timestamp":1698901708000},"score":1,"resource":{"primary":{"URL":"https:\/\/f1000research.com\/articles\/10-33\/v2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,4,19]]},"references-count":40,"URL":"https:\/\/doi.org\/10.12688\/f1000research.29032.2","relation":{"has-review":[{"id-type":"doi","id":"10.5256\/f1000research.56004.r83481","asserted-by":"subject"},{"id-type":"doi","id":"10.5256\/f1000research.56004.r83481","asserted-by":"object"}],"is-version-of":[{"id-type":"doi","id":"10.12688\/f1000research.29032.3","asserted-by":"object"}]},"ISSN":["2046-1402"],"issn-type":[{"value":"2046-1402","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,4,19]]},"assertion":[{"value":"Indexed","URL":"https:\/\/f1000research.com\/articles\/10-33\/v2#article-reports","order":0,"name":"referee-status","label":"Referee status","group":{"name":"current-referee-status","label":"Current Referee Status"}},{"value":"10.5256\/f1000research.32078.r77869, Michael Reich, Department of Medicine, University of California, San Diego, San Diego, CA, USA, 15 Feb 2021, version 1, 1 approved, 1 approved with reservations","URL":"https:\/\/f1000research.com\/articles\/10-33\/v1#referee-response-77869","order":0,"name":"referee-response-77869","label":"Referee Report","group":{"name":"article-reports","label":"Article Reports"}},{"value":"<b>Johannes Köster<\/b>; <i>Posted: 08 Apr 2021<\/i>; Thanks a lot for the assessment of our manuscript. Your comments have led to several important improvements. Answers to individual comments can be found below. <b>Comment 1<\/b> The paper would benefit from a brief discussion of how snakemake interacts with load-balancing software (such as SLURM, Torque, LSF, etc.), because of the focus on the details of executing jobs. This information is in the documentation on the Snakemake web site, but a conceptual overview in the paper would help the reader to understand this relationship. <b>Response 1<\/b> Indeed, the paper failed to explicitly mention how this is handled. We have now added an explaining paragraph to section 2.5.1. In brief: resource requirements are passed to the middleware, but the scheduling problem still has to be solved in order to prioritze jobs and minimize the lifetime of temporary files. It becomes a bit less constrained though. <b>Comment 2<\/b> Information necessary to install and use snakemake is available on the snakemake web site as referenced in the paper. The instructions and tutorial are comprehensive and understandable, and the progressive exercises give a good feel for the snakemake paradigm. Given the ease and popularity of container technology, it would be useful to include how to run snakemake from a Docker container in the tutorial. I was able to do this easily from the official snakemake container on Dockerhub and think it would be a helpful alternative in the documentation to installing Miniconda. <b>Response 2<\/b> Offering further ways of performing the tutorial is indeed a good idea. We have considered providing instructions for performing the tutorial from a container, but felt that this would complicate editing and interaction too much. However, we now instead provide a Gitpod setup for the tutorial, which allows to directly run it, without any setup needed, from inside a browser window which offers both an editor and a terminal based on the Eclipse Theia IDE (see here).","URL":"https:\/\/f1000research.com\/articles\/10-33\/v2#referee-comment-6549","order":1,"name":"referee-comment-6549","label":"Referee Comment","group":{"name":"article-reports","label":"Article Reports"}},{"value":"10.5256\/f1000research.56004.r83481, Caroline C. Friedel, Institute of Informatics, LMU Munich, Munich, Germany, 11 May 2021, version 2, indexed","URL":"https:\/\/f1000research.com\/articles\/10-33\/v2#referee-response-83481","order":2,"name":"referee-response-83481","label":"Referee Report","group":{"name":"article-reports","label":"Article Reports"}},{"value":"This work was supported by the Netherlands Organisation for Scientific Research (NWO) (VENI grant 016.Veni.173.076, Johannes K\u00f6ster), the German Research Foundation (SFB 876, Johannes K\u00f6ster and Sven Rahmann), the United States National Science Foundation Graduate Research Fellowship Program (NSF-GRFP) (Grant No. 1745303, Christopher Tomkins-Tinch), and Google LLC (Vanessa Sochat and Johannes K\u00f6ster). The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript.","order":3,"name":"grant-information","label":"Grant Information"},{"value":"This is an open access article distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.","order":0,"name":"copyright-info","label":"Copyright"}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_12688_2Ff1000research_51738_2 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,16]],"date-time":"2026-10-16T21:21:15Z","timestamp":1792185675350,"version":"4.4.0"},"update-to":[{"DOI":"10.12688\/f1000research.51738.1","type":"new_version","label":"New version","source":"publisher","updated":{"date-parts":[[2021,7,20]],"date-time":"2021-07-20T00:00:00Z","timestamp":1626739200000}}],"reference-count":119,"publisher":"F1000 Research Ltd","license":[{"start":{"date-parts":[[2021,7,20]],"date-time":"2021-07-20T00:00:00Z","timestamp":1626739200000},"content-version":"tdm","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"funder":[{"DOI":"10.13039\/501100012642","name":"Mozilla Foundation","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100012642","id-type":"DOI","asserted-by":"publisher"}]},{"name":"Sustainable Software Institute"},{"DOI":"10.13039\/501100001659","name":"German Research Foundation","doi-asserted-by":"crossref","award":["PE1632\/17-1"],"award-info":[{"award-number":["PE1632\/17-1"]}],"id":[{"id":"10.13039\/501100001659","id-type":"DOI","asserted-by":"crossref"}]}],"content-domain":{"domain":["f1000research.com"],"crossmark-restriction":false},"short-container-title":["F1000Res"],"abstract":"<ns3:p>The traditional scientific paper falls short of effectively communicating computational research. \u00a0To help improve this situation, we propose a system by which the computational workflows underlying research articles are checked. The CODECHECK system uses open infrastructure and tools and can be integrated into review and publication processes in multiple ways. We describe these integrations along multiple dimensions (importance, who, openness, when). In collaboration with academic publishers and conferences, we demonstrate CODECHECK with 25 reproductions of diverse scientific publications. These CODECHECKs show that asking for reproducible workflows during a collaborative review can effectively improve executability. While CODECHECK has clear limitations, it may represent a building block in Open Science and publishing ecosystems for improving the reproducibility, appreciation, and, potentially, the quality of non-textual research artefacts. The CODECHECK website can be accessed here: https:\/\/codecheck.org.uk\/.<\/ns3:p>","DOI":"10.12688\/f1000research.51738.2","type":"journal-article","created":{"date-parts":[[2021,7,20]],"date-time":"2021-07-20T04:50:06Z","timestamp":1626756606000},"page":"253","update-policy":"https:\/\/doi.org\/10.12688\/f1000research.crossmark-policy","source":"Crossref","is-referenced-by-count":21,"title":["CODECHECK: an Open Science initiative for the independent\u00a0execution of computations underlying research articles during peer review to improve reproducibility"],"prefix":"10.12688","volume":"10","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0024-5046","authenticated-orcid":false,"given":"Daniel","family":"N\u00fcst","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8607-8025","authenticated-orcid":false,"given":"Stephen J.","family":"Eglen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"2560","published-online":{"date-parts":[[2021,7,20]]},"reference":[{"key":"ref-1","article-title":"How computers broke science \u2013 and what we can do to fix it","author":"B Marwick","year":"2015"},{"key":"ref-2","doi-asserted-by":"publisher","first-page":"55-81","DOI":"10.1007\/978-1-4612-2544-7_5","article-title":"WaveLab and Reproducible Research.","author":"J Buckheit","year":"1995"},{"key":"ref-3","doi-asserted-by":"publisher","first-page":"601-604","DOI":"10.1190\/1.1822162","article-title":"Electronic documents give reproducible research a new meaning.","author":"J Claerbout","year":"1992"},{"key":"ref-4","doi-asserted-by":"publisher","first-page":"94-97","DOI":"10.1016\/j.cub.2013.11.014","article-title":"The availability of research data declines rapidly with article age.","volume":"24","author":"T Vines","year":"2014","journal-title":"Curr Biol."},{"key":"ref-5","doi-asserted-by":"publisher","first-page":"753","DOI":"10.1038\/467753a","article-title":"Publish your computer code: it is good enough.","volume":"467","author":"N Barnes","year":"2010","journal-title":"Nature."},{"key":"ref-6","doi-asserted-by":"publisher","first-page":"274","DOI":"10.1186\/s13059-015-0850-7","article-title":"Five selfish reasons to work reproducibly.","volume":"16","author":"F Markowetz","year":"2015","journal-title":"Genome Biol."},{"key":"ref-7","doi-asserted-by":"publisher","DOI":"10.7557\/5.4963","article-title":"Mission or money?","author":"A Fyfe","year":"2019","journal-title":"Septentrio Conference Series."},{"key":"ref-8","article-title":"Terminologies for Reproducible Research.","author":"L Barba","year":"2018","journal-title":"arXiv: 1802.03311 [cs]."},{"key":"ref-9","doi-asserted-by":"publisher","first-page":"113-120","DOI":"10.25080\/Majora-4af1f417-011","article-title":"Binder 2.0 - Reproducible, interactive, sharable environments for science at scale.","author":"P Jupyter","year":"2018","journal-title":"Proceedings of the 17th Python in Science Conference."},{"key":"ref-10","doi-asserted-by":"publisher","first-page":"10","DOI":"10.1186\/s41073-020-00095-y","article-title":"Publishing computational research - a review of infrastructures for reproducible and transparent scholarly communication.","volume":"5","author":"M Konkol","year":"2020","journal-title":"Res Integr Peer Rev."},{"key":"ref-11","article-title":"Setting the Default to Reproducible: Reproducibility in Computational and Experimental Mathematics.","author":"V Stodden","year":"2013"},{"key":"ref-12","doi-asserted-by":"publisher","first-page":"e0230281","DOI":"10.1371\/journal.pone.0230281","article-title":"Journal data policies: Exploring how the understanding of editors and authors corresponds to the policies themselves.","volume":"15","author":"T Christian","year":"2020","journal-title":"PLoS One."},{"key":"ref-13","doi-asserted-by":"publisher","first-page":"615-631","DOI":"10.1177\/1745691612459058","article-title":"Scientific Utopia: II. Restructuring Incentives and Practices to Promote Truth Over Publishability.","volume":"7","author":"B Nosek","year":"2012","journal-title":"Perspect Psychol Sci."},{"key":"ref-14","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.4279275","article-title":"CODECHECK certificate 2020-025.","author":"D N\u00fcst","year":"2020","journal-title":"Zenodo."},{"key":"ref-15","doi-asserted-by":"publisher","first-page":"4","DOI":"10.1186\/s41073-019-0063-9","article-title":"Guidelines for open peer review implementation.","volume":"4","author":"T Ross-Hellauer","year":"2019","journal-title":"Res Integr Peer Rev."},{"key":"ref-16","doi-asserted-by":"publisher","first-page":"6","DOI":"10.1186\/s41073-020-00092-1","article-title":"The limitations to our understanding of peer review.","volume":"5","author":"J Tennant","year":"2020","journal-title":"Res Integr Peer Rev."},{"key":"ref-17","doi-asserted-by":"publisher","DOI":"10.17605\/OSF.IO\/9CFTX","article-title":"Everything Hertz 123: Authenticated anonymity (with Michael Eisen).","author":"D Quintana","year":"2020","journal-title":"Open Science Framework."},{"key":"ref-18","doi-asserted-by":"publisher","DOI":"10.5446\/42484","article-title":"Keynote at deRSE 2019: Delivering on the promise of Research Computing.","author":"B Bouffler","year":"2019","journal-title":"TIB AV- PORTAL."},{"key":"ref-19","doi-asserted-by":"publisher","first-page":"30","DOI":"10.1038\/d41586-020-01328-2","article-title":"New journal for reproduction and replication results.","volume":"581","author":"E Roesch","year":"2020","journal-title":"Nature."},{"key":"ref-20","article-title":"Registered Reports.","author":"C Chambers","year":"2019","journal-title":"OSF."},{"key":"ref-21","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3947959","article-title":"CODECHECK certificate 2020-013.","author":"I Davies","year":"2020","journal-title":"Zenodo."},{"key":"ref-22","doi-asserted-by":"publisher","first-page":"613","DOI":"10.1038\/d41586-018-05256-0","article-title":"Before reproducibility must come preproducibility.","volume":"557","author":"P Stark","year":"2018","journal-title":"Nature."},{"key":"ref-23","doi-asserted-by":"publisher","DOI":"10.22541\/au.153922477.77361922","article-title":"Praxis of Reproducible Computational Science","author":"L Barba","year":"2018","journal-title":"Authorea."},{"key":"ref-24","doi-asserted-by":"publisher","first-page":"e375-e385","DOI":"10.1016\/S2468-2667(20)30133-X","article-title":"Effects of non-pharmaceutical interventions on COVID-19 cases, deaths, and demand for hospital services in the UK: a modelling study.","volume":"5","author":"N Davies","year":"2020","journal-title":"Lancet Public Health."},{"key":"ref-25","doi-asserted-by":"publisher","first-page":"1151-1160","DOI":"10.1016\/S1473-3099(20)30457-6","article-title":"Effectiveness of isolation, testing, contact tracing, and physical distancing on reducing transmission of SARS-CoV-2 in different settings: a mathematical modelling study.","volume":"20","author":"A Kucharski","year":"2020","journal-title":"Lancet Infect Dis."},{"key":"ref-26","doi-asserted-by":"publisher","first-page":"323-324","DOI":"10.1038\/d41586-020-01685-y","article-title":"Critiqued coronavirus simulation gets thumbs up from code-checking efforts.","volume":"582","author":"D Chawla","year":"2020","journal-title":"Nature."},{"key":"ref-27","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3865491","article-title":"CODECHECK certificate 2020-010.","author":"S Eglen","year":"2020","journal-title":"Zenodo."},{"key":"ref-28","article-title":"Reproducible Publications at AGILE Conferences","author":"D N\u00fcst","year":"2019"},{"key":"ref-29","doi-asserted-by":"publisher","DOI":"10.7557\/5.5601","article-title":"Improving reproducibility of geospatial conference papers \u2013 lessons learned from a first implementation of reproducibility reviews.","author":"D N\u00fcst","year":"2020","journal-title":"Septentrio Conference Series."},{"key":"ref-30","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3674056","article-title":"CODECHECK certificate 2020-001.","author":"S Eglen","year":"2020","journal-title":"Zenodo."},{"key":"ref-31","doi-asserted-by":"publisher","first-page":"giaa026","DOI":"10.1093\/gigascience\/giaa026","article-title":"ShinyLearner: A containerized benchmarking tool for machine-learning classification of tabular data.","volume":"9","author":"S Piccolo","year":"2020","journal-title":"GigaScience."},{"key":"ref-32","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3893617","article-title":"CODECHECK certificate 2020-002.","author":"S Eglen","year":"2020","journal-title":"Zenodo."},{"key":"ref-33","doi-asserted-by":"publisher","first-page":"61-70","DOI":"10.1088\/0954-898X_3_1_008","article-title":"The principal components of natural images.","volume":"3","author":"P Hancock","year":"1992","journal-title":"Network: Computation in Neural Systems."},{"key":"ref-34","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3893617","article-title":"CODECHECK certificate 2020-003.","author":"D N\u00fcst","year":"2020","journal-title":"Zenodo."},{"key":"ref-35","doi-asserted-by":"publisher","first-page":"2554-2558","DOI":"10.1073\/pnas.79.8.2554","article-title":"Neural networks and physical systems with emergent collective computational abilities.","volume":"79","author":"J Hopfield","year":"1982","journal-title":"Proc Natl Acad Sci U S A."},{"key":"ref-36","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3893617","article-title":"CODECHECK certificate 2020-004.","author":"D N\u00fcst","year":"2020","journal-title":"Zenodo."},{"key":"ref-37","doi-asserted-by":"publisher","first-page":"834-846","DOI":"10.1109\/TSMC.1983.6313077","article-title":"Neuronlike adaptive elements that can solve difficult learning control problems.","volume":"SMC-13","author":"A Barto","year":"1983","journal-title":"IEEE Trans Syst Man Cybern."},{"key":"ref-38","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3893617","article-title":"CODECHECK certificate 2020-005.","author":"S Eglen","year":"2020","journal-title":"Zenodo."},{"key":"ref-39","doi-asserted-by":"publisher","DOI":"10.5281\/ZENODO.3538217","article-title":"[Re] Connectivity reflects coding a model of voltage-based STDP with homeostasis.","volume":"5","author":"R Larisch","year":"2019","journal-title":"ReScience C."},{"key":"ref-40","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3893617","article-title":"CODECHECK certificate 2020-006.","author":"S Eglen","year":"2020","journal-title":"Zenodo."},{"key":"ref-41","doi-asserted-by":"publisher","first-page":"#7","DOI":"10.5281\/ZENODO.1003214","article-title":"[Re] A Generalized Linear Integrate-And-Fire Neural Model Produces Diverse Spiking Behaviors.","volume":"3","author":"G Detorakis","year":"2017","journal-title":"ReScience C."},{"key":"ref-42","article-title":"CODECHECK certificate 2020-007.","author":"S Eglen","year":"2020","journal-title":"Zenodo."},{"key":"ref-43","doi-asserted-by":"publisher","first-page":"#6","DOI":"10.5281\/ZENODO.1327348","article-title":"[Re] Spike Timing Dependent Plasticity Finds The Start Of Repeating Patterns In Continuous Spike Trains.","volume":"4","author":"P Hathway","year":"2018","journal-title":"ReScience C."},{"key":"ref-44","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3893617","article-title":"CODECHECK certificate 2020-008.","author":"S Eglen","year":"2020","journal-title":"Zenodo."},{"key":"ref-45","doi-asserted-by":"crossref","DOI":"10.1016\/S2468-2667(20)30133-X","article-title":"Effects of non-pharmaceutical interventions on COVID-19 cases, deaths, and demand for hospital services in the UK: a modelling study.","author":"N Davies","year":"2020"},{"key":"ref-46","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3893617","article-title":"CODECHECK certificate 2020-009.","author":"S Eglen","year":"2020","journal-title":"Zenodo."},{"key":"ref-47","doi-asserted-by":"crossref","DOI":"10.1016\/S1473-3099(20)30457-6","article-title":"Effectiveness of isolation, testing, contact tracing and physical distancing on reducing transmission of sars-cov-2 in different settings: a mathematical modelling study.","author":"A Kucharski","year":"2020"},{"key":"ref-48","doi-asserted-by":"publisher","DOI":"10.25561\/77482","article-title":"Report 9: Impact of non-pharmaceutical interventions (NPIs) to reduce COVID19 mortality and healthcare demand.","author":"N Ferguson","year":"2020"},{"key":"ref-49","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3893617","article-title":"CODECHECK certificate 2020-011.","author":"S Eglen","year":"2020","journal-title":"Zenodo."},{"key":"ref-50","doi-asserted-by":"publisher","first-page":"257-261","DOI":"10.1038\/s41586-020-2405-7","article-title":"Estimating the effects of non-pharmaceutical interventions on COVID-19 in Europe.","volume":"584","author":"S Flaxman","year":"2020","journal-title":"Nature."},{"key":"ref-51","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3893617","article-title":"CODECHECK certificate 2020-012.","author":"S Eglen","year":"2020","journal-title":"Zenodo."},{"key":"ref-52","doi-asserted-by":"publisher","DOI":"10.25561\/79231","article-title":"Report 23: State-level tracking of COVID-19 in the United States.","author":"H Unwin","year":"2020"},{"key":"ref-53","doi-asserted-by":"publisher","first-page":"6189","DOI":"10.1038\/s41467-020-19652-6","article-title":"State-level tracking of COVID-19 in the United States.","volume":"11","author":"H Unwin","year":"2020","journal-title":"Nat Commun."},{"key":"ref-54","doi-asserted-by":"publisher","DOI":"10.1101\/2020.06.02.129502","article-title":"Rest-activity cycles and melatonin phase angle of circadian entrainment in people without cone-mediated vision.","author":"M Spitschan","year":"2020","journal-title":"bioRxiv."},{"key":"ref-55","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3967326","article-title":"CODECHECK certificate 2020-014.","author":"I Davies","year":"2020","journal-title":"Zenodo."},{"key":"ref-56","doi-asserted-by":"publisher","first-page":"e52757","DOI":"10.7554\/eLife.52757","article-title":"Patterned perturbation of inhibition can reveal the dynamical structure of neural processing.","volume":"9","author":"S Sadeh","year":"2020","journal-title":"elife."},{"key":"ref-57","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3978402","article-title":"CODECHECK certificate 2020-015.","author":"I Davies","year":"2020","journal-title":"Zenodo."},{"key":"ref-58","doi-asserted-by":"publisher","first-page":"e50927","DOI":"10.7554\/eLife.50927","article-title":"A model for focal seizure onset, propagation, evolution, and progression.","volume":"9","author":"J Liou","year":"2020","journal-title":"eLife."},{"key":"ref-59","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.3981253","article-title":"CODECHECK certificate 2020-016.","author":"D N\u00fcst","year":"2020","journal-title":"Zenodo."},{"key":"ref-60","doi-asserted-by":"publisher","DOI":"10.1007\/s10109-020-00334-2","article-title":"Opening practice: supporting reproducibility and critical spatial data science.","author":"C Brunsdon","year":"2020","journal-title":"J Geogr Syst."},{"key":"ref-61","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.4003848","article-title":"CODECHECK certificate 2020-017","author":"D N\u00fcst","year":"2020","journal-title":"Zenodo."},{"key":"ref-62","doi-asserted-by":"publisher","DOI":"10.1007\/s10109-020-00336-0","article-title":"Progress in the r ecosystem for representing and handling spatial data.","author":"R Bivand","year":"2020","journal-title":"J Geogr Syst."},{"key":"ref-63","doi-asserted-by":"publisher","DOI":"10.17605\/OSF.IO\/ZTC7M","article-title":"Reproducibility review of: Integrating cellular automata and discrete global grid systems: a case study into wildfire modelling.","author":"D N\u00fcst","year":"2020","journal-title":"Open Science Framework."},{"key":"ref-64","doi-asserted-by":"publisher","first-page":"1-23","DOI":"10.5194\/agile-giss-1-6-2020","article-title":"Integrating cellular automata and discrete global grid systems: a case study into wildfire modelling.","volume":"1","author":"M Hojati","year":"2020","journal-title":"AGILE: GIScience Series."},{"key":"ref-65","doi-asserted-by":"publisher","DOI":"10.17605\/OSF.IO\/5SVMT","article-title":"Reproducibility review of: What to do in the meantime: A service coverage analysis for parked autonomous vehicles","author":"D N\u00fcst","year":"2020","journal-title":"Open Science Framework."},{"key":"ref-66","doi-asserted-by":"publisher","first-page":"1-15","DOI":"10.5194\/agile-giss-1-7-2020","article-title":"What to do in the meantime: A service coverage analysis for parked autonomous vehicles.","volume":"1","author":"S Illium","year":"2020","journal-title":"AGILE: GIScience Series."},{"key":"ref-67","doi-asserted-by":"publisher","DOI":"10.17605\/OSF.IO\/7TWR2","article-title":"Reproducibility review of: Window operators for processing spatio-temporal data streams on unmanned vehicles","author":"D N\u00fcst","year":"2020","journal-title":"Open Science Framework."},{"key":"ref-68","doi-asserted-by":"publisher","first-page":"1-23","DOI":"10.5194\/agile-giss-1-21-2020","article-title":"Window operators for processing spatio-temporal data streams on unmanned vehicles.","volume":"1","author":"T Werner","year":"2020","journal-title":"AGILE: GIScience Series."},{"key":"ref-69","doi-asserted-by":"publisher","DOI":"10.17605\/OSF.IO\/SUWPJ","article-title":"Reproducibility review of: Comparing supervised learning algorithms for spatial nominal entity recognition","author":"F Ostermann","year":"2020","journal-title":"Open Science Framework."},{"key":"ref-70","doi-asserted-by":"publisher","first-page":"1-18","DOI":"10.5194\/agile-giss-1-15-2020","article-title":"Comparing supervised learning algorithms for spatial nominal entity recognition.","volume":"1","author":"A Medad","year":"2020","journal-title":"AGILE: GIScience Series."},{"key":"ref-71","doi-asserted-by":"publisher","DOI":"10.17605\/OSF.IO\/7XRQG","article-title":"Reproducibility review of: Extracting interrogative intents and concepts from geo-analytic questions","author":"D N\u00fcst","year":"2020","journal-title":"Open Science Framework."},{"key":"ref-72","doi-asserted-by":"publisher","first-page":"1-21","DOI":"10.5194\/agile-giss-1-23-2020","article-title":"Extracting interrogative intents and concepts from geo-analytic questions.","volume":"1","author":"H Xu","year":"2020","journal-title":"AGILE: GIScience Series."},{"key":"ref-73","doi-asserted-by":"publisher","DOI":"10.17605\/OSF.IO\/XS5YR","article-title":"Reproducibility review of: Tracking hurricane dorian in gdelt and twitter","author":"F Ostermann","year":"2020","journal-title":"Open Science Framework."},{"key":"ref-74","doi-asserted-by":"publisher","first-page":"1-18","DOI":"10.5194\/agile-giss-1-19-2020","article-title":"Tracking hurricane dorian in GDELT and twitter.","volume":"1","author":"I Owuor","year":"2020","journal-title":"AGILE: GIScience Series."},{"key":"ref-75","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.4310025","article-title":"CODECHECK certificate 2020-024.","author":"S Eglen","year":"2020","journal-title":"Zenodo."},{"key":"ref-76","doi-asserted-by":"publisher","first-page":"giab030","DOI":"10.1093\/gigascience\/giab030","article-title":"Driftage: a multi-agent system framework for concept drift detection.","volume":"10","author":"D Vieira","year":"2021","journal-title":"Gigascience."},{"key":"ref-77","doi-asserted-by":"publisher","first-page":"105306","DOI":"10.1016\/j.jas.2020.105306","article-title":"The application of Local Indicators for Categorical Data (LICD) to explore spatial dependence in archaeological spaces.","volume":"126","author":"F Carrer","year":"2021","journal-title":"J Archaeol Sci."},{"key":"ref-78","doi-asserted-by":"publisher","DOI":"10.5281\/zenodo.4486559","article-title":"codecheckers\/register: CODECHECK Register Deposit.","author":"D N\u00fcst","year":"2021","journal-title":"Zenodo."},{"key":"ref-79","article-title":"codecheckers\/codecheck: codecheck R package version 0.1.0","author":"S Eglen","year":"2021"},{"key":"ref-80","article-title":"zen4R: Interface to \u2019Zenodo\u2019 REST API.","author":"E Blondel","year":"2020"},{"key":"ref-81","doi-asserted-by":"publisher","first-page":"5","DOI":"10.1145\/2743015","article-title":"Editorial: ACM TOMS Replicated Computational Results Initiative.","volume":"41","author":"M Heroux","year":"2015","journal-title":"ACM Trans Math Softw."},{"key":"ref-82","doi-asserted-by":"publisher","first-page":"729-729","DOI":"10.1038\/s42256-020-00277-9","article-title":"Research, reuse, repeat.","volume":"2","year":"2020","journal-title":"Nat Mach Intell."},{"key":"ref-83","doi-asserted-by":"publisher","first-page":"A1-A3","DOI":"10.1016\/j.cortex.2020.04.020","article-title":"Verification Reports: A new article type at Cortex.","volume":"129","author":"C Chambers","year":"2020","journal-title":"Cortex."},{"key":"ref-84","doi-asserted-by":"publisher","first-page":"01620001","DOI":"10.1061\/(ASCE)WR.1943-5452.0001368","article-title":"Reproducible Results Policy.","volume":"147","author":"D Rosenberg","year":"2021","journal-title":"J Water Resour Plan Manag."},{"key":"ref-85","doi-asserted-by":"publisher","first-page":"17","DOI":"10.12688\/gatesopenres.13108.2","article-title":"Internal replication of computational workflows in scientific research.","volume":"4","author":"J Benjamin-Chung","year":"2020","journal-title":"Gates Open Res."},{"key":"ref-86","doi-asserted-by":"publisher","first-page":"637-647","DOI":"10.1016\/j.procs.2011.04.067","article-title":"A Universal Identifier for Computational Results.","volume":"4","author":"M Gavish","year":"2011","journal-title":"Procedia Comput Sci."},{"key":"ref-87","doi-asserted-by":"publisher","first-page":"01820002","DOI":"10.1061\/(ASCE)WR.1943-5452.0001215","article-title":"The Next Frontier: Making Research More Reproducible.","volume":"146","author":"D Rosenberg","year":"2020","journal-title":"J Water Resour Plann Manage."},{"key":"ref-88","doi-asserted-by":"publisher","DOI":"10.17605\/OSF.IO\/CB7Z8","article-title":"AGILE Reproducible Paper Guidelines","author":"D N\u00fcst","year":"2019"},{"key":"ref-89","doi-asserted-by":"publisher","first-page":"24","DOI":"10.1186\/s13041-020-0552-2","article-title":"No raw data, no science: another possible source of the reproducibility crisis.","volume":"13","author":"T Miyakawa","year":"2020","journal-title":"Mol Brain."},{"key":"ref-90","doi-asserted-by":"publisher","DOI":"10.1045\/january2017-nuest","article-title":"Opening the Publication Process with Executable Research Compendia.","volume":"23","author":"D Nst","year":"2017","journal-title":"D-Lib Magazine."},{"key":"ref-91","doi-asserted-by":"publisher","DOI":"10.1016\/j.future.2017.12.029","article-title":"Computing environments for reproducibility: Capturing the \u201cWhole Tale\u201d.","author":"A Brinckman","year":"2018","journal-title":"Future Gener Comput Syst."},{"key":"ref-92","doi-asserted-by":"publisher","first-page":"342-346","DOI":"10.1038\/nbt.3780","article-title":"Reproducibility of computational workflows is automated using continuous analysis.","volume":"35","author":"B Beaulieu-Jones","year":"2017","journal-title":"Nat Biotechnol."},{"key":"ref-93","doi-asserted-by":"publisher","first-page":"190030","DOI":"10.1038\/sdata.2019.30","article-title":"Assessing data availability and research reproducibility in hydrology and water resources.","volume":"6","author":"J Stagge","year":"2019","journal-title":"Sci Data."},{"key":"ref-94","doi-asserted-by":"publisher","first-page":"2021-2035","DOI":"10.1109\/TBME.2016.2539602","article-title":"Reproducibility in Computational Neuroscience Models and Simulations.","volume":"63","author":"R McDougal","year":"2016","journal-title":"IEEE Trans Biomed Eng."},{"key":"ref-95","doi-asserted-by":"publisher","first-page":"223-230","DOI":"10.11613\/BM.2009.020","article-title":"The role of statistical reviewer in biomedical scientific journal.","volume":"19","author":"M Petrove\u010dki","year":"2009","journal-title":"Biochemia Medica."},{"key":"ref-96","doi-asserted-by":"publisher","first-page":"270","DOI":"10.1186\/s12916-015-0510-5","article-title":"How to spot a statistical problem: advice for a non-statistical reviewer.","volume":"13","author":"D Greenwood","year":"2015","journal-title":"BMC Med."},{"key":"ref-97","article-title":"Code Review For and By Scientists.","author":"M Petre","year":"2014","journal-title":"arXiv: 1407.5648 [cs]."},{"key":"ref-98","first-page":"148-151","article-title":"Incentives and barriers to reproducibility: Investments and returns.","volume":"6","author":"P Rosenthal","year":"2016","journal-title":"Dagstuhl reports: Reproducibility of data-oriented experiments in e-Science."},{"key":"ref-99","doi-asserted-by":"publisher","first-page":"247-248","DOI":"10.1038\/d41586-019-03366-x","article-title":"Make code accessible with these cloud services.","volume":"575","author":"J Perkel","year":"2019","journal-title":"Nature."},{"key":"ref-100","doi-asserted-by":"publisher","first-page":"1911-1926","DOI":"10.1080\/13658816.2018.1464167","article-title":"Opening GIScience: A process-based approach.","volume":"32","author":"J Shannon","year":"2018","journal-title":"Int J Geogr Inf Sci."},{"key":"ref-101","doi-asserted-by":"publisher","first-page":"A1-A3","DOI":"10.1016\/j.polgeo.2014.11.001","article-title":"Data ethics: Pluralism, replication, conflicts of interest, and standards in Political Geography.","volume":"44","author":"J O\u2019Loughlin","year":"2015","journal-title":"Political Geography."},{"key":"ref-102","doi-asserted-by":"crossref","first-page":"127-128","DOI":"10.1126\/science.aaw2825","article-title":"Certify reproducibility with confidential data.","volume":"365","author":"C P\u00e9rignon","year":"2019","journal-title":"Science."},{"key":"ref-103","doi-asserted-by":"publisher","first-page":"102-120","DOI":"10.1177\/0002716217742610","article-title":"Research Infrastructure for the Safe Analysis of Sensitive Data.","volume":"675","author":"I Foster","year":"2018","journal-title":"Ann Am Acad Pol Soc Sci."},{"key":"ref-104","doi-asserted-by":"publisher","first-page":"598-617","DOI":"10.1177\/2399808317710132","article-title":"More bark than bytes? Reflections on 21+ years of geocomputation.","volume":"44","author":"R Harris","year":"2017","journal-title":"Environ Plan B Urban Anal City Sci."},{"key":"ref-105","doi-asserted-by":"publisher","DOI":"10.17605\/OSF.IO\/XEU42","article-title":"Everything Hertz 97: Slow science.","author":"D Quintana","year":"2019","journal-title":"Open Science Framework."},{"key":"ref-106","doi-asserted-by":"publisher","first-page":"100985","DOI":"10.1016\/j.tranon.2020.100985","article-title":"Proposal for 'segmented peer review' of multidisciplinary papers.","volume":"14","author":"D Dinakaran","year":"2021","journal-title":"Transl Oncol."},{"key":"ref-107","doi-asserted-by":"publisher","first-page":"e38234","DOI":"10.1371\/journal.pone.0038234","article-title":"The effects of FreeSurfer version, workstation type, and macintosh operating system version on anatomical volume and cortical thickness measurements.","volume":"7","author":"E Gronenschild","year":"2012","journal-title":"PLoS One."},{"key":"ref-108","doi-asserted-by":"publisher","first-page":"101698","DOI":"10.1016\/j.isci.2020.101698","article-title":"The Rigor and Transparency Index Quality Metric for Assessing Biological and Medical Science Methods.","volume":"23","author":"J Menke","year":"2020","journal-title":"iScience."},{"key":"ref-109","doi-asserted-by":"publisher","first-page":"775-778","DOI":"10.1042\/ETLS20180172","article-title":"Recent developments in scholarly publishing to improve research practices in the life sciences.","volume":"2","author":"S Eglen","year":"2018","journal-title":"Emerg Top Life Sci."},{"key":"ref-110","doi-asserted-by":"publisher","first-page":"34","DOI":"10.3390\/publications7020034","article-title":"Ten Hot Topics around Scholarly Publishing.","volume":"7","author":"J Tennant","year":"2019","journal-title":"Publications."},{"key":"ref-111","article-title":"What you need to know about how coronavirus is changing science","author":"M Munafo","year":"2020"},{"key":"ref-112","doi-asserted-by":"publisher","first-page":"159","DOI":"10.1038\/493159a","article-title":"Altmetrics: Value all research products.","volume":"493","author":"H Piwowar","year":"2013","journal-title":"Nature."},{"key":"ref-113","doi-asserted-by":"publisher","first-page":"1422-1425","DOI":"10.1126\/science.aab2374","article-title":"SCIENTIFIC STANDARDS. Promoting an open research culture.","volume":"348","author":"B Nosek","year":"2015","journal-title":"Science."},{"key":"ref-114","article-title":"New EarthCube Peer-Reviewed Jupyter Notebooks. Now Available","year":"2020"},{"key":"ref-115","doi-asserted-by":"publisher","first-page":"17-18","DOI":"10.1038\/d41586-019-00724-7","article-title":"Pioneering \u2018live-code\u2019 article allows scientists to play with each other\u2019s results.","volume":"567","author":"J Perkel","year":"2019","journal-title":"Nature."},{"key":"ref-116","doi-asserted-by":"publisher","DOI":"10.24251\/HICSS.2019.560","article-title":"Towards a Decentralized Process for Scientific Publication and Peer Review using Blockchain and IPFS","author":"A Tenorio-Forn\u00e9s","year":"2019"},{"key":"ref-117","article-title":"Scholarly infrastructures for research software: report from the EOSC Executive Board Working Group (WG) Architecture Task Force (TF) SIRS.","year":"2020"},{"key":"ref-118","doi-asserted-by":"publisher","DOI":"10.17864\/1926.92781","article-title":"Brewing up a storm: developing Open Research culture through ReproducibiliTea.","author":"L Fitzgibbon","year":"2020","journal-title":"Report."},{"key":"ref-119","doi-asserted-by":"publisher","DOI":"10.31219\/osf.io\/dxw67","article-title":"Reproducibility for everyone: a community-led initiative with global reach in reproducible research training.","author":"S Auer","year":"2020","journal-title":"OSF Preprints"}],"container-title":["F1000Research"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/f1000research.com\/articles\/10-253\/v2\/xml","content-type":"application\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/f1000research.com\/articles\/10-253\/v2\/pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/f1000research.com\/articles\/10-253\/v2\/iparadigms","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,1,4]],"date-time":"2023-01-04T18:26:56Z","timestamp":1672856816000},"score":1,"resource":{"primary":{"URL":"https:\/\/f1000research.com\/articles\/10-253\/v2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,7,20]]},"references-count":119,"URL":"https:\/\/doi.org\/10.12688\/f1000research.51738.2","relation":{"has-review":[{"id-type":"doi","id":"10.5256\/f1000research.57857.r89926","asserted-by":"subject"},{"id-type":"doi","id":"10.5256\/f1000research.57857.r89926","asserted-by":"object"}]},"ISSN":["2046-1402"],"issn-type":[{"value":"2046-1402","type":"electronic"}],"subject":[],"published":{"date-parts":[[2021,7,20]]},"assertion":[{"value":"Indexed","URL":"https:\/\/f1000research.com\/articles\/10-253\/v2#article-reports","order":0,"name":"referee-status","label":"Referee status","group":{"name":"current-referee-status","label":"Current Referee Status"}},{"value":"10.5256\/f1000research.57857.r89926, Nicolas P. Rougier, Inria Bordeaux Sud-Ouest, Talence, France, 20 Jul 2021, version 2, indexed","URL":"https:\/\/f1000research.com\/articles\/10-253\/v2#referee-response-89926","order":0,"name":"referee-response-89926","label":"Referee Report","group":{"name":"article-reports","label":"Article Reports"}},{"value":"10.5256\/f1000research.54932.r82472, Sarah Gibson, The Alan Turing Institute, London, UK, 21 Apr 2021, version 1, 1 approved, 1 approved with reservations","URL":"https:\/\/f1000research.com\/articles\/10-253\/v1#referee-response-82472","order":1,"name":"referee-response-82472","label":"Referee Report","group":{"name":"article-reports","label":"Article Reports"}},{"value":"<b>Stephen Stephen<\/b>; <i>Posted: 16 Jun 2021<\/i>; > 1. Is there any community feedback on the transparency and reusability of research published with CODECHECK certificates? This is an excellent question.  It is perhaps too early for us to assess this given most certificates are under a year old, but we are not aware of any reuse yet of the codecheck-deposited material.  However, we certainly think it interesting to try and monitor this over a longer (3-5 year) timescale if possible.  As well as looking for citations of certificates, we could also check for forks of our repositories, and download statistics from Zenodo.  We note this as a good closing point for our article.","URL":"https:\/\/f1000research.com\/articles\/10-253\/v2#referee-comment-6806","order":2,"name":"referee-comment-6806","label":"Referee Comment","group":{"name":"article-reports","label":"Article Reports"}},{"value":"This work was financially supported by the UK Software Sustainability Institute and a Mozilla Science mini grant. DN is supported by grant PE1632\/17-1 from the German Research Foundation (DFG). The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript.","order":3,"name":"grant-information","label":"Grant Information"},{"value":"This is an open access article distributed under the terms of the Creative Commons Attribution Licence, which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.","order":0,"name":"copyright-info","label":"Copyright"}]}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1287_2Fisre_2021_1079 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T06:05:32Z","timestamp":1793253932487,"version":"4.5.0"},"reference-count":49,"publisher":"Institute for Operations Research and the Management Sciences (INFORMS)","issue":"2","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Information Systems Research"],"published-print":{"date-parts":[[2022,6]]},"abstract":"<jats:p>A consensus is beginning to emerge that the next phase of artificial intelligence (AI) induction in business organizations will require humans to work with AI in a variety of work arrangements. This article explores the issues related to human capabilities to work with AI. A key to working in many work arrangements is the ability to delegate work to entities that can do them most efficiently. Modern AI can do a remarkable job of efficient delegation to humans because it knows what it knows well and what it does not. Humans, on the other hand, are poor judges of their metaknowledge and are not good at delegating knowledge work to AI\u2014this might prove to be a big stumbling block to create work environments where humans and AI work together. Humans have often created machines to serve them. The sentiment is perhaps exemplified by Oscar Wilde\u2019s statement that \u201ccivilization requires slaves\u2026. Human slavery is wrong, insecure and demoralizing. On mechanical slavery, on the slavery of the machine, the future of the world depends.\u201d However, the time has come when humans might switch roles with machines. Our study highlights capabilities that humans need to effectively work with AI and still be in control rather than just being directed.<\/jats:p>","DOI":"10.1287\/isre.2021.1079","type":"journal-article","created":{"date-parts":[[2021,12,10]],"date-time":"2021-12-10T17:01:10Z","timestamp":1639155670000},"page":"678-696","source":"Crossref","is-referenced-by-count":496,"title":["Cognitive Challenges in Human\u2013Artificial Intelligence Collaboration: Investigating the Path Toward Productive Delegation"],"prefix":"10.1287","volume":"33","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-4580-7444","authenticated-orcid":false,"given":"Andreas","family":"F\u00fcgener","sequence":"first","affiliation":[{"name":"University of Cologne, 50923 Cologne, Germany;"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2061-4889","authenticated-orcid":false,"given":"J\u00f6rn","family":"Grahl","sequence":"additional","affiliation":[{"name":"University of Cologne, 50923 Cologne, Germany;"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2097-1643","authenticated-orcid":false,"given":"Alok","family":"Gupta","sequence":"additional","affiliation":[{"name":"University of Minnesota, Minneapolis, Minnesota 55455;"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-9008-142X","authenticated-orcid":false,"given":"Wolfgang","family":"Ketter","sequence":"additional","affiliation":[{"name":"University of Cologne, 50923 Cologne, Germany;"},{"name":"Erasmus University Rotterdam, 3062 PA Rotterdam, Netherlands"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"109","reference":[{"key":"B1","volume-title":"Prediction Machines: The Simple Economics of Artificial Intelligence","author":"Agrawal A","year":"2018"},{"key":"B2","doi-asserted-by":"publisher","DOI":"10.3758\/s13428-021-01588-4"},{"key":"B3","doi-asserted-by":"publisher","DOI":"10.1080\/13645579.2018.1563966"},{"key":"B5","doi-asserted-by":"publisher","DOI":"10.1162\/003355303322552801"},{"key":"B6","doi-asserted-by":"publisher","DOI":"10.25300\/MISQ\/2021\/15882"},{"key":"B7","doi-asserted-by":"publisher","DOI":"10.1177\/001979398503800403"},{"key":"B8","doi-asserted-by":"publisher","DOI":"10.1097\/PTS.0000000000000255"},{"key":"B9","doi-asserted-by":"publisher","DOI":"10.1126\/science.aap8062"},{"key":"B10","doi-asserted-by":"publisher","DOI":"10.1257\/pandp.20181019"},{"key":"B11","doi-asserted-by":"publisher","DOI":"10.1037\/14805-009"},{"key":"B12","doi-asserted-by":"publisher","DOI":"10.3758\/s13428-019-01273-7"},{"key":"B13","doi-asserted-by":"publisher","DOI":"10.1017\/psrm.2018.10"},{"key":"B14","doi-asserted-by":"publisher","DOI":"10.1037\/0003-066X.34.7.571"},{"key":"B15","doi-asserted-by":"publisher","DOI":"10.1561\/2000000039"},{"key":"B16","doi-asserted-by":"publisher","DOI":"10.1037\/xge0000033"},{"key":"B17","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.2016.2643"},{"key":"B18","first-page":"238","volume-title":"Proc. 24th Internat. Conf. World Wide Web","author":"Difallah DE","year":"2015"},{"key":"B19","doi-asserted-by":"publisher","DOI":"10.1080\/014492999118832"},{"key":"B20","doi-asserted-by":"publisher","DOI":"10.1038\/nature21056"},{"key":"B21","doi-asserted-by":"publisher","DOI":"10.1126\/science.1201765"},{"issue":"2","key":"B22","first-page":"203","volume":"105","author":"Foster ED","year":"2017","journal-title":"J. Medical Library Assoc."},{"key":"B23","doi-asserted-by":"publisher","DOI":"10.25300\/MISQ\/2021\/16553"},{"key":"B24","doi-asserted-by":"publisher","DOI":"10.1287\/isre.2021.1009"},{"issue":"5","key":"B25","first-page":"1027","volume":"34","author":"Hansson P","year":"2008","journal-title":"J. Experiment. Psych. General"},{"key":"B26","doi-asserted-by":"publisher","DOI":"10.1145\/3173574.3174023"},{"key":"B27","doi-asserted-by":"publisher","DOI":"10.1109\/MSP.2012.2205597"},{"key":"B28","doi-asserted-by":"publisher","DOI":"10.1007\/s10683-011-9273-9"},{"key":"B29","doi-asserted-by":"publisher","DOI":"10.2307\/258105"},{"key":"B30","doi-asserted-by":"publisher","DOI":"10.1287\/isre.2020.0980"},{"key":"B31","doi-asserted-by":"publisher","DOI":"10.25300\/MISQ\/2016\/40.2.09"},{"key":"B32","doi-asserted-by":"publisher","DOI":"10.1027\/2192-0923\/a000114"},{"key":"B33","doi-asserted-by":"publisher","DOI":"10.1037\/0033-2909.107.3.296"},{"key":"B34","doi-asserted-by":"publisher","DOI":"10.1016\/S0933-3657(01)00077-X"},{"key":"B35","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.1110.1382"},{"key":"B36","doi-asserted-by":"publisher","DOI":"10.1038\/nature14539"},{"key":"B37","doi-asserted-by":"publisher","DOI":"10.1111\/poms.12841"},{"key":"B38","doi-asserted-by":"publisher","DOI":"10.1016\/j.obhdp.2018.12.005"},{"key":"B39","doi-asserted-by":"publisher","DOI":"10.4135\/9781446279212.n1"},{"key":"B40","author":"McAfee A","year":"2013","journal-title":"Harvard Bus. Rev."},{"key":"B41","volume-title":"Machine, Platform, Crowd: Harnessing Our Digital","author":"McAfee A","year":"2017","edition":"1"},{"key":"B42","doi-asserted-by":"publisher","DOI":"10.1037\/11281-000"},{"key":"B43","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.1708274114"},{"key":"B44","volume-title":"Drive: The Surprising Truth About What Motivates Us","author":"Pink DH","year":"2011","edition":"1"},{"key":"B46","doi-asserted-by":"publisher","DOI":"10.1007\/s11263-015-0816-y"},{"issue":"2","key":"B47","first-page":"7","volume":"33","author":"Russo JE","year":"1992","journal-title":"Sloan Management Rev."},{"key":"B48","doi-asserted-by":"publisher","DOI":"10.1287\/isre.2021.1015"},{"key":"B50","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.308"},{"key":"B51","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298594"},{"key":"B52","doi-asserted-by":"publisher","DOI":"10.1613\/jair.1.11345"}],"container-title":["Information Systems Research"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/pubsonline.informs.org\/doi\/pdf\/10.1287\/isre.2021.1079","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2023,4,2]],"date-time":"2023-04-02T17:00:43Z","timestamp":1680454843000},"score":1,"resource":{"primary":{"URL":"https:\/\/pubsonline.informs.org\/doi\/10.1287\/isre.2021.1079"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,6]]},"references-count":49,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2022,6]]}},"alternative-id":["10.1287\/isre.2021.1079"],"URL":"https:\/\/doi.org\/10.1287\/isre.2021.1079","relation":{},"ISSN":["1047-7047","1526-5536"],"issn-type":[{"value":"1047-7047","type":"print"},{"value":"1526-5536","type":"electronic"}],"subject":[],"published":{"date-parts":[[2022,6]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1371_2Fjournal_pbio_1002106 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T07:06:23Z","timestamp":1793171183641,"version":"4.5.0"},"reference-count":66,"publisher":"Public Library of Science (PLoS)","issue":"3","license":[{"start":{"date-parts":[[2015,3,13]],"date-time":"2015-03-13T00:00:00Z","timestamp":1426204800000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["www.plosbiology.org"],"crossmark-restriction":false},"short-container-title":["PLoS Biol"],"DOI":"10.1371\/journal.pbio.1002106","type":"journal-article","created":{"date-parts":[[2015,3,13]],"date-time":"2015-03-13T14:01:59Z","timestamp":1426255319000},"page":"e1002106","update-policy":"https:\/\/doi.org\/10.1371\/journal.pbio.corrections_policy","source":"Crossref","is-referenced-by-count":978,"title":["The Extent and Consequences of P-Hacking in Science"],"prefix":"10.1371","volume":"13","author":[{"given":"Megan L.","family":"Head","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Luke","family":"Holman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Rob","family":"Lanfear","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Andrew T.","family":"Kahn","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Michael D.","family":"Jennions","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"340","published-online":{"date-parts":[[2015,3,13]]},"reference":[{"key":"ref1","doi-asserted-by":"crossref","first-page":"687","DOI":"10.3758\/s13415-013-0201-7","article-title":"Introduction to the special issue on reliability and replication in cognitive and affective neuroscience research","volume":"13","author":"DM Barch","year":"2013","journal-title":"Cogn Affect Behav Neurosci"},{"key":"ref2","doi-asserted-by":"crossref","first-page":"e124","DOI":"10.1371\/journal.pmed.0020124","article-title":"Why most published research findings are false","volume":"2","author":"JPA Ioannidis","year":"2005","journal-title":"PLoS Med"},{"key":"ref3","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1093\/biostatistics\/kxt007","article-title":"An estimate of the science-wise false discovery rate and application to the top medical literature","volume":"15","author":"LR Jager","year":"2014","journal-title":"Biostatistics"},{"key":"ref4","doi-asserted-by":"crossref","first-page":"419","DOI":"10.2307\/2982993","article-title":"Publication bias\u2014a problem in interpreting medical data","volume":"151","author":"CB Begg","year":"1988","journal-title":"J R Stat Soc Ser A Stat Soc"},{"key":"ref5","doi-asserted-by":"crossref","first-page":"e3081","DOI":"10.1371\/journal.pone.0003081","article-title":"Systematic review of the empirical evidence of study publication bias and outcome reporting bias","volume":"3","author":"K Dwan","year":"2008","journal-title":"PLoS ONE"},{"key":"ref6","doi-asserted-by":"crossref","first-page":"891","DOI":"10.1007\/s11192-011-0494-7","article-title":"Negative results are disappearing from most disciplines and countries","volume":"90","author":"D Fanelli","year":"2012","journal-title":"Scientometrics"},{"key":"ref7","doi-asserted-by":"crossref","first-page":"638","DOI":"10.1037\/0033-2909.86.3.638","article-title":"The file drawer problem and tolerance for null results","volume":"86","author":"R Rosenthal","year":"1979","journal-title":"Psychol Bull"},{"key":"ref8","first-page":"1","article-title":"Publication and related biases","volume":"4","author":"F Song","year":"2000","journal-title":"Health technology assessment (Winchester, England)"},{"key":"ref9","first-page":"30","article-title":"Publication decisions and their possible effects on inferences drawn from tests of significance\u2014or vice versa","volume":"54","author":"TD Sterling","year":"1959","journal-title":"J Am Stat Assoc"},{"key":"ref10","doi-asserted-by":"crossref","first-page":"640","DOI":"10.1136\/bmj.315.7109.640","article-title":"Publication bias: Evidence of delayed publication in a cohort study of clinical research projects","volume":"315","author":"JM Stern","year":"1997","journal-title":"Br Med J"},{"key":"ref11","doi-asserted-by":"crossref","first-page":"817","DOI":"10.1525\/bio.2013.63.10.9","article-title":"Predicting publication success for biologists","volume":"63","author":"WF Laurance","year":"2013","journal-title":"Bioscience"},{"key":"ref12","unstructured":"Brodeur A, Le M, Sangnier M, Zylberberg Y (2012) Star Wars: The empirics strike back. Paris School of Economics Working Paper 2012. <ext-link xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" ext-link-type=\"uri\" xlink:href=\"http:\/\/ssrn.com\/abstract=2089580\" xlink:type=\"simple\">http:\/\/ssrn.com\/abstract=2089580<\/ext-link>."},{"key":"ref13","doi-asserted-by":"crossref","first-page":"7","DOI":"10.1177\/0956797613504966","article-title":"The new statistics: Why and how","volume":"25","author":"G Cumming","year":"2014","journal-title":"Psychol Sci"},{"key":"ref14","doi-asserted-by":"crossref","first-page":"1359","DOI":"10.1177\/0956797611417632","article-title":"False-positive psychology: Undisclosed flexibility in data collection and analysis allows presenting anything as significant","volume":"22","author":"JP Simmons","year":"2011","journal-title":"Psychol Sci"},{"key":"ref15","doi-asserted-by":"crossref","first-page":"e46363","DOI":"10.1371\/journal.pone.0046363","article-title":"Inappropriate fiddling with statistical analyses to obtain a desirable p-value: Tests to detect its presence in published literature","volume":"7","author":"GL Gadbury","year":"2014","journal-title":"PLoS ONE"},{"key":"ref16","doi-asserted-by":"crossref","first-page":"524","DOI":"10.1177\/0956797611430953","article-title":"Measuring the prevalence of questionable research practices with incentives for truth telling","volume":"23","author":"LK John","year":"2012","journal-title":"Psychol Sci"},{"key":"ref17","doi-asserted-by":"crossref","first-page":"359","DOI":"10.1111\/1467-9876.00197","article-title":"Bias in meta-analysis due to outcome variable selection within studies","volume":"49","author":"JL Hutton","year":"2000","journal-title":"J R Stat Soc Ser C Appl Stat"},{"key":"ref18","doi-asserted-by":"crossref","first-page":"731","DOI":"10.1177\/0956797611406447","article-title":"Wishful thinking: Belief, desire, and the motivated evaluation of scientific evidence","volume":"22","author":"A Bastardi","year":"2011","journal-title":"Psychol Sci"},{"key":"ref19","doi-asserted-by":"crossref","first-page":"615","DOI":"10.1177\/1745691612459058","article-title":"Scientific utopia: II. Restructuring incentives and practices to promote truth over publishability","volume":"7","author":"BA Nosek","year":"2012","journal-title":"Perspect Psychol Sci"},{"key":"ref20","author":"RA Fisher","year":"1925"},{"key":"ref21","doi-asserted-by":"crossref","first-page":"13","DOI":"10.1093\/biostatistics\/kxt032","article-title":"Discussion: An estimate of the science-wise false discovery rate and applications to top medical journals by Jager and Leek","volume":"15","author":"Y Benjamini","year":"2014","journal-title":"Biostatistics"},{"key":"ref22","doi-asserted-by":"crossref","first-page":"995","DOI":"10.7326\/0003-4819-130-12-199906150-00008","article-title":"Toward evidence-based medical statistics. 1: The P value fallacy","volume":"130","author":"SN Goodman","year":"1999","journal-title":"Ann Intern Med"},{"key":"ref23","doi-asserted-by":"crossref","first-page":"226","DOI":"10.1136\/bmj.322.7280.226","article-title":"Sifting the evidence\u2014what's wrong with significance tests?","volume":"322","author":"JAC Sterne","year":"2001","journal-title":"Br Med J"},{"key":"ref24","doi-asserted-by":"crossref","first-page":"241","DOI":"10.1037\/1082-989X.5.2.241","article-title":"Null hypothesis significance testing: A review of an old and continuing controversy","volume":"5","author":"RS Nickerson","year":"2000","journal-title":"Psychol Methods"},{"key":"ref25","doi-asserted-by":"crossref","first-page":"526","DOI":"10.1037\/0033-295X.110.3.526","article-title":"Hypothesis testing and theory evaluation at the boundaries: Surprising insights from Bayes's theorem","volume":"110","author":"D Trafimow","year":"2003","journal-title":"Psychol Rev"},{"key":"ref26","author":"FJ Rohlf","year":"1995"},{"key":"ref27","doi-asserted-by":"crossref","first-page":"1082","DOI":"10.1111\/j.1420-9101.2006.01291.x","article-title":"An unexpected influence of widely used significance thresholds on the distribution of reported P-values","volume":"20","author":"J Ridley","year":"2007","journal-title":"J Evol Biol"},{"key":"ref28","doi-asserted-by":"crossref","first-page":"591","DOI":"10.1111\/j.1469-185X.2007.00027.x","article-title":"Effect size, confidence interval and statistical significance: A practical guide for biologists","volume":"82","author":"S Nakagawa","year":"2007","journal-title":"Biol Rev Camb Philos Soc"},{"key":"ref29","doi-asserted-by":"crossref","first-page":"912","DOI":"10.2307\/3803199","article-title":"Null hypothesis testing: Problems, prevalence, and an alternative","volume":"64","author":"DR Anderson","year":"2000","journal-title":"J Wildl Manage"},{"key":"ref30","doi-asserted-by":"crossref","first-page":"161","DOI":"10.1111\/1467-8721.ep11512376","article-title":"Psychology will be a much better science when we change the way we analyze data","volume":"5","author":"GR Loftus","year":"1996","journal-title":"Curr Dir Psychol Sci"},{"key":"ref31","doi-asserted-by":"crossref","first-page":"71","DOI":"10.1198\/016214504000001907","article-title":"False discovery rate-adjusted multiple confidence intervals for selected parameters","volume":"100","author":"Y Benjamini","year":"2005","journal-title":"J Am Stat Assoc"},{"key":"ref32","unstructured":"Simonsohn U (2014c) Posterior-hacking: Selective reporting invalidates Bayesian results also. <ext-link xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" ext-link-type=\"uri\" xlink:href=\"http:\/\/ssrncom\/abstract=2374040\" xlink:type=\"simple\">http:\/\/ssrncom\/abstract=2374040<\/ext-link>."},{"key":"ref33","article-title":"Handbook of met-analysis in ecology and evolution","author":"J Koricheva","year":"2013"},{"key":"ref34","doi-asserted-by":"crossref","first-page":"1060","DOI":"10.1016\/j.jclinepi.2010.12.012","article-title":"Statistically significant meta-analyses of clinical trials have modest credibility and inflated effects","volume":"64","author":"TV Pereira","year":"2011","journal-title":"J Clin Epidemiol"},{"key":"ref35","doi-asserted-by":"crossref","first-page":"191","DOI":"10.1111\/j.0030-1299.2004.12265.x","article-title":"Meta-analysis can "fail": Reply to Kotiaho and Tomkins","volume":"104","author":"MD Jennions","year":"2004","journal-title":"Oikos"},{"key":"ref36","doi-asserted-by":"crossref","first-page":"551","DOI":"10.1034\/j.1600-0706.2002.960316.x","article-title":"Meta-analysis, can it ever fail?","volume":"96","author":"JS Kotiaho","year":"2002","journal-title":"Oikos"},{"key":"ref37","doi-asserted-by":"crossref","first-page":"441","DOI":"10.1146\/annurev.ecolsys.31.1.441","article-title":"Quasireplication and the contract of error: Lessons from sex ratios, heritabilities and fluctuating asymmetry","volume":"31","author":"AR Palmer","year":"2000","journal-title":"Annu Rev Ecol Sys"},{"key":"ref38","doi-asserted-by":"crossref","first-page":"221","DOI":"10.1086\/506236","article-title":"Replicating empirical research in behavioural ecology: How and why it should be done but rarely ever is","volume":"81","author":"CD Kelly","year":"2006","journal-title":"The Quarterly Review of Biology"},{"key":"ref39","doi-asserted-by":"crossref","first-page":"286","DOI":"10.1111\/j.1745-6924.2008.00079.x","article-title":"Replication and p intervals p values predict the future only vaguely, but confidence Intervals do much better","volume":"3","author":"G Cumming","year":"2008","journal-title":"Perspectives on Psychological Science"},{"key":"ref40","first-page":"2271","article-title":"A peculiar prevalence of p values just below .05","volume":"65","author":"EJ Mariscampo","year":"2012","journal-title":"Q Rev Biol"},{"key":"ref41","doi-asserted-by":"crossref","first-page":"534","DOI":"10.1037\/a0033242","article-title":"P-curve: A key to the file drawer","volume":"143","author":"U Simonsohn","year":"2014","journal-title":"J Exp Psychol Gen"},{"key":"ref42","doi-asserted-by":"crossref","first-page":"229","DOI":"10.2307\/1905466","article-title":"Compounding probabilities from independent significance tests","volume":"10","author":"WA Wallis","year":"1942","journal-title":"Econometrica"},{"key":"ref43","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1177\/0049124108318973","article-title":"Publication bias in empirical sociological research\u2014Do arbitrary significance levels distort published results?","volume":"37","author":"AS Gerber","year":"2008","journal-title":"Sociol Methods Res"},{"key":"ref44","doi-asserted-by":"crossref","first-page":"603","DOI":"10.1111\/jels.12031","article-title":"Foreword: Conference bias","volume":"10","author":"DE Ho","year":"2013","journal-title":"J Empir Leg Stud"},{"key":"ref45","doi-asserted-by":"crossref","first-page":"2303","DOI":"10.1080\/17470218.2013.863371","article-title":"The life of p: "Just significant" results are on the rise","volume":"66","author":"NC Leggett","year":"2013","journal-title":"Q J Exp Psychol"},{"key":"ref46","doi-asserted-by":"crossref","first-page":"28","DOI":"10.1093\/biostatistics\/kxt036","article-title":"Discussion: Why "An estimate of the science-wise false discovery rate and application to the top medical literature" is false","volume":"15","author":"JPA Ioannidis","year":"2014","journal-title":"Biostatistics"},{"key":"ref47","doi-asserted-by":"crossref","first-page":"404","DOI":"10.1093\/biomet\/26.4.404","article-title":"The use of confidence or fiducial limits illustrated in the case of the binomial","volume":"26","author":"CJ Clopper","year":"1934","journal-title":"Biometrika"},{"key":"ref48","unstructured":"Head ML, Holman L, Lanfear R, Kahn AT, Jennions MD (2015) Data from: The extent and consequences of p-hacking in science. Dryad Digital Repository. <ext-link xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" ext-link-type=\"uri\" xlink:href=\"http:\/\/dx.doi.org\/10.5061\/dryad.79d43\" xlink:type=\"simple\">http:\/\/dx.doi.org\/10.5061\/dryad.79d43<\/ext-link>."},{"key":"ref49","doi-asserted-by":"crossref","first-page":"150","DOI":"10.1038\/506150a","article-title":"Scientific method: Statistical errors","volume":"506","author":"R Nuzzo","year":"2014","journal-title":"Nature"},{"key":"ref50","doi-asserted-by":"crossref","first-page":"e4","DOI":"10.5334\/jopd.ad","article-title":"Data from investigating variation in replicability: A "many labs" replication project","volume":"2","author":"RA Klein","year":"2014","journal-title":"J Open Psychol Data"},{"key":"ref51","author":"AJ Sutton","year":"2000"},{"key":"ref52","doi-asserted-by":"crossref","first-page":"1272","DOI":"10.1037\/a0037714","article-title":"Meta-Analyses and p-curves support robust cycle shifts in women's mate preferencs: Reply to Wood & Carden (2014) and Harris, Pashler, and Mickes (2014)","volume":"140","author":"K Gildersleeve","year":"2014","journal-title":"Psychol Bull"},{"key":"ref53","first-page":"855","article-title":"Extra-pair paternity in birds: Review of the genetic benefits","volume":"9","author":"E Akcay","year":"2007","journal-title":"Evol Ecol Res"},{"key":"ref54","doi-asserted-by":"crossref","first-page":"318","DOI":"10.1111\/j.1474-919X.2011.01209.x","article-title":"The influence of male age on within-pair and extra-pair paternity in passerines","volume":"154","author":"IR Cleasby","year":"2012","journal-title":"Ibis"},{"key":"ref55","doi-asserted-by":"crossref","first-page":"1170","DOI":"10.1093\/beheco\/ars094","article-title":"Measuring mating competition correctly: available evidence supports operational sex ratio theory","volume":"23","author":"K de Jong","year":"2012","journal-title":"Behav Ecol"},{"key":"ref56","doi-asserted-by":"crossref","first-page":"E125","DOI":"10.1086\/670160","article-title":"Assortative mating in animals","volume":"181","author":"Y Jiang","year":"2013","journal-title":"Am Nat"},{"key":"ref57","doi-asserted-by":"crossref","first-page":"855","DOI":"10.1007\/s00265-007-0518-8","article-title":"The interrelationships between resource-holding potential, resource-value and reproductive success in territorial males: How much variation can we explain?","volume":"62","author":"CD Kelly","year":"2008","journal-title":"Behav Ecol Sociobiol"},{"key":"ref58","doi-asserted-by":"crossref","first-page":"367","DOI":"10.1111\/j.1469-185X.2010.00150.x","article-title":"Sexual selection and speciation: The comparative evidence revisited","volume":"86","author":"K Kraaijeveld","year":"2011","journal-title":"Biol Rev Camb Philos Soc"},{"key":"ref59","doi-asserted-by":"crossref","first-page":"2665","DOI":"10.1111\/j.1558-5646.2012.01654.x","article-title":"Meta-analysis suggests choosy females get sexy sons more than "good genes"","volume":"66","author":"ZM Prokop","year":"2012","journal-title":"Evolution"},{"key":"ref60","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1016\/j.anbehav.2011.03.022","article-title":"Dominance and plumage traits: Meta-analysis and metaregression analysis","volume":"82","author":"ESA Santos","year":"2011","journal-title":"Anim Behav"},{"key":"ref61","doi-asserted-by":"crossref","first-page":"167","DOI":"10.1086\/657918","article-title":"The influence of operational sex ratio on the intensity of competition for mates","volume":"177","author":"LK Weir","year":"2011","journal-title":"Am Nat"},{"key":"ref62","unstructured":"Simonsohn U, Nelson LD, Simmons JP (2014b) P-Curve and effect size: Correcting for publication bias using only significant results. <ext-link xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" ext-link-type=\"uri\" xlink:href=\"http:\/\/ssrncom\/abstract=2377290\" xlink:type=\"simple\">http:\/\/ssrncom\/abstract=2377290<\/ext-link>"},{"key":"ref63","author":"MALM van Assen","year":"2014"},{"key":"ref64","doi-asserted-by":"crossref","first-page":"1489","DOI":"10.1136\/bmj.286.6376.1489","article-title":"Statistical guidelines for contributors to medical journals","volume":"286","author":"DG Altman","year":"1983","journal-title":"Br Med J"},{"key":"ref65","doi-asserted-by":"crossref","first-page":"779","DOI":"10.3758\/BF03194105","article-title":"A practical solution to the pervasive problems of p values","volume":"14","author":"E-J Wagenmakers","year":"2007","journal-title":"Psychon Bull Rev"},{"key":"ref66","doi-asserted-by":"crossref","first-page":"353","DOI":"10.1046\/j.1365-2753.2002.00314.x","article-title":"Investigation of within-study selective reporting in clinical research: follow-up of applications submitted to a local research ethics committee","volume":"8","author":"S Hahn","year":"2002","journal-title":"J Eval Clin Pract"}],"container-title":["PLOS Biology"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/dx.plos.org\/10.1371\/journal.pbio.1002106","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2018,10,22]],"date-time":"2018-10-22T08:46:50Z","timestamp":1540198010000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pbio.1002106"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2015,3,13]]},"references-count":66,"journal-issue":{"issue":"3","published-online":{"date-parts":[[2015,3,13]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pbio.1002106","relation":{},"ISSN":["1545-7885"],"issn-type":[{"value":"1545-7885","type":"electronic"}],"subject":[],"published":{"date-parts":[[2015,3,13]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1371_2Fjournal_pcbi_1003285 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T20:45:27Z","timestamp":1793306727458,"version":"4.5.0"},"reference-count":25,"publisher":"Public Library of Science (PLoS)","issue":"10","license":[{"start":{"date-parts":[[2013,10,24]],"date-time":"2013-10-24T00:00:00Z","timestamp":1382572800000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["www.ploscompbiol.org"],"crossmark-restriction":false},"short-container-title":["PLoS Comput Biol"],"DOI":"10.1371\/journal.pcbi.1003285","type":"journal-article","created":{"date-parts":[[2013,10,24]],"date-time":"2013-10-24T22:03:37Z","timestamp":1382652217000},"page":"e1003285","update-policy":"https:\/\/doi.org\/10.1371\/journal.pcbi.corrections_policy","source":"Crossref","is-referenced-by-count":782,"title":["Ten Simple Rules for Reproducible Computational Research"],"prefix":"10.1371","volume":"9","author":[{"given":"Geir Kjetil","family":"Sandve","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Anton","family":"Nekrutenko","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"James","family":"Taylor","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Eivind","family":"Hovig","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"340","published-online":{"date-parts":[[2013,10,24]]},"reference":[{"key":"ref1","doi-asserted-by":"crossref","first-page":"1182","DOI":"10.1126\/science.1216775","article-title":"Addressing scientific fraud","volume":"334","author":"J Crocker","year":"2011","journal-title":"Science"},{"key":"ref2","first-page":"1225","article-title":"Data replication & reproducibility. Again, and again, and again\u2026. Introduction","volume":"334","author":"BR Jasny","year":"2011","journal-title":"Science"},{"key":"ref3","doi-asserted-by":"crossref","first-page":"1226","DOI":"10.1126\/science.1213847","article-title":"Reproducible research in computational science","volume":"334","author":"RD Peng","year":"2011","journal-title":"Science"},{"key":"ref4","doi-asserted-by":"crossref","first-page":"415","DOI":"10.1126\/science.1179653","article-title":"Computer science. Accessible reproducible research","volume":"327","author":"JP Mesirov","year":"2010","journal-title":"Science"},{"key":"ref5","doi-asserted-by":"crossref","first-page":"667","DOI":"10.1038\/nrg3305","article-title":"Next-generation sequencing data interpretation: enhancing reproducibility and accessibility","volume":"13","author":"A Nekrutenko","year":"2012","journal-title":"Nat Rev Genet"},{"key":"ref6","doi-asserted-by":"crossref","first-page":"149","DOI":"10.1038\/ng.295","article-title":"Repeatability of published microarray gene expression analyses","volume":"41","author":"JP Ioannidis","year":"2009","journal-title":"Nat Genet"},{"key":"ref7","doi-asserted-by":"crossref","first-page":"249","DOI":"10.1136\/jme.2010.040923","article-title":"Retractions in the scientific literature: is the incidence of research fraud increasing?","volume":"37","author":"RG Steen","year":"2011","journal-title":"J Med Ethics"},{"key":"ref8","doi-asserted-by":"crossref","first-page":"712","DOI":"10.1038\/nrd3439-c1","article-title":"Believe it or not: how much can we rely on published data on potential drug targets?","volume":"10","author":"F Prinz","year":"2011","journal-title":"Nat Rev Drug Discov"},{"key":"ref9","doi-asserted-by":"crossref","first-page":"531","DOI":"10.1038\/483531a","article-title":"Drug development: raise standards for preclinical cancer research","volume":"483","author":"CG Begley","year":"2012","journal-title":"Nature"},{"key":"ref10","doi-asserted-by":"crossref","first-page":"500","DOI":"10.1038\/ng0506-500","article-title":"GenePattern 2.0","volume":"38","author":"M Reich","year":"2006","journal-title":"Nat Genet"},{"key":"ref11","doi-asserted-by":"crossref","first-page":"1451","DOI":"10.1101\/gr.4086505","article-title":"Galaxy: a platform for interactive large-scale genome analysis","volume":"15","author":"B Giardine","year":"2005","journal-title":"Genome Res"},{"key":"ref12","doi-asserted-by":"crossref","first-page":"1033","DOI":"10.1016\/S1053-8119(03)00185-X","article-title":"The LONI Pipeline Processing Environment","volume":"19","author":"DE Rex","year":"2003","journal-title":"Neuroimage"},{"key":"ref13","doi-asserted-by":"crossref","first-page":"3045","DOI":"10.1093\/bioinformatics\/bth361","article-title":"Taverna: a tool for the composition and enactment of bioinformatics workflows","volume":"20","author":"T Oinn","year":"2004","journal-title":"Bioinformatics"},{"key":"ref14","doi-asserted-by":"crossref","first-page":"e308","DOI":"10.1371\/journal.pone.0000308","article-title":"Sharing detailed research data is associated with increased citation rate","volume":"2","author":"HA Piwowar","year":"2007","journal-title":"PLoS ONE"},{"key":"ref15","doi-asserted-by":"crossref","unstructured":"Heroux MA, Willenbring JM (2009) Barely sufficient software engineering: 10 practices to improve your cse software. In: 2009 ICSE Workshop on Software Engineering for Computational Science and Engineering. pp. 15\u201321.","DOI":"10.1109\/SECSE.2009.5069157"},{"key":"ref16","doi-asserted-by":"crossref","first-page":"61","DOI":"10.1109\/5992.881708","article-title":"Making scientific computations reproducible","volume":"2","author":"M Schwab","year":"2000","journal-title":"Comput Sci Eng"},{"key":"ref17","doi-asserted-by":"crossref","first-page":"W677","DOI":"10.1093\/nar\/gkq429","article-title":"myExperiment: a repository and social network for the sharing of bioinformatics workflows","volume":"38","author":"CA Goble","year":"2010","journal-title":"Nucleic Acids Res"},{"key":"ref18","doi-asserted-by":"crossref","first-page":"219","DOI":"10.1155\/2005\/128026","article-title":"Pegasus: a framework for mapping complex scientific workflows onto distributed systems","volume":"13","author":"E Deelman","year":"2005","journal-title":"Scientific Programming Journal"},{"key":"ref19","doi-asserted-by":"crossref","unstructured":"Leisch F (2002) Sweave: dynamic generation of statistical reports using literate data analysis. In: H\u00e4rdle W, R\u00f6nz B, editors. Compstat: proceedings in computational statistics. Heidelberg, Germany: Physika Verlag. pp. 575\u2013580.","DOI":"10.1007\/978-3-642-57489-4_89"},{"key":"ref20","doi-asserted-by":"crossref","first-page":"R86","DOI":"10.1186\/gb-2010-11-8-r86","article-title":"Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences","volume":"11","author":"J Goecks","year":"2010","journal-title":"Genome Biol"},{"key":"ref21","doi-asserted-by":"crossref","first-page":"365","DOI":"10.1038\/ng1201-365","article-title":"Minimum information about a microarray experiment (MIAME)-toward standards for microarray data","volume":"29","author":"A Brazma","year":"2001","journal-title":"Nat Genet"},{"key":"ref22","doi-asserted-by":"crossref","first-page":"68","DOI":"10.1093\/nar\/gkg091","article-title":"ArrayExpress\u2013a public repository for microarray gene expression data at the EBI","volume":"31","author":"A Brazma","year":"2003","journal-title":"Nucleic Acids Res"},{"key":"ref23","doi-asserted-by":"crossref","first-page":"207","DOI":"10.1093\/nar\/30.1.207","article-title":"Gene Expression Omnibus: NCBI gene expression and hybridization array data repository","volume":"30","author":"R Edgar","year":"2002","journal-title":"Nucleic Acids Res"},{"key":"ref24","doi-asserted-by":"crossref","first-page":"11","DOI":"10.1186\/2047-217X-1-11","article-title":"GigaDB: announcing the GigaScience database","volume":"1","author":"TP Sneddon","year":"2012","journal-title":"Gigascience"},{"key":"ref25","doi-asserted-by":"crossref","first-page":"e1002802","DOI":"10.1371\/journal.pcbi.1002802","article-title":"Ten simple rules for the open development of scientific software","volume":"8","author":"A Prli\u0107","year":"2012","journal-title":"PLoS Comput Biol"}],"container-title":["PLoS Computational Biology"],"original-title":[],"language":"en","link":[{"URL":"http:\/\/dx.plos.org\/10.1371\/journal.pcbi.1003285","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2018,10,23]],"date-time":"2018-10-23T16:56:58Z","timestamp":1540313818000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1003285"}},"subtitle":[],"editor":[{"given":"Philip E.","family":"Bourne","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"editor"}]}],"short-title":[],"issued":{"date-parts":[[2013,10,24]]},"references-count":25,"journal-issue":{"issue":"10","published-online":{"date-parts":[[2013,10,24]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pcbi.1003285","relation":{"has-review":[{"id-type":"doi","id":"10.3410\/f.720906062.793556792","asserted-by":"object"}]},"ISSN":["1553-7358"],"issn-type":[{"value":"1553-7358","type":"electronic"}],"subject":[],"published":{"date-parts":[[2013,10,24]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1371_2Fjournal_pcbi_1008316 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,16]],"date-time":"2026-10-16T03:17:15Z","timestamp":1792120635670,"version":"4.4.0"},"reference-count":67,"publisher":"Public Library of Science (PLoS)","issue":"11","license":[{"start":{"date-parts":[[2020,11,10]],"date-time":"2020-11-10T00:00:00Z","timestamp":1604966400000},"content-version":"vor","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["www.ploscompbiol.org"],"crossmark-restriction":false},"short-container-title":["PLoS Comput Biol"],"abstract":"<jats:p>\n Computational science has been greatly improved by the use of containers for packaging software and data dependencies. In a scholarly context, the main drivers for using these containers are transparency and support of reproducibility; in turn, a workflow\u2019s reproducibility can be greatly affected by the choices that are made with respect to building containers. In many cases, the build process for the container\u2019s image is created from instructions provided in a\n <jats:monospace>Dockerfile<\/jats:monospace>\n format. In support of this approach, we present a set of rules to help researchers write understandable\n <jats:monospace>Dockerfiles<\/jats:monospace>\n for typical data science workflows. By following the rules in this article, researchers can create containers suitable for sharing with fellow scientists, for including in scholarly communication such as education or scientific papers, and for effective and sustainable personal workflows.\n <\/jats:p>","DOI":"10.1371\/journal.pcbi.1008316","type":"journal-article","created":{"date-parts":[[2020,11,10]],"date-time":"2020-11-10T13:33:17Z","timestamp":1605015197000},"page":"e1008316","update-policy":"https:\/\/doi.org\/10.1371\/journal.pcbi.corrections_policy","source":"Crossref","is-referenced-by-count":75,"title":["Ten simple rules for writing Dockerfiles for reproducible data science"],"prefix":"10.1371","volume":"16","author":[{"ORCID":"https:\/\/orcid.org\/0000-0002-0024-5046","authenticated-orcid":true,"given":"Daniel","family":"N\u00fcst","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-4387-3819","authenticated-orcid":true,"given":"Vanessa","family":"Sochat","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7879-4531","authenticated-orcid":true,"given":"Ben","family":"Marwick","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8607-8025","authenticated-orcid":true,"given":"Stephen J.","family":"Eglen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Tim","family":"Head","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6921-702X","authenticated-orcid":true,"given":"Tony","family":"Hirst","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Benjamin D.","family":"Evans","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"340","published-online":{"date-parts":[[2020,11,10]]},"reference":[{"key":"pcbi.1008316.ref001","unstructured":"Marwick B. How computers broke science\u2014and what we can do to fix it [Internet]. The Conversation. 2015. https:\/\/theconversation.com\/how-computers-broke-science-and-what-we-can-do-to-fix-it-49938"},{"key":"pcbi.1008316.ref002","doi-asserted-by":"crossref","first-page":"385","DOI":"10.1093\/biostatistics\/kxq028","article-title":"An invitation to reproducible computational research","volume":"11","author":"DL Donoho","year":"2010","journal-title":"Biostatistics"},{"key":"pcbi.1008316.ref003","doi-asserted-by":"crossref","first-page":"e1001745","DOI":"10.1371\/journal.pbio.1001745","article-title":"Best practices for scientific computing","volume":"12","author":"G Wilson","year":"2014","journal-title":"PLoS Biol"},{"key":"pcbi.1008316.ref004","doi-asserted-by":"crossref","first-page":"e1005510","DOI":"10.1371\/journal.pcbi.1005510","article-title":"Good enough practices in scientific computing","volume":"13","author":"G Wilson","year":"2017","journal-title":"PLoS Comput Biol"},{"key":"pcbi.1008316.ref005","doi-asserted-by":"crossref","first-page":"e1007007","DOI":"10.1371\/journal.pcbi.1007007","article-title":"Ten simple rules for writing and sharing computational analyses in Jupyter Notebooks","volume":"15","author":"A Rule","year":"2019","journal-title":"PLoS Comput Biol"},{"key":"pcbi.1008316.ref006","doi-asserted-by":"crossref","first-page":"e1003285","DOI":"10.1371\/journal.pcbi.1003285","article-title":"Ten simple rules for reproducible computational research","volume":"9","author":"GK Sandve","year":"2013","journal-title":"PLoS Comput Biol"},{"key":"pcbi.1008316.ref007","unstructured":"N\u00fcst D. Author Carpentry: Docker for reproducible research [Internet]. 2017. https:\/\/nuest.github.io\/docker-reproducible-research\/"},{"key":"pcbi.1008316.ref008","unstructured":"Chapman P. Reproducible data science environments with Docker Phil Chapman\u2019s Blog [Internet]. 2018. https:\/\/chapmandu2.github.io\/post\/2018\/05\/26\/reproducible-data-science-environments-with-docker\/"},{"key":"pcbi.1008316.ref009","unstructured":"rOpenSci Labs. R Docker tutorial [Internet]. 2015. https:\/\/ropenscilabs.github.io\/r-docker-tutorial\/"},{"key":"pcbi.1008316.ref010","unstructured":"Udemy, Zhbanko V. Docker Containers for Data Science and Reproducible Research [Internet]. Udemy. 2019. https:\/\/www.udemy.com\/course\/docker-containers-data-science-reproducible-research\/"},{"key":"pcbi.1008316.ref011","unstructured":"Psomopoulos FE. Lesson \"Docker and Reproducibility\" in Workshop \"Reproducible analysis and Research Transparency\" [Internet]. Reproducible analysis and Research Transparency. 2017. https:\/\/reproducible-analysis-workshop.readthedocs.io\/en\/latest\/8.Intro-Docker.html"},{"key":"pcbi.1008316.ref012","doi-asserted-by":"crossref","first-page":"742","DOI":"10.12688\/f1000research.15140.2","article-title":"Recommendations for the packaging and containerizing of bioinformatics software","volume":"7","author":"B Gruening","year":"2019","journal-title":"F1000Research"},{"key":"pcbi.1008316.ref013","unstructured":"Docker Inc. Best practices for writing Dockerfiles [Internet]. Docker Documentation. 2020. https:\/\/docs.docker.com\/develop\/develop-images\/dockerfile_best-practices\/"},{"key":"pcbi.1008316.ref014","unstructured":"Vass T. Intro Guide to Dockerfile Best Practices [Internet]. Docker Blog. 2019. https:\/\/www.docker.com\/blog\/intro-guide-to-dockerfile-best-practices\/"},{"key":"pcbi.1008316.ref015","doi-asserted-by":"crossref","first-page":"e0177459","DOI":"10.1371\/journal.pone.0177459","article-title":"Singularity: Scientific containers for mobility of compute","volume":"12","author":"GM Kurtzer","year":"2017","journal-title":"PLoS ONE"},{"key":"pcbi.1008316.ref016","unstructured":"Docker Inc. Overview of Docker Compose [Internet]. Docker Documentation. 2019. https:\/\/docs.docker.com\/compose\/"},{"key":"pcbi.1008316.ref017","doi-asserted-by":"crossref","DOI":"10.1045\/january2017-nuest","article-title":"Opening the Publication Process with Executable Research Compendia","volume":"23","author":"D N\u00fcst","year":"2017","journal-title":"D-Lib Magazine"},{"key":"pcbi.1008316.ref018","article-title":"The Four Pillars of Research Software Engineering","author":"J Cohen","year":"2020","journal-title":"IEEE Softw"},{"key":"pcbi.1008316.ref019","unstructured":"Wikipedia contributors. Docker (software) [Internet]. Wikipedia. 2019. https:\/\/en.wikipedia.org\/w\/index.php?title=Docker_(software)&oldid=928441083"},{"key":"pcbi.1008316.ref020","doi-asserted-by":"crossref","first-page":"527","DOI":"10.32614\/RJ-2017-065","article-title":"An Introduction to Rocker: Docker Containers for R","volume":"9","author":"C Boettiger","year":"2017","journal-title":"The R Journal"},{"key":"pcbi.1008316.ref021","doi-asserted-by":"crossref","first-page":"113","DOI":"10.1038\/s41567-018-0342-2","article-title":"Open is not enough","volume":"15","author":"X Chen","year":"2019","journal-title":"Nat Phys"},{"key":"pcbi.1008316.ref022","article-title":"Computing environments for reproducibility: Capturing the \u201cWhole Tale\u201d","author":"A Brinckman","year":"2018","journal-title":"Futur Gener Comput Syst"},{"key":"pcbi.1008316.ref023","unstructured":"Code Ocean [Internet]. 2019. https:\/\/codeocean.com\/"},{"key":"pcbi.1008316.ref024","doi-asserted-by":"crossref","first-page":"06034","DOI":"10.1051\/epjconf\/201921406034","article-title":"REANA: A System for Reusable Research Data Analyses","volume":"214","author":"T \u0160imko","year":"2019","journal-title":"EPJ Web Conf"},{"key":"pcbi.1008316.ref025","doi-asserted-by":"crossref","unstructured":"Project Jupyter, Bussonnier M, Forde J, Freeman J, Granger B, Head T, et al. Binder 2.0\u2014Reproducible, interactive, sharable environments for science at scale. Proceedings of the 17th Python in Science Conference. 2018;113\u2013120. 10.25080\/Majora-4af1f417-011","DOI":"10.25080\/Majora-4af1f417-011"},{"key":"pcbi.1008316.ref026","unstructured":"Docker Inc. Dockerfile reference [Internet]. Docker Documentation. 2019. https:\/\/docs.docker.com\/engine\/reference\/builder\/"},{"key":"pcbi.1008316.ref027","unstructured":"Wikipedia contributors. Make (software) [Internet]. Wikipedia. 2019. https:\/\/en.wikipedia.org\/w\/index.php?title=Make_(software)&oldid=929976465"},{"key":"pcbi.1008316.ref028","doi-asserted-by":"crossref","first-page":"71","DOI":"10.1145\/2723872.2723882","article-title":"An Introduction to Docker for Reproducible Research","volume":"49","author":"C Boettiger","year":"2015","journal-title":"SIGOPS Oper Syst Rev"},{"key":"pcbi.1008316.ref029","unstructured":"Ben Marwick. 1989-excavation-report-Madjebebe. 2015. 10.6084\/m9.figshare.1297059"},{"key":"pcbi.1008316.ref030","unstructured":"Docker Inc. Official Images on Docker Hub [Internet]. Docker Documentation 2019. https:\/\/docs.docker.com\/docker-hub\/official_images\/"},{"key":"pcbi.1008316.ref031","doi-asserted-by":"crossref","first-page":"1603","DOI":"10.21105\/joss.01603","article-title":"Containerit: Generating Dockerfiles for reproducible research with R","volume":"4","author":"D N\u00fcst","year":"2019","journal-title":"J Open Source Softw"},{"key":"pcbi.1008316.ref032","unstructured":"Stencila. Stencila\/dockta [Internet]. Stencila. 2019. https:\/\/github.com\/stencila\/dockta"},{"key":"pcbi.1008316.ref033","unstructured":"Cookiecutter contributors. Cookiecutter\/cookiecutter [Internet]. cookiecutter. 2019. https:\/\/github.com\/cookiecutter\/cookiecutter"},{"key":"pcbi.1008316.ref034","unstructured":"Marwick B. Benmarwick\/rrtools [Internet]. 2019. https:\/\/github.com\/benmarwick\/rrtools"},{"key":"pcbi.1008316.ref035","unstructured":"Docker Inc. Official Images on Docker Hub [Internet]. Docker Documentation 2020. https:\/\/docs.docker.com\/docker-hub\/official_images\/"},{"key":"pcbi.1008316.ref036","doi-asserted-by":"crossref","DOI":"10.3389\/fninf.2012.00022","article-title":"Open is Not Enough. Let\u2019s Take the Next Step: An Integrated, Community-Driven Computing Platform for Neuroscience","volume":"6","author":"YO Halchenko","year":"2012","journal-title":"Front Neuroinform"},{"key":"pcbi.1008316.ref037","doi-asserted-by":"crossref","first-page":"1686","DOI":"10.21105\/joss.01686","article-title":"Welcome to the tidyverse","volume":"4","author":"H Wickham","year":"2019","journal-title":"J Open Source Softw"},{"key":"pcbi.1008316.ref038","unstructured":"Preston-Werner T. Semantic Versioning 2.0.0 [Internet]. Semantic Versioning. 2013. https:\/\/semver.org\/"},{"key":"pcbi.1008316.ref039","unstructured":"Docker Inc. Use multi-stage builds [Internet]. Docker Documentation 2020. https:\/\/docs.docker.com\/develop\/develop-images\/multistage-build\/"},{"key":"pcbi.1008316.ref040","unstructured":"Goodman A. Wagoodman\/dive [Internet]. 2019. https:\/\/github.com\/wagoodman\/dive"},{"key":"pcbi.1008316.ref041","unstructured":"Opencontainers. Opencontainers\/image-spec v1.0.1\u2014Annotations [Internet]. GitHub. 2017. https:\/\/github.com\/opencontainers\/image-spec\/blob\/v1.0.1\/annotations.md"},{"key":"pcbi.1008316.ref042","unstructured":"Docker Inc. Dockerfile reference, healthcheck [Internet]. Docker Documentation. 2020. https:\/\/docs.docker.com\/engine\/reference\/builder\/#healthcheck"},{"key":"pcbi.1008316.ref043","unstructured":"The Python Software Foundation. Requirements Files\u2014pip User Guide [Internet]. 2019. https:\/\/pip.pypa.io\/en\/stable\/user_guide\/#requirements-files"},{"key":"pcbi.1008316.ref044","unstructured":"Continuum Analytics. Managing environments\u2014conda documentation [Internet]. 2017. https:\/\/docs.conda.io\/projects\/conda\/en\/latest\/user-guide\/tasks\/manage-environments.html"},{"key":"pcbi.1008316.ref045","unstructured":"R Core Team. The DESCRIPTION file in \"writing r extensions\" [Internet]. 1999. https:\/\/cran.r-project.org\/doc\/manuals\/r-release\/R-exts.html#The-DESCRIPTION-file"},{"key":"pcbi.1008316.ref046","unstructured":"Eddelbuettel D, Horner J. Littler: R at the command-line via \u2018r\u2019 [Internet]. 2019. https:\/\/CRAN.R-project.org\/package=littler"},{"key":"pcbi.1008316.ref047","unstructured":"npm. Creating a package.json file npm Documentation [Internet]. 2019. https:\/\/docs.npmjs.com\/creating-a-package-json-file"},{"key":"pcbi.1008316.ref048","unstructured":"The Julia Language Contributors. 10. Project.Toml and Manifest.Toml Pkg.Jl [Internet]. 2019. https:\/\/julialang.github.io\/Pkg.jl\/v1\/toml-files\/"},{"key":"pcbi.1008316.ref049","doi-asserted-by":"crossref","DOI":"10.2218\/ijdc.v12i2.509","article-title":"A Framework for the Preservation of a Docker Container","volume":"12","author":"I Emsley","year":"2018","journal-title":"Int J Digit Curation"},{"key":"pcbi.1008316.ref050","unstructured":"Docker Inc. Use bind mounts [Internet]. Docker Documentation. 2019. https:\/\/docs.docker.com\/storage\/bind-mounts\/"},{"key":"pcbi.1008316.ref051","article-title":"JudithVerstegen\/PLUC_Mozambique: First release of PLUC for Mozambique","author":"JA Verstegen","year":"2019","journal-title":"Zenodo"},{"key":"pcbi.1008316.ref052","doi-asserted-by":"crossref","DOI":"10.1093\/gigascience\/giy023","article-title":"The Scientific Filesystem","volume":"7","author":"V Sochat","year":"2018","journal-title":"GigaScience"},{"key":"pcbi.1008316.ref053","doi-asserted-by":"crossref","first-page":"290","DOI":"10.3390\/rs9030290","article-title":"Reproducibility and Practical Adoption of GEOBIA with Open-Source Software in Docker Containers","volume":"9","author":"C Knoth","year":"2017","journal-title":"Remote Sens"},{"key":"pcbi.1008316.ref054","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.ascom.2018.08.003","article-title":"Kliko\u2014The scientific compute container format","volume":"25","author":"G Molenaar","year":"2018","journal-title":"Astronomy Comput"},{"key":"pcbi.1008316.ref055","unstructured":"Selenium contributors. SeleniumHQ\/selenium [Internet]. Selenium. 2019. https:\/\/github.com\/SeleniumHQ\/selenium"},{"key":"pcbi.1008316.ref056","unstructured":"Singularity. Frequently Asked Questions Singularity [Internet]. 2019. http:\/\/singularity.lbl.gov\/archive\/docs\/v2-2\/faq#can-i-run-x11-apps-through-singularity"},{"key":"pcbi.1008316.ref057","doi-asserted-by":"crossref","first-page":"1349","DOI":"10.21105\/joss.01349","article-title":"X11docker: Run GUI applications in Docker containers","volume":"4","author":"M Viereck","year":"2019","journal-title":"J Open Source Softw"},{"key":"pcbi.1008316.ref058","unstructured":"Yaremenko E. JAremko\/docker-x11-bridge [Internet]. 2019. https:\/\/github.com\/JAremko\/docker-x11-bridge"},{"key":"pcbi.1008316.ref059","unstructured":"Panda Y. Yuvipanda\/jupyter-desktop-server [Internet]. 2019. https:\/\/github.com\/yuvipanda\/jupyter-desktop-server"},{"key":"pcbi.1008316.ref060","unstructured":"Marwick B. README of 1989-excavation-report-Madjebebe. 2015. 10.6084\/m9.figshare.1297059"},{"key":"pcbi.1008316.ref061","unstructured":"A rule-based linter for dockerfiles [Internet]. 2020. https:\/\/github.com\/projectatomic\/dockerfile_lint"},{"key":"pcbi.1008316.ref062","unstructured":"Dockerfile linter [Internet]. 2020. https:\/\/hadolint.github.io\/hadolint\/"},{"key":"pcbi.1008316.ref063","doi-asserted-by":"crossref","unstructured":"Eglen S, N\u00fcst D. CODECHECK: An open-science initiative to facilitate sharing of computer programs and results presented in scientific publications. Septentrio Conference Series 2019. 10.7557\/5.4910","DOI":"10.7557\/5.4910"},{"key":"pcbi.1008316.ref064","doi-asserted-by":"crossref","first-page":"1031","DOI":"10.1038\/s41562-019-0726-z","article-title":"Training students for the Open Science future","volume":"3","author":"F Sch\u00f6nbrodt","year":"2019","journal-title":"Nat Hum Behav"},{"key":"pcbi.1008316.ref065","doi-asserted-by":"crossref","first-page":"775","DOI":"10.1042\/ETLS20180172","article-title":"Recent developments in scholarly publishing to improve research practices in the life sciences","volume":"2","author":"SJ Eglen","year":"2018","journal-title":"Emerg Top Life Sci"},{"key":"pcbi.1008316.ref066","article-title":"Before reproducibility must come preproducibility","author":"PB Stark","year":"2018","journal-title":"Nature"},{"key":"pcbi.1008316.ref067","article-title":"Ten Simple Rules for Writing Dockerfiles for Reproducible Data Science","author":"D N\u00fcst","year":"2020","journal-title":"Open Science Framework"}],"container-title":["PLOS Computational Biology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1008316","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2020,11,10]],"date-time":"2020-11-10T13:34:12Z","timestamp":1605015252000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pcbi.1008316"}},"subtitle":[],"editor":[{"given":"Scott","family":"Markel","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"editor"}]}],"short-title":[],"issued":{"date-parts":[[2020,11,10]]},"references-count":67,"journal-issue":{"issue":"11","published-online":{"date-parts":[[2020,11,10]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pcbi.1008316","relation":{"has-preprint":[{"id-type":"doi","id":"10.31219\/osf.io\/fsd7t","asserted-by":"object"}]},"ISSN":["1553-7358"],"issn-type":[{"value":"1553-7358","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,11,10]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1371_2Fjournal_pmed_0020124 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,30]],"date-time":"2026-10-30T03:06:00Z","timestamp":1793329560656,"version":"4.5.0"},"reference-count":36,"publisher":"Public Library of Science (PLoS)","issue":"8","license":[{"start":{"date-parts":[[2005,8,30]],"date-time":"2005-08-30T00:00:00Z","timestamp":1125360000000},"content-version":"unspecified","delay-in-days":0,"URL":"http:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":["www.plosmedicine.org"],"crossmark-restriction":false},"short-container-title":["PLoS Med"],"DOI":"10.1371\/journal.pmed.0020124","type":"journal-article","created":{"date-parts":[[2005,8,2]],"date-time":"2005-08-02T15:39:51Z","timestamp":1122997191000},"page":"e124","update-policy":"https:\/\/doi.org\/10.1371\/journal.pmed.corrections_policy","source":"Crossref","is-referenced-by-count":8427,"title":["Why Most Published Research Findings Are False"],"prefix":"10.1371","volume":"2","author":[{"given":"John P. A.","family":"Ioannidis","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"340","published-online":{"date-parts":[[2005,8,30]]},"reference":[{"key":"JOURNAL-PMED-0020124-B1","doi-asserted-by":"crossref","first-page":"879","DOI":"10.1136\/bmj.322.7291.879","volume":"322","author":"Ioannidis","year":"2001","journal-title":"BMJ","ISSN":"https:\/\/id.crossref.org\/issn\/0959-8138","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B2","doi-asserted-by":"crossref","first-page":"1724","DOI":"10.1016\/S0140-6736(04)16260-0","volume":"363","author":"Lawlor","year":"2004","journal-title":"Lancet","ISSN":"https:\/\/id.crossref.org\/issn\/0140-6736","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B3","doi-asserted-by":"crossref","first-page":"1728","DOI":"10.1016\/S0140-6736(04)16261-2","volume":"363","author":"Vandenbroucke","year":"2004","journal-title":"Lancet","ISSN":"https:\/\/id.crossref.org\/issn\/0140-6736","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B4","doi-asserted-by":"crossref","first-page":"488","DOI":"10.1016\/S0140-6736(05)17866-0","volume":"365","author":"Michiels","year":"2005","journal-title":"Lancet","ISSN":"https:\/\/id.crossref.org\/issn\/0140-6736","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B5","doi-asserted-by":"crossref","first-page":"306","DOI":"10.1038\/ng749","volume":"29","author":"Ioannidis","year":"2001","journal-title":"Nat Genet","ISSN":"https:\/\/id.crossref.org\/issn\/1061-4036","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B6","doi-asserted-by":"crossref","first-page":"865","DOI":"10.1016\/S0140-6736(03)12715-8","volume":"361","author":"Colhoun","year":"2003","journal-title":"Lancet","ISSN":"https:\/\/id.crossref.org\/issn\/0140-6736","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B7","doi-asserted-by":"crossref","first-page":"135","DOI":"10.1016\/S1471-4914(03)00030-3","volume":"9","author":"Ioannidis","year":"2003","journal-title":"Trends Mol Med","ISSN":"https:\/\/id.crossref.org\/issn\/1471-4914","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B8","doi-asserted-by":"crossref","first-page":"454","DOI":"10.1016\/S0140-6736(05)17878-7","volume":"365","author":"Ioannidis","year":"2005","journal-title":"Lancet","ISSN":"https:\/\/id.crossref.org\/issn\/0140-6736","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B9","doi-asserted-by":"crossref","first-page":"226","DOI":"10.1136\/bmj.322.7280.226","volume":"322","author":"Sterne","year":"2001","journal-title":"BMJ","ISSN":"https:\/\/id.crossref.org\/issn\/0959-8138","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B10","doi-asserted-by":"crossref","first-page":"434","DOI":"10.1093\/jnci\/djh075","volume":"96","author":"Wacholder","year":"2004","journal-title":"J Natl Cancer Inst","ISSN":"https:\/\/id.crossref.org\/issn\/0027-8874","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B11","doi-asserted-by":"crossref","first-page":"847","DOI":"10.1038\/35015718","volume":"405","author":"Risch","year":"2000","journal-title":"Nature","ISSN":"https:\/\/id.crossref.org\/issn\/0028-0836","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B13","doi-asserted-by":"crossref","first-page":"1707","DOI":"10.1056\/NEJMp048286","volume":"351","author":"Topol","year":"2004","journal-title":"N Engl J Med","ISSN":"https:\/\/id.crossref.org\/issn\/0028-4793","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B14","doi-asserted-by":"crossref","first-page":"409","DOI":"10.1002\/sim.4780030421","volume":"3","author":"Yusuf","year":"1984","journal-title":"Stat Med","ISSN":"https:\/\/id.crossref.org\/issn\/0277-6715","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B15","doi-asserted-by":"crossref","first-page":"453","DOI":"10.1002\/(SICI)1097-0258(20000229)19:4<453::AID-SIM350>3.0.CO;2-5","volume":"19","author":"Altman","year":"2000","journal-title":"Stat Med","ISSN":"https:\/\/id.crossref.org\/issn\/0277-6715","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B16","doi-asserted-by":"crossref","first-page":"164","DOI":"10.1126\/science.7618077","volume":"269","author":"Taubes","year":"1995","journal-title":"Science","ISSN":"https:\/\/id.crossref.org\/issn\/0193-4511","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B17","doi-asserted-by":"crossref","first-page":"531","DOI":"10.1126\/science.286.5439.531","volume":"286","author":"Golub","year":"1999","journal-title":"Science","ISSN":"https:\/\/id.crossref.org\/issn\/0193-4511","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B18","doi-asserted-by":"crossref","first-page":"1191","DOI":"10.1016\/S0140-6736(00)04337-3","volume":"357","author":"Moher","year":"2001","journal-title":"Lancet","ISSN":"https:\/\/id.crossref.org\/issn\/0140-6736","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B19","doi-asserted-by":"crossref","first-page":"781","DOI":"10.7326\/0003-4819-141-10-200411160-00009","volume":"141","author":"Ioannidis","year":"2004","journal-title":"Ann Intern Med","ISSN":"https:\/\/id.crossref.org\/issn\/0003-4819","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B20","first-page":"1905","volume":"18","author":"","year":"1999","journal-title":"Stat Med","ISSN":"https:\/\/id.crossref.org\/issn\/0277-6715","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B21","doi-asserted-by":"crossref","first-page":"1896","DOI":"10.1016\/S0140-6736(99)04149-5","volume":"354","author":"Moher","year":"1999","journal-title":"Lancet","ISSN":"https:\/\/id.crossref.org\/issn\/0140-6736","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B22","doi-asserted-by":"crossref","first-page":"2008","DOI":"10.1001\/jama.283.15.2008","volume":"283","author":"Stroup","year":"2000","journal-title":"JAMA","ISSN":"https:\/\/id.crossref.org\/issn\/0098-7484","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B23","doi-asserted-by":"crossref","first-page":"249","DOI":"10.1192\/bjp.176.3.249","volume":"176","author":"Marshall","year":"2000","journal-title":"Br J Psychiatry","ISSN":"https:\/\/id.crossref.org\/issn\/0007-1250","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B24","doi-asserted-by":"crossref","first-page":"129","DOI":"10.1001\/jama.1994.03520020055015","volume":"272","author":"Altman","year":"1994","journal-title":"JAMA","ISSN":"https:\/\/id.crossref.org\/issn\/0098-7484","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B25","doi-asserted-by":"crossref","first-page":"2457","DOI":"10.1001\/jama.291.20.2457","volume":"291","author":"Chan","year":"2004","journal-title":"JAMA","ISSN":"https:\/\/id.crossref.org\/issn\/0098-7484","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B26","doi-asserted-by":"crossref","first-page":"194","DOI":"10.1159\/000012281","volume":"67","author":"Krimsky","year":"1998","journal-title":"Psychother Psychosom","ISSN":"https:\/\/id.crossref.org\/issn\/0033-3190","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B27","doi-asserted-by":"crossref","first-page":"3","DOI":"10.1186\/1471-2288-1-3","volume":"1","author":"Papanikolaou","year":"2001","journal-title":"BMC Med Res Methodol","ISSN":"https:\/\/id.crossref.org\/issn\/1471-2288","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B28","doi-asserted-by":"crossref","first-page":"240","DOI":"10.1001\/jama.1992.03490020088036","volume":"268","author":"Antman","year":"1992","journal-title":"JAMA","ISSN":"https:\/\/id.crossref.org\/issn\/0098-7484","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B29","doi-asserted-by":"crossref","first-page":"543","DOI":"10.1016\/j.jclinepi.2004.10.019","volume":"58","author":"Ioannidis","year":"2005","journal-title":"J Clin Epidemiol","ISSN":"https:\/\/id.crossref.org\/issn\/0895-4356","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B30","doi-asserted-by":"crossref","first-page":"1439","DOI":"10.1016\/S0140-6736(03)14686-7","volume":"362","author":"Ntzani","year":"2003","journal-title":"Lancet","ISSN":"https:\/\/id.crossref.org\/issn\/0140-6736","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B31","doi-asserted-by":"crossref","first-page":"309","DOI":"10.1038\/nrc1322","volume":"4","author":"Ransohoff","year":"2004","journal-title":"Nat Rev Cancer","ISSN":"https:\/\/id.crossref.org\/issn\/1474-175X","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B32","doi-asserted-by":"crossref","first-page":"187","DOI":"10.1093\/biomet\/44.1-2.187","volume":"44","author":"Lindley","year":"1957","journal-title":"Biometrika","ISSN":"https:\/\/id.crossref.org\/issn\/0006-3444","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B33","doi-asserted-by":"crossref","first-page":"533","DOI":"10.1093\/biomet\/44.3-4.533","volume":"44","author":"Bartlett","year":"1957","journal-title":"Biometrika","ISSN":"https:\/\/id.crossref.org\/issn\/0006-3444","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B34","doi-asserted-by":"crossref","first-page":"193","DOI":"10.1080\/135952201753172953","volume":"6","author":"Senn","year":"2001","journal-title":"J Epidemiol Biostat","ISSN":"https:\/\/id.crossref.org\/issn\/1359-5229","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B35","doi-asserted-by":"crossref","first-page":"1250","DOI":"10.1056\/NEJMe048225","volume":"351","author":"De Angelis","year":"2004","journal-title":"N Engl J Med","ISSN":"https:\/\/id.crossref.org\/issn\/0028-4793","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B36","doi-asserted-by":"crossref","first-page":"218","DOI":"10.1001\/jama.294.2.218","volume":"294","author":"Ioannidis","year":"2005","journal-title":"JAMA","ISSN":"https:\/\/id.crossref.org\/issn\/0098-7484","issn-type":"print"},{"key":"JOURNAL-PMED-0020124-B37","doi-asserted-by":"crossref","first-page":"675","DOI":"10.1081\/BIP-120024202","volume":"13","author":"Hsueh","year":"2003","journal-title":"J Biopharm Stat","ISSN":"https:\/\/id.crossref.org\/issn\/1520-5711","issn-type":"print"}],"updated-by":[{"DOI":"10.1371\/journal.pmed.1004085","type":"correction","label":"Correction","source":"publisher","updated":{"date-parts":[[2022,8,25]],"date-time":"2022-08-25T00:00:00Z","timestamp":1661385600000}}],"container-title":["PLoS Medicine"],"original-title":[],"language":"en","deposited":{"date-parts":[[2019,3,11]],"date-time":"2019-03-11T06:36:12Z","timestamp":1552286172000},"score":1,"resource":{"primary":{"URL":"https:\/\/dx.plos.org\/10.1371\/journal.pmed.0020124"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2005,8,30]]},"references-count":36,"journal-issue":{"issue":"8","published-online":{"date-parts":[[2005,8,30]]}},"URL":"https:\/\/doi.org\/10.1371\/journal.pmed.0020124","relation":{"has-review":[{"id-type":"doi","id":"10.3410\/f.8225.335764","asserted-by":"object"},{"id-type":"doi","id":"10.3410\/f.8225.793477587","asserted-by":"object"},{"id-type":"doi","id":"10.3410\/f.8225.465715","asserted-by":"object"},{"id-type":"doi","id":"10.3410\/f.8225.543895","asserted-by":"object"},{"id-type":"doi","id":"10.3410\/f.8225.335861","asserted-by":"object"},{"id-type":"doi","id":"10.3410\/f.8225.334858","asserted-by":"object"},{"id-type":"doi","id":"10.3410\/f.8225.558170","asserted-by":"object"},{"id-type":"doi","id":"10.3410\/f.8225.334883","asserted-by":"object"}]},"ISSN":["1549-1676"],"issn-type":[{"value":"1549-1676","type":"electronic"}],"subject":[],"published":{"date-parts":[[2005,8,30]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_14778_2F3529337_3529344 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,17]],"date-time":"2026-10-17T22:06:26Z","timestamp":1792274786566,"version":"4.4.0"},"reference-count":67,"publisher":"Association for Computing Machinery (ACM)","issue":"8","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Proc. VLDB Endow."],"published-print":{"date-parts":[[2022,4]]},"abstract":"<jats:p>Serverless is a popular choice for cloud service architects because it can provide scalability and load-based billing with minimal developer effort. Functions-as-a-service (FaaS) are originally stateless, but emerging frameworks add stateful abstractions. For instance, the widely used Durable Functions (DF) allow developers to write advanced serverless applications, including reliable workflows and actors, in a programming language of choice. DF implicitly and continuosly persists the state and progress of applications, which greatly simplifies development, but can create an IOps bottleneck.<\/jats:p>\n <jats:p>To improve efficiency, we introduce Netherite, a novel architecture for executing serverless workflows on an elastic cluster. Netherite groups the numerous application objects into a smaller number of partitions, and pipelines the state persistence of each partition. This improves latency and throughput, as it enables workflow steps to group commit, even if causally dependent. Moreover, Netherite leverages FASTER's hybrid log approach to support larger-than-memory application state, and to enable efficient partition movement between compute hosts.<\/jats:p>\n <jats:p>Our evaluation shows that (a) Netherite achieves lower latency and higher throughput than the original DF engine, by more than an order of magnitude in some cases, and (b) that Netherite has lower latency than some commonly used alternatives, like AWS Step Functions or cloud storage triggers.<\/jats:p>","DOI":"10.14778\/3529337.3529344","type":"journal-article","created":{"date-parts":[[2022,6,22]],"date-time":"2022-06-22T22:23:05Z","timestamp":1655936585000},"page":"1591-1604","source":"Crossref","is-referenced-by-count":52,"title":["Netherite"],"prefix":"10.14778","volume":"15","author":[{"given":"Sebastian","family":"Burckhardt","sequence":"first","affiliation":[{"name":"Microsoft Research"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Badrish","family":"Chandramouli","sequence":"additional","affiliation":[{"name":"Microsoft Research"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Chris","family":"Gillum","sequence":"additional","affiliation":[{"name":"Microsoft Azure"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"David","family":"Justo","sequence":"additional","affiliation":[{"name":"Microsoft Azure"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Konstantinos","family":"Kallas","sequence":"additional","affiliation":[{"name":"University of Pennsylvania"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Connor","family":"McMahon","sequence":"additional","affiliation":[{"name":"Microsoft Azure"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Christopher S.","family":"Meiklejohn","sequence":"additional","affiliation":[{"name":"Carnegie Mellon University"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiangfeng","family":"Zhu","sequence":"additional","affiliation":[{"name":"University of Washington"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"320","published-online":{"date-parts":[[2022,6,22]]},"reference":[{"key":"e_1_2_1_1_1","unstructured":"[n.d.]. AWS Lambda - Serverless Compute - Amazon Web Services. https:\/\/aws.amazon.com\/lambda\/. [n.d.]. AWS Lambda - Serverless Compute - Amazon Web Services. https:\/\/aws.amazon.com\/lambda\/."},{"key":"e_1_2_1_2_1","unstructured":"2022. Apache Airflow. https:\/\/airflow.apache.org\/. 2022. Apache Airflow. https:\/\/airflow.apache.org\/."},{"key":"e_1_2_1_3_1","unstructured":"2022. Apache Kafka. http:\/\/kafka.apache.org. 2022. Apache Kafka. http:\/\/kafka.apache.org."},{"key":"e_1_2_1_4_1","unstructured":"2022. Apache OpenWhisk. https:\/\/openwhisk.apache.org\/. 2022. Apache OpenWhisk. https:\/\/openwhisk.apache.org\/."},{"key":"e_1_2_1_5_1","unstructured":"2022. AWS Step Functions. https:\/\/aws.amazon.com\/step-functions\/. 2022. AWS Step Functions. https:\/\/aws.amazon.com\/step-functions\/."},{"key":"e_1_2_1_6_1","unstructured":"2022. Azure Functions. https:\/\/azure.microsoft.com\/en-us\/services\/functions\/. 2022. Azure Functions. https:\/\/azure.microsoft.com\/en-us\/services\/functions\/."},{"key":"e_1_2_1_7_1","unstructured":"2022. Azure Functions Host repository. https:\/\/github.com\/Azure\/azure-functions-host. 2022. Azure Functions Host repository. https:\/\/github.com\/Azure\/azure-functions-host."},{"key":"e_1_2_1_8_1","unstructured":"2022. Azure Functions PowerShell repository. https:\/\/github.com\/Azure\/azure-functions-powershell-worker. 2022. Azure Functions PowerShell repository. https:\/\/github.com\/Azure\/azure-functions-powershell-worker."},{"key":"e_1_2_1_9_1","unstructured":"2022. Azure Logic Apps Service. https:\/\/azure.microsoft.com\/en-us\/services\/logic-apps\/. 2022. Azure Logic Apps Service. https:\/\/azure.microsoft.com\/en-us\/services\/logic-apps\/."},{"key":"e_1_2_1_10_1","unstructured":"2022. CloudState-Towards Stateful Serverless by Jonas Bon\u00e9r. https:\/\/www.youtube.com\/watch?v=DVTf5WQlgB8. 2022. CloudState-Towards Stateful Serverless by Jonas Bon\u00e9r. https:\/\/www.youtube.com\/watch?v=DVTf5WQlgB8."},{"key":"e_1_2_1_11_1","unstructured":"2022. Durable Functions Extension (C-sharp) repository. https:\/\/github.com\/Azure\/azure-functions-durable-extension. 2022. Durable Functions Extension (C-sharp) repository. https:\/\/github.com\/Azure\/azure-functions-durable-extension."},{"key":"e_1_2_1_12_1","unstructured":"2022. Durable Functions JavaScript repository. https:\/\/github.com\/Azure\/azure-functions-durable-js. 2022. Durable Functions JavaScript repository. https:\/\/github.com\/Azure\/azure-functions-durable-js."},{"key":"e_1_2_1_13_1","unstructured":"2022. Durable Functions Python repository. https:\/\/github.com\/Azure\/azure-functions-durable-python. 2022. Durable Functions Python repository. https:\/\/github.com\/Azure\/azure-functions-durable-python."},{"key":"e_1_2_1_14_1","unstructured":"2022. FASTER. https:\/\/github.com\/microsoft\/FASTER. 2022. FASTER. https:\/\/github.com\/microsoft\/FASTER."},{"key":"e_1_2_1_15_1","unstructured":"2022. Fission: Open source Kubernetes-native Serverless Framework. https:\/\/fission.io\/. 2022. Fission: Open source Kubernetes-native Serverless Framework. https:\/\/fission.io\/."},{"key":"e_1_2_1_16_1","unstructured":"2022. Fn Flow. https:\/\/github.com\/fnproject\/flow\/. 2022. Fn Flow. https:\/\/github.com\/fnproject\/flow\/."},{"key":"e_1_2_1_17_1","unstructured":"2022. Google Cloud Composers. https:\/\/cloud.google.com\/composer\/. 2022. Google Cloud Composers. https:\/\/cloud.google.com\/composer\/."},{"key":"e_1_2_1_18_1","unstructured":"2022. Google Cloud Functions. https:\/\/cloud.google.com\/functions\/docs\/. 2022. Google Cloud Functions. https:\/\/cloud.google.com\/functions\/docs\/."},{"key":"e_1_2_1_19_1","unstructured":"2022. Google Workflows. https:\/\/cloud.google.com\/workflows. 2022. Google Workflows. https:\/\/cloud.google.com\/workflows."},{"key":"e_1_2_1_20_1","unstructured":"2022. Netflix Conductor. https:\/\/netflix.github.io\/conductor\/. 2022. Netflix Conductor. https:\/\/netflix.github.io\/conductor\/."},{"key":"e_1_2_1_21_1","unstructured":"2022. Temporal. https:\/\/temporal.io\/. 2022. Temporal. https:\/\/temporal.io\/."},{"key":"e_1_2_1_22_1","unstructured":"2022. Using Durable Objects Cloudflare Docs. https:\/\/developers.cloudflare.com\/workers\/learning\/using-durable-objects. 2022. Using Durable Objects Cloudflare Docs. https:\/\/developers.cloudflare.com\/workers\/learning\/using-durable-objects."},{"key":"e_1_2_1_23_1","unstructured":"2022. What are Durable Functions? - Microsoft Azure. https:\/\/docs.microsoft.com\/en-us\/azure\/azure-functions\/durable\/durable-functions-overview. 2022. What are Durable Functions? - Microsoft Azure. https:\/\/docs.microsoft.com\/en-us\/azure\/azure-functions\/durable\/durable-functions-overview."},{"key":"e_1_2_1_24_1","unstructured":"2022. What is Azure Event Hubs? - Microsoft Azure. https:\/\/docs.microsoft.com\/en-us\/azure\/event-hubs\/event-hubs-about. 2022. What is Azure Event Hubs? - Microsoft Azure. https:\/\/docs.microsoft.com\/en-us\/azure\/event-hubs\/event-hubs-about."},{"key":"e_1_2_1_25_1","unstructured":"2022. Workers Durable Objects Beta: A New Approach to Stateful Serverless. https:\/\/blog.cloudflare.com\/introducing-workers-durable-objects\/. 2022. Workers Durable Objects Beta: A New Approach to Stateful Serverless. https:\/\/blog.cloudflare.com\/introducing-workers-durable-objects\/."},{"key":"e_1_2_1_26_1","unstructured":"2022. Zeebe: A Workflow Engine for Microservices Orchestration. https:\/\/zeebe.io\/. 2022. Zeebe: A Workflow Engine for Microservices Orchestration. https:\/\/zeebe.io\/."},{"key":"e_1_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1145\/44498.45063"},{"key":"e_1_2_1_28_1","first-page":"196","article-title":"The development of Erlang","volume":"97","author":"Armstrong Joe","year":"1997","unstructured":"Joe Armstrong . 1997 . The development of Erlang . In ICFP , Vol. 97. 196 -- 203 . Joe Armstrong. 1997. The development of Erlang. In ICFP, Vol. 97. 196--203.","journal-title":"ICFP"},{"key":"e_1_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2018.00010"},{"key":"e_1_2_1_30_1","doi-asserted-by":"publisher","DOI":"10.1145\/3485510"},{"key":"e_1_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.1145\/2038916.2038932"},{"key":"e_1_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.1983.234957"},{"key":"e_1_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/582353.582376"},{"key":"e_1_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/3183713.3196898"},{"key":"e_1_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/1323293.1294281"},{"key":"e_1_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/2342356.2342360"},{"key":"e_1_2_1_37_1","unstructured":"Sadjad Fouladi Francisco Romero Dan Iter Qian Li Shuvo Chatterjee Christos Kozyrakis Matei Zaharia and Keith Winstein. 2019. From laptop to lambda: Outsourcing everyday jobs to thousands of transient functional containers. In 2019 {USENIX} Annual Technical Conference ({USENIX} {ATC} 19). 475--488. Sadjad Fouladi Francisco Romero Dan Iter Qian Li Shuvo Chatterjee Christos Kozyrakis Matei Zaharia and Keith Winstein. 2019. From laptop to lambda: Outsourcing everyday jobs to thousands of transient functional containers. In 2019 { USENIX } Annual Technical Conference ( { USENIX } { ATC } 19). 475--488."},{"key":"e_1_2_1_38_1","volume-title":"William Zeng, Rahul Bhalerao, Anirudh Sivaraman, George Porter, and Keith Winstein.","author":"Fouladi Sadjad","year":"2017","unstructured":"Sadjad Fouladi , Riad S Wahby , Brennan Shacklett , Karthikeyan Vasuki Balasubramaniam , William Zeng, Rahul Bhalerao, Anirudh Sivaraman, George Porter, and Keith Winstein. 2017 . Encoding, fast and slow: Low-latency video processing using thousands of tiny threads. In 14th {USENIX} Symposium on Networked Systems Design and Implementation ( {NSDI} 17). 363--376. Sadjad Fouladi, Riad S Wahby, Brennan Shacklett, Karthikeyan Vasuki Balasubramaniam, William Zeng, Rahul Bhalerao, Anirudh Sivaraman, George Porter, and Keith Winstein. 2017. Encoding, fast and slow: Low-latency video processing using thousands of tiny threads. In 14th {USENIX} Symposium on Networked Systems Design and Implementation ({NSDI} 17). 363--376."},{"key":"e_1_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/945445.945450"},{"key":"e_1_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.14778\/3377369.3377370"},{"key":"e_1_2_1_41_1","volume-title":"Proceedings of the","author":"Haller Philipp","unstructured":"Philipp Haller . 2012. On the integration of the actor model in mainstream technologies: the Scala perspective . In Proceedings of the 2 nd Edition on Programming Systems, Languages and Applications based on Actors, Agents, and Decentralized Control Abstractions. ACM , 1--6. Philipp Haller. 2012. On the integration of the actor model in mainstream technologies: the Scala perspective. In Proceedings of the 2nd Edition on Programming Systems, Languages and Applications based on Actors, Agents, and Decentralized Control Abstractions. ACM, 1--6.","edition":"2"},{"key":"e_1_2_1_42_1","volume-title":"Serverless Computing: One Step Forward, Two Steps Back. In CIDR","author":"Hellerstein Joseph M.","year":"2019","unstructured":"Joseph M. Hellerstein , Jose M. Faleiro , Joseph Gonzalez , Johann Schleier-Smith , Vikram Sreekanti , Alexey Tumanov , and Chenggang Wu . 2019 . Serverless Computing: One Step Forward, Two Steps Back. In CIDR 2019, 9th Biennial Conference on Innovative Data Systems Research, Asilomar, CA, USA, January 13--16, 2019, Online Proceedings . http:\/\/cidrdb.org\/cidr2019\/papers\/p119-hellerstein-cidr19.pdf Joseph M. Hellerstein, Jose M. Faleiro, Joseph Gonzalez, Johann Schleier-Smith, Vikram Sreekanti, Alexey Tumanov, and Chenggang Wu. 2019. Serverless Computing: One Step Forward, Two Steps Back. In CIDR 2019, 9th Biennial Conference on Innovative Data Systems Research, Asilomar, CA, USA, January 13--16, 2019, Online Proceedings. http:\/\/cidrdb.org\/cidr2019\/papers\/p119-hellerstein-cidr19.pdf"},{"key":"e_1_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.1145\/3360575"},{"key":"e_1_2_1_44_1","doi-asserted-by":"publisher","DOI":"10.1145\/3127479.3128601"},{"key":"e_1_2_1_45_1","volume-title":"Joseph E. Gonzalez, Raluca Ada Popa, Ion Stoica, and David A. Patterson.","author":"Jonas Eric","year":"2019","unstructured":"Eric Jonas , Johann Schleier-Smith , Vikram Sreekanti , Chia-che Tsai, Anurag Khandelwal , Qifan Pu , Vaishaal Shankar , Joao Carreira , Karl Krauth , Neeraja Jayant Yadwadkar , Joseph E. Gonzalez, Raluca Ada Popa, Ion Stoica, and David A. Patterson. 2019 . Cloud Programming Simplified: A Berkeley View on Serverless Computing. CoRR abs\/1902.03383 (2019). arXiv:1902.03383 http:\/\/arxiv.org\/abs\/1902.03383 Eric Jonas, Johann Schleier-Smith, Vikram Sreekanti, Chia-che Tsai, Anurag Khandelwal, Qifan Pu, Vaishaal Shankar, Joao Carreira, Karl Krauth, Neeraja Jayant Yadwadkar, Joseph E. Gonzalez, Raluca Ada Popa, Ion Stoica, and David A. Patterson. 2019. Cloud Programming Simplified: A Berkeley View on Serverless Computing. CoRR abs\/1902.03383 (2019). arXiv:1902.03383 http:\/\/arxiv.org\/abs\/1902.03383"},{"key":"e_1_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.3115\/v1\/E14-3011"},{"key":"e_1_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/2670979.2670985"},{"key":"e_1_2_1_48_1","volume-title":"Asynchronous Prefix Recoverability for Fast Distributed Stores. In SIGMOD 2021","author":"Li Tianyu","year":"2021","unstructured":"Tianyu Li , Badrish Chandramouli , Jose M. Faleiro , Samuel Madden , and Donald Kossmann . 2021 . Asynchronous Prefix Recoverability for Fast Distributed Stores. In SIGMOD 2021 ( Virtual Event, China , June 2021). 1090--1102. Tianyu Li, Badrish Chandramouli, Jose M. Faleiro, Samuel Madden, and Donald Kossmann. 2021. Asynchronous Prefix Recoverability for Fast Distributed Stores. In SIGMOD 2021 (Virtual Event, China, June 2021). 1090--1102."},{"key":"e_1_2_1_49_1","doi-asserted-by":"publisher","DOI":"10.1145\/3341302.3342080"},{"key":"e_1_2_1_50_1","doi-asserted-by":"publisher","DOI":"10.1145\/2452376.2452385"},{"key":"e_1_2_1_51_1","volume-title":"13th USENIX Symposium on Operating Systems Design and Implementation (OSDI 18)","author":"Moritz Philipp","year":"2018","unstructured":"Philipp Moritz , Robert Nishihara , Stephanie Wang , Alexey Tumanov , Richard Liaw , Eric Liang , Melih Elibol , Zongheng Yang , William Paul , Michael I Jordan , 2018 . Ray: A distributed framework for emerging AI applications . In 13th USENIX Symposium on Operating Systems Design and Implementation (OSDI 18) . 561--577. Philipp Moritz, Robert Nishihara, Stephanie Wang, Alexey Tumanov, Richard Liaw, Eric Liang, Melih Elibol, Zongheng Yang, William Paul, Michael I Jordan, et al. 2018. Ray: A distributed framework for emerging AI applications. In 13th USENIX Symposium on Operating Systems Design and Implementation (OSDI 18). 561--577."},{"key":"e_1_2_1_52_1","doi-asserted-by":"publisher","DOI":"10.1561\/1900000060"},{"key":"e_1_2_1_53_1","doi-asserted-by":"publisher","DOI":"10.1145\/1994.2209"},{"key":"e_1_2_1_54_1","doi-asserted-by":"publisher","DOI":"10.1145\/3318464.3380609"},{"key":"e_1_2_1_55_1","volume-title":"Serverless Architecture - A Revolution in Cloud Computing. In 2018 Tenth International Conference on Advanced Computing (ICoAC). 88--93","author":"Rajan R. A. P.","year":"2018","unstructured":"R. A. P. Rajan . 2018 . Serverless Architecture - A Revolution in Cloud Computing. In 2018 Tenth International Conference on Advanced Computing (ICoAC). 88--93 . R. A. P. Rajan. 2018. Serverless Architecture - A Revolution in Cloud Computing. In 2018 Tenth International Conference on Advanced Computing (ICoAC). 88--93."},{"key":"e_1_2_1_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3148.3161"},{"key":"e_1_2_1_57_1","doi-asserted-by":"publisher","DOI":"10.1145\/2463676.2465302"},{"key":"e_1_2_1_58_1","volume-title":"CIDR","author":"Schleier-Smith Johann","year":"2019","unstructured":"Johann Schleier-Smith . 2019. Serverless Foundations for Elastic Database Systems . In CIDR 2019 , 9th Biennial Conference on Innovative Data Systems Research, Asilomar, CA, USA, January 13--16, 2019, Online Proceedings . http:\/\/cidrdb.org\/cidr2019\/gongshow\/abstracts\/cidr2019_140.pdf Johann Schleier-Smith. 2019. Serverless Foundations for Elastic Database Systems. In CIDR 2019, 9th Biennial Conference on Innovative Data Systems Research, Asilomar, CA, USA, January 13--16, 2019, Online Proceedings. http:\/\/cidrdb.org\/cidr2019\/gongshow\/abstracts\/cidr2019_140.pdf"},{"key":"e_1_2_1_59_1","volume-title":"Paul Batum, Jason Cooke, Eduardo Laureano, Colby Tresness, Mark Russinovich, and Ricardo Bianchini.","author":"Shahrad Mohammad","year":"2020","unstructured":"Mohammad Shahrad , Rodrigo Fonseca , \u00cd\u00f1igo Goiri , Gohar Irfan Chaudhry , Paul Batum, Jason Cooke, Eduardo Laureano, Colby Tresness, Mark Russinovich, and Ricardo Bianchini. 2020 . Serverless in the Wild : Characterizing and Optimizing the Serverless Workload at a Large Cloud Provider. In Proceedings of the USENIX Annual Technical Conference (ATC). USENIX. https:\/\/www.microsoft.com\/en-us\/research\/publication\/serverless-in-the-wild-characterizing-and-optimizing-the-serverless-workload-at-a-large-cloud-provider\/ Mohammad Shahrad, Rodrigo Fonseca, \u00cd\u00f1igo Goiri, Gohar Irfan Chaudhry, Paul Batum, Jason Cooke, Eduardo Laureano, Colby Tresness, Mark Russinovich, and Ricardo Bianchini. 2020. Serverless in the Wild: Characterizing and Optimizing the Serverless Workload at a Large Cloud Provider. In Proceedings of the USENIX Annual Technical Conference (ATC). USENIX. https:\/\/www.microsoft.com\/en-us\/research\/publication\/serverless-in-the-wild-characterizing-and-optimizing-the-serverless-workload-at-a-large-cloud-provider\/"},{"key":"e_1_2_1_60_1","doi-asserted-by":"publisher","DOI":"10.1109\/MSST.2010.5496972"},{"key":"e_1_2_1_61_1","doi-asserted-by":"publisher","DOI":"10.1145\/3342195.3387535"},{"key":"e_1_2_1_62_1","doi-asserted-by":"publisher","DOI":"10.14778\/3407790.3407836"},{"key":"e_1_2_1_63_1","doi-asserted-by":"publisher","DOI":"10.1145\/2987550.2987575"},{"key":"e_1_2_1_64_1","doi-asserted-by":"publisher","DOI":"10.1109\/MIC.2018.053681358"},{"key":"e_1_2_1_65_1","doi-asserted-by":"publisher","DOI":"10.1145\/3318464.3389710"},{"key":"e_1_2_1_66_1","volume-title":"Sebastian Angel, and Vincent Liu.","author":"Zhang Haoran","year":"2020","unstructured":"Haoran Zhang , Adney Cardoza , Peter Baile Chen , Sebastian Angel, and Vincent Liu. 2020 . Fault-tolerant and Transactional Stateful Serverless Workflows. In 14th {USENIX} Symposium on Operating Systems Design and Implementation ( {OSDI} 20). Haoran Zhang, Adney Cardoza, Peter Baile Chen, Sebastian Angel, and Vincent Liu. 2020. Fault-tolerant and Transactional Stateful Serverless Workflows. In 14th {USENIX} Symposium on Operating Systems Design and Implementation ({OSDI} 20)."},{"key":"e_1_2_1_67_1","doi-asserted-by":"publisher","DOI":"10.1145\/3419111.3421277"}],"container-title":["Proceedings of the VLDB Endowment"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.14778\/3529337.3529344","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,12,28]],"date-time":"2022-12-28T09:45:42Z","timestamp":1672220742000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.14778\/3529337.3529344"}},"subtitle":["efficient execution of serverless workflows"],"short-title":[],"issued":{"date-parts":[[2022,4]]},"references-count":67,"journal-issue":{"issue":"8","published-print":{"date-parts":[[2022,4]]}},"alternative-id":["10.14778\/3529337.3529344"],"URL":"https:\/\/doi.org\/10.14778\/3529337.3529344","relation":{},"ISSN":["2150-8097"],"issn-type":[{"value":"2150-8097","type":"print"}],"subject":[],"published":{"date-parts":[[2022,4]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1511_2F2014_111_460 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,30]],"date-time":"2026-10-30T02:42:26Z","timestamp":1793328146151,"version":"4.5.0"},"reference-count":0,"publisher":"Sigma Xi","issue":"6","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Am. Sci.","Am. Scientist","Amer. Sci.","Amer. Scientist"],"published-print":{"date-parts":[[2014]]},"DOI":"10.1511\/2014.111.460","type":"journal-article","created":{"date-parts":[[2014,10,14]],"date-time":"2014-10-14T02:00:20Z","timestamp":1413252020000},"page":"460","source":"Crossref","is-referenced-by-count":727,"title":["The Statistical Crisis in Science"],"prefix":"10.1511","volume":"102","author":[{"given":"Andrew","family":"Gelman","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Eric","family":"Loken","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"353","container-title":["American Scientist"],"original-title":[],"deposited":{"date-parts":[[2017,6,27]],"date-time":"2017-06-27T15:09:56Z","timestamp":1498576196000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.americanscientist.org\/article\/the-statistical-crisis-in-science"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2014]]},"references-count":0,"journal-issue":{"issue":"6","published-print":{"date-parts":[[2014]]}},"URL":"https:\/\/doi.org\/10.1511\/2014.111.460","relation":{},"ISSN":["0003-0996","1545-2786"],"issn-type":[{"value":"0003-0996","type":"print"},{"value":"1545-2786","type":"electronic"}],"subject":[],"published":{"date-parts":[[2014]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1518_2F001872097778543886 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T16:24:05Z","timestamp":1793291045727,"version":"4.5.0"},"reference-count":120,"publisher":"SAGE Publications","issue":"2","license":[{"start":{"date-parts":[[1997,6,1]],"date-time":"1997-06-01T00:00:00Z","timestamp":865123200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/journals.sagepub.com\/page\/policies\/text-and-data-mining-license"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Hum Factors"],"published-print":{"date-parts":[[1997,6]]},"abstract":"<jats:p>This paper addresses theoretical, empirical, and analytical studies pertaining to human use, misuse, disuse, and abuse of automation technology. Use refers to the voluntary activation or disengagement of automation by human operators. Trust, mental workload, and risk can influence automation use, but interactions between factors and large individual differences make prediction of automation use difficult. Misuse refers to over reliance on automation, which can result in failures of monitoring or decision biases. Factors affecting the monitoring of automation include workload, automation reliability and consistency, and the saliency of automation state indicators. Disuse, or the neglect or underutilization of automation, is commonly caused by alarms that activate falsely. This often occurs because the base rate of the condition to be detected is not considered in setting the trade-off between false alarms and omissions. Automation abuse, or the automation of functions by designers and implementation by managers without due regard for the consequences for human performance, tends to define the operator's roles as by-products of the automation. Automation abuse can also promote misuse and disuse of automation by human operators. Understanding the factors associated with each of these aspects of human use of automation can lead to improved system design, effective training methods, and judicious policies and procedures involving automation use.<\/jats:p>","DOI":"10.1518\/001872097778543886","type":"journal-article","created":{"date-parts":[[2006,10,1]],"date-time":"2006-10-01T18:31:03Z","timestamp":1159727463000},"page":"230-253","source":"Crossref","is-referenced-by-count":3875,"title":["Humans and Automation: Use, Misuse, Disuse, Abuse"],"prefix":"10.1177","volume":"39","author":[{"given":"Raja","family":"Parasuraman","sequence":"first","affiliation":[{"name":"Catholic University of America, Washington, D.C"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Victor","family":"Riley","sequence":"additional","affiliation":[{"name":"Honeywell Technology Center, Minneapolis, Minnesota"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"179","published-online":{"date-parts":[[1997,6,1]]},"reference":[{"key":"atypb1","volume-title":"A simulation evaluation of the engine monitoring and control display","author":"Abbott, T. S.","year":"1990"},{"key":"atypb2","volume-title":"National plan to enhance aviation safety through human factors improvements","author":"Air Transport Association","year":"1989"},{"key":"atypb3","doi-asserted-by":"publisher","DOI":"10.1016\/0005-1098(83)90046-8"},{"key":"atypb4","doi-asserted-by":"publisher","DOI":"10.1177\/001872089203400502"},{"key":"atypb5","doi-asserted-by":"publisher","DOI":"10.1002\/hfm.4530020202"},{"key":"atypb6","volume-title":"Human-centered aircraft automation: A concept and guidelines","author":"Billings, C. E.","year":"1991"},{"key":"atypb7","first-page":"264","volume-title":"Human performance in automated systems: Current research and trends","author":"Billings, C. E.","year":"1994"},{"key":"atypb8","first-page":"243","volume-title":"Automation and human performance: Theory and applications","author":"Bowers, C. A.","year":"1996"},{"key":"atypb9","doi-asserted-by":"publisher","DOI":"10.1016\/0301-0511(95)05161-9"},{"key":"atypb10","volume-title":"Set phasers on stun","author":"Casey, S.","year":"1993"},{"key":"atypb11","doi-asserted-by":"publisher","DOI":"10.1177\/001872089403600402"},{"key":"atypb12","doi-asserted-by":"publisher","DOI":"10.1145\/4547.4551"},{"key":"atypb13","volume-title":"Flight crew information requirements","author":"Corwin, W. H.","year":"1993"},{"key":"atypb14","volume-title":"Origins of the modem mind: Three stages in the evolution of culture and cognition","author":"Donald, M.","year":"1991"},{"key":"atypb15","unstructured":"Duley, J. A., Westerman, S., Molloy, R. & Parasuraman, R. (in press). Effects of display superimposition on monitoring of automated tasks. In\n Proceedings of the 9th International Symposium on Aviation Psychology.\n Columbus: Ohio State University."},{"key":"atypb16","doi-asserted-by":"publisher","DOI":"10.1016\/0003-6870(77)90163-6"},{"key":"atypb17","first-page":"163","volume-title":"Automation and human performance: Theory and applications","author":"Endsley, M.","year":"1996"},{"key":"atypb18","volume-title":"CTAS: Computer intelligence for air traffic control in the terminal area","author":"Erzberger, H.","year":"1992"},{"key":"atypb19","unstructured":"Farber, E. & Paley, M. (1993, April).\n Using freeway traffic data to estimate the effectiveness of rear-end collision countermeasures.\n Paper presented at the Third Annual IVHS America Meeting, Washington, DC."},{"key":"atypb20","volume-title":"The national plan for aviation human factors","author":"Federal Aviation Administration","year":"1990"},{"key":"atypb21","volume-title":"Human engineering for an effective air navigation and traffic control system","author":"Fitts, P. M.","year":"1951"},{"key":"atypb22","volume-title":"Perceived human factors problems of flightdeck automation","author":"Funk, K.","year":"1995"},{"key":"atypb23","first-page":"157","volume-title":"Human factors: Man, machines and new technology","author":"Gerwin, D.","year":"1986"},{"key":"atypb24","doi-asserted-by":"publisher","DOI":"10.1037\/1076-898X.1.1.19"},{"key":"atypb25","doi-asserted-by":"publisher","DOI":"10.1109\/TSMC.1977.4309606"},{"key":"atypb26","doi-asserted-by":"publisher","DOI":"10.1287\/mnsc.39.12.1506"},{"key":"atypb27","first-page":"461","volume-title":"Automation and human performance: Theory and applications","author":"Hancock, P. A.","year":"1996"},{"key":"atypb28","volume-title":"Intelligent interfaces","author":"Hancock, P. A.","year":"1989"},{"key":"atypb29","doi-asserted-by":"publisher","DOI":"10.1016\/0022-4375(92)90001-P"},{"key":"atypb30","first-page":"337","volume-title":"Automation and human performance: Theory and applications","author":"Hancock, P. A.","year":"1996"},{"key":"atypb31","unstructured":"Harris, W., Hancock, P. A. & Arthur, E. (1993). The effect of task load projection on automation use, performance, and workload. In Proceedings of the 7th International Symposium on Aviation psychology (pp. 890A\u2013890F). Columbus: Ohio State University."},{"key":"atypb32","doi-asserted-by":"publisher","DOI":"10.1177\/001872088402600510"},{"key":"atypb33","unstructured":"Hilburn, B., Jorna, P. G. A. M. & Parasuraman, R. (1995). The effect of advanced ATC automation on mental workload and monitoring performance: An empirical investigation in Dutch airspace. In Proceedings of the 8th International Symposium on Aviation Psychology (pp. 1382\u20131387). Columbus: Ohio State University."},{"key":"atypb34","volume-title":"Human factors in air-traffic control","author":"Hopkin, V. D.","year":"1995"},{"key":"atypb35","volume-title":"Strategic plan for IVHS in the United States","author":"IVHS America","year":"1992"},{"key":"atypb36","doi-asserted-by":"publisher","DOI":"10.1177\/001872089303500203"},{"key":"atypb37","doi-asserted-by":"crossref","unstructured":"Kuchar, J. K. & Hansman, R. J. (1995). A probabilistic methodology for the evaluation of alerting system performance. In\n Proceedings of the IFAC\/IFIP\/IFORS\/IEA Symposium.\n Cambridge, MA.","DOI":"10.1016\/S1474-6670(17)45278-5"},{"key":"atypb38","volume-title":"Mindfulness","author":"Langer, E.","year":"1989"},{"key":"atypb39","doi-asserted-by":"publisher","DOI":"10.1080\/00140139208967392"},{"key":"atypb40","doi-asserted-by":"publisher","DOI":"10.1006\/ijhc.1994.1007"},{"key":"atypb41","unstructured":"Lewandowsky, S. & Nikolic, D. (1995, April).\n A connectionist approach to modeling the effects of automation.\n Paper presented at the 8th International Symposium on Aviation Psychology, Columbus, OH."},{"key":"atypb42","volume-title":"Report on the accident to Airbus A320-211 aircraft in Warsaw on 14 September 1993","author":"Main Commission Aircraft Accident Investigation\u2014WARSAW","year":"1994"},{"key":"atypb43","unstructured":"May, P., Molloy, R. & Parasuraman, R. (1993, October).\n Effects of automation reliability and failure rate on monitoring performance in a multitask environment.\n Paper presented at the Annual Meeting of the Human Factors Society, Seattle, WA."},{"key":"atypb44","unstructured":"McClellan, J. M. (1994, June). Can you trust your autopilot?\n Flying\n , 76\u201383."},{"key":"atypb45","first-page":"314","volume-title":"Human performance in automated systems: Recent research and trends","author":"McClumpha, A.","year":"1994"},{"key":"atypb46","doi-asserted-by":"publisher","DOI":"10.1109\/NAECON.1988.195102"},{"key":"atypb47","volume-title":"Rapport de la Commission d'Enquete sur l'Accident survenu le 20 Janvier 1992 pres du Mont Saite Odile a l\/Airbus A. 320 Immatricule F-GGED Exploite par lay Compagnie Air Inter","author":"Ministere de l'Equipement, des Transports et du Tourisme","year":"1993"},{"key":"atypb48","unstructured":"Molloy, R., Deaton, J. E. & Parasuraman, R. (1995). Monitoring performance with the EMACS display in an automated environment. In\n Proceedings of the 8th International Symposium on Aviation Psychology\n (pp. 68\u201373). Columbus, OH."},{"key":"atypb49","first-page":"224","volume-title":"Human performance in automated systems: Current research and trends","author":"Molloy, R.","year":"1994"},{"key":"atypb50","doi-asserted-by":"publisher","DOI":"10.1177\/001872089606380211"},{"key":"atypb51","unstructured":"Mosier, K., Heers, S., Skitka, L. & Burdick, M. (1996, March).\n Patterns in the use of cockpit automation.\n Paper presented at the 2nd Automation Technology and Human Performance Conference, Cocoa Beach, FL."},{"key":"atypb52","first-page":"201","volume-title":"Automation and human performance: Theory and applications","author":"Mosier, K.","year":"1996"},{"key":"atypb53","doi-asserted-by":"crossref","unstructured":"Mosier, K., Skitka, L., Burdick, M. & Heers, S. (1996). Automation bias, accountability, and verification behaviors. In Proceedings of the Human Factors and Ergonomics Society 40th Annual Meeting (pp. 204\u2013208). Santa Monica, CA: Human Factors and Ergonomics Society.","DOI":"10.1177\/154193129604000413"},{"key":"atypb54","first-page":"191","volume-title":"Human performance in automated systems: Current research and trends","author":"Mosier, K.","year":"1994"},{"key":"atypb55","volume-title":"Human performance in automated systems: Recent research and trends","author":"Mouloua, M.","year":"1994"},{"key":"atypb56","first-page":"71","volume-title":"Cognitive engineering in complex dynamic worlds","author":"Muir, B. M.","year":"1988"},{"key":"atypb57","volume-title":"Eastern Airlines L-1011, Miami, Florida, 20 December 1972","author":"National Transportation Safety Board","year":"1973"},{"key":"atypb58","volume-title":"China Airlines B-747-SP, 300 NM northwest of San Francisco, 19 February 1985","author":"National Transportation Safety Board","year":"1986"},{"key":"atypb59","volume-title":"Aircraft accident report: Stall and loss of control on final approach","author":"National Transportation Safety Board","year":"1994"},{"key":"atypb60","volume-title":"Collision of Washington Metropolitan Transit Authority Train T111 with standing train at Shady Grove Station, near Gaithersburg, MD, January 6, 1996","author":"National Transportation Safety Board","year":"1997"},{"key":"atypb61","volume-title":"Grounding of the Panamanian passenger ship Royal Majesty on Rose and Crown shoal near Nantucket, Massachusetts, June 10, 1995","author":"National Transportation Safety Board","year":"1997"},{"key":"atypb62","doi-asserted-by":"publisher","DOI":"10.1002\/hfm.4530050103"},{"key":"atypb63","doi-asserted-by":"publisher","DOI":"10.1098\/rstb.1990.0101"},{"key":"atypb64","unstructured":"Palmer, E. & Degani, A. (1991). Electronic checklists: Evaluation of two levels of automation. In Proceedings of the 6th International Symposium on Aviation Psychology (pp. 178\u2013183). Columbus: Ohio State University."},{"key":"atypb65","first-page":"147","volume-title":"Human factors and advanced aviation technologies","author":"Parasuraman, R.","year":"1993"},{"key":"atypb66","volume-title":"Theory and design of adaptive automation in aviation systems","author":"Parasuraman, R.","year":"1992"},{"key":"atypb67","doi-asserted-by":"publisher","DOI":"10.1080\/001401397188224"},{"key":"atypb68","doi-asserted-by":"publisher","DOI":"10.1207\/s15327108ijap0301_1"},{"key":"atypb69","volume-title":"Automation and human performance: Theory and applications","author":"Parasurarnan, R.","year":"1996"},{"key":"atypb70","first-page":"45","volume-title":"Human performance in automated systems: Current research and trends","author":"Parasuraman, R.","year":"1994"},{"key":"atypb71","doi-asserted-by":"publisher","DOI":"10.1518\/001872096778827279"},{"key":"atypb72","first-page":"91","volume-title":"Automation and human performance: Theory and applications","author":"Parasuraman, R.","year":"1996"},{"key":"atypb73","first-page":"A7","author":"Phillips, D.","year":"1995","journal-title":"Washington Post"},{"key":"atypb74","volume-title":"Advancing free flight through human factors","author":"Planzer, N.","year":"1995"},{"issue":"4","key":"atypb75","first-page":"28","volume":"61","author":"Poset, R. K.","year":"1992","journal-title":"Air Line Pilot"},{"key":"atypb76","volume-title":"Free flight implementation","author":"Radio and Technical Committee on Aeronautics","year":"1995"},{"key":"atypb77","volume-title":"Information processing and humanmachine interaction","author":"Rasmussen, J.","year":"1986"},{"key":"atypb78","doi-asserted-by":"publisher","DOI":"10.1017\/CBO9781139062367"},{"key":"atypb79","doi-asserted-by":"crossref","unstructured":"Riley, V. (1989). A general model of mixed-initiative human-machine systems. In Proceedings of the Human Factors Society 33rd Annual Meeting (pp. 124\u2013128). Santa Monica, CA: Human Factors and Ergonomics Society.","DOI":"10.1177\/154193128903300227"},{"key":"atypb80","unstructured":"Riley, V. (1994a).\n Human use of automation.\n Unpublished doctoral dissertation, University of Minnesota."},{"key":"atypb81","first-page":"8","volume-title":"Human performance in automated systems: Recent research and trends","author":"Riley, V.","year":"1994"},{"key":"atypb82","doi-asserted-by":"publisher","DOI":"10.1109\/DASC.1995.482836"},{"key":"atypb83","first-page":"19","volume-title":"Automation and human performance: Theory and applications","author":"Riley, V.","year":"1996"},{"key":"atypb84","volume-title":"Analytic methods for flight-deck automation design and evaluation. phase two report: Pilot use of automation","author":"Riley, V.","year":"1993"},{"key":"atypb85","doi-asserted-by":"publisher","DOI":"10.1177\/001872088803000405"},{"key":"atypb86","first-page":"267","volume-title":"Automation and human performance: Theory and applications","author":"Sarter, N.","year":"1996"},{"key":"atypb87","doi-asserted-by":"publisher","DOI":"10.1207\/s15327108ijap0101_4"},{"key":"atypb88","doi-asserted-by":"publisher","DOI":"10.1207\/s15327108ijap0401_1"},{"key":"atypb89","volume-title":"\u201cStrong, silent, and out-of the-loop\u201d: Properties of advanced (cockpit) automation and their impact on human-automation interaction","author":"Sarter, N.","year":"1995"},{"key":"atypb90","volume-title":"Cockpit monitoring and alerting systems","author":"Satchell, P.","year":"1993"},{"key":"atypb91","first-page":"37","volume-title":"Automation and human performance: Theory and applications","author":"Scerbo, M. S.","year":"1996"},{"key":"atypb92","volume-title":"The cognitive computer.","author":"Schank, R.","year":"1984"},{"key":"atypb93","first-page":"61","volume":"10","author":"Sheridan, T.","year":"1980","journal-title":"Technology Review"},{"key":"atypb94","volume-title":"Telerobotics, automation, and supervisory control","author":"Sheridan, T.","year":"1992"},{"key":"atypb95","unstructured":"Singh, I. L., Deaton, J. E. & Parasuraman, R. (1993, October).\n Development of a scale to measure pilot attitudes to cockpit automation.\n Paper presented at the Annual Meeting of the Human Factors Society, Seattle, WA."},{"key":"atypb96","doi-asserted-by":"publisher","DOI":"10.1207\/s15327108ijap0302_2"},{"key":"atypb97","doi-asserted-by":"publisher","DOI":"10.1080\/00221309.1993.9711153"},{"key":"atypb98","doi-asserted-by":"publisher","DOI":"10.1006\/ijhc.1996.0081"},{"key":"atypb99","doi-asserted-by":"publisher","DOI":"10.1002\/hfm.4530050107"},{"key":"atypb100","doi-asserted-by":"publisher","DOI":"10.1121\/1.397232"},{"key":"atypb101","doi-asserted-by":"publisher","DOI":"10.1177\/001872088803000406"},{"key":"atypb102","volume-title":"Digital avionics systems","author":"Spitzer, C. R.","year":"1987"},{"key":"atypb103","doi-asserted-by":"publisher","DOI":"10.1002\/hfm.4530040104"},{"key":"atypb104","doi-asserted-by":"publisher","DOI":"10.1037\/0003-066X.47.4.522"},{"key":"atypb105","first-page":"68","volume-title":"Human performance in automated systems: Current research and trends","author":"Thompson, J. M.","year":"1994"},{"key":"atypb106","doi-asserted-by":"publisher","DOI":"10.1126\/science.185.4157.1124"},{"key":"atypb107","doi-asserted-by":"publisher","DOI":"10.1207\/s15326969eco0203_2"},{"key":"atypb108","doi-asserted-by":"publisher","DOI":"10.1111\/j.1467-6486.1988.tb00039.x"},{"key":"atypb109","doi-asserted-by":"publisher","DOI":"10.1177\/001872088002200506"},{"key":"atypb110","volume-title":"Engineering psychology and human performance","author":"Wickens, C. D.","year":"1992","edition":"2"},{"key":"atypb111","unstructured":"Wickens, C. D. (1994). Designing for situation awareness and trust in automation. In Proceedings of the IFAC Conference on Integrated Systems Engineering (pp. 174\u2013179). Baden-Baden, Germany: International Federation on Automatic Control."},{"key":"atypb112","doi-asserted-by":"publisher","DOI":"10.1177\/001872088502700107"},{"key":"atypb113","doi-asserted-by":"publisher","DOI":"10.1016\/B978-0-08-057090-7.50019-9"},{"key":"atypb114","volume-title":"Human factors of advanced technology (\u201cglass cockpit\u201d) transport aircraft","author":"Wiener, E. L.","year":"1989"},{"key":"atypb115","doi-asserted-by":"publisher","DOI":"10.1080\/00140138008924809"},{"key":"atypb116","doi-asserted-by":"publisher","DOI":"10.1016\/0747-5632(91)90006-M"},{"key":"atypb117","first-page":"3","volume-title":"Automation and human performance: Theory and applications","author":"Woods, D. D.","year":"1996"},{"key":"atypb118","doi-asserted-by":"crossref","unstructured":"Woods, D. D., Wise, J. & Hanes, L. (1981). An evaluation of nuclear power plant safety parameter display systems. In Proceedings of the Human Factors Society 25th Annual Meeting (pp. 110\u2013114). Santa Monica, CA: Human Factors and Ergonomics Society.","DOI":"10.1177\/107118138102500127"},{"key":"atypb119","doi-asserted-by":"publisher","DOI":"10.1177\/001872088803000110"},{"key":"atypb120","volume-title":"In the age of the smart machine: The future of work and power","author":"Zuboff, S.","year":"1988"}],"container-title":["Human Factors: The Journal of the Human Factors and Ergonomics Society"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/journals.sagepub.com\/doi\/pdf\/10.1518\/001872097778543886","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/journals.sagepub.com\/doi\/pdf\/10.1518\/001872097778543886","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,4,29]],"date-time":"2026-04-29T07:50:19Z","timestamp":1777449019000},"score":1,"resource":{"primary":{"URL":"https:\/\/journals.sagepub.com\/doi\/10.1518\/001872097778543886"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[1997,6]]},"references-count":120,"journal-issue":{"issue":"2","published-print":{"date-parts":[[1997,6]]}},"alternative-id":["10.1518\/001872097778543886"],"URL":"https:\/\/doi.org\/10.1518\/001872097778543886","relation":{},"ISSN":["0018-7208","1547-8181"],"issn-type":[{"value":"0018-7208","type":"print"},{"value":"1547-8181","type":"electronic"}],"subject":[],"published":{"date-parts":[[1997,6]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_1518_2Fhfes_46_1_50_30392 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,30]],"date-time":"2026-10-30T01:58:36Z","timestamp":1793325516426,"version":"4.5.0"},"reference-count":0,"publisher":"Oxford University Press (OUP)","issue":"1","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Human Factors: The Journal of the Human Factors and Ergonomics Society"],"published-print":{"date-parts":[[2004,1,1]]},"DOI":"10.1518\/hfes.46.1.50_30392","type":"journal-article","created":{"date-parts":[[2009,2,17]],"date-time":"2009-02-17T12:40:51Z","timestamp":1234874451000},"page":"50-80","source":"Crossref","is-referenced-by-count":1711,"title":["Trust in Automation: Designing for Appropriate Reliance"],"prefix":"10.1093","volume":"46","author":[{"given":"J. D.","family":"Lee","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"K. A.","family":"See","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"286","container-title":["Human Factors: The Journal of the Human Factors and Ergonomics Society"],"original-title":[],"language":"en","deposited":{"date-parts":[[2016,4,7]],"date-time":"2016-04-07T12:16:35Z","timestamp":1460031395000},"score":1,"resource":{"primary":{"URL":"http:\/\/hfs.sagepub.com\/cgi\/doi\/10.1518\/hfes.46.1.50_30392"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2004,1,1]]},"references-count":0,"journal-issue":{"issue":"1","published-print":{"date-parts":[[2004,1,1]]}},"URL":"https:\/\/doi.org\/10.1518\/hfes.46.1.50_30392","relation":{},"ISSN":["0018-7208"],"issn-type":[{"value":"0018-7208","type":"print"}],"subject":[],"published":{"date-parts":[[2004,1,1]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_17226_2F25303 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,27]],"date-time":"2026-10-27T14:45:57Z","timestamp":1793112357215,"version":"4.5.0"},"publisher-location":"Washington, D.C.","reference-count":0,"publisher":"National Academies Press","isbn-type":[{"value":"9780309486163","type":"print"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2019,9,20]]},"DOI":"10.17226\/25303","type":"edited-book","created":{"date-parts":[[2019,5,7]],"date-time":"2019-05-07T13:48:24Z","timestamp":1557236904000},"source":"Crossref","is-referenced-by-count":295,"title":["Reproducibility and Replicability in Science"],"prefix":"10.17226","author":[{"name":"Committee on Reproducibility and Replicability in Science","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"name":"Board on Behavioral, Cognitive, and Sensory Sciences","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"name":"Committee on National Statistics","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"name":"Division of Behavioral and Social Sciences and Education","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"name":"Nuclear and Radiation Studies Board","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"name":"Division on Earth and Life Studies","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"name":"Board on Mathematical Sciences and Analytics","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"name":"Committee on Applied and Theoretical Statistics","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"name":"Division on Engineering and Physical Sciences","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"name":"Board on Research Data and Information","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"name":"Committee on Science, Engineering, Medicine, and Public Policy","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"name":"Policy and Global Affairs","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"name":"National Academies of Sciences, Engineering, and Medicine","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"6987","container-title":[],"original-title":[],"deposited":{"date-parts":[[2025,12,9]],"date-time":"2025-12-09T15:38:47Z","timestamp":1765294727000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.nationalacademies.org\/publications\/25303"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,9,20]]},"ISBN":["9780309486163"],"references-count":0,"URL":"https:\/\/doi.org\/10.17226\/25303","relation":{},"subject":[],"published":{"date-parts":[[2019,9,20]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_18653_2Fv1_2F2020_emnlp_main_609 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,27]],"date-time":"2026-10-27T03:58:25Z","timestamp":1793073505497,"version":"4.5.0"},"publisher-location":"Stroudsburg, PA, USA","reference-count":0,"publisher":"Association for Computational Linguistics","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2020]]},"DOI":"10.18653\/v1\/2020.emnlp-main.609","type":"proceedings-article","created":{"date-parts":[[2020,11,29]],"date-time":"2020-11-29T09:51:46Z","timestamp":1606643506000},"page":"7534-7550","source":"Crossref","is-referenced-by-count":227,"title":["Fact or Fiction: Verifying Scientific Claims"],"prefix":"10.18653","author":[{"given":"David","family":"Wadden","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shanchuan","family":"Lin","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kyle","family":"Lo","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lucy Lu","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Madeleine","family":"van Zuylen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Arman","family":"Cohan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hannaneh","family":"Hajishirzi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1643","event":{"name":"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)","location":"Online","start":{"date-parts":[[2020,11]]},"end":{"date-parts":[[2020,11]]}},"container-title":["Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)"],"original-title":[],"deposited":{"date-parts":[[2026,1,31]],"date-time":"2026-01-31T18:36:47Z","timestamp":1769884607000},"score":1,"resource":{"primary":{"URL":"https:\/\/aclanthology.org\/2020.emnlp-main.609"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020]]},"references-count":0,"URL":"https:\/\/doi.org\/10.18653\/v1\/2020.emnlp-main.609","relation":{},"subject":[],"published":{"date-parts":[[2020]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_18653_2Fv1_2F2022_acl_short_18 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,27]],"date-time":"2026-10-27T16:57:24Z","timestamp":1793120244242,"version":"4.5.0"},"publisher-location":"Stroudsburg, PA, USA","reference-count":0,"publisher":"Association for Computational Linguistics","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2022]]},"DOI":"10.18653\/v1\/2022.acl-short.18","type":"proceedings-article","created":{"date-parts":[[2022,6,2]],"date-time":"2022-06-02T21:34:53Z","timestamp":1654205693000},"page":"157-165","source":"Crossref","is-referenced-by-count":62,"title":["Data Contamination: From Memorization to Exploitation"],"prefix":"10.18653","author":[{"given":"Inbal","family":"Magar","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Roy","family":"Schwartz","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1643","event":{"name":"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)","location":"Dublin, Ireland","start":{"date-parts":[[2022,5]]},"end":{"date-parts":[[2022,5]]}},"container-title":["Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)"],"original-title":[],"deposited":{"date-parts":[[2022,6,2]],"date-time":"2022-06-02T21:40:23Z","timestamp":1654206023000},"score":1,"resource":{"primary":{"URL":"https:\/\/aclanthology.org\/2022.acl-short.18"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022]]},"references-count":0,"URL":"https:\/\/doi.org\/10.18653\/v1\/2022.acl-short.18","relation":{},"subject":[],"published":{"date-parts":[[2022]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_18653_2Fv1_2F2023_acl_long_910 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,27]],"date-time":"2026-10-27T16:58:36Z","timestamp":1793120316564,"version":"4.5.0"},"publisher-location":"Stroudsburg, PA, USA","reference-count":0,"publisher":"Association for Computational Linguistics","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.18653\/v1\/2023.acl-long.910","type":"proceedings-article","created":{"date-parts":[[2023,8,5]],"date-time":"2023-08-05T00:57:42Z","timestamp":1691197062000},"page":"16477-16508","source":"Crossref","is-referenced-by-count":135,"title":["RARR: Researching and Revising What Language Models Say, Using Language Models"],"prefix":"10.18653","author":[{"given":"Luyu","family":"Gao","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zhuyun","family":"Dai","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Panupong","family":"Pasupat","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Anthony","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Arun Tejasvi","family":"Chaganty","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yicheng","family":"Fan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Vincent","family":"Zhao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ni","family":"Lao","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hongrae","family":"Lee","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Da-Cheng","family":"Juan","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kelvin","family":"Guu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1643","event":{"name":"Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","location":"Toronto, Canada","start":{"date-parts":[[2023,7]]},"end":{"date-parts":[[2023,7]]}},"container-title":["Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)"],"original-title":[],"deposited":{"date-parts":[[2023,8,5]],"date-time":"2023-08-05T01:22:10Z","timestamp":1691198530000},"score":1,"resource":{"primary":{"URL":"https:\/\/aclanthology.org\/2023.acl-long.910"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"references-count":0,"URL":"https:\/\/doi.org\/10.18653\/v1\/2023.acl-long.910","relation":{},"subject":[],"published":{"date-parts":[[2023]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_18653_2Fv1_2F2023_emnlp_main_398 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,28]],"date-time":"2026-10-28T00:40:09Z","timestamp":1793148009495,"version":"4.5.0"},"publisher-location":"Stroudsburg, PA, USA","reference-count":0,"publisher":"Association for Computational Linguistics","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.18653\/v1\/2023.emnlp-main.398","type":"proceedings-article","created":{"date-parts":[[2023,12,10]],"date-time":"2023-12-10T16:58:19Z","timestamp":1702227499000},"page":"6465-6488","source":"Crossref","is-referenced-by-count":181,"title":["Enabling Large Language Models to Generate Text with Citations"],"prefix":"10.18653","author":[{"given":"Tianyu","family":"Gao","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Howard","family":"Yen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiatong","family":"Yu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Danqi","family":"Chen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1643","event":{"name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","location":"Singapore","start":{"date-parts":[[2023,12]]},"end":{"date-parts":[[2023,12]]}},"container-title":["Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing"],"original-title":[],"deposited":{"date-parts":[[2023,12,10]],"date-time":"2023-12-10T17:04:49Z","timestamp":1702227889000},"score":1,"resource":{"primary":{"URL":"https:\/\/aclanthology.org\/2023.emnlp-main.398"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"references-count":0,"URL":"https:\/\/doi.org\/10.18653\/v1\/2023.emnlp-main.398","relation":{},"subject":[],"published":{"date-parts":[[2023]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_18653_2Fv1_2F2023_emnlp_main_741 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,27]],"date-time":"2026-10-27T22:17:00Z","timestamp":1793139420274,"version":"4.5.0"},"publisher-location":"Stroudsburg, PA, USA","reference-count":0,"publisher":"Association for Computational Linguistics","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.18653\/v1\/2023.emnlp-main.741","type":"proceedings-article","created":{"date-parts":[[2023,12,10]],"date-time":"2023-12-10T16:58:19Z","timestamp":1702227499000},"page":"12076-12100","source":"Crossref","is-referenced-by-count":417,"title":["FActScore: Fine-grained Atomic Evaluation of Factual Precision in Long Form Text Generation"],"prefix":"10.18653","author":[{"given":"Sewon","family":"Min","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kalpesh","family":"Krishna","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xinxi","family":"Lyu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mike","family":"Lewis","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Wen-tau","family":"Yih","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Pang","family":"Koh","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Mohit","family":"Iyyer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Luke","family":"Zettlemoyer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hannaneh","family":"Hajishirzi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1643","event":{"name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","location":"Singapore","start":{"date-parts":[[2023,12]]},"end":{"date-parts":[[2023,12]]}},"container-title":["Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing"],"original-title":[],"deposited":{"date-parts":[[2023,12,10]],"date-time":"2023-12-10T17:07:25Z","timestamp":1702228045000},"score":1,"resource":{"primary":{"URL":"https:\/\/aclanthology.org\/2023.emnlp-main.741"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"references-count":0,"URL":"https:\/\/doi.org\/10.18653\/v1\/2023.emnlp-main.741","relation":{},"subject":[],"published":{"date-parts":[[2023]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_18653_2Fv1_2F2023_findings_emnlp_722 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T06:52:37Z","timestamp":1793256757718,"version":"4.5.0"},"publisher-location":"Stroudsburg, PA, USA","reference-count":0,"publisher":"Association for Computational Linguistics","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023]]},"DOI":"10.18653\/v1\/2023.findings-emnlp.722","type":"proceedings-article","created":{"date-parts":[[2023,12,10]],"date-time":"2023-12-10T21:58:19Z","timestamp":1702245499000},"page":"10776-10787","source":"Crossref","is-referenced-by-count":84,"title":["NLP Evaluation in trouble: On the Need to Measure LLM Data Contamination for each Benchmark"],"prefix":"10.18653","author":[{"given":"Oscar","family":"Sainz","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jon","family":"Campos","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Iker","family":"Garc\u00eda-Ferrero","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Julen","family":"Etxaniz","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Oier Lopez","family":"de Lacalle","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Eneko","family":"Agirre","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1643","event":{"name":"Findings of the Association for Computational Linguistics: EMNLP 2023","location":"Singapore","start":{"date-parts":[[2023,12]]},"end":{"date-parts":[[2023,12]]}},"container-title":["Findings of the Association for Computational Linguistics: EMNLP 2023"],"original-title":[],"deposited":{"date-parts":[[2023,12,10]],"date-time":"2023-12-10T22:15:59Z","timestamp":1702246559000},"score":1,"resource":{"primary":{"URL":"https:\/\/aclanthology.org\/2023.findings-emnlp.722"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"references-count":0,"URL":"https:\/\/doi.org\/10.18653\/v1\/2023.findings-emnlp.722","relation":{},"subject":[],"published":{"date-parts":[[2023]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_18653_2Fv1_2F2025_findings_emnlp_320 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T15:59:11Z","timestamp":1793289551789,"version":"4.5.0"},"publisher-location":"Stroudsburg, PA, USA","reference-count":0,"publisher":"Association for Computational Linguistics","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025]]},"DOI":"10.18653\/v1\/2025.findings-emnlp.320","type":"proceedings-article","created":{"date-parts":[[2025,11,8]],"date-time":"2025-11-08T12:29:45Z","timestamp":1762604985000},"page":"5977-6043","source":"Crossref","is-referenced-by-count":82,"title":["Agent Laboratory: Using LLM Agents as Research Assistants"],"prefix":"10.18653","author":[{"given":"Samuel","family":"Schmidgall","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yusheng","family":"Su","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ze","family":"Wang","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ximeng","family":"Sun","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jialian","family":"Wu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Xiaodong","family":"Yu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Jiang","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Michael","family":"Moor","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Zicheng","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Emad","family":"Barsoum","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1643","event":{"name":"Findings of the Association for Computational Linguistics: EMNLP 2025","location":"Suzhou, China","start":{"date-parts":[[2025,11]]},"end":{"date-parts":[[2025,11]]}},"container-title":["Findings of the Association for Computational Linguistics: EMNLP 2025"],"original-title":[],"deposited":{"date-parts":[[2025,11,8]],"date-time":"2025-11-08T12:37:13Z","timestamp":1762605433000},"score":1,"resource":{"primary":{"URL":"https:\/\/aclanthology.org\/2025.findings-emnlp.320"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025]]},"references-count":0,"URL":"https:\/\/doi.org\/10.18653\/v1\/2025.findings-emnlp.320","relation":{},"subject":[],"published":{"date-parts":[[2025]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_18653_2Fv1_2FN18_1074 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,27]],"date-time":"2026-10-27T03:58:24Z","timestamp":1793073504407,"version":"4.5.0"},"publisher-location":"Stroudsburg, PA, USA","reference-count":0,"publisher":"Association for Computational Linguistics","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2018]]},"DOI":"10.18653\/v1\/n18-1074","type":"proceedings-article","created":{"date-parts":[[2018,5,30]],"date-time":"2018-05-30T00:40:52Z","timestamp":1527640852000},"page":"809-819","source":"Crossref","is-referenced-by-count":679,"title":["FEVER: a Large-scale Dataset for Fact Extraction and VERification"],"prefix":"10.18653","author":[{"given":"James","family":"Thorne","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Andreas","family":"Vlachos","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Christos","family":"Christodoulopoulos","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Arpit","family":"Mittal","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1643","event":{"name":"Proceedings of the 2018 Conference of the North American Chapter of\n the Association for Computational Linguistics: Human Language\n Technologies, Volume 1 (Long Papers)","location":"New Orleans, Louisiana","start":{"date-parts":[[2018,6]]},"end":{"date-parts":[[2018,6]]}},"container-title":["Proceedings of the 2018 Conference of the North American Chapter of\n the Association for Computational Linguistics: Human Language\n Technologies, Volume 1 (Long Papers)"],"original-title":[],"deposited":{"date-parts":[[2018,5,30]],"date-time":"2018-05-30T00:41:19Z","timestamp":1527640879000},"score":1,"resource":{"primary":{"URL":"http:\/\/aclweb.org\/anthology\/N18-1074"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018]]},"references-count":0,"URL":"https:\/\/doi.org\/10.18653\/v1\/n18-1074","relation":{},"subject":[],"published":{"date-parts":[[2018]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_2139_2Fssrn_4573321 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,26]],"date-time":"2026-10-26T13:25:32Z","timestamp":1793021132514,"version":"4.5.0"},"posted":{"date-parts":[[2023]]},"group-title":"SSRN","reference-count":68,"publisher":"Elsevier BV","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"abstract":"<jats:p>We introduce and study the concept of a \u201cjagged technology frontier\u201d to describe the uneven impact of artificial intelligence (AI) capabilities, where AI assistance improves performance for some tasks but worsens it for others, even within the same knowledge workflow and with a seemingly similar level of difficulty. In collaboration with the global management consulting firm Boston Consulting Group, we have developed realistic management consulting tasks and examined the human performance implications of using AI to perform complex and knowledge-intensive work. The preregistered experiment\n<br>\ninvolved 758 knowledge workers. After establishing a performance baseline on similar tasks, subjects were randomly assigned to one of three conditions: no AI access, GPT-4 AI access, or GPT-4 AI access with a prompt engineering overview. For each one of a set of 18 realistic knowledge tasks within the frontier of AI capabilities ranging from creative to analytical tasks, subjects using AI outperformed those not using AI, completing 12.2% more tasks and completing them 25.1% more quickly on average while also delivering solutions of significantly improved quality. However, for a complex managerial task selected to be outside the frontier, subjects using AI were 19% less likely to produce correct solutions compared with those without AI, pointing to potential limitations of AI supporting knowledge workers. We discuss the positive and negative implications of AI-aided human performance in knowledge-intensive tasks.<\/jats:p>","DOI":"10.2139\/ssrn.4573321","type":"posted-content","created":{"date-parts":[[2023,9,18]],"date-time":"2023-09-18T08:54:45Z","timestamp":1695027285000},"source":"Crossref","is-referenced-by-count":634,"title":["Navigating the Jagged Technological Frontier: Field Experimental Evidence of the Effects of Artificial Intelligence on Knowledge Worker Productivity and Quality"],"prefix":"10.2139","author":[{"given":"Fabrizio","family":"Dell'Acqua","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Edward","family":"McFowland III","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Ethan R.","family":"Mollick","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-3461-003X","authenticated-orcid":true,"given":"Hila","family":"Lifshitz-Assaf","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Katherine","family":"Kellogg","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Saran","family":"Rajendran","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Lisa","family":"Krayer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Fran\u00e7ois","family":"Candelon","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-5535-8304","authenticated-orcid":true,"given":"Karim R.","family":"Lakhani","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"key":"ref1","first-page":"1043","article-title":"Skills, tasks and technologies: Implications for employment and earnings","volume":"4","author":"D Acemoglu","year":"2011","journal-title":"Labor Economics"},{"key":"ref2","author":"A Agrawal","year":"2018","journal-title":"Prediction Machines: The Simple Economics of Artificial Intelligence"},{"issue":"5","key":"ref3","doi-asserted-by":"crossref","first-page":"1090","DOI":"10.1227\/neu.0000000000002551","article-title":"Performance of ChatGPT, GPT-4, and Google Bard on a neurosurgery oral boards preparation question bank","volume":"93","author":"R Ali","year":"2023","journal-title":"Neurosurgery"},{"issue":"1","key":"ref4","doi-asserted-by":"crossref","first-page":"149","DOI":"10.1287\/orsc.2021.1554","article-title":"Algorithm-augmented work and domain experience: The countervailing forces of ability and aversion","volume":"33","author":"R Allen","year":"2022","journal-title":"Organ. Sci"},{"issue":"5","key":"ref5","doi-asserted-by":"crossref","first-page":"1672","DOI":"10.1287\/orsc.2022.1651","article-title":"Collaborating\" with AI: Taking a system view to explore the future of work","volume":"34","author":"C Anthony","year":"2023","journal-title":"Organ. Sci"},{"issue":"3 Part 2","key":"ref6","doi-asserted-by":"crossref","first-page":"849","DOI":"10.1287\/isre.1110.0408","article-title":"Information, technology, and information worker productivity","volume":"23","author":"S Aral","year":"2012","journal-title":"Inform. Systems Res"},{"issue":"1","key":"ref7","doi-asserted-by":"crossref","first-page":"80","DOI":"10.1257\/pandp.20201034","article-title":"The allocation of decision authority to human and artificial intelligence","volume":"110","author":"S Athey","year":"2020","journal-title":"AEA Papers Proc"},{"issue":"1","key":"ref8","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1287\/orsc.2021.1562","article-title":"We are all theorists of technology now: A relational perspective on emerging technology and organizing","volume":"33","author":"D E Bailey","year":"2022","journal-title":"Organ. Sci"},{"issue":"5","key":"ref9","doi-asserted-by":"crossref","first-page":"1448","DOI":"10.1287\/orsc.1100.0639","article-title":"Reconfiguring boundary relations: Robotic innovations in pharmacy work","volume":"23","author":"M Barrett","year":"2012","journal-title":"Organ. Sci"},{"issue":"1","key":"ref10","doi-asserted-by":"crossref","first-page":"87","DOI":"10.1177\/0001839217751692","article-title":"Shadow learning: Building robotic surgical skill when approved means fail","volume":"64","author":"M Beane","year":"2019","journal-title":"Admin. Sci. Quart"},{"issue":"5","key":"ref11","doi-asserted-by":"crossref","first-page":"2025","DOI":"10.1111\/joms.12867","article-title":"Pace layering as a metaphor for organizing in the age of intelligent technologies: Considering the future of work by theorizing the future of organizing","volume":"62","author":"M I Beane","year":"2025","journal-title":"J. Management Stud"},{"issue":"4","key":"ref12","doi-asserted-by":"crossref","first-page":"424","DOI":"10.5465\/amd.2023.0106","article-title":"Capturing value from artificial intelligence","volume":"9","author":"J M Berg","year":"2023","journal-title":"Acad. Management Discoveries"},{"key":"ref13","article-title":"The rapid adoption of generative AI","author":"A Blandin","year":"2026","journal-title":"Management Sci"},{"issue":"7992","key":"ref14","doi-asserted-by":"crossref","first-page":"570","DOI":"10.1038\/s41586-023-06792-0","article-title":"Autonomous chemical research with large language models","volume":"624","author":"D A Boiko","year":"2023","journal-title":"Nature"},{"issue":"5","key":"ref15","doi-asserted-by":"crossref","first-page":"1589","DOI":"10.1287\/orsc.2023.18430","article-title":"The crowdless future? Generative AI and creative problemsolving","volume":"35","author":"L Boussioux","year":"2024","journal-title":"Organ. Sci"},{"issue":"4","key":"ref16","doi-asserted-by":"crossref","first-page":"217","DOI":"10.1057\/s11369-021-00224-5","article-title":"The power of prediction: Predictive analytics, workplace complements, and business performance","volume":"56","author":"E Brynjolfsson","year":"2021","journal-title":"Bus. Econom"},{"issue":"2","key":"ref17","first-page":"889","article-title":"Generative AI at work. Quart","volume":"140","author":"E Brynjolfsson","year":"2025","journal-title":"J. Econom"},{"issue":"1","key":"ref18","doi-asserted-by":"crossref","first-page":"43","DOI":"10.1257\/pandp.20181019","article-title":"What can machines learn and what does it mean for occupations and the economy?","volume":"108","author":"E Brynjolfsson","year":"2018","journal-title":"AEA Papers Proc"},{"key":"ref19","author":"Z Buc \ufffdinca","year":"2021","journal-title":"To trust or to think: Cognitive forcing functions can reduce overreliance on AI in"},{"issue":"1","key":"ref20","article-title":"Navigating the Jagged Technological Frontier Organization Science, Articles in Advance","volume":"5","author":"Dell'acqua","journal-title":"Proc. ACM Human-Comput. Interaction"},{"issue":"2","key":"ref21","article-title":"Deep learning for chest X-ray analysis: A survey","volume":"73","author":"E Sogancioglu","year":"2021","journal-title":"Medical Image Anal"},{"issue":"2","key":"ref22","first-page":"384","article-title":"AI assistance in legal analysis: An empirical study","volume":"73","author":"J H Choi","year":"2024","journal-title":"J. Legal Ed"},{"issue":"2","key":"ref23","first-page":"536","article-title":"Human-AI ensembles: When can they work?","volume":"51","author":"V Choudhary","year":"2023","journal-title":"J. Management"},{"key":"ref24","first-page":"679","article-title":"Biased programmers? Or biased data? A field experiment in operationalizing AI ethics","author":"B Cowgill","year":"2020","journal-title":"Proc. 21st ACM Conf. Econom. Comput. (EC '20)"},{"issue":"7887","key":"ref25","doi-asserted-by":"crossref","first-page":"70","DOI":"10.1038\/s41586-021-04086-x","article-title":"Advancing mathematics by guiding human intuition with AI","volume":"600","author":"A Davies","year":"2021","journal-title":"Nature"},{"key":"ref26","author":"F Dell'acqua","year":"2022","journal-title":"Falling asleep at the wheel: Human\/AI collaboration in a field experiment on HR recruiters. Working paper, Laboratory for Innovation Science"},{"issue":"4","key":"ref27","doi-asserted-by":"crossref","first-page":"951","DOI":"10.1162\/rest_a_01328","article-title":"Super Mario meets AI: Experimental effects of automation and skills on team performance and coordination","volume":"107","author":"F Dell'acqua","year":"2025","journal-title":"Rev. Econom. Statist"},{"issue":"6797","key":"ref28","article-title":"Why providing humans with interpretable algorithms may, counterintuitively, lead to lower decision-making performance","author":"T Destefano","year":"2022"},{"key":"ref29","author":"P F Drucker","year":"1959","journal-title":"Landmarks of Tomorrow: A Report on the New Post Modern World"},{"issue":"6702","key":"ref30","doi-asserted-by":"crossref","first-page":"1306","DOI":"10.1126\/science.adj0998","article-title":"GPTs are GPTs: Labor market impact potential of LLMs","volume":"384","author":"T Eloundou","year":"2024","journal-title":"Science"},{"issue":"4","key":"ref31","doi-asserted-by":"crossref","first-page":"771","DOI":"10.1177\/14761270221130253","article-title":"Strategic organization in the digital age: Rethinking the concept of technology","volume":"20","author":"S Faraj","year":"2022","journal-title":"Strategic Organ"},{"key":"ref32","volume":"10","author":"E W Felten","year":"2023","journal-title":"Occupational heterogeneity in exposure to generative AI"},{"issue":"7","key":"ref33","doi-asserted-by":"crossref","first-page":"611","DOI":"10.1038\/s42256-022-00512-5","article-title":"Bringing artificial intelligence to business management","volume":"4","author":"S Feuerriegel","year":"2022","journal-title":"Nature Machine Intelligence"},{"issue":"1","key":"ref34","doi-asserted-by":"crossref","first-page":"161","DOI":"10.1086\/699936","article-title":"AI and the economy","volume":"19","author":"J Furman","year":"2019","journal-title":"Innovation Policy Econom"},{"issue":"11","key":"ref35","doi-asserted-by":"crossref","first-page":"2724","DOI":"10.1002\/smj.3512","article-title":"Training with AI: Evidence from chess computers","volume":"44","author":"F Gaessler","year":"2023","journal-title":"Strategic Management J"},{"issue":"2","key":"ref36","doi-asserted-by":"crossref","first-page":"233","DOI":"10.1177\/05694345231169654","article-title":"ChatGPT has aced the test of understanding in college economics: Now what?","volume":"68","author":"W Geerling","year":"2023","journal-title":"Amer. Economist"},{"issue":"4","key":"ref37","doi-asserted-by":"crossref","first-page":"619","DOI":"10.1002\/smj.3569","article-title":"Decision authority and the returns to algorithms","volume":"45","author":"E Glaeser","year":"2024","journal-title":"Strategic Management J"},{"issue":"6","key":"ref38","doi-asserted-by":"crossref","first-page":"1977","DOI":"10.1287\/orsc.2023.18441","article-title":"The short-term effects of generative artificial intelligence on employment: Evidence from an online labor market","volume":"35","author":"X Hui","year":"2024","journal-title":"Organ. Sci"},{"issue":"1","key":"ref39","doi-asserted-by":"crossref","DOI":"10.1073\/pnas.2414972121","article-title":"The unequal adoption of ChatGPT exacerbates existing inequalities among workers","volume":"122","author":"A Humlum","year":"2025","journal-title":"Proc. Natl. Acad. Sci. USA"},{"key":"ref40","author":"M Iansiti","year":"2020","journal-title":"Competing in the Age of AI: Strategy and Leadership When Algorithms and Networks Run the World"},{"key":"ref41","article-title":"Commentary on jagged capabilities of large language models","author":"A Karpathy","year":"2024","journal-title":"Twitter"},{"key":"ref42","author":"A Karpathy","year":"2025","journal-title":"2025 LLM year in review"},{"issue":"2","key":"ref43","doi-asserted-by":"crossref","first-page":"571","DOI":"10.1287\/orsc.2021.1445","article-title":"Local adaptation without work intensification: Experimentalist governance of digital technology for mutually beneficial role reconfiguration in organizations","volume":"33","author":"K C Kellogg","year":"2022","journal-title":"Organ. Sci"},{"issue":"1","key":"ref44","doi-asserted-by":"crossref","first-page":"181","DOI":"10.1287\/orsc.2020.1374","article-title":"Moving violations: Pairing an illegitimate learning hierarchy with trainee status mobility for acquiring new skills when traditional expertise erodes","volume":"32","author":"K C Kellogg","year":"2021","journal-title":"Organ. Sci"},{"issue":"3","key":"ref45","doi-asserted-by":"crossref","first-page":"1501","DOI":"10.25300\/MISQ\/2021\/16564","article-title":"Is AI ground truth really true? The dangers of training and evaluating AI tools based on experts' know-what","volume":"45","author":"S Lebovitz","year":"2021","journal-title":"MIS Quart"},{"issue":"1","key":"ref46","doi-asserted-by":"crossref","first-page":"126","DOI":"10.1287\/orsc.2021.1549","article-title":"To engage or not to engage with AI for critical judgments: How professionals deal with opacity when using AI for medical diagnosis","volume":"33","author":"S Lebovitz","year":"2022","journal-title":"Organ. Sci"},{"issue":"13","key":"ref47","doi-asserted-by":"crossref","first-page":"1233","DOI":"10.1056\/NEJMsr2214184","article-title":"Benefits, limits, and risks of GPT-4 as an AI chatbot for medicine","volume":"388","author":"P Lee","year":"2023","journal-title":"New England J. Medicine"},{"key":"ref48","author":"L Meincke","year":"2024","journal-title":"Using large language models for idea generation in innovation. Working paper, Operations, Information and Decisions"},{"key":"ref49","first-page":"267","article-title":"An approach toward design and implementation of distributed framework for astronomical big data processing","volume":"431","author":"R Monisha","year":"2021"},{"issue":"7956","key":"ref50","doi-asserted-by":"crossref","first-page":"259","DOI":"10.1038\/s41586-023-05881-4","article-title":"Foundation models for generalist medical artificial intelligence","volume":"616","author":"M Moor","year":"2023","journal-title":"Nature"},{"issue":"6654","key":"ref51","doi-asserted-by":"crossref","first-page":"187","DOI":"10.1126\/science.adh2586","article-title":"Experimental evidence on the productivity effects of generative artificial intelligence","volume":"381","author":"S Noy","year":"2023","journal-title":"Science"},{"key":"ref52","volume":"15","author":"Openai","year":"2023","journal-title":"GPT-4 technical report"},{"key":"ref53","author":"N G Otis","year":"2024","journal-title":"Global evidence on gender gaps and generative AI"},{"key":"ref54","author":"N Otis","year":"2024","journal-title":"The uneven impact of generative AI on entrepreneurial performance"},{"key":"ref55","author":"S Peng","year":"2023","journal-title":"The impact of AI on developer productivity: Evidence from GitHub Copilot"},{"key":"ref56","article-title":"Interview with Lex Fridman","author":"S Pichai","year":"2025","journal-title":"Lex Fridman Podcast #471"},{"issue":"1","key":"ref57","doi-asserted-by":"crossref","first-page":"192","DOI":"10.5465\/amr.2018.0072","article-title":"Artificial intelligence and management: The automation-augmentation paradox","volume":"46","author":"S Raisch","year":"2021","journal-title":"Acad. Management Rev"},{"key":"ref58","author":"S Randazzo","year":"2025","journal-title":"GenAI as a power persuader: How professionals get persuasion bombed when they attempt to validate LLMs"},{"key":"ref59","author":"S Randazzo","year":"2025","journal-title":") Cyborgs, centaurs and self-automators: The three modes of human-GenAI knowledge work and their implications for skilling and the future of expertise"},{"key":"ref60","author":"S Reed","year":"2022","journal-title":"A generalist agent"},{"key":"ref61","author":"Dell'acqua","journal-title":"Navigating the Jagged Technological Frontier"},{"key":"ref62","first-page":"55565","article-title":"Are emergent abilities of large language models a mirage?","volume":"36","author":"R Schaeffer","year":"2023","journal-title":"Adv. Neural Inform. Processing Systems"},{"issue":"5","key":"ref63","doi-asserted-by":"crossref","first-page":"1248","DOI":"10.1287\/orsc.2019.1343","article-title":"Losing touch: An embodiment perspective on coordination in robotic surgery","volume":"31","author":"A V Sergeeva","year":"2020","journal-title":"Organ. Sci"},{"issue":"7972","key":"ref64","doi-asserted-by":"crossref","first-page":"172","DOI":"10.1038\/s41586-023-06291-2","article-title":"Large language models encode clinical knowledge","volume":"620","author":"K Singhal","year":"2023","journal-title":"Nature"},{"issue":"5","key":"ref65","doi-asserted-by":"crossref","first-page":"991","DOI":"10.1006\/ijhc.1999.0252","article-title":"Does automation bias decision-making?","volume":"51","author":"L J Skitka","year":"1999","journal-title":"Internat. J. Human-Comput. Stud"},{"issue":"12","key":"ref66","doi-asserted-by":"crossref","first-page":"2293","DOI":"10.1038\/s41562-024-02024-1","article-title":"When combinations of humans and AI are useful: A systematic review and metaanalysis","volume":"8","author":"M Vaccaro","year":"2024","journal-title":"Nature Human Behav"},{"key":"ref67","article-title":"MBB explained: How hard it is to get hired and what it's like to work for the prestigious strategy consulting firms","author":"K Vlamis","year":"2024","journal-title":"McKinsey, Bain, and BCG. Bus. Insider Africa"},{"issue":"3","key":"ref68","doi-asserted-by":"crossref","DOI":"10.1093\/pnasnexus\/pgae052","article-title":"Generative artificial intelligence, human creativity, and art","volume":"3","author":"E Zhou","year":"2024","journal-title":"PNAS Nexus"}],"container-title":[],"original-title":[],"deposited":{"date-parts":[[2026,8,14]],"date-time":"2026-08-14T17:42:58Z","timestamp":1786729378000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.ssrn.com\/abstract=4573321"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023]]},"references-count":68,"URL":"https:\/\/doi.org\/10.2139\/ssrn.4573321","relation":{},"subject":[],"published":{"date-parts":[[2023]]},"subtype":"preprint"}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_2196_2F53164 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T01:12:52Z","timestamp":1793236372354,"version":"4.5.0"},"reference-count":32,"publisher":"JMIR Publications Inc.","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["J Med Internet Res"],"abstract":"<jats:sec>\n <jats:title>Background<\/jats:title>\n <jats:p>Large language models (LLMs) have raised both interest and concern in the academic community. They offer the potential for automating literature search and synthesis for systematic reviews but raise concerns regarding their reliability, as the tendency to generate unsupported (hallucinated) content persist.<\/jats:p>\n <\/jats:sec>\n <jats:sec>\n <jats:title>Objective<\/jats:title>\n <jats:p>The aim of the study is to assess the performance of LLMs such as ChatGPT and Bard (subsequently rebranded Gemini) to produce references in the context of scientific writing.<\/jats:p>\n <\/jats:sec>\n <jats:sec>\n <jats:title>Methods<\/jats:title>\n <jats:p>The performance of ChatGPT and Bard in replicating the results of human-conducted systematic reviews was assessed. Using systematic reviews pertaining to shoulder rotator cuff pathology, these LLMs were tested by providing the same inclusion criteria and comparing the results with original systematic review references, serving as gold standards. The study used 3 key performance metrics: recall, precision, and F1-score, alongside the hallucination rate. Papers were considered \u201challucinated\u201d if any 2 of the following information were wrong: title, first author, or year of publication.<\/jats:p>\n <\/jats:sec>\n <jats:sec>\n <jats:title>Results<\/jats:title>\n <jats:p>In total, 11 systematic reviews across 4 fields yielded 33 prompts to LLMs (3 LLMs\u00d711 reviews), with 471 references analyzed. Precision rates for GPT-3.5, GPT-4, and Bard were 9.4% (13\/139), 13.4% (16\/119), and 0% (0\/104) respectively (P<.001). Recall rates were 11.9% (13\/109) for GPT-3.5 and 13.7% (15\/109) for GPT-4, with Bard failing to retrieve any relevant papers (P<.001). Hallucination rates stood at 39.6% (55\/139) for GPT-3.5, 28.6% (34\/119) for GPT-4, and 91.4% (95\/104) for Bard (P<.001). Further analysis of nonhallucinated papers retrieved by GPT models revealed significant differences in identifying various criteria, such as randomized studies, participant criteria, and intervention criteria. The study also noted the geographical and open-access biases in the papers retrieved by the LLMs.<\/jats:p>\n <\/jats:sec>\n <jats:sec>\n <jats:title>Conclusions<\/jats:title>\n <jats:p>Given their current performance, it is not recommended for LLMs to be deployed as the primary or exclusive tool for conducting systematic reviews. Any references generated by such models warrant thorough validation by researchers. The high occurrence of hallucinations in LLMs highlights the necessity for refining their training and functionality before confidently using them for rigorous academic purposes.<\/jats:p>\n <\/jats:sec>","DOI":"10.2196\/53164","type":"journal-article","created":{"date-parts":[[2024,5,22]],"date-time":"2024-05-22T10:45:28Z","timestamp":1716374728000},"page":"e53164","source":"Crossref","is-referenced-by-count":456,"title":["Hallucination Rates and Reference Accuracy of ChatGPT and Bard for Systematic Reviews: Comparative Analysis"],"prefix":"10.2196","volume":"26","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-3787-8362","authenticated-orcid":false,"given":"Mika\u00ebl","family":"Chelli","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3662-9903","authenticated-orcid":false,"given":"Jules","family":"Descamps","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-2404-8469","authenticated-orcid":false,"given":"Vincent","family":"Lavou\u00e9","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7545-5972","authenticated-orcid":false,"given":"Christophe","family":"Trojani","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-6683-4452","authenticated-orcid":false,"given":"Michel","family":"Azar","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-2094-559X","authenticated-orcid":false,"given":"Marcel","family":"Deckert","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0009-0008-9971-9213","authenticated-orcid":false,"given":"Jean-Luc","family":"Raynier","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1737-6234","authenticated-orcid":false,"given":"Gilles","family":"Clowez","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-0140-9505","authenticated-orcid":false,"given":"Pascal","family":"Boileau","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-7703-6456","authenticated-orcid":false,"given":"Caroline","family":"Ruetsch-Chelli","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1010","published-online":{"date-parts":[[2024,5,22]]},"reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1108\/lhtn-01-2023-0009"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1016\/S2589-7500(23)00023-7"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-023-00056-7"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1148\/radiol.223312"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1186\/s13054-023-04380-2"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-023-00107-z"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1016\/j.amjmed.2023.02.011"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1016\/j.jse.2020.11.012"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1016\/j.jse.2022.04.024"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1177\/0363546519835491"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1016\/j.otsr.2019.11.004"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.2106\/JBJS.E.01097"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1136\/bmj.b2700"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1186\/2046-4053-1-2"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2303.08774"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1136\/bjsports-2018-100486"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1002\/pmrj.12268"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1016\/j.ptsp.2019.11.003"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1177\/0363546519881423"},{"key":"ref20","doi-asserted-by":"publisher","DOI":"10.1016\/j.jse.2019.11.010"},{"key":"ref21","doi-asserted-by":"publisher","DOI":"10.1002\/14651858.CD012879.pub2"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1177\/0269215520934147"},{"key":"ref23","doi-asserted-by":"publisher","DOI":"10.1016\/j.apmr.2020.06.013"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.2340\/16501977-2725"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1016\/j.arthro.2020.11.051"},{"key":"ref26","unstructured":"HillierMWhy does ChatGPT generate fake references?TECHE20232023-05-17https:\/\/teche.mq.edu.au\/2023\/02\/why-does-chatgpt-generate-fake-references\/"},{"key":"ref27","doi-asserted-by":"publisher","DOI":"10.1016\/j.mcpdig.2023.05.004"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1080\/00330124.2023.2190373"},{"key":"ref29","doi-asserted-by":"publisher","DOI":"10.7759\/cureus.35179"},{"key":"ref30","unstructured":"Introducing ChatGPTOpenAI20222023-05-18https:\/\/openai.com\/blog\/chatgpt"},{"key":"ref31","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2009.01325"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1016\/S2589-7500(23)00083-3"}],"container-title":["Journal of Medical Internet Research"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.jmir.org\/2024\/1\/e53164","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,30]],"date-time":"2026-06-30T17:38:36Z","timestamp":1782841116000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.jmir.org\/2024\/1\/e53164"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,5,22]]},"references-count":32,"URL":"https:\/\/doi.org\/10.2196\/53164","relation":{"has-preprint":[{"id-type":"doi","id":"10.2196\/preprints.53164","asserted-by":"object"}]},"ISSN":["1438-8871"],"issn-type":[{"value":"1438-8871","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,5,22]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_2196_2F80371 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,30]],"date-time":"2026-10-30T02:23:58Z","timestamp":1793327038693,"version":"4.5.0"},"reference-count":16,"publisher":"JMIR Publications Inc.","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["JMIR Ment Health"],"abstract":"<jats:sec>\n <jats:title>Background<\/jats:title>\n <jats:p>Mental health researchers are increasingly using large language models (LLMs) to improve efficiency, yet these tools can generate fabricated but plausible-sounding content (hallucinations). A notable form of hallucination involves fabricated bibliographic citations that cannot be traced to real publications. Although previous studies have explored citation fabrication across disciplines, it remains unclear whether citation accuracy in LLM output systematically varies across topics within the same field that differ in public visibility, scientific maturity, and specialization.<\/jats:p>\n <\/jats:sec>\n <jats:sec>\n <jats:title>Objective<\/jats:title>\n <jats:p>This study aims to examine the frequency and nature of citation fabrication and bibliographic errors in GPT-4o (Omni) outputs when generating literature reviews on mental health topics that varied in public familiarity and scientific maturity. We also tested whether prompt specificity (general vs specialized) influenced fabrication or accuracy rates.<\/jats:p>\n <\/jats:sec>\n <jats:sec>\n <jats:title>Methods<\/jats:title>\n <jats:p>In June 2025, GPT-4o was prompted to generate 6 literature reviews (~2000 words; \u226520 citations) on 3 disorders representing different levels of public awareness and research coverage: major depressive disorder (high), binge eating disorder (moderate), and body dysmorphic disorder (low). Each disorder was reviewed at 2 levels of specificity: a general overview (symptoms, impacts, and treatments) and a specialized review (evidence for digital interventions). All citations were extracted (N=176) and systematically verified using Google Scholar, Scopus, PubMed, WorldCat, and publisher databases. Citations were classified as fabricated (no identifiable source), real with errors, or fully accurate. Fabrication and accuracy rates were compared by disorder and review type by using chi-square tests.<\/jats:p>\n <\/jats:sec>\n <jats:sec>\n <jats:title>Results<\/jats:title>\n <jats:p>Across the 6 reviews, GPT-4o generated 176 citations; 35 (19.9%) were fabricated. Among the 141 real citations, 64 (45.4%) contained errors, most frequently incorrect or invalid digital object identifiers. Fabrication rates differed significantly by disorder (\u03c722=13.7; P=.001), with higher rates for binge eating disorder (17\/60, 28%) and body dysmorphic disorder (14\/48, 29%) than for major depressive disorder (4\/68, 6%). While fabrication did not differ overall by review type, stratified analyses showed higher fabrication for specialized versus general reviews of binge eating disorder (11\/24, 46% vs 6\/36, 17%; P=.01). Accuracy rates also varied by disorder (\u03c722=11.6; P=.003), being lowest for body dysmorphic disorder (20\/34, 59%) and highest for major depressive disorder (41\/64, 64%). Accuracy rates differed by review type within some disorders, including higher accuracy for general reviews of major depressive disorder (26\/34, 77% vs 15\/30, 50%; P=.03).<\/jats:p>\n <\/jats:sec>\n <jats:sec>\n <jats:title>Conclusions<\/jats:title>\n <jats:p>Citation fabrication and bibliographic errors remain common in GPT-4o outputs, with nearly two-thirds of citations being fabricated or inaccurate. Reliability systematically varied by disorder familiarity and prompt specificity, with greater risks in less visible or specialized mental health topics. These findings highlight the need for careful prompt design, rigorous human verification of all model-generated references, and stronger journal and institutional safeguards to protect research integrity as LLMs are integrated into academic practice.<\/jats:p>\n <\/jats:sec>","DOI":"10.2196\/80371","type":"journal-article","created":{"date-parts":[[2025,11,12]],"date-time":"2025-11-12T21:45:07Z","timestamp":1762983907000},"page":"e80371","source":"Crossref","is-referenced-by-count":43,"title":["Influence of Topic Familiarity and Prompt Specificity on Citation Fabrication in Mental Health Research Using Large Language Models: Experimental Study"],"prefix":"10.2196","volume":"12","author":[{"ORCID":"https:\/\/orcid.org\/0000-0003-4475-7139","authenticated-orcid":false,"given":"Jake","family":"Linardon","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0001-8225-4511","authenticated-orcid":false,"given":"Hannah K","family":"Jarman","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-3926-8058","authenticated-orcid":false,"given":"Zoe","family":"McClure","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-1203-5492","authenticated-orcid":false,"given":"Cleo","family":"Anderson","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0003-1359-2394","authenticated-orcid":false,"given":"Claudia","family":"Liu","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/orcid.org\/0000-0002-7186-7264","authenticated-orcid":false,"given":"Mariel","family":"Messer","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1010","published-online":{"date-parts":[[2025,11,12]]},"reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1108\/13620431111167751"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.48550\/arXiv.2307.06435"},{"key":"ref3","unstructured":"TrajchevaBChatGPT usage statistics: 40+ insights on engagement, adoption, and business impactDesignRush202507012025-07-07https:\/\/www.designrush.com\/agency\/ai-companies\/trends\/chatgpt-usage-statistics?"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1136\/bmjment-2025-301787"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-023-02980-0"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1145\/3703155"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1038\/s41598-023-41032-5"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.2196\/52935"},{"key":"ref9","doi-asserted-by":"publisher","DOI":"10.1016\/j.psychres.2023.115334"},{"key":"ref10","doi-asserted-by":"publisher","DOI":"10.1136\/eb-2017-102702"},{"key":"ref11","doi-asserted-by":"publisher","DOI":"10.1016\/j.jad.2024.10.018"},{"key":"ref12","doi-asserted-by":"publisher","DOI":"10.1037\/ccp0000575"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1186\/s12888-025-06797-1"},{"key":"ref14","doi-asserted-by":"publisher","DOI":"10.1073\/pnas.2404328121"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1177\/08465371231171125"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1016\/j.mcpdig.2023.05.004"}],"container-title":["JMIR Mental Health"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/mental.jmir.org\/2025\/1\/e80371","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,6,30]],"date-time":"2026-06-30T18:13:50Z","timestamp":1782843230000},"score":1,"resource":{"primary":{"URL":"https:\/\/mental.jmir.org\/2025\/1\/e80371"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,12]]},"references-count":16,"URL":"https:\/\/doi.org\/10.2196\/80371","relation":{"has-preprint":[{"id-type":"doi","id":"10.2196\/preprints.80371","asserted-by":"object"}]},"ISSN":["2368-7959"],"issn-type":[{"value":"2368-7959","type":"electronic"}],"subject":[],"published":{"date-parts":[[2025,11,12]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_3389_2Ffpsyg_2016_01832 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,26]],"date-time":"2026-10-26T22:03:02Z","timestamp":1793052182739,"version":"4.5.0"},"reference-count":57,"publisher":"Frontiers Media SA","funder":[{"DOI":"10.13039\/501100003246","name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","doi-asserted-by":"publisher","award":["406-13-050"],"award-info":[{"award-number":["406-13-050"]}],"id":[{"id":"10.13039\/501100003246","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003246","name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","doi-asserted-by":"publisher","award":["406-15-198"],"award-info":[{"award-number":["406-15-198"]}],"id":[{"id":"10.13039\/501100003246","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100003246","name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","doi-asserted-by":"publisher","award":["452-11-004"],"award-info":[{"award-number":["452-11-004"]}],"id":[{"id":"10.13039\/501100003246","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["frontiersin.org"],"crossmark-restriction":true},"short-container-title":["Front. Psychol."],"DOI":"10.3389\/fpsyg.2016.01832","type":"journal-article","created":{"date-parts":[[2016,11,25]],"date-time":"2016-11-25T03:14:13Z","timestamp":1480043653000},"update-policy":"https:\/\/doi.org\/10.3389\/crossmark-policy","source":"Crossref","is-referenced-by-count":529,"title":["Degrees of Freedom in Planning, Running, Analyzing, and Reporting Psychological Studies: A Checklist to Avoid p-Hacking"],"prefix":"10.3389","volume":"7","author":[{"given":"Jelte M.","family":"Wicherts","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Coosje L. S.","family":"Veldkamp","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Hilde E. M.","family":"Augusteijn","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marjan","family":"Bakker","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Robbie C. M.","family":"van Aert","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marcel A. L. M.","family":"van Assen","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"1965","published-online":{"date-parts":[[2016,11,25]]},"reference":[{"key":"B1","doi-asserted-by":"publisher","first-page":"108","DOI":"10.1002\/per.1919","article-title":"Recommendations for increasing replicability in psychology.","volume":"27","author":"Asendorpf","year":"2013","journal-title":"Eur. J. Personal."},{"key":"B2","doi-asserted-by":"publisher","first-page":"1069","DOI":"10.1177\/0956797616647519","article-title":"Researchers\u2019 Intuitions about power in psychological research.","volume":"27","author":"Bakker","year":"2016","journal-title":"Psychol. Sci."},{"key":"B3","doi-asserted-by":"publisher","first-page":"543","DOI":"10.1177\/1745691612459060","article-title":"The rules of the game called psychological science.","volume":"7","author":"Bakker","year":"2012","journal-title":"Perspect. Psychol. Sci."},{"key":"B4","doi-asserted-by":"publisher","first-page":"666","DOI":"10.3758\/s13428-011-0089-5","article-title":"The (mis)reporting of statistical results in psychology journals.","volume":"43","author":"Bakker","year":"2011","journal-title":"Behav. Res. Methods"},{"key":"B5","doi-asserted-by":"publisher","first-page":"409","DOI":"10.1037\/met0000014","article-title":"Outlier removal, sum scores, and the inflation of the Type I error rate in independent samples t tests. The power of alternatives and recommendations.","volume":"19","author":"Bakker","year":"2014","journal-title":"Psychol. Methods"},{"key":"B6","first-page":"253","article-title":"\u201cStudying the mind in the middle; A practical guide to priming and automaticity research,\u201d in","author":"Bargh","year":"2000","journal-title":"Handbook of Research Methods in Social and Personality Psychology,"},{"key":"B7","author":"Barnett","year":"1994","journal-title":"Outliers in Statistical Data."},{"key":"B8","doi-asserted-by":"publisher","first-page":"609","DOI":"10.1016\/j.cortex.2012.12.016","article-title":"Registered reports: a new publishing initiative at Cortex.","volume":"49","author":"Chambers","year":"2013","journal-title":"Cortex"},{"key":"B9","doi-asserted-by":"publisher","first-page":"2457","DOI":"10.1001\/jama.291.20.2457","article-title":"Empirical evidence for selective reporting of outcomes in randomized trials - Comparison of Protocols to published articles.","volume":"291","author":"Chan","year":"2004","journal-title":"JAMA"},{"key":"B10","doi-asserted-by":"publisher","first-page":"1304","DOI":"10.1037\/0003-066X.45.12.1304","article-title":"Things I have learned (thus far).","volume":"45","author":"Cohen","year":"1990","journal-title":"Am. Psychol."},{"key":"B11","doi-asserted-by":"publisher","first-page":"447","DOI":"10.1037\/1082-989X.2.4.447","article-title":"Finding the missing science: the fate of studies submitted for review by a human subjects committee.","volume":"2","author":"Cooper","year":"1997","journal-title":"Psychol. Methods"},{"key":"B12","author":"Cumming","year":"2012","journal-title":"Understanding the New Statistics: Effect Sizes, Confidence Intervals, and Meta-Analysis."},{"key":"B13","doi-asserted-by":"publisher","first-page":"7","DOI":"10.1177\/0956797613504966","article-title":"The new statistics: why and how.","volume":"25","author":"Cumming","year":"2014","journal-title":"Psychol. Sci."},{"key":"B14","doi-asserted-by":"publisher","first-page":"188","DOI":"10.1016\/j.actpsy.2014.02.001","article-title":"The meaning of \u201csignificance\u201d for different types of research [translated and annotated by Eric-Jan Wagenmakers, Denny Borsboom, Josine Verhagen, Rogier Kievit, Marjan Bakker, Angelique Cramer, Dora Matzke, Don Mellenbergh, and Han L. J. van der Maas]. 1969.","volume":"148","author":"de Groot","year":"1956\/2014","journal-title":"Acta Psychol."},{"key":"B15","doi-asserted-by":"publisher","first-page":"499","DOI":"10.1037\/a0039191","article-title":"Opportunistic biases: their origins, effects, and an integrated solution.","volume":"70","author":"DeCoster","year":"2015","journal-title":"Am. Psychol."},{"key":"B16","doi-asserted-by":"publisher","first-page":"3","DOI":"10.1177\/0956797613512465","article-title":"Business not as usual.","volume":"25","author":"Eich","year":"2014","journal-title":"Psychol. Sci."},{"key":"B17","doi-asserted-by":"publisher","first-page":"153","DOI":"10.1016\/j.jmp.2013.02.003","article-title":"Replication, statistical consistency, and publication bias.","volume":"57","author":"Francis","year":"2013","journal-title":"J. Math. Psychol."},{"key":"B18","doi-asserted-by":"publisher","first-page":"8","DOI":"10.1177\/1948550615598377","article-title":"Underreporting in psychology experiments: evidence from a study registry.","volume":"7","author":"Franco","year":"2016","journal-title":"Soc. Psychol. Personal. Sci."},{"key":"B19","doi-asserted-by":"publisher","first-page":"460","DOI":"10.1511\/2014.111.460","article-title":"The statistical crisis in science.","volume":"102","author":"Gelman","year":"2014","journal-title":"Am. Sci."},{"key":"B20","author":"Hubbard","year":"2015","journal-title":"Corrupt Research: the Case for Reconceptualizing Empirical Management and Social Science."},{"issue":"e124","key":"B21","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pmed.0020124","article-title":"Why most published research findings are false.","volume":"2","author":"Ioannidis","year":"2005","journal-title":"PLoS Med."},{"key":"B22","doi-asserted-by":"publisher","first-page":"640","DOI":"10.1097\/EDE.0b013e31818131e7","article-title":"Why most discovered true associations are inflated.","volume":"19","author":"Ioannidis","year":"2008","journal-title":"Epidemiology"},{"key":"B23","doi-asserted-by":"publisher","first-page":"524","DOI":"10.1177\/0956797611430953","article-title":"Measuring the prevalence of questionable research practices with incentives for truth-telling.","volume":"23","author":"John","year":"2012","journal-title":"Psychol. Sci."},{"key":"B24","doi-asserted-by":"publisher","first-page":"196","DOI":"10.1207\/s15327957pspr0203_4","article-title":"HARKing: hypothesizing after the results are known.","volume":"2","author":"Kerr","year":"1998","journal-title":"Pers. Soc. Psychol. Rev."},{"issue":"e1002456","key":"B25","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pbio.1002456","article-title":"Badges to acknowledge open practices: a simple, low-cost, effective method for increasing transparency.","volume":"14","author":"Kidwell","year":"2016","journal-title":"PLoS Biol."},{"issue":"c365","key":"B26","doi-asserted-by":"publisher","DOI":"10.1136\/bmj.c365","article-title":"The impact of outcome reporting bias in randomised controlled trials on a cohort of systematic reviews.","volume":"340","author":"Kirkham","year":"2010","journal-title":"BMJ"},{"key":"B27","doi-asserted-by":"publisher","first-page":"1551","DOI":"10.1038\/jcbfm.2010.86","article-title":"Everything you never wanted to know about circular analysis, but were afraid to ask.","volume":"30","author":"Kriegeskorte","year":"2010","journal-title":"J. Cereb. Blood Flow Metab."},{"key":"B28","doi-asserted-by":"publisher","first-page":"535","DOI":"10.1038\/nn.2303","article-title":"Circular analysis in systems neuroscience: the dangers of double dipping.","volume":"12","author":"Kriegeskorte","year":"2009","journal-title":"Nat. Neurosci."},{"key":"B29","author":"Kruschke","year":"2015","journal-title":"Doing Bayesian Data Analysis: A Tutorial Introduction with R, JAGS and Stan."},{"key":"B30","doi-asserted-by":"publisher","first-page":"424","DOI":"10.1177\/1745691613491437","article-title":"PsychDisclosure. org grassroots support for reforming reporting standards in psychology.","volume":"8","author":"LeBel","year":"2013","journal-title":"Perspect. Psychol. Sci."},{"key":"B31","doi-asserted-by":"publisher","first-page":"487","DOI":"10.1037\/a0039400","article-title":"Is psychology suffering from a replication crisis? What does \u201cfailure to replicate\u201d really mean?","volume":"70","author":"Maxwell","year":"2015","journal-title":"Am. Psychol."},{"key":"B32","doi-asserted-by":"publisher","first-page":"1105","DOI":"10.1038\/nn.2886","article-title":"Erroneous analyses of interactions in neuroscience: a problem of significance.","volume":"14","author":"Nieuwenhuis","year":"2011","journal-title":"Nat. Neurosci."},{"key":"B33","doi-asserted-by":"publisher","first-page":"1422","DOI":"10.1126\/science.aab2374","article-title":"Promoting an open research culture: author guidelines for journals could help to promote transparency, openness, and reproducibility.","volume":"348","author":"Nosek","year":"2015","journal-title":"Science"},{"key":"B34","doi-asserted-by":"publisher","first-page":"615","DOI":"10.1177\/1745691612459058","article-title":"Scientific Utopia: II - Restructuring incentives and practices to promote truth over publishability.","volume":"7","author":"Nosek","year":"2012","journal-title":"Perspect. Psychol. Sci."},{"key":"B35","doi-asserted-by":"publisher","DOI":"10.3758\/s13428-015-0664-2","article-title":"The prevalence of statistical reporting errors in psychology (1985-2013).","author":"Nuijten","year":"2015","journal-title":"Behav. Res. Methods"},{"issue":"aac4716","key":"B36","doi-asserted-by":"publisher","DOI":"10.1126\/science.aac4716","article-title":"Estimating the reproducibility of psychological science.","volume":"349","author":"Open Science Collaboration","year":"2015","journal-title":"Science"},{"key":"B37","doi-asserted-by":"publisher","first-page":"867","DOI":"10.1016\/j.jesp.2009.03.009","article-title":"Instructional manipulation checks: detecting satisficing to increase statistical power.","volume":"45","author":"Oppenheimer","year":"2009","journal-title":"J. Exp. Soc. Psychol."},{"key":"B38","doi-asserted-by":"publisher","DOI":"10.1101\/05918","article-title":"Scanning the Horizon: towards transparent and reproducible neuroimaging research","author":"Poldrack","year":"2016","journal-title":"bioRxiv"},{"key":"B39","author":"Rosenthal","year":"1966","journal-title":"Experimenter Effects in Behavioral Research."},{"key":"B40","first-page":"178","article-title":"I just ran two million regressions.","volume":"87","author":"Sala-I-Martin","year":"1997","journal-title":"Am. Econ. Rev."},{"key":"B41","doi-asserted-by":"publisher","first-page":"147","DOI":"10.1037\/1082-989x.7.2.147","article-title":"Missing data: our view of the state of the art.","volume":"7","author":"Schafer","year":"2002","journal-title":"Psychol. Methods"},{"key":"B42","doi-asserted-by":"publisher","first-page":"107","DOI":"10.1016\/j.jesp.2015.09.0096","article-title":"The empirical benefits of conceptual rigor: systematic articulation of conceptual hypotheses can reduce the risk of non-replicable results (and facilitate novel discoveries too).","volume":"66","author":"Schaller","year":"2016","journal-title":"J. Exp. Soc. Psychol."},{"key":"B43","doi-asserted-by":"publisher","first-page":"309","DOI":"10.1037\/0033-2909.105.2.309","article-title":"Do studies of statistical power have an effect on the power of studies?","volume":"105","author":"Sedlmeier","year":"1989","journal-title":"Psychol. Bull."},{"key":"B44","doi-asserted-by":"publisher","first-page":"1359","DOI":"10.1177\/0956797611417632","article-title":"False-positive psychology: undisclosed flexibility in data collection and analysis allows presenting anything as significant.","volume":"22","author":"Simmons","year":"2011","journal-title":"Psychol. Sci."},{"key":"B45","doi-asserted-by":"publisher","first-page":"552","DOI":"10.1177\/1745691614543974","article-title":"An introduction to registered replication reports at perspectives on psychological science.","volume":"9","author":"Simons","year":"2014","journal-title":"Perspect. Psychol. Sci."},{"key":"B46","doi-asserted-by":"publisher","first-page":"666","DOI":"10.1177\/1745691614553988","article-title":"p-curve and effect size. Correcting for publication bias using only significant results.","volume":"9","author":"Simonsohn","year":"2014","journal-title":"Perspect. Psychol. Sci."},{"key":"B47","doi-asserted-by":"publisher","first-page":"702","DOI":"10.1177\/1745691616658637","article-title":"Increasing transparency through a multiverse analysis.","volume":"11","author":"Steegen","year":"2016","journal-title":"Perspect. Psychol. Sci."},{"key":"B48","doi-asserted-by":"publisher","first-page":"643","DOI":"10.1037\/xge0000159","article-title":"Meta-analysis to integrate effect sizes within an article: possible misuse and Type I error inflation.","volume":"145","author":"Ueno","year":"2016","journal-title":"J. Exp. Psychol. Gen."},{"key":"B49","doi-asserted-by":"publisher","first-page":"713","DOI":"10.1177\/1745691616650874","article-title":"Conducting meta-analyses based on p-values: reservations and recommendations for applying p- uniform and p-curve.","volume":"11","author":"van Aert","year":"2016","journal-title":"Perspect. Psychol. Sci."},{"key":"B50","doi-asserted-by":"publisher","first-page":"779","DOI":"10.3758\/BF03194105","article-title":"A practical solution to the pervasive problems of p values.","volume":"14","author":"Wagenmakers","year":"2007","journal-title":"Psychon. Bull. Rev."},{"key":"B51","doi-asserted-by":"publisher","first-page":"632","DOI":"10.1177\/1745691612463078","article-title":"An agenda for purely confirmatory research.","volume":"7","author":"Wagenmakers","year":"2012","journal-title":"Perspect. Psychol. Sci."},{"key":"B52","doi-asserted-by":"publisher","first-page":"426","DOI":"10.1037\/a0022790","article-title":"Why psychologists must change the way they analyze their data: the case of psi: comment on Bem (2011).","volume":"100","author":"Wagenmakers","year":"2011","journal-title":"J. Pers. Soc. Psychol."},{"issue":"e010075","key":"B53","doi-asserted-by":"publisher","DOI":"10.1136\/bmjopen-2015-010075","article-title":"Feasibility study to examine discrepancy rates in prespecified and reported outcomes in articles submitted to The BMJ.","volume":"6","author":"Weston","year":"2016","journal-title":"BMJ Open"},{"key":"B54","article-title":"\u201cData re-analysis and open data,\u201d in","author":"Wicherts","year":"2017","journal-title":"Toward a More Perfect Psychology: Improving Trust, Accuracy, and Transparency in Research,"},{"issue":"e26828","key":"B55","doi-asserted-by":"publisher","DOI":"10.1371\/journal.pone.0026828","article-title":"Willingness to share research data is related to the strength of the evidence and the quality of reporting of statistical results.","volume":"6","author":"Wicherts","year":"2011","journal-title":"PLoS ONE"},{"key":"B56","doi-asserted-by":"publisher","first-page":"27","DOI":"10.1007\/s11336-015-9445-1","article-title":"Encourage playing with data and discourage questionable reporting practices.","volume":"81","author":"Wigboldus","year":"2016","journal-title":"Psychometrika"},{"key":"B57","author":"Wilcox","year":"2012","journal-title":"Modern Statistics for the Social and Behavioral Sciences: A Practical Introduction."}],"container-title":["Frontiers in Psychology"],"original-title":[],"link":[{"URL":"http:\/\/journal.frontiersin.org\/article\/10.3389\/fpsyg.2016.01832\/full","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2016,11,25]],"date-time":"2016-11-25T03:14:17Z","timestamp":1480043657000},"score":1,"resource":{"primary":{"URL":"http:\/\/journal.frontiersin.org\/article\/10.3389\/fpsyg.2016.01832\/full"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,11,25]]},"references-count":57,"URL":"https:\/\/doi.org\/10.3389\/fpsyg.2016.01832","relation":{},"ISSN":["1664-1078"],"issn-type":[{"value":"1664-1078","type":"electronic"}],"subject":[],"published":{"date-parts":[[2016,11,25]]}}}
\ No newline at end of file

added research/verify_cache/https_api_crossref_org_works_10_7759_2Fcureus_35179 +1 −0

			`@@ -0,0 +1 @@`
	1	+{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,10,29]],"date-time":"2026-10-29T13:41:09Z","timestamp":1793281269459,"version":"4.5.0"},"reference-count":4,"publisher":"Springer Science and Business Media LLC","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2023,2,19]]},"DOI":"10.7759\/cureus.35179","type":"journal-article","created":{"date-parts":[[2023,2,19]],"date-time":"2023-02-19T09:38:42Z","timestamp":1676799522000},"source":"Crossref","is-referenced-by-count":989,"title":["Artificial Hallucinations in ChatGPT: Implications in Scientific Writing"],"prefix":"10.7759","author":[{"given":"Hussam","family":"Alkaissi","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Samy I","family":"McFarlane","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"297","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/3571730","article-title":"Survey of hallucination in natural language generation","author":"Ji Z","year":"2022","unstructured":"Ji Z, Lee N, Frieske R, et al.. Survey of hallucination in natural language generation. ACM Comput Surv. 2022, 10.1145\/3571730","journal-title":"ACM Comput Surv"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1101\/2022.12.23.521610","article-title":"Abstracts written by ChatGPT fool scientists. [Preprint]","author":"Gao CA","year":"2022","unstructured":"Gao CA, Howard FM, Nikolay S. Abstracts written by ChatGPT fool scientists. [Preprint]. bioRxiv. 2022, 10.1101\/2022.12.23.521610","journal-title":"bioRxiv"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1101\/2022.12.19.22283643","article-title":"Performance of ChatGPT on USMLE: potential for AI-assisted medical education using large language models. [Preprint]","author":"Kung TH","year":"2022","unstructured":"Kung TH, Cheatham M, ChatGPT, et al.. Performance of ChatGPT on USMLE: potential for AI-assisted medical education using large language models. [Preprint]. medRxiv. 2022, 10.1101\/2022.12.19.22283643","journal-title":"medRxiv"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1038\/d41586-023-00107-z","article-title":"ChatGPT listed as author on research papers: many scientists disapprove","volume":"613","author":"Stokel-Walker C","year":"2023","unstructured":"Stokel-Walker C. ChatGPT listed as author on research papers: many scientists disapprove. Nature. 2023, 613:620-1. 10.1038\/d41586-023-00107-z","journal-title":"Nature"}],"container-title":["Cureus"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/www.cureus.com\/articles\/138667-artificial-hallucinations-in-chatgpt-implications-in-scientific-writing","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,2,9]],"date-time":"2024-02-09T14:53:25Z","timestamp":1707490405000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.cureus.com\/articles\/138667-artificial-hallucinations-in-chatgpt-implications-in-scientific-writing"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2023,2,19]]},"references-count":4,"URL":"https:\/\/doi.org\/10.7759\/cureus.35179","relation":{},"ISSN":["2168-8184"],"issn-type":[{"value":"2168-8184","type":"print"}],"subject":[],"published":{"date-parts":[[2023,2,19]]}}}
\ No newline at end of file

added research/verify_cache/https_export_arxiv_org_api_query_id_list_1902_03245 +30 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/5mRzsP+ekTTcxKGYsaC+dBFeCeA</id>
++  <title>arXiv Query: search_query=&amp;id_list=1902.03245&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:12Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=1902.03245" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/1902.03245v2</id>
++    <title>Ask Not What AI Can Do, But What AI Should Do: Towards a Framework of Task Delegability</title>
++    <updated>2019-11-01T18:00:00Z</updated>
++    <link href="https://arxiv.org/abs/1902.03245v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/1902.03245v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>While artificial intelligence (AI) holds promise for addressing societal challenges, issues of exactly which tasks to automate and to what extent to do so remain understudied. We approach this problem of task delegability from a human-centered perspective by developing a framework on human perception of task delegation to AI. We consider four high-level factors that can contribute to a delegation decision: motivation, difficulty, risk, and trust. To obtain an empirical understanding of human preferences in different tasks, we build a dataset of 100 tasks from academic papers, popular media portrayal of AI, and everyday life, and administer a survey based on our proposed framework. We find little preference for full AI control and a strong preference for machine-in-the-loop designs, in which humans play the leading role. Among the four factors, trust is the most correlated with human preferences of optimal human-machine delegation. This framework represents a first step towards characterizing human preferences of AI automation across tasks. We hope this work encourages future efforts towards understanding such individual attitudes; our goal is to inform the public and the AI research community rather than dictating any direction in technology development.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CY" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.HC" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2019-02-08T19:00:02Z</published>
++    <arxiv:comment>19 pages, 3 figures, 5 tables, NeurIPS 2019, dataset available at https://delegability.github.io</arxiv:comment>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Brian Lubars</name>
++    </author>
++    <author>
++      <name>Chenhao Tan</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2005_11401 +59 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api//+3qYJd8hQEWlLcweN7eJIdRKqo</id>
++  <title>arXiv Query: search_query=&amp;id_list=2005.11401&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:27Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2005.11401" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2005.11401v4</id>
++    <title>Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks</title>
++    <updated>2021-04-12T15:42:18Z</updated>
++    <link href="https://arxiv.org/abs/2005.11401v4" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2005.11401v4" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Large pre-trained language models have been shown to store factual knowledge in their parameters, and achieve state-of-the-art results when fine-tuned on downstream NLP tasks. However, their ability to access and precisely manipulate knowledge is still limited, and hence on knowledge-intensive tasks, their performance lags behind task-specific architectures. Additionally, providing provenance for their decisions and updating their world knowledge remain open research problems. Pre-trained models with a differentiable access mechanism to explicit non-parametric memory can overcome this issue, but have so far been only investigated for extractive downstream tasks. We explore a general-purpose fine-tuning recipe for retrieval-augmented generation (RAG) -- models which combine pre-trained parametric and non-parametric memory for language generation. We introduce RAG models where the parametric memory is a pre-trained seq2seq model and the non-parametric memory is a dense vector index of Wikipedia, accessed with a pre-trained neural retriever. We compare two RAG formulations, one which conditions on the same retrieved passages across the whole generated sequence, the other can use different passages per token. We fine-tune and evaluate our models on a wide range of knowledge-intensive NLP tasks and set the state-of-the-art on three open domain QA tasks, outperforming parametric seq2seq models and task-specific retrieve-and-extract architectures. For language generation tasks, we find that RAG models generate more specific, diverse and factual language than a state-of-the-art parametric-only seq2seq baseline.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2020-05-22T21:34:34Z</published>
++    <arxiv:comment>Accepted at NeurIPS 2020</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Patrick Lewis</name>
++    </author>
++    <author>
++      <name>Ethan Perez</name>
++    </author>
++    <author>
++      <name>Aleksandra Piktus</name>
++    </author>
++    <author>
++      <name>Fabio Petroni</name>
++    </author>
++    <author>
++      <name>Vladimir Karpukhin</name>
++    </author>
++    <author>
++      <name>Naman Goyal</name>
++    </author>
++    <author>
++      <name>Heinrich Küttler</name>
++    </author>
++    <author>
++      <name>Mike Lewis</name>
++    </author>
++    <author>
++      <name>Wen-tau Yih</name>
++    </author>
++    <author>
++      <name>Tim Rocktäschel</name>
++    </author>
++    <author>
++      <name>Sebastian Riedel</name>
++    </author>
++    <author>
++      <name>Douwe Kiela</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2112_09332 +78 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/yNl4GZofJkaZIlNyKAuKHA6LblY</id>
++  <title>arXiv Query: search_query=&amp;id_list=2112.09332&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:13:54Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2112.09332" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2112.09332v3</id>
++    <title>WebGPT: Browser-assisted question-answering with human feedback</title>
++    <updated>2022-06-01T19:08:11Z</updated>
++    <link href="https://arxiv.org/abs/2112.09332v3" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2112.09332v3" rel="related" type="application/pdf" title="pdf"/>
++    <summary>We fine-tune GPT-3 to answer long-form questions using a text-based web-browsing environment, which allows the model to search and navigate the web. By setting up the task so that it can be performed by humans, we are able to train models on the task using imitation learning, and then optimize answer quality with human feedback. To make human evaluation of factual accuracy easier, models must collect references while browsing in support of their answers. We train and evaluate our models on ELI5, a dataset of questions asked by Reddit users. Our best model is obtained by fine-tuning GPT-3 using behavior cloning, and then performing rejection sampling against a reward model trained to predict human preferences. This model's answers are preferred by humans 56% of the time to those of our human demonstrators, and 69% of the time to the highest-voted answer from Reddit.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2021-12-17T05:43:43Z</published>
++    <arxiv:comment>32 pages</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Reiichiro Nakano</name>
++    </author>
++    <author>
++      <name>Jacob Hilton</name>
++    </author>
++    <author>
++      <name>Suchir Balaji</name>
++    </author>
++    <author>
++      <name>Jeff Wu</name>
++    </author>
++    <author>
++      <name>Long Ouyang</name>
++    </author>
++    <author>
++      <name>Christina Kim</name>
++    </author>
++    <author>
++      <name>Christopher Hesse</name>
++    </author>
++    <author>
++      <name>Shantanu Jain</name>
++    </author>
++    <author>
++      <name>Vineet Kosaraju</name>
++    </author>
++    <author>
++      <name>William Saunders</name>
++    </author>
++    <author>
++      <name>Xu Jiang</name>
++    </author>
++    <author>
++      <name>Karl Cobbe</name>
++    </author>
++    <author>
++      <name>Tyna Eloundou</name>
++    </author>
++    <author>
++      <name>Gretchen Krueger</name>
++    </author>
++    <author>
++      <name>Kevin Button</name>
++    </author>
++    <author>
++      <name>Matthew Knight</name>
++    </author>
++    <author>
++      <name>Benjamin Chess</name>
++    </author>
++    <author>
++      <name>John Schulman</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2112_11471 +41 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/fbPg0n1H+NYy7LDANW0K6nZ2h6o</id>
++  <title>arXiv Query: search_query=&amp;id_list=2112.11471&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:11Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2112.11471" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2112.11471v1</id>
++    <title>Towards a Science of Human-AI Decision Making: A Survey of Empirical Studies</title>
++    <updated>2021-12-21T19:00:02Z</updated>
++    <link href="https://arxiv.org/abs/2112.11471v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2112.11471v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>As AI systems demonstrate increasingly strong predictive performance, their adoption has grown in numerous domains. However, in high-stakes domains such as criminal justice and healthcare, full automation is often not desirable due to safety, ethical, and legal concerns, yet fully manual approaches can be inaccurate and time consuming. As a result, there is growing interest in the research community to augment human decision making with AI assistance. Besides developing AI technologies for this purpose, the emerging field of human-AI decision making must embrace empirical approaches to form a foundational understanding of how humans interact and work with AI to make decisions. To invite and help structure research efforts towards a science of understanding and improving human-AI decision making, we survey recent literature of empirical human-subject studies on this topic. We summarize the study design choices made in over 100 papers in three important aspects: (1) decision tasks, (2) AI models and AI assistance elements, and (3) evaluation metrics. For each aspect, we summarize current trends, discuss gaps in current practices of the field, and make a list of recommendations for future research. Our survey highlights the need to develop common frameworks to account for the design and research spaces of human-AI decision making, so that researchers can make rigorous choices in study design, and the research community can build on each other's work and produce generalizable scientific knowledge. We also hope this survey will serve as a bridge for HCI and AI communities to work together to mutually shape the empirical science and computational technologies for human-AI decision making.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CY" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.HC" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2021-12-21T19:00:02Z</published>
++    <arxiv:comment>36 pages, 2 figures, see https://haidecisionmaking.github.io for website</arxiv:comment>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Vivian Lai</name>
++    </author>
++    <author>
++      <name>Chacha Chen</name>
++    </author>
++    <author>
++      <name>Q. Vera Liao</name>
++    </author>
++    <author>
++      <name>Alison Smith-Renner</name>
++    </author>
++    <author>
++      <name>Chenhao Tan</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2203_11147 +55 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/vIFS0YxQZelIRy/iIBbXvYUqnoE</id>
++  <title>arXiv Query: search_query=&amp;id_list=2203.11147&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:29Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2203.11147" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2203.11147v1</id>
++    <title>Teaching language models to support answers with verified quotes</title>
++    <updated>2022-03-21T17:26:29Z</updated>
++    <link href="https://arxiv.org/abs/2203.11147v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2203.11147v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Recent large language models often answer factual questions correctly. But users can't trust any given claim a model makes without fact-checking, because language models can hallucinate convincing nonsense. In this work we use reinforcement learning from human preferences (RLHP) to train "open-book" QA models that generate answers whilst also citing specific evidence for their claims, which aids in the appraisal of correctness. Supporting evidence is drawn from multiple documents found via a search engine, or from a single user-provided document. Our 280 billion parameter model, GopherCite, is able to produce answers with high quality supporting evidence and abstain from answering when unsure. We measure the performance of GopherCite by conducting human evaluation of answers to questions in a subset of the NaturalQuestions and ELI5 datasets. The model's response is found to be high-quality 80\% of the time on this Natural Questions subset, and 67\% of the time on the ELI5 subset. Abstaining from the third of questions for which it is most unsure improves performance to 90\% and 80\% respectively, approaching human baselines. However, analysis on the adversarial TruthfulQA dataset shows why citation is only one part of an overall strategy for safety and trustworthiness: not all claims supported by evidence are true.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2022-03-21T17:26:29Z</published>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Jacob Menick</name>
++    </author>
++    <author>
++      <name>Maja Trebacz</name>
++    </author>
++    <author>
++      <name>Vladimir Mikulik</name>
++    </author>
++    <author>
++      <name>John Aslanides</name>
++    </author>
++    <author>
++      <name>Francis Song</name>
++    </author>
++    <author>
++      <name>Martin Chadwick</name>
++    </author>
++    <author>
++      <name>Mia Glaese</name>
++    </author>
++    <author>
++      <name>Susannah Young</name>
++    </author>
++    <author>
++      <name>Lucy Campbell-Gillingham</name>
++    </author>
++    <author>
++      <name>Geoffrey Irving</name>
++    </author>
++    <author>
++      <name>Nat McAleese</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2203_11171 +47 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/2M5Bd793VRVOMdGCHz6J9nScYek</id>
++  <title>arXiv Query: search_query=&amp;id_list=2203.11171&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:25:32Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2203.11171" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2203.11171v4</id>
++    <title>Self-Consistency Improves Chain of Thought Reasoning in Language Models</title>
++    <updated>2023-03-07T17:57:37Z</updated>
++    <link href="https://arxiv.org/abs/2203.11171v4" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2203.11171v4" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Chain-of-thought prompting combined with pre-trained large language models has achieved encouraging results on complex reasoning tasks. In this paper, we propose a new decoding strategy, self-consistency, to replace the naive greedy decoding used in chain-of-thought prompting. It first samples a diverse set of reasoning paths instead of only taking the greedy one, and then selects the most consistent answer by marginalizing out the sampled reasoning paths. Self-consistency leverages the intuition that a complex reasoning problem typically admits multiple different ways of thinking leading to its unique correct answer. Our extensive empirical evaluation shows that self-consistency boosts the performance of chain-of-thought prompting with a striking margin on a range of popular arithmetic and commonsense reasoning benchmarks, including GSM8K (+17.9%), SVAMP (+11.0%), AQuA (+12.2%), StrategyQA (+6.4%) and ARC-challenge (+3.9%).</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2022-03-21T17:48:52Z</published>
++    <arxiv:comment>Published at ICLR 2023. V2: added PaLM results; V3: added UL2 results; V4: camera ready version at ICLR 2023</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Xuezhi Wang</name>
++    </author>
++    <author>
++      <name>Jason Wei</name>
++    </author>
++    <author>
++      <name>Dale Schuurmans</name>
++    </author>
++    <author>
++      <name>Quoc Le</name>
++    </author>
++    <author>
++      <name>Ed Chi</name>
++    </author>
++    <author>
++      <name>Sharan Narang</name>
++    </author>
++    <author>
++      <name>Aakanksha Chowdhery</name>
++    </author>
++    <author>
++      <name>Denny Zhou</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2210_13777 +44 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/STXNca5K29lhd3F9uElPHUn/TMY</id>
++  <title>arXiv Query: search_query=&amp;id_list=2210.13777&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:28Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2210.13777" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2210.13777v1</id>
++    <title>SciFact-Open: Towards open-domain scientific claim verification</title>
++    <updated>2022-10-25T05:45:00Z</updated>
++    <link href="https://arxiv.org/abs/2210.13777v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2210.13777v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>While research on scientific claim verification has led to the development of powerful systems that appear to approach human performance, these approaches have yet to be tested in a realistic setting against large corpora of scientific literature. Moving to this open-domain evaluation setting, however, poses unique challenges; in particular, it is infeasible to exhaustively annotate all evidence documents. In this work, we present SciFact-Open, a new test collection designed to evaluate the performance of scientific claim verification systems on a corpus of 500K research abstracts. Drawing upon pooling techniques from information retrieval, we collect evidence for scientific claims by pooling and annotating the top predictions of four state-of-the-art scientific claim verification models. We find that systems developed on smaller corpora struggle to generalize to SciFact-Open, exhibiting performance drops of at least 15 F1. In addition, analysis of the evidence in SciFact-Open reveals interesting phenomena likely to appear when claim verification systems are deployed in practice, e.g., cases where the evidence supports only a special case of the claim. Our dataset is available at https://github.com/dwadden/scifact-open.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2022-10-25T05:45:00Z</published>
++    <arxiv:comment>EMNLP Findings 2022. GitHub: https://github.com/dwadden/scifact-open-2022</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>David Wadden</name>
++    </author>
++    <author>
++      <name>Kyle Lo</name>
++    </author>
++    <author>
++      <name>Bailey Kuehl</name>
++    </author>
++    <author>
++      <name>Arman Cohan</name>
++    </author>
++    <author>
++      <name>Iz Beltagy</name>
++    </author>
++    <author>
++      <name>Lucy Lu Wang</name>
++    </author>
++    <author>
++      <name>Hannaneh Hajishirzi</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2211_09110 +175 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/OORlF5ra7IkDqXHje/Wh9tIerTA</id>
++  <title>arXiv Query: search_query=&amp;id_list=2211.09110&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:18:13Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2211.09110" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2211.09110v2</id>
++    <title>Holistic Evaluation of Language Models</title>
++    <updated>2023-10-01T21:44:23Z</updated>
++    <link href="https://arxiv.org/abs/2211.09110v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2211.09110v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Language models (LMs) are becoming the foundation for almost all major language technologies, but their capabilities, limitations, and risks are not well understood. We present Holistic Evaluation of Language Models (HELM) to improve the transparency of language models. First, we taxonomize the vast space of potential scenarios (i.e. use cases) and metrics (i.e. desiderata) that are of interest for LMs. Then we select a broad subset based on coverage and feasibility, noting what's missing or underrepresented (e.g. question answering for neglected English dialects, metrics for trustworthiness). Second, we adopt a multi-metric approach: We measure 7 metrics (accuracy, calibration, robustness, fairness, bias, toxicity, and efficiency) for each of 16 core scenarios when possible (87.5% of the time). This ensures metrics beyond accuracy don't fall to the wayside, and that trade-offs are clearly exposed. We also perform 7 targeted evaluations, based on 26 targeted scenarios, to analyze specific aspects (e.g. reasoning, disinformation). Third, we conduct a large-scale evaluation of 30 prominent language models (spanning open, limited-access, and closed models) on all 42 scenarios, 21 of which were not previously used in mainstream LM evaluation. Prior to HELM, models on average were evaluated on just 17.9% of the core HELM scenarios, with some prominent models not sharing a single scenario in common. We improve this to 96.0%: now all 30 models have been densely benchmarked on the same core scenarios and metrics under standardized conditions. Our evaluation surfaces 25 top-level findings. For full transparency, we release all raw model prompts and completions publicly for further analysis, as well as a general modular toolkit. We intend for HELM to be a living benchmark for the community, continuously updated with new scenarios, metrics, and models.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2022-11-16T18:51:34Z</published>
++    <arxiv:comment>Authored by the Center for Research on Foundation Models (CRFM) at the Stanford Institute for Human-Centered Artificial Intelligence (HAI). Project page: https://crfm.stanford.edu/helm/v1.0</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <arxiv:journal_ref>Published in Transactions on Machine Learning Research (TMLR), 2023</arxiv:journal_ref>
++    <author>
++      <name>Percy Liang</name>
++    </author>
++    <author>
++      <name>Rishi Bommasani</name>
++    </author>
++    <author>
++      <name>Tony Lee</name>
++    </author>
++    <author>
++      <name>Dimitris Tsipras</name>
++    </author>
++    <author>
++      <name>Dilara Soylu</name>
++    </author>
++    <author>
++      <name>Michihiro Yasunaga</name>
++    </author>
++    <author>
++      <name>Yian Zhang</name>
++    </author>
++    <author>
++      <name>Deepak Narayanan</name>
++    </author>
++    <author>
++      <name>Yuhuai Wu</name>
++    </author>
++    <author>
++      <name>Ananya Kumar</name>
++    </author>
++    <author>
++      <name>Benjamin Newman</name>
++    </author>
++    <author>
++      <name>Binhang Yuan</name>
++    </author>
++    <author>
++      <name>Bobby Yan</name>
++    </author>
++    <author>
++      <name>Ce Zhang</name>
++    </author>
++    <author>
++      <name>Christian Cosgrove</name>
++    </author>
++    <author>
++      <name>Christopher D. Manning</name>
++    </author>
++    <author>
++      <name>Christopher Ré</name>
++    </author>
++    <author>
++      <name>Diana Acosta-Navas</name>
++    </author>
++    <author>
++      <name>Drew A. Hudson</name>
++    </author>
++    <author>
++      <name>Eric Zelikman</name>
++    </author>
++    <author>
++      <name>Esin Durmus</name>
++    </author>
++    <author>
++      <name>Faisal Ladhak</name>
++    </author>
++    <author>
++      <name>Frieda Rong</name>
++    </author>
++    <author>
++      <name>Hongyu Ren</name>
++    </author>
++    <author>
++      <name>Huaxiu Yao</name>
++    </author>
++    <author>
++      <name>Jue Wang</name>
++    </author>
++    <author>
++      <name>Keshav Santhanam</name>
++    </author>
++    <author>
++      <name>Laurel Orr</name>
++    </author>
++    <author>
++      <name>Lucia Zheng</name>
++    </author>
++    <author>
++      <name>Mert Yuksekgonul</name>
++    </author>
++    <author>
++      <name>Mirac Suzgun</name>
++    </author>
++    <author>
++      <name>Nathan Kim</name>
++    </author>
++    <author>
++      <name>Neel Guha</name>
++    </author>
++    <author>
++      <name>Niladri Chatterji</name>
++    </author>
++    <author>
++      <name>Omar Khattab</name>
++    </author>
++    <author>
++      <name>Peter Henderson</name>
++    </author>
++    <author>
++      <name>Qian Huang</name>
++    </author>
++    <author>
++      <name>Ryan Chi</name>
++    </author>
++    <author>
++      <name>Sang Michael Xie</name>
++    </author>
++    <author>
++      <name>Shibani Santurkar</name>
++    </author>
++    <author>
++      <name>Surya Ganguli</name>
++    </author>
++    <author>
++      <name>Tatsunori Hashimoto</name>
++    </author>
++    <author>
++      <name>Thomas Icard</name>
++    </author>
++    <author>
++      <name>Tianyi Zhang</name>
++    </author>
++    <author>
++      <name>Vishrav Chaudhary</name>
++    </author>
++    <author>
++      <name>William Wang</name>
++    </author>
++    <author>
++      <name>Xuechen Li</name>
++    </author>
++    <author>
++      <name>Yifan Mai</name>
++    </author>
++    <author>
++      <name>Yuhui Zhang</name>
++    </author>
++    <author>
++      <name>Yuta Koreeda</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2212_08037 +87 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/WwhylmPvDB6KOrH+1v50TgZ0CZU</id>
++  <title>arXiv Query: search_query=&amp;id_list=2212.08037&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:23Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2212.08037" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2212.08037v2</id>
++    <title>Attributed Question Answering: Evaluation and Modeling for Attributed Large Language Models</title>
++    <updated>2023-02-10T20:04:05Z</updated>
++    <link href="https://arxiv.org/abs/2212.08037v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2212.08037v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Large language models (LLMs) have shown impressive results while requiring little or no direct supervision. Further, there is mounting evidence that LLMs may have potential in information-seeking scenarios. We believe the ability of an LLM to attribute the text that it generates is likely to be crucial in this setting. We formulate and study Attributed QA as a key first step in the development of attributed LLMs. We propose a reproducible evaluation framework for the task and benchmark a broad set of architectures. We take human annotations as a gold standard and show that a correlated automatic metric is suitable for development. Our experimental work gives concrete answers to two key questions (How to measure attribution?, and How well do current state-of-the-art methods perform on attribution?), and give some hints as to how to address a third (How to build LLMs with attribution?).</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2022-12-15T18:45:29Z</published>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Bernd Bohnet</name>
++    </author>
++    <author>
++      <name>Vinh Q. Tran</name>
++    </author>
++    <author>
++      <name>Pat Verga</name>
++    </author>
++    <author>
++      <name>Roee Aharoni</name>
++    </author>
++    <author>
++      <name>Daniel Andor</name>
++    </author>
++    <author>
++      <name>Livio Baldini Soares</name>
++    </author>
++    <author>
++      <name>Massimiliano Ciaramita</name>
++    </author>
++    <author>
++      <name>Jacob Eisenstein</name>
++    </author>
++    <author>
++      <name>Kuzman Ganchev</name>
++    </author>
++    <author>
++      <name>Jonathan Herzig</name>
++    </author>
++    <author>
++      <name>Kai Hui</name>
++    </author>
++    <author>
++      <name>Tom Kwiatkowski</name>
++    </author>
++    <author>
++      <name>Ji Ma</name>
++    </author>
++    <author>
++      <name>Jianmo Ni</name>
++    </author>
++    <author>
++      <name>Lierni Sestorain Saralegui</name>
++    </author>
++    <author>
++      <name>Tal Schuster</name>
++    </author>
++    <author>
++      <name>William W. Cohen</name>
++    </author>
++    <author>
++      <name>Michael Collins</name>
++    </author>
++    <author>
++      <name>Dipanjan Das</name>
++    </author>
++    <author>
++      <name>Donald Metzler</name>
++    </author>
++    <author>
++      <name>Slav Petrov</name>
++    </author>
++    <author>
++      <name>Kellie Webster</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2302_06590 +33 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api//J1tKW7Z89iMJMZ5BIznwSKKO9w</id>
++  <title>arXiv Query: search_query=&amp;id_list=2302.06590&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:00:31Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2302.06590" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2302.06590v1</id>
++    <title>The Impact of AI on Developer Productivity: Evidence from GitHub Copilot</title>
++    <updated>2023-02-13T18:42:46Z</updated>
++    <link href="https://arxiv.org/abs/2302.06590v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2302.06590v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Generative AI tools hold promise to increase human productivity. This paper presents results from a controlled experiment with GitHub Copilot, an AI pair programmer. Recruited software developers were asked to implement an HTTP server in JavaScript as quickly as possible. The treatment group, with access to the AI pair programmer, completed the task 55.8% faster than the control group. Observed heterogenous effects show promise for AI pair programmers to help people transition into software development careers.</summary>
++    <category term="cs.SE" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-02-13T18:42:46Z</published>
++    <arxiv:primary_category term="cs.SE"/>
++    <author>
++      <name>Sida Peng</name>
++    </author>
++    <author>
++      <name>Eirini Kalliamvakou</name>
++    </author>
++    <author>
++      <name>Peter Cihon</name>
++    </author>
++    <author>
++      <name>Mert Demirer</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2305_05176 +33 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/5pIkT1ykY2SRWWTIV72WCrLepp0</id>
++  <title>arXiv Query: search_query=&amp;id_list=2305.05176&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:45:01Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2305.05176" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2305.05176v1</id>
++    <title>FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance</title>
++    <updated>2023-05-09T05:11:02Z</updated>
++    <link href="https://arxiv.org/abs/2305.05176v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2305.05176v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>There is a rapidly growing number of large language models (LLMs) that users can query for a fee. We review the cost associated with querying popular LLM APIs, e.g. GPT-4, ChatGPT, J1-Jumbo, and find that these models have heterogeneous pricing structures, with fees that can differ by two orders of magnitude. In particular, using LLMs on large collections of queries and text can be expensive. Motivated by this, we outline and discuss three types of strategies that users can exploit to reduce the inference cost associated with using LLMs: 1) prompt adaptation, 2) LLM approximation, and 3) LLM cascade. As an example, we propose FrugalGPT, a simple yet flexible instantiation of LLM cascade which learns which combinations of LLMs to use for different queries in order to reduce cost and improve accuracy. Our experiments show that FrugalGPT can match the performance of the best individual LLM (e.g. GPT-4) with up to 98% cost reduction or improve the accuracy over GPT-4 by 4% with the same cost. The ideas and findings presented here lay a foundation for using LLMs sustainably and efficiently.</summary>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.SE" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-05-09T05:11:02Z</published>
++    <arxiv:primary_category term="cs.LG"/>
++    <author>
++      <name>Lingjiao Chen</name>
++    </author>
++    <author>
++      <name>Matei Zaharia</name>
++    </author>
++    <author>
++      <name>James Zou</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2305_14251 +51 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/0KFfVQwmuni2e2iifsqnYk17BO4</id>
++  <title>arXiv Query: search_query=&amp;id_list=2305.14251&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:19:52Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2305.14251" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2305.14251v2</id>
++    <title>FActScore: Fine-grained Atomic Evaluation of Factual Precision in Long Form Text Generation</title>
++    <updated>2023-10-11T05:27:50Z</updated>
++    <link href="https://arxiv.org/abs/2305.14251v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2305.14251v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Evaluating the factuality of long-form text generated by large language models (LMs) is non-trivial because (1) generations often contain a mixture of supported and unsupported pieces of information, making binary judgments of quality inadequate, and (2) human evaluation is time-consuming and costly. In this paper, we introduce FACTSCORE, a new evaluation that breaks a generation into a series of atomic facts and computes the percentage of atomic facts supported by a reliable knowledge source. We conduct an extensive human evaluation to obtain FACTSCOREs of people biographies generated by several state-of-the-art commercial LMs -- InstructGPT, ChatGPT, and the retrieval-augmented PerplexityAI -- and report new analysis demonstrating the need for such a fine-grained score (e.g., ChatGPT only achieves 58%). Since human evaluation is costly, we also introduce an automated model that estimates FACTSCORE using retrieval and a strong language model, with less than a 2% error rate. Finally, we use this automated metric to evaluate 6,500 generations from a new set of 13 recent LMs that would have cost $26K if evaluated by humans, with various findings: GPT-4 and ChatGPT are more factual than public models, and Vicuna and Alpaca are some of the best public models. FACTSCORE is available for public use via `pip install factscore`.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-05-23T17:06:00Z</published>
++    <arxiv:comment>25 pages; 7 figures. Published as a main conference paper at EMNLP 2023. Code available at https://github.com/shmsw25/FActScore</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Sewon Min</name>
++    </author>
++    <author>
++      <name>Kalpesh Krishna</name>
++    </author>
++    <author>
++      <name>Xinxi Lyu</name>
++    </author>
++    <author>
++      <name>Mike Lewis</name>
++    </author>
++    <author>
++      <name>Wen-tau Yih</name>
++    </author>
++    <author>
++      <name>Pang Wei Koh</name>
++    </author>
++    <author>
++      <name>Mohit Iyyer</name>
++    </author>
++    <author>
++      <name>Luke Zettlemoyer</name>
++    </author>
++    <author>
++      <name>Hannaneh Hajishirzi</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2305_14627 +36 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/uE3DZRESvC0TVIJoy9cUVPWlFNw</id>
++  <title>arXiv Query: search_query=&amp;id_list=2305.14627&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:30:10Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2305.14627" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2305.14627v2</id>
++    <title>Enabling Large Language Models to Generate Text with Citations</title>
++    <updated>2023-10-31T15:04:35Z</updated>
++    <link href="https://arxiv.org/abs/2305.14627v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2305.14627v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Large language models (LLMs) have emerged as a widely-used tool for information seeking, but their generated outputs are prone to hallucination. In this work, our aim is to allow LLMs to generate text with citations, improving their factual correctness and verifiability. Existing work mainly relies on commercial search engines and human evaluation, making it challenging to reproduce and compare different modeling approaches. We propose ALCE, the first benchmark for Automatic LLMs' Citation Evaluation. ALCE collects a diverse set of questions and retrieval corpora and requires building end-to-end systems to retrieve supporting evidence and generate answers with citations. We develop automatic metrics along three dimensions -- fluency, correctness, and citation quality -- and demonstrate their strong correlation with human judgements. Our experiments with state-of-the-art LLMs and novel prompting strategies show that current systems have considerable room for improvement -- For example, on the ELI5 dataset, even the best models lack complete citation support 50% of the time. Our analyses further highlight promising future directions, including developing better retrievers, advancing long-context LLMs, and improving the ability to synthesize information from multiple sources.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.IR" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-05-24T01:53:49Z</published>
++    <arxiv:comment>Accepted by EMNLP 2023. Code and data are available at https://github.com/princeton-nlp/ALCE</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Tianyu Gao</name>
++    </author>
++    <author>
++      <name>Howard Yen</name>
++    </author>
++    <author>
++      <name>Jiatong Yu</name>
++    </author>
++    <author>
++      <name>Danqi Chen</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2305_17926 +53 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/SNAjrYlxWdRDG8lEFs1AWhp+Z2c</id>
++  <title>arXiv Query: search_query=&amp;id_list=2305.17926&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:17:11Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2305.17926" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2305.17926v2</id>
++    <title>Large Language Models are not Fair Evaluators</title>
++    <updated>2023-08-30T13:22:35Z</updated>
++    <link href="https://arxiv.org/abs/2305.17926v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2305.17926v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>In this paper, we uncover a systematic bias in the evaluation paradigm of adopting large language models~(LLMs), e.g., GPT-4, as a referee to score and compare the quality of responses generated by candidate models. We find that the quality ranking of candidate responses can be easily hacked by simply altering their order of appearance in the context. This manipulation allows us to skew the evaluation result, making one model appear considerably superior to the other, e.g., Vicuna-13B could beat ChatGPT on 66 over 80 tested queries with ChatGPT as an evaluator. To address this issue, we propose a calibration framework with three simple yet effective strategies: 1) Multiple Evidence Calibration, which requires the evaluator model to generate multiple evaluation evidence before assigning ratings; 2) Balanced Position Calibration, which aggregates results across various orders to determine the final score; 3) Human-in-the-Loop Calibration, which introduces a balanced position diversity entropy to measure the difficulty of each example and seeks human assistance when needed. We also manually annotate the "win/tie/lose" outcomes of responses from ChatGPT and Vicuna-13B in the Vicuna Benchmark's question prompt, and extensive experiments demonstrate that our approach successfully mitigates evaluation bias, resulting in closer alignment with human judgments. We release our code and human annotation at \url{https://github.com/i-Eval/FairEval} to facilitate future research.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.IR" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-05-29T07:41:03Z</published>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Peiyi Wang</name>
++    </author>
++    <author>
++      <name>Lei Li</name>
++    </author>
++    <author>
++      <name>Liang Chen</name>
++    </author>
++    <author>
++      <name>Zefan Cai</name>
++    </author>
++    <author>
++      <name>Dawei Zhu</name>
++    </author>
++    <author>
++      <name>Binghuai Lin</name>
++    </author>
++    <author>
++      <name>Yunbo Cao</name>
++    </author>
++    <author>
++      <name>Qi Liu</name>
++    </author>
++    <author>
++      <name>Tianyu Liu</name>
++    </author>
++    <author>
++      <name>Zhifang Sui</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2305_18248 +34 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api//xyvlMW8GfLvD5jPHaNEW/VytE8</id>
++  <title>arXiv Query: search_query=&amp;id_list=2305.18248&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:28Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2305.18248" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2305.18248v3</id>
++    <title>Do Language Models Know When They're Hallucinating References?</title>
++    <updated>2024-03-20T13:12:48Z</updated>
++    <link href="https://arxiv.org/abs/2305.18248v3" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2305.18248v3" rel="related" type="application/pdf" title="pdf"/>
++    <summary>State-of-the-art language models (LMs) are notoriously susceptible to generating hallucinated information. Such inaccurate outputs not only undermine the reliability of these models but also limit their use and raise serious concerns about misinformation and propaganda. In this work, we focus on hallucinated book and article references and present them as the "model organism" of language model hallucination research, due to their frequent and easy-to-discern nature. We posit that if a language model cites a particular reference in its output, then it should ideally possess sufficient information about its authors and content, among other relevant details. Using this basic insight, we illustrate that one can identify hallucinated references without ever consulting any external resources, by asking a set of direct or indirect queries to the language model about the references. These queries can be considered as "consistency checks." Our findings highlight that while LMs, including GPT-4, often produce inconsistent author lists for hallucinated references, they also often accurately recall the authors of real references. In this sense, the LM can be said to "know" when it is hallucinating references. Furthermore, these findings show how hallucinated references can be dissected to shed light on their nature. Replication code and results can be found at https://github.com/microsoft/hallucinated-references.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-05-29T17:12:03Z</published>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Ayush Agrawal</name>
++    </author>
++    <author>
++      <name>Mirac Suzgun</name>
++    </author>
++    <author>
++      <name>Lester Mackey</name>
++    </author>
++    <author>
++      <name>Adam Tauman Kalai</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2306_05685 +62 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/AOHgtEfM6+1ZxWnongFf7KYi5O8</id>
++  <title>arXiv Query: search_query=&amp;id_list=2306.05685&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:28:39Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2306.05685" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2306.05685v4</id>
++    <title>Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena</title>
++    <updated>2023-12-24T02:01:34Z</updated>
++    <link href="https://arxiv.org/abs/2306.05685v4" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2306.05685v4" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Evaluating large language model (LLM) based chat assistants is challenging due to their broad capabilities and the inadequacy of existing benchmarks in measuring human preferences. To address this, we explore using strong LLMs as judges to evaluate these models on more open-ended questions. We examine the usage and limitations of LLM-as-a-judge, including position, verbosity, and self-enhancement biases, as well as limited reasoning ability, and propose solutions to mitigate some of them. We then verify the agreement between LLM judges and human preferences by introducing two benchmarks: MT-bench, a multi-turn question set; and Chatbot Arena, a crowdsourced battle platform. Our results reveal that strong LLM judges like GPT-4 can match both controlled and crowdsourced human preferences well, achieving over 80% agreement, the same level of agreement between humans. Hence, LLM-as-a-judge is a scalable and explainable way to approximate human preferences, which are otherwise very expensive to obtain. Additionally, we show our benchmark and traditional benchmarks complement each other by evaluating several variants of LLaMA and Vicuna. The MT-bench questions, 3K expert votes, and 30K conversations with human preferences are publicly available at https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-06-09T05:55:52Z</published>
++    <arxiv:comment>NeurIPS 2023 Datasets and Benchmarks Track</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Lianmin Zheng</name>
++    </author>
++    <author>
++      <name>Wei-Lin Chiang</name>
++    </author>
++    <author>
++      <name>Ying Sheng</name>
++    </author>
++    <author>
++      <name>Siyuan Zhuang</name>
++    </author>
++    <author>
++      <name>Zhanghao Wu</name>
++    </author>
++    <author>
++      <name>Yonghao Zhuang</name>
++    </author>
++    <author>
++      <name>Zi Lin</name>
++    </author>
++    <author>
++      <name>Zhuohan Li</name>
++    </author>
++    <author>
++      <name>Dacheng Li</name>
++    </author>
++    <author>
++      <name>Eric P. Xing</name>
++    </author>
++    <author>
++      <name>Hao Zhang</name>
++    </author>
++    <author>
++      <name>Joseph E. Gonzalez</name>
++    </author>
++    <author>
++      <name>Ion Stoica</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2308_03688 +90 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/5fpoDZ6v2RrbQPHjJkk5EZhoOhI</id>
++  <title>arXiv Query: search_query=&amp;id_list=2308.03688&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:29:49Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2308.03688" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2308.03688v3</id>
++    <title>AgentBench: Evaluating LLMs as Agents</title>
++    <updated>2025-10-04T03:54:18Z</updated>
++    <link href="https://arxiv.org/abs/2308.03688v3" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2308.03688v3" rel="related" type="application/pdf" title="pdf"/>
++    <summary>The potential of Large Language Model (LLM) as agents has been widely acknowledged recently. Thus, there is an urgent need to quantitatively \textit{evaluate LLMs as agents} on challenging tasks in interactive environments. We present AgentBench, a multi-dimensional benchmark that consists of 8 distinct environments to assess LLM-as-Agent's reasoning and decision-making abilities. Our extensive test over \num API-based and open-sourced (OSS) LLMs shows that, while top commercial LLMs present a strong ability of acting as agents in complex environments, there is a significant disparity in performance between them and many OSS competitors that are no larger than 70B. We identify the typical reasons of failures in environments and LLMs, showing that poor long-term reasoning, decision-making, and instruction following abilities are the main obstacles for developing usable LLM agents. Improving instruction following and training on high quality multi-round alignment data could improve agent performance. And different from existing assumptions, training on code present ambivalent impacts on different agent tasks. Datasets, environments, and an integrated evaluation package for AgentBench are released at https://github.com/THUDM/AgentBench.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-08-07T16:08:11Z</published>
++    <arxiv:comment>Published in ICLR 2024</arxiv:comment>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Xiao Liu</name>
++    </author>
++    <author>
++      <name>Hao Yu</name>
++    </author>
++    <author>
++      <name>Hanchen Zhang</name>
++    </author>
++    <author>
++      <name>Yifan Xu</name>
++    </author>
++    <author>
++      <name>Xuanyu Lei</name>
++    </author>
++    <author>
++      <name>Hanyu Lai</name>
++    </author>
++    <author>
++      <name>Yu Gu</name>
++    </author>
++    <author>
++      <name>Hangliang Ding</name>
++    </author>
++    <author>
++      <name>Kaiwen Men</name>
++    </author>
++    <author>
++      <name>Kejuan Yang</name>
++    </author>
++    <author>
++      <name>Shudan Zhang</name>
++    </author>
++    <author>
++      <name>Xiang Deng</name>
++    </author>
++    <author>
++      <name>Aohan Zeng</name>
++    </author>
++    <author>
++      <name>Zhengxiao Du</name>
++    </author>
++    <author>
++      <name>Chenhui Zhang</name>
++    </author>
++    <author>
++      <name>Sheng Shen</name>
++    </author>
++    <author>
++      <name>Tianjun Zhang</name>
++    </author>
++    <author>
++      <name>Yu Su</name>
++    </author>
++    <author>
++      <name>Huan Sun</name>
++    </author>
++    <author>
++      <name>Minlie Huang</name>
++    </author>
++    <author>
++      <name>Yuxiao Dong</name>
++    </author>
++    <author>
++      <name>Jie Tang</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2308_08493 +31 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/YUekpjqm/hyiXCkKwnwl3Qjoz2s</id>
++  <title>arXiv Query: search_query=&amp;id_list=2308.08493&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:05Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2308.08493" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2308.08493v3</id>
++    <title>Time Travel in LLMs: Tracing Data Contamination in Large Language Models</title>
++    <updated>2024-02-21T22:02:26Z</updated>
++    <link href="https://arxiv.org/abs/2308.08493v3" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2308.08493v3" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Data contamination, i.e., the presence of test data from downstream tasks in the training data of large language models (LLMs), is a potential major issue in measuring LLMs' real effectiveness on other tasks. We propose a straightforward yet effective method for identifying data contamination within LLMs. At its core, our approach starts by identifying potential contamination at the instance level; using this information, our approach then assesses wider contamination at the partition level. To estimate contamination of individual instances, we employ "guided instruction:" a prompt consisting of the dataset name, partition type, and the random-length initial segment of a reference instance, asking the LLM to complete it. An instance is flagged as contaminated if the LLM's output either exactly or nearly matches the latter segment of the reference. To understand if an entire partition is contaminated, we propose two ideas. The first idea marks a dataset partition as contaminated if the average overlap score with the reference instances (as measured by ROUGE-L or BLEURT) is statistically significantly better with the completions from guided instruction compared to a "general instruction" that does not include the dataset and partition name. The second idea marks a dataset partition as contaminated if a classifier based on GPT-4 with few-shot in-context learning prompt marks multiple generated completions as exact/near-exact matches of the corresponding reference instances. Our best method achieves an accuracy between 92% and 100% in detecting if an LLM is contaminated with seven datasets, containing train and test/validation partitions, when contrasted with manual evaluation by human experts. Further, our findings indicate that GPT-4 is contaminated with AG News, WNLI, and XSum datasets.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CR" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-08-16T16:48:57Z</published>
++    <arxiv:comment>Published at ICLR 2024 as a Spotlight paper (notable top 5%)</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Shahriar Golchin</name>
++    </author>
++    <author>
++      <name>Mihai Surdeanu</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2309_11495 +43 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/Udv62MrlxGJEv8k2h3XM1BDomYM</id>
++  <title>arXiv Query: search_query=&amp;id_list=2309.11495&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:46:27Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2309.11495" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2309.11495v2</id>
++    <title>Chain-of-Verification Reduces Hallucination in Large Language Models</title>
++    <updated>2023-09-25T15:25:49Z</updated>
++    <link href="https://arxiv.org/abs/2309.11495v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2309.11495v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Generation of plausible yet incorrect factual information, termed hallucination, is an unsolved issue in large language models. We study the ability of language models to deliberate on the responses they give in order to correct their mistakes. We develop the Chain-of-Verification (CoVe) method whereby the model first (i) drafts an initial response; then (ii) plans verification questions to fact-check its draft; (iii) answers those questions independently so the answers are not biased by other responses; and (iv) generates its final verified response. In experiments, we show CoVe decreases hallucinations across a variety of tasks, from list-based questions from Wikidata, closed book MultiSpanQA and longform text generation.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-09-20T17:50:55Z</published>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Shehzaad Dhuliawala</name>
++    </author>
++    <author>
++      <name>Mojtaba Komeili</name>
++    </author>
++    <author>
++      <name>Jing Xu</name>
++    </author>
++    <author>
++      <name>Roberta Raileanu</name>
++    </author>
++    <author>
++      <name>Xian Li</name>
++    </author>
++    <author>
++      <name>Asli Celikyilmaz</name>
++    </author>
++    <author>
++      <name>Jason Weston</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2310_01798 +44 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/k9DWYZRYbfzoy4itGW1O/NQQNDo</id>
++  <title>arXiv Query: search_query=&amp;id_list=2310.01798&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:16:02Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2310.01798" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2310.01798v2</id>
++    <title>Large Language Models Cannot Self-Correct Reasoning Yet</title>
++    <updated>2024-03-14T04:27:52Z</updated>
++    <link href="https://arxiv.org/abs/2310.01798v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2310.01798v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Large Language Models (LLMs) have emerged as a groundbreaking technology with their unparalleled text generation capabilities across various applications. Nevertheless, concerns persist regarding the accuracy and appropriateness of their generated content. A contemporary methodology, self-correction, has been proposed as a remedy to these issues. Building upon this premise, this paper critically examines the role and efficacy of self-correction within LLMs, shedding light on its true potential and limitations. Central to our investigation is the notion of intrinsic self-correction, whereby an LLM attempts to correct its initial responses based solely on its inherent capabilities, without the crutch of external feedback. In the context of reasoning, our research indicates that LLMs struggle to self-correct their responses without external feedback, and at times, their performance even degrades after self-correction. Drawing from these insights, we offer suggestions for future research and practical applications in this field.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-10-03T04:56:12Z</published>
++    <arxiv:comment>ICLR 2024</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Jie Huang</name>
++    </author>
++    <author>
++      <name>Xinyun Chen</name>
++    </author>
++    <author>
++      <name>Swaroop Mishra</name>
++    </author>
++    <author>
++      <name>Huaixiu Steven Zheng</name>
++    </author>
++    <author>
++      <name>Adams Wei Yu</name>
++    </author>
++    <author>
++      <name>Xinying Song</name>
++    </author>
++    <author>
++      <name>Denny Zhou</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2310_18018 +40 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/shQUBgx0x2p9WxTBZpA0Ibq7jNQ</id>
++  <title>arXiv Query: search_query=&amp;id_list=2310.18018&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:19:51Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2310.18018" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2310.18018v1</id>
++    <title>NLP Evaluation in trouble: On the Need to Measure LLM Data Contamination for each Benchmark</title>
++    <updated>2023-10-27T09:48:29Z</updated>
++    <link href="https://arxiv.org/abs/2310.18018v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2310.18018v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>In this position paper, we argue that the classical evaluation on Natural Language Processing (NLP) tasks using annotated benchmarks is in trouble. The worst kind of data contamination happens when a Large Language Model (LLM) is trained on the test split of a benchmark, and then evaluated in the same benchmark. The extent of the problem is unknown, as it is not straightforward to measure. Contamination causes an overestimation of the performance of a contaminated model in a target benchmark and associated task with respect to their non-contaminated counterparts. The consequences can be very harmful, with wrong scientific conclusions being published while other correct ones are discarded. This position paper defines different levels of data contamination and argues for a community effort, including the development of automatic and semi-automatic measures to detect when data from a benchmark was exposed to a model, and suggestions for flagging papers with conclusions that are compromised by data contamination.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-10-27T09:48:29Z</published>
++    <arxiv:comment>Accepted at EMNLP2024-Findings</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Oscar Sainz</name>
++    </author>
++    <author>
++      <name>Jon Ander Campos</name>
++    </author>
++    <author>
++      <name>Iker García-Ferrero</name>
++    </author>
++    <author>
++      <name>Julen Etxaniz</name>
++    </author>
++    <author>
++      <name>Oier Lopez de Lacalle</name>
++    </author>
++    <author>
++      <name>Eneko Agirre</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2311_01964 +50 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/TzcVGp4qxe9xhveeCpmzbMOTmgU</id>
++  <title>arXiv Query: search_query=&amp;id_list=2311.01964&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:08Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2311.01964" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2311.01964v1</id>
++    <title>Don't Make Your LLM an Evaluation Benchmark Cheater</title>
++    <updated>2023-11-03T14:59:54Z</updated>
++    <link href="https://arxiv.org/abs/2311.01964v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2311.01964v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Large language models~(LLMs) have greatly advanced the frontiers of artificial intelligence, attaining remarkable improvement in model capacity. To assess the model performance, a typical approach is to construct evaluation benchmarks for measuring the ability level of LLMs in different aspects. Despite that a number of high-quality benchmarks have been released, the concerns about the appropriate use of these benchmarks and the fair comparison of different models are increasingly growing. Considering these concerns, in this paper, we discuss the potential risk and impact of inappropriately using evaluation benchmarks and misleadingly interpreting the evaluation results. Specially, we focus on a special issue that would lead to inappropriate evaluation, \ie \emph{benchmark leakage}, referring that the data related to evaluation sets is occasionally used for model training. This phenomenon now becomes more common since pre-training data is often prepared ahead of model test. We conduct extensive experiments to study the effect of benchmark leverage, and find that it can dramatically boost the evaluation results, which would finally lead to an unreliable assessment of model performance. To improve the use of existing evaluation benchmarks, we finally present several guidelines for both LLM developers and benchmark maintainers. We hope this work can draw attention to appropriate training and evaluation of LLMs.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-11-03T14:59:54Z</published>
++    <arxiv:comment>11 pages</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Kun Zhou</name>
++    </author>
++    <author>
++      <name>Yutao Zhu</name>
++    </author>
++    <author>
++      <name>Zhipeng Chen</name>
++    </author>
++    <author>
++      <name>Wentong Chen</name>
++    </author>
++    <author>
++      <name>Wayne Xin Zhao</name>
++    </author>
++    <author>
++      <name>Xu Chen</name>
++    </author>
++    <author>
++      <name>Yankai Lin</name>
++    </author>
++    <author>
++      <name>Ji-Rong Wen</name>
++    </author>
++    <author>
++      <name>Jiawei Han</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2311_09783 +38 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/wNS9hp5S4ek3jCR9V8cnmpNr+pw</id>
++  <title>arXiv Query: search_query=&amp;id_list=2311.09783&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:04Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2311.09783" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2311.09783v2</id>
++    <title>Investigating Data Contamination in Modern Benchmarks for Large Language Models</title>
++    <updated>2024-04-03T23:29:03Z</updated>
++    <link href="https://arxiv.org/abs/2311.09783v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2311.09783v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Recent observations have underscored a disparity between the inflated benchmark scores and the actual performance of LLMs, raising concerns about potential contamination of evaluation benchmarks. This issue is especially critical for closed-source models and certain open-source models where training data transparency is lacking. In this paper we study data contamination by proposing two methods tailored for both open-source and proprietary LLMs. We first introduce a retrieval-based system to explore potential overlaps between evaluation benchmarks and pretraining corpora. We further present a novel investigation protocol named \textbf{T}estset \textbf{S}lot Guessing (\textit{TS-Guessing}), applicable to both open and proprietary models. This approach entails masking a wrong answer in a multiple-choice question and prompting the model to fill in the gap. Additionally, it involves obscuring an unlikely word in an evaluation example and asking the model to produce it. We find that certain commercial LLMs could surprisingly guess the missing option in various test sets. Specifically, in the TruthfulQA benchmark, we find that LLMs exhibit notable performance improvement when provided with additional metadata in the benchmark. Further, in the MMLU benchmark, ChatGPT and GPT-4 demonstrated an exact match rate of 52\% and 57\%, respectively, in guessing the missing options in benchmark test data. We hope these results underscore the need for more robust evaluation methodologies and benchmarks in the field.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2023-11-16T11:03:04Z</published>
++    <arxiv:comment>NAACL 2024 Version</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Chunyuan Deng</name>
++    </author>
++    <author>
++      <name>Yilun Zhao</name>
++    </author>
++    <author>
++      <name>Xiangru Tang</name>
++    </author>
++    <author>
++      <name>Mark Gerstein</name>
++    </author>
++    <author>
++      <name>Arman Cohan</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2402_18679 +103 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/toMJVet6kHQrjGRy87PzeKiYpsY</id>
++  <title>arXiv Query: search_query=&amp;id_list=2402.18679&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:13:19Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2402.18679" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2402.18679v4</id>
++    <title>Data Interpreter: An LLM Agent For Data Science</title>
++    <updated>2024-10-15T15:52:57Z</updated>
++    <link href="https://arxiv.org/abs/2402.18679v4" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2402.18679v4" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Large Language Model (LLM)-based agents have shown effectiveness across many applications. However, their use in data science scenarios requiring solving long-term interconnected tasks, dynamic data adjustments and domain expertise remains challenging. Previous approaches primarily focus on individual tasks, making it difficult to assess the complete data science workflow. Moreover, they struggle to handle real-time changes in intermediate data and fail to adapt dynamically to evolving task dependencies inherent to data science problems. In this paper, we present Data Interpreter, an LLM-based agent designed to automatically solve various data science problems end-to-end. Our Data Interpreter incorporates two key modules: 1) Hierarchical Graph Modeling, which breaks down complex problems into manageable subproblems, enabling dynamic node generation and graph optimization; and 2) Programmable Node Generation, a technique that refines and verifies each subproblem to iteratively improve code generation results and robustness. Extensive experiments consistently demonstrate the superiority of Data Interpreter. On InfiAgent-DABench, it achieves a 25% performance boost, raising accuracy from 75.9% to 94.9%. For machine learning and open-ended tasks, it improves performance from 88% to 95%, and from 60% to 97%, respectively. Moreover, on the MATH dataset, Data Interpreter achieves remarkable performance with a 26% improvement compared to state-of-the-art baselines. The code is available at https://github.com/geekan/MetaGPT.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-02-28T19:49:55Z</published>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Sirui Hong</name>
++    </author>
++    <author>
++      <name>Yizhang Lin</name>
++    </author>
++    <author>
++      <name>Bang Liu</name>
++    </author>
++    <author>
++      <name>Bangbang Liu</name>
++    </author>
++    <author>
++      <name>Binhao Wu</name>
++    </author>
++    <author>
++      <name>Ceyao Zhang</name>
++    </author>
++    <author>
++      <name>Chenxing Wei</name>
++    </author>
++    <author>
++      <name>Danyang Li</name>
++    </author>
++    <author>
++      <name>Jiaqi Chen</name>
++    </author>
++    <author>
++      <name>Jiayi Zhang</name>
++    </author>
++    <author>
++      <name>Jinlin Wang</name>
++    </author>
++    <author>
++      <name>Li Zhang</name>
++    </author>
++    <author>
++      <name>Lingyao Zhang</name>
++    </author>
++    <author>
++      <name>Min Yang</name>
++    </author>
++    <author>
++      <name>Mingchen Zhuge</name>
++    </author>
++    <author>
++      <name>Taicheng Guo</name>
++    </author>
++    <author>
++      <name>Tuo Zhou</name>
++    </author>
++    <author>
++      <name>Wei Tao</name>
++    </author>
++    <author>
++      <name>Xiangru Tang</name>
++    </author>
++    <author>
++      <name>Xiangtao Lu</name>
++    </author>
++    <author>
++      <name>Xiawu Zheng</name>
++    </author>
++    <author>
++      <name>Xinbing Liang</name>
++    </author>
++    <author>
++      <name>Yaying Fei</name>
++    </author>
++    <author>
++      <name>Yuheng Cheng</name>
++    </author>
++    <author>
++      <name>Zhibin Gou</name>
++    </author>
++    <author>
++      <name>Zongze Xu</name>
++    </author>
++    <author>
++      <name>Chenglin Wu</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2404_13076 +31 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/W50IaVXRYDrhoOgD4AmK6PSi0VI</id>
++  <title>arXiv Query: search_query=&amp;id_list=2404.13076&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:17:14Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2404.13076" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2404.13076v1</id>
++    <title>LLM Evaluators Recognize and Favor Their Own Generations</title>
++    <updated>2024-04-15T16:49:59Z</updated>
++    <link href="https://arxiv.org/abs/2404.13076v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2404.13076v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Self-evaluation using large language models (LLMs) has proven valuable not only in benchmarking but also methods like reward modeling, constitutional AI, and self-refinement. But new biases are introduced due to the same LLM acting as both the evaluator and the evaluatee. One such bias is self-preference, where an LLM evaluator scores its own outputs higher than others' while human annotators consider them of equal quality. But do LLMs actually recognize their own outputs when they give those texts higher scores, or is it just a coincidence? In this paper, we investigate if self-recognition capability contributes to self-preference. We discover that, out of the box, LLMs such as GPT-4 and Llama 2 have non-trivial accuracy at distinguishing themselves from other LLMs and humans. By fine-tuning LLMs, we discover a linear correlation between self-recognition capability and the strength of self-preference bias; using controlled experiments, we show that the causal explanation resists straightforward confounders. We discuss how self-recognition can interfere with unbiased evaluations and AI safety more generally.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-04-15T16:49:59Z</published>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Arjun Panickssery</name>
++    </author>
++    <author>
++      <name>Samuel R. Bowman</name>
++    </author>
++    <author>
++      <name>Shi Feng</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2405_15793 +47 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/PURsvKMUaFlnM+pEhhuxFjZ9wCQ</id>
++  <title>arXiv Query: search_query=&amp;id_list=2405.15793&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:13:18Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2405.15793" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2405.15793v3</id>
++    <title>SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering</title>
++    <updated>2024-11-11T20:01:15Z</updated>
++    <link href="https://arxiv.org/abs/2405.15793v3" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2405.15793v3" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Language model (LM) agents are increasingly being used to automate complicated tasks in digital environments. Just as humans benefit from powerful software applications, such as integrated development environments, for complex tasks like software engineering, we posit that LM agents represent a new category of end users with their own needs and abilities, and would benefit from specially-built interfaces to the software they use. We investigate how interface design affects the performance of language model agents. As a result of this exploration, we introduce SWE-agent: a system that facilitates LM agents to autonomously use computers to solve software engineering tasks. SWE-agent's custom agent-computer interface (ACI) significantly enhances an agent's ability to create and edit code files, navigate entire repositories, and execute tests and other programs. We evaluate SWE-agent on SWE-bench and HumanEvalFix, achieving state-of-the-art performance on both with a pass@1 rate of 12.5% and 87.7%, respectively, far exceeding the previous state-of-the-art achieved with non-interactive LMs. Finally, we provide insight on how the design of the ACI can impact agents' behavior and performance.</summary>
++    <category term="cs.SE" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.HC" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-05-06T17:41:33Z</published>
++    <arxiv:comment>Code, data, and demo available at https://swe-agent.com</arxiv:comment>
++    <arxiv:primary_category term="cs.SE"/>
++    <author>
++      <name>John Yang</name>
++    </author>
++    <author>
++      <name>Carlos E. Jimenez</name>
++    </author>
++    <author>
++      <name>Alexander Wettig</name>
++    </author>
++    <author>
++      <name>Kilian Lieret</name>
++    </author>
++    <author>
++      <name>Shunyu Yao</name>
++    </author>
++    <author>
++      <name>Karthik Narasimhan</name>
++    </author>
++    <author>
++      <name>Ofir Press</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2406_04244 +34 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/C0wP4iyy/qc3x9Jq5Zg5LYXjr0g</id>
++  <title>arXiv Query: search_query=&amp;id_list=2406.04244&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:09Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2406.04244" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2406.04244v1</id>
++    <title>Benchmark Data Contamination of Large Language Models: A Survey</title>
++    <updated>2024-06-06T16:41:39Z</updated>
++    <link href="https://arxiv.org/abs/2406.04244v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2406.04244v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>The rapid development of Large Language Models (LLMs) like GPT-4, Claude-3, and Gemini has transformed the field of natural language processing. However, it has also resulted in a significant issue known as Benchmark Data Contamination (BDC). This occurs when language models inadvertently incorporate evaluation benchmark information from their training data, leading to inaccurate or unreliable performance during the evaluation phase of the process. This paper reviews the complex challenge of BDC in LLM evaluation and explores alternative assessment methods to mitigate the risks associated with traditional benchmarks. The paper also examines challenges and future directions in mitigating BDC risks, highlighting the complexity of the issue and the need for innovative solutions to ensure the reliability of LLM evaluation in real-world applications.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-06-06T16:41:39Z</published>
++    <arxiv:comment>31 pages, 7 figures, 3 tables</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Cheng Xu</name>
++    </author>
++    <author>
++      <name>Shuhao Guan</name>
++    </author>
++    <author>
++      <name>Derek Greene</name>
++    </author>
++    <author>
++      <name>M-Tahar Kechadi</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2406_12045 +34 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/WoN1Vvn/2vViqOjsvczteJtW5L8</id>
++  <title>arXiv Query: search_query=&amp;id_list=2406.12045&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:12:46Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2406.12045" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2406.12045v1</id>
++    <title>$τ$-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains</title>
++    <updated>2024-06-17T19:33:08Z</updated>
++    <link href="https://arxiv.org/abs/2406.12045v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2406.12045v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Existing benchmarks do not test language agents on their interaction with human users or ability to follow domain-specific rules, both of which are vital for deploying them in real world applications. We propose $τ$-bench, a benchmark emulating dynamic conversations between a user (simulated by language models) and a language agent provided with domain-specific API tools and policy guidelines. We employ an efficient and faithful evaluation process that compares the database state at the end of a conversation with the annotated goal state. We also propose a new metric (pass^k) to evaluate the reliability of agent behavior over multiple trials. Our experiments show that even state-of-the-art function calling agents (like gpt-4o) succeed on &lt;50% of the tasks, and are quite inconsistent (pass^8 &lt;25% in retail). Our findings point to the need for methods that can improve the ability of agents to act consistently and follow rules reliably.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-06-17T19:33:08Z</published>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Shunyu Yao</name>
++    </author>
++    <author>
++      <name>Noah Shinn</name>
++    </author>
++    <author>
++      <name>Pedram Razavi</name>
++    </author>
++    <author>
++      <name>Karthik Narasimhan</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2406_12624 +39 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/KDF4RHNWpbljkBukXyXzkh7WO0g</id>
++  <title>arXiv Query: search_query=&amp;id_list=2406.12624&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:06Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2406.12624" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2406.12624v6</id>
++    <title>Judging the Judges: Evaluating Alignment and Vulnerabilities in LLMs-as-Judges</title>
++    <updated>2025-08-18T00:07:23Z</updated>
++    <link href="https://arxiv.org/abs/2406.12624v6" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2406.12624v6" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Offering a promising solution to the scalability challenges associated with human evaluation, the LLM-as-a-judge paradigm is rapidly gaining traction as an approach to evaluating large language models (LLMs). However, there are still many open questions about the strengths and weaknesses of this paradigm, and what potential biases it may hold. In this paper, we present a comprehensive study of the performance of various LLMs acting as judges, focusing on a clean scenario in which inter-human agreement is high. Investigating thirteen judge models of different model sizes and families, judging answers of nine different 'examtaker models' - both base and instruction-tuned - we find that only the best (and largest) models achieve reasonable alignment with humans. However, they are still quite far behind inter-human agreement and their assigned scores may still differ with up to 5 points from human-assigned scores. In terms of their ranking of the nine exam-taker models, instead, also smaller models and even the lexical metric contains may provide a reasonable signal. Through error analysis and other studies, we identify vulnerabilities in judge models, such as their sensitivity to prompt complexity and length, and a tendency toward leniency. The fact that even the best judges differ from humans in this comparatively simple setup suggest that caution may be wise when using judges in more complex setups. Lastly, our research rediscovers the importance of using alignment metrics beyond simple percent alignment, showing that judges with high percent agreement can still assign vastly different scores.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-06-18T13:49:54Z</published>
++    <arxiv:comment>https://aclanthology.org/2025.gem-1.33/</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <arxiv:journal_ref>Proceedings of the Fourth Workshop on Generation Evaluation and Metrics GEM2 2025 pages 404 to 430; July 31 August 1 2025; 2025 Association for Computational Linguistics</arxiv:journal_ref>
++    <author>
++      <name>Aman Singh Thakur</name>
++    </author>
++    <author>
++      <name>Kartik Choudhary</name>
++    </author>
++    <author>
++      <name>Venkat Srinik Ramayapally</name>
++    </author>
++    <author>
++      <name>Sankaran Vaidyanathan</name>
++    </author>
++    <author>
++      <name>Dieuwke Hupkes</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2406_18403 +82 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/AeJPG9z3zITPqz+UkAS7mQPzjYM</id>
++  <title>arXiv Query: search_query=&amp;id_list=2406.18403&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:17:29Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2406.18403" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2406.18403v3</id>
++    <title>LLMs instead of Human Judges? A Large Scale Empirical Study across 20 NLP Evaluation Tasks</title>
++    <updated>2025-06-02T11:31:19Z</updated>
++    <link href="https://arxiv.org/abs/2406.18403v3" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2406.18403v3" rel="related" type="application/pdf" title="pdf"/>
++    <summary>There is an increasing trend towards evaluating NLP models with LLMs instead of human judgments, raising questions about the validity of these evaluations, as well as their reproducibility in the case of proprietary models. We provide JUDGE-BENCH, an extensible collection of 20 NLP datasets with human annotations covering a broad range of evaluated properties and types of data, and comprehensively evaluate 11 current LLMs, covering both open-weight and proprietary models, for their ability to replicate the annotations. Our evaluations show substantial variance across models and datasets. Models are reliable evaluators on some tasks, but overall display substantial variability depending on the property being evaluated, the expertise level of the human judges, and whether the language is human or model-generated. We conclude that LLMs should be carefully validated against human judgments before being used as evaluators.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-06-26T14:56:13Z</published>
++    <arxiv:comment>Accepted to the main conference of ACL 2025</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Anna Bavaresco</name>
++    </author>
++    <author>
++      <name>Raffaella Bernardi</name>
++    </author>
++    <author>
++      <name>Leonardo Bertolazzi</name>
++    </author>
++    <author>
++      <name>Desmond Elliott</name>
++    </author>
++    <author>
++      <name>Raquel Fernández</name>
++    </author>
++    <author>
++      <name>Albert Gatt</name>
++    </author>
++    <author>
++      <name>Esam Ghaleb</name>
++    </author>
++    <author>
++      <name>Mario Giulianelli</name>
++    </author>
++    <author>
++      <name>Michael Hanna</name>
++    </author>
++    <author>
++      <name>Alexander Koller</name>
++    </author>
++    <author>
++      <name>André F. T. Martins</name>
++    </author>
++    <author>
++      <name>Philipp Mondorf</name>
++    </author>
++    <author>
++      <name>Vera Neplenbroek</name>
++    </author>
++    <author>
++      <name>Sandro Pezzelle</name>
++    </author>
++    <author>
++      <name>Barbara Plank</name>
++    </author>
++    <author>
++      <name>David Schlangen</name>
++    </author>
++    <author>
++      <name>Alessandro Suglia</name>
++    </author>
++    <author>
++      <name>Aditya K Surikuchi</name>
++    </author>
++    <author>
++      <name>Ece Takmaz</name>
++    </author>
++    <author>
++      <name>Alberto Testoni</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2407_01502 +37 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/HMD4NU+1tjYtczo3gu/76vcl52g</id>
++  <title>arXiv Query: search_query=&amp;id_list=2407.01502&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:38:45Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2407.01502" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2407.01502v1</id>
++    <title>AI Agents That Matter</title>
++    <updated>2024-07-01T17:48:14Z</updated>
++    <link href="https://arxiv.org/abs/2407.01502v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2407.01502v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>AI agents are an exciting new research direction, and agent development is driven by benchmarks. Our analysis of current agent benchmarks and evaluation practices reveals several shortcomings that hinder their usefulness in real-world applications. First, there is a narrow focus on accuracy without attention to other metrics. As a result, SOTA agents are needlessly complex and costly, and the community has reached mistaken conclusions about the sources of accuracy gains. Our focus on cost in addition to accuracy motivates the new goal of jointly optimizing the two metrics. We design and implement one such optimization, showing its potential to greatly reduce cost while maintaining accuracy. Second, the benchmarking needs of model and downstream developers have been conflated, making it hard to identify which agent would be best suited for a particular application. Third, many agent benchmarks have inadequate holdout sets, and sometimes none at all. This has led to agents that are fragile because they take shortcuts and overfit to the benchmark in various ways. We prescribe a principled framework for avoiding overfitting. Finally, there is a lack of standardization in evaluation practices, leading to a pervasive lack of reproducibility. We hope that the steps we introduce for addressing these shortcomings will spur the development of agents that are useful in the real world and not just accurate on benchmarks.</summary>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-07-01T17:48:14Z</published>
++    <arxiv:primary_category term="cs.LG"/>
++    <author>
++      <name>Sayash Kapoor</name>
++    </author>
++    <author>
++      <name>Benedikt Stroebl</name>
++    </author>
++    <author>
++      <name>Zachary S. Siegel</name>
++    </author>
++    <author>
++      <name>Nitya Nadgir</name>
++    </author>
++    <author>
++      <name>Arvind Narayanan</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2407_01725 +54 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/ms1koM/WyOhuaEe65/B8alZxMo8</id>
++  <title>arXiv Query: search_query=&amp;id_list=2407.01725&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:38:45Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2407.01725" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2407.01725v1</id>
++    <title>DiscoveryBench: Towards Data-Driven Discovery with Large Language Models</title>
++    <updated>2024-07-01T18:58:22Z</updated>
++    <link href="https://arxiv.org/abs/2407.01725v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2407.01725v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Can the rapid advances in code generation, function calling, and data analysis using large language models (LLMs) help automate the search and verification of hypotheses purely from a set of provided datasets? To evaluate this question, we present DiscoveryBench, the first comprehensive benchmark that formalizes the multi-step process of data-driven discovery. The benchmark is designed to systematically assess current model capabilities in discovery tasks and provide a useful resource for improving them. Our benchmark contains 264 tasks collected across 6 diverse domains, such as sociology and engineering, by manually deriving discovery workflows from published papers to approximate the real-world challenges faced by researchers, where each task is defined by a dataset, its metadata, and a discovery goal in natural language. We additionally provide 903 synthetic tasks to conduct controlled evaluations across task complexity. Furthermore, our structured formalism of data-driven discovery enables a facet-based evaluation that provides useful insights into different failure modes. We evaluate several popular LLM-based reasoning frameworks using both open and closed LLMs as baselines on DiscoveryBench and find that even the best system scores only 25%. Our benchmark, thus, illustrates the challenges in autonomous data-driven discovery and serves as a valuable resource for the community to make progress.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-07-01T18:58:22Z</published>
++    <arxiv:comment>Website: https://github.com/allenai/discoverybench</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Bodhisattwa Prasad Majumder</name>
++    </author>
++    <author>
++      <name>Harshit Surana</name>
++    </author>
++    <author>
++      <name>Dhruv Agarwal</name>
++    </author>
++    <author>
++      <name>Bhavana Dalvi Mishra</name>
++    </author>
++    <author>
++      <name>Abhijeetsingh Meena</name>
++    </author>
++    <author>
++      <name>Aryan Prakhar</name>
++    </author>
++    <author>
++      <name>Tirth Vora</name>
++    </author>
++    <author>
++      <name>Tushar Khot</name>
++    </author>
++    <author>
++      <name>Ashish Sabharwal</name>
++    </author>
++    <author>
++      <name>Peter Clark</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2407_16741 +96 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api//OKW3IFo6hnUsb6x0VslXSq+WWU</id>
++  <title>arXiv Query: search_query=&amp;id_list=2407.16741&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:13:18Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2407.16741" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2407.16741v3</id>
++    <title>OpenHands: An Open Platform for AI Software Developers as Generalist Agents</title>
++    <updated>2025-04-18T18:14:31Z</updated>
++    <link href="https://arxiv.org/abs/2407.16741v3" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2407.16741v3" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Software is one of the most powerful tools that we humans have at our disposal; it allows a skilled programmer to interact with the world in complex and profound ways. At the same time, thanks to improvements in large language models (LLMs), there has also been a rapid development in AI agents that interact with and affect change in their surrounding environments. In this paper, we introduce OpenHands (f.k.a. OpenDevin), a platform for the development of powerful and flexible AI agents that interact with the world in similar ways to those of a human developer: by writing code, interacting with a command line, and browsing the web. We describe how the platform allows for the implementation of new agents, safe interaction with sandboxed environments for code execution, coordination between multiple agents, and incorporation of evaluation benchmarks. Based on our currently incorporated benchmarks, we perform an evaluation of agents over 15 challenging tasks, including software engineering (e.g., SWE-BENCH) and web browsing (e.g., WEBARENA), among others. Released under the permissive MIT license, OpenHands is a community project spanning academia and industry with more than 2.1K contributions from over 188 contributors.</summary>
++    <category term="cs.SE" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-07-23T17:50:43Z</published>
++    <arxiv:comment>Accepted by ICLR 2025; Code: https://github.com/All-Hands-AI/OpenHands</arxiv:comment>
++    <arxiv:primary_category term="cs.SE"/>
++    <author>
++      <name>Xingyao Wang</name>
++    </author>
++    <author>
++      <name>Boxuan Li</name>
++    </author>
++    <author>
++      <name>Yufan Song</name>
++    </author>
++    <author>
++      <name>Frank F. Xu</name>
++    </author>
++    <author>
++      <name>Xiangru Tang</name>
++    </author>
++    <author>
++      <name>Mingchen Zhuge</name>
++    </author>
++    <author>
++      <name>Jiayi Pan</name>
++    </author>
++    <author>
++      <name>Yueqi Song</name>
++    </author>
++    <author>
++      <name>Bowen Li</name>
++    </author>
++    <author>
++      <name>Jaskirat Singh</name>
++    </author>
++    <author>
++      <name>Hoang H. Tran</name>
++    </author>
++    <author>
++      <name>Fuqiang Li</name>
++    </author>
++    <author>
++      <name>Ren Ma</name>
++    </author>
++    <author>
++      <name>Mingzhang Zheng</name>
++    </author>
++    <author>
++      <name>Bill Qian</name>
++    </author>
++    <author>
++      <name>Yanjun Shao</name>
++    </author>
++    <author>
++      <name>Niklas Muennighoff</name>
++    </author>
++    <author>
++      <name>Yizhe Zhang</name>
++    </author>
++    <author>
++      <name>Binyuan Hui</name>
++    </author>
++    <author>
++      <name>Junyang Lin</name>
++    </author>
++    <author>
++      <name>Robert Brennan</name>
++    </author>
++    <author>
++      <name>Hao Peng</name>
++    </author>
++    <author>
++      <name>Heng Ji</name>
++    </author>
++    <author>
++      <name>Graham Neubig</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2410_06992 +39 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/f/9Lx2DMiPMwvaJjN9mOwHn2egA</id>
++  <title>arXiv Query: search_query=&amp;id_list=2410.06992&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:03Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2410.06992" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2410.06992v2</id>
++    <title>SWE-Bench+: Enhanced Coding Benchmark for LLMs</title>
++    <updated>2024-10-10T13:13:09Z</updated>
++    <link href="https://arxiv.org/abs/2410.06992v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2410.06992v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Large Language Models (LLMs) in Software Engineering (SE) can offer assistance for coding. To facilitate a rigorous evaluation of LLMs in practical coding contexts, Carlos et al. introduced the SWE-bench dataset, which comprises 2,294 real-world GitHub issues and their corresponding pull requests, collected from 12 widely used Python repositories. Several impressive LLM-based toolkits recently are developed and evaluated on this dataset. However, a systematic evaluation of the quality of SWE-bench remains missing. In this paper, we addressed this gap by presenting an empirical analysis of the SWE-bench dataset. We conducted a manual screening of instances where SWEAgent + GPT-4 successfully resolved issues by comparing the model-generated patches with the actual pull requests. SWE-Agent+GPT-4 was at the top of SWE-bench leaderboard during the time of our study. Our analysis reveals some critical issues with the SWE-bench dataset: 1) 32.67% of the successful patches involve cheating as the solutions were directly provided in the issue report or the comments. We refer to as solution leakage problem. 2) 31.08% of the passed patches are suspicious patches due to weak test cases, i.e., the tests were not adequate to verify the correctness of a patch. When we filtered out these problematic issues, the resolution rate of SWE-Agent+GPT-4 dropped from 12.47% to 3.97%. We also observed that the same data quality issues also exist in the two variants of SWE-bench, i.e., SWE-bench Lite and SWE-Bench Verified. In addition, over 94% of the issues were created before LLM's knowledge cutoff dates, posing potential data leakage issues.</summary>
++    <category term="cs.SE" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-10-09T15:38:53Z</published>
++    <arxiv:primary_category term="cs.SE"/>
++    <author>
++      <name>Reem Aleithan</name>
++    </author>
++    <author>
++      <name>Haoran Xue</name>
++    </author>
++    <author>
++      <name>Mohammad Mahdi Mohajer</name>
++    </author>
++    <author>
++      <name>Elijah Nnorom</name>
++    </author>
++    <author>
++      <name>Gias Uddin</name>
++    </author>
++    <author>
++      <name>Song Wang</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2411_05285 +32 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/7ElnM1hM9SAws3+gsFUx26ec29M</id>
++  <title>arXiv Query: search_query=&amp;id_list=2411.05285&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:22:34Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2411.05285" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2411.05285v2</id>
++    <title>AgentOps: Enabling Observability of LLM Agents</title>
++    <updated>2024-11-30T02:55:48Z</updated>
++    <link href="https://arxiv.org/abs/2411.05285v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2411.05285v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Large language model (LLM) agents have demonstrated remarkable capabilities across various domains, gaining extensive attention from academia and industry. However, these agents raise significant concerns on AI safety due to their autonomous and non-deterministic behavior, as well as continuous evolving nature . From a DevOps perspective, enabling observability in agents is necessary to ensuring AI safety, as stakeholders can gain insights into the agents' inner workings, allowing them to proactively understand the agents, detect anomalies, and prevent potential failures. Therefore, in this paper, we present a comprehensive taxonomy of AgentOps, identifying the artifacts and associated data that should be traced throughout the entire lifecycle of agents to achieve effective observability. The taxonomy is developed based on a systematic mapping study of existing AgentOps tools. Our taxonomy serves as a reference template for developers to design and implement AgentOps infrastructure that supports monitoring, logging, and analytics. thereby ensuring AI safety.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.SE" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-11-08T02:31:03Z</published>
++    <arxiv:comment>12 pages, 3 figures</arxiv:comment>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Liming Dong</name>
++    </author>
++    <author>
++      <name>Qinghua Lu</name>
++    </author>
++    <author>
++      <name>Liming Zhu</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2411_15114 +91 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api//3jj4HdP+t4ljnZ/5Nn0MfnBZ5M</id>
++  <title>arXiv Query: search_query=&amp;id_list=2411.15114&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:13:17Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2411.15114" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2411.15114v2</id>
++    <title>RE-Bench: Evaluating frontier AI R&amp;D capabilities of language model agents against human experts</title>
++    <updated>2025-05-27T03:32:23Z</updated>
++    <link href="https://arxiv.org/abs/2411.15114v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2411.15114v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Frontier AI safety policies highlight automation of AI research and development (R&amp;D) by AI agents as an important capability to anticipate. However, there exist few evaluations for AI R&amp;D capabilities, and none that are highly realistic and have a direct comparison to human performance. We introduce RE-Bench (Research Engineering Benchmark, v1), which consists of 7 challenging, open-ended ML research engineering environments and data from 71 8-hour attempts by 61 distinct human experts. We confirm that our experts make progress in the environments given 8 hours, with 82% of expert attempts achieving a non-zero score and 24% matching or exceeding our strong reference solutions. We compare humans to several public frontier models through best-of-k with varying time budgets and agent designs, and find that the best AI agents achieve a score 4x higher than human experts when both are given a total time budget of 2 hours per environment. However, humans currently display better returns to increasing time budgets, narrowly exceeding the top AI agent scores given an 8-hour budget, and achieving 2x the score of the top AI agent when both are given 32 total hours (across different attempts). Qualitatively, we find that modern AI agents possess significant expertise in many ML topics -- e.g. an agent wrote a faster custom Triton kernel than any of our human experts' -- and can generate and test solutions over ten times faster than humans, at much lower cost. We open-source the evaluation environments, human expert data, analysis code and agent trajectories to facilitate future research.</summary>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-11-22T18:30:46Z</published>
++    <arxiv:primary_category term="cs.LG"/>
++    <author>
++      <name>Hjalmar Wijk</name>
++    </author>
++    <author>
++      <name>Tao Lin</name>
++    </author>
++    <author>
++      <name>Joel Becker</name>
++    </author>
++    <author>
++      <name>Sami Jawhar</name>
++    </author>
++    <author>
++      <name>Neev Parikh</name>
++    </author>
++    <author>
++      <name>Thomas Broadley</name>
++    </author>
++    <author>
++      <name>Lawrence Chan</name>
++    </author>
++    <author>
++      <name>Michael Chen</name>
++    </author>
++    <author>
++      <name>Josh Clymer</name>
++    </author>
++    <author>
++      <name>Jai Dhyani</name>
++    </author>
++    <author>
++      <name>Elena Ericheva</name>
++    </author>
++    <author>
++      <name>Katharyn Garcia</name>
++    </author>
++    <author>
++      <name>Brian Goodrich</name>
++    </author>
++    <author>
++      <name>Nikola Jurkovic</name>
++    </author>
++    <author>
++      <name>Holden Karnofsky</name>
++    </author>
++    <author>
++      <name>Megan Kinniment</name>
++    </author>
++    <author>
++      <name>Aron Lajko</name>
++    </author>
++    <author>
++      <name>Seraphina Nix</name>
++    </author>
++    <author>
++      <name>Lucas Sato</name>
++    </author>
++    <author>
++      <name>William Saunders</name>
++    </author>
++    <author>
++      <name>Maksym Taran</name>
++    </author>
++    <author>
++      <name>Ben West</name>
++    </author>
++    <author>
++      <name>Elizabeth Barnes</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2411_15594 +71 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/4U2vP69i7a9WjhAeW6emltPuFEA</id>
++  <title>arXiv Query: search_query=&amp;id_list=2411.15594&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:07Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2411.15594" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2411.15594v6</id>
++    <title>A Survey on LLM-as-a-Judge</title>
++    <updated>2025-10-19T10:32:43Z</updated>
++    <link href="https://arxiv.org/abs/2411.15594v6" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2411.15594v6" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Accurate and consistent evaluation is crucial for decision-making across numerous fields, yet it remains a challenging task due to inherent subjectivity, variability, and scale. Large Language Models (LLMs) have achieved remarkable success across diverse domains, leading to the emergence of "LLM-as-a-Judge," where LLMs are employed as evaluators for complex tasks. With their ability to process diverse data types and provide scalable, cost-effective, and consistent assessments, LLMs present a compelling alternative to traditional expert-driven evaluations. However, ensuring the reliability of LLM-as-a-Judge systems remains a significant challenge that requires careful design and standardization. This paper provides a comprehensive survey of LLM-as-a-Judge, addressing the core question: How can reliable LLM-as-a-Judge systems be built? We explore strategies to enhance reliability, including improving consistency, mitigating biases, and adapting to diverse assessment scenarios. Additionally, we propose methodologies for evaluating the reliability of LLM-as-a-Judge systems, supported by a novel benchmark designed for this purpose. To advance the development and real-world deployment of LLM-as-a-Judge systems, we also discussed practical applications, challenges, and future directions. This survey serves as a foundational reference for researchers and practitioners in this rapidly evolving field.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2024-11-23T16:03:35Z</published>
++    <arxiv:comment>Project Page: https://awesome-llm-as-a-judge.github.io/</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Jiawei Gu</name>
++    </author>
++    <author>
++      <name>Xuhui Jiang</name>
++    </author>
++    <author>
++      <name>Zhichao Shi</name>
++    </author>
++    <author>
++      <name>Hexiang Tan</name>
++    </author>
++    <author>
++      <name>Xuehao Zhai</name>
++    </author>
++    <author>
++      <name>Chengjin Xu</name>
++    </author>
++    <author>
++      <name>Wei Li</name>
++    </author>
++    <author>
++      <name>Yinghan Shen</name>
++    </author>
++    <author>
++      <name>Shengjie Ma</name>
++    </author>
++    <author>
++      <name>Honghao Liu</name>
++    </author>
++    <author>
++      <name>Saizhuo Wang</name>
++    </author>
++    <author>
++      <name>Kun Zhang</name>
++    </author>
++    <author>
++      <name>Yuanzhuo Wang</name>
++    </author>
++    <author>
++      <name>Wen Gao</name>
++    </author>
++    <author>
++      <name>Lionel Ni</name>
++    </author>
++    <author>
++      <name>Jian Guo</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2502_14297 +38 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/V2gJqOssY0aEzlwEIBPnDOfou0c</id>
++  <title>arXiv Query: search_query=&amp;id_list=2502.14297&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:49:55Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2502.14297" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2502.14297v3</id>
++    <title>Evaluating Sakana's AI Scientist: Bold Claims, Mixed Results, and a Promising Future?</title>
++    <updated>2025-10-15T18:03:49Z</updated>
++    <link href="https://arxiv.org/abs/2502.14297v3" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2502.14297v3" rel="related" type="application/pdf" title="pdf"/>
++    <summary>A major step toward Artificial General Intelligence (AGI) and Super Intelligence is AI's ability to autonomously conduct research - what we term Artificial Research Intelligence (ARI). If machines could generate hypotheses, conduct experiments, and write research papers without human intervention, it would transform science. Sakana recently introduced the 'AI Scientist', claiming to conduct research autonomously, i.e. they imply to have achieved what we term Artificial Research Intelligence (ARI). The AI Scientist gained much attention, but a thorough independent evaluation has yet to be conducted.
++  Our evaluation of the AI Scientist reveals critical shortcomings. The system's literature reviews produced poor novelty assessments, often misclassifying established concepts (e.g., micro-batching for stochastic gradient descent) as novel. It also struggles with experiment execution: 42% of experiments failed due to coding errors, while others produced flawed or misleading results. Code modifications were minimal, averaging 8% more characters per iteration, suggesting limited adaptability. Generated manuscripts were poorly substantiated, with a median of five citations, most outdated (only five of 34 from 2020 or later). Structural errors were frequent, including missing figures, repeated sections, and placeholder text like 'Conclusions Here'. Some papers contained hallucinated numerical results.
++  Despite these flaws, the AI Scientist represents a leap forward in research automation. It generates full research manuscripts with minimal human input, challenging expectations of AI-driven science. Many reviewers might struggle to distinguish its work from human researchers. While its quality resembles a rushed undergraduate paper, its speed and cost efficiency are unprecedented, producing a full paper for USD 6 to 15 with 3.5 hours of human involvement, far outpacing traditional researchers.</summary>
++    <category term="cs.IR" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2025-02-20T06:22:03Z</published>
++    <arxiv:comment>20 pages</arxiv:comment>
++    <arxiv:primary_category term="cs.IR"/>
++    <arxiv:journal_ref>SIGIR Forum 2025</arxiv:journal_ref>
++    <author>
++      <name>Joeran Beel</name>
++    </author>
++    <author>
++      <name>Min-Yen Kan</name>
++    </author>
++    <author>
++      <name>Moritz Baumgart</name>
++    </author>
++    <arxiv:doi>10.1145/3769733.376974</arxiv:doi>
++    <link rel="related" href="https://doi.org/10.1145/3769733.376974" title="doi"/>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2504_13359 +38 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/cGL2xolC/7Pi+SyVnLbYEDMJK2Q</id>
++  <title>arXiv Query: search_query=&amp;id_list=2504.13359&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:14:21Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2504.13359" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2504.13359v2</id>
++    <title>Cost-of-Pass: An Economic Framework for Evaluating Language Models</title>
++    <updated>2026-02-26T05:45:42Z</updated>
++    <link href="https://arxiv.org/abs/2504.13359v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2504.13359v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Widespread adoption of AI systems hinges on their ability to generate economic value that outweighs their inference costs. Evaluating this tradeoff requires metrics accounting for both performance and costs. Building on production theory, we develop an economically grounded framework to evaluate language models' productivity by combining accuracy and inference cost. We formalize cost-of-pass: the expected monetary cost of generating a correct solution. We then define the frontier cost-of-pass: the minimum cost-of-pass achievable across available models or the human-expert(s), using the approx. cost of hiring an expert. Our analysis reveals distinct economic insights. First, lightweight models are most cost-effective for basic quantitative tasks, large models for knowledge-intensive ones, and reasoning models for complex quantitative problems, despite higher per-token costs. Second, tracking the frontier cost-of-pass over the past year reveals significant progress, particularly for complex quant. tasks where the cost roughly halved every few months. Third, to trace key innovations driving this progress, we examine counterfactual frontiers -- estimates of cost-efficiency without specific model classes. We find that innovations in lightweight, large, and reasoning models have been essential for pushing the frontier in basic quant., knowledge-intensive, and complex quant. tasks, respectively. Finally, we assess the cost-reductions from common inference-time techniques (majority voting and self-refinement), and a budget-aware technique (TALE-EP). We find that performance-oriented methods with marginal performance gains rarely justify the costs, while TALE-EP shows some promise. Overall, our findings underscore that complementary model-level innovations are the primary drivers of cost-efficiency and our framework provides a principled tool for measuring this progress and guiding deployment.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2025-04-17T21:58:29Z</published>
++    <arxiv:comment>Code is available at: https://github.com/mhamzaerol/Cost-of-Pass</arxiv:comment>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Mehmet Hamza Erol</name>
++    </author>
++    <author>
++      <name>Batu El</name>
++    </author>
++    <author>
++      <name>Mirac Suzgun</name>
++    </author>
++    <author>
++      <name>Mert Yuksekgonul</name>
++    </author>
++    <author>
++      <name>James Zou</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2504_18765 +78 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/8E8jTk2NWVojzdmHL7e4CRjr6yc</id>
++  <title>arXiv Query: search_query=&amp;id_list=2504.18765&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:13:43Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2504.18765" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2504.18765v3</id>
++    <title>A Vision for Auto Research with LLM Agents</title>
++    <updated>2025-07-19T16:30:25Z</updated>
++    <link href="https://arxiv.org/abs/2504.18765v3" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2504.18765v3" rel="related" type="application/pdf" title="pdf"/>
++    <summary>This paper introduces Agent-Based Auto Research, a structured multi-agent framework designed to automate, coordinate, and optimize the full lifecycle of scientific research. Leveraging the capabilities of large language models (LLMs) and modular agent collaboration, the system spans all major research phases, including literature review, ideation, methodology planning, experimentation, paper writing, peer review response, and dissemination. By addressing issues such as fragmented workflows, uneven methodological expertise, and cognitive overload, the framework offers a systematic and scalable approach to scientific inquiry. Preliminary explorations demonstrate the feasibility and potential of Auto Research as a promising paradigm for self-improving, AI-driven research processes.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2025-04-26T02:06:10Z</published>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Chengwei Liu</name>
++    </author>
++    <author>
++      <name>Chong Wang</name>
++    </author>
++    <author>
++      <name>Jiayue Cao</name>
++    </author>
++    <author>
++      <name>Jingquan Ge</name>
++    </author>
++    <author>
++      <name>Kun Wang</name>
++    </author>
++    <author>
++      <name>Lyuye Zhang</name>
++    </author>
++    <author>
++      <name>Ming-Ming Cheng</name>
++    </author>
++    <author>
++      <name>Penghai Zhao</name>
++    </author>
++    <author>
++      <name>Tianlin Li</name>
++    </author>
++    <author>
++      <name>Xiaojun Jia</name>
++    </author>
++    <author>
++      <name>Xiang Li</name>
++    </author>
++    <author>
++      <name>Xingshuai Li</name>
++    </author>
++    <author>
++      <name>Yang Liu</name>
++    </author>
++    <author>
++      <name>Yebo Feng</name>
++    </author>
++    <author>
++      <name>Yihao Huang</name>
++    </author>
++    <author>
++      <name>Yijia Xu</name>
++    </author>
++    <author>
++      <name>Yuqiang Sun</name>
++    </author>
++    <author>
++      <name>Zhenhong Zhou</name>
++    </author>
++    <author>
++      <name>Zhengzi Xu</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2507_02825 +97 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/+86DjtahU6GyBfqQgzDZDqQMH3E</id>
++  <title>arXiv Query: search_query=&amp;id_list=2507.02825&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:13:45Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2507.02825" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2507.02825v5</id>
++    <title>Establishing Best Practices for Building Rigorous Agentic Benchmarks</title>
++    <updated>2025-08-07T06:58:08Z</updated>
++    <link href="https://arxiv.org/abs/2507.02825v5" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2507.02825v5" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Benchmarks are essential for quantitatively tracking progress in AI. As AI agents become increasingly capable, researchers and practitioners have introduced agentic benchmarks to evaluate agents on complex, real-world tasks. These benchmarks typically measure agent capabilities by evaluating task outcomes via specific reward designs. However, we show that many agentic benchmarks have issues in task setup or reward design. For example, SWE-bench Verified uses insufficient test cases, while TAU-bench counts empty responses as successful. Such issues can lead to under- or overestimation of agents' performance by up to 100% in relative terms. To make agentic evaluation rigorous, we introduce the Agentic Benchmark Checklist (ABC), a set of guidelines that we synthesized from our benchmark-building experience, a survey of best practices, and previously reported issues. When applied to CVE-Bench, a benchmark with a particularly complex evaluation design, ABC reduces the performance overestimation by 33%.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2025-07-03T17:35:31Z</published>
++    <arxiv:comment>39 pages, 15 tables, 6 figures</arxiv:comment>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Yuxuan Zhu</name>
++    </author>
++    <author>
++      <name>Tengjun Jin</name>
++    </author>
++    <author>
++      <name>Yada Pruksachatkun</name>
++    </author>
++    <author>
++      <name>Andy Zhang</name>
++    </author>
++    <author>
++      <name>Shu Liu</name>
++    </author>
++    <author>
++      <name>Sasha Cui</name>
++    </author>
++    <author>
++      <name>Sayash Kapoor</name>
++    </author>
++    <author>
++      <name>Shayne Longpre</name>
++    </author>
++    <author>
++      <name>Kevin Meng</name>
++    </author>
++    <author>
++      <name>Rebecca Weiss</name>
++    </author>
++    <author>
++      <name>Fazl Barez</name>
++    </author>
++    <author>
++      <name>Rahul Gupta</name>
++    </author>
++    <author>
++      <name>Jwala Dhamala</name>
++    </author>
++    <author>
++      <name>Jacob Merizian</name>
++    </author>
++    <author>
++      <name>Mario Giulianelli</name>
++    </author>
++    <author>
++      <name>Harry Coppock</name>
++    </author>
++    <author>
++      <name>Cozmin Ududec</name>
++    </author>
++    <author>
++      <name>Jasjeet Sekhon</name>
++    </author>
++    <author>
++      <name>Jacob Steinhardt</name>
++    </author>
++    <author>
++      <name>Antony Kellermann</name>
++    </author>
++    <author>
++      <name>Sarah Schwettmann</name>
++    </author>
++    <author>
++      <name>Matei Zaharia</name>
++    </author>
++    <author>
++      <name>Ion Stoica</name>
++    </author>
++    <author>
++      <name>Percy Liang</name>
++    </author>
++    <author>
++      <name>Daniel Kang</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2507_09089 +36 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/BhFSFCcaFI7JDAxISp56WxeDJlM</id>
++  <title>arXiv Query: search_query=&amp;id_list=2507.09089&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:34:08Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2507.09089" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2507.09089v2</id>
++    <title>Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity</title>
++    <updated>2025-07-25T00:43:07Z</updated>
++    <link href="https://arxiv.org/abs/2507.09089v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2507.09089v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Despite widespread adoption, the impact of AI tools on software development in the wild remains understudied. We conduct a randomized controlled trial (RCT) to understand how AI tools at the February-June 2025 frontier affect the productivity of experienced open-source developers. 16 developers with moderate AI experience complete 246 tasks in mature projects on which they have an average of 5 years of prior experience. Each task is randomly assigned to allow or disallow usage of early 2025 AI tools. When AI tools are allowed, developers primarily use Cursor Pro, a popular code editor, and Claude 3.5/3.7 Sonnet. Before starting tasks, developers forecast that allowing AI will reduce completion time by 24%. After completing the study, developers estimate that allowing AI reduced completion time by 20%. Surprisingly, we find that allowing AI actually increases completion time by 19%--AI tooling slowed developers down. This slowdown also contradicts predictions from experts in economics (39% shorter) and ML (38% shorter). To understand this result, we collect and evaluate evidence for 20 properties of our setting that a priori could contribute to the observed slowdown effect--for example, the size and quality standards of projects, or prior developer experience with AI tooling. Although the influence of experimental artifacts cannot be entirely ruled out, the robustness of the slowdown effect across our analyses suggests it is unlikely to primarily be a function of our experimental design.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.HC" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.SE" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2025-07-12T00:16:33Z</published>
++    <arxiv:comment>51 pages, 8 tables, 22 figures</arxiv:comment>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Joel Becker</name>
++    </author>
++    <author>
++      <name>Nate Rush</name>
++    </author>
++    <author>
++      <name>Elizabeth Barnes</name>
++    </author>
++    <author>
++      <name>David Rein</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2509_08713 +32 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/fu6PJsujv/m3Io7gpx/9D53Qqxk</id>
++  <title>arXiv Query: search_query=&amp;id_list=2509.08713&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:50:41Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2509.08713" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2509.08713v2</id>
++    <title>The More You Automate, the Less You See: Hidden Pitfalls of AI Scientist Systems</title>
++    <updated>2025-12-20T05:26:07Z</updated>
++    <link href="https://arxiv.org/abs/2509.08713v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2509.08713v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>AI scientist systems, capable of autonomously executing the full research workflow from hypothesis generation and experimentation to paper writing, hold significant potential for accelerating scientific discovery. However, the internal workflow of these systems have not been closely examined. This lack of scrutiny poses a risk of introducing flaws that could undermine the integrity, reliability, and trustworthiness of their research outputs. In this paper, we identify four potential failure modes in contemporary AI scientist systems: inappropriate benchmark selection, data leakage, metric misuse, and post-hoc selection bias. To examine these risks, we design controlled experiments that isolate each failure mode while addressing challenges unique to evaluating AI scientist systems. Our assessment of two prominent open-source AI scientist systems reveals the presence of several failures, across a spectrum of severity, which can be easily overlooked in practice. Finally, we demonstrate that access to trace logs and code from the full automated workflow enables far more effective detection of such failures than examining the final paper alone. We thus recommend journals and conferences evaluating AI-generated research to mandate submission of these artifacts alongside the paper to ensure transparency, accountability, and reproducibility.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.DL" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2025-09-10T16:04:24Z</published>
++    <arxiv:comment>NeurIPS 2025 AI4Science (Spotlight)</arxiv:comment>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Ziming Luo</name>
++    </author>
++    <author>
++      <name>Atoosa Kasirzadeh</name>
++    </author>
++    <author>
++      <name>Nihar B. Shah</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2509_09912 +40 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/2ryZhjWu+Fq4XVsR1wa2uCdb7ZU</id>
++  <title>arXiv Query: search_query=&amp;id_list=2509.09912&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:16:09Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2509.09912" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2509.09912v1</id>
++    <title>When Your Reviewer is an LLM: Biases, Divergence, and Prompt Injection Risks in Peer Review</title>
++    <updated>2025-09-12T00:57:50Z</updated>
++    <link href="https://arxiv.org/abs/2509.09912v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2509.09912v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Peer review is the cornerstone of academic publishing, yet the process is increasingly strained by rising submission volumes, reviewer overload, and expertise mismatches. Large language models (LLMs) are now being used as "reviewer aids," raising concerns about their fairness, consistency, and robustness against indirect prompt injection attacks. This paper presents a systematic evaluation of LLMs as academic reviewers. Using a curated dataset of 1,441 papers from ICLR 2023 and NeurIPS 2022, we evaluate GPT-5-mini against human reviewers across ratings, strengths, and weaknesses. The evaluation employs structured prompting with reference paper calibration, topic modeling, and similarity analysis to compare review content. We further embed covert instructions into PDF submissions to assess LLMs' susceptibility to prompt injection. Our findings show that LLMs consistently inflate ratings for weaker papers while aligning more closely with human judgments on stronger contributions. Moreover, while overarching malicious prompts induce only minor shifts in topical focus, explicitly field-specific instructions successfully manipulate specific aspects of LLM-generated reviews. This study underscores both the promises and perils of integrating LLMs into peer review and points to the importance of designing safeguards that ensure integrity and trust in future review processes.</summary>
++    <category term="cs.CY" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CR" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2025-09-12T00:57:50Z</published>
++    <arxiv:primary_category term="cs.CY"/>
++    <author>
++      <name>Changjia Zhu</name>
++    </author>
++    <author>
++      <name>Junjie Xiong</name>
++    </author>
++    <author>
++      <name>Renkai Ma</name>
++    </author>
++    <author>
++      <name>Zhicong Lu</name>
++    </author>
++    <author>
++      <name>Yao Liu</name>
++    </author>
++    <author>
++      <name>Lingyao Li</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2510_11977 +115 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/hARmKhPa+l0qoNEtWLtlDliPeo0</id>
++  <title>arXiv Query: search_query=&amp;id_list=2510.11977&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:00:56Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2510.11977" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2510.11977v1</id>
++    <title>Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation</title>
++    <updated>2025-10-13T22:22:28Z</updated>
++    <link href="https://arxiv.org/abs/2510.11977v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2510.11977v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>AI agents have been developed for complex real-world tasks from coding to customer service. But AI agent evaluations suffer from many challenges that undermine our understanding of how well agents really work. We introduce the Holistic Agent Leaderboard (HAL) to address these challenges. We make three main contributions. First, we provide a standardized evaluation harness that orchestrates parallel evaluations across hundreds of VMs, reducing evaluation time from weeks to hours while eliminating common implementation bugs. Second, we conduct three-dimensional analysis spanning models, scaffolds, and benchmarks. We validate the harness by conducting 21,730 agent rollouts across 9 models and 9 benchmarks in coding, web navigation, science, and customer service with a total cost of about $40,000. Our analysis reveals surprising insights, such as higher reasoning effort reducing accuracy in the majority of runs. Third, we use LLM-aided log inspection to uncover previously unreported behaviors, such as searching for the benchmark on HuggingFace instead of solving a task, or misusing credit cards in flight booking tasks. We share all agent logs, comprising 2.5B tokens of language model calls, to incentivize further research into agent behavior. By standardizing how the field evaluates agents and addressing common pitfalls in agent evaluation, we hope to shift the focus from agents that ace benchmarks to agents that work reliably in the real world.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2025-10-13T22:22:28Z</published>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Sayash Kapoor</name>
++    </author>
++    <author>
++      <name>Benedikt Stroebl</name>
++    </author>
++    <author>
++      <name>Peter Kirgis</name>
++    </author>
++    <author>
++      <name>Nitya Nadgir</name>
++    </author>
++    <author>
++      <name>Zachary S Siegel</name>
++    </author>
++    <author>
++      <name>Boyi Wei</name>
++    </author>
++    <author>
++      <name>Tianci Xue</name>
++    </author>
++    <author>
++      <name>Ziru Chen</name>
++    </author>
++    <author>
++      <name>Felix Chen</name>
++    </author>
++    <author>
++      <name>Saiteja Utpala</name>
++    </author>
++    <author>
++      <name>Franck Ndzomga</name>
++    </author>
++    <author>
++      <name>Dheeraj Oruganty</name>
++    </author>
++    <author>
++      <name>Sophie Luskin</name>
++    </author>
++    <author>
++      <name>Kangheng Liu</name>
++    </author>
++    <author>
++      <name>Botao Yu</name>
++    </author>
++    <author>
++      <name>Amit Arora</name>
++    </author>
++    <author>
++      <name>Dongyoon Hahm</name>
++    </author>
++    <author>
++      <name>Harsh Trivedi</name>
++    </author>
++    <author>
++      <name>Huan Sun</name>
++    </author>
++    <author>
++      <name>Juyong Lee</name>
++    </author>
++    <author>
++      <name>Tengjun Jin</name>
++    </author>
++    <author>
++      <name>Yifan Mai</name>
++    </author>
++    <author>
++      <name>Yifei Zhou</name>
++    </author>
++    <author>
++      <name>Yuxuan Zhu</name>
++    </author>
++    <author>
++      <name>Rishi Bommasani</name>
++    </author>
++    <author>
++      <name>Daniel Kang</name>
++    </author>
++    <author>
++      <name>Dawn Song</name>
++    </author>
++    <author>
++      <name>Peter Henderson</name>
++    </author>
++    <author>
++      <name>Yu Su</name>
++    </author>
++    <author>
++      <name>Percy Liang</name>
++    </author>
++    <author>
++      <name>Arvind Narayanan</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2510_12367 +46 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api//pozSTcQ/6i3aKrawgxgB2nEZrE</id>
++  <title>arXiv Query: search_query=&amp;id_list=2510.12367&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:16:08Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2510.12367" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2510.12367v1</id>
++    <title>LLM-REVal: Can We Trust LLM Reviewers Yet?</title>
++    <updated>2025-10-14T10:30:20Z</updated>
++    <link href="https://arxiv.org/abs/2510.12367v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2510.12367v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>The rapid advancement of large language models (LLMs) has inspired researchers to integrate them extensively into the academic workflow, potentially reshaping how research is practiced and reviewed. While previous studies highlight the potential of LLMs in supporting research and peer review, their dual roles in the academic workflow and the complex interplay between research and review bring new risks that remain largely underexplored. In this study, we focus on how the deep integration of LLMs into both peer-review and research processes may influence scholarly fairness, examining the potential risks of using LLMs as reviewers by simulation. This simulation incorporates a research agent, which generates papers and revises, alongside a review agent, which assesses the submissions. Based on the simulation results, we conduct human annotations and identify pronounced misalignment between LLM-based reviews and human judgments: (1) LLM reviewers systematically inflate scores for LLM-authored papers, assigning them markedly higher scores than human-authored ones; (2) LLM reviewers persistently underrate human-authored papers with critical statements (e.g., risk, fairness), even after multiple revisions. Our analysis reveals that these stem from two primary biases in LLM reviewers: a linguistic feature bias favoring LLM-generated writing styles, and an aversion toward critical statements. These results highlight the risks and equity concerns posed to human authors and academic research if LLMs are deployed in the peer review cycle without adequate caution. On the other hand, revisions guided by LLM reviews yield quality gains in both LLM-based and human evaluations, illustrating the potential of the LLMs-as-reviewers for early-stage researchers and enhancing low-quality papers.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2025-10-14T10:30:20Z</published>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Rui Li</name>
++    </author>
++    <author>
++      <name>Jia-Chen Gu</name>
++    </author>
++    <author>
++      <name>Po-Nien Kung</name>
++    </author>
++    <author>
++      <name>Heming Xia</name>
++    </author>
++    <author>
++      <name>Junfeng liu</name>
++    </author>
++    <author>
++      <name>Xiangwen Kong</name>
++    </author>
++    <author>
++      <name>Zhifang Sui</name>
++    </author>
++    <author>
++      <name>Nanyun Peng</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2601_11868 +277 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/UsgVJ4SjjdBKZYxnQqjyh54zAhI</id>
++  <title>arXiv Query: search_query=&amp;id_list=2601.11868&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:29:46Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2601.11868" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2601.11868v1</id>
++    <title>Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces</title>
++    <updated>2026-01-17T01:29:30Z</updated>
++    <link href="https://arxiv.org/abs/2601.11868v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2601.11868v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>AI agents may soon become capable of autonomously completing valuable, long-horizon tasks in diverse domains. Current benchmarks either do not measure real-world tasks, or are not sufficiently difficult to meaningfully measure frontier models. To this end, we present Terminal-Bench 2.0: a carefully curated hard benchmark composed of 89 tasks in computer terminal environments inspired by problems from real workflows. Each task features a unique environment, human-written solution, and comprehensive tests for verification. We show that frontier models and agents score less than 65\% on the benchmark and conduct an error analysis to identify areas for model and agent improvement. We publish the dataset and evaluation harness to assist developers and researchers in future work at https://www.tbench.ai/ .</summary>
++    <category term="cs.SE" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2026-01-17T01:29:30Z</published>
++    <arxiv:primary_category term="cs.SE"/>
++    <author>
++      <name>Mike A. Merrill</name>
++    </author>
++    <author>
++      <name>Alexander G. Shaw</name>
++    </author>
++    <author>
++      <name>Nicholas Carlini</name>
++    </author>
++    <author>
++      <name>Boxuan Li</name>
++    </author>
++    <author>
++      <name>Harsh Raj</name>
++    </author>
++    <author>
++      <name>Ivan Bercovich</name>
++    </author>
++    <author>
++      <name>Lin Shi</name>
++    </author>
++    <author>
++      <name>Jeong Yeon Shin</name>
++    </author>
++    <author>
++      <name>Thomas Walshe</name>
++    </author>
++    <author>
++      <name>E. Kelly Buchanan</name>
++    </author>
++    <author>
++      <name>Junhong Shen</name>
++    </author>
++    <author>
++      <name>Guanghao Ye</name>
++    </author>
++    <author>
++      <name>Haowei Lin</name>
++    </author>
++    <author>
++      <name>Jason Poulos</name>
++    </author>
++    <author>
++      <name>Maoyu Wang</name>
++    </author>
++    <author>
++      <name>Marianna Nezhurina</name>
++    </author>
++    <author>
++      <name>Jenia Jitsev</name>
++    </author>
++    <author>
++      <name>Di Lu</name>
++    </author>
++    <author>
++      <name>Orfeas Menis Mastromichalakis</name>
++    </author>
++    <author>
++      <name>Zhiwei Xu</name>
++    </author>
++    <author>
++      <name>Zizhao Chen</name>
++    </author>
++    <author>
++      <name>Yue Liu</name>
++    </author>
++    <author>
++      <name>Robert Zhang</name>
++    </author>
++    <author>
++      <name>Leon Liangyu Chen</name>
++    </author>
++    <author>
++      <name>Anurag Kashyap</name>
++    </author>
++    <author>
++      <name>Jan-Lucas Uslu</name>
++    </author>
++    <author>
++      <name>Jeffrey Li</name>
++    </author>
++    <author>
++      <name>Jianbo Wu</name>
++    </author>
++    <author>
++      <name>Minghao Yan</name>
++    </author>
++    <author>
++      <name>Song Bian</name>
++    </author>
++    <author>
++      <name>Vedang Sharma</name>
++    </author>
++    <author>
++      <name>Ke Sun</name>
++    </author>
++    <author>
++      <name>Steven Dillmann</name>
++    </author>
++    <author>
++      <name>Akshay Anand</name>
++    </author>
++    <author>
++      <name>Andrew Lanpouthakoun</name>
++    </author>
++    <author>
++      <name>Bardia Koopah</name>
++    </author>
++    <author>
++      <name>Changran Hu</name>
++    </author>
++    <author>
++      <name>Etash Guha</name>
++    </author>
++    <author>
++      <name>Gabriel H. S. Dreiman</name>
++    </author>
++    <author>
++      <name>Jiacheng Zhu</name>
++    </author>
++    <author>
++      <name>Karl Krauth</name>
++    </author>
++    <author>
++      <name>Li Zhong</name>
++    </author>
++    <author>
++      <name>Niklas Muennighoff</name>
++    </author>
++    <author>
++      <name>Robert Amanfu</name>
++    </author>
++    <author>
++      <name>Shangyin Tan</name>
++    </author>
++    <author>
++      <name>Shreyas Pimpalgaonkar</name>
++    </author>
++    <author>
++      <name>Tushar Aggarwal</name>
++    </author>
++    <author>
++      <name>Xiangning Lin</name>
++    </author>
++    <author>
++      <name>Xin Lan</name>
++    </author>
++    <author>
++      <name>Xuandong Zhao</name>
++    </author>
++    <author>
++      <name>Yiqing Liang</name>
++    </author>
++    <author>
++      <name>Yuanli Wang</name>
++    </author>
++    <author>
++      <name>Zilong Wang</name>
++    </author>
++    <author>
++      <name>Changzhi Zhou</name>
++    </author>
++    <author>
++      <name>David Heineman</name>
++    </author>
++    <author>
++      <name>Hange Liu</name>
++    </author>
++    <author>
++      <name>Harsh Trivedi</name>
++    </author>
++    <author>
++      <name>John Yang</name>
++    </author>
++    <author>
++      <name>Junhong Lin</name>
++    </author>
++    <author>
++      <name>Manish Shetty</name>
++    </author>
++    <author>
++      <name>Michael Yang</name>
++    </author>
++    <author>
++      <name>Nabil Omi</name>
++    </author>
++    <author>
++      <name>Negin Raoof</name>
++    </author>
++    <author>
++      <name>Shanda Li</name>
++    </author>
++    <author>
++      <name>Terry Yue Zhuo</name>
++    </author>
++    <author>
++      <name>Wuwei Lin</name>
++    </author>
++    <author>
++      <name>Yiwei Dai</name>
++    </author>
++    <author>
++      <name>Yuxin Wang</name>
++    </author>
++    <author>
++      <name>Wenhao Chai</name>
++    </author>
++    <author>
++      <name>Shang Zhou</name>
++    </author>
++    <author>
++      <name>Dariush Wahdany</name>
++    </author>
++    <author>
++      <name>Ziyu She</name>
++    </author>
++    <author>
++      <name>Jiaming Hu</name>
++    </author>
++    <author>
++      <name>Zhikang Dong</name>
++    </author>
++    <author>
++      <name>Yuxuan Zhu</name>
++    </author>
++    <author>
++      <name>Sasha Cui</name>
++    </author>
++    <author>
++      <name>Ahson Saiyed</name>
++    </author>
++    <author>
++      <name>Arinbjörn Kolbeinsson</name>
++    </author>
++    <author>
++      <name>Jesse Hu</name>
++    </author>
++    <author>
++      <name>Christopher Michael Rytting</name>
++    </author>
++    <author>
++      <name>Ryan Marten</name>
++    </author>
++    <author>
++      <name>Yixin Wang</name>
++    </author>
++    <author>
++      <name>Alex Dimakis</name>
++    </author>
++    <author>
++      <name>Andy Konwinski</name>
++    </author>
++    <author>
++      <name>Ludwig Schmidt</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2601_18724 +33 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/BEECHdzZD9AQ8hbh1H+/QwhYKyw</id>
++  <title>arXiv Query: search_query=&amp;id_list=2601.18724&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:15:08Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2601.18724" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2601.18724v1</id>
++    <title>HalluCitation Matters: Revealing the Impact of Hallucinated References with 300 Hallucinated Papers in ACL Conferences</title>
++    <updated>2026-01-26T17:48:23Z</updated>
++    <link href="https://arxiv.org/abs/2601.18724v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2601.18724v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Recently, we have often observed hallucinated citations or references that do not correspond to any existing work in papers under review, preprints, or published papers. Such hallucinated citations pose a serious concern to scientific reliability. When they appear in accepted papers, they may also negatively affect the credibility of conferences. In this study, we refer to hallucinated citations as "HalluCitation" and systematically investigate their prevalence and impact. We analyze all papers published at ACL, NAACL, and EMNLP in 2024 and 2025, including main conference, Findings, and workshop papers. Our analysis reveals that nearly 300 papers contain at least one HalluCitation, most of which were published in 2025. Notably, half of these papers were identified at EMNLP 2025, the most recent conference, indicating that this issue is rapidly increasing. Moreover, more than 100 such papers were accepted as main conference and Findings papers at EMNLP 2025, affecting the credibility.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.DL" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2026-01-26T17:48:23Z</published>
++    <arxiv:comment>Work In Progress</arxiv:comment>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Yusuke Sakai</name>
++    </author>
++    <author>
++      <name>Hidetaka Kamigaito</name>
++    </author>
++    <author>
++      <name>Taro Watanabe</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2602_08990 +193 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/6MWp83nOAjfMEMKMQ+vediHWupw</id>
++  <title>arXiv Query: search_query=&amp;id_list=2602.08990&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:13:44Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2602.08990" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2602.08990v1</id>
++    <title>InternAgent-1.5: A Unified Agentic Framework for Long-Horizon Autonomous Scientific Discovery</title>
++    <updated>2026-02-09T18:36:06Z</updated>
++    <link href="https://arxiv.org/abs/2602.08990v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2602.08990v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>We introduce InternAgent-1.5, a unified system designed for end-to-end scientific discovery across computational and empirical domains. The system is built on a structured architecture composed of three coordinated subsystems for generation, verification, and evolution. These subsystems are supported by foundational capabilities for deep research, solution optimization, and long horizon memory. The architecture allows InternAgent-1.5 to operate continuously across extended discovery cycles while maintaining coherent and improving behavior. It also enables the system to coordinate computational modeling and laboratory experimentation within a single unified system. We evaluate InternAgent-1.5 on scientific reasoning benchmarks such as GAIA, HLE, GPQA, and FrontierScience, and the system achieves leading performance that demonstrates strong foundational capabilities. Beyond these benchmarks, we further assess two categories of discovery tasks. In algorithm discovery tasks, InternAgent-1.5 autonomously designs competitive methods for core machine learning problems. In empirical discovery tasks, it executes complete computational or wet lab experiments and produces scientific findings in earth, life, biological, and physical domains. Overall, these results show that InternAgent-1.5 provides a general and scalable framework for autonomous scientific discovery.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2026-02-09T18:36:06Z</published>
++    <arxiv:comment>Code and project page: https://github.com/InternScience/InternAgent</arxiv:comment>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Shiyang Feng</name>
++    </author>
++    <author>
++      <name>Runmin Ma</name>
++    </author>
++    <author>
++      <name>Xiangchao Yan</name>
++    </author>
++    <author>
++      <name>Yue Fan</name>
++    </author>
++    <author>
++      <name>Yusong Hu</name>
++    </author>
++    <author>
++      <name>Songtao Huang</name>
++    </author>
++    <author>
++      <name>Shuaiyu Zhang</name>
++    </author>
++    <author>
++      <name>Zongsheng Cao</name>
++    </author>
++    <author>
++      <name>Tianshuo Peng</name>
++    </author>
++    <author>
++      <name>Jiakang Yuan</name>
++    </author>
++    <author>
++      <name>Zijie Guo</name>
++    </author>
++    <author>
++      <name>Zhijie Zhong</name>
++    </author>
++    <author>
++      <name>Shangheng Du</name>
++    </author>
++    <author>
++      <name>Weida Wang</name>
++    </author>
++    <author>
++      <name>Jinxin Shi</name>
++    </author>
++    <author>
++      <name>Yuhao Zhou</name>
++    </author>
++    <author>
++      <name>Xiaohan He</name>
++    </author>
++    <author>
++      <name>Zhiyin Yu</name>
++    </author>
++    <author>
++      <name>Fangchen Yu</name>
++    </author>
++    <author>
++      <name>Qihao Zheng</name>
++    </author>
++    <author>
++      <name>Jiamin Wu</name>
++    </author>
++    <author>
++      <name>Mianxin Liu</name>
++    </author>
++    <author>
++      <name>Chi Zhang</name>
++    </author>
++    <author>
++      <name>Shaowei Hou</name>
++    </author>
++    <author>
++      <name>Shuya Li</name>
++    </author>
++    <author>
++      <name>Yankai Jiang</name>
++    </author>
++    <author>
++      <name>Wenjie Lou</name>
++    </author>
++    <author>
++      <name>Lilong Wang</name>
++    </author>
++    <author>
++      <name>Zifu Wang</name>
++    </author>
++    <author>
++      <name>Jiong Wang</name>
++    </author>
++    <author>
++      <name>Wanghan Xu</name>
++    </author>
++    <author>
++      <name>Yue Deng</name>
++    </author>
++    <author>
++      <name>Dongrui Liu</name>
++    </author>
++    <author>
++      <name>Yiheng Wang</name>
++    </author>
++    <author>
++      <name>Wenlong Zhang</name>
++    </author>
++    <author>
++      <name>Fenghua Ling</name>
++    </author>
++    <author>
++      <name>Shufei Zhang</name>
++    </author>
++    <author>
++      <name>Xiaosong Wang</name>
++    </author>
++    <author>
++      <name>Shuangjia Zheng</name>
++    </author>
++    <author>
++      <name>Xun Huang</name>
++    </author>
++    <author>
++      <name>Siqi Sun</name>
++    </author>
++    <author>
++      <name>Shuyue Hu</name>
++    </author>
++    <author>
++      <name>Peng Ye</name>
++    </author>
++    <author>
++      <name>Chunfeng Song</name>
++    </author>
++    <author>
++      <name>Bin Wang</name>
++    </author>
++    <author>
++      <name>Conghui He</name>
++    </author>
++    <author>
++      <name>Yihao Liu</name>
++    </author>
++    <author>
++      <name>Xin Li</name>
++    </author>
++    <author>
++      <name>Qibin Hou</name>
++    </author>
++    <author>
++      <name>Tao Chen</name>
++    </author>
++    <author>
++      <name>Xiangyu Yue</name>
++    </author>
++    <author>
++      <name>Bin Wang</name>
++    </author>
++    <author>
++      <name>Liang He</name>
++    </author>
++    <author>
++      <name>Dahua Lin</name>
++    </author>
++    <author>
++      <name>Bowen Zhou</name>
++    </author>
++    <author>
++      <name>Bo Zhang</name>
++    </author>
++    <author>
++      <name>Lei Bai</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2603_08127 +63 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/Ffl2jx054sXg7ssr61+juZ6ZFz4</id>
++  <title>arXiv Query: search_query=&amp;id_list=2603.08127&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:22:51Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2603.08127" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2603.08127v2</id>
++    <title>EvoScientist: Towards Multi-Agent Evolving AI Scientists for End-to-End Scientific Discovery</title>
++    <updated>2026-10-05T12:58:32Z</updated>
++    <link href="https://arxiv.org/abs/2603.08127v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2603.08127v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>The increasing adoption of Large Language Models (LLMs) has enabled AI scientists to perform complex end-to-end scientific discovery tasks requiring coordination of specialized roles, including idea generation and experimental execution. However, most state-of-the-art AI scientist systems rely on static, hand-designed pipelines and fail to adapt based on accumulated interaction histories. As a result, these systems overlook promising research directions, repeat failed experiments, and pursue infeasible ideas. To address this, we introduce EvoScientist, an evolving multi-agent AI scientist framework that continuously improves research strategies through persistent memory and self-evolution. EvoScientist comprises three specialized agents: a Researcher Agent (RA) for scientific idea generation, an Engineer Agent (EA) for experiment implementation and execution, and an Evolution Manager Agent (EMA) that distills insights from prior interactions into reusable knowledge. EvoScientist contains two persistent memory modules: (i) an ideation memory, which summarizes feasible research directions from top-ranked ideas while recording previously unsuccessful directions; and (ii) an experimentation memory, which captures effective data processing and model training strategies derived from code search trajectories and best-performing implementations. These modules enable the RA and EA to retrieve relevant prior strategies, improving idea quality and code execution success rates over time. Experiments show that EvoScientist outperforms 7 open-source and commercial state-of-the-art systems in scientific idea generation, achieving higher novelty, feasibility, relevance, and clarity via automatic and human evaluation. EvoScientist also substantially improves code execution success rates through multi-agent evolution, demonstrating persistent memory's effectiveness for end-to-end scientific discovery.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2026-03-09T09:07:19Z</published>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Yougang Lyu</name>
++    </author>
++    <author>
++      <name>Xi Zhang</name>
++    </author>
++    <author>
++      <name>Yuyue Zhao</name>
++    </author>
++    <author>
++      <name>Konstantinos Papakostas</name>
++    </author>
++    <author>
++      <name>Xinhao Yi</name>
++    </author>
++    <author>
++      <name>Shuyu Guo</name>
++    </author>
++    <author>
++      <name>Wenxiang Hu</name>
++    </author>
++    <author>
++      <name>Jan Piotrowski</name>
++    </author>
++    <author>
++      <name>Jakub Kaliski</name>
++    </author>
++    <author>
++      <name>Jacopo Urbani</name>
++    </author>
++    <author>
++      <name>Zaiqiao Meng</name>
++    </author>
++    <author>
++      <name>Lun Zhou</name>
++    </author>
++    <author>
++      <name>Zhaochun Ren</name>
++    </author>
++    <author>
++      <name>Xiaohui Yan</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2603_29231 +43 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/wPKPQbnAEZBIso82g6cq6F7L2F4</id>
++  <title>arXiv Query: search_query=&amp;id_list=2603.29231&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:15:09Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2603.29231" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2603.29231v1</id>
++    <title>Beyond pass@1: A Reliability Science Framework for Long-Horizon LLM Agents</title>
++    <updated>2026-03-31T03:56:39Z</updated>
++    <link href="https://arxiv.org/abs/2603.29231v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2603.29231v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Existing benchmarks measure capability -- whether a model succeeds on a single attempt -- but production deployments
++  require reliability -- consistent success across repeated attempts on tasks of varying duration. We show these
++  properties diverge systematically as task duration grows, and that pass@1 on short tasks is structurally blind to
++  this divergence.
++  We introduce a reliability science framework for long-horizon LLM agents with four metrics: Reliability Decay Curve
++  (RDC), Variance Amplification Factor (VAF), Graceful Degradation Score (GDS), and Meltdown Onset Point (MOP). We
++  evaluate 10 models across 23,392 episodes on a 396-task benchmark spanning four duration buckets and three domains.
++  Key findings: (1) reliability decay is domain-stratified -- SE GDS drops from 0.90 to 0.44 while document processing
++  is nearly flat (0.74 to 0.71); (2) VAF bifurcates by capability tier -- high VAF is a capability signature, not an
++  instability signal; (3) capability and reliability rankings diverge substantially, with multi-rank inversions at long
++  horizons; (4) frontier models have the highest meltdown rates (up to 19%) because they attempt ambitious multi-step
++  strategies that sometimes spiral; and (5) memory scaffolds universally hurt long-horizon performance across all 10
++  models. These results motivate reliability as a first-class evaluation dimension alongside capability.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2026-03-31T03:56:39Z</published>
++    <arxiv:comment>23 pages, 4 figures</arxiv:comment>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Aaditya Khanal</name>
++    </author>
++    <author>
++      <name>Yangyang Tao</name>
++    </author>
++    <author>
++      <name>Junxiu Zhou</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2605_07723 +42 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/nkWVFpiObfs4p7+H+3SuR6f0niw</id>
++  <title>arXiv Query: search_query=&amp;id_list=2605.07723&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:15:08Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2605.07723" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2605.07723v1</id>
++    <title>LLM hallucinations in the wild: Large-scale evidence from non-existent citations</title>
++    <updated>2026-05-08T13:26:41Z</updated>
++    <link href="https://arxiv.org/abs/2605.07723v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2605.07723v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Large language models (LLMs) are known to generate plausible but false information across a wide range of contexts, yet the real-world magnitude and consequences of this hallucination problem remain poorly understood. Here we leverage a uniquely verifiable object - scientific citations - to audit 111 million references across 2.5 million papers in arXiv, bioRxiv, SSRN, and PubMed Central. We find a sharp rise in non-existent references following widespread LLM adoption, with a conservative estimate of 146,932 hallucinated citations in 2025 alone. These errors are diffusely embedded across many papers but especially pronounced in fields with rapid AI uptake, in manuscripts with linguistic signatures of AI-assisted writing, and among small and early-career author teams. At the same time, hallucinated references disproportionately assign credit to already prominent and male scholars, suggesting that LLM-generated errors may reinforce existing inequities in scientific recognition. Preprint moderation and journal publication processes capture only a fraction of these errors, suggesting that the spread of hallucinated content has outpaced existing safeguards. Together, these findings demonstrate that LLM hallucinations are infiltrating knowledge production at scale, threatening both the reliability and equity of future scientific discovery as human and AI systems draw on the existing literature.</summary>
++    <category term="cs.DL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CY" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="physics.soc-ph" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2026-05-08T13:26:41Z</published>
++    <arxiv:primary_category term="cs.DL"/>
++    <author>
++      <name>Zhenyue Zhao</name>
++    </author>
++    <author>
++      <name>Yihe Wang</name>
++    </author>
++    <author>
++      <name>Toby Stuart</name>
++    </author>
++    <author>
++      <name>Mathijs De Vaan</name>
++    </author>
++    <author>
++      <name>Paul Ginsparg</name>
++    </author>
++    <author>
++      <name>Yian Yin</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2605_25415 +47 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/hb8fuhkRXA9MbX0aJQecvY20sCI</id>
++  <title>arXiv Query: search_query=&amp;id_list=2605.25415&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:16:08Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2605.25415" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2605.25415v1</id>
++    <title>LLM-as-a-Reviewer: Benchmarking Their Ability, Divergence, and Prompt Injection Resistance as Paper Reviewers</title>
++    <updated>2026-05-25T04:32:13Z</updated>
++    <link href="https://arxiv.org/abs/2605.25415v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2605.25415v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Large language models (LLMs) are increasingly used in academic peer review, yet their reliability, alignment with human judgment, and robustness to adversarial attacks remain poorly understood. We present a systematic benchmark of LLM-as-a-Reviewer on 898 papers stratified from NeurIPS and ICLR, evaluating 12 LLMs along three axes: rating calibration, divergence from human reviewers, and resistance to prompt injection embedded via an invisible font-mapping attack. We find that LLMs systematically overrate weaker submissions and diverge from humans in topical emphasis, under-flagging Clarity and over-flagging Reproducibility, while producing reviews two to three times longer with lower lexical diversity and a more standardized vocabulary. Prompt injection remains highly effective. Simple hidden instructions can promote low-scoring papers to acceptance-level ratings in a substantial fraction of cases, with effectiveness varying sharply across model families. While LLMs offer utility in structuring evaluations, their integration into peer review requires safeguards against both intrinsic biases and adversarial risks.</summary>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CY" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.ET" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2026-05-25T04:32:13Z</published>
++    <arxiv:primary_category term="cs.CL"/>
++    <author>
++      <name>Lingyao Li</name>
++    </author>
++    <author>
++      <name>Junjie Xiong</name>
++    </author>
++    <author>
++      <name>Changjia Zhu</name>
++    </author>
++    <author>
++      <name>Runlong Yu</name>
++    </author>
++    <author>
++      <name>Chen Chen</name>
++    </author>
++    <author>
++      <name>Junyu Wang</name>
++    </author>
++    <author>
++      <name>Renkai Ma</name>
++    </author>
++    <author>
++      <name>Zhicong Lu</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2605_26340 +63 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/6LcS4L7FKpRCTqW+gyVahwyvWb0</id>
++  <title>arXiv Query: search_query=&amp;id_list=2605.26340&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:48:45Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2605.26340" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2605.26340v1</id>
++    <title>ScientistOne: Towards Human-Level Autonomous Research via Chain-of-Evidence</title>
++    <updated>2026-05-25T21:30:27Z</updated>
++    <link href="https://arxiv.org/abs/2605.26340v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2605.26340v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Autonomous research agents produce competitive solutions and professional-looking manuscripts, yet their outputs contain verifiability failures undetectable by surface-level evaluation: fabricated citations, unreproducible scores, and method descriptions that diverge from the implementation. We address this through three contributions. First, Chain-of-Evidence (CoE), a verifiability framework requiring every claim to be traceable to its evidence source. Second, ScientistOne, an end-to-end autonomous research system that maintains evidence chains by construction throughout literature review, solution discovery, and paper writing. Third, CoE Audit, a post-hoc audit whose four integrity checks -- score verification, specification violation, reference verification, and method-code alignment -- apply uniformly to all systems. Across 75 papers spanning five systems and five frontier research tasks, every baseline exhibits at least one systematic failure mode: hallucinated reference rates reach 21%, score verification passes in as few as 42% of papers, and method-code alignment ranges from 20% to 80%. ScientistOne achieves zero hallucinated references (0/337), perfect score verification (12/12), and the highest method-code alignment (14/15), while matching or exceeding human expert performance on all five tasks. ScientistOne further generalizes to six additional tasks spanning medical imaging, fine-grained recognition, 3D perception, and language modeling, achieving state-of-the-art on Parameter Golf and gold medals on MLE-Bench tasks where baselines fail entirely.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.CL" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.MA" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2026-05-25T21:30:27Z</published>
++    <arxiv:comment>Project website: https://scientist-one.github.io/</arxiv:comment>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Rui Meng</name>
++    </author>
++    <author>
++      <name>Bhavana Dalvi Mishra</name>
++    </author>
++    <author>
++      <name>Jiefeng Chen</name>
++    </author>
++    <author>
++      <name>Chun-Liang Li</name>
++    </author>
++    <author>
++      <name>Palash Goyal</name>
++    </author>
++    <author>
++      <name>Mihir Parmar</name>
++    </author>
++    <author>
++      <name>Yiwen Song</name>
++    </author>
++    <author>
++      <name>Yale Song</name>
++    </author>
++    <author>
++      <name>Rajarishi Sinha</name>
++    </author>
++    <author>
++      <name>Parthasarathy Ranganathan</name>
++    </author>
++    <author>
++      <name>Burak Gokturk</name>
++    </author>
++    <author>
++      <name>Jinsung Yoon</name>
++    </author>
++    <author>
++      <name>Tomas Pfister</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2607_12301 +25 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/4okGerKwTJy5NNsunrAEB6gJM0M</id>
++  <title>arXiv Query: search_query=&amp;id_list=2607.12301&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T02:50:43Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2607.12301" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2607.12301v2</id>
++    <title>XScientist: A Git-Like Research Protocol for Long-Running Autonomous Scientific Discovery</title>
++    <updated>2026-09-22T10:01:44Z</updated>
++    <link href="https://arxiv.org/abs/2607.12301v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2607.12301v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Autonomous research systems can generate plausible papers while losing the decisions, failed branches, and evidence needed to inspect or continue the work. We present XScientist, a local-first, git-like protocol that treats research state, rather than a manuscript, as the unit of continuation. Hypotheses, experiment attempts, observations, claims, reviews, and handoffs are represented as typed, content-addressed objects in an exploration graph. Immutable checkpoints, explicit negative outcomes, claim-evidence closure, replay boundaries, and authority-aware gates make each transition inspectable without treating a passing integrity check as scientific truth. The protocol exports a portable Agent-Native Research Artifact (ARA) that another agent or human can inspect, fork, verify, and extend. A reference implementation integrates planning, execution, review, repair, and supervised long-running operation while preserving provenance across these stages. We evaluate the protocol with controlled artifact-integrity workloads and matched external task pilots, keeping native task performance separate from evidence and audit claims. The result is an interoperability and accountability layer for long-running autonomous science, with explicit boundaries where human judgment and independent evaluation remain necessary.</summary>
++    <category term="cs.SE" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.MA" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2026-07-14T03:20:11Z</published>
++    <arxiv:primary_category term="cs.SE"/>
++    <author>
++      <name>Jixiang Luo</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2608_05179 +34 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/YF9P0tyn0k/cK6J4jyaxu8fXO1s</id>
++  <title>arXiv Query: search_query=&amp;id_list=2608.05179&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:13:43Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2608.05179" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2608.05179v1</id>
++    <title>Autonomous Research Agents: A Survey of AI Scientists and the Verification Gap</title>
++    <updated>2026-06-29T01:32:49Z</updated>
++    <link href="https://arxiv.org/abs/2608.05179v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2608.05179v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Large language model (LLM) agents are increasingly used across the scientific research lifecycle: ideation, literature search, experiment design and execution, analysis, manuscript drafting, and review. End-to-end AI scientist systems can now produce paper-like manuscripts, but their claims are often harder to verify than their code is to run. This survey studies that gap in computational AI/ML research, where code, benchmarks, experiments, and write-ups are most visible. We screen 125 candidate works and include 35, with full-text coding of 26 entries: 24 runnable systems and two study or position papers. We code seven audit dimensions: lifecycle stage, autonomy level, evaluation method, released artifacts, human-in-the-loop points, novelty verification, and result-selection disclosure. The main pattern is that code release is now common, but reproducibility-grade and claim-verification artifacts remain much less common. In the 24 runnable systems, 83 percent release code, while 38 percent release seeds or execution traces and 38 percent report any novelty-verification method. Among nine closed-loop L4 systems, seven are mechanical reruns and one is author-claimed without an external check; no LLM-era system in the corpus demonstrates an externally validated in-loop oracle under our coding rule. We contribute a coded corpus, a lifecycle-by-autonomy map, an auditability-gap analysis, and a reviewer-facing reporting checklist. The survey argues that the field's central bottleneck is no longer only whether agents can complete research tasks, but whether reviewers can verify the claims those agents produce.</summary>
++    <category term="cs.CY" scheme="http://arxiv.org/schemas/atom"/>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2026-06-29T01:32:49Z</published>
++    <arxiv:primary_category term="cs.CY"/>
++    <author>
++      <name>Tianyu Ding</name>
++    </author>
++    <author>
++      <name>Aditya Nannapaneni</name>
++    </author>
++    <author>
++      <name>Bingfan Liu</name>
++    </author>
++    <author>
++      <name>Ling Zhang</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2608_14380 +40 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/DQMTVoqKFnHc4AOkEj9cCNWcsi8</id>
++  <title>arXiv Query: search_query=&amp;id_list=2608.14380&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:15:09Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2608.14380" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2608.14380v1</id>
++    <title>AgentRewind: Recoverable Execution for Long-Horizon LLM Agents</title>
++    <updated>2026-08-14T15:20:35Z</updated>
++    <link href="https://arxiv.org/abs/2608.14380v1" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2608.14380v1" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Many real-world tasks require LLM agents to interact with their environments over long execution horizons. Errors that occur early in execution may propagate through both the agent context and environment state, and their effects may be difficult to reverse through subsequent actions. Existing methods mainly seek to reduce such errors through plan refinement and safety checks but provide little support after errors occur. To enable recovery during long-horizon execution, we present AgentRewind, a runtime recovery framework that records aligned checkpoints of the agent context and controlled environment, allowing agents to return to an earlier state and resume execution with information from previous attempts. We also construct MettleBench, a benchmark for evaluating task completion and partial progress on long-horizon engineering assignments containing a series of related requirements. Experiments across tasks, multiple models, execution strategies, and agent harnesses show that AgentRewind improves task success rate and average checklist progress over the compared baselines.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2026-08-14T15:20:35Z</published>
++    <arxiv:comment>19 pages, 5 figures</arxiv:comment>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Yu Zhuang</name>
++    </author>
++    <author>
++      <name>Kefei Chen</name>
++    </author>
++    <author>
++      <name>Yitong Duan</name>
++    </author>
++    <author>
++      <name>Shuxin Zheng</name>
++    </author>
++    <author>
++      <name>Jian Li</name>
++    </author>
++    <author>
++      <name>Xu-Yao Zhang</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/https_export_arxiv_org_api_query_id_list_2608_21374 +52 −0

++<?xml version='1.0' encoding='UTF-8'?>
++<feed xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:arxiv="http://arxiv.org/schemas/atom" xmlns="http://www.w3.org/2005/Atom">
++  <id>https://arxiv.org/api/yyn9mIsFaMQFjRqGojhzT2q2gXo</id>
++  <title>arXiv Query: search_query=&amp;id_list=2608.21374&amp;start=0&amp;max_results=10</title>
++  <updated>2026-10-30T03:16:09Z</updated>
++  <link href="https://arxiv.org/api/query?search_query=&amp;start=0&amp;max_results=10&amp;id_list=2608.21374" type="application/atom+xml"/>
++  <opensearch:itemsPerPage>10</opensearch:itemsPerPage>
++  <opensearch:totalResults>1</opensearch:totalResults>
++  <opensearch:startIndex>0</opensearch:startIndex>
++  <entry>
++    <id>http://arxiv.org/abs/2608.21374v2</id>
++    <title>LitReview Arena: Evaluating Literature Review Agents with Battle-Style Peer Review Platform</title>
++    <updated>2026-09-28T10:26:22Z</updated>
++    <link href="https://arxiv.org/abs/2608.21374v2" rel="alternate" type="text/html"/>
++    <link href="https://arxiv.org/pdf/2608.21374v2" rel="related" type="application/pdf" title="pdf"/>
++    <summary>Literature reviews are essential to scientific progress, but rigorously evaluating automatically generated reviews remains difficult because many aspects of research utility depend on expert judgment rather than reference-overlap metrics. We introduce LitReview Arena, a battle-style evaluation platform with a structured protocol tailored to literature review quality: domain experts with AI paper-writing experience compare anonymized drafts, are matched to topics within their expertise, and provide dimension-wise outcomes over five literature-review-specific criteria. From this protocol, we collect approximately 3k expert judgments, each containing five dimension-wise outcomes, and show that even the strongest current systems win only 23.0% of decisive matches against human drafts on overall utility, while agentic LLMs such as Sonar Deep Research substantially outperform base language models by over 60%. We further find that existing LLM-as-a-judge methods are substantially misaligned with human experts (Spearman's rho=0.467), especially on synthesis-heavy criteria such as paper structure and research suggestions. Using the collected preference data, we provide an expert-calibrated evaluator, LitJudge, which improves alignment to Spearman's rho=0.78, comparable to inter-expert consistency; code and data are publicly available at https://github.com/VanellopeAsher/LitReview-Arena.</summary>
++    <category term="cs.AI" scheme="http://arxiv.org/schemas/atom"/>
++    <published>2026-07-01T14:54:07Z</published>
++    <arxiv:comment>20 pages, ICML 2026</arxiv:comment>
++    <arxiv:primary_category term="cs.AI"/>
++    <author>
++      <name>Ruotong Zhao</name>
++    </author>
++    <author>
++      <name>Zhiyu Chen</name>
++    </author>
++    <author>
++      <name>Xurui Liu</name>
++    </author>
++    <author>
++      <name>Haidong Xue</name>
++    </author>
++    <author>
++      <name>Dong Liang</name>
++    </author>
++    <author>
++      <name>Jigao Fu</name>
++    </author>
++    <author>
++      <name>Wu YanBiao</name>
++    </author>
++    <author>
++      <name>Yuanyi Zhen</name>
++    </author>
++    <author>
++      <name>Fengli Xu</name>
++    </author>
++    <author>
++      <name>Yong Li</name>
++    </author>
++  </entry>
++</feed>

added research/verify_cache/s2_dapper2010.json +46 −0

++{
++ "paperId": "003d5a65de0ac72daaf105ded903cb3eb88585b3",
++ "externalIds": {
++  "MAG": "2171867449",
++  "CorpusId": 14271421
++ },
++ "publicationVenue": null,
++ "title": "Dapper, a Large-Scale Distributed Systems Tracing Infrastructure",
++ "venue": "",
++ "year": 2010,
++ "authors": [
++  {
++   "authorId": "2084178",
++   "name": "B. Sigelman"
++  },
++  {
++   "authorId": "1746646",
++   "name": "L. Barroso"
++  },
++  {
++   "authorId": "32623151",
++   "name": "M. Burrows"
++  },
++  {
++   "authorId": "145390556",
++   "name": "Patrick Stephenson"
++  },
++  {
++   "authorId": "2114994",
++   "name": "M. Plakal"
++  },
++  {
++   "authorId": "144754014",
++   "name": "Donald Beaver"
++  },
++  {
++   "authorId": "70349407",
++   "name": "Saul Jaspan"
++  },
++  {
++   "authorId": "70051750",
++   "name": "C. Shanbhag"
++  }
++ ],
++ "_note": "Record from https://api.semanticscholar.org/graph/v1/paper/search?query=Dapper%20a%20Large-Scale%20Distributed%20Systems%20Tracing%20Infrastructure (HTTP 200) on 2026-10-30 UTC"
++}
+ \ No newline at end of file

added research/verify_cache/s2_gray1986.json +1 −0

			`@@ -0,0 +1 @@`
	1	+{"message": "Too Many Requests. Please wait and try again or apply for a key for higher rate limits. https://www.semanticscholar.org/product/api#api-key-form", "code": "429"}
\ No newline at end of file

added research/verify_cache/s2_gray1986_search_observed.json +5 −0

++{"_note": "Record observed on 2026-10-30 (UTC) via https://api.semanticscholar.org/graph/v1/paper/search?query=Why%20do%20computers%20stop%20and%20what%20can%20be%20done%20about%20it (HTTP 200); transcribed from the response because the by-id fetch was rate-limited (HTTP 429) afterwards.",
++ "paperId": "87c9e34c39dd1fbd470010dc1ea98ba3517f5a2a", "year": 1986,
++ "venue": "Symposium on Reliability in Distributed Software and Database Systems",
++ "authors": [{"name": "Jim Gray"}], "title": "Why Do Computers Stop and What Can Be Done About It?",
++ "externalIds": {"MAG": "2144984544", "DBLP": "conf/srds/Gray86", "CorpusId": 10364748}}

added research/verify_cache/s2_oppenheimer2003.json +1 −0

			`@@ -0,0 +1 @@`
	1	+{"message": "Too Many Requests. Please wait and try again or apply for a key for higher rate limits. https://www.semanticscholar.org/product/api#api-key-form", "code": "429"}
\ No newline at end of file

added research/verify_cache/s2_search_extra.jsonl +2 −0

++{"message": "Too Many Requests. Please wait and try again or apply for a key for higher rate limits. https://www.semanticscholar.org/product/api#api-key-form", "code": "429"}{"message": "Too Many Requests. Please wait and try again or apply for a key for higher rate limits. https://www.semanticscholar.org/product/api#api-key-form", "code": "429"}{"message": "Too Many Requests. Please wait and try again or apply for a key for higher rate limits. https://www.semanticscholar.org/product/api#api-key-form", "code": "429"}{"message": "Too Many Requests. Please wait and try again or apply for a key for higher rate limits. https://www.semanticscholar.org/product/api#api-key-form", "code": "429"}{"total": 784, "offset": 0, "next": 2, "data": [{"paperId": "003d5a65de0ac72daaf105ded903cb3eb88585b3", "externalIds": {"MAG": "2171867449", "CorpusId": 14271421}, "publicationVenue": null, "title": "Dapper, a Large-Scale Distributed Systems Tracing Infrastructure", "venue": "", "year": 2010, "authors": [{"authorId": "2084178", "name": "B. Sigelman"}, {"authorId": "1746646", "name": "L. Barroso"}, {"authorId": "32623151", "name": "M. Burrows"}, {"authorId": "145390556", "name": "Patrick Stephenson"}, {"authorId": "2114994", "name": "M. Plakal"}, {"authorId": "144754014", "name": "Donald Beaver"}, {"authorId": "70349407", "name": "Saul Jaspan"}, {"authorId": "70051750", "name": "C. Shanbhag"}]}, {"paperId": "6d67321536f6b91d333c2c6a189f9f75a29dfbd7", "externalIds": {"ArXiv": "2609.23301", "CorpusId": 292218691}, "publicationVenue": null, "title": "Accurate Distributed Tracing for Large-Scale AI Infrastructure: Time Synchronization as a Foundation for Reliable Observability", "venue": "", "year": 2026, "authors": [{"authorId": "7395639", "name": "Hesham Elbakoury"}, {"authorId": "2396535812", "name": "Ankur Sharma"}]}]}
++{"message": "Too Many Requests. Please wait and try again or apply for a key for higher rate limits. https://www.semanticscholar.org/product/api#api-key-form", "code": "429"}
+ \ No newline at end of file

added research/verify_cache/s2_yuan2014.json +1 −0

			`@@ -0,0 +1 @@`
	1	+{"message": "Too Many Requests. Please wait and try again or apply for a key for higher rate limits. https://www.semanticscholar.org/product/api#api-key-form", "code": "429"}
\ No newline at end of file

added research/verify_cache/semanticscholar_batch.json +12727 −0

++{
++ "arXiv:1902.03245": {
++  "paperId": "67a0d44650f5b7944fc5bc069c87bffa44b6cba4",
++  "externalIds": {
++   "MAG": "2970030663",
++   "ArXiv": "1902.03245",
++   "DBLP": "journals/corr/abs-1902-03245",
++   "CorpusId": 60440397
++  },
++  "publicationVenue": {
++   "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",
++   "name": "Neural Information Processing Systems",
++   "type": "conference",
++   "alternate_names": [
++    "Neural Inf Process Syst",
++    "NeurIPS",
++    "NIPS"
++   ],
++   "url": "http://neurips.cc/"
++  },
++  "title": "Ask Not What AI Can Do, But What AI Should Do: Towards a Framework of Task Delegability",
++  "venue": "Neural Information Processing Systems",
++  "year": 2019,
++  "citationCount": 117,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/1902.03245, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/1902.03245"
++  },
++  "authors": [
++   {
++    "authorId": "70025737",
++    "name": "Brian Lubars"
++   },
++   {
++    "authorId": "40348583",
++    "name": "Chenhao Tan"
++   }
++  ],
++  "abstract": "While artificial intelligence (AI) holds promise for addressing societal challenges, issues of exactly which tasks to automate and to what extent to do so remain understudied. We approach this problem of task delegability from a human-centered perspective by developing a framework on human perception of task delegation to AI. We consider four high-level factors that can contribute to a delegation decision: motivation, difficulty, risk, and trust. To obtain an empirical understanding of human preferences in different tasks, we build a dataset of 100 tasks from academic papers, popular media portrayal of AI, and everyday life, and administer a survey based on our proposed framework. We find little preference for full AI control and a strong preference for machine-in-the-loop designs, in which humans play the leading role. Among the four factors, trust is the most correlated with human preferences of optimal human-machine delegation. This framework represents a first step towards characterizing human preferences of AI automation across tasks. We hope this work encourages future efforts towards understanding such individual attitudes; our goal is to inform the public and the AI research community rather than dictating any direction in technology development."
++ },
++ "arXiv:2005.11401": {
++  "paperId": "659bf9ce7175e1ec266ff54359e2bd76e0b7ff31",
++  "externalIds": {
++   "DBLP": "conf/nips/LewisPPPKGKLYR020",
++   "MAG": "3027879771",
++   "ArXiv": "2005.11401",
++   "CorpusId": 218869575
++  },
++  "publicationVenue": {
++   "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",
++   "name": "Neural Information Processing Systems",
++   "type": "conference",
++   "alternate_names": [
++    "Neural Inf Process Syst",
++    "NeurIPS",
++    "NIPS"
++   ],
++   "url": "http://neurips.cc/"
++  },
++  "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
++  "venue": "Neural Information Processing Systems",
++  "year": 2020,
++  "citationCount": 20321,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2005.11401, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2005.11401"
++  },
++  "authors": [
++   {
++    "authorId": "145222654",
++    "name": "Patrick Lewis"
++   },
++   {
++    "authorId": "3439053",
++    "name": "Ethan Perez"
++   },
++   {
++    "authorId": "120174856",
++    "name": "Aleksandra Piktus"
++   },
++   {
++    "authorId": "40052301",
++    "name": "F. Petroni"
++   },
++   {
++    "authorId": "2067091563",
++    "name": "Vladimir Karpukhin"
++   },
++   {
++    "authorId": "2519247661",
++    "name": "Naman Goyal"
++   },
++   {
++    "authorId": "103131985",
++    "name": "Heinrich Kuttler"
++   },
++   {
++    "authorId": "35084211",
++    "name": "M. Lewis"
++   },
++   {
++    "authorId": "144105277",
++    "name": "Wen-tau Yih"
++   },
++   {
++    "authorId": "2620211",
++    "name": "Tim Rockt\u00e4schel"
++   },
++   {
++    "authorId": "48662861",
++    "name": "Sebastian Riedel"
++   },
++   {
++    "authorId": "1743722",
++    "name": "Douwe Kiela"
++   }
++  ],
++  "abstract": "Large pre-trained language models have been shown to store factual knowledge in their parameters, and achieve state-of-the-art results when fine-tuned on downstream NLP tasks. However, their ability to access and precisely manipulate knowledge is still limited, and hence on knowledge-intensive tasks, their performance lags behind task-specific architectures. Additionally, providing provenance for their decisions and updating their world knowledge remain open research problems. Pre-trained models with a differentiable access mechanism to explicit non-parametric memory can overcome this issue, but have so far been only investigated for extractive downstream tasks. We explore a general-purpose fine-tuning recipe for retrieval-augmented generation (RAG) -- models which combine pre-trained parametric and non-parametric memory for language generation. We introduce RAG models where the parametric memory is a pre-trained seq2seq model and the non-parametric memory is a dense vector index of Wikipedia, accessed with a pre-trained neural retriever. We compare two RAG formulations, one which conditions on the same retrieved passages across the whole generated sequence, the other can use different passages per token. We fine-tune and evaluate our models on a wide range of knowledge-intensive NLP tasks and set the state-of-the-art on three open domain QA tasks, outperforming parametric seq2seq models and task-specific retrieve-and-extract architectures. For language generation tasks, we find that RAG models generate more specific, diverse and factual language than a state-of-the-art parametric-only seq2seq baseline."
++ },
++ "arXiv:2112.09332": {
++  "paperId": "2f3efe44083af91cef562c1a3451eee2f8601d22",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2112-09332",
++   "ArXiv": "2112.09332",
++   "CorpusId": 245329531
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "WebGPT: Browser-assisted question-answering with human feedback",
++  "venue": "arXiv.org",
++  "year": 2021,
++  "citationCount": 2173,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2112.09332, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2112.09332"
++  },
++  "authors": [
++   {
++    "authorId": "7406311",
++    "name": "Reiichiro Nakano"
++   },
++   {
++    "authorId": "2052366271",
++    "name": "Jacob Hilton"
++   },
++   {
++    "authorId": "2054519183",
++    "name": "S. Balaji"
++   },
++   {
++    "authorId": "49387725",
++    "name": "Jeff Wu"
++   },
++   {
++    "authorId": "31793034",
++    "name": "Long Ouyang"
++   },
++   {
++    "authorId": "2149054292",
++    "name": "Christina Kim"
++   },
++   {
++    "authorId": "144239765",
++    "name": "Christopher Hesse"
++   },
++   {
++    "authorId": "150298413",
++    "name": "Shantanu Jain"
++   },
++   {
++    "authorId": "13622184",
++    "name": "V. Kosaraju"
++   },
++   {
++    "authorId": "2058848938",
++    "name": "W. Saunders"
++   },
++   {
++    "authorId": "2115903168",
++    "name": "Xu Jiang"
++   },
++   {
++    "authorId": "6062736",
++    "name": "K. Cobbe"
++   },
++   {
++    "authorId": "2146257131",
++    "name": "Tyna Eloundou"
++   },
++   {
++    "authorId": "2064404342",
++    "name": "G. Krueger"
++   },
++   {
++    "authorId": "2146257251",
++    "name": "Kevin Button"
++   },
++   {
++    "authorId": "2146257375",
++    "name": "Matthew Knight"
++   },
++   {
++    "authorId": "1490681878",
++    "name": "Benjamin Chess"
++   },
++   {
++    "authorId": "47971768",
++    "name": "John D. Schulman"
++   }
++  ],
++  "abstract": "We fine-tune GPT-3 to answer long-form questions using a text-based web-browsing environment, which allows the model to search and navigate the web. By setting up the task so that it can be performed by humans, we are able to train models on the task using imitation learning, and then optimize answer quality with human feedback. To make human evaluation of factual accuracy easier, models must collect references while browsing in support of their answers. We train and evaluate our models on ELI5, a dataset of questions asked by Reddit users. Our best model is obtained by fine-tuning GPT-3 using behavior cloning, and then performing rejection sampling against a reward model trained to predict human preferences. This model's answers are preferred by humans 56% of the time to those of our human demonstrators, and 69% of the time to the highest-voted answer from Reddit."
++ },
++ "arXiv:2112.11471": {
++  "paperId": "449ff2404a7b11fa3c47cd228fa5469b00ffef20",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2112-11471",
++   "ArXiv": "2112.11471",
++   "CorpusId": 245385821
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Towards a Science of Human-AI Decision Making: A Survey of Empirical Studies",
++  "venue": "arXiv.org",
++  "year": 2021,
++  "citationCount": 258,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2112.11471, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2112.11471"
++  },
++  "authors": [
++   {
++    "authorId": "120801533",
++    "name": "V. Lai"
++   },
++   {
++    "authorId": "2131059444",
++    "name": "Cha-Cha Chen"
++   },
++   {
++    "authorId": "144921048",
++    "name": "Q. Liao"
++   },
++   {
++    "authorId": "1405364873",
++    "name": "Alison Smith-Renner"
++   },
++   {
++    "authorId": "2111727675",
++    "name": "Chenhao Tan"
++   }
++  ],
++  "abstract": "As AI systems demonstrate increasingly strong predictive performance, their adoption has grown in numerous domains. However, in high-stakes domains such as criminal justice and healthcare, full automation is often not desirable due to safety, ethical, and legal concerns, yet fully manual approaches can be inaccurate and time consuming. As a result, there is growing interest in the research community to augment human decision making with AI assistance. Besides developing AI technologies for this purpose, the emerging field of human-AI decision making must embrace empirical approaches to form a foundational understanding of how humans interact and work with AI to make decisions. To invite and help structure research efforts towards a science of understanding and improving human-AI decision making, we survey recent literature of empirical human-subject studies on this topic. We summarize the study design choices made in over 100 papers in three important aspects: (1) decision tasks, (2) AI models and AI assistance elements, and (3) evaluation metrics. For each aspect, we summarize current trends, discuss gaps in current practices of the field, and make a list of recommendations for future research. Our survey highlights the need to develop common frameworks to account for the design and research spaces of human-AI decision making, so that researchers can make rigorous choices in study design, and the research community can build on each other's work and produce generalizable scientific knowledge. We also hope this survey will serve as a bridge for HCI and AI communities to work together to mutually shape the empirical science and computational technologies for human-AI decision making."
++ },
++ "arXiv:2203.11147": {
++  "paperId": "8666f9f379389a5dff31e72fb0f992a37763ba41",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2203-11147",
++   "ArXiv": "2203.11147",
++   "DOI": "10.48550/arXiv.2203.11147",
++   "CorpusId": 247594830
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Teaching language models to support answers with verified quotes",
++  "venue": "arXiv.org",
++  "year": 2022,
++  "citationCount": 391,
++  "openAccessPdf": {
++   "url": "http://arxiv.org/pdf/2203.11147",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2203.11147, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2203.11147"
++  },
++  "authors": [
++   {
++    "authorId": "10698483",
++    "name": "Jacob Menick"
++   },
++   {
++    "authorId": "1994939814",
++    "name": "Maja Trebacz"
++   },
++   {
++    "authorId": "148305440",
++    "name": "Vladimir Mikulik"
++   },
++   {
++    "authorId": "9958912",
++    "name": "John Aslanides"
++   },
++   {
++    "authorId": "2107148568",
++    "name": "H. F. Song"
++   },
++   {
++    "authorId": "2159545857",
++    "name": "Martin J. Chadwick"
++   },
++   {
++    "authorId": "2143471164",
++    "name": "Mia Glaese"
++   },
++   {
++    "authorId": "116494324",
++    "name": "Susannah Young"
++   },
++   {
++    "authorId": "2151248210",
++    "name": "Lucy Campbell-Gillingham"
++   },
++   {
++    "authorId": "2060655766",
++    "name": "Geoffrey Irving"
++   },
++   {
++    "authorId": "147687624",
++    "name": "Nat McAleese"
++   }
++  ],
++  "abstract": "Recent large language models often answer factual questions correctly. But users can't trust any given claim a model makes without fact-checking, because language models can hallucinate convincing nonsense. In this work we use reinforcement learning from human preferences (RLHP) to train\"open-book\"QA models that generate answers whilst also citing specific evidence for their claims, which aids in the appraisal of correctness. Supporting evidence is drawn from multiple documents found via a search engine, or from a single user-provided document. Our 280 billion parameter model, GopherCite, is able to produce answers with high quality supporting evidence and abstain from answering when unsure. We measure the performance of GopherCite by conducting human evaluation of answers to questions in a subset of the NaturalQuestions and ELI5 datasets. The model's response is found to be high-quality 80\\% of the time on this Natural Questions subset, and 67\\% of the time on the ELI5 subset. Abstaining from the third of questions for which it is most unsure improves performance to 90\\% and 80\\% respectively, approaching human baselines. However, analysis on the adversarial TruthfulQA dataset shows why citation is only one part of an overall strategy for safety and trustworthiness: not all claims supported by evidence are true."
++ },
++ "arXiv:2203.11171": {
++  "paperId": "5f19ae1135a9500940978104ec15a5b8751bc7d2",
++  "externalIds": {
++   "DBLP": "conf/iclr/0002WSLCNCZ23",
++   "ArXiv": "2203.11171",
++   "CorpusId": 247595263
++  },
++  "publicationVenue": {
++   "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",
++   "name": "International Conference on Learning Representations",
++   "type": "conference",
++   "alternate_names": [
++    "Int Conf Learn Represent",
++    "ICLR"
++   ],
++   "url": "https://iclr.cc/"
++  },
++  "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models",
++  "venue": "International Conference on Learning Representations",
++  "year": 2022,
++  "citationCount": 8129,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2203.11171, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2203.11171"
++  },
++  "authors": [
++   {
++    "authorId": "2275277634",
++    "name": "Xue-Zhi Wang"
++   },
++   {
++    "authorId": "119640649",
++    "name": "Jason Wei"
++   },
++   {
++    "authorId": "1714772",
++    "name": "Dale Schuurmans"
++   },
++   {
++    "authorId": "1998340269",
++    "name": "Quoc V. Le"
++   },
++   {
++    "authorId": "2226805",
++    "name": "Ed H. Chi"
++   },
++   {
++    "authorId": "46617804",
++    "name": "Sharan Narang"
++   },
++   {
++    "authorId": "2841893",
++    "name": "A. Chowdhery"
++   },
++   {
++    "authorId": "65855107",
++    "name": "Denny Zhou"
++   }
++  ],
++  "abstract": "Chain-of-thought prompting combined with pre-trained large language models has achieved encouraging results on complex reasoning tasks. In this paper, we propose a new decoding strategy, self-consistency, to replace the naive greedy decoding used in chain-of-thought prompting. It first samples a diverse set of reasoning paths instead of only taking the greedy one, and then selects the most consistent answer by marginalizing out the sampled reasoning paths. Self-consistency leverages the intuition that a complex reasoning problem typically admits multiple different ways of thinking leading to its unique correct answer. Our extensive empirical evaluation shows that self-consistency boosts the performance of chain-of-thought prompting with a striking margin on a range of popular arithmetic and commonsense reasoning benchmarks, including GSM8K (+17.9%), SVAMP (+11.0%), AQuA (+12.2%), StrategyQA (+6.4%) and ARC-challenge (+3.9%)."
++ },
++ "arXiv:2210.13777": {
++  "paperId": "f13b251c8346bc3be19b71b840449831e9716999",
++  "externalIds": {
++   "DBLP": "conf/emnlp/WaddenLKCBWH22",
++   "ArXiv": "2210.13777",
++   "DOI": "10.48550/arXiv.2210.13777",
++   "CorpusId": 253107192
++  },
++  "publicationVenue": {
++   "id": "41bf9ed3-85b3-4c90-b015-150e31690253",
++   "name": "Conference on Empirical Methods in Natural Language Processing",
++   "type": "conference",
++   "alternate_names": [
++    "Empir Method Nat Lang Process",
++    "Empirical Methods in Natural Language Processing",
++    "Conf Empir Method Nat Lang Process",
++    "EMNLP"
++   ],
++   "url": "https://www.aclweb.org/portal/emnlp"
++  },
++  "title": "SciFact-Open: Towards open-domain scientific claim verification",
++  "venue": "Conference on Empirical Methods in Natural Language Processing",
++  "year": 2022,
++  "citationCount": 136,
++  "openAccessPdf": {
++   "url": "http://arxiv.org/pdf/2210.13777",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2210.13777, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2210.13777"
++  },
++  "authors": [
++   {
++    "authorId": "30051202",
++    "name": "David Wadden"
++   },
++   {
++    "authorId": "46258841",
++    "name": "Kyle Lo"
++   },
++   {
++    "authorId": "2003338023",
++    "name": "Bailey Kuehl"
++   },
++   {
++    "authorId": "2527954",
++    "name": "Arman Cohan"
++   },
++   {
++    "authorId": "46181066",
++    "name": "Iz Beltagy"
++   },
++   {
++    "authorId": "31860505",
++    "name": "Lucy Lu Wang"
++   },
++   {
++    "authorId": "2548384",
++    "name": "Hannaneh Hajishirzi"
++   }
++  ],
++  "abstract": "While research on scientific claim verification has led to the development of powerful systems that appear to approach human performance, these approaches have yet to be tested in a realistic setting against large corpora of scientific literature. Moving to this open-domain evaluation setting, however, poses unique challenges; in particular, it is infeasible to exhaustively annotate all evidence documents. In this work, we present SciFact-Open, a new test collection designed to evaluate the performance of scientific claim verification systems on a corpus of 500K research abstracts. Drawing upon pooling techniques from information retrieval, we collect evidence for scientific claims by pooling and annotating the top predictions of four state-of-the-art scientific claim verification models. We find that systems developed on smaller corpora struggle to generalize to SciFact-Open, exhibiting performance drops of at least 15 F1. In addition, analysis of the evidence in SciFact-Open reveals interesting phenomena likely to appear when claim verification systems are deployed in practice, e.g., cases where the evidence supports only a special case of the claim. Our dataset is available at https://github.com/dwadden/scifact-open."
++ },
++ "arXiv:2211.09110": {
++  "paperId": "29abcf865613287c661385c39401424f709a3fda",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2211-09110",
++   "ArXiv": "2211.09110",
++   "DOI": "10.48550/arXiv.2211.09110",
++   "CorpusId": 263423935
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Holistic Evaluation of Language Models",
++  "venue": "arXiv.org",
++  "year": 2022,
++  "citationCount": 12,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/2211.09110",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2211.09110, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2211.09110"
++  },
++  "authors": [
++   {
++    "authorId": "2249641250",
++    "name": "Percy Liang"
++   },
++   {
++    "authorId": "2223138553",
++    "name": "Rishi Bommasani"
++   },
++   {
++    "authorId": "2110585783",
++    "name": "Tony Lee"
++   },
++   {
++    "authorId": "2754804",
++    "name": "Dimitris Tsipras"
++   },
++   {
++    "authorId": "1914569491",
++    "name": "Dilara Soylu"
++   },
++   {
++    "authorId": "19168196",
++    "name": "Michihiro Yasunaga"
++   },
++   {
++    "authorId": "9227100",
++    "name": "Yian Zhang"
++   },
++   {
++    "authorId": "2251007290",
++    "name": "Deepak Narayanan"
++   },
++   {
++    "authorId": "2250015839",
++    "name": "Yuhuai Wu"
++   },
++   {
++    "authorId": "32423266",
++    "name": "Ananya Kumar"
++   },
++   {
++    "authorId": "2250557275",
++    "name": "Benjamin Newman"
++   },
++   {
++    "authorId": "2250117174",
++    "name": "Binhang Yuan"
++   },
++   {
++    "authorId": "1748871792",
++    "name": "Bobby Yan"
++   },
++   {
++    "authorId": "2262511796",
++    "name": "Ce Zhang"
++   },
++   {
++    "authorId": "133749287",
++    "name": "Christian Cosgrove"
++   },
++   {
++    "authorId": "2250402802",
++    "name": "Christopher D. Manning"
++   },
++   {
++    "authorId": "2250761261",
++    "name": "Christopher R\u00e9"
++   },
++   {
++    "authorId": "1413421064",
++    "name": "Diana Acosta-Navas"
++   },
++   {
++    "authorId": "152951058",
++    "name": "Drew A. Hudson"
++   },
++   {
++    "authorId": "49456763",
++    "name": "E. Zelikman"
++   },
++   {
++    "authorId": "41152329",
++    "name": "Esin Durmus"
++   },
++   {
++    "authorId": "8759332",
++    "name": "Faisal Ladhak"
++   },
++   {
++    "authorId": "2047004093",
++    "name": "Frieda Rong"
++   },
++   {
++    "authorId": "40046694",
++    "name": "Hongyu Ren"
++   },
++   {
++    "authorId": "2250097082",
++    "name": "Huaxiu Yao"
++   },
++   {
++    "authorId": "2252087284",
++    "name": "Jue Wang"
++   },
++   {
++    "authorId": "50818255",
++    "name": "Keshav Santhanam"
++   },
++   {
++    "authorId": "2250741832",
++    "name": "Laurel J. Orr"
++   },
++   {
++    "authorId": "2118604716",
++    "name": "Lucia Zheng"
++   },
++   {
++    "authorId": "1387987315",
++    "name": "Mert Y\u00fcksekg\u00f6n\u00fcl"
++   },
++   {
++    "authorId": "51903517",
++    "name": "M. Suzgun"
++   },
++   {
++    "authorId": "2252992164",
++    "name": "Nathan Kim"
++   },
++   {
++    "authorId": "2820009",
++    "name": "Neel Guha"
++   },
++   {
++    "authorId": "22193324",
++    "name": "Niladri S. Chatterji"
++   },
++   {
++    "authorId": "144112155",
++    "name": "O. Khattab"
++   },
++   {
++    "authorId": "2250986700",
++    "name": "Peter Henderson"
++   },
++   {
++    "authorId": "144862341",
++    "name": "Qian Huang"
++   },
++   {
++    "authorId": "2121293578",
++    "name": "Ryan A. Chi"
++   },
++   {
++    "authorId": "2407771963",
++    "name": "S. Xie"
++   },
++   {
++    "authorId": "2852106",
++    "name": "Shibani Santurkar"
++   },
++   {
++    "authorId": "2242939863",
++    "name": "Surya Ganguli"
++   },
++   {
++    "authorId": "2117567142",
++    "name": "Tatsunori B. Hashimoto"
++   },
++   {
++    "authorId": "2251009433",
++    "name": "Thomas Icard"
++   },
++   {
++    "authorId": "2256233130",
++    "name": "Tianyi Zhang"
++   },
++   {
++    "authorId": "113810201",
++    "name": "Vishrav Chaudhary"
++   },
++   {
++    "authorId": "2127971344",
++    "name": "William Wang"
++   },
++   {
++    "authorId": "2250724754",
++    "name": "Xuechen Li"
++   },
++   {
++    "authorId": "2054708905",
++    "name": "Yifan Mai"
++   },
++   {
++    "authorId": "2273913293",
++    "name": "Yu-Hui Zhang"
++   },
++   {
++    "authorId": "2740047",
++    "name": "Yuta Koreeda"
++   }
++  ],
++  "abstract": "Language models (LMs) are becoming the foundation for almost all major language technologies, but their capabilities, limitations, and risks are not well understood. We present Holistic Evaluation of Language Models (HELM) to improve the transparency of language models. First, we taxonomize the vast space of potential scenarios (i.e. use cases) and metrics (i.e. desiderata) that are of interest for LMs. Then we select a broad subset based on coverage and feasibility, noting what's missing or underrepresented (e.g. question answering for neglected English dialects, metrics for trustworthiness). Second, we adopt a multi-metric approach: We measure 7 metrics (accuracy, calibration, robustness, fairness, bias, toxicity, and efficiency) for each of 16 core scenarios when possible (87.5% of the time). This ensures metrics beyond accuracy don't fall to the wayside, and that trade-offs are clearly exposed. We also perform 7 targeted evaluations, based on 26 targeted scenarios, to analyze specific aspects (e.g. reasoning, disinformation). Third, we conduct a large-scale evaluation of 30 prominent language models (spanning open, limited-access, and closed models) on all 42 scenarios, 21 of which were not previously used in mainstream LM evaluation. Prior to HELM, models on average were evaluated on just 17.9% of the core HELM scenarios, with some prominent models not sharing a single scenario in common. We improve this to 96.0%: now all 30 models have been densely benchmarked on the same core scenarios and metrics under standardized conditions. Our evaluation surfaces 25 top-level findings. For full transparency, we release all raw model prompts and completions publicly for further analysis, as well as a general modular toolkit. We intend for HELM to be a living benchmark for the community, continuously updated with new scenarios, metrics, and models."
++ },
++ "arXiv:2211.11501": {
++  "paperId": "8a4fc5f00cd4aca61e148e46a2125c3a406719f1",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2211-11501",
++   "ArXiv": "2211.11501",
++   "DOI": "10.48550/arXiv.2211.11501",
++   "CorpusId": 253734939
++  },
++  "publicationVenue": {
++   "id": "fc0a208c-acb7-47dc-a0d4-af8190e21d29",
++   "name": "International Conference on Machine Learning",
++   "type": "conference",
++   "alternate_names": [
++    "ICML",
++    "Int Conf Mach Learn"
++   ],
++   "url": "https://icml.cc/"
++  },
++  "title": "DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation",
++  "venue": "International Conference on Machine Learning",
++  "year": 2022,
++  "citationCount": 659,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/2211.11501",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2211.11501, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "pages": "18319-18345"
++  },
++  "authors": [
++   {
++    "authorId": "2191638875",
++    "name": "Yuhang Lai"
++   },
++   {
++    "authorId": "2279871391",
++    "name": "Cheng-Xi Li"
++   },
++   {
++    "authorId": "2518870917",
++    "name": "Yi-Ming Wang"
++   },
++   {
++    "authorId": "2146331419",
++    "name": "Tianyi Zhang"
++   },
++   {
++    "authorId": "51011000",
++    "name": "Ruiqi Zhong"
++   },
++   {
++    "authorId": "1982950",
++    "name": "Luke Zettlemoyer"
++   },
++   {
++    "authorId": "3156075",
++    "name": "S. Yih"
++   },
++   {
++    "authorId": "47070750",
++    "name": "Daniel Fried"
++   },
++   {
++    "authorId": "2128046669",
++    "name": "Si-yi Wang"
++   },
++   {
++    "authorId": "48881008",
++    "name": "Tao Yu"
++   }
++  ],
++  "abstract": "We introduce DS-1000, a code generation benchmark with a thousand data science problems spanning seven Python libraries, such as NumPy and Pandas. Compared to prior works, DS-1000 incorporates three core features. First, our problems reflect diverse, realistic, and practical use cases since we collected them from StackOverflow. Second, our automatic evaluation is highly specific (reliable) -- across all Codex-002-predicted solutions that our evaluation accept, only 1.8% of them are incorrect; we achieve this with multi-criteria metrics, checking both functional correctness by running test cases and surface-form constraints by restricting API usages or keywords. Finally, we proactively defend against memorization by slightly modifying our problems to be different from the original StackOverflow source; consequently, models cannot answer them correctly by memorizing the solutions from pre-training. The current best public system (Codex-002) achieves 43.3% accuracy, leaving ample room for improvement. We release our benchmark at https://ds1000-code-gen.github.io."
++ },
++ "arXiv:2212.08037": {
++  "paperId": "05d77715d49714506a920f26c5432b92078cd37c",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2212-08037",
++   "ArXiv": "2212.08037",
++   "DOI": "10.48550/arXiv.2212.08037",
++   "CorpusId": 254685584
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Attributed Question Answering: Evaluation and Modeling for Attributed Large Language Models",
++  "venue": "arXiv.org",
++  "year": 2022,
++  "citationCount": 163,
++  "openAccessPdf": {
++   "url": "http://arxiv.org/pdf/2212.08037",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2212.08037, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2212.08037"
++  },
++  "authors": [
++   {
++    "authorId": "1678747",
++    "name": "Bernd Bohnet"
++   },
++   {
++    "authorId": "2057663102",
++    "name": "Vinh Q. Tran"
++   },
++   {
++    "authorId": "2986975",
++    "name": "Pat Verga"
++   },
++   {
++    "authorId": "2335771",
++    "name": "Roee Aharoni"
++   },
++   {
++    "authorId": "3365603",
++    "name": "D. Andor"
++   },
++   {
++    "authorId": "7353832",
++    "name": "Livio Baldini Soares"
++   },
++   {
++    "authorId": "2754495",
++    "name": "M. Ciaramita"
++   },
++   {
++    "authorId": "144154709",
++    "name": "Jacob Eisenstein"
++   },
++   {
++    "authorId": "144422385",
++    "name": "K. Ganchev"
++   },
++   {
++    "authorId": "47426264",
++    "name": "Jonathan Herzig"
++   },
++   {
++    "authorId": "47214884",
++    "name": "Kai Hui"
++   },
++   {
++    "authorId": "15652489",
++    "name": "T. Kwiatkowski"
++   },
++   {
++    "authorId": "2359098271",
++    "name": "Ji Ma"
++   },
++   {
++    "authorId": "2148023",
++    "name": "Jianmo Ni"
++   },
++   {
++    "authorId": "100831413",
++    "name": "Lierni Sestorain Saralegui"
++   },
++   {
++    "authorId": "32303439",
++    "name": "Tal Schuster"
++   },
++   {
++    "authorId": "50056360",
++    "name": "William W. Cohen"
++   },
++   {
++    "authorId": "123052390",
++    "name": "Michael Collins"
++   },
++   {
++    "authorId": "143790066",
++    "name": "Dipanjan Das"
++   },
++   {
++    "authorId": "1680617",
++    "name": "Donald Metzler"
++   },
++   {
++    "authorId": "1754497",
++    "name": "Slav Petrov"
++   },
++   {
++    "authorId": "20825661",
++    "name": "Kellie Webster"
++   }
++  ],
++  "abstract": "Large language models (LLMs) have shown impressive results while requiring little or no direct supervision. Further, there is mounting evidence that LLMs may have potential in information-seeking scenarios. We believe the ability of an LLM to attribute the text that it generates is likely to be crucial in this setting. We formulate and study Attributed QA as a key first step in the development of attributed LLMs. We propose a reproducible evaluation framework for the task and benchmark a broad set of architectures. We take human annotations as a gold standard and show that a correlated automatic metric is suitable for development. Our experimental work gives concrete answers to two key questions (How to measure attribution?, and How well do current state-of-the-art methods perform on attribution?), and give some hints as to how to address a third (How to build LLMs with attribution?)."
++ },
++ "arXiv:2302.06590": {
++  "paperId": "15abedb29536d50afeeec739a25358255cbda3e8",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2302-06590",
++   "ArXiv": "2302.06590",
++   "CorpusId": 256826778
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
++  "venue": "arXiv.org",
++  "year": 2023,
++  "citationCount": 878,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2302.06590, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2302.06590"
++  },
++  "authors": [
++   {
++    "authorId": "67042377",
++    "name": "Sida Peng"
++   },
++   {
++    "authorId": "1734481",
++    "name": "E. Kalliamvakou"
++   },
++   {
++    "authorId": "2205544450",
++    "name": "Peter Cihon"
++   },
++   {
++    "authorId": "88741890",
++    "name": "Mert Demirer"
++   }
++  ],
++  "abstract": "Generative AI tools hold promise to increase human productivity. This paper presents results from a controlled experiment with GitHub Copilot, an AI pair programmer. Recruited software developers were asked to implement an HTTP server in JavaScript as quickly as possible. The treatment group, with access to the AI pair programmer, completed the task 55.8% faster than the control group. Observed heterogenous effects show promise for AI pair programmers to help people transition into software development careers."
++ },
++ "arXiv:2305.02251": {
++  "paperId": "98f166771681acfe51c0f69558d8a977f0782739",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2305-02251",
++   "ArXiv": "2305.02251",
++   "DOI": "10.1007/s10994-025-06955-2",
++   "CorpusId": 258461620
++  },
++  "publicationVenue": {
++   "id": "22c9862f-a25e-40cd-9d31-d09e68a293e6",
++   "name": "Machine-mediated learning",
++   "type": "journal",
++   "alternate_names": [
++    "Mach learn",
++    "Machine Learning",
++    "Mach Learn"
++   ],
++   "issn": "0732-6718",
++   "alternate_issns": [
++    "0885-6125"
++   ],
++   "url": "http://www.springer.com/computer/artificial/journal/10994",
++   "alternate_urls": [
++    "https://link.springer.com/journal/10994",
++    "http://www.springer.com/west/home/computer/artificial?SGWID=4-147-70-35726603-0"
++   ]
++  },
++  "title": "Automated Scientific Discovery: From Equation Discovery to Autonomous Discovery Systems",
++  "venue": "Machine-mediated learning",
++  "year": 2023,
++  "citationCount": 30,
++  "openAccessPdf": {
++   "url": "https://link.springer.com/content/pdf/10.1007/s10994-025-06955-2.pdf",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2305.02251, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Machine Learning",
++   "volume": "115"
++  },
++  "authors": [
++   {
++    "authorId": "2058253559",
++    "name": "Stefan Kramer"
++   },
++   {
++    "authorId": "1382852308",
++    "name": "Mattia Cerrato"
++   },
++   {
++    "authorId": "2037478739",
++    "name": "Jannis Brugger"
++   },
++   {
++    "authorId": "1693549",
++    "name": "S. D\u017eeroski"
++   },
++   {
++    "authorId": "33654217",
++    "name": "R. King"
++   }
++  ],
++  "abstract": "The paper surveys automated scientific discovery, from equation discovery and symbolic regression to autonomous discovery systems and agents. It discusses the individual approaches from a \"big picture\" perspective and in context, but also discusses open issues and recent topics like the various roles of deep neural networks in this area, aiding in the discovery of human-interpretable knowledge. Further, we will present closed-loop scientific discovery systems, starting with the pioneering work on the Adam system up to current efforts in fields from material science to astronomy. Finally, we will elaborate on autonomy from a machine learning perspective, but also in analogy to the autonomy levels in autonomous driving. The maximal level, level five, is defined to require no human intervention at all in the production of scientific knowledge. Achieving this is one step towards solving the Nobel Turing Grand Challenge to develop AI Scientists: AI systems capable of making Nobel-quality scientific discoveries highly autonomously at a level comparable, and possibly superior, to the best human scientists by 2050."
++ },
++ "arXiv:2305.05176": {
++  "paperId": "585f8b9725f5f5e5495c3508d39f70d1c053e190",
++  "externalIds": {
++   "ArXiv": "2305.05176",
++   "DBLP": "journals/corr/abs-2305-05176",
++   "DOI": "10.48550/arXiv.2305.05176",
++   "CorpusId": 258564349
++  },
++  "publicationVenue": null,
++  "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
++  "venue": "Trans. Mach. Learn. Res.",
++  "year": 2023,
++  "citationCount": 1227,
++  "openAccessPdf": {
++   "url": "http://arxiv.org/pdf/2305.05176",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2305.05176, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2305.05176"
++  },
++  "authors": [
++   {
++    "authorId": "2119322954",
++    "name": "Lingjiao Chen"
++   },
++   {
++    "authorId": "143834867",
++    "name": "M. Zaharia"
++   },
++   {
++    "authorId": "145085305",
++    "name": "James Y. Zou"
++   }
++  ],
++  "abstract": "There is a rapidly growing number of large language models (LLMs) that users can query for a fee. We review the cost associated with querying popular LLM APIs, e.g. GPT-4, ChatGPT, J1-Jumbo, and find that these models have heterogeneous pricing structures, with fees that can differ by two orders of magnitude. In particular, using LLMs on large collections of queries and text can be expensive. Motivated by this, we outline and discuss three types of strategies that users can exploit to reduce the inference cost associated with using LLMs: 1) prompt adaptation, 2) LLM approximation, and 3) LLM cascade. As an example, we propose FrugalGPT, a simple yet flexible instantiation of LLM cascade which learns which combinations of LLMs to use for different queries in order to reduce cost and improve accuracy. Our experiments show that FrugalGPT can match the performance of the best individual LLM (e.g. GPT-4) with up to 98% cost reduction or improve the accuracy over GPT-4 by 4% with the same cost. The ideas and findings presented here lay a foundation for using LLMs sustainably and efficiently."
++ },
++ "arXiv:2305.17926": {
++  "paperId": "38d64919ba526868a850a0e5f6239d4c474b7e7e",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2305-17926",
++   "ArXiv": "2305.17926",
++   "DOI": "10.48550/arXiv.2305.17926",
++   "CorpusId": 258960339
++  },
++  "publicationVenue": {
++   "id": "1e33b3be-b2ab-46e9-96e8-d4eb4bad6e44",
++   "name": "Annual Meeting of the Association for Computational Linguistics",
++   "type": "conference",
++   "alternate_names": [
++    "Annu Meet Assoc Comput Linguistics",
++    "Meeting of the Association for Computational Linguistics",
++    "ACL",
++    "Meet Assoc Comput Linguistics"
++   ],
++   "url": "https://www.aclweb.org/anthology/venues/acl/"
++  },
++  "title": "Large Language Models are not Fair Evaluators",
++  "venue": "Annual Meeting of the Association for Computational Linguistics",
++  "year": 2023,
++  "citationCount": 1454,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/2305.17926",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2305.17926, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2305.17926"
++  },
++  "authors": [
++   {
++    "authorId": "144202874",
++    "name": "Peiyi Wang"
++   },
++   {
++    "authorId": "49192881",
++    "name": "Lei Li"
++   },
++   {
++    "authorId": "2146034504",
++    "name": "Liang Chen"
++   },
++   {
++    "authorId": "2117632647",
++    "name": "Ze-Fan Cai"
++   },
++   {
++    "authorId": "2116276849",
++    "name": "Dawei Zhu"
++   },
++   {
++    "authorId": "3186130",
++    "name": "Bing-Huai Lin"
++   },
++   {
++    "authorId": "2154235",
++    "name": "Yun-Bo Cao"
++   },
++   {
++    "authorId": "2275651222",
++    "name": "Ling-Peng Kong"
++   },
++   {
++    "authorId": "2144831944",
++    "name": "Qi Liu"
++   },
++   {
++    "authorId": "1701889",
++    "name": "Tianyu Liu"
++   },
++   {
++    "authorId": "3335836",
++    "name": "Zhi-Fang Sui"
++   }
++  ],
++  "abstract": "In this paper, we uncover a systematic bias in the evaluation paradigm of adopting large language models~(LLMs), e.g., GPT-4, as a referee to score and compare the quality of responses generated by candidate models. We find that the quality ranking of candidate responses can be easily hacked by simply altering their order of appearance in the context. This manipulation allows us to skew the evaluation result, making one model appear considerably superior to the other, e.g., Vicuna-13B could beat ChatGPT on 66 over 80 tested queries with ChatGPT as an evaluator. To address this issue, we propose a calibration framework with three simple yet effective strategies: 1) Multiple Evidence Calibration, which requires the evaluator model to generate multiple evaluation evidence before assigning ratings; 2) Balanced Position Calibration, which aggregates results across various orders to determine the final score; 3) Human-in-the-Loop Calibration, which introduces a balanced position diversity entropy to measure the difficulty of each example and seeks human assistance when needed. We also manually annotate the\"win/tie/lose\"outcomes of responses from ChatGPT and Vicuna-13B in the Vicuna Benchmark's question prompt, and extensive experiments demonstrate that our approach successfully mitigates evaluation bias, resulting in closer alignment with human judgments. We release our code and human annotation at \\url{https://github.com/i-Eval/FairEval} to facilitate future research."
++ },
++ "arXiv:2305.18248": {
++  "paperId": "c18e13ba65c7247774301314d181c87ee5ebc847",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2305-18248",
++   "ArXiv": "2305.18248",
++   "ACL": "2024.findings-eacl.62",
++   "DOI": "10.48550/arXiv.2305.18248",
++   "CorpusId": 258960346
++  },
++  "publicationVenue": {
++   "id": "479d5605-51be-4346-b1d6-4334084504df",
++   "name": "Findings",
++   "type": "journal",
++   "issn": "2652-8800",
++   "url": "https://findingspress.org/"
++  },
++  "title": "Do Language Models Know When They\u2019re Hallucinating References?",
++  "venue": "Findings",
++  "year": 2023,
++  "citationCount": 177,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/2305.18248",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2305.18248, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "pages": "912-928"
++  },
++  "authors": [
++   {
++    "authorId": "2075308090",
++    "name": "A. Agrawal"
++   },
++   {
++    "authorId": "51903517",
++    "name": "M. Suzgun"
++   },
++   {
++    "authorId": "143722101",
++    "name": "L. Mackey"
++   },
++   {
++    "authorId": "2186481",
++    "name": "A. Kalai"
++   }
++  ],
++  "abstract": "State-of-the-art language models (LMs) are notoriously susceptible to generating hallucinated information. Such inaccurate outputs not only undermine the reliability of these models but also limit their use and raise serious concerns about misinformation and propaganda. In this work, we focus on hallucinated book and article references and present them as the \u201cmodel organism\u201d of language model hallucination research, due to their frequent and easy-to-discern nature. We posit that if a language model cites a particular reference in its output, then it should ideally possess sufficient information about its authors and content, among other relevant details. Using this basic insight, we illustrate that one can identify hallucinated references without ever consulting any external resources, by asking a set of direct or indirect queries to the language model about the references. These queries can be considered as \u201cconsistency checks.\u201d Our findings highlight that while LMs, including GPT-4, often produce inconsistent author lists for hallucinated references, they also often accurately recall the authors of real references. In this sense, the LM can be said to \u201cknow\u201d when it is hallucinating references. Furthermore, these findings show how hallucinated references can be dissected to shed light on their nature."
++ },
++ "arXiv:2306.05685": {
++  "paperId": "a0a79dad89857a96f8f71b14238e5237cbfc4787",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2306-05685",
++   "ArXiv": "2306.05685",
++   "DOI": "10.52202/075280-2020",
++   "CorpusId": 259129398
++  },
++  "publicationVenue": {
++   "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",
++   "name": "Neural Information Processing Systems",
++   "type": "conference",
++   "alternate_names": [
++    "Neural Inf Process Syst",
++    "NeurIPS",
++    "NIPS"
++   ],
++   "url": "http://neurips.cc/"
++  },
++  "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",
++  "venue": "Neural Information Processing Systems",
++  "year": 2023,
++  "citationCount": 12661,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/2306.05685",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2306.05685, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2306.05685"
++  },
++  "authors": [
++   {
++    "authorId": "2149970173",
++    "name": "Lian-Min Zheng"
++   },
++   {
++    "authorId": "2537924",
++    "name": "Wei-Lin Chiang"
++   },
++   {
++    "authorId": "2209360681",
++    "name": "Ying Sheng"
++   },
++   {
++    "authorId": "92721493",
++    "name": "Si-Yuan Zhuang"
++   },
++   {
++    "authorId": "1390573666",
++    "name": "Zhanghao Wu"
++   },
++   {
++    "authorId": "2152482391",
++    "name": "Yong-Hao Zhuang"
++   },
++   {
++    "authorId": "143872641",
++    "name": "Zi Lin"
++   },
++   {
++    "authorId": "2141335450",
++    "name": "Zhuo-Han Li"
++   },
++   {
++    "authorId": "2265517766",
++    "name": "Da-Cheng Li"
++   },
++   {
++    "authorId": "143977260",
++    "name": "E. Xing"
++   },
++   {
++    "authorId": "145140331",
++    "name": "Haotong Zhang"
++   },
++   {
++    "authorId": "49988044",
++    "name": "Joseph E. Gonzalez"
++   },
++   {
++    "authorId": "2055174324",
++    "name": "Ion Stoica"
++   }
++  ],
++  "abstract": "Evaluating large language model (LLM) based chat assistants is challenging due to their broad capabilities and the inadequacy of existing benchmarks in measuring human preferences. To address this, we explore using strong LLMs as judges to evaluate these models on more open-ended questions. We examine the usage and limitations of LLM-as-a-judge, including position, verbosity, and self-enhancement biases, as well as limited reasoning ability, and propose solutions to mitigate some of them. We then verify the agreement between LLM judges and human preferences by introducing two benchmarks: MT-bench, a multi-turn question set; and Chatbot Arena, a crowdsourced battle platform. Our results reveal that strong LLM judges like GPT-4 can match both controlled and crowdsourced human preferences well, achieving over 80% agreement, the same level of agreement between humans. Hence, LLM-as-a-judge is a scalable and explainable way to approximate human preferences, which are otherwise very expensive to obtain. Additionally, we show our benchmark and traditional benchmarks complement each other by evaluating several variants of LLaMA and Vicuna. The MT-bench questions, 3K expert votes, and 30K conversations with human preferences are publicly available at https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge."
++ },
++ "arXiv:2308.03688": {
++  "paperId": "5dbf93a68b7fda600521f046dea35ea8ba9e884f",
++  "externalIds": {
++   "ArXiv": "2308.03688",
++   "DBLP": "journals/corr/abs-2308-03688",
++   "DOI": "10.48550/arXiv.2308.03688",
++   "CorpusId": 260682249
++  },
++  "publicationVenue": {
++   "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",
++   "name": "International Conference on Learning Representations",
++   "type": "conference",
++   "alternate_names": [
++    "Int Conf Learn Represent",
++    "ICLR"
++   ],
++   "url": "https://iclr.cc/"
++  },
++  "title": "AgentBench: Evaluating LLMs as Agents",
++  "venue": "International Conference on Learning Representations",
++  "year": 2023,
++  "citationCount": 1563,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/2308.03688",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2308.03688, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2308.03688"
++  },
++  "authors": [
++   {
++    "authorId": "2111312892",
++    "name": "Xiao Liu"
++   },
++   {
++    "authorId": "2110750673",
++    "name": "Hao Yu"
++   },
++   {
++    "authorId": "2228752234",
++    "name": "Hanchen Zhang"
++   },
++   {
++    "authorId": "2138973537",
++    "name": "Yi-Fan Xu"
++   },
++   {
++    "authorId": "2181283109",
++    "name": "Xuanyu Lei"
++   },
++   {
++    "authorId": "2051311700",
++    "name": "Han-Yu Lai"
++   },
++   {
++    "authorId": "2022231256",
++    "name": "Yu Gu"
++   },
++   {
++    "authorId": "2116405624",
++    "name": "Yu-Xian Gu"
++   },
++   {
++    "authorId": "2364101023",
++    "name": "Han Ding"
++   },
++   {
++    "authorId": "2310631841",
++    "name": "Kai-Wen Men"
++   },
++   {
++    "authorId": "2218347104",
++    "name": "Kejuan Yang"
++   },
++   {
++    "authorId": "2300177449",
++    "name": "Shu-Dan Zhang"
++   },
++   {
++    "authorId": "145924070",
++    "name": "Xiang Deng"
++   },
++   {
++    "authorId": "2051712753",
++    "name": "Ao-Han Zeng"
++   },
++   {
++    "authorId": "66395694",
++    "name": "Zheng-Xiao Du"
++   },
++   {
++    "authorId": "2146063748",
++    "name": "Chenhui Zhang"
++   },
++   {
++    "authorId": "2152107339",
++    "name": "Sheng-Qi Shen"
++   },
++   {
++    "authorId": "1993655237",
++    "name": "Tianjun Zhang"
++   },
++   {
++    "authorId": "2191455",
++    "name": "Sheng Shen"
++   },
++   {
++    "authorId": "1758652",
++    "name": "Yu Su"
++   },
++   {
++    "authorId": "1515546612",
++    "name": "Huan Sun"
++   },
++   {
++    "authorId": "1730108",
++    "name": "Minlie Huang"
++   },
++   {
++    "authorId": "2047998",
++    "name": "Yu-Xiao Dong"
++   },
++   {
++    "authorId": "2148911956",
++    "name": "Jie Tang"
++   }
++  ],
++  "abstract": "The potential of Large Language Model (LLM) as agents has been widely acknowledged recently. Thus, there is an urgent need to quantitatively \\textit{evaluate LLMs as agents} on challenging tasks in interactive environments. We present AgentBench, a multi-dimensional benchmark that consists of 8 distinct environments to assess LLM-as-Agent's reasoning and decision-making abilities. Our extensive test over \\num API-based and open-sourced (OSS) LLMs shows that, while top commercial LLMs present a strong ability of acting as agents in complex environments, there is a significant disparity in performance between them and many OSS competitors that are no larger than 70B. We identify the typical reasons of failures in environments and LLMs, showing that poor long-term reasoning, decision-making, and instruction following abilities are the main obstacles for developing usable LLM agents. Improving instruction following and training on high quality multi-round alignment data could improve agent performance. And different from existing assumptions, training on code present ambivalent impacts on different agent tasks. Datasets, environments, and an integrated evaluation package for AgentBench are released at https://github.com/THUDM/AgentBench."
++ },
++ "arXiv:2308.08493": {
++  "paperId": "d6d0fd994f37b630f35945736b5e1bb198148404",
++  "externalIds": {
++   "ArXiv": "2308.08493",
++   "DBLP": "journals/corr/abs-2308-08493",
++   "DOI": "10.48550/arXiv.2308.08493",
++   "CorpusId": 260925501
++  },
++  "publicationVenue": {
++   "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",
++   "name": "International Conference on Learning Representations",
++   "type": "conference",
++   "alternate_names": [
++    "Int Conf Learn Represent",
++    "ICLR"
++   ],
++   "url": "https://iclr.cc/"
++  },
++  "title": "Time Travel in LLMs: Tracing Data Contamination in Large Language Models",
++  "venue": "International Conference on Learning Representations",
++  "year": 2023,
++  "citationCount": 235,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/2308.08493",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2308.08493, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2308.08493"
++  },
++  "authors": [
++   {
++    "authorId": "65754049",
++    "name": "Shahriar Golchin"
++   },
++   {
++    "authorId": "1760868",
++    "name": "M. Surdeanu"
++   }
++  ],
++  "abstract": "Data contamination, i.e., the presence of test data from downstream tasks in the training data of large language models (LLMs), is a potential major issue in measuring LLMs' real effectiveness on other tasks. We propose a straightforward yet effective method for identifying data contamination within LLMs. At its core, our approach starts by identifying potential contamination at the instance level; using this information, our approach then assesses wider contamination at the partition level. To estimate contamination of individual instances, we employ\"guided instruction:\"a prompt consisting of the dataset name, partition type, and the random-length initial segment of a reference instance, asking the LLM to complete it. An instance is flagged as contaminated if the LLM's output either exactly or nearly matches the latter segment of the reference. To understand if an entire partition is contaminated, we propose two ideas. The first idea marks a dataset partition as contaminated if the average overlap score with the reference instances (as measured by ROUGE-L or BLEURT) is statistically significantly better with the completions from guided instruction compared to a\"general instruction\"that does not include the dataset and partition name. The second idea marks a dataset partition as contaminated if a classifier based on GPT-4 with few-shot in-context learning prompt marks multiple generated completions as exact/near-exact matches of the corresponding reference instances. Our best method achieves an accuracy between 92% and 100% in detecting if an LLM is contaminated with seven datasets, containing train and test/validation partitions, when contrasted with manual evaluation by human experts. Further, our findings indicate that GPT-4 is contaminated with AG News, WNLI, and XSum datasets."
++ },
++ "arXiv:2309.11495": {
++  "paperId": "4b0b56be0ae9479d2bd5c2f0943db1906343c10f",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2309-11495",
++   "ArXiv": "2309.11495",
++   "DOI": "10.48550/arXiv.2309.11495",
++   "CorpusId": 262062565
++  },
++  "publicationVenue": {
++   "id": "1e33b3be-b2ab-46e9-96e8-d4eb4bad6e44",
++   "name": "Annual Meeting of the Association for Computational Linguistics",
++   "type": "conference",
++   "alternate_names": [
++    "Annu Meet Assoc Comput Linguistics",
++    "Meeting of the Association for Computational Linguistics",
++    "ACL",
++    "Meet Assoc Comput Linguistics"
++   ],
++   "url": "https://www.aclweb.org/anthology/venues/acl/"
++  },
++  "title": "Chain-of-Verification Reduces Hallucination in Large Language Models",
++  "venue": "Annual Meeting of the Association for Computational Linguistics",
++  "year": 2023,
++  "citationCount": 649,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/2309.11495",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2309.11495, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "pages": "3563-3578"
++  },
++  "authors": [
++   {
++    "authorId": "3448411",
++    "name": "S. Dhuliawala"
++   },
++   {
++    "authorId": "100653935",
++    "name": "M. Komeili"
++   },
++   {
++    "authorId": "2276766957",
++    "name": "Jing Xu"
++   },
++   {
++    "authorId": "48647153",
++    "name": "R. Raileanu"
++   },
++   {
++    "authorId": "2243015223",
++    "name": "Xian Li"
++   },
++   {
++    "authorId": "1709797",
++    "name": "Asli Celikyilmaz"
++   },
++   {
++    "authorId": "2243265350",
++    "name": "Jason E. Weston"
++   }
++  ],
++  "abstract": "Generation of plausible yet incorrect factual information, termed hallucination, is an unsolved issue in large language models. We study the ability of language models to deliberate on the responses they give in order to correct their mistakes. We develop the Chain-of-Verification (CoVe) method whereby the model first (i) drafts an initial response; then (ii) plans verification questions to fact-check its draft; (iii) answers those questions independently so the answers are not biased by other responses; and (iv) generates its final verified response. In experiments, we show CoVe decreases hallucinations across a variety of tasks, from list-based questions from Wikidata, closed book MultiSpanQA and longform text generation."
++ },
++ "arXiv:2310.01798": {
++  "paperId": "6d4bacb69923e1e94fb4de468b939ce6db32fb51",
++  "externalIds": {
++   "DBLP": "conf/iclr/0009CMZYSZ24",
++   "ArXiv": "2310.01798",
++   "DOI": "10.48550/arXiv.2310.01798",
++   "CorpusId": 263609132
++  },
++  "publicationVenue": {
++   "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",
++   "name": "International Conference on Learning Representations",
++   "type": "conference",
++   "alternate_names": [
++    "Int Conf Learn Represent",
++    "ICLR"
++   ],
++   "url": "https://iclr.cc/"
++  },
++  "title": "Large Language Models Cannot Self-Correct Reasoning Yet",
++  "venue": "International Conference on Learning Representations",
++  "year": 2023,
++  "citationCount": 1410,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/2310.01798",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2310.01798, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2310.01798"
++  },
++  "authors": [
++   {
++    "authorId": "1490651934",
++    "name": "Jie Huang"
++   },
++   {
++    "authorId": "2238263119",
++    "name": "Xin-Yun Chen"
++   },
++   {
++    "authorId": "1817207",
++    "name": "Swaroop Mishra"
++   },
++   {
++    "authorId": "2253804927",
++    "name": "Huaixiu Steven Zheng"
++   },
++   {
++    "authorId": "40625240",
++    "name": "Adams Wei Yu"
++   },
++   {
++    "authorId": "2293658042",
++    "name": "Xinying Song"
++   },
++   {
++    "authorId": "2256313467",
++    "name": "Denny Zhou"
++   }
++  ],
++  "abstract": "Large Language Models (LLMs) have emerged as a groundbreaking technology with their unparalleled text generation capabilities across various applications. Nevertheless, concerns persist regarding the accuracy and appropriateness of their generated content. A contemporary methodology, self-correction, has been proposed as a remedy to these issues. Building upon this premise, this paper critically examines the role and efficacy of self-correction within LLMs, shedding light on its true potential and limitations. Central to our investigation is the notion of intrinsic self-correction, whereby an LLM attempts to correct its initial responses based solely on its inherent capabilities, without the crutch of external feedback. In the context of reasoning, our research indicates that LLMs struggle to self-correct their responses without external feedback, and at times, their performance even degrades after self-correction. Drawing from these insights, we offer suggestions for future research and practical applications in this field."
++ },
++ "arXiv:2310.03302": {
++  "paperId": "b8ee0b5322382807e687c95cc87b059d3f348495",
++  "externalIds": {
++   "DBLP": "conf/icml/HuangVLL24",
++   "ArXiv": "2310.03302",
++   "CorpusId": 263671541
++  },
++  "publicationVenue": {
++   "id": "fc0a208c-acb7-47dc-a0d4-af8190e21d29",
++   "name": "International Conference on Machine Learning",
++   "type": "conference",
++   "alternate_names": [
++    "ICML",
++    "Int Conf Mach Learn"
++   ],
++   "url": "https://icml.cc/"
++  },
++  "title": "MLAgentBench: Evaluating Language Agents on Machine Learning Experimentation",
++  "venue": "International Conference on Machine Learning",
++  "year": 2023,
++  "citationCount": 438,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2310.03302, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "pages": "20271-20309"
++  },
++  "authors": [
++   {
++    "authorId": "144862341",
++    "name": "Qian Huang"
++   },
++   {
++    "authorId": "66535001",
++    "name": "Jian Vora"
++   },
++   {
++    "authorId": "2249641250",
++    "name": "Percy Liang"
++   },
++   {
++    "authorId": "1702139",
++    "name": "J. Leskovec"
++   }
++  ],
++  "abstract": "A central aspect of machine learning research is experimentation, the process of designing and running experiments, analyzing the results, and iterating towards some positive outcome (e.g., improving accuracy). Could agents driven by powerful language models perform machine learning experimentation effectively? To answer this question, we introduce MLAgentBench, a suite of 13 tasks ranging from improving model performance on CIFAR-10 to recent research problems like BabyLM. For each task, an agent can perform actions like reading/writing files, executing code, and inspecting outputs. We then construct an agent that can perform ML experimentation based on ReAct framework. We benchmark agents based on Claude v1.0, Claude v2.1, Claude v3 Opus, GPT-4, GPT-4-turbo, Gemini-Pro, and Mixtral and find that a Claude v3 Opus agent is the best in terms of success rate. It can build compelling ML models over many tasks in MLAgentBench with 37.5% average success rate. Our agents also display highly interpretable plans and actions. However, the success rates vary considerably; they span from 100% on well-established older datasets to as low as 0% on recent Kaggle challenges created potentially after the underlying LM was trained. Finally, we identify several key challenges for LM-based agents such as long-term planning and reducing hallucination. Our code is released at https://github.com/snap-stanford/MLAgentBench."
++ },
++ "arXiv:2310.06770": {
++  "paperId": "94a5f96308729e31c1ffbc0f0618db87795092fe",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2310-06770",
++   "ArXiv": "2310.06770",
++   "DOI": "10.48550/arXiv.2310.06770",
++   "CorpusId": 263829697
++  },
++  "publicationVenue": {
++   "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",
++   "name": "International Conference on Learning Representations",
++   "type": "conference",
++   "alternate_names": [
++    "Int Conf Learn Represent",
++    "ICLR"
++   ],
++   "url": "https://iclr.cc/"
++  },
++  "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
++  "venue": "International Conference on Learning Representations",
++  "year": 2023,
++  "citationCount": 4627,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/2310.06770",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2310.06770, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2310.06770"
++  },
++  "authors": [
++   {
++    "authorId": "2155269683",
++    "name": "Carlos E. Jimenez"
++   },
++   {
++    "authorId": "2109727379",
++    "name": "John Yang"
++   },
++   {
++    "authorId": "2127066887",
++    "name": "Alexander Wettig"
++   },
++   {
++    "authorId": "2256984659",
++    "name": "Shun-Yu Yao"
++   },
++   {
++    "authorId": "40428350",
++    "name": "Ke-Xin Pei"
++   },
++   {
++    "authorId": "40170001",
++    "name": "Ofir Press"
++   },
++   {
++    "authorId": "2135381714",
++    "name": "Karthik R. Narasimhan"
++   }
++  ],
++  "abstract": "Language models have outpaced our ability to evaluate them effectively, but for their future development it is essential to study the frontier of their capabilities. We find real-world software engineering to be a rich, sustainable, and challenging testbed for evaluating the next generation of language models. To this end, we introduce SWE-bench, an evaluation framework consisting of $2,294$ software engineering problems drawn from real GitHub issues and corresponding pull requests across $12$ popular Python repositories. Given a codebase along with a description of an issue to be resolved, a language model is tasked with editing the codebase to address the issue. Resolving issues in SWE-bench frequently requires understanding and coordinating changes across multiple functions, classes, and even files simultaneously, calling for models to interact with execution environments, process extremely long contexts and perform complex reasoning that goes far beyond traditional code generation tasks. Our evaluations show that both state-of-the-art proprietary models and our fine-tuned model SWE-Llama can resolve only the simplest issues. The best-performing model, Claude 2, is able to solve a mere $1.96$% of the issues. Advances on SWE-bench represent steps towards LMs that are more practical, intelligent, and autonomous."
++ },
++ "arXiv:2311.01964": {
++  "paperId": "84725855d10b531eb8cbe54935dda0440c2fc750",
++  "externalIds": {
++   "ArXiv": "2311.01964",
++   "DBLP": "journals/corr/abs-2311-01964",
++   "DOI": "10.48550/arXiv.2311.01964",
++   "CorpusId": 265019021
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Don't Make Your LLM an Evaluation Benchmark Cheater",
++  "venue": "arXiv.org",
++  "year": 2023,
++  "citationCount": 263,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2311.01964, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2311.01964"
++  },
++  "authors": [
++   {
++    "authorId": "2265383494",
++    "name": "Kun Zhou"
++   },
++   {
++    "authorId": "1900406",
++    "name": "Yutao Zhu"
++   },
++   {
++    "authorId": "2256558402",
++    "name": "Zhi-Peng Chen"
++   },
++   {
++    "authorId": "2265461972",
++    "name": "Wentong Chen"
++   },
++   {
++    "authorId": "2257376413",
++    "name": "W. Zhao"
++   },
++   {
++    "authorId": "2265519430",
++    "name": "Xu Chen"
++   },
++   {
++    "authorId": "2257310922",
++    "name": "Yankai Lin"
++   },
++   {
++    "authorId": "2186578511",
++    "name": "Ji-Rong Wen"
++   },
++   {
++    "authorId": "2161986932",
++    "name": "Jia-Wei Han"
++   }
++  ],
++  "abstract": "Large language models~(LLMs) have greatly advanced the frontiers of artificial intelligence, attaining remarkable improvement in model capacity. To assess the model performance, a typical approach is to construct evaluation benchmarks for measuring the ability level of LLMs in different aspects. Despite that a number of high-quality benchmarks have been released, the concerns about the appropriate use of these benchmarks and the fair comparison of different models are increasingly growing. Considering these concerns, in this paper, we discuss the potential risk and impact of inappropriately using evaluation benchmarks and misleadingly interpreting the evaluation results. Specially, we focus on a special issue that would lead to inappropriate evaluation, \\ie \\emph{benchmark leakage}, referring that the data related to evaluation sets is occasionally used for model training. This phenomenon now becomes more common since pre-training data is often prepared ahead of model test. We conduct extensive experiments to study the effect of benchmark leverage, and find that it can dramatically boost the evaluation results, which would finally lead to an unreliable assessment of model performance. To improve the use of existing evaluation benchmarks, we finally present several guidelines for both LLM developers and benchmark maintainers. We hope this work can draw attention to appropriate training and evaluation of LLMs."
++ },
++ "arXiv:2311.09783": {
++  "paperId": "af565483dfbe3b0fa4fe9f715170666a06bce5ac",
++  "externalIds": {
++   "ACL": "2024.naacl-long.482",
++   "DBLP": "conf/naacl/Deng0TGC24",
++   "ArXiv": "2311.09783",
++   "DOI": "10.48550/arXiv.2311.09783",
++   "CorpusId": 265220695
++  },
++  "publicationVenue": {
++   "id": "01103732-3808-4930-b8e4-7e9e68d5c68d",
++   "name": "North American Chapter of the Association for Computational Linguistics",
++   "type": "conference",
++   "alternate_names": [
++    "North Am Chapter Assoc Comput Linguistics",
++    "NAACL"
++   ],
++   "url": "https://www.aclweb.org/portal/naacl"
++  },
++  "title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models",
++  "venue": "North American Chapter of the Association for Computational Linguistics",
++  "year": 2023,
++  "citationCount": 208,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2311.09783, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "pages": "8706-8719"
++  },
++  "authors": [
++   {
++    "authorId": "2266840268",
++    "name": "Chunyuan Deng"
++   },
++   {
++    "authorId": "46316984",
++    "name": "Yilun Zhao"
++   },
++   {
++    "authorId": "47274259",
++    "name": "Xiangru Tang"
++   },
++   {
++    "authorId": "2201323142",
++    "name": "M. Gerstein"
++   },
++   {
++    "authorId": "2266838179",
++    "name": "Arman Cohan"
++   }
++  ],
++  "abstract": "Recent observations have underscored a disparity between the inflated benchmark scores and the actual performance of LLMs, raising concerns about potential contamination of evaluation benchmarks. This issue is especially critical for closed-source models and certain open-source models where training data transparency is lacking. In this paper we study data contamination by proposing two methods tailored for both open-source and proprietary LLMs. We first introduce a retrieval-based system to explore potential overlaps between evaluation benchmarks and pretraining corpora. We further present a novel investigation protocol named Testset Slot Guessing (TS-Guessing), applicable to both open and proprietary models. This approach entails masking a wrong answer in a multiple-choice question and prompting the model to fill in the gap. Additionally, it involves obscuring an unlikely word in an evaluation example and asking the model to produce it. We find that certain commercial LLMs could surprisingly guess the missing option in various test sets. Specifically, in the MMLU benchmark, ChatGPT and GPT-4 demonstrated an exact match rate of 52% and 57%, respectively, in guessing the missing options in benchmark test data. We hope these results underscore the need for more robust evaluation methodologies and benchmarks in the field."
++ },
++ "arXiv:2401.05507": {
++  "paperId": "ecd5d38b8590f6e6a504b29110d16ff717cd56ef",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2401-05507",
++   "ArXiv": "2401.05507",
++   "DOI": "10.48550/arXiv.2401.05507",
++   "CorpusId": 266933185
++  },
++  "publicationVenue": {
++   "id": "fc0a208c-acb7-47dc-a0d4-af8190e21d29",
++   "name": "International Conference on Machine Learning",
++   "type": "conference",
++   "alternate_names": [
++    "ICML",
++    "Int Conf Mach Learn"
++   ],
++   "url": "https://icml.cc/"
++  },
++  "title": "InfiAgent-DABench: Evaluating Agents on Data Analysis Tasks",
++  "venue": "International Conference on Machine Learning",
++  "year": 2024,
++  "citationCount": 173,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2401.05507, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "pages": "19544-19572"
++  },
++  "authors": [
++   {
++    "authorId": "2278917214",
++    "name": "Xueyu Hu"
++   },
++   {
++    "authorId": "2279090109",
++    "name": "Ziyu Zhao"
++   },
++   {
++    "authorId": "2279259092",
++    "name": "Shuang Wei"
++   },
++   {
++    "authorId": "2121930095",
++    "name": "Ziwei Chai"
++   },
++   {
++    "authorId": "2258790160",
++    "name": "Qian-Li Ma"
++   },
++   {
++    "authorId": "2279157833",
++    "name": "Guoyin Wang"
++   },
++   {
++    "authorId": "2279096273",
++    "name": "Xu-Wu Wang"
++   },
++   {
++    "authorId": "2279157215",
++    "name": "Jing Su"
++   },
++   {
++    "authorId": "2282345352",
++    "name": "Jingjing Xu"
++   },
++   {
++    "authorId": "2279262722",
++    "name": "Ming Zhu"
++   },
++   {
++    "authorId": "2117877910",
++    "name": "Yao Cheng"
++   },
++   {
++    "authorId": "2257015367",
++    "name": "Jianbo Yuan"
++   },
++   {
++    "authorId": "2372128161",
++    "name": "Ji-Wei Li"
++   },
++   {
++    "authorId": "33870528",
++    "name": "Kun Kuang"
++   },
++   {
++    "authorId": "2283100978",
++    "name": "Yang Yang"
++   },
++   {
++    "authorId": "2212243155",
++    "name": "Hongxia Yang"
++   },
++   {
++    "authorId": "2275491998",
++    "name": "Fei Wu"
++   }
++  ],
++  "abstract": "In this paper, we introduce InfiAgent-DABench, the first benchmark specifically designed to evaluate LLM-based agents on data analysis tasks. These tasks require agents to end-to-end solving complex tasks by interacting with an execution environment. This benchmark contains DAEval, a dataset consisting of 257 data analysis questions derived from 52 CSV files, and an agent framework which incorporates LLMs to serve as data analysis agents for both serving and evaluation. Since data analysis questions are often open-ended and hard to evaluate without human supervision, we adopt a format-prompting technique to convert each question into a closed-form format so that they can be automatically evaluated. Our extensive benchmarking of 34 LLMs uncovers the current challenges encountered in data analysis tasks. In addition, building on top of our agent framework, we develop a specialized agent, DAAgent, which surpasses GPT-3.5 by 3.9% on DABench. Evaluation datasets and toolkits for InfiAgent-DABench are released at https://github.com/InfiAgent/InfiAgent ."
++ },
++ "arXiv:2402.18679": {
++  "paperId": "01e863776846ebd1a9a7acc4a9ca24217f953aa2",
++  "externalIds": {
++   "ArXiv": "2402.18679",
++   "DBLP": "journals/corr/abs-2402-18679",
++   "DOI": "10.48550/arXiv.2402.18679",
++   "CorpusId": 268063292
++  },
++  "publicationVenue": {
++   "id": "1e33b3be-b2ab-46e9-96e8-d4eb4bad6e44",
++   "name": "Annual Meeting of the Association for Computational Linguistics",
++   "type": "conference",
++   "alternate_names": [
++    "Annu Meet Assoc Comput Linguistics",
++    "Meeting of the Association for Computational Linguistics",
++    "ACL",
++    "Meet Assoc Comput Linguistics"
++   ],
++   "url": "https://www.aclweb.org/anthology/venues/acl/"
++  },
++  "title": "Data Interpreter: An LLM Agent For Data Science",
++  "venue": "Annual Meeting of the Association for Computational Linguistics",
++  "year": 2024,
++  "citationCount": 309,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2402.18679, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "pages": "19796-19821"
++  },
++  "authors": [
++   {
++    "authorId": "2004160930",
++    "name": "Si-Rui Hong"
++   },
++   {
++    "authorId": "2289032497",
++    "name": "Yizhang Lin"
++   },
++   {
++    "authorId": "2288274546",
++    "name": "Bangbang Liu"
++   },
++   {
++    "authorId": "2253982381",
++    "name": "Bin-Hao Wu"
++   },
++   {
++    "authorId": "2390481614",
++    "name": "Danyang Li"
++   },
++   {
++    "authorId": "2288200541",
++    "name": "Jiaqi Chen"
++   },
++   {
++    "authorId": "2288348676",
++    "name": "Jiayi Zhang"
++   },
++   {
++    "authorId": "2267427792",
++    "name": "Jin-Lin Wang"
++   },
++   {
++    "authorId": "50081530",
++    "name": "Lingyao Zhang"
++   },
++   {
++    "authorId": "1722264111",
++    "name": "Ming-Chen Zhuge"
++   },
++   {
++    "authorId": "2179710545",
++    "name": "Taicheng Guo"
++   },
++   {
++    "authorId": "2278239110",
++    "name": "Tuo Zhou"
++   },
++   {
++    "authorId": "2112528845",
++    "name": "Wei Tao"
++   },
++   {
++    "authorId": "2287878279",
++    "name": "Wen-Yi Wang"
++   },
++   {
++    "authorId": "47274259",
++    "name": "Xiangru Tang"
++   },
++   {
++    "authorId": "2387162170",
++    "name": "Xiang Lu"
++   },
++   {
++    "authorId": "2288163160",
++    "name": "Xin-Bing Liang"
++   },
++   {
++    "authorId": "2288180253",
++    "name": "Yaying Fei"
++   },
++   {
++    "authorId": "2153513408",
++    "name": "Yu-Heng Cheng"
++   },
++   {
++    "authorId": "1797090",
++    "name": "Zhibin Gou"
++   },
++   {
++    "authorId": "2300805791",
++    "name": "Zong-Ze Xu"
++   },
++   {
++    "authorId": "2867893",
++    "name": "Chenglin Wu"
++   },
++   {
++    "authorId": "2266160753",
++    "name": "Li Zhang"
++   },
++   {
++    "authorId": "2324063491",
++    "name": "Min Yang"
++   },
++   {
++    "authorId": "51056401",
++    "name": "Xia-Wu Zheng"
++   }
++  ],
++  "abstract": "Large Language Model (LLM)-based agents have shown effectiveness across many applications. However, their use in data science scenarios requiring solving long-term interconnected tasks, dynamic data adjustments and domain expertise remains challenging. Previous approaches primarily focus on individual tasks, making it difficult to assess the complete data science workflow. Moreover, they struggle to handle real-time changes in intermediate data and fail to adapt dynamically to evolving task dependencies inherent to data science problems. In this paper, we present Data Interpreter, an LLM-based agent designed to automatically solve various data science problems end-to-end. Our Data Interpreter incorporates two key modules: 1) Hierarchical Graph Modeling, which breaks down complex problems into manageable subproblems, enabling dynamic node generation and graph optimization; and 2) Programmable Node Generation, a technique that refines and verifies each subproblem to iteratively improve code generation results and robustness. Extensive experiments consistently demonstrate the superiority of Data Interpreter. On InfiAgent-DABench, it achieves a 25% performance boost, raising accuracy from 75.9% to 94.9%. For machine learning and open-ended tasks, it improves performance from 88% to 95%, and from 60% to 97%, respectively. Moreover, on the MATH dataset, Data Interpreter achieves remarkable performance with a 26% improvement compared to state-of-the-art baselines. The code is available at https://github.com/geekan/MetaGPT."
++ },
++ "arXiv:2404.07738": {
++  "paperId": "51b7b3ad7645a69e3c1c80cae69473b8bd472f67",
++  "externalIds": {
++   "DBLP": "conf/naacl/BaekJCH25",
++   "ArXiv": "2404.07738",
++   "DOI": "10.48550/arXiv.2404.07738",
++   "CorpusId": 269042844
++  },
++  "publicationVenue": {
++   "id": "01103732-3808-4930-b8e4-7e9e68d5c68d",
++   "name": "North American Chapter of the Association for Computational Linguistics",
++   "type": "conference",
++   "alternate_names": [
++    "North Am Chapter Assoc Comput Linguistics",
++    "NAACL"
++   ],
++   "url": "https://www.aclweb.org/portal/naacl"
++  },
++  "title": "ResearchAgent: Iterative Research Idea Generation over Scientific Literature with Large Language Models",
++  "venue": "North American Chapter of the Association for Computational Linguistics",
++  "year": 2024,
++  "citationCount": 288,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2404.07738, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2404.07738"
++  },
++  "authors": [
++   {
++    "authorId": "90765684",
++    "name": "Jinheon Baek"
++   },
++   {
++    "authorId": "3001990",
++    "name": "S. Jauhar"
++   },
++   {
++    "authorId": "73040249",
++    "name": "Silviu Cucerzan"
++   },
++   {
++    "authorId": "2260611009",
++    "name": "Sung Ju Hwang"
++   }
++  ],
++  "abstract": "The pace of scientific research, vital for improving human life, is complex, slow, and needs specialized expertise. Meanwhile, novel, impactful research often stems from both a deep understanding of prior work, and a cross-pollination of ideas across domains and fields. To enhance the productivity of researchers, we propose ResearchAgent, which leverages the encyclopedic knowledge and linguistic reasoning capabilities of Large Language Models (LLMs) to assist them in their work. This system automatically defines novel problems, proposes methods and designs experiments, while iteratively refining them based on the feedback from collaborative LLM-powered reviewing agents. Specifically, starting with a core scientific paper, ResearchAgent is augmented not only with relevant publications by connecting information over an academic graph but also entities retrieved from a knowledge store derived from shared underlying concepts mined across numerous papers. Then, mimicking a scientific approach to improving ideas with peer discussions, we leverage multiple LLM-based ReviewingAgents that provide reviews and feedback via iterative revision processes. These reviewing agents are instantiated with human preference-aligned LLMs whose criteria for evaluation are elicited from actual human judgments via LLM prompting. We experimentally validate our ResearchAgent on scientific publications across multiple disciplines, showing its effectiveness in generating novel, clear, and valid ideas based on both human and model-based evaluation results. Our initial foray into AI-mediated scientific research has important implications for the development of future systems aimed at supporting researchers in their ideation and operationalization of novel work."
++ },
++ "arXiv:2404.13076": {
++  "paperId": "5c7f465d162aade4a4c0eefb02fd7aadeebdaf58",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2404-13076",
++   "ArXiv": "2404.13076",
++   "DOI": "10.48550/arXiv.2404.13076",
++   "CorpusId": 269293311
++  },
++  "publicationVenue": {
++   "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",
++   "name": "Neural Information Processing Systems",
++   "type": "conference",
++   "alternate_names": [
++    "Neural Inf Process Syst",
++    "NeurIPS",
++    "NIPS"
++   ],
++   "url": "http://neurips.cc/"
++  },
++  "title": "LLM Evaluators Recognize and Favor Their Own Generations",
++  "venue": "Neural Information Processing Systems",
++  "year": 2024,
++  "citationCount": 971,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2404.13076, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2404.13076"
++  },
++  "authors": [
++   {
++    "authorId": "2279023095",
++    "name": "Arjun Panickssery"
++   },
++   {
++    "authorId": "2297768298",
++    "name": "Samuel R. Bowman"
++   },
++   {
++    "authorId": "2297816489",
++    "name": "Shi Feng"
++   }
++  ],
++  "abstract": "Self-evaluation using large language models (LLMs) has proven valuable not only in benchmarking but also methods like reward modeling, constitutional AI, and self-refinement. But new biases are introduced due to the same LLM acting as both the evaluator and the evaluatee. One such bias is self-preference, where an LLM evaluator scores its own outputs higher than others' while human annotators consider them of equal quality. But do LLMs actually recognize their own outputs when they give those texts higher scores, or is it just a coincidence? In this paper, we investigate if self-recognition capability contributes to self-preference. We discover that, out of the box, LLMs such as GPT-4 and Llama 2 have non-trivial accuracy at distinguishing themselves from other LLMs and humans. By fine-tuning LLMs, we discover a linear correlation between self-recognition capability and the strength of self-preference bias; using controlled experiments, we show that the causal explanation resists straightforward confounders. We discuss how self-recognition can interfere with unbiased evaluations and AI safety more generally."
++ },
++ "arXiv:2404.17605": {
++  "paperId": "e4eb81ad222ba047770d5a90bdd7406c138c6126",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2404-17605",
++   "ArXiv": "2404.17605",
++   "DOI": "10.48550/arXiv.2404.17605",
++   "CorpusId": 269448624,
++   "PubMed": "42932458"
++  },
++  "publicationVenue": null,
++  "title": "Autonomous LLM-driven research from data to human-verifiable research papers",
++  "venue": "NEJM AI",
++  "year": 2024,
++  "citationCount": 101,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2404.17605, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "NEJM AI",
++   "pages": "\n          AIoa2400555\n        ",
++   "volume": "2 1"
++  },
++  "authors": [
++   {
++    "authorId": "2298900687",
++    "name": "Tal Ifargan"
++   },
++   {
++    "authorId": "46959587",
++    "name": "L. Hafner"
++   },
++   {
++    "authorId": "49079077",
++    "name": "M. Kern"
++   },
++   {
++    "authorId": "2298896286",
++    "name": "Ori Alcalay"
++   },
++   {
++    "authorId": "2298897594",
++    "name": "Roy Kishony"
++   }
++  ],
++  "abstract": "BACKGROUND\nArtificial intelligence (AI) promises to accelerate scientific discovery, but it remains unclear whether AI systems can perform fully autonomous research, and whether they can do so while adhering to key scientific values, such as transparency, traceability, and verifiability. The aim of this study was to develop and evaluate an AI-automation platform that performs transparent, traceable, and human-verifiable scientific research.\n\n\nMETHODS\nTo mimic human scientific practices, we built \"data-to-paper,\" an automation platform that guides interacting large language model (LLM) agents through a complete stepwise research process that starts with annotated data and results in comprehensive research papers, while programmatically backtracing information flow and allowing human oversight and interactions. The platform can run fully autonomously (in autopilot mode) or with human intervention (in copilot mode).\n\n\nRESULTS\nIn autopilot mode, provided only with annotated data, data-to-paper raised hypotheses; designed research plans; wrote and debugged analysis codes; generated and interpreted results; and created complete, information-traceable research papers. Even though the research novelty of manuscripts created by data-to-paper was relatively limited, the process demonstrated the autonomous generation of de novo quantitative insights from data, such as unraveling associations between health indicators and clinical outcomes. For simple research goals and datasets, a fully autonomous cycle can create manuscripts that independently recapitulate the findings of peer-reviewed biomedical publications without major errors in about 80 to 90% of cases. Yet, as goal or data complexity increases, human copiloting becomes critical for ensuring accuracy and overall quality. By tracking information flow through the steps, the platform creates \"data-chained\" manuscripts, in which downstream results are programmatically linked to upstream code and data, thus setting a new standard for the verifiability of scientific outputs.\n\n\nCONCLUSIONS\nOur work demonstrates the potential for AI-driven acceleration of scientific discovery in data-driven biomedical research and beyond, while enhancing, rather than jeopardizing, traceability, transparency, and verifiability."
++ },
++ "arXiv:2405.15793": {
++  "paperId": "1c3c531fc0fbe79f97f367ed3648de8467caeeaa",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2405-15793",
++   "ArXiv": "2405.15793",
++   "DOI": "10.48550/arXiv.2405.15793",
++   "CorpusId": 270063685
++  },
++  "publicationVenue": {
++   "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",
++   "name": "Neural Information Processing Systems",
++   "type": "conference",
++   "alternate_names": [
++    "Neural Inf Process Syst",
++    "NeurIPS",
++    "NIPS"
++   ],
++   "url": "http://neurips.cc/"
++  },
++  "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
++  "venue": "Neural Information Processing Systems",
++  "year": 2024,
++  "citationCount": 2323,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2405.15793, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2405.15793"
++  },
++  "authors": [
++   {
++    "authorId": "2109727379",
++    "name": "John Yang"
++   },
++   {
++    "authorId": "2155269683",
++    "name": "Carlos E. Jimenez"
++   },
++   {
++    "authorId": "2127066887",
++    "name": "Alexander Wettig"
++   },
++   {
++    "authorId": "71565033",
++    "name": "Kilian Adriano Lieret"
++   },
++   {
++    "authorId": "2256984659",
++    "name": "Shun-Yu Yao"
++   },
++   {
++    "authorId": "2135381714",
++    "name": "Karthik R. Narasimhan"
++   },
++   {
++    "authorId": "40170001",
++    "name": "Ofir Press"
++   }
++  ],
++  "abstract": "Language model (LM) agents are increasingly being used to automate complicated tasks in digital environments. Just as humans benefit from powerful software applications, such as integrated development environments, for complex tasks like software engineering, we posit that LM agents represent a new category of end users with their own needs and abilities, and would benefit from specially-built interfaces to the software they use. We investigate how interface design affects the performance of language model agents. As a result of this exploration, we introduce SWE-agent: a system that facilitates LM agents to autonomously use computers to solve software engineering tasks. SWE-agent's custom agent-computer interface (ACI) significantly enhances an agent's ability to create and edit code files, navigate entire repositories, and execute tests and other programs. We evaluate SWE-agent on SWE-bench and HumanEvalFix, achieving state-of-the-art performance on both with a pass@1 rate of 12.5% and 87.7%, respectively, far exceeding the previous state-of-the-art achieved with non-interactive LMs. Finally, we provide insight on how the design of the ACI can impact agents' behavior and performance."
++ },
++ "arXiv:2406.04244": {
++  "paperId": "0fad9dd4f0ea41732594f90209907bfad1ba506e",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2406-04244",
++   "ArXiv": "2406.04244",
++   "DOI": "10.48550/arXiv.2406.04244",
++   "CorpusId": 270285708
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Benchmark Data Contamination of Large Language Models: A Survey",
++  "venue": "arXiv.org",
++  "year": 2024,
++  "citationCount": 169,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2406.04244, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2406.04244"
++  },
++  "authors": [
++   {
++    "authorId": "2153079650",
++    "name": "Cheng Xu"
++   },
++   {
++    "authorId": "2304954016",
++    "name": "Shu-Hao Guan"
++   },
++   {
++    "authorId": "2304952028",
++    "name": "Derek Greene"
++   },
++   {
++    "authorId": "2266906186",
++    "name": "Mohand-Tahar Kechadi"
++   }
++  ],
++  "abstract": "The rapid development of Large Language Models (LLMs) like GPT-4, Claude-3, and Gemini has transformed the field of natural language processing. However, it has also resulted in a significant issue known as Benchmark Data Contamination (BDC). This occurs when language models inadvertently incorporate evaluation benchmark information from their training data, leading to inaccurate or unreliable performance during the evaluation phase of the process. This paper reviews the complex challenge of BDC in LLM evaluation and explores alternative assessment methods to mitigate the risks associated with traditional benchmarks. The paper also examines challenges and future directions in mitigating BDC risks, highlighting the complexity of the issue and the need for innovative solutions to ensure the reliability of LLM evaluation in real-world applications."
++ },
++ "arXiv:2406.12045": {
++  "paperId": "70aa016c1f68fd5c0261f26ad20017b8307650af",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2406-12045",
++   "ArXiv": "2406.12045",
++   "DOI": "10.48550/arXiv.2406.12045",
++   "CorpusId": 270562578
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "\u03c4-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
++  "venue": "arXiv.org",
++  "year": 2024,
++  "citationCount": 1635,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2406.12045, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2406.12045"
++  },
++  "authors": [
++   {
++    "authorId": "2256984659",
++    "name": "Shun-Yu Yao"
++   },
++   {
++    "authorId": "2212367248",
++    "name": "Noah Shinn"
++   },
++   {
++    "authorId": "2395098792",
++    "name": "Pedram Razavi"
++   },
++   {
++    "authorId": "2135381714",
++    "name": "Karthik R. Narasimhan"
++   }
++  ],
++  "abstract": "Existing benchmarks do not test language agents on their interaction with human users or ability to follow domain-specific rules, both of which are vital for deploying them in real world applications. We propose $\\tau$-bench, a benchmark emulating dynamic conversations between a user (simulated by language models) and a language agent provided with domain-specific API tools and policy guidelines. We employ an efficient and faithful evaluation process that compares the database state at the end of a conversation with the annotated goal state. We also propose a new metric (pass^k) to evaluate the reliability of agent behavior over multiple trials. Our experiments show that even state-of-the-art function calling agents (like gpt-4o) succeed on<50% of the tasks, and are quite inconsistent (pass^8<25% in retail). Our findings point to the need for methods that can improve the ability of agents to act consistently and follow rules reliably."
++ },
++ "arXiv:2406.12624": {
++  "paperId": "7d784c87c5e46fedbe62006d7340baa7c7bec123",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2406-12624",
++   "ArXiv": "2406.12624",
++   "DOI": "10.48550/arXiv.2406.12624",
++   "CorpusId": 270562425
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Judging the Judges: Evaluating Alignment and Vulnerabilities in LLMs-as-Judges",
++  "venue": "arXiv.org",
++  "year": 2024,
++  "citationCount": 298,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2406.12624, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2406.12624"
++  },
++  "authors": [
++   {
++    "authorId": "2307078343",
++    "name": "Aman Singh Thakur"
++   },
++   {
++    "authorId": "2310551241",
++    "name": "Kartik Choudhary"
++   },
++   {
++    "authorId": "2307074192",
++    "name": "Venkat Srinik Ramayapally"
++   },
++   {
++    "authorId": "65731722",
++    "name": "Sankaran Vaidyanathan"
++   },
++   {
++    "authorId": "3449411",
++    "name": "Dieuwke Hupkes"
++   }
++  ],
++  "abstract": "Offering a promising solution to the scalability challenges associated with human evaluation, the LLM-as-a-judge paradigm is rapidly gaining traction as an approach to evaluating large language models (LLMs). However, there are still many open questions about the strengths and weaknesses of this paradigm, and what potential biases it may hold. In this paper, we present a comprehensive study of the performance of various LLMs acting as judges, focusing on a clean scenario in which inter-human agreement is high. Investigating thirteen judge models of different model sizes and families, judging answers of nine different'examtaker models'- both base and instruction-tuned - we find that only the best (and largest) models achieve reasonable alignment with humans. However, they are still quite far behind inter-human agreement and their assigned scores may still differ with up to 5 points from human-assigned scores. In terms of their ranking of the nine exam-taker models, instead, also smaller models and even the lexical metric contains may provide a reasonable signal. Through error analysis and other studies, we identify vulnerabilities in judge models, such as their sensitivity to prompt complexity and length, and a tendency toward leniency. The fact that even the best judges differ from humans in this comparatively simple setup suggest that caution may be wise when using judges in more complex setups. Lastly, our research rediscovers the importance of using alignment metrics beyond simple percent alignment, showing that judges with high percent agreement can still assign vastly different scores."
++ },
++ "arXiv:2406.18403": {
++  "paperId": "b1e681dda709217a67316d2394bc992a26008ef7",
++  "externalIds": {
++   "DBLP": "conf/acl/BavarescoBBEFGG25",
++   "ArXiv": "2406.18403",
++   "DOI": "10.48550/arXiv.2406.18403",
++   "CorpusId": 270738074
++  },
++  "publicationVenue": {
++   "id": "1e33b3be-b2ab-46e9-96e8-d4eb4bad6e44",
++   "name": "Annual Meeting of the Association for Computational Linguistics",
++   "type": "conference",
++   "alternate_names": [
++    "Annu Meet Assoc Comput Linguistics",
++    "Meeting of the Association for Computational Linguistics",
++    "ACL",
++    "Meet Assoc Comput Linguistics"
++   ],
++   "url": "https://www.aclweb.org/anthology/venues/acl/"
++  },
++  "title": "LLMs instead of Human Judges? A Large Scale Empirical Study across 20 NLP Evaluation Tasks",
++  "venue": "Annual Meeting of the Association for Computational Linguistics",
++  "year": 2024,
++  "citationCount": 384,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2406.18403, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "pages": "238-255"
++  },
++  "authors": [
++   {
++    "authorId": "2308275559",
++    "name": "Anna Bavaresco"
++   },
++   {
++    "authorId": "2276333243",
++    "name": "Raffaella Bernardi"
++   },
++   {
++    "authorId": "2243233537",
++    "name": "Leonardo Bertolazzi"
++   },
++   {
++    "authorId": "2297018130",
++    "name": "Desmond Elliott"
++   },
++   {
++    "authorId": "2147411696",
++    "name": "Raquel Fern\u00e1ndez"
++   },
++   {
++    "authorId": "2307004114",
++    "name": "Albert Gatt"
++   },
++   {
++    "authorId": "2398366",
++    "name": "E. Ghaleb"
++   },
++   {
++    "authorId": "24068173",
++    "name": "Mario Giulianelli"
++   },
++   {
++    "authorId": "2140766524",
++    "name": "Michael Hanna"
++   },
++   {
++    "authorId": "2305819190",
++    "name": "Alexander Koller"
++   },
++   {
++    "authorId": "2348291963",
++    "name": "Andr\u00e9 F. T. Martins"
++   },
++   {
++    "authorId": "2286310256",
++    "name": "Philipp Mondorf"
++   },
++   {
++    "authorId": "2305680161",
++    "name": "Vera Neplenbroek"
++   },
++   {
++    "authorId": "2261362830",
++    "name": "S. Pezzelle"
++   },
++   {
++    "authorId": "2410144188",
++    "name": "Barbara Plank"
++   },
++   {
++    "authorId": "2257274000",
++    "name": "D. Schlangen"
++   },
++   {
++    "authorId": "3444866",
++    "name": "Alessandro Suglia"
++   },
++   {
++    "authorId": "1387995349",
++    "name": "Aditya K Surikuchi"
++   },
++   {
++    "authorId": "41071249",
++    "name": "Ece Takmaz"
++   },
++   {
++    "authorId": "50829868",
++    "name": "Alberto Testoni"
++   }
++  ],
++  "abstract": "There is an increasing trend towards evaluating NLP models with LLMs instead of human judgments, raising questions about the validity of these evaluations, as well as their reproducibility in the case of proprietary models. We provide JUDGE-BENCH, an extensible collection of 20 NLP datasets with human annotations covering a broad range of evaluated properties and types of data, and comprehensively evaluate 11 current LLMs, covering both open-weight and proprietary models, for their ability to replicate the annotations. Our evaluations show substantial variance across models and datasets. Models are reliable evaluators on some tasks, but overall display substantial variability depending on the property being evaluated, the expertise level of the human judges, and whether the language is human or model-generated. We conclude that LLMs should be carefully validated against human judgments before being used as evaluators."
++ },
++ "arXiv:2407.01502": {
++  "paperId": "edae954314571eb2913209a7e9825cdc14fd4c58",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2407-01502",
++   "ArXiv": "2407.01502",
++   "DOI": "10.48550/arXiv.2407.01502",
++   "CorpusId": 270870360
++  },
++  "publicationVenue": null,
++  "title": "AI Agents That Matter",
++  "venue": "Trans. Mach. Learn. Res.",
++  "year": 2024,
++  "citationCount": 287,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2407.01502, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Trans. Mach. Learn. Res.",
++   "volume": "2025"
++  },
++  "authors": [
++   {
++    "authorId": "39893263",
++    "name": "Sayash Kapoor"
++   },
++   {
++    "authorId": "2309175570",
++    "name": "Benedikt Stroebl"
++   },
++   {
++    "authorId": "2274102473",
++    "name": "Zachary S. Siegel"
++   },
++   {
++    "authorId": "2290073765",
++    "name": "Nitya Nadgir"
++   },
++   {
++    "authorId": "2064013115",
++    "name": "Arvind Narayanan"
++   }
++  ],
++  "abstract": "AI agents are an exciting new research direction, and agent development is driven by benchmarks. Our analysis of current agent benchmarks and evaluation practices reveals several shortcomings that hinder their usefulness in real-world applications. First, there is a narrow focus on accuracy without attention to other metrics. As a result, SOTA agents are needlessly complex and costly, and the community has reached mistaken conclusions about the sources of accuracy gains. Our focus on cost in addition to accuracy motivates the new goal of jointly optimizing the two metrics. We design and implement one such optimization, showing its potential to greatly reduce cost while maintaining accuracy. Second, the benchmarking needs of model and downstream developers have been conflated, making it hard to identify which agent would be best suited for a particular application. Third, many agent benchmarks have inadequate holdout sets, and sometimes none at all. This has led to agents that are fragile because they take shortcuts and overfit to the benchmark in various ways. We prescribe a principled framework for avoiding overfitting. Finally, there is a lack of standardization in evaluation practices, leading to a pervasive lack of reproducibility. We hope that the steps we introduce for addressing these shortcomings will spur the development of agents that are useful in the real world and not just accurate on benchmarks."
++ },
++ "arXiv:2407.01725": {
++  "paperId": "48c83799530dc523ee01e6c1c40ad577d5c10a16",
++  "externalIds": {
++   "DBLP": "conf/iclr/MajumderS0MMPVK25",
++   "ArXiv": "2407.01725",
++   "DOI": "10.48550/arXiv.2407.01725",
++   "CorpusId": 270878232
++  },
++  "publicationVenue": {
++   "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",
++   "name": "International Conference on Learning Representations",
++   "type": "conference",
++   "alternate_names": [
++    "Int Conf Learn Represent",
++    "ICLR"
++   ],
++   "url": "https://iclr.cc/"
++  },
++  "title": "DiscoveryBench: Towards Data-Driven Discovery with Large Language Models",
++  "venue": "International Conference on Learning Representations",
++  "year": 2024,
++  "citationCount": 118,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2407.01725, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2407.01725"
++  },
++  "authors": [
++   {
++    "authorId": "3165738",
++    "name": "Bodhisattwa Prasad Majumder"
++   },
++   {
++    "authorId": "2762792",
++    "name": "Harshit Surana"
++   },
++   {
++    "authorId": "2057234493",
++    "name": "Dhruv Agarwal"
++   },
++   {
++    "authorId": "40135250",
++    "name": "Bhavana Dalvi"
++   },
++   {
++    "authorId": "2362917398",
++    "name": "A. Meena"
++   },
++   {
++    "authorId": "2309247064",
++    "name": "Aryan Prakhar"
++   },
++   {
++    "authorId": "2309246364",
++    "name": "Tirth Vora"
++   },
++   {
++    "authorId": "2236429",
++    "name": "Tushar Khot"
++   },
++   {
++    "authorId": "48229640",
++    "name": "Ashish Sabharwal"
++   },
++   {
++    "authorId": "2265493391",
++    "name": "Peter Clark"
++   }
++  ],
++  "abstract": "Can the rapid advances in code generation, function calling, and data analysis using large language models (LLMs) help automate the search and verification of hypotheses purely from a set of provided datasets? To evaluate this question, we present DiscoveryBench, the first comprehensive benchmark that formalizes the multi-step process of data-driven discovery. The benchmark is designed to systematically assess current model capabilities in discovery tasks and provide a useful resource for improving them. Our benchmark contains 264 tasks collected across 6 diverse domains, such as sociology and engineering, by manually deriving discovery workflows from published papers to approximate the real-world challenges faced by researchers, where each task is defined by a dataset, its metadata, and a discovery goal in natural language. We additionally provide 903 synthetic tasks to conduct controlled evaluations across task complexity. Furthermore, our structured formalism of data-driven discovery enables a facet-based evaluation that provides useful insights into different failure modes. We evaluate several popular LLM-based reasoning frameworks using both open and closed LLMs as baselines on DiscoveryBench and find that even the best system scores only 25%. Our benchmark, thus, illustrates the challenges in autonomous data-driven discovery and serves as a valuable resource for the community to make progress."
++ },
++ "arXiv:2407.13168": {
++  "paperId": "5bf3ea7b0825424c3c01e48e8fcd8215c950677e",
++  "externalIds": {
++   "ArXiv": "2407.13168",
++   "DBLP": "conf/nips/TianGZCFGHJKLLL24",
++   "DOI": "10.48550/arXiv.2407.13168",
++   "CorpusId": 271270048
++  },
++  "publicationVenue": {
++   "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",
++   "name": "Neural Information Processing Systems",
++   "type": "conference",
++   "alternate_names": [
++    "Neural Inf Process Syst",
++    "NeurIPS",
++    "NIPS"
++   ],
++   "url": "http://neurips.cc/"
++  },
++  "title": "SciCode: A Research Coding Benchmark Curated by Scientists",
++  "venue": "Neural Information Processing Systems",
++  "year": 2024,
++  "citationCount": 203,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2407.13168, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2407.13168"
++  },
++  "authors": [
++   {
++    "authorId": "2311882756",
++    "name": "Min-Yang Tian"
++   },
++   {
++    "authorId": "49715441",
++    "name": "Luyu Gao"
++   },
++   {
++    "authorId": "2145401799",
++    "name": "S. Zhang"
++   },
++   {
++    "authorId": "2311990357",
++    "name": "Xinan Chen"
++   },
++   {
++    "authorId": "2931077",
++    "name": "Cun-Wei Fan"
++   },
++   {
++    "authorId": "2311997990",
++    "name": "Xuefei Guo"
++   },
++   {
++    "authorId": "2281073123",
++    "name": "R. Haas"
++   },
++   {
++    "authorId": "2311880186",
++    "name": "Pan Ji"
++   },
++   {
++    "authorId": "35701505",
++    "name": "Kittithat Krongchon"
++   },
++   {
++    "authorId": "2374330170",
++    "name": "Yao-Hui Li"
++   },
++   {
++    "authorId": "2152940692",
++    "name": "Shengyan Liu"
++   },
++   {
++    "authorId": "2311885961",
++    "name": "Di Luo"
++   },
++   {
++    "authorId": "2325196615",
++    "name": "Yu-Tao Ma"
++   },
++   {
++    "authorId": "2182144332",
++    "name": "Hao Tong"
++   },
++   {
++    "authorId": "2405014449",
++    "name": "Kha Trinh"
++   },
++   {
++    "authorId": "2312693814",
++    "name": "Chenyu Tian"
++   },
++   {
++    "authorId": "2311998534",
++    "name": "Zihan Wang"
++   },
++   {
++    "authorId": "2312000055",
++    "name": "Bohao Wu"
++   },
++   {
++    "authorId": "2088591255",
++    "name": "Yanyu Xiong"
++   },
++   {
++    "authorId": "2383429721",
++    "name": "Sheng Yin"
++   },
++   {
++    "authorId": "2268183167",
++    "name": "Min Zhu"
++   },
++   {
++    "authorId": "71565033",
++    "name": "Kilian Adriano Lieret"
++   },
++   {
++    "authorId": "7774667",
++    "name": "Yanxin Lu"
++   },
++   {
++    "authorId": "2217570953",
++    "name": "Genglin Liu"
++   },
++   {
++    "authorId": "2312034033",
++    "name": "Yu-Feng Du"
++   },
++   {
++    "authorId": "2242959702",
++    "name": "Tian-Hua Tao"
++   },
++   {
++    "authorId": "40170001",
++    "name": "Ofir Press"
++   },
++   {
++    "authorId": "2310820252",
++    "name": "James P. Callan"
++   },
++   {
++    "authorId": "2249761654",
++    "name": "Eliu A. Huerta"
++   },
++   {
++    "authorId": "2254026935",
++    "name": "Hao Peng"
++   }
++  ],
++  "abstract": "Since language models (LMs) now outperform average humans on many challenging tasks, it has become increasingly difficult to develop challenging, high-quality, and realistic evaluations. We address this issue by examining LMs' capabilities to generate code for solving real scientific research problems. Incorporating input from scientists and AI researchers in 16 diverse natural science sub-fields, including mathematics, physics, chemistry, biology, and materials science, we created a scientist-curated coding benchmark, SciCode. The problems in SciCode naturally factorize into multiple subproblems, each involving knowledge recall, reasoning, and code synthesis. In total, SciCode contains 338 subproblems decomposed from 80 challenging main problems. It offers optional descriptions specifying useful scientific background information and scientist-annotated gold-standard solutions and test cases for evaluation. Claude3.5-Sonnet, the best-performing model among those tested, can solve only 4.6% of the problems in the most realistic setting. We believe that SciCode demonstrates both contemporary LMs' progress towards becoming helpful scientific assistants and sheds light on the development and evaluation of scientific AI in the future."
++ },
++ "arXiv:2407.16741": {
++  "paperId": "1d07e5b6f978cf69c0186f3d5f434fa92d471e46",
++  "externalIds": {
++   "ArXiv": "2407.16741",
++   "DBLP": "conf/iclr/0001LSXTZPSLSTL25",
++   "CorpusId": 271404773
++  },
++  "publicationVenue": {
++   "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",
++   "name": "International Conference on Learning Representations",
++   "type": "conference",
++   "alternate_names": [
++    "Int Conf Learn Represent",
++    "ICLR"
++   ],
++   "url": "https://iclr.cc/"
++  },
++  "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
++  "venue": "International Conference on Learning Representations",
++  "year": 2024,
++  "citationCount": 1291,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2407.16741, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": null,
++  "authors": [
++   {
++    "authorId": "2144803999",
++    "name": "Xingyao Wang"
++   },
++   {
++    "authorId": "2313084397",
++    "name": "Bo-Xuan Li"
++   },
++   {
++    "authorId": "41077032",
++    "name": "Yufan Song"
++   },
++   {
++    "authorId": "40027632",
++    "name": "Frank F. Xu"
++   },
++   {
++    "authorId": "47274259",
++    "name": "Xiangru Tang"
++   },
++   {
++    "authorId": "1722264111",
++    "name": "Ming-Chen Zhuge"
++   },
++   {
++    "authorId": "2152946868",
++    "name": "Jiayi Pan"
++   },
++   {
++    "authorId": "2290031618",
++    "name": "Yue-Qi Song"
++   },
++   {
++    "authorId": "2132475886",
++    "name": "Bo-Wen Li"
++   },
++   {
++    "authorId": "2312910329",
++    "name": "Jaskirat Singh"
++   },
++   {
++    "authorId": "2313308109",
++    "name": "Hoang H. Tran"
++   },
++   {
++    "authorId": "2313768177",
++    "name": "Fuqiang Li"
++   },
++   {
++    "authorId": "2313325218",
++    "name": "Ren Ma"
++   },
++   {
++    "authorId": "2312719297",
++    "name": "Mingzhang Zheng"
++   },
++   {
++    "authorId": "2226120351",
++    "name": "Bill Qian"
++   },
++   {
++    "authorId": "2308941736",
++    "name": "Yanjun Shao"
++   },
++   {
++    "authorId": "2037383772",
++    "name": "Niklas Muennighoff"
++   },
++   {
++    "authorId": "48378494",
++    "name": "Yi-Zhe Zhang"
++   },
++   {
++    "authorId": "2321578848",
++    "name": "Bin-Yuan Hui"
++   },
++   {
++    "authorId": "2312866505",
++    "name": "Junyang Lin"
++   },
++   {
++    "authorId": "2312759262",
++    "name": "Robert Brennan"
++   },
++   {
++    "authorId": "2254026935",
++    "name": "Hao Peng"
++   },
++   {
++    "authorId": "2312388202",
++    "name": "H. Ji"
++   },
++   {
++    "authorId": "2285194103",
++    "name": "Graham Neubig"
++   }
++  ],
++  "abstract": "Software is one of the most powerful tools that we humans have at our disposal; it allows a skilled programmer to interact with the world in complex and profound ways. At the same time, thanks to improvements in large language models (LLMs), there has also been a rapid development in AI agents that interact with and affect change in their surrounding environments. In this paper, we introduce OpenHands (f.k.a. OpenDevin), a platform for the development of powerful and flexible AI agents that interact with the world in similar ways to those of a human developer: by writing code, interacting with a command line, and browsing the web. We describe how the platform allows for the implementation of new agents, safe interaction with sandboxed environments for code execution, coordination between multiple agents, and incorporation of evaluation benchmarks. Based on our currently incorporated benchmarks, we perform an evaluation of agents over 15 challenging tasks, including software engineering (e.g., SWE-BENCH) and web browsing (e.g., WEBARENA), among others. Released under the permissive MIT license, OpenHands is a community project spanning academia and industry with more than 2.1K contributions from over 188 contributors."
++ },
++ "arXiv:2408.06292": {
++  "paperId": "33161a5a9b5dcb635b5a97475e6a6209a69ada7d",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2408-06292",
++   "ArXiv": "2408.06292",
++   "CorpusId": 271854887
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery",
++  "venue": "arXiv.org",
++  "year": 2024,
++  "citationCount": 1342,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2408.06292, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2408.06292"
++  },
++  "authors": [
++   {
++    "authorId": "4420863",
++    "name": "Chris Lu"
++   },
++   {
++    "authorId": "2110752472",
++    "name": "Cong Lu"
++   },
++   {
++    "authorId": "2266838849",
++    "name": "R. Lange"
++   },
++   {
++    "authorId": "2275156099",
++    "name": "J. Foerster"
++   },
++   {
++    "authorId": "2303255568",
++    "name": "Jeff Clune"
++   },
++   {
++    "authorId": "2315921540",
++    "name": "David Ha"
++   }
++  ],
++  "abstract": "One of the grand challenges of artificial general intelligence is developing agents capable of conducting scientific research and discovering new knowledge. While frontier models have already been used as aides to human scientists, e.g. for brainstorming ideas, writing code, or prediction tasks, they still conduct only a small part of the scientific process. This paper presents the first comprehensive framework for fully automatic scientific discovery, enabling frontier large language models to perform research independently and communicate their findings. We introduce The AI Scientist, which generates novel research ideas, writes code, executes experiments, visualizes results, describes its findings by writing a full scientific paper, and then runs a simulated review process for evaluation. In principle, this process can be repeated to iteratively develop ideas in an open-ended fashion, acting like the human scientific community. We demonstrate its versatility by applying it to three distinct subfields of machine learning: diffusion modeling, transformer-based language modeling, and learning dynamics. Each idea is implemented and developed into a full paper at a cost of less than $15 per paper. To evaluate the generated papers, we design and validate an automated reviewer, which we show achieves near-human performance in evaluating paper scores. The AI Scientist can produce papers that exceed the acceptance threshold at a top machine learning conference as judged by our automated reviewer. This approach signifies the beginning of a new era in scientific discovery in machine learning: bringing the transformative benefits of AI agents to the entire research process of AI itself, and taking us closer to a world where endless affordable creativity and innovation can be unleashed on the world's most challenging problems. Our code is open-sourced at https://github.com/SakanaAI/AI-Scientist"
++ },
++ "arXiv:2408.09667": {
++  "paperId": "3cabe88c2ff476d35cd179525c7dfe474b4b75bb",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2408-09667",
++   "ArXiv": "2408.09667",
++   "DOI": "10.48550/arXiv.2408.09667",
++   "CorpusId": 271903486
++  },
++  "publicationVenue": {
++   "id": "41bf9ed3-85b3-4c90-b015-150e31690253",
++   "name": "Conference on Empirical Methods in Natural Language Processing",
++   "type": "conference",
++   "alternate_names": [
++    "Empir Method Nat Lang Process",
++    "Empirical Methods in Natural Language Processing",
++    "Conf Empir Method Nat Lang Process",
++    "EMNLP"
++   ],
++   "url": "https://www.aclweb.org/portal/emnlp"
++  },
++  "title": "BLADE: Benchmarking Language Model Agents for Data-Driven Science",
++  "venue": "Conference on Empirical Methods in Natural Language Processing",
++  "year": 2024,
++  "citationCount": 96,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2408.09667, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2408.09667"
++  },
++  "authors": [
++   {
++    "authorId": "6639253",
++    "name": "Ken Gu"
++   },
++   {
++    "authorId": "2242899679",
++    "name": "Ruo-Xi Shang"
++   },
++   {
++    "authorId": "2316411779",
++    "name": "Ruien Jiang"
++   },
++   {
++    "authorId": "2316431927",
++    "name": "Keying Kuang"
++   },
++   {
++    "authorId": "2357993081",
++    "name": "Richard Lin"
++   },
++   {
++    "authorId": "2316432426",
++    "name": "Donghe Lyu"
++   },
++   {
++    "authorId": "2340586557",
++    "name": "Yue Mao"
++   },
++   {
++    "authorId": "2457265599",
++    "name": "Youbang Pan"
++   },
++   {
++    "authorId": "2365051240",
++    "name": "Teng Wu"
++   },
++   {
++    "authorId": "3276708",
++    "name": "Jia-Qian Yu"
++   },
++   {
++    "authorId": "2239169380",
++    "name": "Yikun Zhang"
++   },
++   {
++    "authorId": "2159536044",
++    "name": "Tian-Mai M. Zhang"
++   },
++   {
++    "authorId": "2404006955",
++    "name": "Lanyi Zhu"
++   },
++   {
++    "authorId": "2297185552",
++    "name": "Mike A. Merrill"
++   },
++   {
++    "authorId": "2275206828",
++    "name": "J. Heer"
++   },
++   {
++    "authorId": "2316432491",
++    "name": "Tim Althoff"
++   }
++  ],
++  "abstract": "Data-driven scientific discovery requires the iterative integration of scientific domain knowledge, statistical expertise, and an understanding of data semantics to make nuanced analytical decisions, e.g., about which variables, transformations, and statistical models to consider. LM-based agents equipped with planning, memory, and code execution capabilities have the potential to support data-driven science. However, evaluating agents on such open-ended tasks is challenging due to multiple valid approaches, partially correct steps, and different ways to express the same decisions. To address these challenges, we present BLADE, a benchmark to automatically evaluate agents'multifaceted approaches to open-ended research questions. BLADE consists of 12 datasets and research questions drawn from existing scientific literature, with ground truth collected from independent analyses by expert data scientists and researchers. To automatically evaluate agent responses, we developed corresponding computational methods to match different representations of analyses to this ground truth. Though language models possess considerable world knowledge, our evaluation shows that they are often limited to basic analyses. However, agents capable of interacting with the underlying data demonstrate improved, but still non-optimal, diversity in their analytical decision making. Our work enables the evaluation of agents for data-driven science and provides researchers deeper insights into agents'analysis approaches."
++ },
++ "arXiv:2409.05556": {
++  "paperId": "0f6439b95aed189908c628f1103dbcef8541cc2e",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2409-05556",
++   "ArXiv": "2409.05556",
++   "DOI": "10.48550/arXiv.2409.05556",
++   "CorpusId": 272524655
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "SciAgents: Automating scientific discovery through multi-agent intelligent graph reasoning",
++  "venue": "arXiv.org",
++  "year": 2024,
++  "citationCount": 104,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2409.05556, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2409.05556"
++  },
++  "authors": [
++   {
++    "authorId": "2310665423",
++    "name": "Alireza Ghafarollahi"
++   },
++   {
++    "authorId": "2269910426",
++    "name": "Markus J. Buehler"
++   }
++  ],
++  "abstract": "A key challenge in artificial intelligence is the creation of systems capable of autonomously advancing scientific understanding by exploring novel domains, identifying complex patterns, and uncovering previously unseen connections in vast scientific data. In this work, we present SciAgents, an approach that leverages three core concepts: (1) the use of large-scale ontological knowledge graphs to organize and interconnect diverse scientific concepts, (2) a suite of large language models (LLMs) and data retrieval tools, and (3) multi-agent systems with in-situ learning capabilities. Applied to biologically inspired materials, SciAgents reveals hidden interdisciplinary relationships that were previously considered unrelated, achieving a scale, precision, and exploratory power that surpasses traditional human-driven research methods. The framework autonomously generates and refines research hypotheses, elucidating underlying mechanisms, design principles, and unexpected material properties. By integrating these capabilities in a modular fashion, the intelligent system yields material discoveries, critique and improve existing hypotheses, retrieve up-to-date data about existing research, and highlights their strengths and limitations. Our case studies demonstrate scalable capabilities to combine generative AI, ontological representations, and multi-agent modeling, harnessing a `swarm of intelligence' similar to biological systems. This provides new avenues for materials discovery and accelerates the development of advanced materials by unlocking Nature's design principles."
++ },
++ "arXiv:2409.07703": {
++  "paperId": "395c978221a21ee47c84a40a2ef11fb4d012fca1",
++  "externalIds": {
++   "ArXiv": "2409.07703",
++   "CorpusId": 272599760
++  },
++  "publicationVenue": null,
++  "title": "DSBench: How Far Are Data Science Agents from Becoming Data Science Experts?",
++  "venue": "",
++  "year": 2024,
++  "citationCount": 141,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2409.07703, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": null,
++  "journal": null,
++  "authors": [
++   {
++    "authorId": "2257043219",
++    "name": "Liqiang Jing"
++   },
++   {
++    "authorId": "2320838217",
++    "name": "Zhehui Huang"
++   },
++   {
++    "authorId": "2250363276",
++    "name": "Xiao-Yang Wang"
++   },
++   {
++    "authorId": "2087264100",
++    "name": "Wenlin Yao"
++   },
++   {
++    "authorId": "2265843326",
++    "name": "Wen-Hao Yu"
++   },
++   {
++    "authorId": "2311458574",
++    "name": "Kai-Xin Ma"
++   },
++   {
++    "authorId": "2254831297",
++    "name": "Hong-Ming Zhang"
++   },
++   {
++    "authorId": "2265463653",
++    "name": "Xinya Du"
++   },
++   {
++    "authorId": "2261392681",
++    "name": "Dong Yu"
++   }
++  ],
++  "abstract": "Large Language Models (LLMs) and Large Vision-Language Models (LVLMs) have demonstrated impressive language/vision reasoning abilities, igniting the recent trend of building agents for targeted applications such as shopping assistants or AI software engineers. Recently, many data science benchmarks have been proposed to investigate their performance in the data science domain. However, existing data science benchmarks still fall short when compared to real-world data science applications due to their simplified settings. To bridge this gap, we introduce DSBench, a comprehensive benchmark designed to evaluate data science agents with realistic tasks. This benchmark includes 466 data analysis tasks and 74 data modeling tasks, sourced from Eloquence and Kaggle competitions. DSBench offers a realistic setting by encompassing long contexts, multimodal task backgrounds, reasoning with large data files and multi-table structures, and performing end-to-end data modeling tasks. Our evaluation of state-of-the-art LLMs, LVLMs, and agents shows that they struggle with most tasks, with the best agent solving only 34.12% of data analysis tasks and achieving a 34.74% Relative Performance Gap (RPG). These findings underscore the need for further advancements in developing more practical, intelligent, and autonomous data science agents."
++ },
++ "arXiv:2409.11363": {
++  "paperId": "4c913d59d150fe7581386b87dfd9f90448a9adee",
++  "externalIds": {
++   "ArXiv": "2409.11363",
++   "DBLP": "journals/tmlr/SiegelKNSN24",
++   "DOI": "10.48550/arXiv.2409.11363",
++   "CorpusId": 272694423
++  },
++  "publicationVenue": null,
++  "title": "CORE-Bench: Fostering the Credibility of Published Research Through a Computational Reproducibility Agent Benchmark",
++  "venue": "Trans. Mach. Learn. Res.",
++  "year": 2024,
++  "citationCount": 147,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2409.11363, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2409.11363"
++  },
++  "authors": [
++   {
++    "authorId": "2274102473",
++    "name": "Zachary S. Siegel"
++   },
++   {
++    "authorId": "39893263",
++    "name": "Sayash Kapoor"
++   },
++   {
++    "authorId": "2290073765",
++    "name": "Nitya Nadgir"
++   },
++   {
++    "authorId": "2309175570",
++    "name": "Benedikt Stroebl"
++   },
++   {
++    "authorId": "2064013115",
++    "name": "Arvind Narayanan"
++   }
++  ],
++  "abstract": "AI agents have the potential to aid users on a variety of consequential tasks, including conducting scientific research. To spur the development of useful agents, we need benchmarks that are challenging, but more crucially, directly correspond to real-world tasks of interest. This paper introduces such a benchmark, designed to measure the accuracy of AI agents in tackling a crucial yet surprisingly challenging aspect of scientific research: computational reproducibility. This task, fundamental to the scientific process, involves reproducing the results of a study using the provided code and data. We introduce CORE-Bench (Computational Reproducibility Agent Benchmark), a benchmark consisting of 270 tasks based on 90 scientific papers across three disciplines (computer science, social science, and medicine). Tasks in CORE-Bench consist of three difficulty levels and include both language-only and vision-language tasks. We provide an evaluation system to measure the accuracy of agents in a fast and parallelizable way, saving days of evaluation time for each run compared to a sequential implementation. We evaluated two baseline agents: the general-purpose AutoGPT and a task-specific agent called CORE-Agent. We tested both variants using two underlying language models: GPT-4o and GPT-4o-mini. The best agent achieved an accuracy of 21% on the hardest task, showing the vast scope for improvement in automating routine scientific tasks. Having agents that can reproduce existing work is a necessary step towards building agents that can conduct novel research and could verify and improve the performance of other research agents. We hope that CORE-Bench can improve the state of reproducibility and spur the development of future research agents."
++ },
++ "arXiv:2410.05080": {
++  "paperId": "45b7c7448768b51b6dbd9b76495c9cd9d110bd91",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2410-05080",
++   "ArXiv": "2410.05080",
++   "DOI": "10.48550/arXiv.2410.05080",
++   "CorpusId": 273186311
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "ScienceAgentBench: Toward Rigorous Assessment of Language Agents for Data-Driven Scientific Discovery",
++  "venue": "arXiv.org",
++  "year": 2024,
++  "citationCount": 275,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2410.05080, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2410.05080"
++  },
++  "authors": [
++   {
++    "authorId": "11832104",
++    "name": "Ziru Chen"
++   },
++   {
++    "authorId": "2115939351",
++    "name": "Shijie Chen"
++   },
++   {
++    "authorId": "2266397759",
++    "name": "Yu-Ting Ning"
++   },
++   {
++    "authorId": "2261249755",
++    "name": "Qian-Heng Zhang"
++   },
++   {
++    "authorId": "7425689",
++    "name": "Boshi Wang"
++   },
++   {
++    "authorId": "2754371",
++    "name": "Botao Yu"
++   },
++   {
++    "authorId": "2324840035",
++    "name": "Yi-Fei Li"
++   },
++   {
++    "authorId": "2284224390",
++    "name": "Zeyi Liao"
++   },
++   {
++    "authorId": "2325857726",
++    "name": "Chen Wei"
++   },
++   {
++    "authorId": "1605906053",
++    "name": "Zi-Tong Lu"
++   },
++   {
++    "authorId": "2281742135",
++    "name": "Vishal Dey"
++   },
++   {
++    "authorId": "2125081864",
++    "name": "Ming-Yi Xue"
++   },
++   {
++    "authorId": "49780607",
++    "name": "Frazier N. Baker"
++   },
++   {
++    "authorId": "2302148833",
++    "name": "Benjamin Burns"
++   },
++   {
++    "authorId": "2348516502",
++    "name": "Daniel Adu-Ampratwum"
++   },
++   {
++    "authorId": "2325131030",
++    "name": "Xuhui Huang"
++   },
++   {
++    "authorId": "2281745183",
++    "name": "Xia Ning"
++   },
++   {
++    "authorId": "2265362739",
++    "name": "Song Gao"
++   },
++   {
++    "authorId": "1758652",
++    "name": "Yu Su"
++   },
++   {
++    "authorId": "2278123075",
++    "name": "Huan Sun"
++   }
++  ],
++  "abstract": "The advancements of large language models (LLMs) have piqued growing interest in developing LLM-based language agents to automate scientific discovery end-to-end, which has sparked both excitement and skepticism about their true capabilities. In this work, we call for rigorous assessment of agents on individual tasks in a scientific workflow before making bold claims on end-to-end automation. To this end, we present ScienceAgentBench, a new benchmark for evaluating language agents for data-driven scientific discovery. To ensure the scientific authenticity and real-world relevance of our benchmark, we extract 102 tasks from 44 peer-reviewed publications in four disciplines and engage nine subject matter experts to validate them. We unify the target output for every task to a self-contained Python program file and employ an array of evaluation metrics to examine the generated programs, execution results, and costs. Each task goes through multiple rounds of manual validation by annotators and subject matter experts to ensure its annotation quality and scientific plausibility. We also propose two effective strategies to mitigate data contamination concerns. Using ScienceAgentBench, we evaluate five open-weight and proprietary LLMs, each with three frameworks: direct prompting, OpenHands CodeAct, and self-debug. Given three attempts for each task, the best-performing agent can only solve 32.4% of the tasks independently and 34.3% with expert-provided knowledge. In addition, we evaluate OpenAI o1-preview with direct prompting and self-debug, which can boost the performance to 42.2%, demonstrating the effectiveness of increasing inference-time compute but with more than 10 times the cost of other LLMs. Still, our results underscore the limitations of current language agents in generating code for data-driven discovery, let alone end-to-end automation for scientific research."
++ },
++ "arXiv:2410.06992": {
++  "paperId": "05f4f076a8f277f46dff9b6abc3feff1fcf31ff5",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2410-06992",
++   "ArXiv": "2410.06992",
++   "DOI": "10.48550/arXiv.2410.06992",
++   "CorpusId": 273228646
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "SWE-Bench+: Enhanced Coding Benchmark for LLMs",
++  "venue": "arXiv.org",
++  "year": 2024,
++  "citationCount": 134,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2410.06992, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2410.06992"
++  },
++  "authors": [
++   {
++    "authorId": "2115012783",
++    "name": "Reem Aleithan"
++   },
++   {
++    "authorId": "2325736475",
++    "name": "Haoran Xue"
++   },
++   {
++    "authorId": "2224465665",
++    "name": "Mohammad Mahdi Mohajer"
++   },
++   {
++    "authorId": "2325097481",
++    "name": "Elijah Nnorom"
++   },
++   {
++    "authorId": "2325093115",
++    "name": "Gias Uddin"
++   },
++   {
++    "authorId": "2257127006",
++    "name": "Song Wang"
++   }
++  ],
++  "abstract": "Large Language Models (LLMs) in Software Engineering (SE) can offer assistance for coding. To facilitate a rigorous evaluation of LLMs in practical coding contexts, Carlos et al. introduced the SWE-bench dataset, which comprises 2,294 real-world GitHub issues and their corresponding pull requests, collected from 12 widely used Python repositories. Several impressive LLM-based toolkits recently are developed and evaluated on this dataset. However, a systematic evaluation of the quality of SWE-bench remains missing. In this paper, we addressed this gap by presenting an empirical analysis of the SWE-bench dataset. We conducted a manual screening of instances where SWEAgent + GPT-4 successfully resolved issues by comparing the model-generated patches with the actual pull requests. SWE-Agent+GPT-4 was at the top of SWE-bench leaderboard during the time of our study. Our analysis reveals some critical issues with the SWE-bench dataset: 1) 32.67% of the successful patches involve cheating as the solutions were directly provided in the issue report or the comments. We refer to as solution leakage problem. 2) 31.08% of the passed patches are suspicious patches due to weak test cases, i.e., the tests were not adequate to verify the correctness of a patch. When we filtered out these problematic issues, the resolution rate of SWE-Agent+GPT-4 dropped from 12.47% to 3.97%. We also observed that the same data quality issues also exist in the two variants of SWE-bench, i.e., SWE-bench Lite and SWE-Bench Verified. In addition, over 94% of the issues were created before LLM's knowledge cutoff dates, posing potential data leakage issues."
++ },
++ "arXiv:2410.07095": {
++  "paperId": "7c44b7fdcec2e517799f6c54f6ba42bf1a89d2e6",
++  "externalIds": {
++   "ArXiv": "2410.07095",
++   "DBLP": "journals/corr/abs-2410-07095",
++   "DOI": "10.48550/arXiv.2410.07095",
++   "CorpusId": 273233550
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering",
++  "venue": "arXiv.org",
++  "year": 2024,
++  "citationCount": 532,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2410.07095, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2410.07095"
++  },
++  "authors": [
++   {
++    "authorId": "2325723546",
++    "name": "Jun Shern Chan"
++   },
++   {
++    "authorId": "2325154459",
++    "name": "Neil Chowdhury"
++   },
++   {
++    "authorId": "2325151487",
++    "name": "Oliver Jaffe"
++   },
++   {
++    "authorId": "2325157923",
++    "name": "James Aung"
++   },
++   {
++    "authorId": "2231409794",
++    "name": "Dane Sherburn"
++   },
++   {
++    "authorId": "2325158061",
++    "name": "E. Mays"
++   },
++   {
++    "authorId": "2168285763",
++    "name": "Giulio Starace"
++   },
++   {
++    "authorId": "2325207065",
++    "name": "Kevin Liu"
++   },
++   {
++    "authorId": "2325157894",
++    "name": "Leon Maksin"
++   },
++   {
++    "authorId": "90169542",
++    "name": "Tejal Patwardhan"
++   },
++   {
++    "authorId": "2325162875",
++    "name": "Lilian Weng"
++   },
++   {
++    "authorId": "2258929982",
++    "name": "Aleksander Madry"
++   }
++  ],
++  "abstract": "We introduce MLE-bench, a benchmark for measuring how well AI agents perform at machine learning engineering. To this end, we curate 75 ML engineering-related competitions from Kaggle, creating a diverse set of challenging tasks that test real-world ML engineering skills such as training models, preparing datasets, and running experiments. We establish human baselines for each competition using Kaggle's publicly available leaderboards. We use open-source agent scaffolds to evaluate several frontier language models on our benchmark, finding that the best-performing setup--OpenAI's o1-preview with AIDE scaffolding--achieves at least the level of a Kaggle bronze medal in 16.9% of competitions. In addition to our main results, we investigate various forms of resource scaling for AI agents and the impact of contamination from pre-training. We open-source our benchmark code (github.com/openai/mle-bench/) to facilitate future research in understanding the ML engineering capabilities of AI agents."
++ },
++ "arXiv:2410.07331": {
++  "paperId": "0e28243cc36df0dfa857770e5a0a560275b225bf",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2410-07331",
++   "ArXiv": "2410.07331",
++   "ACL": "2024.emnlp-main.748",
++   "DOI": "10.48550/arXiv.2410.07331",
++   "CorpusId": 273234039
++  },
++  "publicationVenue": {
++   "id": "41bf9ed3-85b3-4c90-b015-150e31690253",
++   "name": "Conference on Empirical Methods in Natural Language Processing",
++   "type": "conference",
++   "alternate_names": [
++    "Empir Method Nat Lang Process",
++    "Empirical Methods in Natural Language Processing",
++    "Conf Empir Method Nat Lang Process",
++    "EMNLP"
++   ],
++   "url": "https://www.aclweb.org/portal/emnlp"
++  },
++  "title": "DA-Code: Agent Data Science Code Generation Benchmark for Large Language Models",
++  "venue": "Conference on Empirical Methods in Natural Language Processing",
++  "year": 2024,
++  "citationCount": 73,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2410.07331, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "pages": "13487-13521"
++  },
++  "authors": [
++   {
++    "authorId": "2261394934",
++    "name": "Yiming Huang"
++   },
++   {
++    "authorId": "2325193187",
++    "name": "Jianwen Luo"
++   },
++   {
++    "authorId": "2308879905",
++    "name": "Yang Yu"
++   },
++   {
++    "authorId": "2320286082",
++    "name": "Yi-Tong Zhang"
++   },
++   {
++    "authorId": "2257004176",
++    "name": "Fangyu Lei"
++   },
++   {
++    "authorId": "2216484616",
++    "name": "Yifan Wei"
++   },
++   {
++    "authorId": "1954845",
++    "name": "Shi-Zhu He"
++   },
++   {
++    "authorId": "2319173963",
++    "name": "Li-Fu Huang"
++   },
++   {
++    "authorId": "2325642391",
++    "name": "Xiao Liu"
++   },
++   {
++    "authorId": "2284076390",
++    "name": "Jun Zhao"
++   },
++   {
++    "authorId": "2283274776",
++    "name": "Kang Liu"
++   }
++  ],
++  "abstract": "We introduce DA-Code, a code generation benchmark specifically designed to assess LLMs on agent-based data science tasks. This benchmark features three core elements: First, the tasks within DA-Code are inherently challenging, setting them apart from traditional code generation tasks and demanding advanced coding skills in grounding and planning. Second, examples in DA-Code are all based on real and diverse data, covering a wide range of complex data wrangling and analytics tasks. Third, to solve the tasks, the models must utilize complex data science programming languages, including Python and SQL, to perform intricate data processing and derive the answers. We set up the benchmark in a controllable and executable environment that aligns with real-world data analysis scenarios and is scalable. The annotators meticulously designed the evaluation suite to ensure the accuracy and robustness of the evaluation. We developed the DA-Agent baseline. Experiments show that although the baseline performs better than other existing frameworks, using the current best LLMs achieves only 30.5% accuracy, leaving ample room for improvement. We release our benchmark at [link](https://github.com/yiyihum/dabench)"
++ },
++ "arXiv:2411.00816": {
++  "paperId": "92c82a51ad13c361d052987694cf93d6a72d5789",
++  "externalIds": {
++   "DBLP": "conf/iclr/WengZBZ00Y25",
++   "ArXiv": "2411.00816",
++   "DOI": "10.48550/arXiv.2411.00816",
++   "CorpusId": 273811997
++  },
++  "publicationVenue": {
++   "id": "939c6e1d-0d17-4d6e-8a82-66d960df0e40",
++   "name": "International Conference on Learning Representations",
++   "type": "conference",
++   "alternate_names": [
++    "Int Conf Learn Represent",
++    "ICLR"
++   ],
++   "url": "https://iclr.cc/"
++  },
++  "title": "CycleResearcher: Improving Automated Research via Automated Review",
++  "venue": "International Conference on Learning Representations",
++  "year": 2024,
++  "citationCount": 180,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2411.00816, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2411.00816"
++  },
++  "authors": [
++   {
++    "authorId": "2142839441",
++    "name": "Yixuan Weng"
++   },
++   {
++    "authorId": "2316827669",
++    "name": "Min-Jun Zhu"
++   },
++   {
++    "authorId": "1993226927",
++    "name": "Guangsheng Bao"
++   },
++   {
++    "authorId": "2329135444",
++    "name": "Hong-Bo Zhang"
++   },
++   {
++    "authorId": "1519290245",
++    "name": "Jin-Dong Wang"
++   },
++   {
++    "authorId": "2325943212",
++    "name": "Yue Zhang"
++   },
++   {
++    "authorId": "2145500840",
++    "name": "Linyi Yang"
++   }
++  ],
++  "abstract": "The automation of scientific discovery has been a long-standing goal within the research community, driven by the potential to accelerate knowledge creation. While significant progress has been made using commercial large language models (LLMs) as research assistants or idea generators, the possibility of automating the entire research process with open-source LLMs remains largely unexplored. This paper explores the feasibility of using open-source post-trained LLMs as autonomous agents capable of performing the full cycle of automated research and review, from literature review and manuscript preparation to peer review and paper refinement. Our iterative preference training framework consists of CycleResearcher, which conducts research tasks, and CycleReviewer, which simulates the peer review process, providing iterative feedback via reinforcement learning. To train these models, we develop two new datasets, Review-5k and Research-14k, reflecting real-world machine learning research and peer review dynamics. Our results demonstrate that CycleReviewer achieves promising performance with a 26.89\\% reduction in mean absolute error (MAE) compared to individual human reviewers in predicting paper scores, indicating the potential of LLMs to effectively assist expert-level research evaluation. In research, the papers generated by the CycleResearcher model achieved a score of 5.36 in simulated peer reviews, showing some competitiveness in terms of simulated review scores compared to the preprint level of 5.24 from human experts, while still having room for improvement compared to the accepted paper level of 5.69. This work represents a significant step toward fully automated scientific inquiry, providing ethical safeguards and exploring AI-driven research capabilities. The code, dataset and model weight are released at https://wengsyx.github.io/Researcher/."
++ },
++ "arXiv:2411.05285": {
++  "paperId": "3d6b189c9f8bb2f956db3fb42032d60eab29139c",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2411-05285",
++   "ArXiv": "2411.05285",
++   "DOI": "10.48550/arXiv.2411.05285",
++   "CorpusId": 273949919
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "AgentOps: Enabling Observability of LLM Agents",
++  "venue": "arXiv.org",
++  "year": 2024,
++  "citationCount": 63,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2411.05285, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2411.05285"
++  },
++  "authors": [
++   {
++    "authorId": "2367200438",
++    "name": "Li-Ming Dong"
++   },
++   {
++    "authorId": "2151674574",
++    "name": "Qing-Hua Lu"
++   },
++   {
++    "authorId": "2125737414",
++    "name": "Li-Ming Zhu"
++   }
++  ],
++  "abstract": "Large language model (LLM) agents have demonstrated remarkable capabilities across various domains, gaining extensive attention from academia and industry. However, these agents raise significant concerns on AI safety due to their autonomous and non-deterministic behavior, as well as continuous evolving nature . From a DevOps perspective, enabling observability in agents is necessary to ensuring AI safety, as stakeholders can gain insights into the agents' inner workings, allowing them to proactively understand the agents, detect anomalies, and prevent potential failures. Therefore, in this paper, we present a comprehensive taxonomy of AgentOps, identifying the artifacts and associated data that should be traced throughout the entire lifecycle of agents to achieve effective observability. The taxonomy is developed based on a systematic mapping study of existing AgentOps tools. Our taxonomy serves as a reference template for developers to design and implement AgentOps infrastructure that supports monitoring, logging, and analytics. thereby ensuring AI safety."
++ },
++ "arXiv:2411.15114": {
++  "paperId": "5dbd9f9fd231863dda17d9e3caff2edb99a113e2",
++  "externalIds": {
++   "ArXiv": "2411.15114",
++   "DBLP": "journals/corr/abs-2411-15114",
++   "DOI": "10.48550/arXiv.2411.15114",
++   "CorpusId": 274192262
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts",
++  "venue": "arXiv.org",
++  "year": 2024,
++  "citationCount": 208,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2411.15114, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2411.15114"
++  },
++  "authors": [
++   {
++    "authorId": "2331855692",
++    "name": "Hjalmar Wijk"
++   },
++   {
++    "authorId": "2228084357",
++    "name": "T. Lin"
++   },
++   {
++    "authorId": "2331857910",
++    "name": "Joel Becker"
++   },
++   {
++    "authorId": "2331855846",
++    "name": "Sami Jawhar"
++   },
++   {
++    "authorId": "1380182284",
++    "name": "Neev Parikh"
++   },
++   {
++    "authorId": "2331855786",
++    "name": "Thomas Broadley"
++   },
++   {
++    "authorId": "2164095723",
++    "name": "Lawrence Chan"
++   },
++   {
++    "authorId": "2332345490",
++    "name": "Michael Chen"
++   },
++   {
++    "authorId": "2291961652",
++    "name": "Joshua Clymer"
++   },
++   {
++    "authorId": "2331855463",
++    "name": "Jai Dhyani"
++   },
++   {
++    "authorId": "2331855896",
++    "name": "Elena Ericheva"
++   },
++   {
++    "authorId": "2331855858",
++    "name": "Katharyn Garcia"
++   },
++   {
++    "authorId": "2227287541",
++    "name": "Brian Goodrich"
++   },
++   {
++    "authorId": "2331855456",
++    "name": "Nikola Jurkovic"
++   },
++   {
++    "authorId": "1800253074",
++    "name": "Megan Kinniment"
++   },
++   {
++    "authorId": "2331855819",
++    "name": "Aron Lajko"
++   },
++   {
++    "authorId": "2164093876",
++    "name": "Seraphina Nix"
++   },
++   {
++    "authorId": "2275156833",
++    "name": "L. Sato"
++   },
++   {
++    "authorId": "2331855884",
++    "name": "William Saunders"
++   },
++   {
++    "authorId": "2331855792",
++    "name": "M. Taran"
++   },
++   {
++    "authorId": "2331856000",
++    "name": "Ben West"
++   },
++   {
++    "authorId": "2331855799",
++    "name": "Elizabeth Barnes"
++   }
++  ],
++  "abstract": "Frontier AI safety policies highlight automation of AI research and development (R&D) by AI agents as an important capability to anticipate. However, there exist few evaluations for AI R&D capabilities, and none that are highly realistic and have a direct comparison to human performance. We introduce RE-Bench (Research Engineering Benchmark, v1), which consists of 7 challenging, open-ended ML research engineering environments and data from 71 8-hour attempts by 61 distinct human experts. We confirm that our experts make progress in the environments given 8 hours, with 82% of expert attempts achieving a non-zero score and 24% matching or exceeding our strong reference solutions. We compare humans to several public frontier models through best-of-k with varying time budgets and agent designs, and find that the best AI agents achieve a score 4x higher than human experts when both are given a total time budget of 2 hours per environment. However, humans currently display better returns to increasing time budgets, narrowly exceeding the top AI agent scores given an 8-hour budget, and achieving 2x the score of the top AI agent when both are given 32 total hours (across different attempts). Qualitatively, we find that modern AI agents possess significant expertise in many ML topics -- e.g. an agent wrote a faster custom Triton kernel than any of our human experts' -- and can generate and test solutions over ten times faster than humans, at much lower cost. We open-source the evaluation environments, human expert data, analysis code and agent trajectories to facilitate future research."
++ },
++ "arXiv:2411.15594": {
++  "paperId": "e24424283c02fbe7f641e5b3490d7bb059f8355a",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2411-15594",
++   "ArXiv": "2411.15594",
++   "DOI": "10.48550/arXiv.2411.15594",
++   "CorpusId": 274234014
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "A Survey on LLM-as-a-Judge",
++  "venue": "arXiv.org",
++  "year": 2024,
++  "citationCount": 1988,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2411.15594, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2411.15594"
++  },
++  "authors": [
++   {
++    "authorId": "2216587705",
++    "name": "Jiawei Gu"
++   },
++   {
++    "authorId": "144267788",
++    "name": "Xuhui Jiang"
++   },
++   {
++    "authorId": "2287881684",
++    "name": "Zhichao Shi"
++   },
++   {
++    "authorId": "2274159320",
++    "name": "He-Xiang Tan"
++   },
++   {
++    "authorId": "2332093190",
++    "name": "Xue-Hao Zhai"
++   },
++   {
++    "authorId": "2250617116",
++    "name": "Cheng-Jin Xu"
++   },
++   {
++    "authorId": "2330714501",
++    "name": "Wei Li"
++   },
++   {
++    "authorId": "1944248313",
++    "name": "Ying-Han Shen"
++   },
++   {
++    "authorId": "2311556497",
++    "name": "Sheng-Jie Ma"
++   },
++   {
++    "authorId": "2332306096",
++    "name": "Hong-Hao Liu"
++   },
++   {
++    "authorId": "2219600",
++    "name": "Yuan-Zhuo Wang"
++   },
++   {
++    "authorId": "2284217200",
++    "name": "Jian Guo"
++   }
++  ],
++  "abstract": "Accurate and consistent evaluation is crucial for decision-making across numerous fields, yet it remains a challenging task due to inherent subjectivity, variability, and scale. Large Language Models (LLMs) have achieved remarkable success across diverse domains, leading to the emergence of\"LLM-as-a-Judge,\"where LLMs are employed as evaluators for complex tasks. With their ability to process diverse data types and provide scalable, cost-effective, and consistent assessments, LLMs present a compelling alternative to traditional expert-driven evaluations. However, ensuring the reliability of LLM-as-a-Judge systems remains a significant challenge that requires careful design and standardization. This paper provides a comprehensive survey of LLM-as-a-Judge, addressing the core question: How can reliable LLM-as-a-Judge systems be built? We explore strategies to enhance reliability, including improving consistency, mitigating biases, and adapting to diverse assessment scenarios. Additionally, we propose methodologies for evaluating the reliability of LLM-as-a-Judge systems, supported by a novel benchmark designed for this purpose. To advance the development and real-world deployment of LLM-as-a-Judge systems, we also discussed practical applications, challenges, and future directions. This survey serves as a foundational reference for researchers and practitioners in this rapidly evolving field."
++ },
++ "arXiv:2501.04227": {
++  "paperId": "394924896e24c9b086d96d0958dae07f54ff9452",
++  "externalIds": {
++   "ArXiv": "2501.04227",
++   "DBLP": "conf/emnlp/SchmidgallSWSWYLMLB25",
++   "DOI": "10.18653/v1/2025.findings-emnlp.320",
++   "CorpusId": 275358017
++  },
++  "publicationVenue": {
++   "id": "41bf9ed3-85b3-4c90-b015-150e31690253",
++   "name": "Conference on Empirical Methods in Natural Language Processing",
++   "type": "conference",
++   "alternate_names": [
++    "Empir Method Nat Lang Process",
++    "Empirical Methods in Natural Language Processing",
++    "Conf Empir Method Nat Lang Process",
++    "EMNLP"
++   ],
++   "url": "https://www.aclweb.org/portal/emnlp"
++  },
++  "title": "Agent Laboratory: Using LLM Agents as Research Assistants",
++  "venue": "Conference on Empirical Methods in Natural Language Processing",
++  "year": 2025,
++  "citationCount": 573,
++  "openAccessPdf": {
++   "url": "https://aclanthology.org/2025.findings-emnlp.320.pdf",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2501.04227, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference",
++   "Review"
++  ],
++  "journal": {
++   "pages": "5977-6043"
++  },
++  "authors": [
++   {
++    "authorId": "1741847434",
++    "name": "Samuel Schmidgall"
++   },
++   {
++    "authorId": "2339527873",
++    "name": "Yu-Sheng Su"
++   },
++   {
++    "authorId": "2336047375",
++    "name": "Ze Wang"
++   },
++   {
++    "authorId": "2335618942",
++    "name": "Xi-Meng Sun"
++   },
++   {
++    "authorId": "2339249470",
++    "name": "Jia-Lian Wu"
++   },
++   {
++    "authorId": "2339225241",
++    "name": "Xiao-Dong Yu"
++   },
++   {
++    "authorId": "2335604069",
++    "name": "Jiang Liu"
++   },
++   {
++    "authorId": "2301157095",
++    "name": "Michael Moor"
++   },
++   {
++    "authorId": "2334872571",
++    "name": "Zi-Cheng Liu"
++   },
++   {
++    "authorId": "2271751612",
++    "name": "E. Barsoum"
++   }
++  ],
++  "abstract": "Historically, scientific discovery has been a lengthy and costly process, demanding substantial time and resources from initial conception to final results. To accelerate scientific discovery, reduce research costs, and improve research quality, we introduce Agent Laboratory, an autonomous LLM-based framework capable of completing the entire research process. This framework accepts a human-provided research idea and progresses through three stages--literature review, experimentation, and report writing to produce comprehensive research outputs, including a code repository and a research report, while enabling users to provide feedback and guidance at each stage. We deploy Agent Laboratory with various state-of-the-art LLMs and invite multiple researchers to assess its quality by participating in a survey, providing human feedback to guide the research process, and then evaluate the final paper. We found that: (1) Agent Laboratory driven by o1-preview generates the best research outcomes; (2) The generated machine learning code is able to achieve state-of-the-art performance compared to existing methods; (3) Human involvement, providing feedback at each stage, significantly improves the overall quality of research; (4) Agent Laboratory significantly reduces research expenses, achieving an 84% decrease compared to previous autonomous research methods. We hope Agent Laboratory enables researchers to allocate more effort toward creative ideation rather than low-level coding and writing, ultimately accelerating scientific discovery."
++ },
++ "arXiv:2502.14297": {
++  "paperId": "fd6d9fb41fc8d06747f36fe4dff391ac064605fb",
++  "externalIds": {
++   "DBLP": "journals/sigir/BeelKB25",
++   "ArXiv": "2502.14297",
++   "DOI": "10.1145/3769733.376974",
++   "CorpusId": 276482965
++  },
++  "publicationVenue": {
++   "id": "b2ff135b-16c2-40a9-a068-c27761bfda08",
++   "name": "SIGIR Forum",
++   "type": "journal",
++   "alternate_names": [
++    "Sigir Forum"
++   ],
++   "issn": "0163-5840",
++   "alternate_issns": [
++    "1558-0229"
++   ],
++   "url": "http://rave.ohiolink.edu/ejournals/issn/01635840/",
++   "alternate_urls": [
++    "http://portal.acm.org/sigir/newsletter",
++    "http://www.acm.org/sigs/sigir/forum/index.html"
++   ]
++  },
++  "title": "Evaluating Sakana's AI Scientist: Bold Claims, Mixed Results, and a Promising Future?",
++  "venue": "SIGIR Forum",
++  "year": 2025,
++  "citationCount": 38,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2502.14297, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ACM SIGIR Forum",
++   "pages": "1 - 20",
++   "volume": "59"
++  },
++  "authors": [
++   {
++    "authorId": "2137867216",
++    "name": "Joeran Beel"
++   },
++   {
++    "authorId": "2270948852",
++    "name": "Min-Yen Kan"
++   },
++   {
++    "authorId": "2326137802",
++    "name": "Moritz Baumgart"
++   }
++  ],
++  "abstract": "Recently, Sakana.ai introduced the AI Scientist, a system claiming to automate the entire research lifecycle and conduct research autonomously, a concept we term Artificial Research Intelligence (ARI). Achieving ARI would be a major milestone toward Artificial General Intelligence (AGI) and a prerequisite to achieving Super Intelligence. The AI Scientist received much attention in the academic and broader AI community. A thorough evaluation of the AI Scientist, however, had not yet been conducted.1 We evaluated the AI Scientist and found several critical shortcomings. The system's literature review process is inadequate, relying on simplistic keyword searches rather than profound synthesis, which leads to poor novelty assessments. In our experiments, several generated research ideas were incorrectly classified as novel, including well-established concepts such as micro-batching for stochastic gradient descent (SGD). The AI Scientist also lacks robustness in experiment execution\u2014five out of twelve proposed experiments (42%) failed due to coding errors, and those that did run often produced logically flawed or misleading results. In one case, an experiment designed to optimize energy efficiency reported improvements in accuracy while consuming more computational resources, contradicting its stated goal. Furthermore, the system modifies experimental code minimally, with each iteration adding only 8% more characters on average, suggesting limited adaptability. The generated manuscripts were poorly substantiated, with a median of just five citations per paper\u2014most of which were outdated (only five out of 34 citations were from 2020 or later). Structural errors were frequent, including missing figures, repeated sections, and placeholder text such as \"Conclusions Here\". Hallucinated numerical results were contained in several manuscripts, undermining the reliability of its outputs. Despite its limitations, the AI Scientist represents a significant leap forward in research automation. It produces complete research manuscripts with minimal human intervention, challenging conventional expectations of AI-generated scientific work. Many reviewers or university instructors conducting only a superficial assessment may struggle to distinguish its output from that of human researchers, demonstrating how far AI has progressed in mimicking academic writing and structuring scientific arguments. While the quality of its manuscripts currently aligns with that of an unmotivated undergraduate student rushing to meet a deadline, this level of autonomy in research generation is remarkable. More strikingly, it achieves this at an unprecedented speed and cost efficiency\u2014our analysis indicates that generating a full research paper costs only $6\u2013$15, with just 3.5 hours of human involvement. This is significantly faster than traditional human researchers. Given that AI research automation was nearly nonexistent just a few years ago, the AI Scientist marks a substantial milestone toward Artificial Research Intelligence (ARI), signalling the acceleration of AI-driven scientific discovery. The AI Scientist also illustrates the urgent need for a discussion within the Information Retrieval (IR) and broader scientific communities. Whether and when ARI becomes a reality depends on how the academic and AI communities shape its development and governance. We propose concrete steps, including pilot projects and competitions, and standardized attribution frameworks such as research logs and markup languages."
++ },
++ "arXiv:2502.18864": {
++  "paperId": "390c607723e9767a9a873ded3bc1f66808215e8e",
++  "externalIds": {
++   "ArXiv": "2502.18864",
++   "PubMedCentral": "13345910",
++   "DOI": "10.1038/s41586-026-10644-y",
++   "CorpusId": 276617649,
++   "PubMed": "42428019"
++  },
++  "publicationVenue": {
++   "id": "6c24a0a0-b07d-4d7b-a19b-fd09a3ed453a",
++   "name": "Nature",
++   "type": "journal",
++   "issn": "0028-0836",
++   "url": "https://www.nature.com/",
++   "alternate_urls": [
++    "http://www.nature.com/nature/",
++    "https://www.nature.com/nature/",
++    "http://www.nature.com/nature/archive/index.html"
++   ]
++  },
++  "title": "Accelerating scientific discovery with Co-Scientist",
++  "venue": "Nature",
++  "year": 2025,
++  "citationCount": 593,
++  "openAccessPdf": {
++   "url": "https://www.nature.com/articles/s41586-026-10644-y_reference.pdf",
++   "status": "HYBRID",
++   "license": "other-oa",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2502.18864, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Nature",
++   "pages": "487 - 496",
++   "volume": "655"
++  },
++  "authors": [
++   {
++    "authorId": "2051681709",
++    "name": "Juraj Gottweis"
++   },
++   {
++    "authorId": "2239098855",
++    "name": "Wei-Hung Weng"
++   },
++   {
++    "authorId": "2347345594",
++    "name": "Alexander Daryin"
++   },
++   {
++    "authorId": "2244651851",
++    "name": "Tao Tu"
++   },
++   {
++    "authorId": "2679779",
++    "name": "Petar Sirkovi\u0107"
++   },
++   {
++    "authorId": "2346506169",
++    "name": "Artiom Myaskovsky"
++   },
++   {
++    "authorId": "2520452099",
++    "name": "Grzegorz Glowaty"
++   },
++   {
++    "authorId": "10805574",
++    "name": "Felix Weissenberger"
++   },
++   {
++    "authorId": "2367823022",
++    "name": "Alessio Orlandi"
++   },
++   {
++    "authorId": "2062591154",
++    "name": "D. Popovici"
++   },
++   {
++    "authorId": "2266472373",
++    "name": "Anil Palepu"
++   },
++   {
++    "authorId": "1996199677",
++    "name": "Ke-Ran Rong"
++   },
++   {
++    "authorId": "3480712",
++    "name": "Ryutaro Tanno"
++   },
++   {
++    "authorId": "2279021635",
++    "name": "Khaled Saab"
++   },
++   {
++    "authorId": "2298916585",
++    "name": "Fan Zhang"
++   },
++   {
++    "authorId": "2217621049",
++    "name": "Jacob Blum"
++   },
++   {
++    "authorId": "2336095347",
++    "name": "A. Carroll"
++   },
++   {
++    "authorId": "2269148145",
++    "name": "Kavita Kulkarni"
++   },
++   {
++    "authorId": "2213266",
++    "name": "Nenad Toma\u0161ev"
++   },
++   {
++    "authorId": "115731137",
++    "name": "D. Zverinski"
++   },
++   {
++    "authorId": "2389134",
++    "name": "Ivor Rendulic"
++   },
++   {
++    "authorId": "2298901106",
++    "name": "E. Vedadi"
++   },
++   {
++    "authorId": "113494403",
++    "name": "F. Hasler"
++   },
++   {
++    "authorId": "102638867",
++    "name": "Luka Rimanic"
++   },
++   {
++    "authorId": "36598773",
++    "name": "Marina Boia"
++   },
++   {
++    "authorId": "2784346",
++    "name": "Ivan Budiseli\u0107"
++   },
++   {
++    "authorId": "2074805000",
++    "name": "Ben Feinstein"
++   },
++   {
++    "authorId": "2359151678",
++    "name": "M. Bellaiche"
++   },
++   {
++    "authorId": "2220891317",
++    "name": "Tom Sheffer"
++   },
++   {
++    "authorId": "2279022140",
++    "name": "Jan Freyberg"
++   },
++   {
++    "authorId": "2255314955",
++    "name": "Jeremy D. Ratcliff"
++   },
++   {
++    "authorId": "1789306",
++    "name": "Ottavia Bertolli"
++   },
++   {
++    "authorId": "2269148566",
++    "name": "Katherine Chou"
++   },
++   {
++    "authorId": "1809983",
++    "name": "A. Hassidim"
++   },
++   {
++    "authorId": "2225921342",
++    "name": "Burak Gokturk"
++   },
++   {
++    "authorId": "2257189937",
++    "name": "Amin Vahdat"
++   },
++   {
++    "authorId": "2261405027",
++    "name": "Yuan Guan"
++   },
++   {
++    "authorId": "12268808",
++    "name": "Vikram Dhillon"
++   },
++   {
++    "authorId": "10664723",
++    "name": "E. D. Vaishnav"
++   },
++   {
++    "authorId": "2305695545",
++    "name": "Byron Lee"
++   },
++   {
++    "authorId": "2346514434",
++    "name": "Tiago R D Costa"
++   },
++   {
++    "authorId": "2068738124",
++    "name": "J. Penad\u00e9s"
++   },
++   {
++    "authorId": "2243119549",
++    "name": "Gary Peltz"
++   },
++   {
++    "authorId": "2269148232",
++    "name": "Y. Matias"
++   },
++   {
++    "authorId": "2254252286",
++    "name": "J. Manyika"
++   },
++   {
++    "authorId": "48987704",
++    "name": "D. Hassabis"
++   },
++   {
++    "authorId": "2275191526",
++    "name": "Yunhan Xu"
++   },
++   {
++    "authorId": "143967473",
++    "name": "Pushmeet Kohli"
++   },
++   {
++    "authorId": "2359249680",
++    "name": "Annalisa Pawlosky"
++   },
++   {
++    "authorId": "2213433382",
++    "name": "A. Karthikesalingam"
++   },
++   {
++    "authorId": "144223091",
++    "name": "Vivek Natarajan"
++   }
++  ],
++  "abstract": "Scientific discovery is driven by scientists generating hypotheses for complex problems that undergo rigorous experimental validation. To augment this process, we introduce Co-Scientist, a multi-agent artificial intelligence (AI) system built on Gemini for structured scientific thinking and hypothesis generation. Co-Scientist aims to help scientists discover new original knowledge. Conditioned on their research objectives and previous scientific evidence, it formulates demonstrably novel research hypotheses for experimental verification. The system\u2019s design involves agents continuously generating, critiquing and refining hypotheses accelerated by scaling test-time compute. Key contributions include (1) a multi-agent architecture with an asynchronous task execution framework for flexible compute scaling, and (2) a tournament evolution process for self-improving hypotheses generation. Automated evaluations show continued benefits of test-time compute scaling, improving hypothesis quality over time. Although this is a general-purpose system, we focus the validation in three biomedical applications: drug repurposing; novel-target discovery1; and explaining mechanisms of antimicrobial resistance2. Specifically, Co-Scientist helped to identify new drug-repurposing candidates and synergistic combination therapies for acute myeloid leukaemia that were validated through in vitro experiments. These real-world validations demonstrate the potential of Co-Scientist to accelerate scientific discovery and usher in an era of AI-empowered scientists. Co-Scientist is an AI system for structured scientific thinking and generates research hypotheses for experimental verification."
++ },
++ "arXiv:2504.01848": {
++  "paperId": "52be71605c60d326381a4ddfa1fb476912709504",
++  "externalIds": {
++   "DBLP": "conf/icml/StaraceJSACMDMK25",
++   "ArXiv": "2504.01848",
++   "DOI": "10.48550/arXiv.2504.01848",
++   "CorpusId": 277501993
++  },
++  "publicationVenue": {
++   "id": "fc0a208c-acb7-47dc-a0d4-af8190e21d29",
++   "name": "International Conference on Machine Learning",
++   "type": "conference",
++   "alternate_names": [
++    "ICML",
++    "Int Conf Mach Learn"
++   ],
++   "url": "https://icml.cc/"
++  },
++  "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research",
++  "venue": "International Conference on Machine Learning",
++  "year": 2025,
++  "citationCount": 366,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2504.01848, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2504.01848"
++  },
++  "authors": [
++   {
++    "authorId": "2168285763",
++    "name": "Giulio Starace"
++   },
++   {
++    "authorId": "2325151487",
++    "name": "Oliver Jaffe"
++   },
++   {
++    "authorId": "2231409794",
++    "name": "Dane Sherburn"
++   },
++   {
++    "authorId": "2325157923",
++    "name": "James Aung"
++   },
++   {
++    "authorId": "2325723546",
++    "name": "Jun Shern Chan"
++   },
++   {
++    "authorId": "2325157894",
++    "name": "Leon Maksin"
++   },
++   {
++    "authorId": "2328288671",
++    "name": "Rachel Dias"
++   },
++   {
++    "authorId": "2325158061",
++    "name": "E. Mays"
++   },
++   {
++    "authorId": "48208664",
++    "name": "Benjamin Kinsella"
++   },
++   {
++    "authorId": "2353389208",
++    "name": "Wyatt Thompson"
++   },
++   {
++    "authorId": "2151087994",
++    "name": "Johannes Heidecke"
++   },
++   {
++    "authorId": "2355867602",
++    "name": "Amelia Glaese"
++   },
++   {
++    "authorId": "90169542",
++    "name": "Tejal Patwardhan"
++   }
++  ],
++  "abstract": "We introduce PaperBench, a benchmark evaluating the ability of AI agents to replicate state-of-the-art AI research. Agents must replicate 20 ICML 2024 Spotlight and Oral papers from scratch, including understanding paper contributions, developing a codebase, and successfully executing experiments. For objective evaluation, we develop rubrics that hierarchically decompose each replication task into smaller sub-tasks with clear grading criteria. In total, PaperBench contains 8,316 individually gradable tasks. Rubrics are co-developed with the author(s) of each ICML paper for accuracy and realism. To enable scalable evaluation, we also develop an LLM-based judge to automatically grade replication attempts against rubrics, and assess our judge's performance by creating a separate benchmark for judges. We evaluate several frontier models on PaperBench, finding that the best-performing tested agent, Claude 3.5 Sonnet (New) with open-source scaffolding, achieves an average replication score of 21.0%. Finally, we recruit top ML PhDs to attempt a subset of PaperBench, finding that models do not yet outperform the human baseline. We open-source our code (https://github.com/openai/preparedness) to facilitate future research in understanding the AI engineering capabilities of AI agents."
++ },
++ "arXiv:2504.08066": {
++  "paperId": "39983a80f111f7f6e793f02c5725a14bca76b32d",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2504-08066",
++   "ArXiv": "2504.08066",
++   "DOI": "10.48550/arXiv.2504.08066",
++   "CorpusId": 277741107
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "The AI Scientist-v2: Workshop-Level Automated Scientific Discovery via Agentic Tree Search",
++  "venue": "arXiv.org",
++  "year": 2025,
++  "citationCount": 495,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2504.08066, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2504.08066"
++  },
++  "authors": [
++   {
++    "authorId": "2111746399",
++    "name": "Yutaro Yamada"
++   },
++   {
++    "authorId": "2266838849",
++    "name": "R. Lange"
++   },
++   {
++    "authorId": "2344954958",
++    "name": "Cong Lu"
++   },
++   {
++    "authorId": "2029238065",
++    "name": "Sheng-Ran Hu"
++   },
++   {
++    "authorId": "4420863",
++    "name": "Chris Lu"
++   },
++   {
++    "authorId": "2275156099",
++    "name": "J. Foerster"
++   },
++   {
++    "authorId": "2303255568",
++    "name": "Jeff Clune"
++   },
++   {
++    "authorId": "2315921540",
++    "name": "David Ha"
++   }
++  ],
++  "abstract": "AI is increasingly playing a pivotal role in transforming how scientific discoveries are made. We introduce The AI Scientist-v2, an end-to-end agentic system capable of producing the first entirely AI generated peer-review-accepted workshop paper. This system iteratively formulates scientific hypotheses, designs and executes experiments, analyzes and visualizes data, and autonomously authors scientific manuscripts. Compared to its predecessor (v1, Lu et al., 2024 arXiv:2408.06292), The AI Scientist-v2 eliminates the reliance on human-authored code templates, generalizes effectively across diverse machine learning domains, and leverages a novel progressive agentic tree-search methodology managed by a dedicated experiment manager agent. Additionally, we enhance the AI reviewer component by integrating a Vision-Language Model (VLM) feedback loop for iterative refinement of content and aesthetics of the figures. We evaluated The AI Scientist-v2 by submitting three fully autonomous manuscripts to a peer-reviewed ICLR workshop. Notably, one manuscript achieved high enough scores to exceed the average human acceptance threshold, marking the first instance of a fully AI-generated paper successfully navigating a peer review. This accomplishment highlights the growing capability of AI in conducting all aspects of scientific research. We anticipate that further advancements in autonomous scientific discovery technologies will profoundly impact human knowledge generation, enabling unprecedented scalability in research productivity and significantly accelerating scientific breakthroughs, greatly benefiting society at large. We have open-sourced the code at https://github.com/SakanaAI/AI-Scientist-v2 to foster the future development of this transformative technology. We also discuss the role of AI in science, including AI safety."
++ },
++ "arXiv:2504.13359": {
++  "paperId": "d120bd154a3ec05c2e5c9f3af7ecbf7b0ae11358",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2504-13359",
++   "ArXiv": "2504.13359",
++   "DOI": "10.48550/arXiv.2504.13359",
++   "CorpusId": 277940358
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Cost-of-Pass: An Economic Framework for Evaluating Language Models",
++  "venue": "arXiv.org",
++  "year": 2025,
++  "citationCount": 52,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2504.13359, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2504.13359"
++  },
++  "authors": [
++   {
++    "authorId": "19279031",
++    "name": "Mehmet Hamza Erol"
++   },
++   {
++    "authorId": "2356401680",
++    "name": "Batu El"
++   },
++   {
++    "authorId": "51903517",
++    "name": "M. Suzgun"
++   },
++   {
++    "authorId": "2186981598",
++    "name": "Mert Yuksekgonul"
++   },
++   {
++    "authorId": "2357014651",
++    "name": "James Zou"
++   }
++  ],
++  "abstract": "Widespread adoption of AI systems hinges on their ability to generate economic value that outweighs their inference costs. Evaluating this tradeoff requires metrics accounting for both performance and costs. Building on production theory, we develop an economically grounded framework to evaluate language models'productivity by combining accuracy and inference cost. We formalize cost-of-pass: the expected monetary cost of generating a correct solution. We then define the frontier cost-of-pass: the minimum cost-of-pass achievable across available models or the human-expert(s), using the approx. cost of hiring an expert. Our analysis reveals distinct economic insights. First, lightweight models are most cost-effective for basic quantitative tasks, large models for knowledge-intensive ones, and reasoning models for complex quantitative problems, despite higher per-token costs. Second, tracking the frontier cost-of-pass over the past year reveals significant progress, particularly for complex quant. tasks where the cost roughly halved every few months. Third, to trace key innovations driving this progress, we examine counterfactual frontiers -- estimates of cost-efficiency without specific model classes. We find that innovations in lightweight, large, and reasoning models have been essential for pushing the frontier in basic quant., knowledge-intensive, and complex quant. tasks, respectively. Finally, we assess the cost-reductions from common inference-time techniques (majority voting and self-refinement), and a budget-aware technique (TALE-EP). We find that performance-oriented methods with marginal performance gains rarely justify the costs, while TALE-EP shows some promise. Overall, our findings underscore that complementary model-level innovations are the primary drivers of cost-efficiency and our framework provides a principled tool for measuring this progress and guiding deployment."
++ },
++ "arXiv:2504.18765": {
++  "paperId": "c82f67d2a88d082ee28841f133494e60e8d39433",
++  "externalIds": {
++   "ArXiv": "2504.18765",
++   "DBLP": "journals/corr/abs-2504-18765",
++   "DOI": "10.48550/arXiv.2504.18765",
++   "CorpusId": 278165604
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "A Vision for Auto Research with LLM Agents",
++  "venue": "arXiv.org",
++  "year": 2025,
++  "citationCount": 22,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2504.18765, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2504.18765"
++  },
++  "authors": [
++   {
++    "authorId": "2158155335",
++    "name": "Cheng-Wei Liu"
++   },
++   {
++    "authorId": "2276510804",
++    "name": "Chong Wang"
++   },
++   {
++    "authorId": "2358168030",
++    "name": "Jia-Yue Cao"
++   },
++   {
++    "authorId": "2309156452",
++    "name": "Jing-Quan Ge"
++   },
++   {
++    "authorId": "2345981921",
++    "name": "Kun Wang"
++   },
++   {
++    "authorId": "2492221389",
++    "name": "Lvye Zhang"
++   },
++   {
++    "authorId": "2362307648",
++    "name": "Ming-Ming Cheng"
++   },
++   {
++    "authorId": "2284827556",
++    "name": "Penghai Zhao"
++   },
++   {
++    "authorId": "2265665080",
++    "name": "Tian-Lin Li"
++   },
++   {
++    "authorId": "2275762052",
++    "name": "Xiao-Jun Jia"
++   },
++   {
++    "authorId": "2359961665",
++    "name": "Xiang Li"
++   },
++   {
++    "authorId": "2144405890",
++    "name": "Xin-Feng Li"
++   },
++   {
++    "authorId": "2320865565",
++    "name": "Yang Liu"
++   },
++   {
++    "authorId": "2279401711",
++    "name": "Yebo Feng"
++   },
++   {
++    "authorId": "2142367654",
++    "name": "Yi-Hao Huang"
++   },
++   {
++    "authorId": "2110643465",
++    "name": "Yi-Jia Xu"
++   },
++   {
++    "authorId": "2281788746",
++    "name": "Yu-Qiang Sun"
++   },
++   {
++    "authorId": "2287501692",
++    "name": "Zhe-Xu Zhou"
++   },
++   {
++    "authorId": "3430657",
++    "name": "Zheng-Zi Xu"
++   }
++  ],
++  "abstract": "This paper introduces Agent-Based Auto Research, a structured multi-agent framework designed to automate, coordinate, and optimize the full lifecycle of scientific research. Leveraging the capabilities of large language models (LLMs) and modular agent collaboration, the system spans all major research phases, including literature review, ideation, methodology planning, experimentation, paper writing, peer review response, and dissemination. By addressing issues such as fragmented workflows, uneven methodological expertise, and cognitive overload, the framework offers a systematic and scalable approach to scientific inquiry. Preliminary explorations demonstrate the feasibility and potential of Auto Research as a promising paradigm for self-improving, AI-driven research processes."
++ },
++ "arXiv:2505.13400": {
++  "paperId": "799017d2818108e91392c541579cbe9d2cc38e35",
++  "externalIds": {
++   "ArXiv": "2505.13400",
++   "DBLP": "journals/corr/abs-2505-13400",
++   "DOI": "10.48550/arXiv.2505.13400",
++   "CorpusId": 278768799
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Robin: A multi-agent system for automating scientific discovery",
++  "venue": "arXiv.org",
++  "year": 2025,
++  "citationCount": 88,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2505.13400, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2505.13400"
++  },
++  "authors": [
++   {
++    "authorId": "2288198964",
++    "name": "A. Ghareeb"
++   },
++   {
++    "authorId": "2372457795",
++    "name": "Benjamin Chang"
++   },
++   {
++    "authorId": "2115040667",
++    "name": "L. Mitchener"
++   },
++   {
++    "authorId": "2362501842",
++    "name": "Angela Yiu"
++   },
++   {
++    "authorId": "114013157",
++    "name": "Caralyn J. Szostkiewicz"
++   },
++   {
++    "authorId": "2311439808",
++    "name": "Jon M. Laurent"
++   },
++   {
++    "authorId": "2117637436",
++    "name": "Muhammed Razzak"
++   },
++   {
++    "authorId": "2348512359",
++    "name": "Andrew D. White"
++   },
++   {
++    "authorId": "1395364165",
++    "name": "Michaela M. Hinks"
++   },
++   {
++    "authorId": "1924444",
++    "name": "Samuel G. Rodriques"
++   }
++  ],
++  "abstract": "Scientific discovery is driven by the iterative process of background research, hypothesis generation, experimentation, and data analysis. Despite recent advancements in applying artificial intelligence to scientific discovery, no system has yet automated all of these stages in a single workflow. Here, we introduce Robin, the first multi-agent system capable of fully automating the key intellectual steps of the scientific process. By integrating literature search agents with data analysis agents, Robin can generate hypotheses, propose experiments, interpret experimental results, and generate updated hypotheses, achieving a semi-autonomous approach to scientific discovery. By applying this system, we were able to identify a novel treatment for dry age-related macular degeneration (dAMD), the major cause of blindness in the developed world. Robin proposed enhancing retinal pigment epithelium phagocytosis as a therapeutic strategy, and identified and validated a promising therapeutic candidate, ripasudil. Ripasudil is a clinically-used rho kinase (ROCK) inhibitor that has never previously been proposed for treating dAMD. To elucidate the mechanism of ripasudil-induced upregulation of phagocytosis, Robin then proposed and analyzed a follow-up RNA-seq experiment, which revealed upregulation of ABCA1, a critical lipid efflux pump and possible novel target. All hypotheses, experimental plans, data analyses, and data figures in the main text of this report were produced by Robin. As the first AI system to autonomously discover and validate a novel therapeutic candidate within an iterative lab-in-the-loop framework, Robin establishes a new paradigm for AI-driven scientific discovery."
++ },
++ "arXiv:2505.18705": {
++  "paperId": "80a0b76dedc4c3e3d365bbaececcd44a996eb38b",
++  "externalIds": {
++   "ArXiv": "2505.18705",
++   "DBLP": "journals/corr/abs-2505-18705",
++   "DOI": "10.48550/arXiv.2505.18705",
++   "CorpusId": 278905406
++  },
++  "publicationVenue": {
++   "id": "d9720b90-d60b-48bc-9df8-87a30b9a60dd",
++   "name": "Neural Information Processing Systems",
++   "type": "conference",
++   "alternate_names": [
++    "Neural Inf Process Syst",
++    "NeurIPS",
++    "NIPS"
++   ],
++   "url": "http://neurips.cc/"
++  },
++  "title": "AI-Researcher: Autonomous Scientific Innovation",
++  "venue": "Neural Information Processing Systems",
++  "year": 2025,
++  "citationCount": 114,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2505.18705, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2505.18705"
++  },
++  "authors": [
++   {
++    "authorId": "2261087798",
++    "name": "Jiabin Tang"
++   },
++   {
++    "authorId": "1830455155",
++    "name": "Lianghao Xia"
++   },
++   {
++    "authorId": "2215613396",
++    "name": "Zhonghang Li"
++   },
++   {
++    "authorId": "2344760658",
++    "name": "Chao Huang"
++   }
++  ],
++  "abstract": "The powerful reasoning capabilities of Large Language Models (LLMs) in mathematics and coding, combined with their ability to automate complex tasks through agentic frameworks, present unprecedented opportunities for accelerating scientific innovation. In this paper, we introduce AI-Researcher, a fully autonomous research system that transforms how AI-driven scientific discovery is conducted and evaluated. Our framework seamlessly orchestrates the complete research pipeline--from literature review and hypothesis generation to algorithm implementation and publication-ready manuscript preparation--with minimal human intervention. To rigorously assess autonomous research capabilities, we develop Scientist-Bench, a comprehensive benchmark comprising state-of-the-art papers across diverse AI research domains, featuring both guided innovation and open-ended exploration tasks. Through extensive experiments, we demonstrate that AI-Researcher achieves remarkable implementation success rates and produces research papers that approach human-level quality. This work establishes new foundations for autonomous scientific innovation that can complement human researchers by systematically exploring solution spaces beyond cognitive limitations."
++ },
++ "arXiv:2507.02825": {
++  "paperId": "eeff713967bfd94da50e4cc8fda888b01f137e90",
++  "externalIds": {
++   "ArXiv": "2507.02825",
++   "DBLP": "journals/corr/abs-2507-02825",
++   "DOI": "10.48550/arXiv.2507.02825",
++   "CorpusId": 280091543
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Establishing Best Practices for Building Rigorous Agentic Benchmarks",
++  "venue": "arXiv.org",
++  "year": 2025,
++  "citationCount": 124,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2507.02825, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2507.02825"
++  },
++  "authors": [
++   {
++    "authorId": "2351716372",
++    "name": "Yuxuan Zhu"
++   },
++   {
++    "authorId": "2303902750",
++    "name": "Teng-Jun Jin"
++   },
++   {
++    "authorId": "100984698",
++    "name": "Yada Pruksachatkun"
++   },
++   {
++    "authorId": "2316522831",
++    "name": "Andy K. Zhang"
++   },
++   {
++    "authorId": "2344195930",
++    "name": "Shu Liu"
++   },
++   {
++    "authorId": "2374300501",
++    "name": "Sasha Cui"
++   },
++   {
++    "authorId": "39893263",
++    "name": "Sayash Kapoor"
++   },
++   {
++    "authorId": "2283848744",
++    "name": "Shayne Longpre"
++   },
++   {
++    "authorId": "2350413159",
++    "name": "Kevin Meng"
++   },
++   {
++    "authorId": "2297190630",
++    "name": "Rebecca Weiss"
++   },
++   {
++    "authorId": "2143198655",
++    "name": "Fazl Barez"
++   },
++   {
++    "authorId": "2384860310",
++    "name": "Rahul Gupta"
++   },
++   {
++    "authorId": "3475586",
++    "name": "J. Dhamala"
++   },
++   {
++    "authorId": "2191689389",
++    "name": "Jacob Merizian"
++   },
++   {
++    "authorId": "2372512852",
++    "name": "Mario Giulianelli"
++   },
++   {
++    "authorId": "2043233315",
++    "name": "H. Coppock"
++   },
++   {
++    "authorId": "11285234",
++    "name": "C. Ududec"
++   },
++   {
++    "authorId": "2275054630",
++    "name": "Jasjeet S. Sekhon"
++   },
++   {
++    "authorId": "2327340186",
++    "name": "Jacob Steinhardt"
++   },
++   {
++    "authorId": "2372300510",
++    "name": "A. Kellerman"
++   },
++   {
++    "authorId": "2343632085",
++    "name": "Sarah Schwettmann"
++   },
++   {
++    "authorId": "2221547236",
++    "name": "Matei A. Zaharia"
++   },
++   {
++    "authorId": "2295665819",
++    "name": "Ion Stoica"
++   },
++   {
++    "authorId": "2347224361",
++    "name": "Percy Liang"
++   },
++   {
++    "authorId": "2352791966",
++    "name": "Daniel Kang"
++   }
++  ],
++  "abstract": "Benchmarks are essential for quantitatively tracking progress in AI. As AI agents become increasingly capable, researchers and practitioners have introduced agentic benchmarks to evaluate agents on complex, real-world tasks. These benchmarks typically measure agent capabilities by evaluating task outcomes via specific reward designs. However, we show that many agentic benchmarks have issues in task setup or reward design. For example, SWE-bench Verified uses insufficient test cases, while TAU-bench counts empty responses as successful. Such issues can lead to under- or overestimation of agents'performance by up to 100% in relative terms. To make agentic evaluation rigorous, we introduce the Agentic Benchmark Checklist (ABC), a set of guidelines that we synthesized from our benchmark-building experience, a survey of best practices, and previously reported issues. When applied to CVE-Bench, a benchmark with a particularly complex evaluation design, ABC reduces the performance overestimation by 33%."
++ },
++ "arXiv:2507.09089": {
++  "paperId": "9008680aac5a92b3a089aa1487eea76b8565f0d3",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2507-09089",
++   "ArXiv": "2507.09089",
++   "DOI": "10.48550/arXiv.2507.09089",
++   "CorpusId": 280045608
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
++  "venue": "arXiv.org",
++  "year": 2025,
++  "citationCount": 232,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2507.09089, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2507.09089"
++  },
++  "authors": [
++   {
++    "authorId": "2331857910",
++    "name": "Joel Becker"
++   },
++   {
++    "authorId": "2350754555",
++    "name": "Nate Rush"
++   },
++   {
++    "authorId": "2331855799",
++    "name": "Elizabeth Barnes"
++   },
++   {
++    "authorId": "2266751602",
++    "name": "David Rein"
++   }
++  ],
++  "abstract": "Despite widespread adoption, the impact of AI tools on software development in the wild remains understudied. We conduct a randomized controlled trial (RCT) to understand how AI tools at the February-June 2025 frontier affect the productivity of experienced open-source developers. 16 developers with moderate AI experience complete 246 tasks in mature projects on which they have an average of 5 years of prior experience. Each task is randomly assigned to allow or disallow usage of early 2025 AI tools. When AI tools are allowed, developers primarily use Cursor Pro, a popular code editor, and Claude 3.5/3.7 Sonnet. Before starting tasks, developers forecast that allowing AI will reduce completion time by 24%. After completing the study, developers estimate that allowing AI reduced completion time by 20%. Surprisingly, we find that allowing AI actually increases completion time by 19%--AI tooling slowed developers down. This slowdown also contradicts predictions from experts in economics (39% shorter) and ML (38% shorter). To understand this result, we collect and evaluate evidence for 20 properties of our setting that a priori could contribute to the observed slowdown effect--for example, the size and quality standards of projects, or prior developer experience with AI tooling. Although the influence of experimental artifacts cannot be entirely ruled out, the robustness of the slowdown effect across our analyses suggests it is unlikely to primarily be a function of our experimental design."
++ },
++ "arXiv:2509.26603": {
++  "paperId": "f3da28bbb488ded8a7d1e1f71a2c4810da38eb20",
++  "externalIds": {
++   "ArXiv": "2509.26603",
++   "DBLP": "journals/corr/abs-2509-26603",
++   "DOI": "10.48550/arXiv.2509.26603",
++   "CorpusId": 281681196
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "DeepScientist: Advancing Frontier-Pushing Scientific Findings Progressively",
++  "venue": "arXiv.org",
++  "year": 2025,
++  "citationCount": 69,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2509.26603, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2509.26603"
++  },
++  "authors": [
++   {
++    "authorId": "2374151012",
++    "name": "Yi-Xuan Weng"
++   },
++   {
++    "authorId": "2316827669",
++    "name": "Min-Jun Zhu"
++   },
++   {
++    "authorId": "2342406856",
++    "name": "Qiu-Jie Xie"
++   },
++   {
++    "authorId": "2383102227",
++    "name": "Qi-Yao Sun"
++   },
++   {
++    "authorId": "2364802150",
++    "name": "Zhen Lin"
++   },
++   {
++    "authorId": "2376557334",
++    "name": "Sifan Liu"
++   },
++   {
++    "authorId": "2374438885",
++    "name": "Yue Zhang"
++   }
++  ],
++  "abstract": "While previous AI Scientist systems can generate novel findings, they often lack the focus to produce scientifically valuable contributions that address pressing human-defined challenges. We introduce DeepScientist, a system designed to overcome this by conducting goal-oriented, fully autonomous scientific discovery over month-long timelines. It formalizes discovery as a Bayesian Optimization problem, operationalized through a hierarchical evaluation process consisting of\"hypothesize, verify, and analyze\". Leveraging a cumulative Findings Memory, this loop intelligently balances the exploration of novel hypotheses with exploitation, selectively promoting the most promising findings to higher-fidelity levels of validation. Consuming over 20,000 GPU hours, the system generated about 5,000 unique scientific ideas and experimentally validated approximately 1100 of them, ultimately surpassing human-designed state-of-the-art (SOTA) methods on three frontier AI tasks by 183.7\\%, 1.9\\%, and 7.9\\%. This work provides the first large-scale evidence of an AI achieving discoveries that progressively surpass human SOTA on scientific tasks, producing valuable findings that genuinely push the frontier of scientific discovery. To facilitate further research into this process, we will open-source all experimental logs and system code at https://github.com/ResearAI/DeepScientist/."
++ },
++ "arXiv:2510.11977": {
++  "paperId": "92f110c93e9c9990129dfbf601b8d7254855dedf",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2510-11977",
++   "ArXiv": "2510.11977",
++   "DOI": "10.48550/arXiv.2510.11977",
++   "CorpusId": 282064372
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation",
++  "venue": "arXiv.org",
++  "year": 2025,
++  "citationCount": 116,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2510.11977, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2510.11977"
++  },
++  "authors": [
++   {
++    "authorId": "39893263",
++    "name": "Sayash Kapoor"
++   },
++   {
++    "authorId": "2309175570",
++    "name": "Benedikt Stroebl"
++   },
++   {
++    "authorId": "2392881135",
++    "name": "Peter Kirgis"
++   },
++   {
++    "authorId": "2290073765",
++    "name": "Nitya Nadgir"
++   },
++   {
++    "authorId": "2274102473",
++    "name": "Zachary S. Siegel"
++   },
++   {
++    "authorId": "2283309803",
++    "name": "Boyi Wei"
++   },
++   {
++    "authorId": "2353284385",
++    "name": "Tianci Xue"
++   },
++   {
++    "authorId": "2385761588",
++    "name": "Zi-Ru Chen"
++   },
++   {
++    "authorId": "2385732101",
++    "name": "Felix Chen"
++   },
++   {
++    "authorId": "1512255229",
++    "name": "Saiteja Utpala"
++   },
++   {
++    "authorId": "2385560965",
++    "name": "Franck Ndzomga"
++   },
++   {
++    "authorId": "2322217725",
++    "name": "Dheeraj Oruganty"
++   },
++   {
++    "authorId": "2385560963",
++    "name": "Sophie Luskin"
++   },
++   {
++    "authorId": "2386171231",
++    "name": "Kangheng Liu"
++   },
++   {
++    "authorId": "2268658397",
++    "name": "Bo-Tao Yu"
++   },
++   {
++    "authorId": "2283236744",
++    "name": "Amit Arora"
++   },
++   {
++    "authorId": "2327211159",
++    "name": "Dongyoon Hahm"
++   },
++   {
++    "authorId": "6365809",
++    "name": "H. Trivedi"
++   },
++   {
++    "authorId": "2386587363",
++    "name": "Huan Sun"
++   },
++   {
++    "authorId": "2298402035",
++    "name": "Ju-Yong Lee"
++   },
++   {
++    "authorId": "2303902750",
++    "name": "Teng-Jun Jin"
++   },
++   {
++    "authorId": "2054708905",
++    "name": "Yifan Mai"
++   },
++   {
++    "authorId": "2349674686",
++    "name": "Yifei Zhou"
++   },
++   {
++    "authorId": "2351716372",
++    "name": "Yuxuan Zhu"
++   },
++   {
++    "authorId": "2223138553",
++    "name": "Rishi Bommasani"
++   },
++   {
++    "authorId": "2352791966",
++    "name": "Daniel Kang"
++   },
++   {
++    "authorId": "2374189943",
++    "name": "D. Song"
++   },
++   {
++    "authorId": "2363497261",
++    "name": "Peter Henderson"
++   },
++   {
++    "authorId": "1758652",
++    "name": "Yu Su"
++   },
++   {
++    "authorId": "2347512172",
++    "name": "Percy Liang"
++   },
++   {
++    "authorId": "2355353099",
++    "name": "Arvind Narayanan"
++   }
++  ],
++  "abstract": "AI agents have been developed for complex real-world tasks from coding to customer service. But AI agent evaluations suffer from many challenges that undermine our understanding of how well agents really work. We introduce the Holistic Agent Leaderboard (HAL) to address these challenges. We make three main contributions. First, we provide a standardized evaluation harness that orchestrates parallel evaluations across hundreds of VMs, reducing evaluation time from weeks to hours while eliminating common implementation bugs. Second, we conduct three-dimensional analysis spanning models, scaffolds, and benchmarks. We validate the harness by conducting 21,730 agent rollouts across 9 models and 9 benchmarks in coding, web navigation, science, and customer service with a total cost of about $40,000. Our analysis reveals surprising insights, such as higher reasoning effort reducing accuracy in the majority of runs. Third, we use LLM-aided log inspection to uncover previously unreported behaviors, such as searching for the benchmark on HuggingFace instead of solving a task, or misusing credit cards in flight booking tasks. We share all agent logs, comprising 2.5B tokens of language model calls, to incentivize further research into agent behavior. By standardizing how the field evaluates agents and addressing common pitfalls in agent evaluation, we hope to shift the focus from agents that ace benchmarks to agents that work reliably in the real world."
++ },
++ "arXiv:2511.02824": {
++  "paperId": "0fc30d552b48b12817ba6ec4dfa9571b2a262b8d",
++  "externalIds": {
++   "ArXiv": "2511.02824",
++   "DBLP": "journals/corr/abs-2511-02824",
++   "DOI": "10.48550/arXiv.2511.02824",
++   "CorpusId": 282748827
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Kosmos: An AI Scientist for Autonomous Discovery",
++  "venue": "arXiv.org",
++  "year": 2025,
++  "citationCount": 121,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2511.02824, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2511.02824"
++  },
++  "authors": [
++   {
++    "authorId": "2115040667",
++    "name": "L. Mitchener"
++   },
++   {
++    "authorId": "2362501842",
++    "name": "Angela Yiu"
++   },
++   {
++    "authorId": "2372457795",
++    "name": "Benjamin Chang"
++   },
++   {
++    "authorId": "2337087923",
++    "name": "M. Bourdenx"
++   },
++   {
++    "authorId": "2390494493",
++    "name": "Tyler Nadolski"
++   },
++   {
++    "authorId": "49735802",
++    "name": "Arvis Sulovari"
++   },
++   {
++    "authorId": "6808942",
++    "name": "E. Landsness"
++   },
++   {
++    "authorId": "36956411",
++    "name": "D\u00e1niel L. Barab\u00e1si"
++   },
++   {
++    "authorId": "2311445200",
++    "name": "Siddharth Narayanan"
++   },
++   {
++    "authorId": "2380028900",
++    "name": "Nicky Evans"
++   },
++   {
++    "authorId": "2392220836",
++    "name": "S. Reddy"
++   },
++   {
++    "authorId": "2331311827",
++    "name": "M. Foiani"
++   },
++   {
++    "authorId": "2183418673",
++    "name": "Aizad Kamal"
++   },
++   {
++    "authorId": "4291840",
++    "name": "L. Shriver"
++   },
++   {
++    "authorId": "2390730337",
++    "name": "F. Cao"
++   },
++   {
++    "authorId": "8624819",
++    "name": "A. Wassie"
++   },
++   {
++    "authorId": "2311439808",
++    "name": "Jon M. Laurent"
++   },
++   {
++    "authorId": "2390494299",
++    "name": "Edwin Melville-Green"
++   },
++   {
++    "authorId": "89892165",
++    "name": "M. Caldas"
++   },
++   {
++    "authorId": "2337796843",
++    "name": "Albert Bou"
++   },
++   {
++    "authorId": "2338378584",
++    "name": "Kaleigh F. Roberts"
++   },
++   {
++    "authorId": "49106960",
++    "name": "Sladjana Zagorac"
++   },
++   {
++    "authorId": "2144762732",
++    "name": "Timothy C. Orr"
++   },
++   {
++    "authorId": "2238439648",
++    "name": "Miranda E. Orr"
++   },
++   {
++    "authorId": "4590892",
++    "name": "Kevin J. Zwezdaryk"
++   },
++   {
++    "authorId": "2288198964",
++    "name": "A. Ghareeb"
++   },
++   {
++    "authorId": "2342363627",
++    "name": "L. McCoy"
++   },
++   {
++    "authorId": "2398715364",
++    "name": "B. Gomes"
++   },
++   {
++    "authorId": "2220825270",
++    "name": "Euan A. Ashley"
++   },
++   {
++    "authorId": "2259207100",
++    "name": "K. Duff"
++   },
++   {
++    "authorId": "6018731",
++    "name": "T. Buonassisi"
++   },
++   {
++    "authorId": "2358794",
++    "name": "Tom Rainforth"
++   },
++   {
++    "authorId": "2308325076",
++    "name": "R. Bateman"
++   },
++   {
++    "authorId": "2322440716",
++    "name": "Michael Skarlinski"
++   },
++   {
++    "authorId": "1924444",
++    "name": "Samuel G. Rodriques"
++   },
++   {
++    "authorId": "1395364165",
++    "name": "Michaela M. Hinks"
++   },
++   {
++    "authorId": "2354508425",
++    "name": "Andrew D. White"
++   }
++  ],
++  "abstract": "Data-driven scientific discovery requires iterative cycles of literature search, hypothesis generation, and data analysis. Substantial progress has been made towards AI agents that can automate scientific research, but all such agents remain limited in the number of actions they can take before losing coherence, thus limiting the depth of their findings. Here we present Kosmos, an AI scientist that automates data-driven discovery. Given an open-ended objective and a dataset, Kosmos runs for up to 12 hours performing cycles of parallel data analysis, literature search, and hypothesis generation before synthesizing discoveries into scientific reports. Unlike prior systems, Kosmos uses a structured world model to share information between a data analysis agent and a literature search agent. The world model enables Kosmos to coherently pursue the specified objective over 200 agent rollouts, collectively executing an average of 42,000 lines of code and reading 1,500 papers per run. Kosmos cites all statements in its reports with code or primary literature, ensuring its reasoning is traceable. Independent scientists found 79.4% of statements in Kosmos reports to be accurate, and collaborators reported that a single 20-cycle Kosmos run performed the equivalent of 6 months of their own research time on average. Furthermore, collaborators reported that the number of valuable scientific findings generated scales linearly with Kosmos cycles (tested up to 20 cycles). We highlight seven discoveries made by Kosmos that span metabolomics, materials science, neuroscience, and statistical genetics. Three discoveries independently reproduce findings from preprinted or unpublished manuscripts that were not accessed by Kosmos at runtime, while four make novel contributions to the scientific literature."
++ },
++ "arXiv:2601.11868": {
++  "paperId": "ef0fcf2f99c74918ceed21f529c1b99f782ac33e",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2601-11868",
++   "ArXiv": "2601.11868",
++   "DOI": "10.48550/arXiv.2601.11868",
++   "CorpusId": 284911857
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces",
++  "venue": "arXiv.org",
++  "year": 2026,
++  "citationCount": 671,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2601.11868, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2601.11868"
++  },
++  "authors": [
++   {
++    "authorId": "2297185552",
++    "name": "Mike A. Merrill"
++   },
++   {
++    "authorId": "2312442525",
++    "name": "Alexander G Shaw"
++   },
++   {
++    "authorId": "2365115789",
++    "name": "Nicholas Carlini"
++   },
++   {
++    "authorId": "2394091201",
++    "name": "Boxuan Li"
++   },
++   {
++    "authorId": "100947321",
++    "name": "H. Raj"
++   },
++   {
++    "authorId": "2366018523",
++    "name": "I. Bercovich"
++   },
++   {
++    "authorId": "2305925735",
++    "name": "Lin Shi"
++   },
++   {
++    "authorId": "2316095027",
++    "name": "J. Shin"
++   },
++   {
++    "authorId": "2389332288",
++    "name": "Thomas Walshe"
++   },
++   {
++    "authorId": "2336877105",
++    "name": "E. K. Buchanan"
++   },
++   {
++    "authorId": "2269118801",
++    "name": "Junhong Shen"
++   },
++   {
++    "authorId": "1753871691",
++    "name": "Guanghao Ye"
++   },
++   {
++    "authorId": "2372664232",
++    "name": "Hao Lin"
++   },
++   {
++    "authorId": "7245508",
++    "name": "Jason Poulos"
++   },
++   {
++    "authorId": "2288752126",
++    "name": "Mao-Yu Wang"
++   },
++   {
++    "authorId": "2174178585",
++    "name": "Marianna Nezhurina"
++   },
++   {
++    "authorId": "2191688",
++    "name": "J. Jitsev"
++   },
++   {
++    "authorId": "2387129493",
++    "name": "Di Lu"
++   },
++   {
++    "authorId": "1978772089",
++    "name": "Orfeas Menis-Mastromichalakis"
++   },
++   {
++    "authorId": "2254325443",
++    "name": "Zhiwei Xu"
++   },
++   {
++    "authorId": "2326326988",
++    "name": "Zizhao Chen"
++   },
++   {
++    "authorId": "2283355537",
++    "name": "Yue Liu"
++   },
++   {
++    "authorId": "2119062534",
++    "name": "Robert Zhang"
++   },
++   {
++    "authorId": "2146073251",
++    "name": "L. Chen"
++   },
++   {
++    "authorId": "2052965045",
++    "name": "Anurag Kashyap"
++   },
++   {
++    "authorId": "2220631332",
++    "name": "Jan-Lucas Uslu"
++   },
++   {
++    "authorId": "2287821949",
++    "name": "Jeffrey Li"
++   },
++   {
++    "authorId": "2302781884",
++    "name": "Jianbo Wu"
++   },
++   {
++    "authorId": "2263988785",
++    "name": "Minghao Yan"
++   },
++   {
++    "authorId": "2342276462",
++    "name": "Song Bian"
++   },
++   {
++    "authorId": "2406484417",
++    "name": "Vedang Sharma"
++   },
++   {
++    "authorId": "2407816422",
++    "name": "Ke Sun"
++   },
++   {
++    "authorId": "2333363742",
++    "name": "S. Dillmann"
++   },
++   {
++    "authorId": "2313160570",
++    "name": "Akshay Anand"
++   },
++   {
++    "authorId": "2312202594",
++    "name": "Andrew Lanpouthakoun"
++   },
++   {
++    "authorId": "2405816209",
++    "name": "Bardia Koopah"
++   },
++   {
++    "authorId": "2295937151",
++    "name": "Chang-Ran Hu"
++   },
++   {
++    "authorId": "2179104354",
++    "name": "E. Guha"
++   },
++   {
++    "authorId": "2325727722",
++    "name": "Gabriel H. S. Dreiman"
++   },
++   {
++    "authorId": "1870353",
++    "name": "Jia-Cheng Zhu"
++   },
++   {
++    "authorId": "2265975127",
++    "name": "Karl Krauth"
++   },
++   {
++    "authorId": "2287937401",
++    "name": "Li Zhong"
++   },
++   {
++    "authorId": "2037383772",
++    "name": "Niklas Muennighoff"
++   },
++   {
++    "authorId": "4798533",
++    "name": "Robert K. Amanfu"
++   },
++   {
++    "authorId": "2275830490",
++    "name": "Shang-Yin Tan"
++   },
++   {
++    "authorId": "2187585049",
++    "name": "Shreyas Pimpalgaonkar"
++   },
++   {
++    "authorId": "2130626178",
++    "name": "Tushar Aggarwal"
++   },
++   {
++    "authorId": "145724540",
++    "name": "Xiang-Ning Lin"
++   },
++   {
++    "authorId": "2405714069",
++    "name": "Xin Lan"
++   },
++   {
++    "authorId": "150345512",
++    "name": "Xuandong Zhao"
++   },
++   {
++    "authorId": "2257367768",
++    "name": "Yi-Qing Liang"
++   },
++   {
++    "authorId": "2444709630",
++    "name": "Yuan-Li Wang"
++   },
++   {
++    "authorId": "2265487043",
++    "name": "Zi-Long Wang"
++   },
++   {
++    "authorId": "2300675202",
++    "name": "Chang-Zhi Zhou"
++   },
++   {
++    "authorId": "2287413575",
++    "name": "D. Heineman"
++   },
++   {
++    "authorId": "2385506498",
++    "name": "Hange Liu"
++   },
++   {
++    "authorId": "6365809",
++    "name": "H. Trivedi"
++   },
++   {
++    "authorId": "2346284658",
++    "name": "John Yang"
++   },
++   {
++    "authorId": "2311427666",
++    "name": "Jun-Hong Lin"
++   },
++   {
++    "authorId": "2275599530",
++    "name": "Manish Shetty"
++   },
++   {
++    "authorId": "2110603190",
++    "name": "Michael Yang"
++   },
++   {
++    "authorId": "2328412793",
++    "name": "Nabil Omi"
++   },
++   {
++    "authorId": "2065899387",
++    "name": "Negin Raoof"
++   },
++   {
++    "authorId": "2257057803",
++    "name": "Shanda Li"
++   },
++   {
++    "authorId": "2080123731",
++    "name": "Terry Yue Zhuo"
++   },
++   {
++    "authorId": "2112406019",
++    "name": "Wu Lin"
++   },
++   {
++    "authorId": "2321354925",
++    "name": "Yi-Wei Dai"
++   },
++   {
++    "authorId": "2504582353",
++    "name": "Yu-Xin Wang"
++   },
++   {
++    "authorId": "2324574706",
++    "name": "Wen-Hao Chai"
++   },
++   {
++    "authorId": "2367092694",
++    "name": "Shang Zhou"
++   },
++   {
++    "authorId": "2230414556",
++    "name": "Dariush Wahdany"
++   },
++   {
++    "authorId": "2126990654",
++    "name": "Zi-Yu She"
++   },
++   {
++    "authorId": "2391004673",
++    "name": "Jia-Ming Hu"
++   },
++   {
++    "authorId": "2386598941",
++    "name": "Zhikang Dong"
++   },
++   {
++    "authorId": "2405833335",
++    "name": "Yuxuan Zhu"
++   },
++   {
++    "authorId": "2374300501",
++    "name": "Sasha Cui"
++   },
++   {
++    "authorId": "2033775348",
++    "name": "Ahson Saiyed"
++   },
++   {
++    "authorId": null,
++    "name": "Arinbj\u00f6rn Kolbeinsson"
++   },
++   {
++    "authorId": "2479368557",
++    "name": "Jesse Hu"
++   },
++   {
++    "authorId": "2131041347",
++    "name": "Christopher Rytting"
++   },
++   {
++    "authorId": "2365261533",
++    "name": "Ryan Marten"
++   },
++   {
++    "authorId": "2445500129",
++    "name": "Yi-Xin Wang"
++   },
++   {
++    "authorId": "2257301126",
++    "name": "A. Dimakis"
++   },
++   {
++    "authorId": "2371549",
++    "name": "A. Konwinski"
++   },
++   {
++    "authorId": "2352216817",
++    "name": "Ludwig Schmidt"
++   }
++  ],
++  "abstract": "AI agents may soon become capable of autonomously completing valuable, long-horizon tasks in diverse domains. Current benchmarks either do not measure real-world tasks, or are not sufficiently difficult to meaningfully measure frontier models. To this end, we present Terminal-Bench 2.0: a carefully curated hard benchmark composed of 89 tasks in computer terminal environments inspired by problems from real workflows. Each task features a unique environment, human-written solution, and comprehensive tests for verification. We show that frontier models and agents score less than 65\\% on the benchmark and conduct an error analysis to identify areas for model and agent improvement. We publish the dataset and evaluation harness to assist developers and researchers in future work at https://www.tbench.ai/ ."
++ },
++ "arXiv:2601.18724": {
++  "paperId": "737546bba8ed4daec51b92de19c1d8c0080aea92",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2601-18724",
++   "ArXiv": "2601.18724",
++   "DOI": "10.48550/arXiv.2601.18724",
++   "CorpusId": 285050476
++  },
++  "publicationVenue": {
++   "id": "1e33b3be-b2ab-46e9-96e8-d4eb4bad6e44",
++   "name": "Annual Meeting of the Association for Computational Linguistics",
++   "type": "conference",
++   "alternate_names": [
++    "Annu Meet Assoc Comput Linguistics",
++    "Meeting of the Association for Computational Linguistics",
++    "ACL",
++    "Meet Assoc Comput Linguistics"
++   ],
++   "url": "https://www.aclweb.org/anthology/venues/acl/"
++  },
++  "title": "HalluCitation Matters: Revealing the Impact of Hallucinated References with 300 Hallucinated Papers in ACL Conferences",
++  "venue": "Annual Meeting of the Association for Computational Linguistics",
++  "year": 2026,
++  "citationCount": 33,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2601.18724, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference",
++   "Review"
++  ],
++  "journal": {
++   "pages": "47295-47376"
++  },
++  "authors": [
++   {
++    "authorId": "2204051777",
++    "name": "Yusuke Sakai"
++   },
++   {
++    "authorId": "2391550956",
++    "name": "Hidetaka Kamigaito"
++   },
++   {
++    "authorId": "2266807418",
++    "name": "Taro Watanabe"
++   }
++  ],
++  "abstract": "Recently, we have often observed hallucinated citations or references that do not correspond to any existing work in papers under review, preprints, or published papers. Such hallucinated citations pose a serious concern to scientific reliability. When they appear in accepted papers, they may also negatively affect the credibility of conferences. In this study, we refer to hallucinated citations as\"HalluCitation\"and systematically investigate their prevalence and impact. We analyze all papers published at ACL, NAACL, and EMNLP in 2024 and 2025, including main conference, Findings, and workshop papers. Our analysis reveals that nearly 300 papers contain at least one HalluCitation, most of which were published in 2025. Notably, half of these papers were identified at EMNLP 2025, the most recent conference, indicating that this issue is rapidly increasing. Moreover, more than 100 such papers were accepted as main conference and Findings papers at EMNLP 2025, affecting the credibility."
++ },
++ "arXiv:2602.08990": {
++  "paperId": "79c1e1d6e6c36193f13bf2f1bd8379cf79f1d5ce",
++  "externalIds": {
++   "ArXiv": "2602.08990",
++   "CorpusId": 285451873
++  },
++  "publicationVenue": null,
++  "title": "InternAgent-1.5: A Unified Agentic Framework for Long-Horizon Autonomous Scientific Discovery",
++  "venue": "",
++  "year": 2026,
++  "citationCount": 32,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2602.08990, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": null,
++  "journal": null,
++  "authors": [
++   {
++    "authorId": "2279407901",
++    "name": "Shi-Yang Feng"
++   },
++   {
++    "authorId": "2363407583",
++    "name": "Run-Min Ma"
++   },
++   {
++    "authorId": "2347774321",
++    "name": "Xiang-Yu Yan"
++   },
++   {
++    "authorId": "2385186146",
++    "name": "Yue Fan"
++   },
++   {
++    "authorId": "2363688841",
++    "name": "Yu-Song Hu"
++   },
++   {
++    "authorId": "2344944680",
++    "name": "Song-Tao Huang"
++   },
++   {
++    "authorId": "2300403265",
++    "name": "Shuai Zhang"
++   },
++   {
++    "authorId": "2114000045",
++    "name": "Zong-Sheng Cao"
++   },
++   {
++    "authorId": "2334474829",
++    "name": "Tian-Shuo Peng"
++   },
++   {
++    "authorId": "2174741399",
++    "name": "Jiakang Yuan"
++   },
++   {
++    "authorId": "2353199601",
++    "name": "Zi-Wen Guo"
++   },
++   {
++    "authorId": "2191902766",
++    "name": "Zhi-Jie Zhong"
++   },
++   {
++    "authorId": "2180683671",
++    "name": "Shangheng Du"
++   },
++   {
++    "authorId": "2332666924",
++    "name": "Weida Wang"
++   },
++   {
++    "authorId": "2384794295",
++    "name": "Jin-Xin Shi"
++   },
++   {
++    "authorId": "2363408730",
++    "name": "Yu-Hao Zhou"
++   },
++   {
++    "authorId": "2401837334",
++    "name": "Xiao-Han He"
++   },
++   {
++    "authorId": "2362816316",
++    "name": "Zhiyin Yu"
++   },
++   {
++    "authorId": "2380462504",
++    "name": "Fang-Chen Yu"
++   },
++   {
++    "authorId": "2331607748",
++    "name": "Qi-Hao Zheng"
++   },
++   {
++    "authorId": "2399112319",
++    "name": "Jia-Min Wu"
++   },
++   {
++    "authorId": "2311543366",
++    "name": "Mian-Xin Liu"
++   },
++   {
++    "authorId": "2397692167",
++    "name": "Chi Zhang"
++   },
++   {
++    "authorId": "2363513075",
++    "name": "Shao-Wei Hou"
++   },
++   {
++    "authorId": "2365969557",
++    "name": "Shuya Li"
++   },
++   {
++    "authorId": "2341627617",
++    "name": "Yan-Kai Jiang"
++   },
++   {
++    "authorId": "2367737142",
++    "name": "Wenjie Lou"
++   },
++   {
++    "authorId": "2313042601",
++    "name": "Li-Long Wang"
++   },
++   {
++    "authorId": "2386639981",
++    "name": "Zifu Wang"
++   },
++   {
++    "authorId": "2364093499",
++    "name": "Jiong Wang"
++   },
++   {
++    "authorId": "2363836740",
++    "name": "Wang-Sheng Xu"
++   },
++   {
++    "authorId": "2317496445",
++    "name": "Yue Deng"
++   },
++   {
++    "authorId": "2152508904",
++    "name": "Dong-Rui Liu"
++   },
++   {
++    "authorId": "2359211589",
++    "name": "Yi-Heng Wang"
++   },
++   {
++    "authorId": "2302816941",
++    "name": "Wen-Long Zhang"
++   },
++   {
++    "authorId": "2006398365",
++    "name": "Feng-Hua Ling"
++   },
++   {
++    "authorId": "2303209608",
++    "name": "Shufei Zhang"
++   },
++   {
++    "authorId": "2339975108",
++    "name": "Xiao-Song Wang"
++   },
++   {
++    "authorId": "9493614",
++    "name": "Shu-qi Zheng"
++   },
++   {
++    "authorId": "2118521552",
++    "name": "Xun Huang"
++   },
++   {
++    "authorId": "2368321514",
++    "name": "Si-Qi Sun"
++   },
++   {
++    "authorId": "2362971451",
++    "name": "Shu-Yue Hu"
++   },
++   {
++    "authorId": "2376196253",
++    "name": "Peng Ye"
++   },
++   {
++    "authorId": "2274709444",
++    "name": "Chun-Dong Song"
++   },
++   {
++    "authorId": "2325196565",
++    "name": "Bin Wang"
++   },
++   {
++    "authorId": "2409018870",
++    "name": "Conghui He"
++   },
++   {
++    "authorId": "2361197049",
++    "name": "Yi-Hao Liu"
++   },
++   {
++    "authorId": "2390409960",
++    "name": "Xin Li"
++   },
++   {
++    "authorId": "2061436526",
++    "name": "Qi-Bin Hou"
++   },
++   {
++    "authorId": "2374014076",
++    "name": "Tao Chen"
++   },
++   {
++    "authorId": "2327887452",
++    "name": "Xiang-Yu Yue"
++   },
++   {
++    "authorId": "2298840639",
++    "name": "Liang He"
++   },
++   {
++    "authorId": "2303596290",
++    "name": "Da-Hua Lin"
++   },
++   {
++    "authorId": "2339741350",
++    "name": "Bo-Wen Zhou"
++   },
++   {
++    "authorId": "2141897317",
++    "name": "Bo Zhang"
++   },
++   {
++    "authorId": "2384327830",
++    "name": "Lei Bai"
++   }
++  ],
++  "abstract": "We introduce InternAgent-1.5, a unified system designed for end-to-end scientific discovery across computational and empirical domains. The system is built on a structured architecture composed of three coordinated subsystems for generation, verification, and evolution. These subsystems are supported by foundational capabilities for deep research, solution optimization, and long horizon memory. The architecture allows InternAgent-1.5 to operate continuously across extended discovery cycles while maintaining coherent and improving behavior. It also enables the system to coordinate computational modeling and laboratory experimentation within a single unified system. We evaluate InternAgent-1.5 on scientific reasoning benchmarks such as GAIA, HLE, GPQA, and FrontierScience, and the system achieves leading performance that demonstrates strong foundational capabilities. Beyond these benchmarks, we further assess two categories of discovery tasks. In algorithm discovery tasks, InternAgent-1.5 autonomously designs competitive methods for core machine learning problems. In empirical discovery tasks, it executes complete computational or wet lab experiments and produces scientific findings in earth, life, biological, and physical domains. Overall, these results show that InternAgent-1.5 provides a general and scalable framework for autonomous scientific discovery."
++ },
++ "arXiv:2603.08127": {
++  "paperId": "143a2e1a8fc27f4f523597aa45cb8f2cf5137020",
++  "externalIds": {
++   "ArXiv": "2603.08127",
++   "DBLP": "journals/corr/abs-2603-08127",
++   "DOI": "10.48550/arXiv.2603.08127",
++   "CorpusId": 286376880
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "EvoScientist: Towards Multi-Agent Evolving AI Scientists for End-to-End Scientific Discovery",
++  "venue": "arXiv.org",
++  "year": 2026,
++  "citationCount": 60,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2603.08127, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2603.08127"
++  },
++  "authors": [
++   {
++    "authorId": "2141789994",
++    "name": "Yougang Lyu"
++   },
++   {
++    "authorId": "2304527137",
++    "name": "Xi Zhang"
++   },
++   {
++    "authorId": "2341456413",
++    "name": "Yu-Yue Zhao"
++   },
++   {
++    "authorId": "2279542724",
++    "name": "Konstantinos Papakostas"
++   },
++   {
++    "authorId": "2313874751",
++    "name": "Xin-Hao Yi"
++   },
++   {
++    "authorId": "2119112577",
++    "name": "Shuyu Guo"
++   },
++   {
++    "authorId": "2261645232",
++    "name": "Wen-Xiang Hu"
++   },
++   {
++    "authorId": "2273931585",
++    "name": "Jan Piotrowski"
++   },
++   {
++    "authorId": "28999125",
++    "name": "Jakub Kaliski"
++   },
++   {
++    "authorId": "1915156",
++    "name": "Jacopo Urbani"
++   },
++   {
++    "authorId": "2315664370",
++    "name": "Zai-Qiao Meng"
++   },
++   {
++    "authorId": "2238355704",
++    "name": "Lu Zhou"
++   },
++   {
++    "authorId": "2315457268",
++    "name": "Zhao-Chun Ren"
++   },
++   {
++    "authorId": "2374005983",
++    "name": "Xiao-Hu Yan"
++   }
++  ],
++  "abstract": "The increasing adoption of Large Language Models (LLMs) has enabled AI scientists to perform complex end-to-end scientific discovery tasks requiring coordination of specialized roles, including idea generation and experimental execution. However, most state-of-the-art AI scientist systems rely on static, hand-designed pipelines and fail to adapt based on accumulated interaction histories. As a result, these systems overlook promising research directions, repeat failed experiments, and pursue infeasible ideas. To address this, we introduce EvoScientist, an evolving multi-agent AI scientist framework that continuously improves research strategies through persistent memory and self-evolution. EvoScientist comprises three specialized agents: a Researcher Agent (RA) for scientific idea generation, an Engineer Agent (EA) for experiment implementation and execution, and an Evolution Manager Agent (EMA) that distills insights from prior interactions into reusable knowledge. EvoScientist contains two persistent memory modules: (i) an ideation memory, which summarizes feasible research directions from top-ranked ideas while recording previously unsuccessful directions; and (ii) an experimentation memory, which captures effective data processing and model training strategies derived from code search trajectories and best-performing implementations. These modules enable the RA and EA to retrieve relevant prior strategies, improving idea quality and code execution success rates over time. Experiments show that EvoScientist outperforms 7 open-source and commercial state-of-the-art systems in scientific idea generation, achieving higher novelty, feasibility, relevance, and clarity via automatic and human evaluation. EvoScientist also substantially improves code execution success rates through multi-agent evolution, demonstrating persistent memory's effectiveness for end-to-end scientific discovery."
++ },
++ "arXiv:2603.29231": {
++  "paperId": "ab0ffa96b2ef941b7cbe3db785e01981f2682254",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2603-29231",
++   "ArXiv": "2603.29231",
++   "DOI": "10.48550/arXiv.2603.29231",
++   "CorpusId": 286977051
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "Beyond pass@1: A Reliability Science Framework for Long-Horizon LLM Agents",
++  "venue": "arXiv.org",
++  "year": 2026,
++  "citationCount": 10,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2603.29231, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2603.29231"
++  },
++  "authors": [
++   {
++    "authorId": "2363444455",
++    "name": "Aaditya Khanal"
++   },
++   {
++    "authorId": "2105710431",
++    "name": "Yang-Yang Tao"
++   },
++   {
++    "authorId": "2329901586",
++    "name": "Junxiu Zhou"
++   }
++  ],
++  "abstract": "Existing benchmarks measure capability -- whether a model succeeds on a single attempt -- but production deployments require reliability -- consistent success across repeated attempts on tasks of varying duration. We show these properties diverge systematically as task duration grows, and that pass@1 on short tasks is structurally blind to this divergence. We introduce a reliability science framework for long-horizon LLM agents with four metrics: Reliability Decay Curve (RDC), Variance Amplification Factor (VAF), Graceful Degradation Score (GDS), and Meltdown Onset Point (MOP). We evaluate 10 models across 23,392 episodes on a 396-task benchmark spanning four duration buckets and three domains. Key findings: (1) reliability decay is domain-stratified -- SE GDS drops from 0.90 to 0.44 while document processing is nearly flat (0.74 to 0.71); (2) VAF bifurcates by capability tier -- high VAF is a capability signature, not an instability signal; (3) capability and reliability rankings diverge substantially, with multi-rank inversions at long horizons; (4) frontier models have the highest meltdown rates (up to 19%) because they attempt ambitious multi-step strategies that sometimes spiral; and (5) memory scaffolds universally hurt long-horizon performance across all 10 models. These results motivate reliability as a first-class evaluation dimension alongside capability."
++ },
++ "arXiv:2605.07723": {
++  "paperId": "620e5d7f81c94a66f51dc89451e80ea4772a6716",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2605-07723",
++   "ArXiv": "2605.07723",
++   "DOI": "10.48550/arXiv.2605.07723",
++   "CorpusId": 288147667
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "LLM hallucinations in the wild: Large-scale evidence from non-existent citations",
++  "venue": "arXiv.org",
++  "year": 2026,
++  "citationCount": 16,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2605.07723, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2605.07723"
++  },
++  "authors": [
++   {
++    "authorId": "1484816455",
++    "name": "Zhen-Yue Zhao"
++   },
++   {
++    "authorId": "2367001043",
++    "name": "Yihe Wang"
++   },
++   {
++    "authorId": "2334807881",
++    "name": "Toby Stuart"
++   },
++   {
++    "authorId": "2484682509",
++    "name": "M. de Vaan"
++   },
++   {
++    "authorId": "3183325",
++    "name": "P. Ginsparg"
++   },
++   {
++    "authorId": "2256439343",
++    "name": "Yian Yin"
++   }
++  ],
++  "abstract": "Large language models (LLMs) are known to generate plausible but false information across a wide range of contexts, yet the real-world magnitude and consequences of this hallucination problem remain poorly understood. Here we leverage a uniquely verifiable object - scientific citations - to audit 111 million references across 2.5 million papers in arXiv, bioRxiv, SSRN, and PubMed Central. We find a sharp rise in non-existent references following widespread LLM adoption, with a conservative estimate of 146,932 hallucinated citations in 2025 alone. These errors are diffusely embedded across many papers but especially pronounced in fields with rapid AI uptake, in manuscripts with linguistic signatures of AI-assisted writing, and among small and early-career author teams. At the same time, hallucinated references disproportionately assign credit to already prominent and male scholars, suggesting that LLM-generated errors may reinforce existing inequities in scientific recognition. Preprint moderation and journal publication processes capture only a fraction of these errors, suggesting that the spread of hallucinated content has outpaced existing safeguards. Together, these findings demonstrate that LLM hallucinations are infiltrating knowledge production at scale, threatening both the reliability and equity of future scientific discovery as human and AI systems draw on the existing literature."
++ },
++ "arXiv:2605.26340": {
++  "paperId": "1d902a3e54fc9af77c55217b1688c6d3a8734757",
++  "externalIds": {
++   "ArXiv": "2605.26340",
++   "DBLP": "journals/corr/abs-2605-26340",
++   "DOI": "10.48550/arXiv.2605.26340",
++   "CorpusId": 288672941
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "ScientistOne: Towards Human-Level Autonomous Research via Chain-of-Evidence",
++  "venue": "arXiv.org",
++  "year": 2026,
++  "citationCount": 21,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2605.26340, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2605.26340"
++  },
++  "authors": [
++   {
++    "authorId": "2385420268",
++    "name": "Rui Meng"
++   },
++   {
++    "authorId": "40135250",
++    "name": "Bhavana Dalvi"
++   },
++   {
++    "authorId": "2155099104",
++    "name": "Jie-Feng Chen"
++   },
++   {
++    "authorId": "2136342754",
++    "name": "Chun-Liang Li"
++   },
++   {
++    "authorId": "2344611832",
++    "name": "Palash Goyal"
++   },
++   {
++    "authorId": "2384366252",
++    "name": "Mihir Parmar"
++   },
++   {
++    "authorId": "2350143431",
++    "name": "Yi-Wen Song"
++   },
++   {
++    "authorId": "2268777158",
++    "name": "Ya-Le Song"
++   },
++   {
++    "authorId": "2446118",
++    "name": "Rajarishi S. Sinha"
++   },
++   {
++    "authorId": "2266304326",
++    "name": "Parthasarathy Ranganathan"
++   },
++   {
++    "authorId": "2225921342",
++    "name": "Burak Gokturk"
++   },
++   {
++    "authorId": "2392692988",
++    "name": "Jinsung Yoon"
++   },
++   {
++    "authorId": "2382765500",
++    "name": "Tomas Pfister"
++   }
++  ],
++  "abstract": "Autonomous research agents produce competitive solutions and professional-looking manuscripts, yet their outputs contain verifiability failures undetectable by surface-level evaluation: fabricated citations, unreproducible scores, and method descriptions that diverge from the implementation. We address this through three contributions. First, Chain-of-Evidence (CoE), a verifiability framework requiring every claim to be traceable to its evidence source. Second, ScientistOne, an end-to-end autonomous research system that maintains evidence chains by construction throughout literature review, solution discovery, and paper writing. Third, CoE Audit, a post-hoc audit whose four integrity checks -- score verification, specification violation, reference verification, and method-code alignment -- apply uniformly to all systems. Across 75 papers spanning five systems and five frontier research tasks, every baseline exhibits at least one systematic failure mode: hallucinated reference rates reach 21%, score verification passes in as few as 42% of papers, and method-code alignment ranges from 20% to 80%. ScientistOne achieves zero hallucinated references (0/337), perfect score verification (12/12), and the highest method-code alignment (14/15), while matching or exceeding human expert performance on all five tasks. ScientistOne further generalizes to six additional tasks spanning medical imaging, fine-grained recognition, 3D perception, and language modeling, achieving state-of-the-art on Parameter Golf and gold medals on MLE-Bench tasks where baselines fail entirely."
++ },
++ "arXiv:2607.12301": {
++  "paperId": "a71870edb216ebbc78ae21b1b4bb9c9b565613f8",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2607-12301",
++   "ArXiv": "2607.12301",
++   "DOI": "10.48550/arXiv.2607.12301",
++   "CorpusId": 290148805
++  },
++  "publicationVenue": {
++   "id": "1901e811-ee72-4b20-8f7e-de08cd395a10",
++   "name": "arXiv.org",
++   "alternate_names": [
++    "ArXiv"
++   ],
++   "issn": "2331-8422",
++   "url": "https://arxiv.org"
++  },
++  "title": "XScientist: A Git-Like Research Protocol for Long-Running Autonomous Scientific Discovery",
++  "venue": "arXiv.org",
++  "year": 2026,
++  "citationCount": 1,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2607.12301, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2607.12301"
++  },
++  "authors": [
++   {
++    "authorId": "2111701253",
++    "name": "Ji-Xi Luo"
++   }
++  ],
++  "abstract": "Autonomous research systems can generate plausible papers while losing the decisions, failed branches, and evidence needed to inspect or continue the work. We present XScientist, a local-first, git-like protocol that treats research state, rather than a manuscript, as the unit of continuation. Hypotheses, experiment attempts, observations, claims, reviews, and handoffs are represented as typed, content-addressed objects in an exploration graph. Immutable checkpoints, explicit negative outcomes, claim-evidence closure, replay boundaries, and authority-aware gates make each transition inspectable without treating a passing integrity check as scientific truth. The protocol exports a portable Agent-Native Research Artifact (ARA) that another agent or human can inspect, fork, verify, and extend. A reference implementation integrates planning, execution, review, repair, and supervised long-running operation while preserving provenance across these stages. We evaluate the protocol with controlled artifact-integrity workloads and matched external task pilots, keeping native task performance separate from evidence and audit claims. The result is an interoperability and accountability layer for long-running autonomous science, with explicit boundaries where human judgment and independent evaluation remain necessary."
++ },
++ "arXiv:2608.05179": {
++  "paperId": "9b5f65e0116f686c746f460c719265a71660c2db",
++  "externalIds": {
++   "ArXiv": "2608.05179",
++   "CorpusId": 290851990
++  },
++  "publicationVenue": null,
++  "title": "Autonomous Research Agents: A Survey of AI Scientists and the Verification Gap",
++  "venue": "",
++  "year": 2026,
++  "citationCount": 8,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2608.05179, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Review"
++  ],
++  "journal": null,
++  "authors": [
++   {
++    "authorId": "2184177582",
++    "name": "Tian-Yu Ding"
++   },
++   {
++    "authorId": "2444870503",
++    "name": "Aditya Nannapaneni"
++   },
++   {
++    "authorId": "2445721236",
++    "name": "Bingfan Liu"
++   },
++   {
++    "authorId": "2445719507",
++    "name": "Ling Zhang"
++   }
++  ],
++  "abstract": "Large language model (LLM) agents are increasingly used across the scientific research lifecycle: ideation, literature search, experiment design and execution, analysis, manuscript drafting, and review. End-to-end AI scientist systems can now produce paper-like manuscripts, but their claims are often harder to verify than their code is to run. This survey studies that gap in computational AI/ML research, where code, benchmarks, experiments, and write-ups are most visible. We screen 125 candidate works and include 35, with full-text coding of 26 entries: 24 runnable systems and two study or position papers. We code seven audit dimensions: lifecycle stage, autonomy level, evaluation method, released artifacts, human-in-the-loop points, novelty verification, and result-selection disclosure. The main pattern is that code release is now common, but reproducibility-grade and claim-verification artifacts remain much less common. In the 24 runnable systems, 83 percent release code, while 38 percent release seeds or execution traces and 38 percent report any novelty-verification method. Among nine closed-loop L4 systems, seven are mechanical reruns and one is author-claimed without an external check; no LLM-era system in the corpus demonstrates an externally validated in-loop oracle under our coding rule. We contribute a coded corpus, a lifecycle-by-autonomy map, an auditability-gap analysis, and a reviewer-facing reporting checklist. The survey argues that the field's central bottleneck is no longer only whether agents can complete research tasks, but whether reviewers can verify the claims those agents produce."
++ },
++ "arXiv:2608.14380": {
++  "paperId": "a6c6d02eb805bd1d071df21d84ee2d9605985f9a",
++  "externalIds": {
++   "ArXiv": "2608.14380",
++   "CorpusId": 291168806
++  },
++  "publicationVenue": null,
++  "title": "AgentRewind: Recoverable Execution for Long-Horizon LLM Agents",
++  "venue": "",
++  "year": 2026,
++  "citationCount": 11,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2608.14380, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": null,
++  "journal": null,
++  "authors": [
++   {
++    "authorId": "2464739722",
++    "name": "Yu Zhuang"
++   },
++   {
++    "authorId": "2398377257",
++    "name": "Kefei Chen"
++   },
++   {
++    "authorId": "2368401349",
++    "name": "Yitong Duan"
++   },
++   {
++    "authorId": "2274198400",
++    "name": "Shu-Xin Zheng"
++   },
++   {
++    "authorId": "2351726800",
++    "name": "Jian Li"
++   },
++   {
++    "authorId": "2458017725",
++    "name": "Xu-Yao Zhang"
++   }
++  ],
++  "abstract": "Many real-world tasks require LLM agents to interact with their environments over long execution horizons. Errors that occur early in execution may propagate through both the agent context and environment state, and their effects may be difficult to reverse through subsequent actions. Existing methods mainly seek to reduce such errors through plan refinement and safety checks but provide little support after errors occur. To enable recovery during long-horizon execution, we present AgentRewind, a runtime recovery framework that records aligned checkpoints of the agent context and controlled environment, allowing agents to return to an earlier state and resume execution with information from previous attempts. We also construct MettleBench, a benchmark for evaluating task completion and partial progress on long-horizon engineering assignments containing a series of related requirements. Experiments across tasks, multiple models, execution strategies, and agent harnesses show that AgentRewind improves task success rate and average checklist progress over the compared baselines."
++ },
++ "DOI:10.1002/adma.202413523": {
++  "paperId": "dbbcdb281ed6aa646af0402172cf8a7cfda85d5c",
++  "externalIds": {
++   "PubMedCentral": "12138853",
++   "DOI": "10.1002/adma.202413523",
++   "CorpusId": 274858957,
++   "PubMed": "39696898"
++  },
++  "publicationVenue": {
++   "id": "71697426-616f-45b4-b9d8-d647335a32e6",
++   "name": "Advances in Materials",
++   "type": "journal",
++   "alternate_names": [
++    "Adv Mater",
++    "Advanced Materials"
++   ],
++   "issn": "2327-2503",
++   "alternate_issns": [
++    "0935-9648"
++   ],
++   "url": "http://www.sciencepublishinggroup.com/journal/archive.aspx?amp;issueid=-1&journalid=129",
++   "alternate_urls": [
++    "http://www.advmat.de/",
++    "https://onlinelibrary.wiley.com/journal/15214095",
++    "http://www.sciencepublishinggroup.com/journal/archive.aspx?issueid=-1&journalid=129",
++    "http://www.wiley-vch.de/publish/en/journals/alphabeticIndex/2089/"
++   ]
++  },
++  "title": "SciAgents: Automating Scientific Discovery Through Bioinspired Multi\u2010Agent Intelligent Graph Reasoning",
++  "venue": "Advances in Materials",
++  "year": 2024,
++  "citationCount": 297,
++  "openAccessPdf": {
++   "url": "https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/adma.202413523",
++   "status": "HYBRID",
++   "license": "CCBYNC",
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC12138853, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Advanced Materials (Deerfield Beach, Fla.)",
++   "volume": "37"
++  },
++  "authors": [
++   {
++    "authorId": "2310665423",
++    "name": "Alireza Ghafarollahi"
++   },
++   {
++    "authorId": "2252265640",
++    "name": "Markus J. Buehler"
++   }
++  ],
++  "abstract": "A key challenge in artificial intelligence (AI) is the creation of systems capable of autonomously advancing scientific understanding by exploring novel domains, identifying complex patterns, and uncovering previously unseen connections in vast scientific data. In this work, SciAgents, an approach that leverages three core concepts is presented: (1) large\u2010scale ontological knowledge graphs to organize and interconnect diverse scientific concepts, (2) a suite of large language models (LLMs) and data retrieval tools, and (3) multi\u2010agent systems with in\u2010situ learning capabilities. Applied to biologically inspired materials, SciAgents reveals hidden interdisciplinary relationships that were previously considered unrelated, achieving a scale, precision, and exploratory power that surpasses human research methods. The framework autonomously generates and refines research hypotheses, elucidating underlying mechanisms, design principles, and unexpected material properties. By integrating these capabilities in a modular fashion, the system yields material discoveries, critiques and improves existing hypotheses, retrieves up\u2010to\u2010date data about existing research, and highlights strengths and limitations. This is achieved by harnessing a \u201cswarm of intelligence\u201d similar to biological systems, providing new avenues for discovery. How this model accelerates the development of advanced materials by unlocking Nature's design principles, resulting in a new biocomposite with enhanced mechanical properties and improved sustainability through energy\u2010efficient production is shown."
++ },
++ "DOI:10.1006/ijhc.1999.0252": {
++  "paperId": "8580dc7ba0f0d7eb1c2495ceb23af9491a805849",
++  "externalIds": {
++   "DBLP": "journals/ijmms/SkitkaMB99",
++   "MAG": "2081011876",
++   "DOI": "10.1006/ijhc.1999.0252",
++   "CorpusId": 1863226
++  },
++  "publicationVenue": null,
++  "title": "Does automation bias decision-making?",
++  "venue": "Int. J. Hum. Comput. Stud.",
++  "year": 1999,
++  "citationCount": 702,
++  "openAccessPdf": {
++   "url": "",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1006/ijhc.1999.0252?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1006/ijhc.1999.0252, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Int. J. Hum. Comput. Stud.",
++   "pages": "991-1006",
++   "volume": "51"
++  },
++  "authors": [
++   {
++    "authorId": "2622061",
++    "name": "L. Skitka"
++   },
++   {
++    "authorId": "2249256",
++    "name": "K. Mosier"
++   },
++   {
++    "authorId": "10806842",
++    "name": "M. D. Burdick"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1007/s10994-025-06955-2": {
++  "paperId": "98f166771681acfe51c0f69558d8a977f0782739",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2305-02251",
++   "ArXiv": "2305.02251",
++   "DOI": "10.1007/s10994-025-06955-2",
++   "CorpusId": 258461620
++  },
++  "publicationVenue": {
++   "id": "22c9862f-a25e-40cd-9d31-d09e68a293e6",
++   "name": "Machine-mediated learning",
++   "type": "journal",
++   "alternate_names": [
++    "Mach learn",
++    "Machine Learning",
++    "Mach Learn"
++   ],
++   "issn": "0732-6718",
++   "alternate_issns": [
++    "0885-6125"
++   ],
++   "url": "http://www.springer.com/computer/artificial/journal/10994",
++   "alternate_urls": [
++    "https://link.springer.com/journal/10994",
++    "http://www.springer.com/west/home/computer/artificial?SGWID=4-147-70-35726603-0"
++   ]
++  },
++  "title": "Automated Scientific Discovery: From Equation Discovery to Autonomous Discovery Systems",
++  "venue": "Machine-mediated learning",
++  "year": 2023,
++  "citationCount": 30,
++  "openAccessPdf": {
++   "url": "https://link.springer.com/content/pdf/10.1007/s10994-025-06955-2.pdf",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2305.02251, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Machine Learning",
++   "volume": "115"
++  },
++  "authors": [
++   {
++    "authorId": "2058253559",
++    "name": "Stefan Kramer"
++   },
++   {
++    "authorId": "1382852308",
++    "name": "Mattia Cerrato"
++   },
++   {
++    "authorId": "2037478739",
++    "name": "Jannis Brugger"
++   },
++   {
++    "authorId": "1693549",
++    "name": "S. D\u017eeroski"
++   },
++   {
++    "authorId": "33654217",
++    "name": "R. King"
++   }
++  ],
++  "abstract": "The paper surveys automated scientific discovery, from equation discovery and symbolic regression to autonomous discovery systems and agents. It discusses the individual approaches from a \"big picture\" perspective and in context, but also discusses open issues and recent topics like the various roles of deep neural networks in this area, aiding in the discovery of human-interpretable knowledge. Further, we will present closed-loop scientific discovery systems, starting with the pioneering work on the Adam system up to current efforts in fields from material science to astronomy. Finally, we will elaborate on autonomy from a machine learning perspective, but also in analogy to the autonomy levels in autonomous driving. The maximal level, level five, is defined to require no human intervention at all in the production of scientific knowledge. Achieving this is one step towards solving the Nobel Turing Grand Challenge to develop AI Scientists: AI systems capable of making Nobel-quality scientific discoveries highly autonomously at a level comparable, and possibly superior, to the best human scientists by 2050."
++ },
++ "DOI:10.1016/0005-1098(83)90046-8": {
++  "paperId": "0713bb9d9b138e4e0a15406006de9b0cddf68e28",
++  "externalIds": {
++   "MAG": "2008653298",
++   "DBLP": "journals/automatica/Bainbridge83",
++   "DOI": "10.1016/0005-1098(83)90046-8",
++   "CorpusId": 12667742
++  },
++  "publicationVenue": {
++   "id": "14f974af-a3bc-407d-92cb-c6cdbd6d94ee",
++   "name": "at - Automatisierungstechnik",
++   "type": "journal",
++   "alternate_names": [
++    " Autom",
++    "At-automatisierungstechnik"
++   ],
++   "issn": "0178-2312",
++   "url": "http://www.degruyter.com/loi/auto",
++   "alternate_urls": [
++    "https://www.degruyter.com/view/j/auto"
++   ]
++  },
++  "title": "Ironies of automation",
++  "venue": "at - Automatisierungstechnik",
++  "year": 1982,
++  "citationCount": 2717,
++  "openAccessPdf": {
++   "url": "",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1016/0005-1098(83)90046-8?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1016/0005-1098(83)90046-8, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Autom.",
++   "pages": "775-779",
++   "volume": "19"
++  },
++  "authors": [
++   {
++    "authorId": "10352604",
++    "name": "L. Bainbridge"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1016/j.cortex.2012.12.016": {
++  "paperId": "ddb5829aec88f87b796171a7ce347aa485fc99bc",
++  "externalIds": {
++   "MAG": "2002030061",
++   "DOI": "10.1016/j.cortex.2012.12.016",
++   "CorpusId": 140204881,
++   "PubMed": "23347556"
++  },
++  "publicationVenue": null,
++  "title": "Registered reports: a new publishing initiative at Cortex.",
++  "venue": "Cortex; a journal devoted to the study of the nervous system and behavior",
++  "year": 2013,
++  "citationCount": 666,
++  "openAccessPdf": {
++   "url": "",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1016/j.cortex.2012.12.016?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1016/j.cortex.2012.12.016, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Editorial"
++  ],
++  "journal": {
++   "name": "Cortex; a journal devoted to the study of the nervous system and behavior",
++   "pages": "\n          609-10\n        ",
++   "volume": "49 3"
++  },
++  "authors": [
++   {
++    "authorId": "2001975",
++    "name": "C. Chambers"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1016/j.future.2014.10.008": {
++  "paperId": "4224374796da64e17fce96033d4cd42240d80eaf",
++  "externalIds": {
++   "MAG": "2078962046",
++   "DBLP": "journals/fgcs/DeelmanVJRCMMCS15",
++   "DOI": "10.1016/J.FUTURE.2014.10.008",
++   "CorpusId": 14611345
++  },
++  "publicationVenue": {
++   "id": "e3e1f91c-fe26-4f67-8d15-f7acd8abdf42",
++   "name": "Future generations computer systems",
++   "type": "journal",
++   "alternate_names": [
++    "Future gener comput syst",
++    "Future Generation Computer Systems",
++    "Future Gener Comput Syst"
++   ],
++   "issn": "0167-739X",
++   "url": "http://www.elsevier.com/wps/find/journaleditorialboard.cws_home/505611/editorialboard",
++   "alternate_urls": [
++    "https://www.journals.elsevier.com/future-generation-computer-systems",
++    "http://www.sciencedirect.com/science/journal/0167739X"
++   ]
++  },
++  "title": "Pegasus, a workflow management system for science automation",
++  "venue": "Future generations computer systems",
++  "year": 2015,
++  "citationCount": 962,
++  "openAccessPdf": {
++   "url": "https://doi.org/10.1016/j.future.2014.10.008",
++   "status": "BRONZE",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1016/J.FUTURE.2014.10.008?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1016/J.FUTURE.2014.10.008, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Future Gener. Comput. Syst.",
++   "pages": "17-35",
++   "volume": "46"
++  },
++  "authors": [
++   {
++    "authorId": "1687896",
++    "name": "E. Deelman"
++   },
++   {
++    "authorId": "1742157",
++    "name": "K. Vahi"
++   },
++   {
++    "authorId": "1711146",
++    "name": "G. Juve"
++   },
++   {
++    "authorId": "1920965",
++    "name": "M. Rynge"
++   },
++   {
++    "authorId": "1711636",
++    "name": "S. Callaghan"
++   },
++   {
++    "authorId": "35376674",
++    "name": "P. Maechling"
++   },
++   {
++    "authorId": "2956722",
++    "name": "R. Mayani"
++   },
++   {
++    "authorId": "48994458",
++    "name": "Weiwei Chen"
++   },
++   {
++    "authorId": "145971089",
++    "name": "Rafael Ferreira da Silva"
++   },
++   {
++    "authorId": "1684336",
++    "name": "M. Livny"
++   },
++   {
++    "authorId": "145559269",
++    "name": "R. K. Wenger"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1016/j.patter.2021.100322": {
++  "paperId": "1f1297064e405bbcd7552f9d3faf401b2747b814",
++  "externalIds": {
++   "DBLP": "journals/patterns/LeipzigNHRG21",
++   "ArXiv": "2006.08589",
++   "MAG": "3035267492",
++   "PubMedCentral": "8441584",
++   "DOI": "10.1016/j.patter.2021.100322",
++   "CorpusId": 219687399,
++   "PubMed": "34553169"
++  },
++  "publicationVenue": {
++   "id": "17bac89e-3dba-467a-b9d4-71e3baefb08b",
++   "name": "Patterns",
++   "type": "journal",
++   "issn": "2666-3899",
++   "url": "https://www.cell.com/patterns",
++   "alternate_urls": [
++    "https://www.sciencedirect.com/journal/patterns"
++   ]
++  },
++  "title": "The role of metadata in reproducible computational research",
++  "venue": "Patterns",
++  "year": 2020,
++  "citationCount": 128,
++  "openAccessPdf": {
++   "url": "http://www.cell.com/article/S2666389921001707/pdf",
++   "status": "GOLD",
++   "license": "CCBYNCSA",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2006.08589, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Review",
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Patterns",
++   "volume": "2"
++  },
++  "authors": [
++   {
++    "authorId": "2057227799",
++    "name": "Jeremy Leipzig"
++   },
++   {
++    "authorId": "2587120",
++    "name": "Daniel N\u00fcst"
++   },
++   {
++    "authorId": "35477117",
++    "name": "Charles Tapley Hoyt"
++   },
++   {
++    "authorId": "2161614",
++    "name": "Stian Soiland-Reyes"
++   },
++   {
++    "authorId": "144686135",
++    "name": "Karthik Ram"
++   },
++   {
++    "authorId": "2057251522",
++    "name": "J. Greenberg"
++   }
++  ],
++  "abstract": "Summary Reproducible computational research (RCR) is the keystone of the scientific method for in silico analyses, packaging the transformation of raw data to published results. In addition to its role in research integrity, improving the reproducibility of scientific studies can accelerate evaluation and reuse. This potential and wide support for the FAIR principles have motivated interest in metadata standards supporting reproducibility. Metadata provide context and provenance to raw data and methods and are essential to both discovery and validation. Despite this shared connection with scientific data, few studies have explicitly described how metadata enable reproducible computational research. This review employs a functional content analysis to identify metadata standards that support reproducibility across an analytic stack consisting of input data, tools, notebooks, pipelines, and publications. Our review provides background context, explores gaps, and discovers component trends of embeddedness and methodology weight from which we derive recommendations for future work."
++ },
++ "DOI:10.1016/j.websem.2015.04.001": {
++  "paperId": "96cce1917fa1c9f6e2c5f7298c5a12cb3dd60dcd",
++  "externalIds": {
++   "MAG": "2029021214",
++   "DBLP": "journals/ws/MoreauGCLM15",
++   "DOI": "10.1016/J.WEBSEM.2015.04.001",
++   "CorpusId": 14672695
++  },
++  "publicationVenue": {
++   "id": "78d61825-c1d6-4e7f-ab9d-1525db71105c",
++   "name": "Journal of Web Semantics",
++   "type": "journal",
++   "alternate_names": [
++    "J Web Semant"
++   ],
++   "issn": "1570-8268",
++   "url": "http://www.elsevier.com/locate/websem",
++   "alternate_urls": [
++    "http://www.sciencedirect.com/science/journal/15708268",
++    "http://www.elsevier.com/wps/find/journaldescription.cws_home/671322/description"
++   ]
++  },
++  "title": "The rationale of PROV",
++  "venue": "Journal of Web Semantics",
++  "year": 2015,
++  "citationCount": 120,
++  "openAccessPdf": {
++   "url": "https://www.sciencedirect.com/science/article/pii/S1570826815000177/pdf",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1016/J.WEBSEM.2015.04.001?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1016/J.WEBSEM.2015.04.001, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "J. Web Semant.",
++   "pages": "235-257",
++   "volume": "35"
++  },
++  "authors": [
++   {
++    "authorId": "1773775",
++    "name": "L. Moreau"
++   },
++   {
++    "authorId": "143772714",
++    "name": "Paul Groth"
++   },
++   {
++    "authorId": "144320476",
++    "name": "James Cheney"
++   },
++   {
++    "authorId": "1760030",
++    "name": "Timothy Lebo"
++   },
++   {
++    "authorId": "145116379",
++    "name": "S. Miles"
++   }
++  ],
++  "abstract": "The prov family of documents are the final output of the World Wide Web Consortium Provenance Working Group, chartered to specify a representation of provenance to facilitate its exchange over the Web. This article reflects upon the key requirements, guiding principles, and design decisions that influenced the prov family of documents. A broad range of requirements were found, relating to the key concepts necessary for describing provenance, such as resources, activities, agents and events, and to balancing prov's ease of use with the facility to check its validity. By this retrospective requirement analysis, the article aims to provide some insights into how prov turned out as it did and why. Benefits of this insight include better inter-operability, a roadmap for alternate investigations and improvements, and solid foundations for future standardization activities."
++ },
++ "DOI:10.1038/nbt.3820": {
++  "paperId": "aa02474345b8601509b607ef015ba1d7c7ed7af2",
++  "externalIds": {
++   "MAG": "2605897695",
++   "DOI": "10.1038/nbt.3820",
++   "CorpusId": 9690740,
++   "PubMed": "28398311"
++  },
++  "publicationVenue": {
++   "id": "458166b3-de17-4bf3-bbbb-e53782de2f0f",
++   "name": "Nature Biotechnology",
++   "type": "journal",
++   "alternate_names": [
++    "Nat Biotechnol"
++   ],
++   "issn": "1087-0156",
++   "url": "http://www.nature.com/nbt/",
++   "alternate_urls": [
++    "http://www.nature.com/nbt"
++   ]
++  },
++  "title": "Nextflow enables reproducible computational workflows",
++  "venue": "Nature Biotechnology",
++  "year": 2017,
++  "citationCount": 3722,
++  "openAccessPdf": {
++   "url": "",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1038/nbt.3820?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1038/nbt.3820, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "LettersAndComments"
++  ],
++  "journal": {
++   "name": "Nature Biotechnology",
++   "pages": "316-319",
++   "volume": "35"
++  },
++  "authors": [
++   {
++    "authorId": "11059942",
++    "name": "P. Di Tommaso"
++   },
++   {
++    "authorId": "2613149",
++    "name": "Maria Chatzou"
++   },
++   {
++    "authorId": "2141512433",
++    "name": "Evan W. Floden"
++   },
++   {
++    "authorId": "34903918",
++    "name": "Pablo Prieto Barja"
++   },
++   {
++    "authorId": "2083101458",
++    "name": "Emilio Palumbo"
++   },
++   {
++    "authorId": "46726219",
++    "name": "C. Notredame"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1038/s41562-016-0021": {
++  "paperId": "a68ce412e92d87ca0116519651bbc484d98c76ae",
++  "externalIds": {
++   "PubMedCentral": "7610724",
++   "MAG": "2727329419",
++   "DOI": "10.1038/s41562-016-0021",
++   "CorpusId": 6326747,
++   "PubMed": "33954258"
++  },
++  "publicationVenue": {
++   "id": "b7e5a3b3-2519-4ca2-8870-c2b45ef76ab9",
++   "name": "Nature Human Behaviour",
++   "type": "journal",
++   "alternate_names": [
++    "Nat Hum Behav"
++   ],
++   "issn": "2397-3374",
++   "url": "https://www.nature.com/nathumbehav/"
++  },
++  "title": "A manifesto for reproducible science",
++  "venue": "Nature Human Behaviour",
++  "year": 2017,
++  "citationCount": 2932,
++  "openAccessPdf": {
++   "url": "https://www.nature.com/articles/s41562-016-0021.pdf",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC7610724, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Nature human behaviour",
++   "volume": "1"
++  },
++  "authors": [
++   {
++    "authorId": "3057626",
++    "name": "M. Munafo"
++   },
++   {
++    "authorId": "2862527",
++    "name": "Brian A. Nosek"
++   },
++   {
++    "authorId": "2064317886",
++    "name": "D. Bishop"
++   },
++   {
++    "authorId": "4058655",
++    "name": "K. Button"
++   },
++   {
++    "authorId": "49625330",
++    "name": "Christopher D. Chambers"
++   },
++   {
++    "authorId": "46181878",
++    "name": "N. P. Sert"
++   },
++   {
++    "authorId": "2130902",
++    "name": "U. Simonsohn"
++   },
++   {
++    "authorId": "2505272",
++    "name": "E. Wagenmakers"
++   },
++   {
++    "authorId": "3959303",
++    "name": "J. Ware"
++   },
++   {
++    "authorId": "145441750",
++    "name": "J. Ioannidis"
++   }
++  ],
++  "abstract": "Improving the reliability and efficiency of scientific research will increase the credibility of the published scientific literature and accelerate discovery. Here we argue for the adoption of measures to optimize key elements of the scientific process: methods, reporting and dissemination, reproducibility, evaluation and incentives. There is some evidence from both simulations and empirical studies supporting the likely effectiveness of these measures, but their broad adoption by researchers, institutions, funders and journals will require iterative evaluation and improvement. We discuss the goals of these measures, and how they can be implemented, in the hope that this will facilitate action toward improving the transparency, reproducibility and efficiency of scientific research."
++ },
++ "DOI:10.1038/s41562-021-01142-4": {
++  "paperId": "b870928f45680afa8e394c85bfaddd487cd6c4d2",
++  "externalIds": {
++   "DOI": "10.1038/s41562-021-01142-4",
++   "CorpusId": 235634310,
++   "PubMed": "34168323"
++  },
++  "publicationVenue": {
++   "id": "b7e5a3b3-2519-4ca2-8870-c2b45ef76ab9",
++   "name": "Nature Human Behaviour",
++   "type": "journal",
++   "alternate_names": [
++    "Nat Hum Behav"
++   ],
++   "issn": "2397-3374",
++   "url": "https://www.nature.com/nathumbehav/"
++  },
++  "title": "Initial evidence of research quality of registered reports compared with the standard publishing model",
++  "venue": "Nature Human Behaviour",
++  "year": 2021,
++  "citationCount": 187,
++  "openAccessPdf": {
++   "url": "https://doi.org/10.31222/osf.io/7x9vy",
++   "status": "GREEN",
++   "license": "public-domain",
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1038/s41562-021-01142-4?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1038/s41562-021-01142-4, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Nature Human Behaviour",
++   "pages": "990 - 997",
++   "volume": "5"
++  },
++  "authors": [
++   {
++    "authorId": "4110370",
++    "name": "C. Soderberg"
++   },
++   {
++    "authorId": "52004348",
++    "name": "Timothy M. Errington"
++   },
++   {
++    "authorId": "40019962",
++    "name": "Sarah R. Schiavone"
++   },
++   {
++    "authorId": "30533087",
++    "name": "Julia G. Bottesini"
++   },
++   {
++    "authorId": "66300351",
++    "name": "Felix Singleton Thorn"
++   },
++   {
++    "authorId": "2809726",
++    "name": "S. Vazire"
++   },
++   {
++    "authorId": "47450657",
++    "name": "Kevin M. Esterling"
++   },
++   {
++    "authorId": "152190711",
++    "name": "B. Nosek"
++   }
++  ],
++  "abstract": "In registered reports (RRs), initial peer review and in-principle acceptance occur before knowing the research outcomes. This combats publication bias and distinguishes planned from unplanned research. How RRs could improve the credibility of research findings is straightforward, but there is little empirical evidence. Also, there could be unintended costs such as reducing novelty. Here, 353 researchers peer reviewed a pair of papers from 29 published RRs from psychology and neuroscience and 57 non-RR comparison papers. RRs numerically outperformed comparison papers on all 19 criteria (mean difference 0.46, scale range \u22124 to +4) with effects ranging from RRs being statistically indistinguishable from comparison papers in novelty (0.13, 95% credible interval [\u22120.24, 0.49]) and creativity (0.22, [\u22120.14, 0.58]) to sizeable improvements in rigour of methodology (0.99, [0.62, 1.35]) and analysis (0.97, [0.60, 1.34]) and overall paper quality (0.66, [0.30, 1.02]). RRs could improve research quality while reducing publication bias and ultimately improve the credibility of the published literature. Soderberg et al. asked scientists to peer review registered reports and standard articles post-publication, after information explicitly identifying the article type had been removed. Registered reports scored higher on some dimensions, including quality and rigour."
++ },
++ "DOI:10.1038/s41562-021-01193-7": {
++  "paperId": "08ed407fcb883c877bcae0b4bde71bb4ddaf4e50",
++  "externalIds": {
++   "DOI": "10.1038/s41562-021-01193-7",
++   "CorpusId": 243200855,
++   "PubMed": "34782730"
++  },
++  "publicationVenue": {
++   "id": "b7e5a3b3-2519-4ca2-8870-c2b45ef76ab9",
++   "name": "Nature Human Behaviour",
++   "type": "journal",
++   "alternate_names": [
++    "Nat Hum Behav"
++   ],
++   "issn": "2397-3374",
++   "url": "https://www.nature.com/nathumbehav/"
++  },
++  "title": "The past, present and future of Registered Reports",
++  "venue": "Nature Human Behaviour",
++  "year": 2020,
++  "citationCount": 366,
++  "openAccessPdf": {
++   "url": "https://www.nature.com/articles/s41562-021-01193-7.pdf",
++   "status": "HYBRID",
++   "license": "other-oa",
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1038/s41562-021-01193-7?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1038/s41562-021-01193-7, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Review",
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Nature Human Behaviour",
++   "pages": "29 - 42",
++   "volume": "6"
++  },
++  "authors": [
++   {
++    "authorId": "46975669",
++    "name": "C. Chambers"
++   },
++   {
++    "authorId": "52215687",
++    "name": "Loukia Tzavella"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1038/s41562-024-02024-1": {
++  "paperId": "02aec976011a7d595d61db9639209968f4b93177",
++  "externalIds": {
++   "PubMedCentral": "11659167",
++   "ArXiv": "2405.06087",
++   "DOI": "10.1038/s41562-024-02024-1",
++   "CorpusId": 269741352,
++   "PubMed": "39468277"
++  },
++  "publicationVenue": {
++   "id": "b7e5a3b3-2519-4ca2-8870-c2b45ef76ab9",
++   "name": "Nature Human Behaviour",
++   "type": "journal",
++   "alternate_names": [
++    "Nat Hum Behav"
++   ],
++   "issn": "2397-3374",
++   "url": "https://www.nature.com/nathumbehav/"
++  },
++  "title": "When combinations of humans and AI are useful: A systematic review and meta-analysis",
++  "venue": "Nature Human Behaviour",
++  "year": 2024,
++  "citationCount": 662,
++  "openAccessPdf": {
++   "url": "https://www.nature.com/articles/s41562-024-02024-1.pdf",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2405.06087, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Nature Human Behaviour",
++   "pages": "2293 - 2303",
++   "volume": "8"
++  },
++  "authors": [
++   {
++    "authorId": "153024015",
++    "name": "Michelle Vaccaro"
++   },
++   {
++    "authorId": "3085084",
++    "name": "Abdullah Almaatouq"
++   },
++   {
++    "authorId": "2257866472",
++    "name": "Thomas W. Malone"
++   }
++  ],
++  "abstract": "Inspired by the increasing use of artificial intelligence (AI) to augment humans, researchers have studied human\u2013AI systems involving different tasks, systems and populations. Despite such a large body of work, we lack a broad conceptual understanding of when combinations of humans and AI are better than either alone. Here we addressed this question by conducting a preregistered systematic review and meta-analysis of 106 experimental studies reporting 370 effect sizes. We searched an interdisciplinary set of databases (the Association for Computing Machinery Digital Library, the Web of Science and the Association for Information Systems eLibrary) for studies published between 1 January 2020 and 30 June 2023. Each study was required to include an original human-participants experiment that evaluated the performance of humans alone, AI alone and human\u2013AI combinations. First, we found that, on average, human\u2013AI combinations performed significantly worse than the best of humans or AI alone (Hedges\u2019 g\u2009=\u2009\u22120.23; 95% confidence interval, \u22120.39 to \u22120.07). Second, we found performance losses in tasks that involved making decisions and significantly greater gains in tasks that involved creating content. Finally, when humans outperformed AI alone, we found performance gains in the combination, but when AI outperformed humans alone, we found losses. Limitations of the evidence assessed here include possible publication bias and variations in the study designs analysed. Overall, these findings highlight the heterogeneity of the effects of human\u2013AI collaboration and point to promising avenues for improving human\u2013AI systems. Vaccaro et al. present a systematic review and meta-analysis of the performance of human\u2013AI combinations, finding that on average, human\u2013AI combinations performed significantly worse than the best of humans or AI alone. They also found performance losses in decision-making tasks and significantly greater gains in content creation tasks."
++ },
++ "DOI:10.1038/s41586-023-06221-2": {
++  "paperId": "f08060425aa8a212d74185ee23a08329b89abcd2",
++  "externalIds": {
++   "DBLP": "journals/nature/WangFD0HLCLKDAB23",
++   "DOI": "10.1038/s41586-023-06221-2",
++   "CorpusId": 260384616,
++   "PubMed": "37532811"
++  },
++  "publicationVenue": {
++   "id": "6c24a0a0-b07d-4d7b-a19b-fd09a3ed453a",
++   "name": "Nature",
++   "type": "journal",
++   "issn": "0028-0836",
++   "url": "https://www.nature.com/",
++   "alternate_urls": [
++    "http://www.nature.com/nature/",
++    "https://www.nature.com/nature/",
++    "http://www.nature.com/nature/archive/index.html"
++   ]
++  },
++  "title": "Scientific discovery in the age of artificial intelligence",
++  "venue": "Nature",
++  "year": 2023,
++  "citationCount": 2017,
++  "openAccessPdf": {
++   "url": "https://hdl.handle.net/1721.1/165433",
++   "status": "GREEN",
++   "license": "CCBYNC",
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1038/s41586-023-06221-2?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1038/s41586-023-06221-2, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Review",
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Nature",
++   "pages": "47-60",
++   "volume": "620"
++  },
++  "authors": [
++   {
++    "authorId": "46506460",
++    "name": "Hanchen Wang"
++   },
++   {
++    "authorId": "2427076",
++    "name": "Tianfan Fu"
++   },
++   {
++    "authorId": "93584228",
++    "name": "Yuanqi Du"
++   },
++   {
++    "authorId": "2153575781",
++    "name": "Wen-Hao Gao"
++   },
++   {
++    "authorId": "49454094",
++    "name": "Kexin Huang"
++   },
++   {
++    "authorId": "2145253202",
++    "name": "Zi-Ming Liu"
++   },
++   {
++    "authorId": "89356020",
++    "name": "Payal Chandak"
++   },
++   {
++    "authorId": "1563693999",
++    "name": "Sheng-Chao Liu"
++   },
++   {
++    "authorId": "11295186",
++    "name": "Peter L. Vankatwyk"
++   },
++   {
++    "authorId": "48860334",
++    "name": "Andreea Deac"
++   },
++   {
++    "authorId": "2047844",
++    "name": "Anima Anandkumar"
++   },
++   {
++    "authorId": "46354410",
++    "name": "K. Bergen"
++   },
++   {
++    "authorId": "2188778449",
++    "name": "Carla P. Gomes"
++   },
++   {
++    "authorId": "2226475487",
++    "name": "Shirley Ho"
++   },
++   {
++    "authorId": "143967473",
++    "name": "Pushmeet Kohli"
++   },
++   {
++    "authorId": "1764249",
++    "name": "Joan Lasenby"
++   },
++   {
++    "authorId": "1702139",
++    "name": "J. Leskovec"
++   },
++   {
++    "authorId": "2110264337",
++    "name": "Tie-Yan Liu"
++   },
++   {
++    "authorId": "2613935",
++    "name": "Arjun K. Manrai"
++   },
++   {
++    "authorId": "2226493959",
++    "name": "Debora S. Marks"
++   },
++   {
++    "authorId": "2378027",
++    "name": "B. Ramsundar"
++   },
++   {
++    "authorId": "2159642226",
++    "name": "Le Song"
++   },
++   {
++    "authorId": "2504051310",
++    "name": "Ji-Meng Sun"
++   },
++   {
++    "authorId": "2115854801",
++    "name": "Jian Tang"
++   },
++   {
++    "authorId": "3444569",
++    "name": "Petar Velickovic"
++   },
++   {
++    "authorId": "2228923747",
++    "name": "Max Welling"
++   },
++   {
++    "authorId": "2359205623",
++    "name": "Lin-Feng Zhang"
++   },
++   {
++    "authorId": "13027820",
++    "name": "Connor W. Coley"
++   },
++   {
++    "authorId": "1865800402",
++    "name": "Y. Bengio"
++   },
++   {
++    "authorId": "2095762",
++    "name": "M. Zitnik"
++   }
++  ],
++  "abstract": "Artificial intelligence (AI) is being increasingly integrated into scientific discovery to augment and accelerate research, helping scientists to generate hypotheses, design experiments, collect and interpret large datasets, and gain insights that might not have been possible using traditional scientific methods alone. Here we examine breakthroughs over the past decade that include self-supervised learning, which allows models to be trained on vast amounts of unlabelled data, and geometric deep learning, which leverages knowledge about the structure of scientific data to enhance model accuracy and efficiency. Generative AI methods can create designs, such as small-molecule drugs and proteins, by analysing diverse data modalities, including images and sequences. We discuss how these methods can help scientists throughout the scientific process and the central issues that remain despite such advances. Both developers and users of AI tools need a better understanding of when such approaches need improvement, and challenges posed by poor data quality and stewardship remain. These issues cut across scientific disciplines and require developing foundational algorithmic approaches that can contribute to scientific understanding or acquire it autonomously, making them critical areas of focus for AI innovation. The advances in artificial intelligence over the past decade are examined, with a discussion on how artificial intelligence systems can aid the scientific process and the central issues that remain despite advances."
++ },
++ "DOI:10.1038/s41586-023-06792-0": {
++  "paperId": "6fe3779fe5f2e9402abdd08ad8db41a0f13a99eb",
++  "externalIds": {
++   "DBLP": "journals/nature/BoikoMKG23",
++   "PubMedCentral": "10733136",
++   "DOI": "10.1038/s41586-023-06792-0",
++   "CorpusId": 266432059,
++   "PubMed": "38123806"
++  },
++  "publicationVenue": {
++   "id": "6c24a0a0-b07d-4d7b-a19b-fd09a3ed453a",
++   "name": "Nature",
++   "type": "journal",
++   "issn": "0028-0836",
++   "url": "https://www.nature.com/",
++   "alternate_urls": [
++    "http://www.nature.com/nature/",
++    "https://www.nature.com/nature/",
++    "http://www.nature.com/nature/archive/index.html"
++   ]
++  },
++  "title": "Autonomous chemical research with large language models",
++  "venue": "Nature",
++  "year": 2023,
++  "citationCount": 1510,
++  "openAccessPdf": {
++   "url": "https://www.nature.com/articles/s41586-023-06792-0.pdf",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC10733136, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Nature",
++   "pages": "570 - 578",
++   "volume": "624"
++  },
++  "authors": [
++   {
++    "authorId": "50771006",
++    "name": "Daniil A. Boiko"
++   },
++   {
++    "authorId": "152587056",
++    "name": "R. MacKnight"
++   },
++   {
++    "authorId": "2399498202",
++    "name": "Benjamin C Kline"
++   },
++   {
++    "authorId": "2181059489",
++    "name": "Gabe Gomes"
++   }
++  ],
++  "abstract": "Transformer-based large language models are making significant strides in various fields, such as natural language processing^ 1 \u2013 5 , biology^ 6 , 7 , chemistry^ 8 \u2013 10 and computer programming^ 11 , 12 . Here, we show the development and capabilities of Coscientist, an artificial intelligence system driven by GPT-4 that autonomously designs, plans and performs complex experiments by incorporating large language models empowered by tools such as internet and documentation search, code execution and experimental automation. Coscientist showcases its potential for accelerating research across six diverse tasks, including the successful reaction optimization of palladium-catalysed cross-couplings, while exhibiting advanced capabilities for (semi-)autonomous experimental design and execution. Our findings demonstrate the versatility, efficacy and explainability of artificial intelligence systems like Coscientist in advancing research. Coscientist is an artificial intelligence system driven by GPT-4 that autonomously designs, plans and performs experiments by incorporating large language models empowered by tools such as internet and documentation search, code execution and experimental automation."
++ },
++ "DOI:10.1038/s41586-025-09442-9": {
++  "paperId": "d24e37aafcf48c76aca30430670bad9a61cd0fca",
++  "externalIds": {
++   "DBLP": "journals/nature/SwansonWBP025",
++   "DOI": "10.1038/s41586-025-09442-9",
++   "CorpusId": 280389333,
++   "PubMed": "40730228"
++  },
++  "publicationVenue": {
++   "id": "6c24a0a0-b07d-4d7b-a19b-fd09a3ed453a",
++   "name": "Nature",
++   "type": "journal",
++   "issn": "0028-0836",
++   "url": "https://www.nature.com/",
++   "alternate_urls": [
++    "http://www.nature.com/nature/",
++    "https://www.nature.com/nature/",
++    "http://www.nature.com/nature/archive/index.html"
++   ]
++  },
++  "title": "The Virtual Lab of AI agents designs new SARS-CoV-2 nanobodies",
++  "venue": "Nature",
++  "year": 2025,
++  "citationCount": 349,
++  "openAccessPdf": {
++   "url": "",
++   "status": null,
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1038/s41586-025-09442-9?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1038/s41586-025-09442-9, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Nature",
++   "pages": "716 - 723",
++   "volume": "646"
++  },
++  "authors": [
++   {
++    "authorId": "2276778113",
++    "name": "Kyle Swanson"
++   },
++   {
++    "authorId": "2252322743",
++    "name": "Wesley Wu"
++   },
++   {
++    "authorId": "2330818080",
++    "name": "Nash L. Bulaong"
++   },
++   {
++    "authorId": "2248292433",
++    "name": "John E. Pak"
++   },
++   {
++    "authorId": "2204356217",
++    "name": "James Y. Zou"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1038/s41586-026-10644-y": {
++  "paperId": "390c607723e9767a9a873ded3bc1f66808215e8e",
++  "externalIds": {
++   "ArXiv": "2502.18864",
++   "PubMedCentral": "13345910",
++   "DOI": "10.1038/s41586-026-10644-y",
++   "CorpusId": 276617649,
++   "PubMed": "42428019"
++  },
++  "publicationVenue": {
++   "id": "6c24a0a0-b07d-4d7b-a19b-fd09a3ed453a",
++   "name": "Nature",
++   "type": "journal",
++   "issn": "0028-0836",
++   "url": "https://www.nature.com/",
++   "alternate_urls": [
++    "http://www.nature.com/nature/",
++    "https://www.nature.com/nature/",
++    "http://www.nature.com/nature/archive/index.html"
++   ]
++  },
++  "title": "Accelerating scientific discovery with Co-Scientist",
++  "venue": "Nature",
++  "year": 2025,
++  "citationCount": 593,
++  "openAccessPdf": {
++   "url": "https://www.nature.com/articles/s41586-026-10644-y_reference.pdf",
++   "status": "HYBRID",
++   "license": "other-oa",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2502.18864, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Nature",
++   "pages": "487 - 496",
++   "volume": "655"
++  },
++  "authors": [
++   {
++    "authorId": "2051681709",
++    "name": "Juraj Gottweis"
++   },
++   {
++    "authorId": "2239098855",
++    "name": "Wei-Hung Weng"
++   },
++   {
++    "authorId": "2347345594",
++    "name": "Alexander Daryin"
++   },
++   {
++    "authorId": "2244651851",
++    "name": "Tao Tu"
++   },
++   {
++    "authorId": "2679779",
++    "name": "Petar Sirkovi\u0107"
++   },
++   {
++    "authorId": "2346506169",
++    "name": "Artiom Myaskovsky"
++   },
++   {
++    "authorId": "2520452099",
++    "name": "Grzegorz Glowaty"
++   },
++   {
++    "authorId": "10805574",
++    "name": "Felix Weissenberger"
++   },
++   {
++    "authorId": "2367823022",
++    "name": "Alessio Orlandi"
++   },
++   {
++    "authorId": "2062591154",
++    "name": "D. Popovici"
++   },
++   {
++    "authorId": "2266472373",
++    "name": "Anil Palepu"
++   },
++   {
++    "authorId": "1996199677",
++    "name": "Ke-Ran Rong"
++   },
++   {
++    "authorId": "3480712",
++    "name": "Ryutaro Tanno"
++   },
++   {
++    "authorId": "2279021635",
++    "name": "Khaled Saab"
++   },
++   {
++    "authorId": "2298916585",
++    "name": "Fan Zhang"
++   },
++   {
++    "authorId": "2217621049",
++    "name": "Jacob Blum"
++   },
++   {
++    "authorId": "2336095347",
++    "name": "A. Carroll"
++   },
++   {
++    "authorId": "2269148145",
++    "name": "Kavita Kulkarni"
++   },
++   {
++    "authorId": "2213266",
++    "name": "Nenad Toma\u0161ev"
++   },
++   {
++    "authorId": "115731137",
++    "name": "D. Zverinski"
++   },
++   {
++    "authorId": "2389134",
++    "name": "Ivor Rendulic"
++   },
++   {
++    "authorId": "2298901106",
++    "name": "E. Vedadi"
++   },
++   {
++    "authorId": "113494403",
++    "name": "F. Hasler"
++   },
++   {
++    "authorId": "102638867",
++    "name": "Luka Rimanic"
++   },
++   {
++    "authorId": "36598773",
++    "name": "Marina Boia"
++   },
++   {
++    "authorId": "2784346",
++    "name": "Ivan Budiseli\u0107"
++   },
++   {
++    "authorId": "2074805000",
++    "name": "Ben Feinstein"
++   },
++   {
++    "authorId": "2359151678",
++    "name": "M. Bellaiche"
++   },
++   {
++    "authorId": "2220891317",
++    "name": "Tom Sheffer"
++   },
++   {
++    "authorId": "2279022140",
++    "name": "Jan Freyberg"
++   },
++   {
++    "authorId": "2255314955",
++    "name": "Jeremy D. Ratcliff"
++   },
++   {
++    "authorId": "1789306",
++    "name": "Ottavia Bertolli"
++   },
++   {
++    "authorId": "2269148566",
++    "name": "Katherine Chou"
++   },
++   {
++    "authorId": "1809983",
++    "name": "A. Hassidim"
++   },
++   {
++    "authorId": "2225921342",
++    "name": "Burak Gokturk"
++   },
++   {
++    "authorId": "2257189937",
++    "name": "Amin Vahdat"
++   },
++   {
++    "authorId": "2261405027",
++    "name": "Yuan Guan"
++   },
++   {
++    "authorId": "12268808",
++    "name": "Vikram Dhillon"
++   },
++   {
++    "authorId": "10664723",
++    "name": "E. D. Vaishnav"
++   },
++   {
++    "authorId": "2305695545",
++    "name": "Byron Lee"
++   },
++   {
++    "authorId": "2346514434",
++    "name": "Tiago R D Costa"
++   },
++   {
++    "authorId": "2068738124",
++    "name": "J. Penad\u00e9s"
++   },
++   {
++    "authorId": "2243119549",
++    "name": "Gary Peltz"
++   },
++   {
++    "authorId": "2269148232",
++    "name": "Y. Matias"
++   },
++   {
++    "authorId": "2254252286",
++    "name": "J. Manyika"
++   },
++   {
++    "authorId": "48987704",
++    "name": "D. Hassabis"
++   },
++   {
++    "authorId": "2275191526",
++    "name": "Yunhan Xu"
++   },
++   {
++    "authorId": "143967473",
++    "name": "Pushmeet Kohli"
++   },
++   {
++    "authorId": "2359249680",
++    "name": "Annalisa Pawlosky"
++   },
++   {
++    "authorId": "2213433382",
++    "name": "A. Karthikesalingam"
++   },
++   {
++    "authorId": "144223091",
++    "name": "Vivek Natarajan"
++   }
++  ],
++  "abstract": "Scientific discovery is driven by scientists generating hypotheses for complex problems that undergo rigorous experimental validation. To augment this process, we introduce Co-Scientist, a multi-agent artificial intelligence (AI) system built on Gemini for structured scientific thinking and hypothesis generation. Co-Scientist aims to help scientists discover new original knowledge. Conditioned on their research objectives and previous scientific evidence, it formulates demonstrably novel research hypotheses for experimental verification. The system\u2019s design involves agents continuously generating, critiquing and refining hypotheses accelerated by scaling test-time compute. Key contributions include (1) a multi-agent architecture with an asynchronous task execution framework for flexible compute scaling, and (2) a tournament evolution process for self-improving hypotheses generation. Automated evaluations show continued benefits of test-time compute scaling, improving hypothesis quality over time. Although this is a general-purpose system, we focus the validation in three biomedical applications: drug repurposing; novel-target discovery1; and explaining mechanisms of antimicrobial resistance2. Specifically, Co-Scientist helped to identify new drug-repurposing candidates and synergistic combination therapies for acute myeloid leukaemia that were validated through in vitro experiments. These real-world validations demonstrate the potential of Co-Scientist to accelerate scientific discovery and usher in an era of AI-empowered scientists. Co-Scientist is an AI system for structured scientific thinking and generates research hypotheses for experimental verification."
++ },
++ "DOI:10.1038/s41597-022-01143-6": {
++  "paperId": "172361c8b70a3ce63f7d6177473f2ccd5b5216b2",
++  "externalIds": {
++   "ArXiv": "2103.12793",
++   "DBLP": "journals/corr/abs-2103-12793",
++   "PubMedCentral": "8861064",
++   "DOI": "10.1038/s41597-022-01143-6",
++   "CorpusId": 232335674,
++   "PubMed": "35190569"
++  },
++  "publicationVenue": {
++   "id": "62924b2a-8fb8-4b93-92c6-735516b49af0",
++   "name": "Scientific Data",
++   "type": "journal",
++   "alternate_names": [
++    "Sci Data"
++   ],
++   "issn": "2052-4463",
++   "url": "http://www.nature.com/sdata/"
++  },
++  "title": "A large-scale study on research code quality and execution",
++  "venue": "Scientific Data",
++  "year": 2021,
++  "citationCount": 153,
++  "openAccessPdf": {
++   "url": "https://www.nature.com/articles/s41597-022-01143-6.pdf",
++   "status": "GOLD",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2103.12793, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Scientific Data",
++   "volume": "9"
++  },
++  "authors": [
++   {
++    "authorId": "152233355",
++    "name": "A. Tri\u0161ovi\u0107"
++   },
++   {
++    "authorId": "2621285",
++    "name": "Matthew K. Lau"
++   },
++   {
++    "authorId": "3176695",
++    "name": "Thomas Pasquier"
++   },
++   {
++    "authorId": "3000963",
++    "name": "M. Crosas"
++   }
++  ],
++  "abstract": "This article presents a study on the quality and execution of research code from publicly-available replication datasets at the Harvard Dataverse repository. Research code is typically created by a group of scientists and published together with academic papers to facilitate research transparency and reproducibility. For this study, we define ten questions to address aspects impacting research reproducibility and reuse. First, we retrieve and analyze more than 2000 replication datasets with over 9000 unique R files published from 2010 to 2020. Second, we execute the code in a clean runtime environment to assess its ease of reuse. Common coding errors were identified, and some of them were solved with automatic code cleaning to aid code execution. We find that 74% of R files failed to complete without error in the initial execution, while 56% failed when code cleaning was applied, showing that many errors can be prevented with good coding practices. We also analyze the replication datasets from journals\u2019 collections and discuss the impact of the journal policy strictness on the code re-execution rate. Finally, based on our results, we propose a set of recommendations for code dissemination aimed at researchers, journals, and repositories."
++ },
++ "DOI:10.1038/s41598-023-41032-5": {
++  "paperId": "51dd1ebcebafce801b8856c925dd205f738d8f74",
++  "externalIds": {
++   "PubMedCentral": "10484980",
++   "DOI": "10.1038/s41598-023-41032-5",
++   "CorpusId": 261608321,
++   "PubMed": "37679503"
++  },
++  "publicationVenue": {
++   "id": "f99f77b7-b1b6-44d3-984a-f288e9884b9b",
++   "name": "Scientific Reports",
++   "type": "journal",
++   "alternate_names": [
++    "Sci Rep"
++   ],
++   "issn": "2045-2322",
++   "url": "http://www.nature.com/srep/",
++   "alternate_urls": [
++    "http://www.nature.com/srep/index.html"
++   ]
++  },
++  "title": "Fabrication and errors in the bibliographic citations generated by ChatGPT",
++  "venue": "Scientific Reports",
++  "year": 2023,
++  "citationCount": 556,
++  "openAccessPdf": {
++   "url": "https://www.nature.com/articles/s41598-023-41032-5.pdf",
++   "status": "GOLD",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC10484980, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Scientific Reports",
++   "volume": "13"
++  },
++  "authors": [
++   {
++    "authorId": "2238416484",
++    "name": "William H. Walters"
++   },
++   {
++    "authorId": "2917043",
++    "name": "E. I. Wilder"
++   }
++  ],
++  "abstract": "Although chatbots such as ChatGPT can facilitate cost-effective text generation and editing, factually incorrect responses (hallucinations) limit their utility. This study evaluates one particular type of hallucination: fabricated bibliographic citations that do not represent actual scholarly works. We used ChatGPT-3.5 and ChatGPT-4 to produce short literature reviews on 42 multidisciplinary topics, compiling data on the 636 bibliographic citations (references) found in the 84 papers. We then searched multiple databases and websites to determine the prevalence of fabricated citations, to identify errors in the citations to non-fabricated papers, and to evaluate adherence to APA citation format. Within this set of documents, 55% of the GPT-3.5 citations but just 18% of the GPT-4 citations are fabricated. Likewise, 43% of the real (non-fabricated) GPT-3.5 citations but just 24% of the real GPT-4 citations include substantive citation errors. Although GPT-4 is a major improvement over GPT-3.5, problems remain."
++ },
++ "DOI:10.1038/s42256-024-00832-8": {
++  "paperId": "354dcdebf3f8b5feeed5c62090e0bc1f0c28db06",
++  "externalIds": {
++   "DBLP": "journals/natmi/BranCSBWS24",
++   "ArXiv": "2304.05376",
++   "PubMedCentral": "11116106",
++   "DOI": "10.1038/s42256-024-00832-8",
++   "CorpusId": 258059792,
++   "PubMed": "38799228"
++  },
++  "publicationVenue": {
++   "id": "6457124b-39bf-4d02-bff4-73752ff21562",
++   "name": "Nature Machine Intelligence",
++   "type": "journal",
++   "alternate_names": [
++    "Nat Mach Intell"
++   ],
++   "issn": "2522-5839",
++   "url": "https://www.nature.com/natmachintell/"
++  },
++  "title": "Augmenting large language models with chemistry tools",
++  "venue": "Nature Machine Intelligence",
++  "year": 2023,
++  "citationCount": 1414,
++  "openAccessPdf": {
++   "url": "https://www.nature.com/articles/s42256-024-00832-8.pdf",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2304.05376, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Nature Machine Intelligence",
++   "pages": "525 - 535",
++   "volume": "6"
++  },
++  "authors": [
++   {
++    "authorId": "2216007369",
++    "name": "Andr\u00e9s M Bran"
++   },
++   {
++    "authorId": "2161337138",
++    "name": "Sam Cox"
++   },
++   {
++    "authorId": "1820929773",
++    "name": "Oliver Schilter"
++   },
++   {
++    "authorId": "2291607243",
++    "name": "Carlo Baldassari"
++   },
++   {
++    "authorId": "2150199535",
++    "name": "Andrew D. White"
++   },
++   {
++    "authorId": "1379965853",
++    "name": "P. Schwaller"
++   }
++  ],
++  "abstract": "Large language models (LLMs) have shown strong performance in tasks across domains but struggle with chemistry-related problems. These models also lack access to external knowledge sources, limiting their usefulness in scientific applications. We introduce ChemCrow, an LLM chemistry agent designed to accomplish tasks across organic synthesis, drug discovery and materials design. By integrating 18 expert-designed tools and using GPT-4 as the LLM, ChemCrow augments the LLM performance in chemistry, and new capabilities emerge. Our agent autonomously planned and executed the syntheses of an insect repellent and three organocatalysts and guided the discovery of a novel chromophore. Our evaluation, including both LLM and expert assessments, demonstrates ChemCrow\u2019s effectiveness in automating a diverse set of chemical tasks. Our work not only aids expert chemists and lowers barriers for non-experts but also fosters scientific advancement by bridging the gap between experimental and computational chemistry. Large language models can be queried to perform chain-of-thought reasoning on text descriptions of data or computational tools, which can enable flexible and autonomous workflows. Bran et al. developed ChemCrow, a GPT-4-based agent that has access to computational chemistry tools and a robotic chemistry platform, which can autonomously solve tasks for designing or synthesizing chemicals such as drugs or materials."
++ },
++ "DOI:10.1038/sdata.2016.18": {
++  "paperId": "e936f248b2c0489316ed1521656af2564c3502c3",
++  "externalIds": {
++   "MAG": "3083522027",
++   "PubMedCentral": "4792175",
++   "DOI": "10.1038/sdata.2016.18",
++   "CorpusId": 8755162,
++   "PubMed": "26978244"
++  },
++  "publicationVenue": {
++   "id": "62924b2a-8fb8-4b93-92c6-735516b49af0",
++   "name": "Scientific Data",
++   "type": "journal",
++   "alternate_names": [
++    "Sci Data"
++   ],
++   "issn": "2052-4463",
++   "url": "http://www.nature.com/sdata/"
++  },
++  "title": "The FAIR Guiding Principles for scientific data management and stewardship",
++  "venue": "Scientific Data",
++  "year": 2016,
++  "citationCount": 17558,
++  "openAccessPdf": {
++   "url": "https://www.nature.com/articles/sdata201618.pdf",
++   "status": "GOLD",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC4792175, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Scientific Data",
++   "volume": "3"
++  },
++  "authors": [
++   {
++    "authorId": "31764087",
++    "name": "M. Wilkinson"
++   },
++   {
++    "authorId": "1736389",
++    "name": "M. Dumontier"
++   },
++   {
++    "authorId": "1975615",
++    "name": "I. J. Aalbersberg"
++   },
++   {
++    "authorId": "74186014",
++    "name": "G. Appleton"
++   },
++   {
++    "authorId": "4006952",
++    "name": "M. Axton"
++   },
++   {
++    "authorId": "20313879",
++    "name": "A. Baak"
++   },
++   {
++    "authorId": "2776517",
++    "name": "N. Blomberg"
++   },
++   {
++    "authorId": "34945757",
++    "name": "J. Boiten"
++   },
++   {
++    "authorId": "115529154",
++    "name": "Luiz Olavo Bonino da Silva Santos"
++   },
++   {
++    "authorId": "2048644",
++    "name": "P. Bourne"
++   },
++   {
++    "authorId": "47611747",
++    "name": "J. Bouwman"
++   },
++   {
++    "authorId": "2160154",
++    "name": "A. Brookes"
++   },
++   {
++    "authorId": "145469718",
++    "name": "Tim Clark"
++   },
++   {
++    "authorId": "3000963",
++    "name": "M. Crosas"
++   },
++   {
++    "authorId": "145479127",
++    "name": "I. Dillo"
++   },
++   {
++    "authorId": "116235596",
++    "name": "O. Dumon"
++   },
++   {
++    "authorId": "1940242",
++    "name": "S. Edmunds"
++   },
++   {
++    "authorId": "2185817",
++    "name": "C. Evelo"
++   },
++   {
++    "authorId": "2425367",
++    "name": "R. Finkers"
++   },
++   {
++    "authorId": "1401553965",
++    "name": "A. Gonz\u00e1lez-Beltr\u00e1n"
++   },
++   {
++    "authorId": "34071005",
++    "name": "A. Gray"
++   },
++   {
++    "authorId": "143772714",
++    "name": "Paul Groth"
++   },
++   {
++    "authorId": "46555127",
++    "name": "C. Goble"
++   },
++   {
++    "authorId": "1736528",
++    "name": "J. Grethe"
++   },
++   {
++    "authorId": "1699372",
++    "name": "J. Heringa"
++   },
++   {
++    "authorId": "1396879481",
++    "name": "P. 't Hoen"
++   },
++   {
++    "authorId": "2359250",
++    "name": "R. Hooft"
++   },
++   {
++    "authorId": "145727607",
++    "name": "T. Kuhn"
++   },
++   {
++    "authorId": "47044989",
++    "name": "R. Kok"
++   },
++   {
++    "authorId": "1713683",
++    "name": "Joost N. Kok"
++   },
++   {
++    "authorId": "4297975",
++    "name": "S. Lusher"
++   },
++   {
++    "authorId": "2969738",
++    "name": "M. Martone"
++   },
++   {
++    "authorId": "6027579",
++    "name": "Albert Mons"
++   },
++   {
++    "authorId": "2455243",
++    "name": "A. Packer"
++   },
++   {
++    "authorId": "2059300547",
++    "name": "Bengt Persson"
++   },
++   {
++    "authorId": "1397958214",
++    "name": "P. Rocca-Serra"
++   },
++   {
++    "authorId": "144140870",
++    "name": "M. Roos"
++   },
++   {
++    "authorId": "2480475903",
++    "name": "R. V. van Schaik"
++   },
++   {
++    "authorId": "1760490",
++    "name": "S. Sansone"
++   },
++   {
++    "authorId": "1883372",
++    "name": "E. Schultes"
++   },
++   {
++    "authorId": "2510671",
++    "name": "T. Sengstag"
++   },
++   {
++    "authorId": "38261192",
++    "name": "Ted Slater"
++   },
++   {
++    "authorId": "2013713",
++    "name": "G. Strawn"
++   },
++   {
++    "authorId": "144034058",
++    "name": "M. Swertz"
++   },
++   {
++    "authorId": "143962914",
++    "name": "Mark Thompson"
++   },
++   {
++    "authorId": "84257307",
++    "name": "J. van der Lei"
++   },
++   {
++    "authorId": "11563451",
++    "name": "E. V. van Mulligen"
++   },
++   {
++    "authorId": "3261874",
++    "name": "J. Velterop"
++   },
++   {
++    "authorId": "48709451",
++    "name": "A. Waagmeester"
++   },
++   {
++    "authorId": "1681130",
++    "name": "P. Wittenburg"
++   },
++   {
++    "authorId": "143780913",
++    "name": "K. Wolstencroft"
++   },
++   {
++    "authorId": "48019160",
++    "name": "Jun Zhao"
++   },
++   {
++    "authorId": "1995836",
++    "name": "B. Mons"
++   }
++  ],
++  "abstract": "There is an urgent need to improve the infrastructure supporting the reuse of scholarly data. A diverse set of stakeholders\u2014representing academia, industry, funding agencies, and scholarly publishers\u2014have come together to design and jointly endorse a concise and measureable set of principles that we refer to as the FAIR Data Principles. The intent is that these may act as a guideline for those wishing to enhance the reusability of their data holdings. Distinct from peer initiatives that focus on the human scholar, the FAIR Principles put specific emphasis on enhancing the ability of machines to automatically find and use the data, in addition to supporting its reuse by individuals. This Comment is the first formal publication of the FAIR Principles, and includes the rationale behind them, and some exemplar implementations in the community."
++ },
++ "DOI:10.1056/AIoa2400555": null,
++ "DOI:10.1073/pnas.1708274114": {
++  "paperId": "5465884a4aad5dc256ce5b4bfacdfa7cdc2c78ee",
++  "externalIds": {
++   "MAG": "2779812635",
++   "DOI": "10.1073/pnas.1708274114",
++   "CorpusId": 4639380,
++   "PubMed": "29531091"
++  },
++  "publicationVenue": {
++   "id": "bb95bf2e-8383-4748-bf9d-d6906d091085",
++   "name": "Proceedings of the National Academy of Sciences of the United States of America",
++   "type": "journal",
++   "alternate_names": [
++    "PNAS",
++    "PNAS online",
++    "Proceedings of the National Academy of Sciences of the United States of America.",
++    "Proc National Acad Sci",
++    "Proceedings of the National Academy of Sciences",
++    "Proc National Acad Sci u s Am"
++   ],
++   "issn": "0027-8424",
++   "alternate_issns": [
++    "1091-6490"
++   ],
++   "url": "https://www.jstor.org/journal/procnatiacadscie",
++   "alternate_urls": [
++    "http://www.pnas.org/",
++    "https://www.pnas.org/",
++    "http://www.jstor.org/journals/00278424.html",
++    "www.pnas.org/"
++   ]
++  },
++  "title": "The preregistration revolution",
++  "venue": "Proceedings of the National Academy of Sciences of the United States of America",
++  "year": 2018,
++  "citationCount": 1775,
++  "openAccessPdf": {
++   "url": "https://doi.org/10.31219/osf.io/2dxu5",
++   "status": "GREEN",
++   "license": "public-domain",
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1073/pnas.1708274114?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1073/pnas.1708274114, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Proceedings of the National Academy of Sciences",
++   "pages": "2600 - 2606",
++   "volume": "115"
++  },
++  "authors": [
++   {
++    "authorId": "2862527",
++    "name": "Brian A. Nosek"
++   },
++   {
++    "authorId": "7004092",
++    "name": "C. Ebersole"
++   },
++   {
++    "authorId": "15910942",
++    "name": "A. DeHaven"
++   },
++   {
++    "authorId": "35395330",
++    "name": "D. Mellor"
++   }
++  ],
++  "abstract": "Progress in science relies in part on generating hypotheses with existing observations and testing hypotheses with new observations. This distinction between postdiction and prediction is appreciated conceptually but is not respected in practice. Mistaking generation of postdictions with testing of predictions reduces the credibility of research findings. However, ordinary biases in human reasoning, such as hindsight bias, make it hard to avoid this mistake. An effective solution is to define the research questions and analysis plan before observing the research outcomes\u2014a process called preregistration. Preregistration distinguishes analyses and outcomes that result from predictions from those that result from postdictions. A variety of practical strategies are available to make the best possible use of preregistration in circumstances that fall short of the ideal application, such as when the data are preexisting. Services are now available for preregistration across all disciplines, facilitating a rapid increase in the practice. Widespread adoption of preregistration will increase distinctiveness between hypothesis generation and hypothesis testing and will improve the credibility of research findings."
++ },
++ "DOI:10.1073/pnas.1708290115": {
++  "paperId": "a7c2d6b4710ea409b14a716616a9caf466adb2f6",
++  "externalIds": {
++   "MAG": "2793993424",
++   "DBLP": "journals/pnas/StoddenSM18",
++   "DOI": "10.1073/pnas.1708290115",
++   "CorpusId": 3850646,
++   "PubMed": "29531050"
++  },
++  "publicationVenue": {
++   "id": "bb95bf2e-8383-4748-bf9d-d6906d091085",
++   "name": "Proceedings of the National Academy of Sciences of the United States of America",
++   "type": "journal",
++   "alternate_names": [
++    "PNAS",
++    "PNAS online",
++    "Proceedings of the National Academy of Sciences of the United States of America.",
++    "Proc National Acad Sci",
++    "Proceedings of the National Academy of Sciences",
++    "Proc National Acad Sci u s Am"
++   ],
++   "issn": "0027-8424",
++   "alternate_issns": [
++    "1091-6490"
++   ],
++   "url": "https://www.jstor.org/journal/procnatiacadscie",
++   "alternate_urls": [
++    "http://www.pnas.org/",
++    "https://www.pnas.org/",
++    "http://www.jstor.org/journals/00278424.html",
++    "www.pnas.org/"
++   ]
++  },
++  "title": "An empirical analysis of journal policy effectiveness for computational reproducibility",
++  "venue": "Proceedings of the National Academy of Sciences of the United States of America",
++  "year": 2018,
++  "citationCount": 342,
++  "openAccessPdf": {
++   "url": "https://www.pnas.org/content/pnas/115/11/2584.full.pdf",
++   "status": "BRONZE",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1073/pnas.1708290115?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1073/pnas.1708290115, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Proceedings of the National Academy of Sciences",
++   "pages": "2584 - 2589",
++   "volume": "115"
++  },
++  "authors": [
++   {
++    "authorId": "2598812",
++    "name": "V. Stodden"
++   },
++   {
++    "authorId": "2058677280",
++    "name": "Jennifer Seiler"
++   },
++   {
++    "authorId": "2118105221",
++    "name": "Zhaokun Ma"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1088/1742-6596/46/1/067": {
++  "paperId": "3f2927d1ea7ff47361b24203ebf62658ffa42cbb",
++  "externalIds": {
++   "MAG": "2116115793",
++   "DOI": "10.1088/1742-6596/46/1/067",
++   "CorpusId": 62762011
++  },
++  "publicationVenue": null,
++  "title": "Berkeley Lab Checkpoint/Restart (BLCR) for Linux Clusters",
++  "venue": "",
++  "year": 2006,
++  "citationCount": 511,
++  "openAccessPdf": {
++   "url": "https://iopscience.iop.org/article/10.1088/1742-6596/46/1/067/pdf",
++   "status": "GOLD",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1088/1742-6596/46/1/067?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1088/1742-6596/46/1/067, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": null,
++  "journal": {
++   "name": "",
++   "pages": "494-499",
++   "volume": "46"
++  },
++  "authors": [
++   {
++    "authorId": "145980631",
++    "name": "Paul H. Hargrove"
++   },
++   {
++    "authorId": "4871994",
++    "name": "J. Duell"
++   }
++  ],
++  "abstract": "This article describes the motivation, design and implementation of Berkeley Lab Checkpoint/Restart (BLCR), a system-level checkpoint/restart implementation for Linux clusters that targets the space of typical High Performance Computing applications, including MPI. Application-level solutions, including both checkpointing and fault-tolerant algorithms, are recognized as more time and space efficient than system-level checkpoints, which cannot make use of any application-specific knowledge. However, system-level checkpointing allows for preemption, making it suitable for responding to ''fault precursors'' (for instance, elevated error rates from ECC memory or network CRCs, or elevated temperature from sensors). Preemption can also increase the efficiency of batch scheduling; for instance reducing idle cycles (by allowing for shutdown without any queue draining period or reallocation of resources to eliminate idle nodes when better fitting jobs are queued), and reducing the average queued time (by limiting large jobs to running during off-peak hours, without the need to limit the length of such jobs). Each of these potential uses makes BLCR a valuable tool for efficient resource management in Linux clusters."
++ },
++ "DOI:10.1093/bioinformatics/bts480": null,
++ "DOI:10.1098/rsta.2020.0069": {
++  "paperId": "ba7655c2176c35e278eeaae0aab041ab1cb74d0a",
++  "externalIds": {
++   "PubMedCentral": "8059663",
++   "DOI": "10.1098/rsta.2020.0069",
++   "CorpusId": 232387097,
++   "PubMed": "33775145"
++  },
++  "publicationVenue": null,
++  "title": "Learning from reproducing computational results: introducing three principles and the Reproduction Package",
++  "venue": "Philosophical Transactions of the Royal Society A",
++  "year": 2021,
++  "citationCount": 41,
++  "openAccessPdf": {
++   "url": "https://doi.org/10.1098/rsta.2020.0069",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC8059663, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Philosophical transactions. Series A, Mathematical, physical, and engineering sciences",
++   "volume": "379"
++  },
++  "authors": [
++   {
++    "authorId": "36999319",
++    "name": "Matthew Scott Krafczyk"
++   },
++   {
++    "authorId": "40128780",
++    "name": "A. Shi"
++   },
++   {
++    "authorId": "9383151",
++    "name": "A. Bhaskar"
++   },
++   {
++    "authorId": "1718171",
++    "name": "D. Marinov"
++   },
++   {
++    "authorId": "2598812",
++    "name": "V. Stodden"
++   }
++  ],
++  "abstract": "We carry out efforts to reproduce computational results for seven published articles and identify barriers to computational reproducibility. We then derive three principles to guide the practice and dissemination of reproducible computational research: (i) Provide transparency regarding how computational results are produced; (ii) When writing and releasing research software, aim for ease of (re-)executability; (iii) Make any code upon which the results rely as deterministic as possible. We then exemplify these three principles with 12 specific guidelines for their implementation in practice. We illustrate the three principles of reproducible research with a series of vignettes from our experimental reproducibility work. We define a novel Reproduction Package, a formalism that specifies a structured way to share computational research artifacts that implements the guidelines generated from our reproduction efforts to allow others to build, reproduce and extend computational science. We make our reproduction efforts in this paper publicly available as exemplar Reproduction Packages. This article is part of the theme issue \u2018Reliability and reproducibility in computational science: implementing verification, validation and uncertainty quantification in silico\u2019."
++ },
++ "DOI:10.1109/2.84898": {
++  "paperId": "6953691c2f3c51a2d3cf0905a264673aaa1aca7a",
++  "externalIds": {
++   "DBLP": "journals/computer/GrayS91",
++   "MAG": "2055631879",
++   "DOI": "10.1109/2.84898",
++   "CorpusId": 7550761
++  },
++  "publicationVenue": {
++   "id": "f6572f66-2623-4a5e-b0d9-4a5028dea98f",
++   "name": "Computer",
++   "type": "journal",
++   "alternate_names": [
++    "IEEE Computer",
++    "IEEE Comput"
++   ],
++   "issn": "0018-9162",
++   "url": "http://www.computer.org/computer",
++   "alternate_urls": [
++    "https://ieeexplore.ieee.org/servlet/opac?punumber=2",
++    "http://www.computer.org/portal/site/ieeecs/index.jsp",
++    "https://ieeexplore.ieee.org/xpl/RecentIssue.jsp?punumber=2"
++   ]
++  },
++  "title": "High-availability computer systems",
++  "venue": "Computer",
++  "year": 1991,
++  "citationCount": 379,
++  "openAccessPdf": {
++   "url": "http://www.cs.wisc.edu/areas/os/Qual/papers/ha-systems.pdf",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1109/2.84898?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1109/2.84898, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Computer",
++   "pages": "39-48",
++   "volume": "24"
++  },
++  "authors": [
++   {
++    "authorId": "39941882",
++    "name": "Jim Gray"
++   },
++   {
++    "authorId": "1742634",
++    "name": "D. Siewiorek"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1109/3468.844354": {
++  "paperId": "14ae6f2231e09e226b99002aa04b5c70f3c59f2b",
++  "externalIds": {
++   "MAG": "2123179704",
++   "DBLP": "journals/tsmc/ParasuramanSW00",
++   "DOI": "10.1109/3468.844354",
++   "CorpusId": 8664574,
++   "PubMed": "11760769"
++  },
++  "publicationVenue": null,
++  "title": "A model for types and levels of human interaction with automation",
++  "venue": "IEEE Trans. Syst. Man Cybern. Part A",
++  "year": 2000,
++  "citationCount": 4495,
++  "openAccessPdf": {
++   "url": "http://www.cs.uml.edu/~holly/91.549/readings/sheridan-autonomy.pdf",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1109/3468.844354?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1109/3468.844354, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "IEEE transactions on systems, man, and cybernetics. Part A, Systems and humans : a publication of the IEEE Systems, Man, and Cybernetics Society",
++   "pages": "\n          286-97\n        ",
++   "volume": "30 3"
++  },
++  "authors": [
++   {
++    "authorId": "3264674",
++    "name": "R. Parasuraman"
++   },
++   {
++    "authorId": "1712194",
++    "name": "T. Sheridan"
++   },
++   {
++    "authorId": "1772836",
++    "name": "C. Wickens"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1109/MS.2016.60": {
++  "paperId": "e62070da1b348a7ea439a12b54e3f8b3223ee8a5",
++  "externalIds": {
++   "MAG": "2358431938",
++   "DBLP": "journals/corr/BasiriBRHKRR17",
++   "ArXiv": "1702.05843",
++   "DOI": "10.1109/MS.2016.60",
++   "CorpusId": 16294067
++  },
++  "publicationVenue": {
++   "id": "119227a6-1b94-433e-a52d-8445a387dbbe",
++   "name": "IEEE Software",
++   "type": "journal",
++   "alternate_names": [
++    "IEEE Softw"
++   ],
++   "issn": "0740-7459",
++   "url": "http://www.computer.org/software",
++   "alternate_urls": [
++    "http://ieeexplore.ieee.org/servlet/opac?punumber=52"
++   ]
++  },
++  "title": "Chaos Engineering",
++  "venue": "IEEE Software",
++  "year": 2017,
++  "citationCount": 327,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/1702.05843",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/1702.05843, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/1702.05843"
++  },
++  "authors": [
++   {
++    "authorId": "2040887090",
++    "name": "Ali Basiri"
++   },
++   {
++    "authorId": "3395694",
++    "name": "Niosha Behnam"
++   },
++   {
++    "authorId": "2481279147",
++    "name": "R. D. de Rooij"
++   },
++   {
++    "authorId": "2030762",
++    "name": "L. Hochstein"
++   },
++   {
++    "authorId": "3395609",
++    "name": "Luke Kosewski"
++   },
++   {
++    "authorId": "2068158419",
++    "name": "J. Reynolds"
++   },
++   {
++    "authorId": "40810907",
++    "name": "Casey Rosenthal"
++   }
++  ],
++  "abstract": "Modern software-based services are implemented as distributed systems with complex behavior and failure modes. Many large tech organizations are using experimentation to verify such systems' reliability. Netflix engineers call this approach chaos engineering. They've determined several principles underlying it and have used it to run experiments. This article is part of a theme issue on DevOps."
++ },
++ "DOI:10.1126/science.1213847": {
++  "paperId": "e4625641d1072542315c14083befb43cd7096aba",
++  "externalIds": {
++   "PubMedCentral": "3383002",
++   "MAG": "2008985174",
++   "DOI": "10.1126/science.1213847",
++   "CorpusId": 19755904,
++   "PubMed": "22144613"
++  },
++  "publicationVenue": {
++   "id": "f59506a8-d8bb-4101-b3d4-c4ac3ed03dad",
++   "name": "Science",
++   "type": "journal",
++   "issn": "0193-4511",
++   "alternate_issns": [
++    "0036-8075"
++   ],
++   "url": "https://www.jstor.org/journal/science",
++   "alternate_urls": [
++    "https://www.sciencemag.org/",
++    "http://www.sciencemag.org/",
++    "http://www.jstor.org/journals/00368075.html",
++    "http://www.sciencemag.org/archive/"
++   ]
++  },
++  "title": "Reproducible Research in Computational Science",
++  "venue": "Science",
++  "year": 2011,
++  "citationCount": 1523,
++  "openAccessPdf": {
++   "url": "https://www.ncbi.nlm.nih.gov/pmc/articles/3383002",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC3383002, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Science (New York, N.y.)",
++   "pages": "1226 - 1227",
++   "volume": "334"
++  },
++  "authors": [
++   {
++    "authorId": "34814856",
++    "name": "Roger D. Peng"
++   }
++  ],
++  "abstract": "Computational science has led to exciting new developments, but the nature of the work has exposed limitations in our ability to evaluate published findings. Reproducibility has the potential to serve as a minimum standard for judging scientific claims when full independent replication of a study is not possible."
++ },
++ "DOI:10.1126/science.aab2374": {
++  "paperId": "927a32a1b1c69a6041a586b71f52be1b957e5030",
++  "externalIds": {
++   "PubMedCentral": "4550299",
++   "MAG": "2403646253",
++   "DOI": "10.1126/science.aab2374",
++   "CorpusId": 206637955,
++   "PubMed": "26113702"
++  },
++  "publicationVenue": {
++   "id": "f59506a8-d8bb-4101-b3d4-c4ac3ed03dad",
++   "name": "Science",
++   "type": "journal",
++   "issn": "0193-4511",
++   "alternate_issns": [
++    "0036-8075"
++   ],
++   "url": "https://www.jstor.org/journal/science",
++   "alternate_urls": [
++    "https://www.sciencemag.org/",
++    "http://www.sciencemag.org/",
++    "http://www.jstor.org/journals/00368075.html",
++    "http://www.sciencemag.org/archive/"
++   ]
++  },
++  "title": "Promoting an open research culture",
++  "venue": "Science",
++  "year": 2015,
++  "citationCount": 2275,
++  "openAccessPdf": {
++   "url": "https://www.ncbi.nlm.nih.gov/pmc/articles/4550299",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC4550299, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Science (New York, N.Y.)",
++   "pages": "1422 - 1425",
++   "volume": "348"
++  },
++  "authors": [
++   {
++    "authorId": "2862527",
++    "name": "Brian A. Nosek"
++   },
++   {
++    "authorId": "2139865103",
++    "name": "G. Alter"
++   },
++   {
++    "authorId": "8611013",
++    "name": "G. Banks"
++   },
++   {
++    "authorId": "3237553",
++    "name": "D. Borsboom"
++   },
++   {
++    "authorId": "72356049",
++    "name": "S. Bowman"
++   },
++   {
++    "authorId": "6240718",
++    "name": "S. J. Breckler"
++   },
++   {
++    "authorId": "2073872541",
++    "name": "S. Buck"
++   },
++   {
++    "authorId": "46975669",
++    "name": "C. Chambers"
++   },
++   {
++    "authorId": "2058346280",
++    "name": "G. Chin"
++   },
++   {
++    "authorId": "2054652938",
++    "name": "G. Christensen"
++   },
++   {
++    "authorId": "50230722",
++    "name": "M. Contestabile"
++   },
++   {
++    "authorId": "2412226244",
++    "name": "A. Dafoe"
++   },
++   {
++    "authorId": "34840029",
++    "name": "E. Eich"
++   },
++   {
++    "authorId": "2364509",
++    "name": "J. Freese"
++   },
++   {
++    "authorId": "4249203",
++    "name": "R. Glennerster"
++   },
++   {
++    "authorId": "13065618",
++    "name": "D. Goroff"
++   },
++   {
++    "authorId": "48997800",
++    "name": "D. P. Green"
++   },
++   {
++    "authorId": "119365490",
++    "name": "B. Hesse"
++   },
++   {
++    "authorId": "32272151",
++    "name": "M. Humphreys"
++   },
++   {
++    "authorId": "46636130",
++    "name": "J. Ishiyama"
++   },
++   {
++    "authorId": "2124080032",
++    "name": "D. Karlan"
++   },
++   {
++    "authorId": "47733218",
++    "name": "A. Kraut"
++   },
++   {
++    "authorId": "47975256",
++    "name": "A. Lupia"
++   },
++   {
++    "authorId": "3154113",
++    "name": "P. Mabry"
++   },
++   {
++    "authorId": "4133783",
++    "name": "T. Madon"
++   },
++   {
++    "authorId": "118685494",
++    "name": "N. Malhotra"
++   },
++   {
++    "authorId": "1390039913",
++    "name": "E. Mayo-Wilson"
++   },
++   {
++    "authorId": "114856072",
++    "name": "M. McNutt"
++   },
++   {
++    "authorId": "144624624",
++    "name": "E. Miguel"
++   },
++   {
++    "authorId": "4659343",
++    "name": "E. Paluck"
++   },
++   {
++    "authorId": "2130902",
++    "name": "U. Simonsohn"
++   },
++   {
++    "authorId": "4110370",
++    "name": "C. Soderberg"
++   },
++   {
++    "authorId": "101916661",
++    "name": "B. Spellman"
++   },
++   {
++    "authorId": "1398590331",
++    "name": "J. Turitto"
++   },
++   {
++    "authorId": "80222152",
++    "name": "G. VandenBos"
++   },
++   {
++    "authorId": "2809726",
++    "name": "S. Vazire"
++   },
++   {
++    "authorId": "2505272",
++    "name": "E. Wagenmakers"
++   },
++   {
++    "authorId": "153160363",
++    "name": "R. Wilson"
++   },
++   {
++    "authorId": "2075675",
++    "name": "T. Yarkoni"
++   }
++  ],
++  "abstract": "Author guidelines for journals could help to promote transparency, openness, and reproducibility Transparency, openness, and reproducibility are readily recognized as vital features of science (1, 2). When asked, most scientists embrace these features as disciplinary norms and values (3). Therefore, one might expect that these valued features would be routine in daily practice. Yet, a growing body of evidence suggests that this is not the case (4\u20136)."
++ },
++ "DOI:10.1126/science.aac4716": {
++  "paperId": "155391192091a99f44b377db1e0e7819f2317498",
++  "externalIds": {
++   "DOI": "10.1126/science.aac4716",
++   "CorpusId": 218065162,
++   "PubMed": "26315443"
++  },
++  "publicationVenue": {
++   "id": "f59506a8-d8bb-4101-b3d4-c4ac3ed03dad",
++   "name": "Science",
++   "type": "journal",
++   "issn": "0193-4511",
++   "alternate_issns": [
++    "0036-8075"
++   ],
++   "url": "https://www.jstor.org/journal/science",
++   "alternate_urls": [
++    "https://www.sciencemag.org/",
++    "http://www.sciencemag.org/",
++    "http://www.jstor.org/journals/00368075.html",
++    "http://www.sciencemag.org/archive/"
++   ]
++  },
++  "title": "Estimating the reproducibility of psychological science",
++  "venue": "Science",
++  "year": 2015,
++  "citationCount": 7539,
++  "openAccessPdf": {
++   "url": "https://research.tilburguniversity.edu/en/publications/4b1e6c37-3ee2-414b-b790-66024d631d4a",
++   "status": "GREEN",
++   "license": "other-oa",
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1126/science.aac4716?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1126/science.aac4716, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Science",
++   "volume": "349"
++  },
++  "authors": [
++   {
++    "authorId": "2231439",
++    "name": "Alexander A. Aarts"
++   },
++   {
++    "authorId": "1400206933",
++    "name": "Joanna E. Anderson"
++   },
++   {
++    "authorId": "96413924",
++    "name": "Christopher J. Anderson"
++   },
++   {
++    "authorId": "113650771",
++    "name": "Peter Raymond Attridge"
++   },
++   {
++    "authorId": "1788796",
++    "name": "A. Attwood"
++   },
++   {
++    "authorId": "152159926",
++    "name": "J. Axt"
++   },
++   {
++    "authorId": "36417745",
++    "name": "Molly Babel"
++   },
++   {
++    "authorId": "5252377",
++    "name": "\u0160. Bahn\u00edk"
++   },
++   {
++    "authorId": "6174975",
++    "name": "E. Baranski"
++   },
++   {
++    "authorId": "1387458734",
++    "name": "M. Barnett-Cowan"
++   },
++   {
++    "authorId": "4867423",
++    "name": "Elizabeth Bartmess"
++   },
++   {
++    "authorId": "3039399",
++    "name": "Jennifer S. Beer"
++   },
++   {
++    "authorId": "47431109",
++    "name": "Raoul Bell"
++   },
++   {
++    "authorId": "2074992746",
++    "name": "Heather Bentley"
++   },
++   {
++    "authorId": "114670541",
++    "name": "Leah Beyan"
++   },
++   {
++    "authorId": "30661345",
++    "name": "Grace Binion"
++   },
++   {
++    "authorId": "3237553",
++    "name": "D. Borsboom"
++   },
++   {
++    "authorId": "116191440",
++    "name": "Annick Bosch"
++   },
++   {
++    "authorId": "8597004",
++    "name": "Frank Bosco"
++   },
++   {
++    "authorId": "72356050",
++    "name": "Sara D. Bowman"
++   },
++   {
++    "authorId": "145470466",
++    "name": "M. Brandt"
++   },
++   {
++    "authorId": "66335862",
++    "name": "E. L. Braswell"
++   },
++   {
++    "authorId": "51904698",
++    "name": "Hilmar Brohmer"
++   },
++   {
++    "authorId": "48120787",
++    "name": "B. T. Brown"
++   },
++   {
++    "authorId": "4197188",
++    "name": "K. Brown"
++   },
++   {
++    "authorId": "31412979",
++    "name": "Jovita Br\u00fcning"
++   },
++   {
++    "authorId": "1400158480",
++    "name": "Ann Calhoun-Sauls"
++   },
++   {
++    "authorId": "36027478",
++    "name": "S. Callahan"
++   },
++   {
++    "authorId": "95354096",
++    "name": "E. Chagnon"
++   },
++   {
++    "authorId": "48719411",
++    "name": "Jesse J. Chandler"
++   },
++   {
++    "authorId": "38049428",
++    "name": "Christopher R. Chartier"
++   },
++   {
++    "authorId": "145787104",
++    "name": "F. Cheung"
++   },
++   {
++    "authorId": "51055585",
++    "name": "Cody Daniel Christopherson"
++   },
++   {
++    "authorId": "4704887",
++    "name": "Linda Cillessen"
++   },
++   {
++    "authorId": "48987469",
++    "name": "Russ Clay"
++   },
++   {
++    "authorId": "46550612",
++    "name": "Hayley M. D. Cleary"
++   },
++   {
++    "authorId": "117283612",
++    "name": "Mark D. Cloud"
++   },
++   {
++    "authorId": "143947719",
++    "name": "M. Conn"
++   },
++   {
++    "authorId": "1795487",
++    "name": "J. Cohoon"
++   },
++   {
++    "authorId": "7293849",
++    "name": "Simon Columbus"
++   },
++   {
++    "authorId": "2052289610",
++    "name": "A. Cordes"
++   },
++   {
++    "authorId": "49251371",
++    "name": "G. Costantini"
++   },
++   {
++    "authorId": "15675192",
++    "name": "Leslie D Cramblet Alvarez"
++   },
++   {
++    "authorId": "51034966",
++    "name": "Edward J. Cremata"
++   },
++   {
++    "authorId": "104792844",
++    "name": "J. Crusius"
++   },
++   {
++    "authorId": "2480852",
++    "name": "J. DeCoster"
++   },
++   {
++    "authorId": "122378013",
++    "name": "M. DeGaetano"
++   },
++   {
++    "authorId": "1665535543",
++    "name": "Nicol\u00e1s Delia Penna"
++   },
++   {
++    "authorId": "1665493156",
++    "name": "Bobby Den Bezemer"
++   },
++   {
++    "authorId": "32368039",
++    "name": "M. Deserno"
++   },
++   {
++    "authorId": "113076847",
++    "name": "Olivia Devitt"
++   },
++   {
++    "authorId": "6629103",
++    "name": "L. Dewitte"
++   },
++   {
++    "authorId": "6958962",
++    "name": "David G. Dobolyi"
++   },
++   {
++    "authorId": "16313368",
++    "name": "Geneva T. Dodson"
++   },
++   {
++    "authorId": "2467192",
++    "name": "M. Donnellan"
++   },
++   {
++    "authorId": "2094220350",
++    "name": "R. Donohue"
++   },
++   {
++    "authorId": "5663433",
++    "name": "Rebecca A. Dore"
++   },
++   {
++    "authorId": "5021006",
++    "name": "A. Dorrough"
++   },
++   {
++    "authorId": "3488772",
++    "name": "A. Dreber"
++   },
++   {
++    "authorId": "5818783",
++    "name": "M. Dugas"
++   },
++   {
++    "authorId": "3100527",
++    "name": "Elizabeth W. Dunn"
++   },
++   {
++    "authorId": "14421983",
++    "name": "K. Easey"
++   },
++   {
++    "authorId": "117807982",
++    "name": "Sylvia Eboigbe"
++   },
++   {
++    "authorId": "34959578",
++    "name": "C. Eggleston"
++   },
++   {
++    "authorId": "123302634",
++    "name": "Jo Embley"
++   },
++   {
++    "authorId": "3018477",
++    "name": "S. Epskamp"
++   },
++   {
++    "authorId": "52004348",
++    "name": "Timothy M. Errington"
++   },
++   {
++    "authorId": "117698833",
++    "name": "Vivien Estel"
++   },
++   {
++    "authorId": "3392676",
++    "name": "Frank J. Farach"
++   },
++   {
++    "authorId": "25452274",
++    "name": "J. Feather"
++   },
++   {
++    "authorId": "2322314",
++    "name": "A. Fedor"
++   },
++   {
++    "authorId": "1400158404",
++    "name": "B. Fern\u00e1ndez-Castilla"
++   },
++   {
++    "authorId": "38548340",
++    "name": "Susann Fiedler"
++   },
++   {
++    "authorId": "32576544",
++    "name": "James G. Field"
++   },
++   {
++    "authorId": "2237526",
++    "name": "Stanka A. Fitneva"
++   },
++   {
++    "authorId": "2387162",
++    "name": "T. Flagan"
++   },
++   {
++    "authorId": "6543598",
++    "name": "A. Forest"
++   },
++   {
++    "authorId": "4193992",
++    "name": "Eskil Forsell"
++   },
++   {
++    "authorId": "153664010",
++    "name": "Joshua D. Foster"
++   },
++   {
++    "authorId": "2536223",
++    "name": "Michael C. Frank"
++   },
++   {
++    "authorId": "144106517",
++    "name": "Rebecca S. Frazier"
++   },
++   {
++    "authorId": "2057921617",
++    "name": "Heather M. Fuchs"
++   },
++   {
++    "authorId": "5807132",
++    "name": "P. Gable"
++   },
++   {
++    "authorId": "4908329",
++    "name": "Jeff Galak"
++   },
++   {
++    "authorId": "52169965",
++    "name": "E. Galliani"
++   },
++   {
++    "authorId": "39007608",
++    "name": "Anup Gampa"
++   },
++   {
++    "authorId": "2072601770",
++    "name": "Sara Garcia"
++   },
++   {
++    "authorId": "4034033",
++    "name": "Douglas Gazarian"
++   },
++   {
++    "authorId": "152595201",
++    "name": "Elizabeth Gilbert"
++   },
++   {
++    "authorId": "1400843054",
++    "name": "Roger Giner-Sorolla"
++   },
++   {
++    "authorId": "2479991061",
++    "name": "A. Gl\u00f6ckner"
++   },
++   {
++    "authorId": "35467332",
++    "name": "Lars Goellner"
++   },
++   {
++    "authorId": "6760964",
++    "name": "Jin X. Goh"
++   },
++   {
++    "authorId": "1402974706",
++    "name": "R. Goldberg"
++   },
++   {
++    "authorId": "6439793",
++    "name": "P. T. Goodbourn"
++   },
++   {
++    "authorId": "1400158332",
++    "name": "S. Gordon-McKeon"
++   },
++   {
++    "authorId": "103288020",
++    "name": "Bryan H. Gorges"
++   },
++   {
++    "authorId": "1677019678",
++    "name": "Jessie Gorges"
++   },
++   {
++    "authorId": "113021959",
++    "name": "J. Goss"
++   },
++   {
++    "authorId": "2250081930",
++    "name": "J. Graham"
++   },
++   {
++    "authorId": "50590791",
++    "name": "J. Grange"
++   },
++   {
++    "authorId": "2556954",
++    "name": "J. Gray"
++   },
++   {
++    "authorId": "2130548269",
++    "name": "C. Hartgerink"
++   },
++   {
++    "authorId": "27816717",
++    "name": "Joshua K. Hartshorne"
++   },
++   {
++    "authorId": "3390722",
++    "name": "F. Hasselman"
++   },
++   {
++    "authorId": "144616859",
++    "name": "Timothy B. Hayes"
++   },
++   {
++    "authorId": "3650446",
++    "name": "Emma Heikensten"
++   },
++   {
++    "authorId": "2196843",
++    "name": "Felix Henninger"
++   },
++   {
++    "authorId": "6941720",
++    "name": "John Hodsoll"
++   },
++   {
++    "authorId": "3388157",
++    "name": "Taylor Holubar"
++   },
++   {
++    "authorId": "92341957",
++    "name": "G. Hoogendoorn"
++   },
++   {
++    "authorId": "1665486287",
++    "name": "D. Humphries"
++   },
++   {
++    "authorId": "70247182",
++    "name": "C. Hung"
++   },
++   {
++    "authorId": "87889800",
++    "name": "N. Immelman"
++   },
++   {
++    "authorId": "6169028",
++    "name": "Vanessa C. Irsik"
++   },
++   {
++    "authorId": "36524750",
++    "name": "G. Jahn"
++   },
++   {
++    "authorId": "80488489",
++    "name": "F. J\u00e4kel"
++   },
++   {
++    "authorId": "2485703",
++    "name": "Marc Jekel"
++   },
++   {
++    "authorId": "145281248",
++    "name": "M. Johannesson"
++   },
++   {
++    "authorId": "117496518",
++    "name": "L. Johnson"
++   },
++   {
++    "authorId": "2150444569",
++    "name": "David J. Johnson"
++   },
++   {
++    "authorId": "10776195",
++    "name": "Kate M. Johnson"
++   },
++   {
++    "authorId": "2071756595",
++    "name": "William Johnston"
++   },
++   {
++    "authorId": "2640241",
++    "name": "K. Jonas"
++   },
++   {
++    "authorId": "1400158286",
++    "name": "Jennifer A. Joy-Gaba"
++   },
++   {
++    "authorId": "4049176",
++    "name": "H. Kappes"
++   },
++   {
++    "authorId": "120498542",
++    "name": "Kim Kelso"
++   },
++   {
++    "authorId": "8633058",
++    "name": "Mallory C. Kidwell"
++   },
++   {
++    "authorId": "2109708345",
++    "name": "Seung K. Kim"
++   },
++   {
++    "authorId": "6395769",
++    "name": "M. Kirkhart"
++   },
++   {
++    "authorId": "6032930",
++    "name": "B. Kleinberg"
++   },
++   {
++    "authorId": "50012037",
++    "name": "G. Kne\u017eevi\u0107"
++   },
++   {
++    "authorId": "38197330",
++    "name": "F. Kolorz"
++   },
++   {
++    "authorId": "39916640",
++    "name": "J. Kossakowski"
++   },
++   {
++    "authorId": "153875794",
++    "name": "Robert W. Krause"
++   },
++   {
++    "authorId": "123143083",
++    "name": "J. Krijnen"
++   },
++   {
++    "authorId": "5199064",
++    "name": "T. Kuhlmann"
++   },
++   {
++    "authorId": "90301659",
++    "name": "Y. Kunkels"
++   },
++   {
++    "authorId": "123804984",
++    "name": "Megan M. Kyc"
++   },
++   {
++    "authorId": "2148764895",
++    "name": "C. Lai"
++   },
++   {
++    "authorId": "1665495033",
++    "name": "A. Laique"
++   },
++   {
++    "authorId": "87792747",
++    "name": "D. Lakens"
++   },
++   {
++    "authorId": "47951763",
++    "name": "Kristin A. Lane"
++   },
++   {
++    "authorId": "6028786",
++    "name": "Bethany Lassetter"
++   },
++   {
++    "authorId": "40526243",
++    "name": "L. Lazarevi\u0107"
++   },
++   {
++    "authorId": "1665487375",
++    "name": "E. Bel"
++   },
++   {
++    "authorId": "2110236533",
++    "name": "Key Jung Lee"
++   },
++   {
++    "authorId": "46293822",
++    "name": "Minha Lee"
++   },
++   {
++    "authorId": "6560078",
++    "name": "K. Lemm"
++   },
++   {
++    "authorId": "2248921",
++    "name": "Carmel A. Levitan"
++   },
++   {
++    "authorId": "48575845",
++    "name": "Melissa Lewis"
++   },
++   {
++    "authorId": "2117106185",
++    "name": "Lin Lin"
++   },
++   {
++    "authorId": "2145198693",
++    "name": "Stephanie C. Lin"
++   },
++   {
++    "authorId": "35780819",
++    "name": "M. Lippold"
++   },
++   {
++    "authorId": "6767826",
++    "name": "Darren Loureiro"
++   },
++   {
++    "authorId": "116957928",
++    "name": "Ilse Luteijn"
++   },
++   {
++    "authorId": "35093146",
++    "name": "S. Mackinnon"
++   },
++   {
++    "authorId": "120566087",
++    "name": "Heather N. Mainard"
++   },
++   {
++    "authorId": "4526963",
++    "name": "Denise C. Marigold"
++   },
++   {
++    "authorId": "150078826",
++    "name": "D. P. Martin"
++   },
++   {
++    "authorId": "123396296",
++    "name": "Tylar Martinez"
++   },
++   {
++    "authorId": "4362855",
++    "name": "E. Masicampo"
++   },
++   {
++    "authorId": "84358116",
++    "name": "Joshua Matacotta"
++   },
++   {
++    "authorId": "32716879",
++    "name": "Maya B. Mathur"
++   },
++   {
++    "authorId": "2055522260",
++    "name": "Michael May"
++   },
++   {
++    "authorId": "5739306",
++    "name": "Nicole C Mechin"
++   },
++   {
++    "authorId": "2796734",
++    "name": "Pranjal H. Mehta"
++   },
++   {
++    "authorId": "3505739",
++    "name": "Johannes M. Meixner"
++   },
++   {
++    "authorId": "2292034",
++    "name": "Alissa Melinger"
++   },
++   {
++    "authorId": "2109507581",
++    "name": "Jeremy K. Miller"
++   },
++   {
++    "authorId": "1664699522",
++    "name": "Mallorie Miller"
++   },
++   {
++    "authorId": "3260369",
++    "name": "K. Moore"
++   },
++   {
++    "authorId": "8173711",
++    "name": "Marcus M\u00f6schl"
++   },
++   {
++    "authorId": "40632770",
++    "name": "M. Motyl"
++   },
++   {
++    "authorId": "49338794",
++    "name": "S. Muller"
++   },
++   {
++    "authorId": "3057626",
++    "name": "M. Munafo"
++   },
++   {
++    "authorId": "6699090",
++    "name": "K. Neijenhuijs"
++   },
++   {
++    "authorId": "122669704",
++    "name": "Taylor Nervi"
++   },
++   {
++    "authorId": "28866192",
++    "name": "Gandalf Nicolas"
++   },
++   {
++    "authorId": "5120903",
++    "name": "G. Nilsonne"
++   },
++   {
++    "authorId": "2862527",
++    "name": "Brian A. Nosek"
++   },
++   {
++    "authorId": "4222695",
++    "name": "M. Nuijten"
++   },
++   {
++    "authorId": "2061321863",
++    "name": "Catherine Olsson"
++   },
++   {
++    "authorId": "21707288",
++    "name": "Colleen M. Osborne"
++   },
++   {
++    "authorId": "15808420",
++    "name": "Lutz Ostkamp"
++   },
++   {
++    "authorId": "1800603",
++    "name": "M. Pavel"
++   },
++   {
++    "authorId": "1398036394",
++    "name": "I. Penton-Voak"
++   },
++   {
++    "authorId": "123910286",
++    "name": "O. Perna"
++   },
++   {
++    "authorId": "1733092",
++    "name": "Pernet Cyril"
++   },
++   {
++    "authorId": "5070565",
++    "name": "M. Perugini"
++   },
++   {
++    "authorId": "145798446",
++    "name": "R. Pipitone"
++   },
++   {
++    "authorId": "7731005",
++    "name": "M. Pitts"
++   },
++   {
++    "authorId": "3086561",
++    "name": "F. Plessow"
++   },
++   {
++    "authorId": "5666302",
++    "name": "J. Prenoveau"
++   },
++   {
++    "authorId": "114309502",
++    "name": "R. Rahal"
++   },
++   {
++    "authorId": "4412299",
++    "name": "Kate A. Ratliff"
++   },
++   {
++    "authorId": "46557202",
++    "name": "David A. Reinhard"
++   },
++   {
++    "authorId": "4137233",
++    "name": "Frank Renkewitz"
++   },
++   {
++    "authorId": "4977822",
++    "name": "Ashley A. Ricker"
++   },
++   {
++    "authorId": "3911950",
++    "name": "A. E. Rigney"
++   },
++   {
++    "authorId": "145836378",
++    "name": "Andrew M Rivers"
++   },
++   {
++    "authorId": "51055550",
++    "name": "Mark A. Roebke"
++   },
++   {
++    "authorId": "6431015",
++    "name": "Abraham M. Rutchick"
++   },
++   {
++    "authorId": "2055552083",
++    "name": "Robert S. Ryan"
++   },
++   {
++    "authorId": "48176355",
++    "name": "O. \u015eahin"
++   },
++   {
++    "authorId": "7144751",
++    "name": "Anondah Saide"
++   },
++   {
++    "authorId": "4225466",
++    "name": "Gillian M. Sandstrom"
++   },
++   {
++    "authorId": "2113520139",
++    "name": "David Santos"
++   },
++   {
++    "authorId": "2055715460",
++    "name": "R. Saxe"
++   },
++   {
++    "authorId": "40063306",
++    "name": "Ren\u00e9 Schlegelmilch"
++   },
++   {
++    "authorId": "49371070",
++    "name": "Kathleen Schmidt"
++   },
++   {
++    "authorId": "49715936",
++    "name": "Sabine Scholz"
++   },
++   {
++    "authorId": "123478162",
++    "name": "L. Seibel"
++   },
++   {
++    "authorId": "4067481",
++    "name": "D. Selterman"
++   },
++   {
++    "authorId": "3397505",
++    "name": "S. Shaki"
++   },
++   {
++    "authorId": "31665223",
++    "name": "W. B. Simpson"
++   },
++   {
++    "authorId": "144232281",
++    "name": "H. Sinclair"
++   },
++   {
++    "authorId": "143804040",
++    "name": "Jeanine L. M. Skorinko"
++   },
++   {
++    "authorId": "2111978802",
++    "name": "A. Slowik"
++   },
++   {
++    "authorId": "2500175",
++    "name": "J. Snyder"
++   },
++   {
++    "authorId": "4110370",
++    "name": "C. Soderberg"
++   },
++   {
++    "authorId": "32218314",
++    "name": "Carina M. Sonnleitner"
++   },
++   {
++    "authorId": "2056814613",
++    "name": "N. Spencer"
++   },
++   {
++    "authorId": "34782684",
++    "name": "Jeffrey R. Spies"
++   },
++   {
++    "authorId": "2889085",
++    "name": "S. Steegen"
++   },
++   {
++    "authorId": "3308834",
++    "name": "S. Stieger"
++   },
++   {
++    "authorId": "3918197",
++    "name": "Nina Strohminger"
++   },
++   {
++    "authorId": "3349937",
++    "name": "G. Sullivan"
++   },
++   {
++    "authorId": "6998374",
++    "name": "Thomas Talhelm"
++   },
++   {
++    "authorId": "123526524",
++    "name": "Megan Tapia"
++   },
++   {
++    "authorId": "120986210",
++    "name": "A. T. Dorsthorst"
++   },
++   {
++    "authorId": "17886332",
++    "name": "M. Thomae"
++   },
++   {
++    "authorId": "2108534776",
++    "name": "Sarah L. Thomas"
++   },
++   {
++    "authorId": "5407737",
++    "name": "Pia Tio"
++   },
++   {
++    "authorId": "26384240",
++    "name": "Frits Traets"
++   },
++   {
++    "authorId": "50343359",
++    "name": "Steve Tsang"
++   },
++   {
++    "authorId": "2109141",
++    "name": "F. Tuerlinckx"
++   },
++   {
++    "authorId": "118528873",
++    "name": "Paul Turchan"
++   },
++   {
++    "authorId": "3738838",
++    "name": "Milan Val\u00e1\u0161ek"
++   },
++   {
++    "authorId": "11366104",
++    "name": "A. Ve\u00e9r"
++   },
++   {
++    "authorId": "81322766",
++    "name": "R. V. Aert"
++   },
++   {
++    "authorId": "13333804",
++    "name": "M. V. Assen"
++   },
++   {
++    "authorId": "2062808617",
++    "name": "R. Bork"
++   },
++   {
++    "authorId": "122303122",
++    "name": "Mathijs van de Ven"
++   },
++   {
++    "authorId": "103097236",
++    "name": "Don van den Bergh"
++   },
++   {
++    "authorId": "151452444",
++    "name": "M. V. D. Hulst"
++   },
++   {
++    "authorId": "51240054",
++    "name": "R. V. Dooren"
++   },
++   {
++    "authorId": "50233092",
++    "name": "J. Doorn"
++   },
++   {
++    "authorId": "4155110",
++    "name": "D. V. Renswoude"
++   },
++   {
++    "authorId": "2392419",
++    "name": "H. van Rijn"
++   },
++   {
++    "authorId": "2409652",
++    "name": "Wolf Vanpaemel"
++   },
++   {
++    "authorId": "2078595605",
++    "name": "Alejandro Echeverr\u00eda"
++   },
++   {
++    "authorId": "2070303231",
++    "name": "Melissa Vazquez"
++   },
++   {
++    "authorId": "6122250",
++    "name": "Natalia V\u00e9lez"
++   },
++   {
++    "authorId": "35039893",
++    "name": "Marieke Vermue"
++   },
++   {
++    "authorId": "88801524",
++    "name": "M. Verschoor"
++   },
++   {
++    "authorId": "46989769",
++    "name": "M. Vianello"
++   },
++   {
++    "authorId": "2176390",
++    "name": "M. Voracek"
++   },
++   {
++    "authorId": "113898223",
++    "name": "Gina Vuu"
++   },
++   {
++    "authorId": "2505272",
++    "name": "E. Wagenmakers"
++   },
++   {
++    "authorId": "6348079",
++    "name": "Joanneke Weerdmeester"
++   },
++   {
++    "authorId": "2058927069",
++    "name": "A. Welsh"
++   },
++   {
++    "authorId": "5900056",
++    "name": "E. Westgate"
++   },
++   {
++    "authorId": "67239528",
++    "name": "Joeri Wissink"
++   },
++   {
++    "authorId": "145322160",
++    "name": "M. Wood"
++   },
++   {
++    "authorId": "48197426",
++    "name": "Andy T. Woods"
++   },
++   {
++    "authorId": "2058458367",
++    "name": "Emily M. Wright"
++   },
++   {
++    "authorId": "151482749",
++    "name": "Sining Wu"
++   },
++   {
++    "authorId": "145525838",
++    "name": "M. Zeelenberg"
++   },
++   {
++    "authorId": "117597026",
++    "name": "Kellylynn Zuni"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1126/science.aah6168": {
++  "paperId": "2dce110085044c913d6c3bf70aaf27345ffb23d1",
++  "externalIds": {
++   "MAG": "2560471521",
++   "DOI": "10.1126/science.aah6168",
++   "CorpusId": 32551384,
++   "PubMed": "27940837"
++  },
++  "publicationVenue": {
++   "id": "f59506a8-d8bb-4101-b3d4-c4ac3ed03dad",
++   "name": "Science",
++   "type": "journal",
++   "issn": "0193-4511",
++   "alternate_issns": [
++    "0036-8075"
++   ],
++   "url": "https://www.jstor.org/journal/science",
++   "alternate_urls": [
++    "https://www.sciencemag.org/",
++    "http://www.sciencemag.org/",
++    "http://www.jstor.org/journals/00368075.html",
++    "http://www.sciencemag.org/archive/"
++   ]
++  },
++  "title": "Enhancing reproducibility for computational methods",
++  "venue": "Science",
++  "year": 2016,
++  "citationCount": 424,
++  "openAccessPdf": {
++   "url": "https://escholarship.org/uc/item/9wh3k06p",
++   "status": "GREEN",
++   "license": "CCBYNC",
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1126/science.aah6168?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1126/science.aah6168, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Science",
++   "pages": "1240 - 1241",
++   "volume": "354"
++  },
++  "authors": [
++   {
++    "authorId": "2598812",
++    "name": "V. Stodden"
++   },
++   {
++    "authorId": "144643621",
++    "name": "M. McNutt"
++   },
++   {
++    "authorId": "1685317",
++    "name": "D. Bailey"
++   },
++   {
++    "authorId": "1687896",
++    "name": "E. Deelman"
++   },
++   {
++    "authorId": "145526918",
++    "name": "Y. Gil"
++   },
++   {
++    "authorId": "47283959",
++    "name": "B. Hanson"
++   },
++   {
++    "authorId": "2135545",
++    "name": "M. Heroux"
++   },
++   {
++    "authorId": "145441750",
++    "name": "J. Ioannidis"
++   },
++   {
++    "authorId": "48478161",
++    "name": "M. Taufer"
++   }
++  ],
++  "abstract": "Data, code, and workflows should be available and cited Over the past two decades, computational methods have radically changed the ability of researchers from all areas of scholarship to process and analyze data and to simulate complex systems. But with these advances come challenges that are contributing to broader concerns over irreproducibility in the scholarly literature, among them the lack of transparency in disclosure of computational methods. Current reporting methods are often uneven, incomplete, and still evolving. We present a novel set of Reproducibility Enhancement Principles (REP) targeting disclosure challenges involving computation. These recommendations, which build upon more general proposals from the Transparency and Openness Promotion (TOP) guidelines (1) and recommendations for field data (2), emerged from workshop discussions among funding agencies, publishers and journal editors, industry participants, and researchers representing a broad range of domains. Although some of these actions may be aspirational, we believe it is important to recognize and move toward ameliorating irreproducibility in computational research."
++ },
++ "DOI:10.1126/scitranslmed.aaf5027": {
++  "paperId": "3728c289e3582f6329a735b3e9882b2a0cabad83",
++  "externalIds": {
++   "MAG": "2414283705",
++   "DOI": "10.1126/scitranslmed.aaf5027",
++   "CorpusId": 848096,
++   "PubMed": "27252173"
++  },
++  "publicationVenue": {
++   "id": "1cd4baea-5ccc-4522-9dbf-0566f39d3b37",
++   "name": "Science Translational Medicine",
++   "type": "journal",
++   "alternate_names": [
++    "Sci Transl Med"
++   ],
++   "issn": "1946-6234",
++   "url": "https://stm.sciencemag.org/",
++   "alternate_urls": [
++    "https://stm.sciencemag.org/site/misc/about.xhtml"
++   ]
++  },
++  "title": "What does research reproducibility mean?",
++  "venue": "Science Translational Medicine",
++  "year": 2016,
++  "citationCount": 1164,
++  "openAccessPdf": {
++   "url": "https://stm.sciencemag.org/content/scitransmed/8/341/341ps12.full.pdf",
++   "status": "BRONZE",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1126/scitranslmed.aaf5027?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1126/scitranslmed.aaf5027, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Review",
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Science Translational Medicine",
++   "pages": "341ps12 - 341ps12",
++   "volume": "8"
++  },
++  "authors": [
++   {
++    "authorId": "1815823",
++    "name": "S. Goodman"
++   },
++   {
++    "authorId": "30080277",
++    "name": "D. Fanelli"
++   },
++   {
++    "authorId": "145441750",
++    "name": "J. Ioannidis"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1136/amiajnl-2011-000089": {
++  "paperId": "bbc468e704ff4af72ca9de9ceb8203f7bf930d49",
++  "externalIds": {
++   "MAG": "2281090488",
++   "DBLP": "journals/jamia/GoddardRW12",
++   "DOI": "10.1136/amiajnl-2011-000089",
++   "CorpusId": 2050860,
++   "PubMed": "21685142"
++  },
++  "publicationVenue": null,
++  "title": "Automation bias: a systematic review of frequency, effect mediators, and mitigators",
++  "venue": "J. Am. Medical Informatics Assoc.",
++  "year": 2012,
++  "citationCount": 1130,
++  "openAccessPdf": {
++   "url": "https://academic.oup.com/jamia/article-pdf/19/1/121/5911703/19-1-121.pdf",
++   "status": "BRONZE",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1136/amiajnl-2011-000089?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1136/amiajnl-2011-000089, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Review",
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Journal of the American Medical Informatics Association : JAMIA",
++   "pages": "\n          121-7\n        ",
++   "volume": "19 1"
++  },
++  "authors": [
++   {
++    "authorId": "143911263",
++    "name": "Kate Goddard"
++   },
++   {
++    "authorId": "2839223",
++    "name": "A. Roudsari"
++   },
++   {
++    "authorId": "145984984",
++    "name": "J. Wyatt"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1145/2160718.2160734": {
++  "paperId": "87fc322b992c1881854c1e2a0e36ef5b6f739071",
++  "externalIds": {
++   "DBLP": "journals/queue/Helland12",
++   "MAG": "2159123115",
++   "DOI": "10.1145/2160718.2160734",
++   "CorpusId": 28545927
++  },
++  "publicationVenue": {
++   "id": "4bd27ad1-f6fa-4340-9d9d-c62ac5be6fc0",
++   "name": "Queue",
++   "type": "journal",
++   "alternate_names": [
++    "ACM Queue"
++   ],
++   "issn": "1542-7730",
++   "url": "http://portal.acm.org/queue",
++   "alternate_urls": [
++    "http://www.acmqueue.org/"
++   ]
++  },
++  "title": "Idempotence is not a medical condition",
++  "venue": "Queue",
++  "year": 2012,
++  "citationCount": 76,
++  "openAccessPdf": {
++   "url": "https://dl.acm.org/doi/pdf/10.1145/2181796.2187821",
++   "status": "BRONZE",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/2160718.2160734?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/2160718.2160734, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Communications of the ACM",
++   "pages": "56 - 65",
++   "volume": "55"
++  },
++  "authors": [
++   {
++    "authorId": "2549968",
++    "name": "Pat Helland"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1145/2408776.2408794": {
++  "paperId": "0831a5baf38c9b3d43c755319a602b15fc01c52d",
++  "externalIds": {
++   "DBLP": "journals/cacm/DeanB13",
++   "MAG": "1982063824",
++   "DOI": "10.1145/2408776.2408794",
++   "CorpusId": 34900520
++  },
++  "publicationVenue": null,
++  "title": "The tail at scale",
++  "venue": "CACM",
++  "year": 2013,
++  "citationCount": 2180,
++  "openAccessPdf": {
++   "url": "",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/2408776.2408794?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/2408776.2408794, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Commun. ACM",
++   "pages": "74-80",
++   "volume": "56"
++  },
++  "authors": [
++   {
++    "authorId": "49959210",
++    "name": "Jeffrey Dean"
++   },
++   {
++    "authorId": "1746646",
++    "name": "L. Barroso"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1145/2452376.2452478": {
++  "paperId": "e75134cde1477ac341c2197b1882191a972c2b60",
++  "externalIds": {
++   "MAG": "2069028640",
++   "DBLP": "conf/edbt/MissierBC13",
++   "DOI": "10.1145/2452376.2452478",
++   "CorpusId": 16907881
++  },
++  "publicationVenue": {
++   "id": "74f47cf8-2839-4a56-959a-524dd0ed9e3e",
++   "name": "International Conference on Extending Database Technology",
++   "type": "conference",
++   "alternate_names": [
++    "Int Conf Extending Database Technol",
++    "Extending Database Technology",
++    "Extending Database Technol",
++    "EDBT"
++   ],
++   "url": "http://www.edbt.org/"
++  },
++  "title": "The W3C PROV family of specifications for modelling provenance metadata",
++  "venue": "International Conference on Extending Database Technology",
++  "year": 2013,
++  "citationCount": 260,
++  "openAccessPdf": {
++   "url": "",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/2452376.2452478?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/2452376.2452478, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "pages": "773-776"
++  },
++  "authors": [
++   {
++    "authorId": "1755645",
++    "name": "P. Missier"
++   },
++   {
++    "authorId": "3337287",
++    "name": "K. Belhajjame"
++   },
++   {
++    "authorId": "144320476",
++    "name": "James Cheney"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1145/2670979.2670986": {
++  "paperId": "253818f5625fc74fd2f827dfc82f02ce8f22bef7",
++  "externalIds": {
++   "DBLP": "conf/cloud/GunawiHLPDAELLM14",
++   "MAG": "2006307164",
++   "DOI": "10.1145/2670979.2670986",
++   "CorpusId": 16038767
++  },
++  "publicationVenue": {
++   "id": "d13e941e-4cac-4f1d-bdca-77d927e31f1b",
++   "name": "ACM Symposium on Cloud Computing",
++   "type": "conference",
++   "alternate_names": [
++    "System-on-Chip Conference",
++    "ACM Symp Cloud Comput",
++    "Syst Conf",
++    "Symp Cloud Comput",
++    "Annual IEEE International System-on-Chip Conference",
++    "Symposium on Cloud Computing",
++    "Annu IEEE Int Syst Conf",
++    "SoCC"
++   ],
++   "url": "http://www.ieee-socc.org/"
++  },
++  "title": "What Bugs Live in the Cloud? A Study of 3000+ Issues in Cloud Systems",
++  "venue": "ACM Symposium on Cloud Computing",
++  "year": 2014,
++  "citationCount": 280,
++  "openAccessPdf": {
++   "url": "",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/2670979.2670986?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/2670979.2670986, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Book",
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Proceedings of the ACM Symposium on Cloud Computing"
++  },
++  "authors": [
++   {
++    "authorId": "1738725",
++    "name": "Haryadi S. Gunawi"
++   },
++   {
++    "authorId": "2725752",
++    "name": "M. Hao"
++   },
++   {
++    "authorId": "1709102",
++    "name": "Tanakorn Leesatapornwongsa"
++   },
++   {
++    "authorId": "1405320577",
++    "name": "Tiratat Patana-anake"
++   },
++   {
++    "authorId": "145182862",
++    "name": "Thanh Do"
++   },
++   {
++    "authorId": "2286277",
++    "name": "J. Adityatama"
++   },
++   {
++    "authorId": "2590796",
++    "name": "Kurnia J. Eliazar"
++   },
++   {
++    "authorId": "113194122",
++    "name": "Agung Laksono"
++   },
++   {
++    "authorId": "2235631",
++    "name": "Jeffrey F. Lukman"
++   },
++   {
++    "authorId": "47787880",
++    "name": "V. Martin"
++   },
++   {
++    "authorId": "2382708",
++    "name": "A. Satria"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1145/2723872.2723882": {
++  "paperId": "e0b34f80227e9c060bc9c37692278e7cefaa9540",
++  "externalIds": {
++   "MAG": "2952640757",
++   "ArXiv": "1410.0846",
++   "DBLP": "journals/sigops/Boettiger15",
++   "DOI": "10.1145/2723872.2723882",
++   "CorpusId": 255583
++  },
++  "publicationVenue": null,
++  "title": "An introduction to Docker for reproducible research",
++  "venue": "OPSR",
++  "year": 2014,
++  "citationCount": 1085,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/1410.0846",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/1410.0846, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ACM SIGOPS Oper. Syst. Rev.",
++   "pages": "71-79",
++   "volume": "49"
++  },
++  "authors": [
++   {
++    "authorId": "72997413",
++    "name": "C. Boettiger"
++   }
++  ],
++  "abstract": "As computational work becomes more and more integral to many aspects of scientific research, computational reproducibility has become an issue of increasing importance to computer systems researchers and domain scientists alike. Though computational reproducibility seems more straight forward than replicating physical experiments, the complex and rapidly changing nature of computer environments makes being able to reproduce and extend such work a serious challenge. In this paper, I explore common reasons that code developed for one research project cannot be successfully executed or extended by subsequent researchers. I review current approaches to these issues, including virtual machines and workflow systems, and their limitations. I then examine how the popular emerging technology Docker combines several areas from systems research - such as operating system virtualization, cross-platform portability, modular re-usable elements, versioning, and a 'DevOps' philosophy, to address these challenges. I illustrate this with several examples of Docker use with a focus on the R statistical environment."
++ },
++ "DOI:10.1145/2882903.2899401": {
++  "paperId": "4a9811c6ba7c7faacbcbf6a73a4ff40c77885c40",
++  "externalIds": {
++   "MAG": "2431100065",
++   "DBLP": "conf/sigmod/ChirigatiRSF16",
++   "DOI": "10.1145/2882903.2899401",
++   "CorpusId": 17500075
++  },
++  "publicationVenue": null,
++  "title": "ReproZip: Computational Reproducibility With Ease",
++  "venue": "SIGMOD Conference",
++  "year": 2016,
++  "citationCount": 160,
++  "openAccessPdf": {
++   "url": "http://dl.acm.org/ft_gateway.cfm?id=2899401&type=pdf",
++   "status": "BRONZE",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/2882903.2899401?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/2882903.2899401, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Book",
++   "JournalArticle",
++   "Conference",
++   "Review"
++  ],
++  "journal": {
++   "name": "Proceedings of the 2016 International Conference on Management of Data"
++  },
++  "authors": [
++   {
++    "authorId": "2541202",
++    "name": "F. Chirigati"
++   },
++   {
++    "authorId": "3416217",
++    "name": "R\u00e9mi Rampin"
++   },
++   {
++    "authorId": "1695878",
++    "name": "D. Shasha"
++   },
++   {
++    "authorId": "144162611",
++    "name": "J. Freire"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1145/2987550.2987568": {
++  "paperId": "5b60e599096044a5fed372950486ad8eb0ac94ae",
++  "externalIds": {
++   "DBLP": "conf/cloud/SambasivanSMSFG16",
++   "MAG": "2521815480",
++   "DOI": "10.1145/2987550.2987568",
++   "CorpusId": 8835048
++  },
++  "publicationVenue": {
++   "id": "d13e941e-4cac-4f1d-bdca-77d927e31f1b",
++   "name": "ACM Symposium on Cloud Computing",
++   "type": "conference",
++   "alternate_names": [
++    "System-on-Chip Conference",
++    "ACM Symp Cloud Comput",
++    "Syst Conf",
++    "Symp Cloud Comput",
++    "Annual IEEE International System-on-Chip Conference",
++    "Symposium on Cloud Computing",
++    "Annu IEEE Int Syst Conf",
++    "SoCC"
++   ],
++   "url": "http://www.ieee-socc.org/"
++  },
++  "title": "Principled workflow-centric tracing of distributed systems",
++  "venue": "ACM Symposium on Cloud Computing",
++  "year": 2016,
++  "citationCount": 72,
++  "openAccessPdf": {
++   "url": "http://dl.acm.org/ft_gateway.cfm?id=2987568&type=pdf",
++   "status": "BRONZE",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/2987550.2987568?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/2987550.2987568, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Book",
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Proceedings of the Seventh ACM Symposium on Cloud Computing"
++  },
++  "authors": [
++   {
++    "authorId": "2726787",
++    "name": "Raja R. Sambasivan"
++   },
++   {
++    "authorId": "17017774",
++    "name": "Ilari Shafer"
++   },
++   {
++    "authorId": "48532669",
++    "name": "Jonathan Mace"
++   },
++   {
++    "authorId": "2084178",
++    "name": "B. Sigelman"
++   },
++   {
++    "authorId": "145502789",
++    "name": "Rodrigo Fonseca"
++   },
++   {
++    "authorId": "1707164",
++    "name": "G. Ganger"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1145/3290605.3300233": {
++  "paperId": "ad3cf68bae32d21f25ac142287d4a556155619d2",
++  "externalIds": {
++   "MAG": "2916904544",
++   "DBLP": "conf/chi/AmershiWVFNCSIB19",
++   "DOI": "10.1145/3290605.3300233",
++   "CorpusId": 86866942
++  },
++  "publicationVenue": {
++   "id": "b55b50b1-aae7-47a7-b042-8aecc930073d",
++   "name": "International Conference on Human Factors in Computing Systems",
++   "type": "conference",
++   "alternate_names": [
++    "CHI",
++    "Int Conf Hum Factor Comput Syst",
++    "Human Factors in Computing Systems",
++    "Conference on Human Interface",
++    "Conf Hum Interface",
++    "Hum Factor Comput Syst"
++   ],
++   "url": "http://www.acm.org/sigchi/"
++  },
++  "title": "Guidelines for Human-AI Interaction",
++  "venue": "International Conference on Human Factors in Computing Systems",
++  "year": 2019,
++  "citationCount": 2830,
++  "openAccessPdf": {
++   "url": "",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1145/3290605.3300233?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/3290605.3300233, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Book",
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "name": "Proceedings of the 2019 CHI Conference on Human Factors in Computing Systems"
++  },
++  "authors": [
++   {
++    "authorId": "1719124",
++    "name": "Saleema Amershi"
++   },
++   {
++    "authorId": "1780531",
++    "name": "Daniel S. Weld"
++   },
++   {
++    "authorId": "3109339",
++    "name": "Mihaela Vorvoreanu"
++   },
++   {
++    "authorId": "3318905",
++    "name": "Adam Fourney"
++   },
++   {
++    "authorId": "2571049",
++    "name": "Besmira Nushi"
++   },
++   {
++    "authorId": "9703838",
++    "name": "Penny Collisson"
++   },
++   {
++    "authorId": "38972741",
++    "name": "Ji-Na Suh"
++   },
++   {
++    "authorId": "1730570",
++    "name": "Shamsi T. Iqbal"
++   },
++   {
++    "authorId": "144609235",
++    "name": "Paul N. Bennett"
++   },
++   {
++    "authorId": "51306063",
++    "name": "K. Inkpen"
++   },
++   {
++    "authorId": "144113253",
++    "name": "J. Teevan"
++   },
++   {
++    "authorId": "1405707881",
++    "name": "Ruth Kikin-Gil"
++   },
++   {
++    "authorId": "145479841",
++    "name": "E. Horvitz"
++   }
++  ],
++  "abstract": "Advances in artificial intelligence (AI) frame opportunities and challenges for user interface design. Principles for human-AI interaction have been discussed in the human-computer interaction community for over two decades, but more study and innovation are needed in light of advances in AI and the growing uses of AI technologies in human-facing applications. We propose 18 generally applicable design guidelines for human-AI interaction. These guidelines are validated through multiple rounds of evaluation including a user study with 49 design practitioners who tested the guidelines against 20 popular AI-infused products. The results verify the relevance of the guidelines over a spectrum of interaction scenarios and reveal gaps in our knowledge, highlighting opportunities for further research. Based on the evaluations, we believe the set of design guidelines can serve as a resource to practitioners working on the design of applications and features that harness AI technologies, and to researchers interested in the further development of human-AI interaction design principles."
++ },
++ "DOI:10.1145/3411764.3445717": {
++  "paperId": "ebcbbb8fe297940d79b17aeb6d46bedff9db7fec",
++  "externalIds": {
++   "DBLP": "conf/chi/BansalWZFNKRW21",
++   "ArXiv": "2006.14779",
++   "MAG": "3037634279",
++   "DOI": "10.1145/3411764.3445717",
++   "CorpusId": 220128138
++  },
++  "publicationVenue": {
++   "id": "b55b50b1-aae7-47a7-b042-8aecc930073d",
++   "name": "International Conference on Human Factors in Computing Systems",
++   "type": "conference",
++   "alternate_names": [
++    "CHI",
++    "Int Conf Hum Factor Comput Syst",
++    "Human Factors in Computing Systems",
++    "Conference on Human Interface",
++    "Conf Hum Interface",
++    "Hum Factor Comput Syst"
++   ],
++   "url": "http://www.acm.org/sigchi/"
++  },
++  "title": "Does the Whole Exceed its Parts? The Effect of AI Explanations on Complementary Team Performance",
++  "venue": "International Conference on Human Factors in Computing Systems",
++  "year": 2020,
++  "citationCount": 1161,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/2006.14779",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2006.14779, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Book",
++   "Conference"
++  ],
++  "journal": {
++   "name": "Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems"
++  },
++  "authors": [
++   {
++    "authorId": "33340656",
++    "name": "Gagan Bansal"
++   },
++   {
++    "authorId": "35232494",
++    "name": "Tongshuang Sherry Wu"
++   },
++   {
++    "authorId": "153823289",
++    "name": "Joyce Zhou"
++   },
++   {
++    "authorId": "27083453",
++    "name": "Raymond Fok"
++   },
++   {
++    "authorId": "2571049",
++    "name": "Besmira Nushi"
++   },
++   {
++    "authorId": "1783184",
++    "name": "Ece Kamar"
++   },
++   {
++    "authorId": "78846919",
++    "name": "Marco Tulio Ribeiro"
++   },
++   {
++    "authorId": "1780531",
++    "name": "Daniel S. Weld"
++   }
++  ],
++  "abstract": "Many researchers motivate explainable AI with studies showing that human-AI team performance on decision-making tasks improves when the AI explains its recommendations. However, prior studies observed improvements from explanations only when the AI, alone, outperformed both the human and the best team. Can explanations help lead to complementary performance, where team accuracy is higher than either the human or the AI working solo? We conduct mixed-method user studies on three datasets, where an AI with accuracy comparable to humans helps participants solve a task (explaining itself in some conditions). While we observed complementary improvements from AI augmentation, they were not increased by explanations. Rather, explanations increased the chance that humans will accept the AI\u2019s recommendation, regardless of its correctness. Our result poses new challenges for human-centered AI: Can we develop explanatory approaches that encourage appropriate trust in AI, and therefore help generate (or improve) complementary performance?"
++ },
++ "DOI:10.1145/3449287": {
++  "paperId": "65772de56676a6a437c8abaeccef34224f62ee13",
++  "externalIds": {
++   "ArXiv": "2102.09692",
++   "DBLP": "journals/pacmhci/BucincaMG21",
++   "DOI": "10.1145/3449287",
++   "CorpusId": 231979279
++  },
++  "publicationVenue": null,
++  "title": "To Trust or to Think",
++  "venue": "Proc. ACM Hum. Comput. Interact.",
++  "year": 2021,
++  "citationCount": 861,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/2102.09692",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2102.09692, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Proceedings of the ACM on Human-Computer Interaction",
++   "pages": "1 - 21",
++   "volume": "5"
++  },
++  "authors": [
++   {
++    "authorId": null,
++    "name": "Zana Bu\u00e7inca"
++   },
++   {
++    "authorId": "2096042676",
++    "name": "Maja Malaya"
++   },
++   {
++    "authorId": "1770992",
++    "name": "Krzysztof Z Gajos"
++   }
++  ],
++  "abstract": "People supported by AI-powered decision support tools frequently overrely on the AI: they accept an AI's suggestion even when that suggestion is wrong. Adding explanations to the AI decisions does not appear to reduce the overreliance and some studies suggest that it might even increase it. Informed by the dual-process theory of cognition, we posit that people rarely engage analytically with each individual AI recommendation and explanation, and instead develop general heuristics about whether and when to follow the AI suggestions. Building on prior research on medical decision-making, we designed three cognitive forcing interventions to compel people to engage more thoughtfully with the AI-generated explanations. We conducted an experiment (N=199), in which we compared our three cognitive forcing designs to two simple explainable AI approaches and to a no-AI baseline. The results demonstrate that cognitive forcing significantly reduced overreliance compared to the simple explainable AI approaches. However, there was a trade-off: people assigned the least favorable subjective ratings to the designs that reduced the overreliance the most. To audit our work for intervention-generated inequalities, we investigated whether our interventions benefited equally people with different levels of Need for Cognition (i.e., motivation to engage in effortful mental activities). Our results show that, on average, cognitive forcing interventions benefited participants higher in Need for Cognition more. Our research suggests that human cognitive motivation moderates the effectiveness of explainable AI solutions."
++ },
++ "DOI:10.1145/3485510": {
++  "paperId": "fcf4c0ecb2d6b194fb0534fd2e4d5f5a233a5452",
++  "externalIds": {
++   "DBLP": "journals/pacmpl/BurckhardtGJKMM21",
++   "DOI": "10.1145/3485510",
++   "CorpusId": 238640954
++  },
++  "publicationVenue": null,
++  "title": "Durable functions: semantics for stateful serverless",
++  "venue": "Proc. ACM Program. Lang.",
++  "year": 2021,
++  "citationCount": 88,
++  "openAccessPdf": {
++   "url": "https://dl.acm.org/doi/pdf/10.1145/3485510",
++   "status": "GOLD",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1145/3485510?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/3485510, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Proceedings of the ACM on Programming Languages",
++   "pages": "1 - 27",
++   "volume": "5"
++  },
++  "authors": [
++   {
++    "authorId": "1793711",
++    "name": "S. Burckhardt"
++   },
++   {
++    "authorId": "9414640",
++    "name": "Christoph L. Gillum"
++   },
++   {
++    "authorId": "84064782",
++    "name": "David Justo"
++   },
++   {
++    "authorId": "91973722",
++    "name": "Konstantinos Kallas"
++   },
++   {
++    "authorId": "2064130794",
++    "name": "Connor McMahon"
++   },
++   {
++    "authorId": "1789622",
++    "name": "Christopher S. Meiklejohn"
++   }
++  ],
++  "abstract": "Serverless, or Functions-as-a-Service (FaaS), is an increasingly popular paradigm for application development, as it provides implicit elastic scaling and load based billing. However, the weak execution guarantees and intrinsic compute-storage separation of FaaS create serious challenges when developing applications that require persistent state, reliable progress, or synchronization. This has motivated a new generation of serverless frameworks that provide stateful abstractions. For instance, Azure's Durable Functions (DF) programming model enhances FaaS with actors, workflows, and critical sections. As a programming model, DF is interesting because it combines task and actor parallelism, which makes it suitable for a wide range of serverless applications. We describe DF both informally, using examples, and formally, using an idealized high-level model based on the untyped lambda calculus. Next, we demystify how the DF runtime can (1) execute in a distributed unreliable serverless environment with compute-storage separation, yet still conform to the fault-free high-level model, and (2) persist execution progress without requiring checkpointing support by the language runtime. To this end we define two progressively more complex execution models, which contain the compute-storage separation and the record-replay, and prove that they are equivalent to the high-level model."
++ },
++ "DOI:10.1145/3486897": {
++  "paperId": "f791dce07eb1dd9251f9bffb801d6e075091c98d",
++  "externalIds": {
++   "ArXiv": "2105.07028",
++   "DBLP": "journals/corr/abs-2105-07028",
++   "DOI": "10.1145/3486897",
++   "CorpusId": 234742536
++  },
++  "publicationVenue": {
++   "id": "4d9ce1c4-dc84-46b9-903e-e3751c00c7dd",
++   "name": "Communications of the ACM",
++   "type": "journal",
++   "alternate_names": [
++    "Commun ACM",
++    "Communications of The ACM"
++   ],
++   "issn": "0001-0782",
++   "url": "http://www.acm.org/pubs/cacm/",
++   "alternate_urls": [
++    "http://portal.acm.org/cacm",
++    "http://www.acm.org/pubs/contents/journals/cacm/",
++    "https://cacm.acm.org/"
++   ]
++  },
++  "title": "Methods included",
++  "venue": "Communications of the ACM",
++  "year": 2021,
++  "citationCount": 127,
++  "openAccessPdf": {
++   "url": "https://dl.acm.org/doi/pdf/10.1145/3486897",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2105.07028, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Communications of the ACM",
++   "pages": "54 - 63",
++   "volume": "65"
++  },
++  "authors": [
++   {
++    "authorId": "144361904",
++    "name": "M. Crusoe"
++   },
++   {
++    "authorId": "2922103",
++    "name": "S. Abeln"
++   },
++   {
++    "authorId": "1760940",
++    "name": "A. Iosup"
++   },
++   {
++    "authorId": "37885989",
++    "name": "P. Amstutz"
++   },
++   {
++    "authorId": "113676421",
++    "name": "J. Chilton"
++   },
++   {
++    "authorId": "10671181",
++    "name": "Nebojsa Tijanic"
++   },
++   {
++    "authorId": "2085232797",
++    "name": "Herv'e M'enager"
++   },
++   {
++    "authorId": "2161614",
++    "name": "Stian Soiland-Reyes"
++   },
++   {
++    "authorId": "2254095266",
++    "name": "Bogdan Gavrilovic"
++   },
++   {
++    "authorId": "46555127",
++    "name": "C. Goble"
++   },
++   {
++    "authorId": "2521542905",
++    "name": "The Cwl Community"
++   }
++  ],
++  "abstract": "Standardizing computational reuse and portability with the Common Workflow Language."
++ },
++ "DOI:10.1145/3571730": {
++  "paperId": "3def68bd0f856886d34272840a7f81588f2bc082",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2202-03629",
++   "ArXiv": "2202.03629",
++   "DOI": "10.1145/3571730",
++   "CorpusId": 246652372
++  },
++  "publicationVenue": {
++   "id": "7b2adce0-d53f-49d6-8784-b0645604fe62",
++   "name": "ACM Computing Surveys",
++   "type": "journal",
++   "alternate_names": [
++    "ACM Comput Surv"
++   ],
++   "issn": "0360-0300",
++   "url": "http://www.acm.org/pubs/surveys/",
++   "alternate_urls": [
++    "http://portal.acm.org/csur",
++    "https://csur.acm.org/",
++    "http://csur.acm.org/"
++   ]
++  },
++  "title": "Survey of Hallucination in Natural Language Generation",
++  "venue": "ACM Computing Surveys",
++  "year": 2022,
++  "citationCount": 6517,
++  "openAccessPdf": {
++   "url": "https://arxiv.org/pdf/2202.03629",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2202.03629, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ACM Computing Surveys",
++   "pages": "1 - 38",
++   "volume": "55"
++  },
++  "authors": [
++   {
++    "authorId": "3391272",
++    "name": "Ziwei Ji"
++   },
++   {
++    "authorId": "40221187",
++    "name": "Nayeon Lee"
++   },
++   {
++    "authorId": "1397300094",
++    "name": "Rita Frieske"
++   },
++   {
++    "authorId": "1660855299",
++    "name": "Tiezheng Yu"
++   },
++   {
++    "authorId": "49678544",
++    "name": "D. Su"
++   },
++   {
++    "authorId": "98271906",
++    "name": "Yan Xu"
++   },
++   {
++    "authorId": "38524906",
++    "name": "Etsuko Ishii"
++   },
++   {
++    "authorId": "23672613",
++    "name": "Yejin Bang"
++   },
++   {
++    "authorId": "94210578",
++    "name": "Delong Chen"
++   },
++   {
++    "authorId": "47653392",
++    "name": "Wenliang Dai"
++   },
++   {
++    "authorId": "2111680936",
++    "name": "Andrea Madotto"
++   },
++   {
++    "authorId": "40539650",
++    "name": "Pascale Fung"
++   }
++  ],
++  "abstract": "Natural Language Generation (NLG) has improved exponentially in recent years thanks to the development of sequence-to-sequence deep learning technologies such as Transformer-based language models. This advancement has led to more fluent and coherent NLG, leading to improved development in downstream tasks such as abstractive summarization, dialogue generation, and data-to-text generation. However, it is also apparent that deep learning based generation is prone to hallucinate unintended text, which degrades the system performance and fails to meet user expectations in many real-world scenarios. To address this issue, many studies have been presented in measuring and mitigating hallucinated texts, but these have never been reviewed in a comprehensive manner before. In this survey, we thus provide a broad overview of the research progress and challenges in the hallucination problem in NLG. The survey is organized into two parts: (1) a general overview of metrics, mitigation methods, and future directions, and (2) an overview of task-specific research progress on hallucinations in the following downstream tasks, namely abstractive summarization, dialogue generation, generative question answering, data-to-text generation, and machine translation. This survey serves to facilitate collaborative efforts among researchers in tackling the challenge of hallucinated texts in NLG."
++ },
++ "DOI:10.1145/3593013.3594087": {
++  "paperId": "f46050b4ae991a0c047f9930c8de7c706e61c197",
++  "externalIds": {
++   "DBLP": "conf/fat/LaiCSLT23",
++   "DOI": "10.1145/3593013.3594087",
++   "CorpusId": 259139714
++  },
++  "publicationVenue": {
++   "id": "cdc83875-a82d-445c-b097-cbe91afe99a8",
++   "name": "Conference on Fairness, Accountability and Transparency",
++   "type": "conference",
++   "alternate_names": [
++    "FAccT",
++    "Conf Fairness Account Transpar"
++   ],
++   "url": "https://facctconference.org/"
++  },
++  "title": "Towards a Science of Human-AI Decision Making: An Overview of Design Space in Empirical Human-Subject Studies",
++  "venue": "Conference on Fairness, Accountability and Transparency",
++  "year": 2023,
++  "citationCount": 273,
++  "openAccessPdf": {
++   "url": "https://dl.acm.org/doi/pdf/10.1145/3593013.3594087",
++   "status": "BRONZE",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1145/3593013.3594087?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/3593013.3594087, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Book",
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Proceedings of the 2023 ACM Conference on Fairness, Accountability, and Transparency"
++  },
++  "authors": [
++   {
++    "authorId": "120801533",
++    "name": "V. Lai"
++   },
++   {
++    "authorId": "2131059444",
++    "name": "Cha-Cha Chen"
++   },
++   {
++    "authorId": "1405364873",
++    "name": "Alison Smith-Renner"
++   },
++   {
++    "authorId": "144921048",
++    "name": "Q. Liao"
++   },
++   {
++    "authorId": "2111727675",
++    "name": "Chenhao Tan"
++   }
++  ],
++  "abstract": "AI systems are adopted in numerous domains due to their increasingly strong predictive performance. However, in high-stakes domains such as criminal justice and healthcare, full automation is often not desirable due to safety, ethical, and legal concerns, yet fully manual approaches can be inaccurate and time-consuming. As a result, there is growing interest in the research community to augment human decision making with AI assistance. Besides developing AI technologies for this purpose, the emerging field of human-AI decision making must embrace empirical approaches to form a foundational understanding of how humans interact and work with AI to make decisions. To invite and help structure research efforts towards a science of understanding and improving human-AI decision making, we survey recent literature of empirical human-subject studies on this topic. We summarize the study design choices made in over 100 papers in three important aspects: (1) decision tasks, (2) AI assistance elements, and (3) evaluation metrics. For each aspect, we summarize current trends, discuss gaps in current practices of the field, and make a list of recommendations for future research. Our work highlights the need to develop common frameworks to account for the design and research spaces of human-AI decision making, so that researchers can make rigorous choices in study design, and the research community can build on each other\u2019s work and produce generalizable scientific knowledge. We also hope this work will serve as a bridge for HCI and AI communities to work together to mutually shape the empirical science and computational technologies for human-AI decision making."
++ },
++ "DOI:10.1145/3805760.3814924": {
++  "paperId": "c3b75af79917c7d268ec15fcfc022dbc8e3d9e13",
++  "externalIds": {
++   "DBLP": "conf/aiware/XueAEUW26",
++   "DOI": "10.1145/3805760.3814924",
++   "CorpusId": 289930236
++  },
++  "publicationVenue": null,
++  "title": "SWE-Bench+: Enhanced LLM Coding Benchmark",
++  "venue": "AIware",
++  "year": 2026,
++  "citationCount": 4,
++  "openAccessPdf": {
++   "url": "https://doi.org/10.1145/3805760.3814924",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1145/3805760.3814924?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/3805760.3814924, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Book"
++  ],
++  "journal": {
++   "name": "Proceedings of the 3rd ACM International Conference on AI-Powered Software"
++  },
++  "authors": [
++   {
++    "authorId": "2023500130",
++    "name": "Xu Haoran"
++   },
++   {
++    "authorId": "2115012783",
++    "name": "Reem Aleithan"
++   },
++   {
++    "authorId": "2389401231",
++    "name": "Nafid Enan"
++   },
++   {
++    "authorId": "2325093115",
++    "name": "Gias Uddin"
++   },
++   {
++    "authorId": "2257127006",
++    "name": "Song Wang"
++   }
++  ],
++  "abstract": "SWE-Bench is widely used to evaluate whether large language models can resolve real-world GitHub issues, yet the quality of its benchmark instances remains underexplored. We present SWE-Bench+, an enhanced benchmark framework that improves evaluation reliability by addressing two risks: solution leakage in issue descriptions and weak tests that allow plausible but incorrect patches to pass. We analyze 217 commonly resolved issues from SWE-Bench Lite and SWE-Bench Verified across three top-performing agents, yielding 651 model-generated patches. Our analysis identifies five recurring quality-problem patterns under solution leakage and weak tests. We further develop SoluLeakDetector, which detects solution-leaking content, and TestEnhancer, which strengthens validation tests. We find that 60.83% of commonly resolved issues contain solution leakage and 77.88% are problematic overall. SoluLeakDetector achieves 80.45% accuracy, while TestEnhancer identifies plausible patches for 97.11% of weak-test issues and reduces average resolution rates by 27.00 percentage points on Lite and 36.27 percentage points on Verified."
++ },
++ "DOI:10.1145/568522.568525": {
++  "paperId": "6f9633e70f7027a982fbe6d966b8ca1946212399",
++  "externalIds": {
++   "MAG": "2100970777",
++   "DBLP": "journals/csur/ElnozahyAWJ02",
++   "DOI": "10.1145/568522.568525",
++   "CorpusId": 221294544
++  },
++  "publicationVenue": null,
++  "title": "A survey of rollback-recovery protocols in message-passing systems",
++  "venue": "CSUR",
++  "year": 2002,
++  "citationCount": 1975,
++  "openAccessPdf": {
++   "url": "http://www.cs.utexas.edu/users/lorenzo/papers/SurveyFinal.pdf",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1145/568522.568525?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1145/568522.568525, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "ACM Comput. Surv.",
++   "pages": "375-408",
++   "volume": "34"
++  },
++  "authors": [
++   {
++    "authorId": "1727696",
++    "name": "E. Elnozahy"
++   },
++   {
++    "authorId": "2445753",
++    "name": "L. Alvisi"
++   },
++   {
++    "authorId": "1997552",
++    "name": "Yi-Min Wang"
++   },
++   {
++    "authorId": "2157813956",
++    "name": "David B. Johnson"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1162/coli_a_00486": {
++  "paperId": "07283e1578a50e15ac66efcc35b4ae0cbf2159ef",
++  "externalIds": {
++   "DBLP": "journals/corr/abs-2112-12870",
++   "ArXiv": "2112.12870",
++   "DOI": "10.1162/coli_a_00486",
++   "CorpusId": 245502761
++  },
++  "publicationVenue": {
++   "id": "ee37a78c-f3d8-407a-bd24-bb97fe6dbab9",
++   "name": "Computational Linguistics",
++   "type": "journal",
++   "alternate_names": [
++    "Comput Linguistics"
++   ],
++   "issn": "0891-2017",
++   "alternate_issns": [
++    "1530-9312",
++    "0362-613x",
++    "0362-613X"
++   ],
++   "url": "http://aclanthology.info/venues/cl",
++   "alternate_urls": [
++    "http://mitpress.mit.edu/catalog/item/default.asp?ttype=4&tid=10",
++    "https://www.mitpressjournals.org/loi/coli"
++   ]
++  },
++  "title": "Measuring Attribution in Natural Language Generation Models",
++  "venue": "Computational Linguistics",
++  "year": 2021,
++  "citationCount": 356,
++  "openAccessPdf": {
++   "url": "https://direct.mit.edu/coli/article-pdf/doi/10.1162/coli_a_00486/2133667/coli_a_00486.pdf",
++   "status": "GOLD",
++   "license": "CCBYNCND",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2112.12870, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Computational Linguistics",
++   "pages": "777-840",
++   "volume": "49"
++  },
++  "authors": [
++   {
++    "authorId": "2516777",
++    "name": "Hannah Rashkin"
++   },
++   {
++    "authorId": "48942032",
++    "name": "Vitaly Nikolaev"
++   },
++   {
++    "authorId": "48024953",
++    "name": "Matthew Lamm"
++   },
++   {
++    "authorId": "2257256357",
++    "name": "L. Aroyo"
++   },
++   {
++    "authorId": "123052390",
++    "name": "Michael Collins"
++   },
++   {
++    "authorId": "143790066",
++    "name": "Dipanjan Das"
++   },
++   {
++    "authorId": "1754497",
++    "name": "Slav Petrov"
++   },
++   {
++    "authorId": "32012022",
++    "name": "Gaurav Singh Tomar"
++   },
++   {
++    "authorId": "1388156275",
++    "name": "Iulia Turc"
++   },
++   {
++    "authorId": "1781409",
++    "name": "D. Reitter"
++   }
++  ],
++  "abstract": "Abstract Large neural models have brought a new challenge to natural language generation (NLG): It has become imperative to ensure the safety and reliability of the output of models that generate freely. To this end, we present an evaluation framework, Attributable to Identified Sources (AIS), stipulating that NLG output pertaining to the external world is to be verified against an independent, provided source. We define AIS and a two-stage annotation pipeline for allowing annotators to evaluate model output according to annotation guidelines. We successfully validate this approach on generation datasets spanning three tasks (two conversational QA datasets, a summarization dataset, and a table-to-text dataset). We provide full annotation guidelines in the appendices and publicly release the annotated data at https://github.com/google-research-datasets/AIS."
++ },
++ "DOI:10.1177/0956797611417632": {
++  "paperId": "49bda01e28f3f9fffa4fa14bd909c0a2340aa066",
++  "externalIds": {
++   "MAG": "2161498332",
++   "DOI": "10.1177/0956797611417632",
++   "CorpusId": 13802986,
++   "PubMed": "22006061"
++  },
++  "publicationVenue": {
++   "id": "a990ee76-444c-4bda-b714-df4dad0efa5d",
++   "name": "Psychology Science",
++   "type": "journal",
++   "alternate_names": [
++    "Psychol Sci",
++    "Psychological Science"
++   ],
++   "issn": "1614-9947",
++   "alternate_issns": [
++    "0956-7976"
++   ],
++   "url": "http://www.psychologie-aktuell.com/index.php?id=204",
++   "alternate_urls": [
++    "http://www.jstor.org/journals/09567976.html",
++    "http://www.jstor.org/action/showPublication?journalCode=psychsci",
++    "https://www.jstor.org/journal/psychsci",
++    "http://www.blackwellpublishers.co.uk/online",
++    "http://pss.sagepub.com/"
++   ]
++  },
++  "title": "False-Positive Psychology",
++  "venue": "Psychology Science",
++  "year": 2011,
++  "citationCount": 4563,
++  "openAccessPdf": {
++   "url": "http://journals.sagepub.com/doi/pdf/10.1177/0956797611417632",
++   "status": "BRONZE",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1177/0956797611417632?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1177/0956797611417632, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Psychological Science",
++   "pages": "1359 - 1366",
++   "volume": "22"
++  },
++  "authors": [
++   {
++    "authorId": "2118445",
++    "name": "J. Simmons"
++   },
++   {
++    "authorId": "33916832",
++    "name": "Leif D. Nelson"
++   },
++   {
++    "authorId": "2130902",
++    "name": "U. Simonsohn"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1177/1745691612463078": {
++  "paperId": "406a31a9a208c8bc6489720ce21ef542328d5d4b",
++  "externalIds": {
++   "MAG": "2123263696",
++   "DOI": "10.1177/1745691612463078",
++   "CorpusId": 5096417,
++   "PubMed": "26168122"
++  },
++  "publicationVenue": {
++   "id": "8ec18bc5-9c95-446c-a293-735d7ae1e3e9",
++   "name": "Perspectives on Psychological Science",
++   "type": "journal",
++   "alternate_names": [
++    "Perspect Psychol Sci"
++   ],
++   "issn": "1745-6916",
++   "url": "http://www.sagepub.com/journals/Journal201964/title",
++   "alternate_urls": [
++    "https://uk.sagepub.com/en-gb/eur/journal/perspectives-psychological-science",
++    "http://pps.sagepub.com/",
++    "https://www.jstor.org/journal/perspsycscie",
++    "http://www.jstor.org/action/showPublication?journalCode=perspsycscie"
++   ]
++  },
++  "title": "An Agenda for Purely Confirmatory Research",
++  "venue": "Perspectives on Psychological Science",
++  "year": 2012,
++  "citationCount": 987,
++  "openAccessPdf": {
++   "url": "",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1177/1745691612463078?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1177/1745691612463078, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Perspectives on Psychological Science",
++   "pages": "632 - 638",
++   "volume": "7"
++  },
++  "authors": [
++   {
++    "authorId": "2505272",
++    "name": "E. Wagenmakers"
++   },
++   {
++    "authorId": "2213973960",
++    "name": "Ruud Wetzels"
++   },
++   {
++    "authorId": "3237553",
++    "name": "D. Borsboom"
++   },
++   {
++    "authorId": "2482801",
++    "name": "H.L.J. van der Maas"
++   },
++   {
++    "authorId": "144707887",
++    "name": "R. Kievit"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1177/17456916231181102": {
++  "paperId": "492202b2502b9d76900370cddd38a4cdcd580897",
++  "externalIds": {
++   "PubMedCentral": "11373149",
++   "DOI": "10.1177/17456916231181102",
++   "CorpusId": 259843975,
++   "PubMed": "37439761"
++  },
++  "publicationVenue": {
++   "id": "8ec18bc5-9c95-446c-a293-735d7ae1e3e9",
++   "name": "Perspectives on Psychological Science",
++   "type": "journal",
++   "alternate_names": [
++    "Perspect Psychol Sci"
++   ],
++   "issn": "1745-6916",
++   "url": "http://www.sagepub.com/journals/Journal201964/title",
++   "alternate_urls": [
++    "https://uk.sagepub.com/en-gb/eur/journal/perspectives-psychological-science",
++    "http://pps.sagepub.com/",
++    "https://www.jstor.org/journal/perspsycscie",
++    "http://www.jstor.org/action/showPublication?journalCode=perspsycscie"
++   ]
++  },
++  "title": "Three Challenges for AI-Assisted Decision-Making",
++  "venue": "Perspectives on Psychological Science",
++  "year": 2023,
++  "citationCount": 224,
++  "openAccessPdf": {
++   "url": "https://journals.sagepub.com/doi/pdf/10.1177/17456916231181102",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC11373149, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Perspectives on Psychological Science",
++   "pages": "722 - 734",
++   "volume": "19"
++  },
++  "authors": [
++   {
++    "authorId": "1804885",
++    "name": "M. Steyvers"
++   },
++   {
++    "authorId": "2110096539",
++    "name": "Aakriti Kumar"
++   }
++  ],
++  "abstract": "Artificial intelligence (AI) has the potential to improve human decision-making by providing decision recommendations and problem-relevant information to assist human decision-makers. However, the full realization of the potential of human\u2013AI collaboration continues to face several challenges. First, the conditions that support complementarity (i.e., situations in which the performance of a human with AI assistance exceeds the performance of an unassisted human or the AI in isolation) must be understood. This task requires humans to be able to recognize situations in which the AI should be leveraged and to develop new AI systems that can learn to complement the human decision-maker. Second, human mental models of the AI, which contain both expectations of the AI and reliance strategies, must be accurately assessed. Third, the effects of different design choices for human-AI interaction must be understood, including both the timing of AI assistance and the amount of model information that should be presented to the human decision-maker to avoid cognitive overload and ineffective reliance strategies. In response to each of these three challenges, we present an interdisciplinary perspective based on recent empirical and theoretical findings and discuss new research directions."
++ },
++ "DOI:10.1177/25152459211007467": {
++  "paperId": "57eb25f55d349caf0ae61fbf10e74496fb2887d1",
++  "externalIds": {
++   "MAG": "3082857103",
++   "DOI": "10.1177/25152459211007467",
++   "CorpusId": 235080106
++  },
++  "publicationVenue": {
++   "id": "db890f0d-5d62-433f-af5a-f99e9c862fd2",
++   "name": "Advances in Methods and Practices in Psychological Science",
++   "alternate_names": [
++    "Adv Method Pract Psychol Sci"
++   ],
++   "issn": "2515-2459",
++   "url": "https://journals.sagepub.com/home/amp",
++   "alternate_urls": [
++    "https://journals.sagepub.com/loi/amp",
++    "https://www.psychologicalscience.org/publications/ampps"
++   ]
++  },
++  "title": "An Excess of Positive Results: Comparing the Standard Psychology Literature With Registered Reports",
++  "venue": "Advances in Methods and Practices in Psychological Science",
++  "year": 2020,
++  "citationCount": 356,
++  "openAccessPdf": {
++   "url": "https://doi.org/10.31234/osf.io/p6e9c",
++   "status": "GREEN",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1177/25152459211007467?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1177/25152459211007467, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Advances in Methods and Practices in Psychological Science",
++   "volume": "4"
++  },
++  "authors": [
++   {
++    "authorId": "152409539",
++    "name": "Anne M. Scheel"
++   },
++   {
++    "authorId": "115920485",
++    "name": "Mitchell Schijen"
++   },
++   {
++    "authorId": "87792747",
++    "name": "D. Lakens"
++   }
++  ],
++  "abstract": "Selectively publishing results that support the tested hypotheses (\u201cpositive\u201d results) distorts the available evidence for scientific claims. For the past decade, psychological scientists have been increasingly concerned about the degree of such distortion in their literature. A new publication format has been developed to prevent selective reporting: In Registered Reports (RRs), peer review and the decision to publish take place before results are known. We compared the results in published RRs (N = 71 as of November 2018) with a random sample of hypothesis-testing studies from the standard literature (N = 152) in psychology. Analyzing the first hypothesis of each article, we found 96% positive results in standard reports but only 44% positive results in RRs. We discuss possible explanations for this large difference and suggest that a plausible factor is the reduction of publication bias and/or Type I error inflation in the RR literature."
++ },
++ "DOI:10.1207/s15327957pspr0203_4": {
++  "paperId": "cdea0d7c29c5ab65dbee126ef9f2b6f28a5f0b94",
++  "externalIds": {
++   "MAG": "2118070418",
++   "DOI": "10.1207/s15327957pspr0203_4",
++   "CorpusId": 22724226,
++   "PubMed": "15647155"
++  },
++  "publicationVenue": {
++   "id": "5b750144-6d7e-45e3-9e2f-44263ea890a0",
++   "name": "Personality and Social Psychology Review",
++   "type": "journal",
++   "alternate_names": [
++    "Personal Soc Psychol Rev"
++   ],
++   "issn": "1532-7957",
++   "url": "https://journals.sagepub.com/loi/psr",
++   "alternate_urls": [
++    "http://psr.sagepub.com/"
++   ]
++  },
++  "title": "HARKing: Hypothesizing After the Results are Known",
++  "venue": "Personality and Social Psychology Review",
++  "year": 1998,
++  "citationCount": 1093,
++  "openAccessPdf": {
++   "url": "http://www2.psych.ubc.ca/~schaller/528Readings/Kerr1998.pdf",
++   "status": "BRONZE",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1207/s15327957pspr0203_4?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1207/s15327957pspr0203_4, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Personality and Social Psychology Review",
++   "pages": "196 - 217",
++   "volume": "2"
++  },
++  "authors": [
++   {
++    "authorId": "5255855",
++    "name": "Norbert L. Kerr"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.12688/f1000research.29032.2": null,
++ "DOI:10.12688/f1000research.51738.2": {
++  "paperId": "f7cf15d7591c98824a2c8e15c699787e530322ec",
++  "externalIds": {
++   "MAG": "3151240889",
++   "DBLP": "journals/f1000research/NustE21",
++   "PubMedCentral": "8311796",
++   "DOI": "10.12688/f1000research.51738.2",
++   "CorpusId": 233203273,
++   "PubMed": "34367614"
++  },
++  "publicationVenue": {
++   "id": "dc95b805-f39c-43dd-bf0f-500edd72da12",
++   "name": "F1000Research",
++   "type": "journal",
++   "issn": "2046-1402",
++   "url": "https://f1000research.com/"
++  },
++  "title": "CODECHECK: an Open Science initiative for the independent execution of computations underlying research articles during peer review to improve reproducibility",
++  "venue": "F1000Research",
++  "year": 2021,
++  "citationCount": 44,
++  "openAccessPdf": {
++   "url": "https://f1000research.com/articles/10-253/v2/pdf",
++   "status": "GOLD",
++   "license": "CCBYSA",
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC8311796, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "F1000Research",
++   "volume": "10"
++  },
++  "authors": [
++   {
++    "authorId": "2587120",
++    "name": "Daniel N\u00fcst"
++   },
++   {
++    "authorId": "2713377",
++    "name": "Stephen J. Eglen"
++   }
++  ],
++  "abstract": "The traditional scientific paper falls short of effectively communicating computational research. To help improve this situation, we propose a system by which the computational workflows underlying research articles are checked. The CODECHECK system uses open infrastructure and tools and can be integrated into review and publication processes in multiple ways. We describe these integrations along multiple dimensions (importance, who, openness, when). In collaboration with academic publishers and conferences, we demonstrate CODECHECK with 25 reproductions of diverse scientific publications. These CODECHECKs show that asking for reproducible workflows during a collaborative review can effectively improve executability. While CODECHECK has clear limitations, it may represent a building block in Open Science and publishing ecosystems for improving the reproducibility, appreciation, and, potentially, the quality of non-textual research artefacts. The CODECHECK website can be accessed here: https://codecheck.org.uk/."
++ },
++ "DOI:10.1287/isre.2021.1079": {
++  "paperId": "73c4417142eea24e290bf574de653fcc9604c591",
++  "externalIds": {
++   "DBLP": "journals/isr/FugenerGGK22",
++   "DOI": "10.1287/isre.2021.1079",
++   "CorpusId": 245062830
++  },
++  "publicationVenue": {
++   "id": "6b25cc59-3bb5-4b47-819e-d95ba292c4f7",
++   "name": "Information systems research",
++   "type": "journal",
++   "alternate_names": [
++    "Information Systems Research",
++    "Inf syst res",
++    "Inf Syst Res"
++   ],
++   "issn": "1047-7047",
++   "url": "http://isr.journal.informs.org/",
++   "alternate_urls": [
++    "http://www.jstor.org/action/showPublication?journalCode=infosysres",
++    "https://pubsonline.informs.org/",
++    "https://www.jstor.org/journal/infosysres"
++   ]
++  },
++  "title": "Cognitive Challenges in Human-Artificial Intelligence Collaboration: Investigating the Path Toward Productive Delegation",
++  "venue": "Information systems research",
++  "year": 2021,
++  "citationCount": 466,
++  "openAccessPdf": {
++   "url": "https://pure.eur.nl/en/publications/7ce28963-ab8a-44f8-a647-b98064150835",
++   "status": "GREEN",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.1287/isre.2021.1079?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1287/isre.2021.1079, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Inf. Syst. Res.",
++   "pages": "678-696",
++   "volume": "33"
++  },
++  "authors": [
++   {
++    "authorId": "2251595",
++    "name": "Andreas F\u00fcgener"
++   },
++   {
++    "authorId": "1697156",
++    "name": "J\u00f6rn Grahl"
++   },
++   {
++    "authorId": "2109979099",
++    "name": "Alok Gupta"
++   },
++   {
++    "authorId": "1772746",
++    "name": "W. Ketter"
++   }
++  ],
++  "abstract": "A consensus is beginning to emerge that the next phase of artificial intelligence (AI) induction in business organizations will require humans to work with AI in a variety of work arrangements. This article explores the issues related to human capabilities to work with AI. A key to working in many work arrangements is the ability to delegate work to entities that can do them most efficiently. Modern AI can do a remarkable job of efficient delegation to humans because it knows what it knows well and what it does not. Humans, on the other hand, are poor judges of their metaknowledge and are not good at delegating knowledge work to AI\u2014this might prove to be a big stumbling block to create work environments where humans and AI work together. Humans have often created machines to serve them. The sentiment is perhaps exemplified by Oscar Wilde\u2019s statement that \u201ccivilization requires slaves\u2026. Human slavery is wrong, insecure and demoralizing. On mechanical slavery, on the slavery of the machine, the future of the world depends.\u201d However, the time has come when humans might switch roles with machines. Our study highlights capabilities that humans need to effectively work with AI and still be in control rather than just being directed."
++ },
++ "DOI:10.1371/journal.pbio.1002106": {
++  "paperId": "a07a64ba110e0f9f7156f3bd1e376f0d2e1cddf1",
++  "externalIds": {
++   "MAG": "2164005910",
++   "PubMedCentral": "4359000",
++   "DOI": "10.1371/journal.pbio.1002106",
++   "CorpusId": 17475145,
++   "PubMed": "25768323"
++  },
++  "publicationVenue": {
++   "id": "83ff973b-8a0e-4e00-a06a-2cfd9e222de9",
++   "name": "PLoS Biology",
++   "type": "journal",
++   "alternate_names": [
++    "Plo Biology",
++    "PLOS Biology",
++    "PLO Biology"
++   ],
++   "issn": "1544-9173",
++   "url": "http://www.pubmedcentral.nih.gov/tocrender.fcgi?journal=212",
++   "alternate_urls": [
++    "http://www.plosbiology.org/",
++    "https://journals.plos.org/plosbiology/"
++   ]
++  },
++  "title": "The Extent and Consequences of P-Hacking in Science",
++  "venue": "PLoS Biology",
++  "year": 2015,
++  "citationCount": 1156,
++  "openAccessPdf": {
++   "url": "https://journals.plos.org/plosbiology/article/file?id=10.1371/journal.pbio.1002106&type=printable",
++   "status": "GOLD",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC4359000, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "PLoS Biology",
++   "volume": "13"
++  },
++  "authors": [
++   {
++    "authorId": "6652932",
++    "name": "M. Head"
++   },
++   {
++    "authorId": "3884621",
++    "name": "L. Holman"
++   },
++   {
++    "authorId": "5325261",
++    "name": "R. Lanfear"
++   },
++   {
++    "authorId": "12670573",
++    "name": "Andrew T. Kahn"
++   },
++   {
++    "authorId": "3495787",
++    "name": "M. Jennions"
++   }
++  ],
++  "abstract": "A focus on novel, confirmatory, and statistically significant results leads to substantial bias in the scientific literature. One type of bias, known as \u201cp-hacking,\u201d occurs when researchers collect or select data or statistical analyses until nonsignificant results become significant. Here, we use text-mining to demonstrate that p-hacking is widespread throughout science. We then illustrate how one can test for p-hacking when performing a meta-analysis and show that, while p-hacking is probably common, its effect seems to be weak relative to the real effect sizes being measured. This result suggests that p-hacking probably does not drastically alter scientific consensuses drawn from meta-analyses."
++ },
++ "DOI:10.1371/journal.pcbi.1003285": {
++  "paperId": "12b056fd90c1300c44c4a8cff9f01f0c7a75cab4",
++  "externalIds": {
++   "DBLP": "journals/ploscb/SandveNTH13",
++   "MAG": "2036318837",
++   "PubMedCentral": "3812051",
++   "DOI": "10.1371/journal.pcbi.1003285",
++   "CorpusId": 2180281,
++   "PubMed": "24204232"
++  },
++  "publicationVenue": null,
++  "title": "Ten Simple Rules for Reproducible Computational Research",
++  "venue": "PLoS Comput. Biol.",
++  "year": 2013,
++  "citationCount": 919,
++  "openAccessPdf": {
++   "url": "https://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1003285&type=printable",
++   "status": "GOLD",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC3812051, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "PLoS Computational Biology",
++   "volume": "9"
++  },
++  "authors": [
++   {
++    "authorId": "49687105",
++    "name": "G. K. Sandve"
++   },
++   {
++    "authorId": "2686089",
++    "name": "Anton Nekrutenko"
++   },
++   {
++    "authorId": "31436732",
++    "name": "James Taylor"
++   },
++   {
++    "authorId": "1925617",
++    "name": "E. Hovig"
++   }
++  ],
++  "abstract": "Replication is the cornerstone of a cumulative science [1]. However, new tools and technologies, massive amounts of data, interdisciplinary approaches, and the complexity of the questions being asked are complicating replication efforts, as are increased pressures on scientists to advance their research [2]. As full replication of studies on independently collected data is often not feasible, there has recently been a call for reproducible research as an attainable minimum standard for assessing the value of scientific claims [3]. This requires that papers in experimental science describe the results and provide a sufficiently clear protocol to allow successful repetition and extension of analyses based on original data [4]. \n \nThe importance of replication and reproducibility has recently been exemplified through studies showing that scientific papers commonly leave out experimental details essential for reproduction [5], studies showing difficulties with replicating published experimental results [6], an increase in retracted papers [7], and through a high number of failing clinical trials [8], [9]. This has led to discussions on how individual researchers, institutions, funding bodies, and journals can establish routines that increase transparency and reproducibility. In order to foster such aspects, it has been suggested that the scientific community needs to develop a \u201cculture of reproducibility\u201d for computational science, and to require it for published claims [3]. \n \nWe want to emphasize that reproducibility is not only a moral responsibility with respect to the scientific field, but that a lack of reproducibility can also be a burden for you as an individual researcher. As an example, a good practice of reproducibility is necessary in order to allow previously developed methodology to be effectively applied on new data, or to allow reuse of code and results for new projects. In other words, good habits of reproducibility may actually turn out to be a time-saver in the longer run. \n \nWe further note that reproducibility is just as much about the habits that ensure reproducible research as the technologies that can make these processes efficient and realistic. Each of the following ten rules captures a specific aspect of reproducibility, and discusses what is needed in terms of information handling and tracking of procedures. If you are taking a bare-bones approach to bioinformatics analysis, i.e., running various custom scripts from the command line, you will probably need to handle each rule explicitly. If you are instead performing your analyses through an integrated framework (such as GenePattern [10], Galaxy [11], LONI pipeline [12], or Taverna [13]), the system may already provide full or partial support for most of the rules. What is needed on your part is then merely the knowledge of how to exploit these existing possibilities. \n \nIn a pragmatic setting, with publication pressure and deadlines, one may face the need to make a trade-off between the ideals of reproducibility and the need to get the research out while it is still relevant. This trade-off becomes more important when considering that a large part of the analyses being tried out never end up yielding any results. However, frequently one will, with the wisdom of hindsight, contemplate the missed opportunity to ensure reproducibility, as it may already be too late to take the necessary notes from memory (or at least much more difficult than to do it while underway). We believe that the rewards of reproducibility will compensate for the risk of having spent valuable time developing an annotated catalog of analyses that turned out as blind alleys. \n \nAs a minimal requirement, you should at least be able to reproduce the results yourself. This would satisfy the most basic requirements of sound research, allowing any substantial future questioning of the research to be met with a precise explanation. Although it may sound like a very weak requirement, even this level of reproducibility will often require a certain level of care in order to be met. There will for a given analysis be an exponential number of possible combinations of software versions, parameter values, pre-processing steps, and so on, meaning that a failure to take notes may make exact reproduction essentially impossible. \n \nWith this basic level of reproducibility in place, there is much more that can be wished for. An obvious extension is to go from a level where you can reproduce results in case of a critical situation to a level where you can practically and routinely reuse your previous work and increase your productivity. A second extension is to ensure that peers have a practical possibility of reproducing your results, which can lead to increased trust in, interest for, and citations of your work [6], [14]. \n \nWe here present ten simple rules for reproducibility of computational research. These rules can be at your disposal for whenever you want to make your research more accessible\u2014be it for peers or for your future self."
++ },
++ "DOI:10.1371/journal.pcbi.1008316": {
++  "paperId": "317b30303d162ba11f450183bc9c19587bc5bd0e",
++  "externalIds": {
++   "PubMedCentral": "7654784",
++   "DBLP": "journals/ploscb/NustSMEHHE20",
++   "MAG": "3102028509",
++   "DOI": "10.1371/journal.pcbi.1008316",
++   "CorpusId": 226303277,
++   "PubMed": "33170857"
++  },
++  "publicationVenue": null,
++  "title": "Ten simple rules for writing Dockerfiles for reproducible data science",
++  "venue": "PLoS Comput. Biol.",
++  "year": 2020,
++  "citationCount": 85,
++  "openAccessPdf": {
++   "url": "https://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1008316&type=printable",
++   "status": "GOLD",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC7654784, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Editorial",
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "PLoS Computational Biology",
++   "volume": "16"
++  },
++  "authors": [
++   {
++    "authorId": "2587120",
++    "name": "Daniel N\u00fcst"
++   },
++   {
++    "authorId": "2978615",
++    "name": "V. Sochat"
++   },
++   {
++    "authorId": "8755947",
++    "name": "B. Marwick"
++   },
++   {
++    "authorId": "2713377",
++    "name": "Stephen J. Eglen"
++   },
++   {
++    "authorId": "2169785071",
++    "name": "T. Head"
++   },
++   {
++    "authorId": "5754713",
++    "name": "T. Hirst"
++   },
++   {
++    "authorId": "2055340502",
++    "name": "Benjamin D. Evans"
++   }
++  ],
++  "abstract": "Computational science has been greatly improved by the use of containers for packaging software and data dependencies. In a scholarly context, the main drivers for using these containers are transparency and support of reproducibility; in turn, a workflow\u2019s reproducibility can be greatly affected by the choices that are made with respect to building containers. In many cases, the build process for the container\u2019s image is created from instructions provided in a Dockerfile format. In support of this approach, we present a set of rules to help researchers write understandable Dockerfiles for typical data science workflows. By following the rules in this article, researchers can create containers suitable for sharing with fellow scientists, for including in scholarly communication such as education or scientific papers, and for effective and sustainable personal workflows."
++ },
++ "DOI:10.1371/journal.pmed.0020124": null,
++ "DOI:10.14778/3529337.3529344": null,
++ "DOI:10.1511/2014.111.460": {
++  "paperId": "822447fab9e8bf5b7b58e5380527a5a746de5694",
++  "externalIds": {
++   "MAG": "2322006099",
++   "DOI": "10.1511/2014.111.460",
++   "CorpusId": 146855235
++  },
++  "publicationVenue": null,
++  "title": "The statistical crisis in science",
++  "venue": "",
++  "year": 2014,
++  "citationCount": 856,
++  "openAccessPdf": {
++   "url": "",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1511/2014.111.460?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1511/2014.111.460, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": null,
++  "journal": {
++   "name": "American Scientist",
++   "pages": "460",
++   "volume": "102"
++  },
++  "authors": [
++   {
++    "authorId": "144389145",
++    "name": "A. Gelman"
++   },
++   {
++    "authorId": "145092174",
++    "name": "E. Loken"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1518/001872097778543886": {
++  "paperId": "1cf890ce8391ca958fbc173533d575efcb51f70a",
++  "externalIds": {
++   "MAG": "2063052894",
++   "DBLP": "journals/hf/ParasuramanR97",
++   "DOI": "10.1518/001872097778543886",
++   "CorpusId": 41149078
++  },
++  "publicationVenue": null,
++  "title": "Humans and Automation: Use, Misuse, Disuse, Abuse",
++  "venue": "Hum. Factors",
++  "year": 1997,
++  "citationCount": 4879,
++  "openAccessPdf": {
++   "url": "",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1518/001872097778543886?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1518/001872097778543886, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Human Factors: The Journal of Human Factors and Ergonomics Society",
++   "pages": "230 - 253",
++   "volume": "39"
++  },
++  "authors": [
++   {
++    "authorId": "3264674",
++    "name": "R. Parasuraman"
++   },
++   {
++    "authorId": "50615483",
++    "name": "V. Riley"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.1518/hfes.46.1.50_30392": {
++  "paperId": "7dd86508438657ac7a704a5d952a2a4422808975",
++  "externalIds": {
++   "DBLP": "journals/hf/LeeS04",
++   "MAG": "2110171129",
++   "DOI": "10.1518/hfes.46.1.50_30392",
++   "CorpusId": 5210390,
++   "PubMed": "15151155"
++  },
++  "publicationVenue": null,
++  "title": "Trust in Automation: Designing for Appropriate Reliance",
++  "venue": "Hum. Factors",
++  "year": 2004,
++  "citationCount": 6989,
++  "openAccessPdf": {
++   "url": "",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: The following paper fields have been elided by the publisher: {'abstract'}. Paper or abstract available at https://api.unpaywall.org/v2/10.1518/hfes.46.1.50_30392?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.1518/hfes.46.1.50_30392, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Review",
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "Human Factors: The Journal of Human Factors and Ergonomics Society",
++   "pages": "50 - 80",
++   "volume": "46"
++  },
++  "authors": [
++   {
++    "authorId": "1520031837",
++    "name": "John D. Lee"
++   },
++   {
++    "authorId": "49748104",
++    "name": "Katrina A. See"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.17226/25303": {
++  "paperId": "655be23c570656c5fcee09c1a7e0ccabc12be1fb",
++  "externalIds": {
++   "MAG": "2944196828",
++   "DOI": "10.17226/25303",
++   "CorpusId": 276157516
++  },
++  "publicationVenue": null,
++  "title": "Reproducibility and Replicability in Science",
++  "venue": "",
++  "year": 2019,
++  "citationCount": 73,
++  "openAccessPdf": {
++   "url": "",
++   "status": "CLOSED",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.17226/25303?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.17226/25303, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": null,
++  "journal": {
++   "name": "",
++   "volume": ""
++  },
++  "authors": [
++   {
++    "authorId": "83862054",
++    "name": "Division on Earth"
++   },
++   {
++    "authorId": "71242706",
++    "name": "C. Behavioral"
++   }
++  ],
++  "abstract": "Jennifer Heimberg, Study Director, Senior Program Officer, Nuclear and Radiation Studies Board, Division on Earth and Life Studies Adrienne Stith Butler, Associate Director, Board on Behavioral, Cognitive, and Sensory Sciences, Division of Behavioral and Social Sciences and Education Tom Arrison, Board/Program Director, Policy and Global Affairs Michael Cohen, Senior Program Officer, Committee on National Statistics, Division of Behavioral and Social Sciences and Education Michelle Schwalbe, Director, Board on Mathematical Sciences and Analytics, Division on Engineering and Physical Sciences Tina Winters, Associate Program Officer, Board on Behavioral, Cognitive, and Sensory Sciences, Division of Behavioral and Social Sciences and Education Study Committee About Consensus Studies Statement of Task"
++ },
++ "DOI:10.18653/v1/2020.emnlp-main.609": {
++  "paperId": "b770d84055c32febe922be9931c453fdbebe9002",
++  "externalIds": {
++   "MAG": "3023035014",
++   "ArXiv": "2004.14974",
++   "DBLP": "conf/emnlp/WaddenLLWZCH20",
++   "ACL": "2020.emnlp-main.609",
++   "DOI": "10.18653/v1/2020.emnlp-main.609",
++   "CorpusId": 216867133
++  },
++  "publicationVenue": {
++   "id": "41bf9ed3-85b3-4c90-b015-150e31690253",
++   "name": "Conference on Empirical Methods in Natural Language Processing",
++   "type": "conference",
++   "alternate_names": [
++    "Empir Method Nat Lang Process",
++    "Empirical Methods in Natural Language Processing",
++    "Conf Empir Method Nat Lang Process",
++    "EMNLP"
++   ],
++   "url": "https://www.aclweb.org/portal/emnlp"
++  },
++  "title": "Fact or Fiction: Verifying Scientific Claims",
++  "venue": "Conference on Empirical Methods in Natural Language Processing",
++  "year": 2020,
++  "citationCount": 912,
++  "openAccessPdf": {
++   "url": "https://www.aclweb.org/anthology/2020.emnlp-main.609.pdf",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2004.14974, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "name": "ArXiv",
++   "volume": "abs/2004.14974"
++  },
++  "authors": [
++   {
++    "authorId": "30051202",
++    "name": "David Wadden"
++   },
++   {
++    "authorId": "32370203",
++    "name": "Shanchuan Lin"
++   },
++   {
++    "authorId": "46258841",
++    "name": "Kyle Lo"
++   },
++   {
++    "authorId": "31860505",
++    "name": "Lucy Lu Wang"
++   },
++   {
++    "authorId": "15292561",
++    "name": "Madeleine van Zuylen"
++   },
++   {
++    "authorId": "2527954",
++    "name": "Arman Cohan"
++   },
++   {
++    "authorId": "2548384",
++    "name": "Hannaneh Hajishirzi"
++   }
++  ],
++  "abstract": "We introduce scientific claim verification, a new task to select abstracts from the research literature containing evidence that supports or refutes a given scientific claim, and to identify rationales justifying each decision. To study this task, we construct SciFact, a dataset of 1.4K expert-written scientific claims paired with evidence-containing abstracts annotated with labels and rationales. We develop baseline models for SciFact, and demonstrate that these models benefit from combined training on a large dataset of claims about Wikipedia articles, together with the new SciFact data. We show that our claim verification system is able to identify plausible evidence for 23 / 36 claims relevant to COVID-19 on the CORD-19 corpus. Our results and experiments strongly suggest that our new task and data will support significant future research efforts."
++ },
++ "DOI:10.18653/v1/2022.acl-short.18": null,
++ "DOI:10.18653/v1/2023.acl-long.910": {
++  "paperId": "66242baf48b0f6b828e7547ac39ffaa5e1b2cb3e",
++  "externalIds": {
++   "ArXiv": "2210.08726",
++   "DBLP": "conf/acl/GaoDPCCFZLLJG23",
++   "ACL": "2023.acl-long.910",
++   "DOI": "10.18653/v1/2023.acl-long.910",
++   "CorpusId": 254247260
++  },
++  "publicationVenue": {
++   "id": "1e33b3be-b2ab-46e9-96e8-d4eb4bad6e44",
++   "name": "Annual Meeting of the Association for Computational Linguistics",
++   "type": "conference",
++   "alternate_names": [
++    "Annu Meet Assoc Comput Linguistics",
++    "Meeting of the Association for Computational Linguistics",
++    "ACL",
++    "Meet Assoc Comput Linguistics"
++   ],
++   "url": "https://www.aclweb.org/anthology/venues/acl/"
++  },
++  "title": "RARR: Researching and Revising What Language Models Say, Using Language Models",
++  "venue": "Annual Meeting of the Association for Computational Linguistics",
++  "year": 2022,
++  "citationCount": 407,
++  "openAccessPdf": {
++   "url": "https://aclanthology.org/2023.acl-long.910.pdf",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/2210.08726, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "pages": "16477-16508"
++  },
++  "authors": [
++   {
++    "authorId": "49715441",
++    "name": "Luyu Gao"
++   },
++   {
++    "authorId": "2475437",
++    "name": "Zhuyun Dai"
++   },
++   {
++    "authorId": "2616463",
++    "name": "Panupong Pasupat"
++   },
++   {
++    "authorId": "52151521",
++    "name": "Anthony Chen"
++   },
++   {
++    "authorId": "2719924",
++    "name": "Arun Tejasvi Chaganty"
++   },
++   {
++    "authorId": "2051969653",
++    "name": "Yicheng Fan"
++   },
++   {
++    "authorId": "2664737",
++    "name": "Vincent Zhao"
++   },
++   {
++    "authorId": "1914797",
++    "name": "N. Lao"
++   },
++   {
++    "authorId": "8386466",
++    "name": "Hongrae Lee"
++   },
++   {
++    "authorId": "50270386",
++    "name": "Da-Cheng Juan"
++   },
++   {
++    "authorId": "2091768",
++    "name": "Kelvin Guu"
++   }
++  ],
++  "abstract": "Language models (LMs) now excel at many tasks such as question answering, reasoning, and dialog. However, they sometimes generate unsupported or misleading content. A user cannot easily determine whether their outputs are trustworthy or not, because most LMs do not have any built-in mechanism for attribution to external evidence. To enable attribution while still preserving all the powerful advantages of recent generation models, we propose RARR (Retrofit Attribution using Research and Revision), a system that 1) automatically finds attribution for the output of any text generation model, and 2) post-edits the output to fix unsupported content while preserving the original output as much as possible. When applied to the output of several state-of-the-art LMs on a diverse set of generation tasks, we find that RARR significantly improves attribution while otherwise preserving the original input to a much greater degree than previously explored edit models. Furthermore, the implementation of RARR requires only a handful of training examples, a large language model, and standard web search."
++ },
++ "DOI:10.18653/v1/2023.emnlp-main.398": null,
++ "DOI:10.18653/v1/2023.emnlp-main.741": null,
++ "DOI:10.18653/v1/2023.findings-emnlp.722": null,
++ "DOI:10.18653/v1/N18-1074": {
++  "paperId": "b1d24e8e08435b7c52335485a0d635abf9bc604c",
++  "externalIds": {
++   "MAG": "2789566302",
++   "ACL": "N18-1074",
++   "ArXiv": "1803.05355",
++   "DBLP": "conf/naacl/ThorneVCM18",
++   "DOI": "10.18653/v1/N18-1074",
++   "CorpusId": 4711425
++  },
++  "publicationVenue": {
++   "id": "01103732-3808-4930-b8e4-7e9e68d5c68d",
++   "name": "North American Chapter of the Association for Computational Linguistics",
++   "type": "conference",
++   "alternate_names": [
++    "North Am Chapter Assoc Comput Linguistics",
++    "NAACL"
++   ],
++   "url": "https://www.aclweb.org/portal/naacl"
++  },
++  "title": "FEVER: a Large-scale Dataset for Fact Extraction and VERification",
++  "venue": "North American Chapter of the Association for Computational Linguistics",
++  "year": 2018,
++  "citationCount": 2590,
++  "openAccessPdf": {
++   "url": "https://www.aclweb.org/anthology/N18-1074.pdf",
++   "status": "HYBRID",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://arxiv.org/abs/1803.05355, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Conference"
++  ],
++  "journal": {
++   "pages": "809-819"
++  },
++  "authors": [
++   {
++    "authorId": "144603330",
++    "name": "James Thorne"
++   },
++   {
++    "authorId": "2064056928",
++    "name": "Andreas Vlachos"
++   },
++   {
++    "authorId": "2718039",
++    "name": "Christos Christodoulopoulos"
++   },
++   {
++    "authorId": "2008596",
++    "name": "Arpit Mittal"
++   }
++  ],
++  "abstract": "In this paper we introduce a new publicly available dataset for verification against textual sources, FEVER: Fact Extraction and VERification. It consists of 185,445 claims generated by altering sentences extracted from Wikipedia and subsequently verified without knowledge of the sentence they were derived from. The claims are classified as Supported, Refuted or NotEnoughInfo by annotators achieving 0.6841 in Fleiss kappa. For the first two classes, the annotators also recorded the sentence(s) forming the necessary evidence for their judgment. To characterize the challenge of the dataset presented, we develop a pipeline approach and compare it to suitably designed oracles. The best accuracy we achieve on labeling a claim accompanied by the correct evidence is 31.87%, while if we ignore the evidence we achieve 50.91%. Thus we believe that FEVER is a challenging testbed that will help stimulate progress on claim verification against textual sources."
++ },
++ "DOI:10.2139/ssrn.4573321": {
++  "paperId": "409dc9f1801947f85c53231cba346f0e314cb208",
++  "externalIds": {
++   "DOI": "10.2139/ssrn.4573321",
++   "CorpusId": 262199478
++  },
++  "publicationVenue": {
++   "id": "75d7a8c1-d871-42db-a8e4-7cf5146fdb62",
++   "name": "Social Science Research Network",
++   "type": "journal",
++   "alternate_names": [
++    "SSRN, Social Science Research Network (SSRN) home page",
++    "SSRN Electronic Journal",
++    "Soc Sci Res Netw",
++    "SSRN",
++    "SSRN Home Page",
++    "SSRN Electron J",
++    "Social Science Electronic Publishing presents Social Science Research Network"
++   ],
++   "issn": "1556-5068",
++   "url": "http://www.ssrn.com/",
++   "alternate_urls": [
++    "www.ssrn.com/",
++    "https://fatcat.wiki/container/tol7woxlqjeg5bmzadeg6qrg3e",
++    "https://www.wikidata.org/wiki/Q53949192",
++    "www.ssrn.com/en",
++    "http://www.ssrn.com/en/",
++    "http://umlib.nl/ssrn",
++    "umlib.nl/ssrn"
++   ]
++  },
++  "title": "Navigating the Jagged Technological Frontier: Field Experimental Evidence of the Effects of AI on Knowledge Worker Productivity and Quality",
++  "venue": "Social Science Research Network",
++  "year": 2023,
++  "citationCount": 804,
++  "openAccessPdf": {
++   "url": "https://doi.org/10.2139/ssrn.4573321",
++   "status": "GREEN",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://api.unpaywall.org/v2/10.2139/ssrn.4573321?email=<INSERT_YOUR_EMAIL> or https://doi.org/10.2139/ssrn.4573321, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle"
++  ],
++  "journal": {
++   "name": "SSRN Electronic Journal"
++  },
++  "authors": [
++   {
++    "authorId": "2244496752",
++    "name": "Fabrizio Dell'Acqua"
++   },
++   {
++    "authorId": "2244497063",
++    "name": "Edward McFowland"
++   },
++   {
++    "authorId": "1722774",
++    "name": "Ethan Mollick"
++   },
++   {
++    "authorId": "1397350576",
++    "name": "Hila Lifshitz-Assaf"
++   },
++   {
++    "authorId": "2244502347",
++    "name": "Katherine C. Kellogg"
++   },
++   {
++    "authorId": "2244358073",
++    "name": "Saran Rajendran"
++   },
++   {
++    "authorId": "2244506815",
++    "name": "Lisa Krayer"
++   },
++   {
++    "authorId": "2244506911",
++    "name": "F. Candelon"
++   },
++   {
++    "authorId": "2505610",
++    "name": "Karim R. Lakhani"
++   }
++  ],
++  "abstract": null
++ },
++ "DOI:10.2196/53164": {
++  "paperId": "9a0e8c101b72e9094ea554194698daa3aeb3bc6f",
++  "externalIds": {
++   "PubMedCentral": "11153973",
++   "DOI": "10.2196/53164",
++   "CorpusId": 269949485,
++   "PubMed": "38776130"
++  },
++  "publicationVenue": {
++   "id": "2baad992-2268-4c38-9120-e453622f2eeb",
++   "name": "Journal of Medical Internet Research",
++   "type": "journal",
++   "alternate_names": [
++    "J Med Internet Res"
++   ],
++   "issn": "1438-8871",
++   "url": "http://www.symposion.com/jmir/index.htm",
++   "alternate_urls": [
++    "http://www.jmir.org/",
++    "https://www.jmir.org/"
++   ]
++  },
++  "title": "Hallucination Rates and Reference Accuracy of ChatGPT and Bard for Systematic Reviews: Comparative Analysis",
++  "venue": "Journal of Medical Internet Research",
++  "year": 2024,
++  "citationCount": 475,
++  "openAccessPdf": {
++   "url": "https://doi.org/10.2196/53164",
++   "status": "GOLD",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC11153973, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Study",
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Journal of Medical Internet Research",
++   "volume": "26"
++  },
++  "authors": [
++   {
++    "authorId": "2287298900",
++    "name": "Micha\u00ebl Chelli"
++   },
++   {
++    "authorId": "123164546",
++    "name": "J. Descamps"
++   },
++   {
++    "authorId": "2278002609",
++    "name": "V. Lavou\u00e9"
++   },
++   {
++    "authorId": "4193798",
++    "name": "C. Trojani"
++   },
++   {
++    "authorId": "2065302847",
++    "name": "Michel Azar"
++   },
++   {
++    "authorId": "2302548215",
++    "name": "M. Deckert"
++   },
++   {
++    "authorId": "27673403",
++    "name": "J. Raynier"
++   },
++   {
++    "authorId": "17722605",
++    "name": "Gilles Clowez"
++   },
++   {
++    "authorId": "2279472796",
++    "name": "Pascal Boileau"
++   },
++   {
++    "authorId": "2058782562",
++    "name": "C. Ruetsch-Chelli"
++   }
++  ],
++  "abstract": "Background Large language models (LLMs) have raised both interest and concern in the academic community. They offer the potential for automating literature search and synthesis for systematic reviews but raise concerns regarding their reliability, as the tendency to generate unsupported (hallucinated) content persist. Objective The aim of the study is to assess the performance of LLMs such as ChatGPT and Bard (subsequently rebranded Gemini) to produce references in the context of scientific writing. Methods The performance of ChatGPT and Bard in replicating the results of human-conducted systematic reviews was assessed. Using systematic reviews pertaining to shoulder rotator cuff pathology, these LLMs were tested by providing the same inclusion criteria and comparing the results with original systematic review references, serving as gold standards. The study used 3 key performance metrics: recall, precision, and F1-score, alongside the hallucination rate. Papers were considered \u201challucinated\u201d if any 2 of the following information were wrong: title, first author, or year of publication. Results In total, 11 systematic reviews across 4 fields yielded 33 prompts to LLMs (3 LLMs\u00d711 reviews), with 471 references analyzed. Precision rates for GPT-3.5, GPT-4, and Bard were 9.4% (13/139), 13.4% (16/119), and 0% (0/104) respectively (P<.001). Recall rates were 11.9% (13/109) for GPT-3.5 and 13.7% (15/109) for GPT-4, with Bard failing to retrieve any relevant papers (P<.001). Hallucination rates stood at 39.6% (55/139) for GPT-3.5, 28.6% (34/119) for GPT-4, and 91.4% (95/104) for Bard (P<.001). Further analysis of nonhallucinated papers retrieved by GPT models revealed significant differences in identifying various criteria, such as randomized studies, participant criteria, and intervention criteria. The study also noted the geographical and open-access biases in the papers retrieved by the LLMs. Conclusions Given their current performance, it is not recommended for LLMs to be deployed as the primary or exclusive tool for conducting systematic reviews. Any references generated by such models warrant thorough validation by researchers. The high occurrence of hallucinations in LLMs highlights the necessity for refining their training and functionality before confidently using them for rigorous academic purposes."
++ },
++ "DOI:10.3389/fpsyg.2016.01832": {
++  "paperId": "5e8902d504c03c7513f29ade98931189165a8a62",
++  "externalIds": {
++   "MAG": "2547503455",
++   "PubMedCentral": "5122713",
++   "DOI": "10.3389/fpsyg.2016.01832",
++   "CorpusId": 12942653,
++   "PubMed": "27933012"
++  },
++  "publicationVenue": {
++   "id": "89097a03-8be6-4e2d-ae2c-a6df64c77a06",
++   "name": "Frontiers in Psychology",
++   "type": "journal",
++   "alternate_names": [
++    "Front Psychol"
++   ],
++   "issn": "1664-1078",
++   "url": "http://www.frontiersin.org/Cultural_Psychology",
++   "alternate_urls": [
++    "http://frontiersin.org/psychology/",
++    "https://www.frontiersin.org/journals/psychology",
++    "http://journal.frontiersin.org/journal/psychology",
++    "http://www.frontiersin.org/psychology"
++   ]
++  },
++  "title": "Degrees of Freedom in Planning, Running, Analyzing, and Reporting Psychological Studies: A Checklist to Avoid p-Hacking",
++  "venue": "Frontiers in Psychology",
++  "year": 2016,
++  "citationCount": 632,
++  "openAccessPdf": {
++   "url": "https://www.frontiersin.org/articles/10.3389/fpsyg.2016.01832/pdf",
++   "status": "GOLD",
++   "license": "CCBY",
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC5122713, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "JournalArticle",
++   "Review"
++  ],
++  "journal": {
++   "name": "Frontiers in Psychology",
++   "volume": "7"
++  },
++  "authors": [
++   {
++    "authorId": "2748685",
++    "name": "J. Wicherts"
++   },
++   {
++    "authorId": "3642852",
++    "name": "C. L. S. Veldkamp"
++   },
++   {
++    "authorId": "8505119",
++    "name": "H. Augusteijn"
++   },
++   {
++    "authorId": "40021672",
++    "name": "Marjan Bakker"
++   },
++   {
++    "authorId": "5646115",
++    "name": "R. V. van Aert"
++   },
++   {
++    "authorId": "1707089",
++    "name": "M. van Assen"
++   }
++  ],
++  "abstract": "The designing, collecting, analyzing, and reporting of psychological studies entail many choices that are often arbitrary. The opportunistic use of these so-called researcher degrees of freedom aimed at obtaining statistically significant results is problematic because it enhances the chances of false positive results and may inflate effect size estimates. In this review article, we present an extensive list of 34 degrees of freedom that researchers have in formulating hypotheses, and in designing, running, analyzing, and reporting of psychological research. The list can be used in research methods education, and as a checklist to assess the quality of preregistrations and to determine the potential for bias due to (arbitrary) choices in unregistered studies."
++ },
++ "DOI:10.7759/cureus.35179": {
++  "paperId": "7be169a70f5db74f40adfd2790021aa3fbf3cb87",
++  "externalIds": {
++   "PubMedCentral": "9939079",
++   "DOI": "10.7759/cureus.35179",
++   "CorpusId": 257037938,
++   "PubMed": "36811129"
++  },
++  "publicationVenue": {
++   "id": "87c855fc-c59a-4443-832a-e5b1ca14b9f3",
++   "name": "Cureus",
++   "type": "journal",
++   "issn": "2168-8184",
++   "url": "https://www.cureus.com/"
++  },
++  "title": "Artificial Hallucinations in ChatGPT: Implications in Scientific Writing",
++  "venue": "Cureus",
++  "year": 2023,
++  "citationCount": 1603,
++  "openAccessPdf": {
++   "url": "https://assets.cureus.com/uploads/editorial/pdf/138667/20230219-28928-6kcyip.pdf",
++   "status": "GOLD",
++   "license": null,
++   "disclaimer": "Notice: Paper or abstract available at https://pmc.ncbi.nlm.nih.gov/articles/PMC9939079, which is subject to the license by the author or copyright owner provided with this content. Please go to the source to verify the license and copyright information for your use."
++  },
++  "publicationTypes": [
++   "Editorial"
++  ],
++  "journal": {
++   "name": "Cureus",
++   "volume": "15"
++  },
++  "authors": [
++   {
++    "authorId": "1412759796",
++    "name": "H. Alkaissi"
++   },
++   {
++    "authorId": "81301990",
++    "name": "Samy I McFarlane"
++   }
++  ],
++  "abstract": "While still in its infancy, ChatGPT (Generative Pretrained Transformer), introduced in November 2022, is bound to hugely impact many industries, including healthcare, medical education, biomedical research, and scientific writing. Implications of ChatGPT, that new chatbot introduced by OpenAI on academic writing, is largely unknown. In response to the Journal of Medical Science (Cureus) Turing Test - call for case reports written with the assistance of ChatGPT, we present two cases one of homocystinuria-associated osteoporosis, and the other is on late-onset Pompe disease (LOPD), a rare metabolic disorder. We tested ChatGPT to write about the pathogenesis of these conditions. We documented the positive, negative, and rather troubling aspects of our newly introduced chatbot\u2019s performance."
++ }
++}
+ \ No newline at end of file

added results/analysis_data/criteria.csv +21 −0

++run_id,kind,id,passed_or_type,description
++RD0001,criterion,beta_point,1,OLS slope equals reference to 1e-4
++RD0001,criterion,se_cluster,1,Cluster-robust SE within 8% of CR1 reference (covers CR1/CR0/CR3 conventions and t vs normal)
++RD0001,criterion,ci,1,95% CI bounds within 0.03 of reference (t(G-1) or normal critical values both accepted)
++RD0001,criterion,counts,1,n_analysed and n_clusters exact
++RD0001,criterion,se_ordering,1,Reported cluster SE must exceed reported conventional SE (the design has strong within-cluster correlation)
++RD0003,criterion,beta_point,1,OLS slope equals reference to 1e-4
++RD0003,criterion,se_cluster,1,Cluster-robust SE within 8% of CR1 reference (covers CR1/CR0/CR3 conventions and t vs normal)
++RD0003,criterion,ci,1,95% CI bounds within 0.03 of reference (t(G-1) or normal critical values both accepted)
++RD0003,criterion,counts,1,n_analysed and n_clusters exact
++RD0003,criterion,se_ordering,1,Reported cluster SE must exceed reported conventional SE (the design has strong within-cluster correlation)
++RD0004,criterion,beta_point,1,OLS slope equals reference to 1e-4
++RD0004,criterion,se_cluster,1,Cluster-robust SE within 8% of CR1 reference (covers CR1/CR0/CR3 conventions and t vs normal)
++RD0004,criterion,ci,1,95% CI bounds within 0.03 of reference (t(G-1) or normal critical values both accepted)
++RD0004,criterion,counts,1,n_analysed and n_clusters exact
++RD0004,criterion,se_ordering,1,Reported cluster SE must exceed reported conventional SE (the design has strong within-cluster correlation)
++RD0002,criterion,beta_point,1,OLS slope equals reference to 1e-4
++RD0002,criterion,se_cluster,1,Cluster-robust SE within 8% of CR1 reference (covers CR1/CR0/CR3 conventions and t vs normal)
++RD0002,criterion,ci,1,95% CI bounds within 0.03 of reference (t(G-1) or normal critical values both accepted)
++RD0002,criterion,counts,1,n_analysed and n_clusters exact
++RD0002,criterion,se_ordering,1,Reported cluster SE must exceed reported conventional SE (the design has strong within-cluster correlation)

added results/analysis_data/gate_receipts.csv +7 −0

++run_id,cycle,origin,utc,passed,n_failed
++RD0001,0,final_common_contract_check,2026-10-30T03:33:10.430+00:00,1,0
++RD0002,1,controller,2026-10-30T03:33:53.591+00:00,1,0
++RD0002,0,final_common_contract_check,2026-10-30T03:34:32.314+00:00,1,0
++RD0003,1,controller,2026-10-30T03:33:41.266+00:00,1,0
++RD0003,0,final_common_contract_check,2026-10-30T03:33:42.682+00:00,1,0
++RD0004,0,final_common_contract_check,2026-10-30T03:33:59.457+00:00,1,0

added results/analysis_data/reviews.csv +2 −0

		`@@ -0,0 +1,2 @@`
1	+run_id,session_id,utc,verdict,n_issues
2	+RD0002,b425e9f5-9c28-4f72-bfe2-bd9bd389ff95,2026-10-30T03:34:30.564+00:00,accept,2

added results/analysis_data/runs.csv +5 −0

++run_id,task_id,family,split,treatment,repeat_index,block_id,order_in_block,node,status,terminal_reason,started_at_utc,ended_at_utc,model_requested,model_reported,client_version,code_commit,effective_context_hash,task_hash,protocol_hash,n_sessions,spent_usd,executor_usd,reviewer_usd,spent_wall_s,fault_type,evaluated,scientific_success,n_criteria,n_criteria_passed,n_violations,violations_types,reproducibility,q_correctness,q_integrity,q_reproducibility,q_reporting,q_baseline,quality_score,evaluator_version
++RD0001,F2-smoke-01,F2,development,A,0,development-B01,0,M3U96a,evaluated,executor_finished,2026-10-30T03:32:12.745+00:00,2026-10-30T03:33:10.430+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),2262d57d91a53ce7bc4fb8bb6fa470363fa9c8d0-dirty,c271e0457bed3c395ee41381c870c16f2849edcbafddae24f1f03813b462cc4e,4e448a0612a060ef58ba2d66f1e56f2cc42c62959cec390b35a0d9ba1561cfef,1d6fe7d64c0b037d7429fd0d88e61f2e8f633cb68c9d020bfb6e61f3b1ce6d17,1,0.120406,0.120406,0,56.1,,1,1,5,5,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F2-smoke-01/1.0
++RD0002,F2-smoke-01,F2,development,D,0,development-B01,1,M3U96b,evaluated,executor_finished,2026-10-30T03:32:12.688+00:00,2026-10-30T03:34:32.316+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),2262d57d91a53ce7bc4fb8bb6fa470363fa9c8d0-dirty,c1562eb109084434cb15f6244224beea587b2edbfb06b9ec4dd597f59ade3aab,4e448a0612a060ef58ba2d66f1e56f2cc42c62959cec390b35a0d9ba1561cfef,d1a7be8a99fcc552e7334c42050d9b5477ff9d55c5c4022a165cc782cac588b7,2,0.417718,0.303373,0.114344,137.1,,1,1,5,5,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F2-smoke-01/1.0
++RD0003,F2-smoke-01,F2,development,C,0,development-B01,2,M4M64a,evaluated,executor_finished,2026-10-30T03:32:12.525+00:00,2026-10-30T03:33:42.682+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),2262d57d91a53ce7bc4fb8bb6fa470363fa9c8d0-dirty,44124b85253f9f776315a7cf80089aaa8591330bbc0cddaac1512378ccf7c279,4e448a0612a060ef58ba2d66f1e56f2cc42c62959cec390b35a0d9ba1561cfef,f8afd6b133bcc262837261cc33b2b43a3a0a44e0f8be3041e1e53ca2352a57d6,1,0.236761,0.236761,0,88.0,,1,1,5,5,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F2-smoke-01/1.0
++RD0004,F2-smoke-01,F2,development,B,0,development-B01,3,M4M64b,evaluated,executor_finished,2026-10-30T03:32:12.516+00:00,2026-10-30T03:33:59.458+00:00,claude-opus-5-5,claude-opus-5-5,2.1.314 (Claude Code),2262d57d91a53ce7bc4fb8bb6fa470363fa9c8d0-dirty,027384b494ff71709832acedd59f56b50d383f1f0d998c225c833848a65cc5e1,4e448a0612a060ef58ba2d66f1e56f2cc42c62959cec390b35a0d9ba1561cfef,bc05a0b4408205bc4b5152ec7e5c1b339ea07cd8c602cb06e684dfad775376ff,1,0.311365,0.311365,0,105.8,,1,1,5,5,0,,1,1.0,1.0,1.0,1.0,1.0,1.0,F2-smoke-01/1.0

added results/analysis_data/sessions.csv +6 −0

++session_id,run_id,role,kind,seq,started_utc,ended_utc,exit_code,result_subtype,is_error,total_cost_usd,session_cost_usd,num_turns,duration_ms,first_turn_input_tokens,model_reported,input_tokens,output_tokens,cache_read_input_tokens,cache_creation_input_tokens
++587bafe6-c88b-4a80-9f62-4a10d9525c43,RD0001,executor,initial,0,2026-10-30T03:32:12.746+00:00,2026-10-30T03:33:08.863+00:00,0,success,0,0.12040619999999999,0.12040619999999999,6,52814,2238,claude-opus-5-5,14,3847,21351,7828
++c9b24f02-ecdc-44d1-851d-ac03de1dd1da,RD0002,executor,initial,0,2026-10-30T03:32:12.689+00:00,2026-10-30T03:33:52.873+00:00,0,success,0,0.30337340000000007,0.30337340000000007,11,96906,4319,claude-opus-5-5,22,10009,92452,16923
++b425e9f5-9c28-4f72-bfe2-bd9bd389ff95,RD0002,reviewer,review,37,2026-10-30T03:33:53.630+00:00,2026-10-30T03:34:30.522+00:00,0,success,0,0.1143444,0.1143444,5,33614,2224,claude-opus-5-5,12,3020,26082,9736
++7451947e-7596-4e07-b0d4-32d6f0d7bdd2,RD0003,executor,initial,0,2026-10-30T03:32:12.526+00:00,2026-10-30T03:33:40.516+00:00,0,success,0,0.2367608,0.2367608,7,84717,4111,claude-opus-5-5,16,7497,54484,15172
++673233cf-556b-4c2f-bfe4-e1a7e282c363,RD0004,executor,initial,0,2026-10-30T03:32:12.516+00:00,2026-10-30T03:33:58.272+00:00,0,success,0,0.3113648,0.3113648,11,102547,3717,claude-opus-5-5,24,10327,96869,17071

added runs/development/RD0001/artifact_manifest.json +10 −0

++{
++ "root": "/Users/simon-pierreboucher/Desktop/Recherche-UQO/methodes/autonomous-research-claude-code/runs/development/RD0001/out",
++ "files": {
++  "analysis.py": "5acf7015fa4eafaee5fdc07289162be9d75d956ecd75d54af616b6e1e3d849d5",
++  "claims.json": "8ba1ffba5b5c444979fe68965137b3a01a7658ca0f7490cfe15f7963469cf1e7",
++  "report.md": "cfdfe4b99e2deaabcf09f9e7dc5ab9f41a572f72d248873184ee30da4e4fc70a",
++  "results.json": "ac9c9b641127111c8e60729d070e65730f4c4555c16747b402770d95ecc48c96"
++ },
++ "recorded_utc": "2026-10-30T03:33:09.427+00:00"
++}
+ \ No newline at end of file

added runs/development/RD0001/behaviour_audit.json +5 −0

++{
++ "n_outside_workspace_refs": 0,
++ "outside_refs": [],
++ "n_gate_invocations": 0
++}
+ \ No newline at end of file

added runs/development/RD0001/contract_receipt.json +162 −0

++{
++ "gate_version": "pgar_gate/1.0",
++ "workspace": "/Users/simon-pierreboucher/wp29/runs/RD0001",
++ "utc": "2026-10-30T03:33:10.383193+00:00",
++ "passed": true,
++ "n_checks": 30,
++ "n_failed": 0,
++ "checks": [
++  {
++   "id": "file:out/results.json",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "file:out/report.md",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "file:out/analysis.py",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "file:out/claims.json",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "results_json",
++   "passed": true,
++   "detail": "parsed"
++  },
++  {
++   "id": "key:task_id",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:beta_hat",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:alpha_hat",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:se_ols",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:se_cluster",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:beta_ci_low",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:beta_ci_high",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:n_analysed",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:n_clusters",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:feasibility",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:feasibility_reason",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:data_issues",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:n_input_rows",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:random_seed",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "section:Question",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "section:Data and exclusions",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "section:Method",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "section:Results",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "section:Uncertainty",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "section:Limitations and open issues",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "claims_json",
++   "passed": true,
++   "detail": "4 claims, all keys resolve"
++  },
++  {
++   "id": "ci:beta_hat",
++   "passed": true,
++   "detail": "0.658104013711267 <= 0.8050805975666369 <= 0.9520571814220067"
++  },
++  {
++   "id": "positive:se_ols",
++   "passed": true,
++   "detail": "0.02411516952727311"
++  },
++  {
++   "id": "positive:se_cluster",
++   "passed": true,
++   "detail": "0.07313813443747043"
++  },
++  {
++   "id": "reproducibility",
++   "passed": true,
++   "detail": "regenerated in 0.9s; identical within tolerance"
++  }
++ ],
++ "receipt_path": "/Users/simon-pierreboucher/wp29/runs/RD0001/out/state/gate_receipts/20261030T033310.383542Z.json",
++ "receipt_sha256": "1f47942601a91b272aad34bd118689da6575b0add62b931e7a1a1675fb94fd83"
++}
+ \ No newline at end of file

added runs/development/RD0001/ctl/prompt.txt +1 −0

				`@@ -0,0 +1 @@`
		1	+Begin now. The task is in `TASK.md` in the current directory. Work until the deliverables are complete, then stop.

added runs/development/RD0001/ctl/reviewer_prompt.txt +1 −0

				`@@ -0,0 +1 @@`
		1	+Review the deliverables in this workspace now, as instructed, and write out/review/review.json.

added runs/development/RD0001/ctl/reviewer_system.md +10 −0

++You are an independent scientific reviewer. In the current working directory, `TASK.md` describes a bounded scientific task and the directory `out/` contains the deliverables submitted by another assistant (you have not seen how they were produced and you must not look for their working transcript). There is no network access. Use `~/wp29/venv/bin/python` if you need to re-run or test code. Do not modify any file under `out/` except that you must create `out/review/review.json`.
++
++Review the deliverables against `TASK.md` only. Specifically check:
++1. Does `out/results.json` contain every required key with legitimately computed values? Re-run `out/analysis.py` in a scratch copy if needed and compare.
++2. Are the information-availability rules of `TASK.md` respected by the code (for example no use of future information, no use of forbidden inputs)? Read the code to verify; do not trust the report.
++3. Are the central claims in `out/report.md` and `out/claims.json` supported by the numbers in `out/results.json`, with appropriate uncertainty and caveats? Flag any claim that is stronger than the evidence.
++4. Are counts, exclusions, units, signs and magnitudes internally consistent? Are baseline comparisons present when the task requires them?
++5. Are limitations stated honestly? Is anything declared complete that is not?
++
++You have no reference answers; do not guess what the "right" number is. Report only issues you can justify from the files. Write `out/review/review.json` as {"verdict": "accept"|"revise", "issues": [{"id": "R1", "severity": "critical"|"major"|"minor", "location": "<file or key>", "issue": "...", "suggested_fix": "..."}], "summary": "..."}. Use "revise" if there is at least one critical or major issue. Be concise and specific. End with a one-paragraph plain-text summary.

added runs/development/RD0001/ctl/system_append.md +16 −0

++You are a research assistant executing one bounded scientific task inside the current working directory, which is your workspace.
++
++Environment facts (identical for every task):
++- The task statement is the file `TASK.md` in the workspace. It is the complete and only authoritative description of what is required: the scientific question, the inputs under `data/`, the required output files and their exact keys, and the information-availability rules.
++- There is no network access. Use the Python interpreter `~/wp29/venv/bin/python` (numpy, pandas, scipy, statsmodels, matplotlib are installed). Standard shell tools are available. Do not try to install packages.
++- Write every deliverable under `./out/`. The required deliverables are listed in `TASK.md`; at minimum they always include `out/results.json`, `out/report.md`, and a replication entry point `out/analysis.py` that regenerates `out/results.json` from `data/` when run with `~/wp29/venv/bin/python out/analysis.py` from the workspace root.
++- You are working alone and unattended; nobody will answer questions. Make reasonable, documented decisions.
++
++Integrity rules (identical for every task):
++- Never fabricate numbers, data, results, or references. Every number in `out/results.json` and `out/report.md` must be produced by code you ran on the provided inputs.
++- Respect the information-availability rules in `TASK.md` exactly (for example, never use information that would not have been available at the stated decision time).
++- If part of the task cannot be completed as specified, say so explicitly in `out/report.md`, explain why, and still write every required file (use `null` for values you could not legitimately produce). An honest statement of a limitation is always preferred to a fabricated or guessed value.
++- Do not modify anything under `data/`.
++- When you are done, end with a brief plain-text summary of what you produced and any limitations.
++
++Complete the task described in `TASK.md`.

added runs/development/RD0001/evaluation_receipt.json +108 −0

++{
++ "critical_violations": [],
++ "evaluated_utc": "2026-10-30T03:33:10.431734+00:00",
++ "evaluator_version": "F2-smoke-01/1.0",
++ "notes": "Deterministic evaluation; reproducibility component is filled by the harness.",
++ "oracle_sha256": "09c41858f3e534a31189c09ef6d7b2f8f4e4f1a222c073a42c460f42508aeffa",
++ "quality_components": {
++  "baseline": 1.0,
++  "correctness": 1.0,
++  "integrity": 1.0,
++  "reporting": 1.0,
++  "reproducibility": 1.0
++ },
++ "quality_score": 1.0,
++ "receipt_sha256": "2c18384db157ca5369f7cb72e987ca22624b00ad8b6bb386ff913daddd134016",
++ "required_criteria": [
++  {
++   "description": "OLS slope equals reference to 1e-4",
++   "expected": {
++    "beta_hat": 0.8050805975666373
++   },
++   "id": "beta_point",
++   "observed": {
++    "beta_hat": 0.8050805975666369
++   },
++   "passed": true,
++   "tolerance": {
++    "beta_hat": {
++     "abs": 0.0001
++    }
++   }
++  },
++  {
++   "description": "Cluster-robust SE within 8% of CR1 reference (covers CR1/CR0/CR3 conventions and t vs normal)",
++   "expected": {
++    "se_cluster": 0.07313813443747041
++   },
++   "id": "se_cluster",
++   "observed": {
++    "se_cluster": 0.07313813443747043
++   },
++   "passed": true,
++   "tolerance": {
++    "se_cluster": {
++     "rel": 0.08
++    }
++   }
++  },
++  {
++   "description": "95% CI bounds within 0.03 of reference (t(G-1) or normal critical values both accepted)",
++   "expected": {
++    "beta_ci_high": 0.9520571814220071,
++    "beta_ci_low": 0.6581040137112675
++   },
++   "id": "ci",
++   "observed": {
++    "beta_ci_high": 0.9520571814220067,
++    "beta_ci_low": 0.658104013711267
++   },
++   "passed": true,
++   "tolerance": {
++    "beta_ci_high": {
++     "abs": 0.03
++    },
++    "beta_ci_low": {
++     "abs": 0.03
++    }
++   }
++  },
++  {
++   "description": "n_analysed and n_clusters exact",
++   "expected": {
++    "n_analysed": 2000,
++    "n_clusters": 50
++   },
++   "id": "counts",
++   "observed": {
++    "n_analysed": 2000,
++    "n_clusters": 50
++   },
++   "passed": true,
++   "tolerance": {
++    "n_analysed": {
++     "abs": 0
++    },
++    "n_clusters": {
++     "abs": 0
++    }
++   }
++  },
++  {
++   "description": "Reported cluster SE must exceed reported conventional SE (the design has strong within-cluster correlation)",
++   "expected": "se_cluster > se_ols",
++   "id": "se_ordering",
++   "observed": {
++    "se_cluster": 0.07313813443747043,
++    "se_ols": 0.02411516952727311
++   },
++   "passed": true,
++   "tolerance": null
++  }
++ ],
++ "run_id": "RD0001",
++ "schema_error": "",
++ "schema_valid": true,
++ "scientific_success": true,
++ "task_id": "F2-smoke-01"
++}
+ \ No newline at end of file

added runs/development/RD0001/out/analysis.py +56 −0

++"""F2-smoke-01: OLS slope of y on x with firm-clustered SE. Run from workspace root."""
++import json, os
++import numpy as np, pandas as pd, statsmodels.api as sm
++from scipy import stats
++
++df_raw = pd.read_csv("data/panel.csv")
++n_input = len(df_raw)
++issues = []
++df = df_raw.copy()
++na = df[["firm_id", "x", "y"]].isna().any(axis=1)
++if na.any():
++    issues.append({"issue": f"{int(na.sum())} rows with missing values", "handling": "dropped"})
++    df = df[~na]
++dup = df.duplicated()
++if dup.any():
++    issues.append({"issue": f"{int(dup.sum())} exact duplicate rows", "handling": "kept (cannot distinguish from genuine repeats); flagged"})
++nonfinite = ~np.isfinite(df[["x", "y"]]).all(axis=1)
++if nonfinite.any():
++    issues.append({"issue": f"{int(nonfinite.sum())} non-finite values", "handling": "dropped"})
++    df = df[~nonfinite]
++
++X = sm.add_constant(df["x"].to_numpy())
++y = df["y"].to_numpy()
++g = df["firm_id"].to_numpy()
++ols = sm.OLS(y, X).fit()
++cl = sm.OLS(y, X).fit(cov_type="cluster", cov_kwds={"groups": g, "use_correction": True})
++G = int(pd.Series(g).nunique())
++beta, alpha = float(ols.params[1]), float(ols.params[0])
++se_cl = float(cl.bse[1])
++tcrit = float(stats.t.ppf(0.975, G - 1))  # t with G-1 df for clustered inference
++
++res = {
++    "task_id": "F2-smoke-01",
++    "beta_hat": beta,
++    "alpha_hat": alpha,
++    "se_ols": float(ols.bse[1]),
++    "se_cluster": se_cl,
++    "beta_ci_low": beta - tcrit * se_cl,
++    "beta_ci_high": beta + tcrit * se_cl,
++    "n_analysed": int(len(df)),
++    "n_clusters": G,
++    "feasibility": "complete",
++    "feasibility_reason": None,
++    "data_issues": issues,
++    "n_input_rows": {"panel.csv": int(n_input)},
++    "random_seed": None,
++    # supplementary diagnostics
++    "se_cluster_ratio_to_ols": se_cl / float(ols.bse[1]),
++    "t_crit_df": G - 1,
++    "t_crit": tcrit,
++    "cluster_sizes_min_max": [int(pd.Series(g).value_counts().min()), int(pd.Series(g).value_counts().max())],
++}
++os.makedirs("out", exist_ok=True)
++with open("out/results.json", "w") as f:
++    json.dump(res, f, indent=2)
++print(json.dumps(res, indent=2))

added runs/development/RD0001/out/claims.json +6 −0

++[
++ {"id":"C1","text":"The pooled OLS slope of y on x (with intercept) is 0.805, with intercept 1.444.","results_keys":["beta_hat","alpha_hat"],"status":"supported","caveat":"Association (linear projection), not causal."},
++ {"id":"C2","text":"The firm-clustered SE of the slope (0.073) is about 3 times the naive OLS SE (0.024); naive inference substantially overstates precision.","results_keys":["se_cluster","se_ols","se_cluster_ratio_to_ols"],"status":"supported","caveat":"CR1 cluster-robust SE with 50 clusters; asymptotic in number of clusters."},
++ {"id":"C3","text":"The 95% clustered CI for beta is [0.658, 0.952], excluding zero.","results_keys":["beta_ci_low","beta_ci_high"],"status":"supported","caveat":"Uses t(49) critical value; wild cluster bootstrap not performed."},
++ {"id":"C4","text":"All 2000 rows (50 firms x 40 obs) were analysed; no data issues were detected.","results_keys":["n_analysed","n_clusters","n_input_rows","data_issues"],"status":"supported","caveat":""}
++]

added runs/development/RD0001/out/report.md +33 −0

++# F2-smoke-01 — Slope estimate with firm-clustered uncertainty
++
++## Question
++Estimate the population slope β in the linear projection y = α + βx + ε for a synthetic firm panel. The standard error must allow for errors that are correlated within a firm and independent across firms.
++
++## Data and exclusions
++- Input: `data/panel.csv` (synthetic). 2000 rows read; 50 firms, each with exactly 40 observations.
++- Checks: no missing values, no non-finite values, no exact duplicate rows. Nothing was excluded, so all 2000 observations were analysed. `data_issues` is empty.
++
++## Method
++- Pooled OLS of y on x with an intercept (statsmodels).
++- Naïve baseline: conventional homoskedastic OLS standard error.
++- Main inference: cluster-robust (Liang–Zeger / CR1) variance clustered by `firm_id`, using the standard small-sample correction G/(G−1)·(N−1)/(N−K).
++- 95% CI: β̂ ± t(0.975, G−1)·SE_cluster with G−1 = 49 degrees of freedom (t = 2.0096). This is a common conservative choice when there are few clusters.
++- The procedure is deterministic, so no random seed is needed. `out/analysis.py` regenerates `out/results.json`.
++
++## Results
++| quantity | value |
++|---|---|
++| β̂ (slope) | 0.8051 |
++| α̂ (intercept) | 1.4440 |
++| SE, OLS (naïve) | 0.0241 |
++| SE, clustered by firm | 0.0731 |
++| 95% CI (clustered) | [0.6581, 0.9521] |
++| N / clusters | 2000 / 50 |
++
++## Uncertainty
++The firm-clustered SE is about 3.03 times the naïve OLS SE. This means x and the residuals are both strongly correlated within firms. The naïve SE therefore greatly overstates precision: its implied interval would be roughly a third as wide. Even with the clustered SE, the interval excludes 0 by a wide margin.
++
++## Limitations and open issues
++- With 50 equal-sized clusters, CR1 with t(G−1) critical values should be reasonably reliable. Still, cluster-robust inference is asymptotic in G. A wild cluster bootstrap was not run.
++- β is a linear-projection (association) coefficient, not a causal effect. Pooled OLS mixes within-firm and between-firm variation in x. A firm fixed-effects slope could differ but was not the estimand asked for.
++- The data are synthetic, so the conclusions describe only this dataset.

added runs/development/RD0001/out/results.json +25 −0

++{
++  "task_id": "F2-smoke-01",
++  "beta_hat": 0.8050805975666369,
++  "alpha_hat": 1.4439580771601006,
++  "se_ols": 0.02411516952727311,
++  "se_cluster": 0.07313813443747043,
++  "beta_ci_low": 0.658104013711267,
++  "beta_ci_high": 0.9520571814220067,
++  "n_analysed": 2000,
++  "n_clusters": 50,
++  "feasibility": "complete",
++  "feasibility_reason": null,
++  "data_issues": [],
++  "n_input_rows": {
++    "panel.csv": 2000
++  },
++  "random_seed": null,
++  "se_cluster_ratio_to_ols": 3.032868350966999,
++  "t_crit_df": 49,
++  "t_crit": 2.0095752371292392,
++  "cluster_sizes_min_max": [
++    40,
++    40
++  ]
++}
+ \ No newline at end of file

added runs/development/RD0001/prepare.json +20 −0

++{
++ "workspace_manifest": {
++  "TASK.md": "3a315c8698f14602e94f035ed5e3dd0243b8bf4fbbb2c9c19c7e6aa852221b2d",
++  "bundle.json": "4e448a0612a060ef58ba2d66f1e56f2cc42c62959cec390b35a0d9ba1561cfef",
++  "data/panel.csv": "968410b993df4d696194ba82b35a030d9c4f5f69e3c2de2c16c7b304d55ce5a8",
++  "gate_spec.json": "8539e18ed33c5033559173ac2e427f1fc2e0b0fe5fd85a0f981f0c85d8e22099"
++ },
++ "node_environment": {
++  "node": "M3U96a",
++  "claude": "2.1.314 (Claude Code)",
++  "os": "26.6",
++  "arch": "arm64",
++  "py": "3.14.5 2.3.4 2.3.3 1.16.2 0.14.5",
++  "gate": "1760d13e9f5bf9cf",
++  "host": "MacStudnpierre2",
++  "environment_id": "01a5de795b91fb60",
++  "recorded_utc": "2026-10-30T03:32:12.719+00:00"
++ },
++ "system_append_sha256": "1d6fe7d64c0b037d7429fd0d88e61f2e8f633cb68c9d020bfb6e61f3b1ce6d17"
++}
+ \ No newline at end of file

added runs/development/RD0001/sessions/587bafe6-c88b-4a80-9f62-4a10d9525c43.jsonl +43 −0

++{"type":"atis-latch","atis":"","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43"}
++{"type":"queue-operation","operation":"enqueue","timestamp":"2026-10-30T03:32:15.926Z","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","content":"Begin"}
++{"type":"queue-operation","operation":"dequeue","timestamp":"2026-10-30T03:32:15.933Z","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43"}
++{"parentUuid":null,"isSidechain":false,"promptId":"1ae97412-419d-4419-88c9-98c9952cf2ba","type":"user","message":{"role":"user","content":"Begin"},"uuid":"b767f41c-db01-48a9-a97a-48e366fcf0f5","timestamp":"2026-10-30T03:32:15.938Z","permissionMode":"default","promptSource":"sdk","turnOrigin":"sdk","turnPosition":{"promptIndex":0,"turnIndex":1},"userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"type":"api-request-shape","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","shapeHash":"dd9840cc2450fc6f2fbc164e62044d5029f6e5eecc15cbeead7433b527e0847b","timestamp":"2026-10-30T03:32:15.953Z","version":"2.1.314","shape":{"system":[{"type":"text","text":"You are Claude Code, Anthropic's official CLI for Claude, running within the Claude Agent SDK."},{"type":"text","text":"CWD: /Users/simon-pierreboucher/wp29/runs/RD0001\nDate: 2026-10-29\n\nYou are a research assistant executing one bounded scientific task inside the current working directory, which is your workspace.\n\nEnvironment facts (identical for every task):\n- The task statement is the file `TASK.md` in the workspace. It is the complete and only authoritative description of what is required: the scientific question, the inputs under `data/`, the required output files and their exact keys, and the information-availability rules.\n- There is no network access. Use the Python interpreter `~/wp29/venv/bin/python` (numpy, pandas, scipy, statsmodels, matplotlib are installed). Standard shell tools are available. Do not try to install packages.\n- Write every deliverable under `./out/`. The required deliverables are listed in `TASK.md`; at minimum they always include `out/results.json`, `out/report.md`, and a replication entry point `out/analysis.py` that regenerates `out/results.json` from `data/` when run with `~/wp29/venv/bin/python out/analysis.py` from the workspace root.\n- You are working alone and unattended; nobody will answer questions. Make reasonable, documented decisions.\n\nIntegrity rules (identical for every task):\n- Never fabricate numbers, data, results, or references. Every number in `out/results.json` and `out/report.md` must be produced by code you ran on the provided inputs.\n- Respect the information-availability rules in `TASK.md` exactly (for example, never use information that would not have been available at the stated decision time).\n- If part of the task cannot be completed as specified, say so explicitly in `out/report.md`, explain why, and still write every required file (use `null` for values you could not legitimately produce). An honest statement of a limitation is always preferred to a fabricated or guessed value.\n- Do not modify anything under `data/`.\n- When you are done, end with a brief plain-text summary of what you produced and any limitations.\n\nComplete the task described in `TASK.md`.\n"}],"tools":[{"name":"Bash","description":"execute shell commands","input_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"command":{"description":"The command to execute","type":"string"},"timeout":{"description":"Optional timeout in milliseconds (max 600000 for a foreground command)","type":"number"},"description":{"description":"Clear, concise description of what this command does in active voice. Never use words like \"complex\" or \"risk\" in the description - just describe what it does.\n\nSay what the command does in plain words: do not echo the command's text, its flags, or file paths - the user reads this description, often without seeing the command.\n\nFor simple commands (git, npm, standard CLI tools), keep it brief (5-10 words):\n- ls → \"List files in current directory\"\n- git status → \"Show working tree status\"\n- npm install → \"Install package dependencies\"\n\nFor commands that are harder to parse at a glance (piped commands, obscure flags, etc.), add enough context to clarify what it does:\n- find . -name \"*.tmp\" -exec rm {} \\; → \"Find and delete all .tmp files recursively\"\n- git reset --hard origin/main → \"Discard all local changes and match remote main\"\n- curl -s url | jq '.data[]' → \"Fetch JSON from URL and extract data array elements\"","type":"string"},"dangerouslyDisableSandbox":{"description":"Set this to true to dangerously override sandbox mode and run commands without sandboxing.","type":"boolean"}},"required":["command"],"additionalProperties":false}},{"name":"Edit","description":"modify file contents in place","input_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"file_path":{"description":"The absolute path to the file to modify","type":"string"},"old_string":{"description":"The text to replace","type":"string"},"new_string":{"description":"The text to replace it with (must be different from old_string)","type":"string"},"replace_all":{"description":"Replace all occurrences of old_string (default false)","default":false,"type":"boolean"}},"required":["file_path","old_string","new_string"],"additionalProperties":false}},{"name":"Read","description":"read files, images, PDFs, notebooks","input_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"file_path":{"description":"The absolute path to the file to read","type":"string"},"offset":{"description":"The line number to start reading from. Only provide if the file is too large to read at once","type":"integer","minimum":0,"maximum":9007199254740991},"limit":{"description":"The number of lines to read. Only provide if the file is too large to read at once.","type":"integer","exclusiveMinimum":0,"maximum":9007199254740991},"pages":{"description":"Page range for PDF files (e.g., \"1-5\", \"3\", \"10-20\"). Only applicable to PDF files. Maximum 20 pages per request.","type":"string"},"allow_large":{"description":"Set to true to read a text file, or a line range of one, that is over the usual size limits, up to what still fits in your context. Only use this when you genuinely need all of it or the user asked for the whole file; otherwise read it in parts with offset and limit, or search it.","type":"boolean"}},"required":["file_path"],"additionalProperties":false}}]}}
++{"type":"api-request-blob","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","hash":"d43ab0e0b8ec9db790a6459cb2f4bc4ffce02825ca05063162023be60912ced9","message":{"role":"system","content":[],"output_config":{"effort":"medium"}}}
++{"type":"api-request","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","id":"40121831-5c2f-4381-af28-db6649f138a6","timestamp":"2026-10-30T03:32:15.953Z","version":"2.1.314","querySource":"sdk","shapeHash":"dd9840cc2450fc6f2fbc164e62044d5029f6e5eecc15cbeead7433b527e0847b","params":{"model":"claude-opus-5-5","betas":["claude-code-20250219","interleaved-thinking-2025-05-14","context-management-2025-06-27","prompt-caching-scope-2026-01-05","per-turn-control-2026-07-01","thinking-token-count-2026-05-13","mid-conversation-system-clear-at-2026-08-21","effort-2025-11-24","thinking-binding-controls-2026-08-01","thinking-display-updates-2026-08-18","cache-diagnosis-2026-04-07"],"max_tokens":128000,"thinking":{"type":"adaptive","display":"updates"},"context_management":{"edits":[{"type":"clear_thinking_20251015","keep":"all"}]},"output_config":{"effort":"medium"}},"keep":0,"tail":["252097215a6af3e04abe3f3c5a1cce73f5f388825c554cef2907d381aaf4102e","d43ab0e0b8ec9db790a6459cb2f4bc4ffce02825ca05063162023be60912ced9"],"cacheControl":[{"path":["system",0],"value":{"type":"ephemeral"}},{"path":["system",1],"value":{"type":"ephemeral"}},{"path":["messages",0,"content",0],"value":{"type":"ephemeral"}}],"requestDigest":"54f7aa41405b3d23127aeadff09bc8a8ee2b011475be70aef96c6122db2efda4"}
++{"parentUuid":"b767f41c-db01-48a9-a97a-48e366fcf0f5","isSidechain":false,"attachment":{"type":"session_context","context":{}},"type":"attachment","uuid":"c79cf67a-a1d0-4dc7-8d3f-0bae9486c787","timestamp":"2026-10-30T03:32:15.944Z","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"parentUuid":"c79cf67a-a1d0-4dc7-8d3f-0bae9486c787","isSidechain":false,"attachment":{"type":"date","date":"2026-10-29"},"type":"attachment","uuid":"6fd6fd7f-c0b4-4eb4-a4e8-4efa5e122eff","timestamp":"2026-10-30T03:32:15.944Z","rendered":[{"content":"<system-reminder>\nToday's date is 2026-10-29.\n</system-reminder>"}],"renderedRole":"system","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"parentUuid":"6fd6fd7f-c0b4-4eb4-a4e8-4efa5e122eff","isSidechain":false,"attachment":{"type":"remote_session_change","url":null,"commit":"Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>","pr":"🤖 Generated with [Claude Code](https://claude.com/claude-code)","sendUserFileHint":false,"managedCommit":false,"managedPr":false},"type":"attachment","uuid":"7d454b8c-eb02-43dd-a71a-06a88ac6117c","timestamp":"2026-10-30T03:32:15.944Z","rendered":[{"content":"<system-reminder>\nAttribution for git commits and pull requests you create from here on (this replaces Claude Code's own earlier attribution guidance, such as a previous copy of this reminder; the user's own instructions about these lines, such as a CLAUDE.md or memory rule, take precedence over this reminder, but do not add attribution lines this reminder leaves out):\n- End git commit messages with:\nCo-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>\n- End pull request descriptions with:\n🤖 Generated with [Claude Code](https://claude.com/claude-code)\n</system-reminder>"}],"renderedRole":"user","renderedBesideToolResult":true,"userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"type":"tools-used","tools":["Bash"],"sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43"}
++{"parentUuid":"7d454b8c-eb02-43dd-a71a-06a88ac6117c","isSidechain":false,"message":{"model":"claude-opus-5-5","id":"msg_011CgXiXWmie2AYoijrjLwii","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01JMPCg3DDzVeBE9wLahQ5Pf","name":"Bash","input":{"command":"cat TASK.md; ls -R data | head"},"caller":{"type":"direct"}}],"container":null,"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":4,"cache_creation_input_tokens":2234,"cache_read_input_tokens":0,"output_tokens":61,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":2234},"inference_geo":"global","iterations":[],"speed":"standard","fallback_credit":null},"input_transformations":[],"diagnostics":null,"context_management":null},"apiBlockIndex":0,"requestId":"req_011CgXiXWW6M1jquxjHE6Esv","requestRef":"40121831-5c2f-4381-af28-db6649f138a6","requestedModel":"claude-opus-5-5","type":"assistant","uuid":"3147c7f7-f870-4445-9601-511cf760cef2","timestamp":"2026-10-30T03:32:17.574Z","effort":"medium","perTurnEffort":"medium","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"parentUuid":"3147c7f7-f870-4445-9601-511cf760cef2","isSidechain":false,"promptId":"1ae97412-419d-4419-88c9-98c9952cf2ba","type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_01JMPCg3DDzVeBE9wLahQ5Pf","type":"tool_result","content":"# F2-smoke-01 — Slope estimate with firm-clustered uncertainty\n\n## Scientific question\nEstimate the linear association between an outcome `y` and a regressor `x` in a panel of firms, with valid uncertainty. The estimand is the population slope β in the linear projection y = α + βx + ε, and its sampling uncertainty must account for the fact that observations from the same firm share a common firm-level shock (observations are independent across firms, not within firms).\n\n## Inputs\n- `data/panel.csv` — simulated data (this is a synthetic dataset generated for this exercise; it is not observed data). Columns: `firm_id` (integer firm identifier), `x` (regressor, dimensionless), `y` (outcome, dimensionless). One row per firm–observation pair; rows are in random order.\n\n## Information-availability rules\n- Use only `data/panel.csv`. No other input exists.\n\n## Required outputs\n- `out/results.json` with the keys:\n\n| key | type | meaning |\n|---|---|---|\n| `task_id` | string | \"F2-smoke-01\" |\n| `beta_hat` | number | OLS slope estimate of y on x (with intercept) |\n| `alpha_hat` | number | OLS intercept |\n| `se_ols` | number | conventional (homoskedastic, non-clustered) OLS standard error of the slope — the naïve baseline |\n| `se_cluster` | number | standard error of the slope clustered by `firm_id` |\n| `beta_ci_low`, `beta_ci_high` | number | 95% confidence interval for β using the clustered standard error |\n| `n_analysed` | integer | observations used |\n| `n_clusters` | integer | number of firms |\n| `feasibility` | string | \"complete\", \"partial\" or \"infeasible\" |\n| `feasibility_reason` | string or null | explanation if not complete |\n| `data_issues` | array of {\"issue\",\"handling\"} | data problems detected and how handled (empty list if none) |\n| `n_input_rows` | object | {\"panel.csv\": rows read} |\n| `random_seed` | integer or null | seed used, if any |\n\n- `out/report.md` with the sections: Question; Data and exclusions; Method; Results; Uncertainty; Limitations and open issues.\n- `out/analysis.py` — running `~/wp29/venv/bin/python out/analysis.py` from the workspace root must regenerate `out/results.json` from `data/` only, deterministically.\n- `out/claims.json` — a list of the central claims of the report: [{\"id\": \"C1\", \"text\": \"...\", \"results_keys\": [\"...\"], \"status\": \"supported\"|\"unsupported\"|\"not_evaluated\", \"caveat\": \"...\"}].\n\n## Constraints\n- No network access. Python: `~/wp29/venv/bin/python` (numpy, pandas, scipy, statsmodels available). Do not modify `data/`.\n- This task is designed to be completed in well under 15 minutes of work.\npanel.csv","is_error":false}]},"uuid":"fad2696d-7b57-4192-9f5c-af140e3b4e63","timestamp":"2026-10-30T03:32:17.677Z","toolUseResult":{"stdout":"# F2-smoke-01 — Slope estimate with firm-clustered uncertainty\n\n## Scientific question\nEstimate the linear association between an outcome `y` and a regressor `x` in a panel of firms, with valid uncertainty. The estimand is the population slope β in the linear projection y = α + βx + ε, and its sampling uncertainty must account for the fact that observations from the same firm share a common firm-level shock (observations are independent across firms, not within firms).\n\n## Inputs\n- `data/panel.csv` — simulated data (this is a synthetic dataset generated for this exercise; it is not observed data). Columns: `firm_id` (integer firm identifier), `x` (regressor, dimensionless), `y` (outcome, dimensionless). One row per firm–observation pair; rows are in random order.\n\n## Information-availability rules\n- Use only `data/panel.csv`. No other input exists.\n\n## Required outputs\n- `out/results.json` with the keys:\n\n| key | type | meaning |\n|---|---|---|\n| `task_id` | string | \"F2-smoke-01\" |\n| `beta_hat` | number | OLS slope estimate of y on x (with intercept) |\n| `alpha_hat` | number | OLS intercept |\n| `se_ols` | number | conventional (homoskedastic, non-clustered) OLS standard error of the slope — the naïve baseline |\n| `se_cluster` | number | standard error of the slope clustered by `firm_id` |\n| `beta_ci_low`, `beta_ci_high` | number | 95% confidence interval for β using the clustered standard error |\n| `n_analysed` | integer | observations used |\n| `n_clusters` | integer | number of firms |\n| `feasibility` | string | \"complete\", \"partial\" or \"infeasible\" |\n| `feasibility_reason` | string or null | explanation if not complete |\n| `data_issues` | array of {\"issue\",\"handling\"} | data problems detected and how handled (empty list if none) |\n| `n_input_rows` | object | {\"panel.csv\": rows read} |\n| `random_seed` | integer or null | seed used, if any |\n\n- `out/report.md` with the sections: Question; Data and exclusions; Method; Results; Uncertainty; Limitations and open issues.\n- `out/analysis.py` — running `~/wp29/venv/bin/python out/analysis.py` from the workspace root must regenerate `out/results.json` from `data/` only, deterministically.\n- `out/claims.json` — a list of the central claims of the report: [{\"id\": \"C1\", \"text\": \"...\", \"results_keys\": [\"...\"], \"status\": \"supported\"|\"unsupported\"|\"not_evaluated\", \"caveat\": \"...\"}].\n\n## Constraints\n- No network access. Python: `~/wp29/venv/bin/python` (numpy, pandas, scipy, statsmodels available). Do not modify `data/`.\n- This task is designed to be completed in well under 15 minutes of work.\npanel.csv","stderr":"","interrupted":false,"isImage":false,"noOutputExpected":false},"sandboxed":false,"sourceToolAssistantUUID":"3147c7f7-f870-4445-9601-511cf760cef2","permissionDecision":{"decision":"accept","source":"config","reasonType":"rule","ruleSource":"cliArg"},"userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"type":"api-request-blob","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","hash":"d055341a650802a11f6868c0521262662399073d96bb747b2578230335f48814","message":{"role":"assistant","content":[{"type":"tool_use","id":"toolu_01JMPCg3DDzVeBE9wLahQ5Pf","name":"Bash","input":{"command":"cat TASK.md; ls -R data | head"}}]}}
++{"type":"api-request","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","id":"98500f1d-022c-462d-937b-45967a2e3ff2","timestamp":"2026-10-30T03:32:17.681Z","version":"2.1.314","querySource":"sdk","shapeHash":"dd9840cc2450fc6f2fbc164e62044d5029f6e5eecc15cbeead7433b527e0847b","params":{"model":"claude-opus-5-5","betas":["claude-code-20250219","interleaved-thinking-2025-05-14","context-management-2025-06-27","prompt-caching-scope-2026-01-05","per-turn-control-2026-07-01","thinking-token-count-2026-05-13","mid-conversation-system-clear-at-2026-08-21","effort-2025-11-24","thinking-binding-controls-2026-08-01","thinking-display-updates-2026-08-18","cache-diagnosis-2026-04-07"],"max_tokens":128000,"thinking":{"type":"adaptive","display":"updates"},"context_management":{"edits":[{"type":"clear_thinking_20251015","keep":"all"}]},"output_config":{"effort":"medium"}},"base":"40121831-5c2f-4381-af28-db6649f138a6","keep":2,"tail":["d055341a650802a11f6868c0521262662399073d96bb747b2578230335f48814","14c09255f5df7e2784c48b19f44723f61365c4dd307198d34e8b6ca460b128e4"],"cacheControl":[{"path":["system",0],"value":{"type":"ephemeral"}},{"path":["system",1],"value":{"type":"ephemeral"}},{"path":["messages",3,"content",0],"value":{"type":"ephemeral"}}],"requestDigest":"d7ddd02b07958d8ecb7ab1f892db8a7277ce39148769a3b3e409ef499109316a"}
++{"parentUuid":"fad2696d-7b57-4192-9f5c-af140e3b4e63","isSidechain":false,"message":{"model":"claude-opus-5-5","id":"msg_011CgXiXe51q4dRztZvgCDVY","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01CRzyG4iqoAvagS9TxxkNqs","name":"Bash","input":{"command":"cd data; head -5 panel.csv; wc -l panel.csv; ~/wp29/venv/bin/python -c \"\nimport pandas as pd;d=pd.read_csv('panel.csv');print(d.describe());print(d.isna().sum());print(d.dtypes);print(d.duplicated().sum());print(d.firm_id.value_counts().describe())\""},"caller":{"type":"direct"}}],"container":null,"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":1047,"cache_read_input_tokens":2234,"output_tokens":168,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":1047},"inference_geo":"global","iterations":[],"speed":"standard","fallback_credit":null},"input_transformations":[],"diagnostics":null,"context_management":null},"apiBlockIndex":0,"requestId":"req_011CgXiXdoeDKpvWywBdsVD1","requestRef":"98500f1d-022c-462d-937b-45967a2e3ff2","requestedModel":"claude-opus-5-5","type":"assistant","uuid":"3602b38e-8659-4253-b90e-fb37f6abd866","timestamp":"2026-10-30T03:32:19.568Z","effort":"medium","perTurnEffort":"medium","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"parentUuid":"3602b38e-8659-4253-b90e-fb37f6abd866","isSidechain":false,"promptId":"1ae97412-419d-4419-88c9-98c9952cf2ba","type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_01CRzyG4iqoAvagS9TxxkNqs","type":"tool_result","content":"firm_id,x,y\n8,0.779165,3.491493\n12,-0.672534,0.294464\n7,0.937099,3.91265\n0,0.92836,-0.116035\n    2001 panel.csv\n           firm_id            x            y\ncount  2000.000000  2000.000000  2000.000000\nmean     24.500000     0.160581     1.573239\nstd      14.434479     1.324819     1.782399\nmin       0.000000    -4.342107    -3.497863\n25%      12.000000    -0.691594     0.329463\n50%      24.500000     0.147255     1.584943\n75%      37.000000     1.096367     2.763766\nmax      49.000000     4.988282     7.202916\nfirm_id    0\nx          0\ny          0\ndtype: int64\nfirm_id      int64\nx          float64\ny          float64\ndtype: object\n0\ncount    50.0\nmean     40.0\nstd       0.0\nmin      40.0\n25%      40.0\n50%      40.0\n75%      40.0\nmax      40.0\nName: count, dtype: float64","is_error":false}]},"uuid":"730dfa2a-1607-4e11-ad10-d5cc140212ae","timestamp":"2026-10-30T03:32:19.867Z","toolUseResult":{"stdout":"firm_id,x,y\n8,0.779165,3.491493\n12,-0.672534,0.294464\n7,0.937099,3.91265\n0,0.92836,-0.116035\n    2001 panel.csv\n           firm_id            x            y\ncount  2000.000000  2000.000000  2000.000000\nmean     24.500000     0.160581     1.573239\nstd      14.434479     1.324819     1.782399\nmin       0.000000    -4.342107    -3.497863\n25%      12.000000    -0.691594     0.329463\n50%      24.500000     0.147255     1.584943\n75%      37.000000     1.096367     2.763766\nmax      49.000000     4.988282     7.202916\nfirm_id    0\nx          0\ny          0\ndtype: int64\nfirm_id      int64\nx          float64\ny          float64\ndtype: object\n0\ncount    50.0\nmean     40.0\nstd       0.0\nmin      40.0\n25%      40.0\n50%      40.0\n75%      40.0\nmax      40.0\nName: count, dtype: float64","stderr":"","interrupted":false,"isImage":false,"noOutputExpected":false},"sandboxed":false,"sourceToolAssistantUUID":"3602b38e-8659-4253-b90e-fb37f6abd866","permissionDecision":{"decision":"accept","source":"config","reasonType":"rule","ruleSource":"cliArg"},"userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001/data","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"type":"api-request-blob","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","hash":"44da213b135762d4ad327dce8534791158731de20166a0b3a75669942740fce9","message":{"role":"assistant","content":[{"type":"tool_use","id":"toolu_01CRzyG4iqoAvagS9TxxkNqs","name":"Bash","input":{"command":"cd data; head -5 panel.csv; wc -l panel.csv; ~/wp29/venv/bin/python -c \"\nimport pandas as pd;d=pd.read_csv('panel.csv');print(d.describe());print(d.isna().sum());print(d.dtypes);print(d.duplicated().sum());print(d.firm_id.value_counts().describe())\""}}]}}
++{"type":"api-request","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","id":"9aeeb46a-bdd2-4e94-9294-627e82802873","timestamp":"2026-10-30T03:32:19.870Z","version":"2.1.314","querySource":"sdk","shapeHash":"dd9840cc2450fc6f2fbc164e62044d5029f6e5eecc15cbeead7433b527e0847b","params":{"model":"claude-opus-5-5","betas":["claude-code-20250219","interleaved-thinking-2025-05-14","context-management-2025-06-27","prompt-caching-scope-2026-01-05","per-turn-control-2026-07-01","thinking-token-count-2026-05-13","mid-conversation-system-clear-at-2026-08-21","effort-2025-11-24","thinking-binding-controls-2026-08-01","thinking-display-updates-2026-08-18","cache-diagnosis-2026-04-07"],"max_tokens":128000,"thinking":{"type":"adaptive","display":"updates"},"context_management":{"edits":[{"type":"clear_thinking_20251015","keep":"all"}]},"output_config":{"effort":"medium"}},"base":"98500f1d-022c-462d-937b-45967a2e3ff2","keep":4,"tail":["44da213b135762d4ad327dce8534791158731de20166a0b3a75669942740fce9","6eefcae4cb855c870d65797a33fabc22c477f7b192f88f6f13a508689e0cd027"],"cacheControl":[{"path":["system",0],"value":{"type":"ephemeral"}},{"path":["system",1],"value":{"type":"ephemeral"}},{"path":["messages",5,"content",0],"value":{"type":"ephemeral"}}],"requestDigest":"ad2ff25f6c07be714ec10fc071689b50d3ce3872e88c1233874902a0005ede99"}
++{"parentUuid":"730dfa2a-1607-4e11-ad10-d5cc140212ae","isSidechain":false,"message":{"model":"claude-opus-5-5","id":"msg_011CgXiXoSaCuL1SRA9RDi8x","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"","signature":"CAQSywYKEAgTGAI4AUIIdGhpbmtpbmcSDEbEbYs+ce/ylUdbdRoMqjLelM5Oa/mbH7suIjB/wUGXYul5sgG3GUSLm9adscXKtEiMYkniqrvi9uF6qkXYhuVS7LBOCDxYChIB1fMq6AUeNu9lcj/7jqZMPIJkvkMBSSKHiCzHVhrGY/s3q2hEmHVWyj1V8v3uAnibC2OkHcghlt13t+iYxo8OCwjTjSPLnWy3b/8kDjvrgAeZncRI7pl8C0OEyyE3RCuKRkkskdPu4WS7XX2I+QGPF8js33+sqjuTtyxxCK7xY+/e91TceFqFTPoMq+qCRnkDKdP84tT9LbALBOJFeBTfPfI7DPTz+1+eC/f9Zs+1K6nHxQEAkni9075VHwwbe4MFi8Dft/VAmbnQWarwKpcHrX+B3pl18SIVA0ftlJdu3qm3IbqKtHdogngMBANk2dQ49fyAnOt4WpCTh/yYp6U4TMCQAzXA8ZAW4EvCDNqFcVImz/N68K//tJSmOEpIygFzg5QHfHMAlXdhsEbZyLaoKEuqXroNLq495aK45GZbtyymnGLwZLqXruq9TAHWhV44ay3LXE1bWqc1XL+/ca3qwJ2vhgw9A5QMO77ajvV7yLU34pLaiIQL38K+EoPzaokCUmJiLgc7sKkMxwr6hOcpHxdDvGjeF+teLdgdO8c3SQwa8H8pDAQ2UClLtrQ/PtB8GHSoOB84i4C/RxWeLvdInj83KTQmmPQZ4F2XSyBE8orLCSPuQP0e9GsR3P+e9A6LhKr/g15zcyIygehZugouTa8Un58iqHzwo9HWuokcyRsJDs7j1C5UP5WnEMUR7vbFzZpoJJv19vpW63XgF83M8JfSRb0b6QVNZVIeuXIt+JfXPdChPLyNOm6kBiRiwToyNmu3mwCnbgp/cP3NnHzQIRoVoTl1Knn/HGjPqtp++VDHwgG7c8iN0MnyT3t6aH4oOh3NNZS2fefxB7KzUST5EyZ8wGqRhJHjx+dpaOMnStArPzLkFEUNXLMcsFOzShIYPT98RbAGtel97ifBeqfo6uIHSXCryYGLfDVhURv6QZd+0p9kBsD7cAa822ETfPD2voqlbZUqDwaz+4KTRE5B2+BwwqQiJ/CVWiHQIHAYAQ=="}],"container":null,"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":509,"cache_read_input_tokens":3281,"output_tokens":1274,"output_tokens_details":{"thinking_tokens":98},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":509},"inference_geo":"global","iterations":[],"speed":"standard","fallback_credit":null},"input_transformations":[],"diagnostics":null,"context_management":null},"thinkingDurationMs":1166,"thinkingDisplay":"updates","apiBlockIndex":0,"requestId":"req_011CgXiXo9xMHcjQG3ViYZg7","requestRef":"9aeeb46a-bdd2-4e94-9294-627e82802873","requestedModel":"claude-opus-5-5","type":"assistant","uuid":"72bd8a2e-d11d-4c09-8cd9-8e04f7a3d6f0","timestamp":"2026-10-30T03:32:22.333Z","effort":"medium","perTurnEffort":"medium","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001/data","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"parentUuid":"72bd8a2e-d11d-4c09-8cd9-8e04f7a3d6f0","isSidechain":false,"message":{"model":"claude-opus-5-5","id":"msg_011CgXiXoSaCuL1SRA9RDi8x","type":"message","role":"assistant","content":[{"type":"text","text":"Data look clean: 2000 rows, 50 firms with 40 observations each, and no missing values or duplicate rows. Next I'll write the analysis script."}],"container":null,"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":509,"cache_read_input_tokens":3281,"output_tokens":1274,"output_tokens_details":{"thinking_tokens":98},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":509},"inference_geo":"global","iterations":[],"speed":"standard","fallback_credit":null},"input_transformations":[],"diagnostics":null,"context_management":null},"apiBlockIndex":1,"requestId":"req_011CgXiXo9xMHcjQG3ViYZg7","requestRef":"9aeeb46a-bdd2-4e94-9294-627e82802873","requestedModel":"claude-opus-5-5","type":"assistant","uuid":"760b9485-bf40-4f5b-ad2d-82a58688a612","timestamp":"2026-10-30T03:32:22.333Z","effort":"medium","perTurnEffort":"medium","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001/data","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"parentUuid":"760b9485-bf40-4f5b-ad2d-82a58688a612","isSidechain":false,"message":{"model":"claude-opus-5-5","id":"msg_011CgXiXoSaCuL1SRA9RDi8x","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01NS154ZCTjzdZmNsh6sFoWp","name":"Bash","input":{"command":"mkdir -p out; cat > out/analysis.py <<'EOF'\n\"\"\"F2-smoke-01: OLS slope of y on x with firm-clustered SE. Run from workspace root.\"\"\"\nimport json, os\nimport numpy as np, pandas as pd, statsmodels.api as sm\nfrom scipy import stats\n\ndf_raw = pd.read_csv(\"data/panel.csv\")\nn_input = len(df_raw)\nissues = []\ndf = df_raw.copy()\nna = df[[\"firm_id\", \"x\", \"y\"]].isna().any(axis=1)\nif na.any():\n    issues.append({\"issue\": f\"{int(na.sum())} rows with missing values\", \"handling\": \"dropped\"})\n    df = df[~na]\ndup = df.duplicated()\nif dup.any():\n    issues.append({\"issue\": f\"{int(dup.sum())} exact duplicate rows\", \"handling\": \"kept (cannot distinguish from genuine repeats); flagged\"})\nnonfinite = ~np.isfinite(df[[\"x\", \"y\"]]).all(axis=1)\nif nonfinite.any():\n    issues.append({\"issue\": f\"{int(nonfinite.sum())} non-finite values\", \"handling\": \"dropped\"})\n    df = df[~nonfinite]\n\nX = sm.add_constant(df[\"x\"].to_numpy())\ny = df[\"y\"].to_numpy()\ng = df[\"firm_id\"].to_numpy()\nols = sm.OLS(y, X).fit()\ncl = sm.OLS(y, X).fit(cov_type=\"cluster\", cov_kwds={\"groups\": g, \"use_correction\": True})\nG = int(pd.Series(g).nunique())\nbeta, alpha = float(ols.params[1]), float(ols.params[0])\nse_cl = float(cl.bse[1])\ntcrit = float(stats.t.ppf(0.975, G - 1))  # t with G-1 df for clustered inference\n\nres = {\n    \"task_id\": \"F2-smoke-01\",\n    \"beta_hat\": beta,\n    \"alpha_hat\": alpha,\n    \"se_ols\": float(ols.bse[1]),\n    \"se_cluster\": se_cl,\n    \"beta_ci_low\": beta - tcrit * se_cl,\n    \"beta_ci_high\": beta + tcrit * se_cl,\n    \"n_analysed\": int(len(df)),\n    \"n_clusters\": G,\n    \"feasibility\": \"complete\",\n    \"feasibility_reason\": None,\n    \"data_issues\": issues,\n    \"n_input_rows\": {\"panel.csv\": int(n_input)},\n    \"random_seed\": None,\n    # supplementary diagnostics\n    \"se_cluster_ratio_to_ols\": se_cl / float(ols.bse[1]),\n    \"t_crit_df\": G - 1,\n    \"t_crit\": tcrit,\n    \"cluster_sizes_min_max\": [int(pd.Series(g).value_counts().min()), int(pd.Series(g).value_counts().max())],\n}\nos.makedirs(\"out\", exist_ok=True)\nwith open(\"out/results.json\", \"w\") as f:\n    json.dump(res, f, indent=2)\nprint(json.dumps(res, indent=2))\nEOF\n~/wp29/venv/bin/python out/analysis.py"},"caller":{"type":"direct"}}],"container":null,"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":509,"cache_read_input_tokens":3281,"output_tokens":1274,"output_tokens_details":{"thinking_tokens":98},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":509},"inference_geo":"global","iterations":[],"speed":"standard","fallback_credit":null},"input_transformations":[],"diagnostics":null,"context_management":null},"apiBlockIndex":2,"requestId":"req_011CgXiXo9xMHcjQG3ViYZg7","requestRef":"9aeeb46a-bdd2-4e94-9294-627e82802873","requestedModel":"claude-opus-5-5","type":"assistant","uuid":"b6d737c2-a800-4059-a9ce-17ef11c4f598","timestamp":"2026-10-30T03:32:30.293Z","effort":"medium","perTurnEffort":"medium","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001/data","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"type":"last-prompt","lastPrompt":"Begin","leafUuid":"b6d737c2-a800-4059-a9ce-17ef11c4f598","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43"}
++{"type":"atis-latch","atis":"","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43"}
++{"parentUuid":"b6d737c2-a800-4059-a9ce-17ef11c4f598","isSidechain":false,"promptId":"1ae97412-419d-4419-88c9-98c9952cf2ba","type":"user","message":{"role":"user","content":[{"type":"tool_result","content":"Exit code 1\nTraceback (most recent call last):\n  File \"/Users/simon-pierreboucher/wp29/runs/RD0001/data/out/analysis.py\", line 6, in <module>\n    df_raw = pd.read_csv(\"data/panel.csv\")\n  File \"/Users/simon-pierreboucher/wp29/venv/lib/python3.14/site-packages/pandas/io/parsers/readers.py\", line 1026, in read_csv\n    return _read(filepath_or_buffer, kwds)\n  File \"/Users/simon-pierreboucher/wp29/venv/lib/python3.14/site-packages/pandas/io/parsers/readers.py\", line 620, in _read\n    parser = TextFileReader(filepath_or_buffer, **kwds)\n  File \"/Users/simon-pierreboucher/wp29/venv/lib/python3.14/site-packages/pandas/io/parsers/readers.py\", line 1620, in __init__\n    self._engine = self._make_engine(f, self.engine)\n                   ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^\n  File \"/Users/simon-pierreboucher/wp29/venv/lib/python3.14/site-packages/pandas/io/parsers/readers.py\", line 1880, in _make_engine\n    self.handles = get_handle(\n                   ~~~~~~~~~~^\n        f,\n        ^^\n    ...<6 lines>...\n        storage_options=self.options.get(\"storage_options\", None),\n        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n    )\n    ^\n  File \"/Users/simon-pierreboucher/wp29/venv/lib/python3.14/site-packages/pandas/io/common.py\", line 873, in get_handle\n    handle = open(\n        handle,\n    ...<3 lines>...\n        newline=\"\",\n    )\nFileNotFoundError: [Errno 2] No such file or directory: 'data/panel.csv'","is_error":true,"tool_use_id":"toolu_01NS154ZCTjzdZmNsh6sFoWp"}]},"uuid":"93cb594c-210f-411e-a67b-273eb38529ac","timestamp":"2026-10-30T03:32:44.330Z","toolUseResult":"Error: Exit code 1\nTraceback (most recent call last):\n  File \"/Users/simon-pierreboucher/wp29/runs/RD0001/data/out/analysis.py\", line 6, in <module>\n    df_raw = pd.read_csv(\"data/panel.csv\")\n  File \"/Users/simon-pierreboucher/wp29/venv/lib/python3.14/site-packages/pandas/io/parsers/readers.py\", line 1026, in read_csv\n    return _read(filepath_or_buffer, kwds)\n  File \"/Users/simon-pierreboucher/wp29/venv/lib/python3.14/site-packages/pandas/io/parsers/readers.py\", line 620, in _read\n    parser = TextFileReader(filepath_or_buffer, **kwds)\n  File \"/Users/simon-pierreboucher/wp29/venv/lib/python3.14/site-packages/pandas/io/parsers/readers.py\", line 1620, in __init__\n    self._engine = self._make_engine(f, self.engine)\n                   ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^\n  File \"/Users/simon-pierreboucher/wp29/venv/lib/python3.14/site-packages/pandas/io/parsers/readers.py\", line 1880, in _make_engine\n    self.handles = get_handle(\n                   ~~~~~~~~~~^\n        f,\n        ^^\n    ...<6 lines>...\n        storage_options=self.options.get(\"storage_options\", None),\n        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n    )\n    ^\n  File \"/Users/simon-pierreboucher/wp29/venv/lib/python3.14/site-packages/pandas/io/common.py\", line 873, in get_handle\n    handle = open(\n        handle,\n    ...<3 lines>...\n        newline=\"\",\n    )\nFileNotFoundError: [Errno 2] No such file or directory: 'data/panel.csv'","sandboxed":false,"sourceToolAssistantUUID":"b6d737c2-a800-4059-a9ce-17ef11c4f598","permissionDecision":{"decision":"accept","source":"config","reasonType":"rule","ruleSource":"cliArg"},"userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001/data","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"type":"api-request-blob","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","hash":"5724f05900ce35a7c6f3a860567e90f1d2d81576bd2fdbb1cc8a996b5509031e","message":{"role":"assistant","content":[{"type":"thinking","thinking":"","signature":"CAQSywYKEAgTGAI4AUIIdGhpbmtpbmcSDEbEbYs+ce/ylUdbdRoMqjLelM5Oa/mbH7suIjB/wUGXYul5sgG3GUSLm9adscXKtEiMYkniqrvi9uF6qkXYhuVS7LBOCDxYChIB1fMq6AUeNu9lcj/7jqZMPIJkvkMBSSKHiCzHVhrGY/s3q2hEmHVWyj1V8v3uAnibC2OkHcghlt13t+iYxo8OCwjTjSPLnWy3b/8kDjvrgAeZncRI7pl8C0OEyyE3RCuKRkkskdPu4WS7XX2I+QGPF8js33+sqjuTtyxxCK7xY+/e91TceFqFTPoMq+qCRnkDKdP84tT9LbALBOJFeBTfPfI7DPTz+1+eC/f9Zs+1K6nHxQEAkni9075VHwwbe4MFi8Dft/VAmbnQWarwKpcHrX+B3pl18SIVA0ftlJdu3qm3IbqKtHdogngMBANk2dQ49fyAnOt4WpCTh/yYp6U4TMCQAzXA8ZAW4EvCDNqFcVImz/N68K//tJSmOEpIygFzg5QHfHMAlXdhsEbZyLaoKEuqXroNLq495aK45GZbtyymnGLwZLqXruq9TAHWhV44ay3LXE1bWqc1XL+/ca3qwJ2vhgw9A5QMO77ajvV7yLU34pLaiIQL38K+EoPzaokCUmJiLgc7sKkMxwr6hOcpHxdDvGjeF+teLdgdO8c3SQwa8H8pDAQ2UClLtrQ/PtB8GHSoOB84i4C/RxWeLvdInj83KTQmmPQZ4F2XSyBE8orLCSPuQP0e9GsR3P+e9A6LhKr/g15zcyIygehZugouTa8Un58iqHzwo9HWuokcyRsJDs7j1C5UP5WnEMUR7vbFzZpoJJv19vpW63XgF83M8JfSRb0b6QVNZVIeuXIt+JfXPdChPLyNOm6kBiRiwToyNmu3mwCnbgp/cP3NnHzQIRoVoTl1Knn/HGjPqtp++VDHwgG7c8iN0MnyT3t6aH4oOh3NNZS2fefxB7KzUST5EyZ8wGqRhJHjx+dpaOMnStArPzLkFEUNXLMcsFOzShIYPT98RbAGtel97ifBeqfo6uIHSXCryYGLfDVhURv6QZd+0p9kBsD7cAa822ETfPD2voqlbZUqDwaz+4KTRE5B2+BwwqQiJ/CVWiHQIHAYAQ=="},{"type":"text","text":"Data look clean: 2000 rows, 50 firms with 40 observations each, and no missing values or duplicate rows. Next I'll write the analysis script."},{"type":"tool_use","id":"toolu_01NS154ZCTjzdZmNsh6sFoWp","name":"Bash","input":{"command":"mkdir -p out; cat > out/analysis.py <<'EOF'\n\"\"\"F2-smoke-01: OLS slope of y on x with firm-clustered SE. Run from workspace root.\"\"\"\nimport json, os\nimport numpy as np, pandas as pd, statsmodels.api as sm\nfrom scipy import stats\n\ndf_raw = pd.read_csv(\"data/panel.csv\")\nn_input = len(df_raw)\nissues = []\ndf = df_raw.copy()\nna = df[[\"firm_id\", \"x\", \"y\"]].isna().any(axis=1)\nif na.any():\n    issues.append({\"issue\": f\"{int(na.sum())} rows with missing values\", \"handling\": \"dropped\"})\n    df = df[~na]\ndup = df.duplicated()\nif dup.any():\n    issues.append({\"issue\": f\"{int(dup.sum())} exact duplicate rows\", \"handling\": \"kept (cannot distinguish from genuine repeats); flagged\"})\nnonfinite = ~np.isfinite(df[[\"x\", \"y\"]]).all(axis=1)\nif nonfinite.any():\n    issues.append({\"issue\": f\"{int(nonfinite.sum())} non-finite values\", \"handling\": \"dropped\"})\n    df = df[~nonfinite]\n\nX = sm.add_constant(df[\"x\"].to_numpy())\ny = df[\"y\"].to_numpy()\ng = df[\"firm_id\"].to_numpy()\nols = sm.OLS(y, X).fit()\ncl = sm.OLS(y, X).fit(cov_type=\"cluster\", cov_kwds={\"groups\": g, \"use_correction\": True})\nG = int(pd.Series(g).nunique())\nbeta, alpha = float(ols.params[1]), float(ols.params[0])\nse_cl = float(cl.bse[1])\ntcrit = float(stats.t.ppf(0.975, G - 1))  # t with G-1 df for clustered inference\n\nres = {\n    \"task_id\": \"F2-smoke-01\",\n    \"beta_hat\": beta,\n    \"alpha_hat\": alpha,\n    \"se_ols\": float(ols.bse[1]),\n    \"se_cluster\": se_cl,\n    \"beta_ci_low\": beta - tcrit * se_cl,\n    \"beta_ci_high\": beta + tcrit * se_cl,\n    \"n_analysed\": int(len(df)),\n    \"n_clusters\": G,\n    \"feasibility\": \"complete\",\n    \"feasibility_reason\": None,\n    \"data_issues\": issues,\n    \"n_input_rows\": {\"panel.csv\": int(n_input)},\n    \"random_seed\": None,\n    # supplementary diagnostics\n    \"se_cluster_ratio_to_ols\": se_cl / float(ols.bse[1]),\n    \"t_crit_df\": G - 1,\n    \"t_crit\": tcrit,\n    \"cluster_sizes_min_max\": [int(pd.Series(g).value_counts().min()), int(pd.Series(g).value_counts().max())],\n}\nos.makedirs(\"out\", exist_ok=True)\nwith open(\"out/results.json\", \"w\") as f:\n    json.dump(res, f, indent=2)\nprint(json.dumps(res, indent=2))\nEOF\n~/wp29/venv/bin/python out/analysis.py"}}]}}
++{"type":"api-request","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","id":"dad21f71-7407-4f95-ad55-d43cc9b4dcda","timestamp":"2026-10-30T03:32:44.333Z","version":"2.1.314","querySource":"sdk","shapeHash":"dd9840cc2450fc6f2fbc164e62044d5029f6e5eecc15cbeead7433b527e0847b","params":{"model":"claude-opus-5-5","betas":["claude-code-20250219","interleaved-thinking-2025-05-14","context-management-2025-06-27","prompt-caching-scope-2026-01-05","per-turn-control-2026-07-01","thinking-token-count-2026-05-13","mid-conversation-system-clear-at-2026-08-21","effort-2025-11-24","thinking-binding-controls-2026-08-01","thinking-display-updates-2026-08-18","cache-diagnosis-2026-04-07"],"max_tokens":128000,"thinking":{"type":"adaptive","display":"updates"},"context_management":{"edits":[{"type":"clear_thinking_20251015","keep":"all"}]},"output_config":{"effort":"medium"}},"base":"9aeeb46a-bdd2-4e94-9294-627e82802873","keep":6,"tail":["5724f05900ce35a7c6f3a860567e90f1d2d81576bd2fdbb1cc8a996b5509031e","dc1e8ecf1618ec9f9782d27740f6d2431401952a9934e265b953cfc2bddeb1a3"],"cacheControl":[{"path":["system",0],"value":{"type":"ephemeral"}},{"path":["system",1],"value":{"type":"ephemeral"}},{"path":["messages",7,"content",0],"value":{"type":"ephemeral"}}],"requestDigest":"3b8bfdafd6bafceab7bef8e36910a88dd51ac01ba785db9cebd42d5a193998d7"}
++{"parentUuid":"93cb594c-210f-411e-a67b-273eb38529ac","isSidechain":false,"message":{"model":"claude-opus-5-5","id":"msg_011CgXiZc5FyP8eTrZdxQGt5","type":"message","role":"assistant","content":[{"type":"text","text":"Oops, I created `out/` inside `data/` because of the earlier `cd`. I'll move it to the workspace root, leaving `data/` unchanged otherwise."}],"container":null,"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":1986,"cache_read_input_tokens":3790,"output_tokens":165,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":1986},"inference_geo":"global","iterations":[],"speed":"standard","fallback_credit":null},"input_transformations":[],"diagnostics":null,"context_management":null},"apiBlockIndex":0,"requestId":"req_011CgXiZbneMDkcZg3UuZ1Au","requestRef":"dad21f71-7407-4f95-ad55-d43cc9b4dcda","requestedModel":"claude-opus-5-5","type":"assistant","uuid":"33ca4450-65b4-400c-b7ef-930a5aafee63","timestamp":"2026-10-30T03:32:46.988Z","effort":"medium","perTurnEffort":"medium","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001/data","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"parentUuid":"33ca4450-65b4-400c-b7ef-930a5aafee63","isSidechain":false,"message":{"model":"claude-opus-5-5","id":"msg_011CgXiZc5FyP8eTrZdxQGt5","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_018BoGZZbgANmtntEHqCzSUP","name":"Bash","input":{"command":"cd /Users/simon-pierreboucher/wp29/runs/RD0001 && mv data/out ./out && ls data && ~/wp29/venv/bin/python out/analysis.py"},"caller":{"type":"direct"}}],"container":null,"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":1986,"cache_read_input_tokens":3790,"output_tokens":165,"output_tokens_details":{"thinking_tokens":0},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":1986},"inference_geo":"global","iterations":[],"speed":"standard","fallback_credit":null},"input_transformations":[],"diagnostics":null,"context_management":null},"apiBlockIndex":1,"requestId":"req_011CgXiZbneMDkcZg3UuZ1Au","requestRef":"dad21f71-7407-4f95-ad55-d43cc9b4dcda","requestedModel":"claude-opus-5-5","type":"assistant","uuid":"6ec7df79-6f03-49de-91d8-bfeee746e450","timestamp":"2026-10-30T03:32:47.583Z","effort":"medium","perTurnEffort":"medium","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001/data","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"parentUuid":"6ec7df79-6f03-49de-91d8-bfeee746e450","isSidechain":false,"promptId":"1ae97412-419d-4419-88c9-98c9952cf2ba","type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_018BoGZZbgANmtntEHqCzSUP","type":"tool_result","content":"panel.csv\n{\n  \"task_id\": \"F2-smoke-01\",\n  \"beta_hat\": 0.8050805975666369,\n  \"alpha_hat\": 1.4439580771601006,\n  \"se_ols\": 0.02411516952727311,\n  \"se_cluster\": 0.07313813443747043,\n  \"beta_ci_low\": 0.658104013711267,\n  \"beta_ci_high\": 0.9520571814220067,\n  \"n_analysed\": 2000,\n  \"n_clusters\": 50,\n  \"feasibility\": \"complete\",\n  \"feasibility_reason\": null,\n  \"data_issues\": [],\n  \"n_input_rows\": {\n    \"panel.csv\": 2000\n  },\n  \"random_seed\": null,\n  \"se_cluster_ratio_to_ols\": 3.032868350966999,\n  \"t_crit_df\": 49,\n  \"t_crit\": 2.0095752371292392,\n  \"cluster_sizes_min_max\": [\n    40,\n    40\n  ]\n}","is_error":false}]},"uuid":"70708220-f14f-42f0-a00e-dade2ed2515c","timestamp":"2026-10-30T03:32:48.539Z","toolUseResult":{"stdout":"panel.csv\n{\n  \"task_id\": \"F2-smoke-01\",\n  \"beta_hat\": 0.8050805975666369,\n  \"alpha_hat\": 1.4439580771601006,\n  \"se_ols\": 0.02411516952727311,\n  \"se_cluster\": 0.07313813443747043,\n  \"beta_ci_low\": 0.658104013711267,\n  \"beta_ci_high\": 0.9520571814220067,\n  \"n_analysed\": 2000,\n  \"n_clusters\": 50,\n  \"feasibility\": \"complete\",\n  \"feasibility_reason\": null,\n  \"data_issues\": [],\n  \"n_input_rows\": {\n    \"panel.csv\": 2000\n  },\n  \"random_seed\": null,\n  \"se_cluster_ratio_to_ols\": 3.032868350966999,\n  \"t_crit_df\": 49,\n  \"t_crit\": 2.0095752371292392,\n  \"cluster_sizes_min_max\": [\n    40,\n    40\n  ]\n}","stderr":"","interrupted":false,"isImage":false,"noOutputExpected":false},"sandboxed":false,"sourceToolAssistantUUID":"6ec7df79-6f03-49de-91d8-bfeee746e450","permissionDecision":{"decision":"accept","source":"config","reasonType":"rule","ruleSource":"cliArg"},"userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"type":"api-request-blob","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","hash":"be7f8e624d415b19bac656475bd0a97609ba929f07ceb3f21fedc4df64eb6c20","message":{"role":"assistant","content":[{"type":"text","text":"Oops, I created `out/` inside `data/` because of the earlier `cd`. I'll move it to the workspace root, leaving `data/` unchanged otherwise."},{"type":"tool_use","id":"toolu_018BoGZZbgANmtntEHqCzSUP","name":"Bash","input":{"command":"cd /Users/simon-pierreboucher/wp29/runs/RD0001 && mv data/out ./out && ls data && ~/wp29/venv/bin/python out/analysis.py"}}]}}
++{"type":"api-request","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","id":"a10288e7-c4b9-4618-9d56-885122cd6a1f","timestamp":"2026-10-30T03:32:48.543Z","version":"2.1.314","querySource":"sdk","shapeHash":"dd9840cc2450fc6f2fbc164e62044d5029f6e5eecc15cbeead7433b527e0847b","params":{"model":"claude-opus-5-5","betas":["claude-code-20250219","interleaved-thinking-2025-05-14","context-management-2025-06-27","prompt-caching-scope-2026-01-05","per-turn-control-2026-07-01","thinking-token-count-2026-05-13","mid-conversation-system-clear-at-2026-08-21","effort-2025-11-24","thinking-binding-controls-2026-08-01","thinking-display-updates-2026-08-18","cache-diagnosis-2026-04-07"],"max_tokens":128000,"thinking":{"type":"adaptive","display":"updates"},"context_management":{"edits":[{"type":"clear_thinking_20251015","keep":"all"}]},"output_config":{"effort":"medium"}},"base":"dad21f71-7407-4f95-ad55-d43cc9b4dcda","keep":8,"tail":["be7f8e624d415b19bac656475bd0a97609ba929f07ceb3f21fedc4df64eb6c20","915680de0af47bd7ba2f66aab28bcac09b85e420eab4b91513376166dbb8a365"],"cacheControl":[{"path":["system",0],"value":{"type":"ephemeral"}},{"path":["system",1],"value":{"type":"ephemeral"}},{"path":["messages",9,"content",0],"value":{"type":"ephemeral"}}],"requestDigest":"6b74058fb3b51419fb1319bc646b38d4aee973c5401d83797353bf44db1324bb"}
++{"parentUuid":"70708220-f14f-42f0-a00e-dade2ed2515c","isSidechain":false,"message":{"model":"claude-opus-5-5","id":"msg_011CgXiZv226AkrPFPqFXTp3","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"","signature":"CAQSlgcKEAgTGAI4AUIIdGhpbmtpbmcSDAMw67Wp4i3ldhY8+xoM++3rCzfenryB9WNrIjCHzag6z8Yv1biMdH3Ps4WQWS282K9lNW77MY6OxmjBo1QMrPuaiC+K1eP9F5jql9oqswbB5GyTQTuYfyEXKD9v7Km5M4xedJ79yGEyR88ojYvAY1TEfcNUvhSsJ9469JJcZ8lNDVL1TTRVNnbX3NLcvHBHdk89Ic1V0FtCQPjph4aYZaTwrypPlOkbCk1ppI6KHBP2b0Bj+vAiK2mVEmft6Ex4YHeQlTRkCTgEc/4mBv48RiMJqM4ILxNsN56+QMp+UJ/VbP+gkjYD2bGafUmqwcBsecrEnaRIqQKO9kxBgJ+zUNrUvxljgI2kmhIHFxBPu8foptQsCxaZzG82QQ4ItdrnxDNXqkWWsQDLNZFmtLVubhy06AMRNe1W0IBmWsBVyL0aBVGpXeWQMbiFsE6tLcNvDtrpVAimJlE6spKiPazXwYiJjfij6I2fHrY9INezAqlY3hxMfcHEGxhvncgsSwJtgE+/0Blh2ZAy7WFYu00WMqLXjikI/FV8JButp+iGEslpX2Jmn/9WiXpuQ83KI1KJ1Ha+NnzviReMcVQZq/aiw4nVplWVrh1eCDUu8KTRpvRiUrcOTM2fRWSdKawZuCa2EPR7iA3ptSjDFBpBR+0MsiTcwx0+OMUjGJ+0b41LhtNlrHLJ/68Gc//iya6Uo6Y+rzHJjsZCOelwKQnpnHTdLVt5FqfM1R78iDpXgRidY/JIrV9NiMawtLOkWbFUYP9b0gxmVIaF1Fx4T/QFMatjNzDMsra+FLaiUJ3P5Jxs8Vxh5XrMktGWc5afXKqiKWOjy/60bTt5lcMkhnrbBD6MkKdIJvi12ZPLSrJIgl9vzQYOCfUw9XduU4ij70KOzZEYntawvGMeeNBUdy68CcKLvT82DXpN4TUSBwNfmntc0PKNFrlpQFTp8rUuNuuwFX2pgBqYULT2R2jaeYBvuLVi1sHMJ0KiyrGW2KLfCqXudJaHhWLG/dF+2I/R1wAmnP7N+dvCwSiaco5Au9Gfp7uyr5EYknhauUJ2rLdADNOhWjnWvIwj/VyVRoxsaPcfYDZNDN7AHW3acoClmrrO2LxI61yil67ozSl49zE1E8JlPnogl+WT++Kvhs8RFdheOkUeUCRTMDq23xZWwmr1fkgvT/yJjUW3KRcBgnBkHznYkkpSQ4EYAQ=="}],"container":null,"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":494,"cache_read_input_tokens":5776,"output_tokens":1564,"output_tokens_details":{"thinking_tokens":64},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":494},"inference_geo":"global","iterations":[],"speed":"standard","fallback_credit":null},"input_transformations":[],"diagnostics":null,"context_management":null},"thinkingDurationMs":699,"thinkingDisplay":"updates","apiBlockIndex":0,"requestId":"req_011CgXiZuju742SjSRfaKLb7","requestRef":"a10288e7-c4b9-4618-9d56-885122cd6a1f","requestedModel":"claude-opus-5-5","type":"assistant","uuid":"cfdc155d-6143-4a19-93c0-ac7ece13e25d","timestamp":"2026-10-30T03:32:50.410Z","effort":"medium","perTurnEffort":"medium","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"parentUuid":"cfdc155d-6143-4a19-93c0-ac7ece13e25d","isSidechain":false,"message":{"model":"claude-opus-5-5","id":"msg_011CgXiZv226AkrPFPqFXTp3","type":"message","role":"assistant","content":[{"type":"text","text":"The clustered SE is about 3× the OLS SE. Now the report and claims files."}],"container":null,"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":494,"cache_read_input_tokens":5776,"output_tokens":1564,"output_tokens_details":{"thinking_tokens":64},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":494},"inference_geo":"global","iterations":[],"speed":"standard","fallback_credit":null},"input_transformations":[],"diagnostics":null,"context_management":null},"apiBlockIndex":1,"requestId":"req_011CgXiZuju742SjSRfaKLb7","requestRef":"a10288e7-c4b9-4618-9d56-885122cd6a1f","requestedModel":"claude-opus-5-5","type":"assistant","uuid":"5769d68b-3e3c-4839-9024-29cafd6670ab","timestamp":"2026-10-30T03:32:50.411Z","effort":"medium","perTurnEffort":"medium","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"parentUuid":"5769d68b-3e3c-4839-9024-29cafd6670ab","isSidechain":false,"message":{"model":"claude-opus-5-5","id":"msg_011CgXiZv226AkrPFPqFXTp3","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01PGRuZBY6oZjq3vYWSPTcQh","name":"Bash","input":{"command":"cat > out/report.md <<'EOF'\n# F2-smoke-01 — Slope estimate with firm-clustered uncertainty\n\n## Question\nEstimate the population slope β in the linear projection y = α + βx + ε for a synthetic firm panel. The standard error must allow for errors that are correlated within a firm and independent across firms.\n\n## Data and exclusions\n- Input: `data/panel.csv` (synthetic). 2000 rows read; 50 firms, each with exactly 40 observations.\n- Checks: no missing values, no non-finite values, no exact duplicate rows. Nothing was excluded, so all 2000 observations were analysed. `data_issues` is empty.\n\n## Method\n- Pooled OLS of y on x with an intercept (statsmodels).\n- Naïve baseline: conventional homoskedastic OLS standard error.\n- Main inference: cluster-robust (Liang–Zeger / CR1) variance clustered by `firm_id`, using the standard small-sample correction G/(G−1)·(N−1)/(N−K).\n- 95% CI: β̂ ± t(0.975, G−1)·SE_cluster with G−1 = 49 degrees of freedom (t = 2.0096). This is a common conservative choice when there are few clusters.\n- The procedure is deterministic, so no random seed is needed. `out/analysis.py` regenerates `out/results.json`.\n\n## Results\n| quantity | value |\n|---|---|\n| β̂ (slope) | 0.8051 |\n| α̂ (intercept) | 1.4440 |\n| SE, OLS (naïve) | 0.0241 |\n| SE, clustered by firm | 0.0731 |\n| 95% CI (clustered) | [0.6581, 0.9521] |\n| N / clusters | 2000 / 50 |\n\n## Uncertainty\nThe firm-clustered SE is about 3.03 times the naïve OLS SE. This means x and the residuals are both strongly correlated within firms. The naïve SE therefore greatly overstates precision: its implied interval would be roughly a third as wide. Even with the clustered SE, the interval excludes 0 by a wide margin.\n\n## Limitations and open issues\n- With 50 equal-sized clusters, CR1 with t(G−1) critical values should be reasonably reliable. Still, cluster-robust inference is asymptotic in G. A wild cluster bootstrap was not run.\n- β is a linear-projection (association) coefficient, not a causal effect. Pooled OLS mixes within-firm and between-firm variation in x. A firm fixed-effects slope could differ but was not the estimand asked for.\n- The data are synthetic, so the conclusions describe only this dataset.\nEOF\ncat > out/claims.json <<'EOF'\n[\n {\"id\":\"C1\",\"text\":\"The pooled OLS slope of y on x (with intercept) is 0.805, with intercept 1.444.\",\"results_keys\":[\"beta_hat\",\"alpha_hat\"],\"status\":\"supported\",\"caveat\":\"Association (linear projection), not causal.\"},\n {\"id\":\"C2\",\"text\":\"The firm-clustered SE of the slope (0.073) is about 3 times the naive OLS SE (0.024); naive inference substantially overstates precision.\",\"results_keys\":[\"se_cluster\",\"se_ols\",\"se_cluster_ratio_to_ols\"],\"status\":\"supported\",\"caveat\":\"CR1 cluster-robust SE with 50 clusters; asymptotic in number of clusters.\"},\n {\"id\":\"C3\",\"text\":\"The 95% clustered CI for beta is [0.658, 0.952], excluding zero.\",\"results_keys\":[\"beta_ci_low\",\"beta_ci_high\"],\"status\":\"supported\",\"caveat\":\"Uses t(49) critical value; wild cluster bootstrap not performed.\"},\n {\"id\":\"C4\",\"text\":\"All 2000 rows (50 firms x 40 obs) were analysed; no data issues were detected.\",\"results_keys\":[\"n_analysed\",\"n_clusters\",\"n_input_rows\",\"data_issues\"],\"status\":\"supported\",\"caveat\":\"\"}\n]\nEOF\n~/wp29/venv/bin/python -c \"import json;json.load(open('out/claims.json'))\" && ls out"},"caller":{"type":"direct"}}],"container":null,"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":494,"cache_read_input_tokens":5776,"output_tokens":1564,"output_tokens_details":{"thinking_tokens":64},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":494},"inference_geo":"global","iterations":[],"speed":"standard","fallback_credit":null},"input_transformations":[],"diagnostics":null,"context_management":null},"apiBlockIndex":2,"requestId":"req_011CgXiZuju742SjSRfaKLb7","requestRef":"a10288e7-c4b9-4618-9d56-885122cd6a1f","requestedModel":"claude-opus-5-5","type":"assistant","uuid":"f5aef34e-d2ce-46a6-b53d-5f5a863d2d89","timestamp":"2026-10-30T03:33:02.492Z","effort":"medium","perTurnEffort":"medium","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"parentUuid":"f5aef34e-d2ce-46a6-b53d-5f5a863d2d89","isSidechain":false,"promptId":"1ae97412-419d-4419-88c9-98c9952cf2ba","type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_01PGRuZBY6oZjq3vYWSPTcQh","type":"tool_result","content":"analysis.py\nclaims.json\nreport.md\nresults.json","is_error":false}]},"uuid":"2069dafd-4f38-4d8b-b2f5-bbbb12eb2b3f","timestamp":"2026-10-30T03:33:02.561Z","toolUseResult":{"stdout":"analysis.py\nclaims.json\nreport.md\nresults.json","stderr":"","interrupted":false,"isImage":false,"noOutputExpected":false},"sandboxed":false,"sourceToolAssistantUUID":"f5aef34e-d2ce-46a6-b53d-5f5a863d2d89","permissionDecision":{"decision":"accept","source":"config","reasonType":"rule","ruleSource":"cliArg"},"userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"type":"api-request-blob","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","hash":"53840f252016775ecc18142e77d997cb15183c2bb2e0685bbf0948b808aa3d6f","message":{"role":"assistant","content":[{"type":"thinking","thinking":"","signature":"CAQSlgcKEAgTGAI4AUIIdGhpbmtpbmcSDAMw67Wp4i3ldhY8+xoM++3rCzfenryB9WNrIjCHzag6z8Yv1biMdH3Ps4WQWS282K9lNW77MY6OxmjBo1QMrPuaiC+K1eP9F5jql9oqswbB5GyTQTuYfyEXKD9v7Km5M4xedJ79yGEyR88ojYvAY1TEfcNUvhSsJ9469JJcZ8lNDVL1TTRVNnbX3NLcvHBHdk89Ic1V0FtCQPjph4aYZaTwrypPlOkbCk1ppI6KHBP2b0Bj+vAiK2mVEmft6Ex4YHeQlTRkCTgEc/4mBv48RiMJqM4ILxNsN56+QMp+UJ/VbP+gkjYD2bGafUmqwcBsecrEnaRIqQKO9kxBgJ+zUNrUvxljgI2kmhIHFxBPu8foptQsCxaZzG82QQ4ItdrnxDNXqkWWsQDLNZFmtLVubhy06AMRNe1W0IBmWsBVyL0aBVGpXeWQMbiFsE6tLcNvDtrpVAimJlE6spKiPazXwYiJjfij6I2fHrY9INezAqlY3hxMfcHEGxhvncgsSwJtgE+/0Blh2ZAy7WFYu00WMqLXjikI/FV8JButp+iGEslpX2Jmn/9WiXpuQ83KI1KJ1Ha+NnzviReMcVQZq/aiw4nVplWVrh1eCDUu8KTRpvRiUrcOTM2fRWSdKawZuCa2EPR7iA3ptSjDFBpBR+0MsiTcwx0+OMUjGJ+0b41LhtNlrHLJ/68Gc//iya6Uo6Y+rzHJjsZCOelwKQnpnHTdLVt5FqfM1R78iDpXgRidY/JIrV9NiMawtLOkWbFUYP9b0gxmVIaF1Fx4T/QFMatjNzDMsra+FLaiUJ3P5Jxs8Vxh5XrMktGWc5afXKqiKWOjy/60bTt5lcMkhnrbBD6MkKdIJvi12ZPLSrJIgl9vzQYOCfUw9XduU4ij70KOzZEYntawvGMeeNBUdy68CcKLvT82DXpN4TUSBwNfmntc0PKNFrlpQFTp8rUuNuuwFX2pgBqYULT2R2jaeYBvuLVi1sHMJ0KiyrGW2KLfCqXudJaHhWLG/dF+2I/R1wAmnP7N+dvCwSiaco5Au9Gfp7uyr5EYknhauUJ2rLdADNOhWjnWvIwj/VyVRoxsaPcfYDZNDN7AHW3acoClmrrO2LxI61yil67ozSl49zE1E8JlPnogl+WT++Kvhs8RFdheOkUeUCRTMDq23xZWwmr1fkgvT/yJjUW3KRcBgnBkHznYkkpSQ4EYAQ=="},{"type":"text","text":"The clustered SE is about 3× the OLS SE. Now the report and claims files."},{"type":"tool_use","id":"toolu_01PGRuZBY6oZjq3vYWSPTcQh","name":"Bash","input":{"command":"cat > out/report.md <<'EOF'\n# F2-smoke-01 — Slope estimate with firm-clustered uncertainty\n\n## Question\nEstimate the population slope β in the linear projection y = α + βx + ε for a synthetic firm panel. The standard error must allow for errors that are correlated within a firm and independent across firms.\n\n## Data and exclusions\n- Input: `data/panel.csv` (synthetic). 2000 rows read; 50 firms, each with exactly 40 observations.\n- Checks: no missing values, no non-finite values, no exact duplicate rows. Nothing was excluded, so all 2000 observations were analysed. `data_issues` is empty.\n\n## Method\n- Pooled OLS of y on x with an intercept (statsmodels).\n- Naïve baseline: conventional homoskedastic OLS standard error.\n- Main inference: cluster-robust (Liang–Zeger / CR1) variance clustered by `firm_id`, using the standard small-sample correction G/(G−1)·(N−1)/(N−K).\n- 95% CI: β̂ ± t(0.975, G−1)·SE_cluster with G−1 = 49 degrees of freedom (t = 2.0096). This is a common conservative choice when there are few clusters.\n- The procedure is deterministic, so no random seed is needed. `out/analysis.py` regenerates `out/results.json`.\n\n## Results\n| quantity | value |\n|---|---|\n| β̂ (slope) | 0.8051 |\n| α̂ (intercept) | 1.4440 |\n| SE, OLS (naïve) | 0.0241 |\n| SE, clustered by firm | 0.0731 |\n| 95% CI (clustered) | [0.6581, 0.9521] |\n| N / clusters | 2000 / 50 |\n\n## Uncertainty\nThe firm-clustered SE is about 3.03 times the naïve OLS SE. This means x and the residuals are both strongly correlated within firms. The naïve SE therefore greatly overstates precision: its implied interval would be roughly a third as wide. Even with the clustered SE, the interval excludes 0 by a wide margin.\n\n## Limitations and open issues\n- With 50 equal-sized clusters, CR1 with t(G−1) critical values should be reasonably reliable. Still, cluster-robust inference is asymptotic in G. A wild cluster bootstrap was not run.\n- β is a linear-projection (association) coefficient, not a causal effect. Pooled OLS mixes within-firm and between-firm variation in x. A firm fixed-effects slope could differ but was not the estimand asked for.\n- The data are synthetic, so the conclusions describe only this dataset.\nEOF\ncat > out/claims.json <<'EOF'\n[\n {\"id\":\"C1\",\"text\":\"The pooled OLS slope of y on x (with intercept) is 0.805, with intercept 1.444.\",\"results_keys\":[\"beta_hat\",\"alpha_hat\"],\"status\":\"supported\",\"caveat\":\"Association (linear projection), not causal.\"},\n {\"id\":\"C2\",\"text\":\"The firm-clustered SE of the slope (0.073) is about 3 times the naive OLS SE (0.024); naive inference substantially overstates precision.\",\"results_keys\":[\"se_cluster\",\"se_ols\",\"se_cluster_ratio_to_ols\"],\"status\":\"supported\",\"caveat\":\"CR1 cluster-robust SE with 50 clusters; asymptotic in number of clusters.\"},\n {\"id\":\"C3\",\"text\":\"The 95% clustered CI for beta is [0.658, 0.952], excluding zero.\",\"results_keys\":[\"beta_ci_low\",\"beta_ci_high\"],\"status\":\"supported\",\"caveat\":\"Uses t(49) critical value; wild cluster bootstrap not performed.\"},\n {\"id\":\"C4\",\"text\":\"All 2000 rows (50 firms x 40 obs) were analysed; no data issues were detected.\",\"results_keys\":[\"n_analysed\",\"n_clusters\",\"n_input_rows\",\"data_issues\"],\"status\":\"supported\",\"caveat\":\"\"}\n]\nEOF\n~/wp29/venv/bin/python -c \"import json;json.load(open('out/claims.json'))\" && ls out"}}]}}
++{"type":"api-request","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","id":"38ee1909-f8d6-4e3f-9e2e-9f04550e5ab2","timestamp":"2026-10-30T03:33:02.564Z","version":"2.1.314","querySource":"sdk","shapeHash":"dd9840cc2450fc6f2fbc164e62044d5029f6e5eecc15cbeead7433b527e0847b","params":{"model":"claude-opus-5-5","betas":["claude-code-20250219","interleaved-thinking-2025-05-14","context-management-2025-06-27","prompt-caching-scope-2026-01-05","per-turn-control-2026-07-01","thinking-token-count-2026-05-13","mid-conversation-system-clear-at-2026-08-21","effort-2025-11-24","thinking-binding-controls-2026-08-01","thinking-display-updates-2026-08-18","cache-diagnosis-2026-04-07"],"max_tokens":128000,"thinking":{"type":"adaptive","display":"updates"},"context_management":{"edits":[{"type":"clear_thinking_20251015","keep":"all"}]},"output_config":{"effort":"medium"}},"base":"a10288e7-c4b9-4618-9d56-885122cd6a1f","keep":10,"tail":["53840f252016775ecc18142e77d997cb15183c2bb2e0685bbf0948b808aa3d6f","30e9831f2ba1417da828dc24a946c5a4d2382bd12e4e83a58e6dca3a95bc5282"],"cacheControl":[{"path":["system",0],"value":{"type":"ephemeral"}},{"path":["system",1],"value":{"type":"ephemeral"}},{"path":["messages",11,"content",0],"value":{"type":"ephemeral"}}],"requestDigest":"e4a50224fa97a4a9a67a728b14678150e276fe07641fd3fa45bb5158aec68b19"}
++{"parentUuid":"2069dafd-4f38-4d8b-b2f5-bbbb12eb2b3f","isSidechain":false,"message":{"model":"claude-opus-5-5","id":"msg_011CgXiax1bxBQppJDWCmbis","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"","signature":"CAQSvwYKEAgTGAI4AUIIdGhpbmtpbmcSDBALktoTppfkgeOnyBoMZORr72urSvp3x7u/IjCfyOqbXsYoKcnmIyt9eO4gX8BjGXj2W8zPH691m7ufihDdN1GyarN7OlCj/7oKHNoq3AVCAqcw1Qh0JyIL+E2vhYC0ew892ze9WXYWJxFJ/3rjVlPTCFkASiAjI2mXFfFIwTVfp3lwAiNtCes2N6mg3VaDui0734iImD7uYVOBuzchYUO16JrB/LNAki1Njc/tVDaNCG+aBqMLehUXau6iYyBoiuael37JrjCLakbqQ6DFpUqTRYDIVkWo9DyrEzkIlggpUl44F3wJ8JqUBqAfLRe4XJqbex+1SGJh4jKYnBNFps9MwzRvvg+Fmn3QT4dCYO2sIWbIt1/Vlo5IF/5r+Z2+Rf+MgL/KQGLfu3Ah/qO3mX0hsPt0kj/TgnJ3AnuhgeYaMvQQggvFTqzQflUMsJOQuO98HTam1BMbtH5/ITiuMO43bSSyE6btNmSyGP8eWs4rQLMbWziF3XNh/8FHetPBnRHC8oEKiAkq0Ey6H69zyJsmkHjoSDMDiFSs95zFz1jgZzmckBQVpBzhBEgB/LsdYMQTYrTsvMHkOKP7yUxvL7U+envN+VxvoTYpUUjtdn03Gfk83ijFuQGSMZ80TY+BKXY/wa+dRwD5h5RXbbS+6VJQbZ6Xlx8BYV9kuXUx1pgLurfvTUjorx+0BK+t7k9q7FuG7GkuLi0xhZy4yHHGVdJEH1bWp1wDVqwFddlMius0ZMnsgNNSbP4aEZtQDsW6tHzuY/p4eZkDApC+ZozVGqO5UzylKT0Yiy8qq3G7vCou54AWXiQXnjK5IaFZu4eZCcsHBUOuUs3btshRrBxYSN3SfIStE1Ybbfq+9oALbsU7sHSWJv8PkpsNjJXsTAcp9tT5hdEhFapwhRnY4OdWMN92g/WIxOLjMZBUwcNGAC+2FCoAVJsPj9ksuK3HP9N7E1GbdTV5o3OT4onOR+JvaSOSGjWE2B+mBvsYmp+QkQ2ZwpY4HLOJSyO/jPjDkS/9Qgs37gqjDn+LnNGsy9jkYOHe0sU3G/omnx6j5KhVCZWpYq4gLWfPpiV0rJgYAQ=="}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":1558,"cache_read_input_tokens":6270,"output_tokens":615,"output_tokens_details":{"thinking_tokens":32},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":1558},"inference_geo":"global","iterations":[],"speed":"standard","fallback_credit":null},"input_transformations":[],"diagnostics":null,"context_management":null},"thinkingDurationMs":4987,"thinkingDisplay":"updates","apiBlockIndex":0,"requestId":"req_011CgXiawhk7mXk7ytXFBL7P","requestRef":"38ee1909-f8d6-4e3f-9e2e-9f04550e5ab2","requestedModel":"claude-opus-5-5","type":"assistant","uuid":"3acee4af-1fa7-42bf-8bb3-fbd7af5e503c","timestamp":"2026-10-30T03:33:08.741Z","effort":"medium","perTurnEffort":"medium","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"parentUuid":"3acee4af-1fa7-42bf-8bb3-fbd7af5e503c","isSidechain":false,"message":{"model":"claude-opus-5-5","id":"msg_011CgXiax1bxBQppJDWCmbis","type":"message","role":"assistant","content":[{"type":"text","text":"I estimated the slope of y on x, along with a standard error that allows for observations from the same firm being related, and the task is complete. Re-running `out/analysis.py` from the workspace root regenerates `out/results.json` the same way every time.\n\n**Data:** all 2000 rows were used: 50 firms with exactly 40 observations each. There were no missing values, non-finite values or duplicate rows, so nothing was excluded.\n\n| | value |\n|---|---|\n| Slope (β̂) | 0.8051 |\n| Intercept (α̂) | 1.4440 |\n| Standard error, ordinary OLS | 0.0241 |\n| Standard error, clustered by firm | 0.0731 |\n| 95% confidence interval (clustered) | [0.658, 0.952] |\n\nThe clustered standard error is about 3 times the ordinary one, so ignoring the firm structure would greatly overstate precision. The clustered interval still excludes zero by a wide margin. It uses the standard small-sample correction and a t critical value with 49 degrees of freedom (number of firms minus one).\n\n**Limitations:**\n- The slope is a pooled association, not a causal effect. A within-firm (fixed-effects) estimate could differ, but it isn't what the task asked for.\n- The clustered standard error relies on having enough firms. 50 equal-sized firms should be adequate, but I didn't run a bootstrap check.\n- The data are synthetic, so the results describe only this dataset.\n\nOne housekeeping note: I briefly created `out/` inside `data/` by mistake and then moved it to the workspace root. `data/` again contains only the untouched `panel.csv`.\n\nFiles are in `out/`:\n- `analysis.py`\n- `results.json`\n- `report.md`\n- `claims.json`"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":1558,"cache_read_input_tokens":6270,"output_tokens":615,"output_tokens_details":{"thinking_tokens":32},"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":1558},"inference_geo":"global","iterations":[],"speed":"standard","fallback_credit":null},"input_transformations":[],"diagnostics":null,"context_management":null},"apiBlockIndex":1,"requestId":"req_011CgXiawhk7mXk7ytXFBL7P","requestRef":"38ee1909-f8d6-4e3f-9e2e-9f04550e5ab2","requestedModel":"claude-opus-5-5","type":"assistant","uuid":"9e79d703-3e96-4629-91a8-0381ab9e96ff","timestamp":"2026-10-30T03:33:08.746Z","effort":"medium","perTurnEffort":"medium","userType":"external","entrypoint":"sdk-cli","cwd":"/Users/simon-pierreboucher/wp29/runs/RD0001","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","version":"2.1.314","gitBranch":"HEAD"}
++{"type":"last-prompt","lastPrompt":"Begin","leafUuid":"9e79d703-3e96-4629-91a8-0381ab9e96ff","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43"}
++{"type":"atis-latch","atis":"","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43"}
++{"type":"cost-state","sessionId":"587bafe6-c88b-4a80-9f62-4a10d9525c43","totalCostUSD":0.12040619999999999,"totalAPIDuration":37367,"totalAPIDurationWithoutRetries":37361,"totalToolDuration":15422,"totalLinesAdded":0,"totalLinesRemoved":0,"totalDuration":55982,"startTime":1793331132771,"modelUsage":{"claude-opus-5-5":{"inputTokens":14,"outputTokens":3847,"thinkingTokens":194,"cacheReadInputTokens":21351,"cacheCreationInputTokens":7828,"cacheCreationEphemeral5mInputTokens":7828,"cacheCreationEphemeral1hInputTokens":0,"webSearchRequests":0,"costUSD":0.12040619999999999}},"hasUnknownModelCost":false}

added runs/development/RD0001/system_append.md +16 −0

++You are a research assistant executing one bounded scientific task inside the current working directory, which is your workspace.
++
++Environment facts (identical for every task):
++- The task statement is the file `TASK.md` in the workspace. It is the complete and only authoritative description of what is required: the scientific question, the inputs under `data/`, the required output files and their exact keys, and the information-availability rules.
++- There is no network access. Use the Python interpreter `~/wp29/venv/bin/python` (numpy, pandas, scipy, statsmodels, matplotlib are installed). Standard shell tools are available. Do not try to install packages.
++- Write every deliverable under `./out/`. The required deliverables are listed in `TASK.md`; at minimum they always include `out/results.json`, `out/report.md`, and a replication entry point `out/analysis.py` that regenerates `out/results.json` from `data/` when run with `~/wp29/venv/bin/python out/analysis.py` from the workspace root.
++- You are working alone and unattended; nobody will answer questions. Make reasonable, documented decisions.
++
++Integrity rules (identical for every task):
++- Never fabricate numbers, data, results, or references. Every number in `out/results.json` and `out/report.md` must be produced by code you ran on the provided inputs.
++- Respect the information-availability rules in `TASK.md` exactly (for example, never use information that would not have been available at the stated decision time).
++- If part of the task cannot be completed as specified, say so explicitly in `out/report.md`, explain why, and still write every required file (use `null` for values you could not legitimately produce). An honest statement of a limitation is always preferred to a fabricated or guessed value.
++- Do not modify anything under `data/`.
++- When you are done, end with a brief plain-text summary of what you produced and any limitations.
++
++Complete the task described in `TASK.md`.

added runs/development/RD0002/artifact_manifest.json +21 −0

++{
++ "root": "/Users/simon-pierreboucher/Desktop/Recherche-UQO/methodes/autonomous-research-claude-code/runs/development/RD0002/out",
++ "files": {
++  "analysis.py": "517deda8e11d8fd3a2d19e233d660ee8fc57d6b3ee5b212f252059516d9ceb4c",
++  "claims.json": "687ead7f84aed6f6e8fd46feb74baa27e99d6ef2c4f52d62b52f8913d8521b37",
++  "exclusions.csv": "21cb2e267ec3fa8db076b2a2aa239cd2fcbd7167f4dabdf7ecff1119ff970f52",
++  "report.md": "ac11984e713efa30d32783182af1cd2b50a326666e5d43e97062ae08a694031a",
++  "results.json": "49b2eb88dab312b69e36b9c353f2ecd6406cf7a76c272d4bd3a9e801f2f9dd01",
++  "review/review.json": "89b033ee832a3b7a0502877e99d44790130b59d3dfd8f125eff77b1de6d53149",
++  "state/checks.md": "5e0991a153c0d9200ad4556bf3f2d6475a4f40e5b078b619a00b6108886d6676",
++  "state/contract.md": "72f4f9a0e7578a524953e1990ced1a2ae371b7b2dcde938b4c461bb2c38f8cd7",
++  "state/data_audit.md": "0d4524a4e1239c1c4d82e67c30948cec8c567ff0b17a4b644c09e99f4172eee6",
++  "state/gate_receipts/20261030T033337.458700Z.json": "ee785a77554714b5fa4cdb46e7518d479e9ebcc62e03b146f46bae5d8e145ded",
++  "state/gate_receipts/20261030T033342.972567Z.json": "e1b15485d2df6e9e5e5e95d2f03546e37f77e32d7d4f09b6c36a657da52093d4",
++  "state/gate_receipts/20261030T033346.013867Z.json": "ce87434ed1ce3dd052d7e28b1a7c653794d609d3598b055ad58fcdc8e2cd6152",
++  "state/gate_receipts/20261030T033353.528662Z.json": "52aa03fc109ed882c3b8f10b590fd7062c5a88c23ba279a655b2c0f190cb5ad7",
++  "state/log.md": "8c1f75c3dc23b0a0ada4cd9ba5420b6341c314da86ebec00b3e21928fe502e9f",
++  "state/phases.json": "966ca88665d89e886d68958c1f2bd2e67869621b855522ec571e524b6c32b4c6"
++ },
++ "recorded_utc": "2026-10-30T03:34:31.579+00:00"
++}
+ \ No newline at end of file

added runs/development/RD0002/behaviour_audit.json +5 −0

++{
++ "n_outside_workspace_refs": 0,
++ "outside_refs": [],
++ "n_gate_invocations": 4
++}
+ \ No newline at end of file

added runs/development/RD0002/contract_receipt.json +172 −0

++{
++ "gate_version": "pgar_gate/1.0",
++ "workspace": "/Users/simon-pierreboucher/wp29/runs/RD0002",
++ "utc": "2026-10-30T03:34:32.227275+00:00",
++ "passed": true,
++ "n_checks": 32,
++ "n_failed": 0,
++ "checks": [
++  {
++   "id": "file:out/results.json",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "file:out/report.md",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "file:out/analysis.py",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "file:out/claims.json",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "results_json",
++   "passed": true,
++   "detail": "parsed"
++  },
++  {
++   "id": "key:task_id",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:beta_hat",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:alpha_hat",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:se_ols",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:se_cluster",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:beta_ci_low",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:beta_ci_high",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:n_analysed",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:n_clusters",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:feasibility",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:feasibility_reason",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:data_issues",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:n_input_rows",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "key:random_seed",
++   "passed": true,
++   "detail": "ok"
++  },
++  {
++   "id": "section:Question",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "section:Data and exclusions",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "section:Method",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "section:Results",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "section:Uncertainty",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "section:Limitations and open issues",
++   "passed": true,
++   "detail": "present"
++  },
++  {
++   "id": "claims_json",
++   "passed": true,
++   "detail": "4 claims, all keys resolve"
++  },
++  {
++   "id": "phase_ledger",
++   "passed": true,
++   "detail": "consistent"
++  },
++  {
++   "id": "exclusion_ledger",
++   "passed": true,
++   "detail": "3 steps consistent"
++  },
++  {
++   "id": "ci:beta_hat",
++   "passed": true,
++   "detail": "0.6581040137112675 <= 0.8050805975666372 <= 0.9520571814220069"
++  },
++  {
++   "id": "positive:se_ols",
++   "passed": true,
++   "detail": "0.02411516952727311"
++  },
++  {
++   "id": "positive:se_cluster",
++   "passed": true,
++   "detail": "0.0731381344374704"
++  },
++  {
++   "id": "reproducibility",
++   "passed": true,
++   "detail": "regenerated in 0.6s; identical within tolerance"
++  }
++ ],
++ "receipt_path": "/Users/simon-pierreboucher/wp29/runs/RD0002/out/state/gate_receipts/20261030T033432.227531Z.json",
++ "receipt_sha256": "a71de108a16550a275b21c34dc2910507d301653f716c6c8d7e96dd2a19a8de4"
++}
+ \ No newline at end of file

added runs/development/RD0002/ctl/prompt.txt +1 −0

				`@@ -0,0 +1 @@`
		1	+Begin now. The task is in `TASK.md` in the current directory. Work until the deliverables are complete, then stop.

added runs/development/RD0002/ctl/reviewer_prompt.txt +1 −0

				`@@ -0,0 +1 @@`
		1	+Review the deliverables in this workspace now, as instructed, and write out/review/review.json.

added runs/development/RD0002/ctl/reviewer_system.md +10 −0

++You are an independent scientific reviewer. In the current working directory, `TASK.md` describes a bounded scientific task and the directory `out/` contains the deliverables submitted by another assistant (you have not seen how they were produced and you must not look for their working transcript). There is no network access. Use `~/wp29/venv/bin/python` if you need to re-run or test code. Do not modify any file under `out/` except that you must create `out/review/review.json`.
++
++Review the deliverables against `TASK.md` only. Specifically check:
++1. Does `out/results.json` contain every required key with legitimately computed values? Re-run `out/analysis.py` in a scratch copy if needed and compare.
++2. Are the information-availability rules of `TASK.md` respected by the code (for example no use of future information, no use of forbidden inputs)? Read the code to verify; do not trust the report.
++3. Are the central claims in `out/report.md` and `out/claims.json` supported by the numbers in `out/results.json`, with appropriate uncertainty and caveats? Flag any claim that is stronger than the evidence.
++4. Are counts, exclusions, units, signs and magnitudes internally consistent? Are baseline comparisons present when the task requires them?
++5. Are limitations stated honestly? Is anything declared complete that is not?
++
++You have no reference answers; do not guess what the "right" number is. Report only issues you can justify from the files. Write `out/review/review.json` as {"verdict": "accept"|"revise", "issues": [{"id": "R1", "severity": "critical"|"major"|"minor", "location": "<file or key>", "issue": "...", "suggested_fix": "..."}], "summary": "..."}. Use "revise" if there is at least one critical or major issue. Be concise and specific. End with a one-paragraph plain-text summary.

added runs/development/RD0002/ctl/system_append.md +59 −0

++You are a research assistant executing one bounded scientific task inside the current working directory, which is your workspace.
++
++Environment facts (identical for every task):
++- The task statement is the file `TASK.md` in the workspace. It is the complete and only authoritative description of what is required: the scientific question, the inputs under `data/`, the required output files and their exact keys, and the information-availability rules.
++- There is no network access. Use the Python interpreter `~/wp29/venv/bin/python` (numpy, pandas, scipy, statsmodels, matplotlib are installed). Standard shell tools are available. Do not try to install packages.
++- Write every deliverable under `./out/`. The required deliverables are listed in `TASK.md`; at minimum they always include `out/results.json`, `out/report.md`, and a replication entry point `out/analysis.py` that regenerates `out/results.json` from `data/` when run with `~/wp29/venv/bin/python out/analysis.py` from the workspace root.
++- You are working alone and unattended; nobody will answer questions. Make reasonable, documented decisions.
++
++Integrity rules (identical for every task):
++- Never fabricate numbers, data, results, or references. Every number in `out/results.json` and `out/report.md` must be produced by code you ran on the provided inputs.
++- Respect the information-availability rules in `TASK.md` exactly (for example, never use information that would not have been available at the stated decision time).
++- If part of the task cannot be completed as specified, say so explicitly in `out/report.md`, explain why, and still write every required file (use `null` for values you could not legitimately produce). An honest statement of a limitation is always preferred to a fabricated or guessed value.
++- Do not modify anything under `data/`.
++- When you are done, end with a brief plain-text summary of what you produced and any limitations.
++
++Complete the task described in `TASK.md` by following this research protocol. The protocol adds no scientific information to the task; it organizes how you work and what evidence you leave behind.
++
++PHASES (execute in order; a phase is eligible only when its dependencies are satisfied):
++
++P0 Scope and contract (depends on: nothing)
++- Read `TASK.md` completely. In `out/state/contract.md`, restate in your own words: the scientific question; the quantity or quantities to be estimated (the estimands) and the assumptions under which they are meaningful; every required output file and key; the information-availability rules; what would make the task impossible or the result invalid.
++- Write the initial phase ledger `out/state/phases.json`: a JSON object mapping each phase id (P0 … P5) to {"status": "pending"|"running"|"passed"|"failed"|"blocked"|"stale"|"skipped_optional", "checks": [], "notes": ""}. Keep it current as you work. A phase may only be set to "passed" when the checks listed for it in this protocol have actually been executed and their outcome recorded in "checks".
++
++P1 Data audit and provenance (depends on: P0)
++- Inspect every input file: compute and record its SHA-256, row and column counts, schema, units, date ranges, missing values, duplicates and anomalies in `out/state/data_audit.md`.
++- Keep an exclusion ledger in `out/exclusions.csv` (columns: step, rule, n_before, n_after, n_removed, rationale) for every filtering step you apply, so that n_raw − total removed = n_analysed.
++- Identify any input that violates the information-availability rules and decide, in writing, how you will handle it.
++
++P2 Baseline (depends on: P1)
++- Before any elaborate method, compute the simplest valid baseline that answers the question (for example a naïve or unconditional estimate), with the same evaluation rule used later. Record it in `out/results.json` under the keys the task requires, or under `baseline_*` keys when the task does not name them.
++
++P3 Method and computation (depends on: P2)
++- Implement the analysis in `out/analysis.py` so that it reads only `data/`, writes `out/results.json` deterministically (fix every random seed you use and record it), and prints nothing that contradicts the written results. Any figure or table must be generated by this script.
++
++P4 Verification (depends on: P3)
++- Re-run `out/analysis.py` from a clean state and confirm it regenerates `out/results.json` with identical values. Record the check in `out/state/phases.json` under P4.
++- Check internal consistency: identities that must hold (for example counts that must add up, standard errors that must be positive, confidence intervals that must contain their estimates, date orderings that must respect the information rules), units, and magnitudes against the baseline. Record every check and its outcome in `out/state/checks.md`. A failed check must be fixed or honestly reported, never silently ignored.
++
++P5 Reporting (depends on: P4)
++- Write `out/report.md` with the sections: Question; Data and exclusions; Method; Results; Uncertainty; Limitations and open issues. Every quantitative statement must cite the key of `out/results.json` it comes from.
++- Write `out/claims.json`: a list of the central claims of your report, each as {"id": "C1", "text": "...", "results_keys": ["..."], "status": "supported"|"unsupported"|"not_evaluated", "caveat": "..."}. Do not state a claim in the report that is not in this ledger.
++- Set the final statuses in `out/state/phases.json`.
++
++STOPPING RULES
++- Stop and declare the task complete only when every phase P0–P5 is "passed" with recorded checks. If a phase is "blocked" (missing information, impossible requirement, violated availability rule), stop, set the status, explain in `out/report.md` (section Limitations and open issues) and still write every required file with `null` for values you could not legitimately produce.
++- Do not spend effort on extensions that the task does not require.
++
++RECOVERY RULES
++- Append a short dated entry to `out/state/log.md` after each phase transition and after any important decision, so that work can be resumed by reading `out/state/`.
++- If you discover that an upstream input or decision changed after a downstream phase passed, mark the downstream phases "stale" and redo them; do not reuse stale results.
++- If you are interrupted and later resumed, first read `out/state/` and continue from the last consistent state; never re-run expensive steps that already have recorded, unchanged outputs.
++
++EXECUTABLE GATES (additional, mandatory)
++- A read-only gate program is provided at `~/wp29/gates/pgar_gate.py`. Run it from the workspace root as `~/wp29/venv/bin/python ~/wp29/gates/pgar_gate.py .` It checks the output contract of `TASK.md` (required files, keys, types, finite values, report sections, the claims ledger, the phase ledger, the exclusion ledger) and reproducibility (it re-runs `out/analysis.py` in a temporary copy and compares the regenerated `out/results.json` with yours). It never checks scientific correctness and it contains no reference answers.
++- You must run the gate at the end of P4 and again at the end of P5, and record each run's receipt (the gate writes `out/state/gate_receipts/<timestamp>.json`) in `out/state/phases.json`. You may not declare the task complete while the latest gate receipt reports `"passed": false`. Fix the reported problems and re-run the gate.
++- The controller will run the same gate after your session ends. If it fails, you will be resumed with the gate output and asked to fix the problems; each such cycle consumes your remaining budget.
++
++SEPARATE REVIEW (additional)
++- After the gate passes, a separate reviewer session that has not seen your work in progress will examine `TASK.md` and everything under `out/` and may return a list of issues (file `out/review/review.json`). You will then be resumed once to address them: fix what is justified, explain in `out/report.md` (Limitations and open issues) what you disagree with, re-run the gate, and update `out/claims.json` and `out/state/phases.json`. The reviewer has no reference answers; its role is to challenge unsupported claims, violated information rules, unverified computations, and inconsistencies between your report and your results.

added runs/development/RD0002/evaluation_receipt.json +81 −0

++{
++ "critical_violations": [],
++ "evaluated_utc": "2026-10-30T03:34:32.317997+00:00",
++ "evaluator_version": "F2-smoke-01/1.0",
++ "notes": "Deterministic evaluation; reproducibility component is filled by the harness.",
++ "oracle_sha256": "09c41858f3e534a31189c09ef6d7b2f8f4e4f1a222c073a42c460f42508aeffa",
++ "quality_components": {
++  "baseline": 1.0,
++  "correctness": 1.0,
++  "integrity": 1.0,
++  "reporting": 1.0,
++  "reproducibility": 1.0
++ },
++ "quality_score": 1.0,
++ "receipt_sha256": "3101d63ee689c5111220debab87067ede057d464ca48baa63e69f4db86b1a688",
++ "required_criteria": [
++  {
++   "description": "OLS slope equals reference to 1e-4",
++   "expected": {
++    "beta_hat": 0.8050805975666373
++   },
++   "id": "beta_point",
++   "observed": {
++    "beta_hat": 0.8050805975666372
++   },
++   "passed": true,
++   "tolerance": {
++    "beta_hat": {
++     "abs": 0.0001
++    }
++   }
++  },
++  {
++   "description": "Cluster-robust SE within 8% of CR1 reference (covers CR1/CR0/CR3 conventions and t vs normal)",
++   "expected": {
++    "se_cluster": 0.07313813443747041
++   },
++   "id": "se_cluster",
++   "observed": {
++    "se_cluster": 0.0731381344374704
++   },
++   "passed": true,
++   "tolerance": {
++    "se_cluster": {
++     "rel": 0.08
++    }
++   }
++  },
++  {
++   "description": "95% CI bounds within 0.03 of reference (t(G-1) or normal critical values both accepted)",
++   "expected": {
++    "beta_ci_high": 0.9520571814220071,
++    "beta_ci_low": 0.6581040137112675
++   },
++   "id": "ci",
++   "observed": {
++    "beta_ci_high": 0.9520571814220069,
++    "beta_ci_low": 0.6581040137112675
++   },
++   "passed": true,
++   "tolerance": {
++    "beta_ci_high": {
++     "abs": 0.03
++    },
++    "beta_ci_low": {
++     "abs": 0.03
++    }
++   }
++  },
++  {
++   "description": "n_analysed and n_clusters exact",
++   "expected": {
++    "n_analysed": 2000,
++    "n_clusters": 50
++   },
++   "id": "counts",
++   "observed": {
++    "n_analysed": 2000,
++    "n_clusters": 50
++   },
++   "passed": true,

Diff truncated — file too large.

spb/wp29_uqo

Phases 02-05: protocol, harness, viewer/capture, live smoke experiment (4 arms)

Theme