1"""Phase B — search audit and PRISMA counts from the actual records: data/metadata/prisma_counts.json, reports/search_audit.md."""2import json3from pathlib import Path4import pandas as pd56ROOT = Path(__file__).resolve().parents[2]; META = ROOT / "data" / "metadata"789def main():10 log = pd.read_csv(META / "search_log.csv")11 rec = pd.read_csv(META / "records_dedup.csv", low_memory=False)12 summ = json.load(open(META / "screening_summary.json"))13 llm = pd.read_csv(META / "screening_stage1_llm.csv")14 retr = pd.read_csv(META / "retrieval_log.csv")15 pdfs = {p.stem for p in (ROOT / "data" / "raw" / "pdfs").glob("*.pdf")}16 jsons = list((ROOT / "data" / "extracted" / "json").glob("*.json"))17 elig = 0; inel = 018 for f in jsons:19 try:20 j = json.load(open(f)); elig += bool(j.get("eligible")); inel += (not j.get("eligible"))21 except Exception: pass22 cand = pd.read_csv(META / "stage1_candidates.csv")23 merged = cand.merge(llm, on="record_id", how="left")24 hed = merged[merged.reason_code == "ELIGIBLE_HEDONIC"]; ml = merged[merged.reason_code == "ELIGIBLE_ML_PREDICTIVE_ONLY"]; unc = merged[merged.decision == "unclear"]25 sought = set(hed.record_id) | set(unc.record_id)26 retrieved = sought & pdfs27 counts = {28 "records_identified_raw": int(summ["raw_records"]), "by_source": summ["by_source"], "records_after_dedup": int(summ["unique_records"]),29 "records_with_abstract": int(summ["with_abstract"]), "records_screened_rules": int(summ["unique_records"]), "excluded_by_rules": int(summ["unique_records"] - len(cand)),30 "stage1_candidates": int(len(cand)), "screened_by_llm": int(len(llm)), "llm_excluded": int((llm.decision == "exclude").sum()),31 "llm_exclusion_reasons": llm[llm.decision == "exclude"].reason_code.value_counts().to_dict(),32 "llm_unclear": int((llm.decision == "unclear").sum()), "included_hedonic_stage1": int(len(hed)), "included_ml_predictive_stage1": int(len(ml)),33 "fulltext_sought_hedonic_or_unclear": int(len(sought)), "fulltext_retrieved": int(len(retrieved)), "fulltext_not_retrieved": int(len(sought) - len(retrieved)),34 "retrieval_attempts": int(len(retr)), "retrieval_status": retr.status.str.split(":").str[0].value_counts().to_dict(),35 "fulltext_assessed_extracted": int(len(jsons)), "fulltext_eligible": int(elig), "fulltext_ineligible": int(inel),36 }37 (META / "prisma_counts.json").write_text(json.dumps(counts, indent=2))38 lines = ["# WP21 — Search audit (generated by scripts/screening/search_audit.py)", "", f"Date of searches: 2026-10-20. Queries executed: {len(log)}.", "",39 "## Queries and hits", "", "| date | source | mode | query | hits | retrieved |", "|---|---|---|---|---|---|"]40 for r in log.itertuples():41 lines.append(f"| {r.date} | {r.source} | {r.mode} | {str(r.query)[:90]} | {r.hits} | {r.retrieved} |")42 lines += ["", "## Record counts", "", "```json", json.dumps(counts, indent=2), "```", "",43 "## Coverage limitations", "",44 "- OpenAlex: without an API key the service enforces a shared daily budget per IP (HTTP 401 after about 200 records on the search day); it was used only for verification.",45 "- Semantic Scholar: unauthenticated bulk endpoint, frequent HTTP 429; all queries completed with backoff; hit counts are the service's totals, retrieval capped at 20,000 per query (never reached).",46 "- Crossref: relevance-ranked bibliographic search returns hundreds of thousands of fuzzy matches; the first 4,000 per query were kept; most Crossref records carry no abstract, so their stage-1 screening is title-based.",47 "- arXiv: one query failed with HTTP 429 (logged); arXiv's all-field search is fuzzy and most hits are off-topic.",48 "- SSRN, Google Scholar, IDEAS/EconPapers, CORE, DOAJ, institutional repositories: no compliant bulk access from this environment; grey literature is covered only indirectly through Semantic Scholar and arXiv indexing.",49 "- Full texts: only legal open-access copies (Unpaywall locations, Semantic Scholar OA links, arXiv); paywalled studies that passed stage 1 are counted as 'not retrieved' in the PRISMA flow."]50 (ROOT / "reports" / "search_audit.md").write_text("\n".join(lines))51 print(json.dumps({k: v for k, v in counts.items() if not isinstance(v, dict)}, indent=1))525354if __name__ == "__main__":55 main()56