"""Phase B — search audit and PRISMA counts from the actual records: data/metadata/prisma_counts.json, reports/search_audit.md."""
import json
from pathlib import Path
import pandas as pd

ROOT = Path(__file__).resolve().parents[2]; META = ROOT / "data" / "metadata"


def main():
    log = pd.read_csv(META / "search_log.csv")
    rec = pd.read_csv(META / "records_dedup.csv", low_memory=False)
    summ = json.load(open(META / "screening_summary.json"))
    llm = pd.read_csv(META / "screening_stage1_llm.csv")
    retr = pd.read_csv(META / "retrieval_log.csv")
    pdfs = {p.stem for p in (ROOT / "data" / "raw" / "pdfs").glob("*.pdf")}
    jsons = list((ROOT / "data" / "extracted" / "json").glob("*.json"))
    elig = 0; inel = 0
    for f in jsons:
        try:
            j = json.load(open(f)); elig += bool(j.get("eligible")); inel += (not j.get("eligible"))
        except Exception: pass
    cand = pd.read_csv(META / "stage1_candidates.csv")
    merged = cand.merge(llm, on="record_id", how="left")
    hed = merged[merged.reason_code == "ELIGIBLE_HEDONIC"]; ml = merged[merged.reason_code == "ELIGIBLE_ML_PREDICTIVE_ONLY"]; unc = merged[merged.decision == "unclear"]
    sought = set(hed.record_id) | set(unc.record_id)
    retrieved = sought & pdfs
    counts = {
        "records_identified_raw": int(summ["raw_records"]), "by_source": summ["by_source"], "records_after_dedup": int(summ["unique_records"]),
        "records_with_abstract": int(summ["with_abstract"]), "records_screened_rules": int(summ["unique_records"]), "excluded_by_rules": int(summ["unique_records"] - len(cand)),
        "stage1_candidates": int(len(cand)), "screened_by_llm": int(len(llm)), "llm_excluded": int((llm.decision == "exclude").sum()),
        "llm_exclusion_reasons": llm[llm.decision == "exclude"].reason_code.value_counts().to_dict(),
        "llm_unclear": int((llm.decision == "unclear").sum()), "included_hedonic_stage1": int(len(hed)), "included_ml_predictive_stage1": int(len(ml)),
        "fulltext_sought_hedonic_or_unclear": int(len(sought)), "fulltext_retrieved": int(len(retrieved)), "fulltext_not_retrieved": int(len(sought) - len(retrieved)),
        "retrieval_attempts": int(len(retr)), "retrieval_status": retr.status.str.split(":").str[0].value_counts().to_dict(),
        "fulltext_assessed_extracted": int(len(jsons)), "fulltext_eligible": int(elig), "fulltext_ineligible": int(inel),
    }
    (META / "prisma_counts.json").write_text(json.dumps(counts, indent=2))
    lines = ["# WP21 — Search audit (generated by scripts/screening/search_audit.py)", "", f"Date of searches: 2026-10-20. Queries executed: {len(log)}.", "",
             "## Queries and hits", "", "| date | source | mode | query | hits | retrieved |", "|---|---|---|---|---|---|"]
    for r in log.itertuples():
        lines.append(f"| {r.date} | {r.source} | {r.mode} | {str(r.query)[:90]} | {r.hits} | {r.retrieved} |")
    lines += ["", "## Record counts", "", "```json", json.dumps(counts, indent=2), "```", "",
              "## Coverage limitations", "",
              "- OpenAlex: without an API key the service enforces a shared daily budget per IP (HTTP 401 after about 200 records on the search day); it was used only for verification.",
              "- Semantic Scholar: unauthenticated bulk endpoint, frequent HTTP 429; all queries completed with backoff; hit counts are the service's totals, retrieval capped at 20,000 per query (never reached).",
              "- Crossref: relevance-ranked bibliographic search returns hundreds of thousands of fuzzy matches; the first 4,000 per query were kept; most Crossref records carry no abstract, so their stage-1 screening is title-based.",
              "- arXiv: one query failed with HTTP 429 (logged); arXiv's all-field search is fuzzy and most hits are off-topic.",
              "- SSRN, Google Scholar, IDEAS/EconPapers, CORE, DOAJ, institutional repositories: no compliant bulk access from this environment; grey literature is covered only indirectly through Semantic Scholar and arXiv indexing.",
              "- Full texts: only legal open-access copies (Unpaywall locations, Semantic Scholar OA links, arXiv); paywalled studies that passed stage 1 are counted as 'not retrieved' in the PRISMA flow."]
    (ROOT / "reports" / "search_audit.md").write_text("\n".join(lines))
    print(json.dumps({k: v for k, v in counts.items() if not isinstance(v, dict)}, indent=1))


if __name__ == "__main__":
    main()