spb/wpai0002_uqo

1#!/usr/bin/env python32"""Metadata-first discovery + acquisition of PMC Open Access articles (AWS Open Data bucket).34Example: python scripts/discover_pmc.py --candidates 400 --max-docs 100 --seed 1 --out data/raw/pmc5"""6from __future__ import annotations78import argparse9import sys10import time11from pathlib import Path1213ROOT = Path(__file__).resolve().parents[1]14sys.path.insert(0, str(ROOT / "src"))15from htr.ingest.pmc_s3 import discover_and_fetch  # noqa: E40216from htr.licenses.policy import TRAINING_CODES_DEFAULT  # noqa: E40217from htr.utils.io import write_json  # noqa: E40218from htr.utils.provenance import run_header  # noqa: E402192021def main() -> None:22    ap = argparse.ArgumentParser()23    ap.add_argument("--candidates", type=int, default=400)24    ap.add_argument("--max-docs", type=int, default=100)25    ap.add_argument("--seed", type=int, default=1)26    ap.add_argument("--out", default=str(ROOT / "data/raw/pmc"))27    ap.add_argument("--codes", default=",".join(TRAINING_CODES_DEFAULT))28    ap.add_argument("--report", default=None)29    a = ap.parse_args()30    t0 = time.time()31    stats = discover_and_fetch(Path(a.out), a.candidates, a.seed, set(a.codes.split(",")), max_docs=a.max_docs, log_path=Path(a.out) / "_errors.jsonl")32    stats["elapsed_s"] = round(time.time() - t0, 1)33    rep = {**run_header({"script": "discover_pmc", "args": vars(a)}), "stats": stats}34    out = Path(a.report or (ROOT / "data/manifests" / f"discover_pmc_seed{a.seed}.json"))35    write_json(out, rep)36    print(f"fetched={stats['fetched']} candidates={stats['candidates']} missing={stats['missing']} license_excluded={stats['license_excluded']} by_license={stats['by_license']} elapsed={stats['elapsed_s']}s")373839if __name__ == "__main__":40    main()41