spb/wpai0002_uqo

1#!/usr/bin/env python32"""Metadata-first discovery + acquisition of CC-licensed pre-2018 arXiv LaTeX sources (OAI-PMH arXivRaw).34Examples:5  python scripts/discover_arxiv.py --sets q-fin econ --from 2008-01-01 --until 2017-12-31 --max-docs 12 --seed 16  python scripts/discover_arxiv.py --sets q-fin econ --from 2008-01-01 --until 2017-12-31 --metadata-only7"""8from __future__ import annotations910import argparse11import sys12import time13from pathlib import Path1415ROOT = Path(__file__).resolve().parents[1]16sys.path.insert(0, str(ROOT / "src"))17from htr.ingest.arxiv_oai import CC_CODES_DEFAULT, Counters, discover_and_fetch_arxiv18from htr.utils.io import write_json19from htr.utils.provenance import run_header202122def _progress(set_spec: str, upto: str | None, st: Counters) -> None:23    print(f"[{time.strftime('%H:%M:%S')}] set={set_spec} datestamp<={upto} seen={st.records_seen} eligible={st.eligible} license_excluded={st.license_excluded} not_pre2018={st.not_pre2018} before_window={st.before_window}", flush=True)242526def main() -> None:27    ap = argparse.ArgumentParser()28    ap.add_argument("--sets", nargs="+", default=["q-fin", "econ"])29    ap.add_argument("--from", dest="date_from", default="2000-01-01", help="v1 submission date lower bound")30    ap.add_argument("--until", dest="date_until", default="2017-12-31", help="v1 submission date upper bound (capped at 2017-12-31)")31    ap.add_argument("--datestamp-until", default=None, help="metadata-only: OAI datestamp upper bound (default today)")32    ap.add_argument("--max-docs", type=int, default=50)33    ap.add_argument("--seed", type=int, default=1)34    ap.add_argument("--out", default=str(ROOT / "data/raw/arxiv"))35    ap.add_argument("--codes", default=",".join(CC_CODES_DEFAULT))36    ap.add_argument("--metadata-only", action="store_true", help="enumerate eligible records to JSONL, no download")37    ap.add_argument("--metadata-out", default=None)38    ap.add_argument("--report", default=None)39    ap.add_argument("--max-minutes", type=float, default=None, help="stop harvesting after N minutes (partial counts, truncated_at)")40    a = ap.parse_args()41    tag = f"{'-'.join(a.sets)}_{a.date_from[:4]}-{a.date_until[:4]}"42    mout = Path(a.metadata_out) if a.metadata_out else ROOT / "data/manifests" / f"arxiv_eligible_{tag}.jsonl"43    stats = discover_and_fetch_arxiv(44        Path(a.out), a.sets, a.date_from, a.date_until, set(a.codes.split(",")), a.max_docs, a.seed,45        log_path=Path(a.out) / "_skipped.jsonl", metadata_only=a.metadata_only, metadata_out=mout,46        datestamp_until=a.datestamp_until, progress=_progress,47        max_seconds=a.max_minutes * 60 if a.max_minutes else None,48    )49    rep = {**run_header({"script": "discover_arxiv", "args": vars(a)}), "stats": stats}50    kind = "metadata" if a.metadata_only else f"seed{a.seed}"51    out = Path(a.report or (ROOT / "data/manifests" / f"discover_arxiv_{tag}_{kind}.json"))52    write_json(out, rep)53    keys = ("records_seen", "duplicates", "license_excluded", "not_pre2018", "before_window", "eligible", "pdf_only", "fetched", "errors", "by_set", "by_license_eligible", "oai_pages", "elapsed_s", "truncated_at")54    print({k: stats.get(k) for k in keys})55    print("by_license(all seen)", stats["by_license"])56    print("report:", out)575859if __name__ == "__main__":60    main()61