spb/wpai0002_uqo

1#!/usr/bin/env python32"""Metadata-first discovery + acquisition of pre-2018 CC-licensed economics/finance articles via Crossref.34Examples:5  python scripts/discover_crossref.py --count-only6  python scripts/discover_crossref.py --max-docs 15 --per-journal 4 --seed 1 --out data/raw/crossref7"""8from __future__ import annotations910import argparse11import sys12import time13from pathlib import Path1415ROOT = Path(__file__).resolve().parents[1]16sys.path.insert(0, str(ROOT / "src"))17from htr.ingest.crossref import count_crossref, discover_crossref  # noqa: E40218from htr.utils.io import write_json  # noqa: E40219from htr.utils.provenance import run_header  # noqa: E4022021DEFAULT_CODES = "CC-BY,CC-BY-SA,CC-BY-NC,CC-BY-NC-SA,CC0"222324def main() -> None:25    ap = argparse.ArgumentParser()26    ap.add_argument("--config", default=str(ROOT / "configs/data/crossref_journals.json"))27    ap.add_argument("--max-docs", type=int, default=15)28    ap.add_argument("--per-journal", type=int, default=None)29    ap.add_argument("--seed", type=int, default=1)30    ap.add_argument("--until", default="2017-12-31")31    ap.add_argument("--out", default=str(ROOT / "data/raw/crossref"))32    ap.add_argument("--codes", default=DEFAULT_CODES)33    ap.add_argument("--report", default=None)34    ap.add_argument("--count-only", action="store_true", help="screen Crossref metadata only (exact eligible counts), download nothing")35    a = ap.parse_args()36    codes = set(c.strip() for c in a.codes.split(",") if c.strip())37    t0 = time.time()38    if a.count_only:39        stats = count_crossref(a.config, codes, until=a.until)40    else:41        stats = discover_crossref(Path(a.out), a.config, codes, until=a.until, max_docs=a.max_docs, seed=a.seed, per_journal_cap=a.per_journal, log_path=Path(a.out) / "_errors.jsonl")42    stats["elapsed_s"] = round(time.time() - t0, 1)43    rep = {**run_header({"script": "discover_crossref", "args": vars(a)}), "stats": stats}44    name = "discover_crossref_count.json" if a.count_only else f"discover_crossref_seed{a.seed}.json"45    out = Path(a.report or (ROOT / "data/manifests" / name))46    write_json(out, rep)47    if a.count_only:48        print(f"{'journal':58s} {'xref<=until':>11s} {'has_lic':>7s} {'eligible':>8s}  years      by_license")49        for jn, js in stats["screen"].items():50            if "error" in js:51                print(f"{jn:58s} ERROR {js['error']}")52                continue53            yrs = f"{js.get('first_year', '-')}-{js.get('last_year', '-')}"54            print(f"{jn[:58]:58s} {js['crossref_total_until']:>11d} {js['crossref_total_until_has_license']:>7d} {js['eligible']:>8d}  {yrs:9s}  {js['by_license']}")55        print(f"eligible_total={stats['eligible_total']} records_seen={stats['records_seen']} not_pre2018={stats['not_pre2018']} no_license={stats['no_license']} license_excluded={stats['license_excluded']} no_fulltext={stats['no_fulltext']} elapsed={stats['elapsed_s']}s")56    else:57        keys = ("records_seen", "no_license", "license_excluded", "not_pre2018", "no_fulltext", "already_present", "fetched", "fetch_errors")58        print(" ".join(f"{k}={stats[k]}" for k in keys))59        print("by_journal", stats["by_journal"])60        print("by_license", stats["by_license"])61        print("by_route", stats["by_route"])62        if stats.get("blocked_hosts"):63            print("blocked_hosts", stats["blocked_hosts"])64        print(f"elapsed={stats['elapsed_s']}s report={out}")656667if __name__ == "__main__":68    main()69