spb/wpai0002_uqo

1#!/usr/bin/env python32"""Discovery + acquisition for the "books" stratum (pre-2018 English reference / scholarly books).34Examples:5  python scripts/discover_books.py --source gutenberg --max-docs 12 --seed 1 --out data/raw/books6  python scripts/discover_books.py --source bookshelf --max-docs 3 --seed 1 --out data/raw/books7  python scripts/discover_books.py --source openstax  --max-docs 3 --seed 1 --out data/raw/books8  python scripts/discover_books.py --source gutenberg --pool-only     # eligible-pool estimate from the catalog910Raw files and ``*.meta.json`` stubs go to ``<out>/<source>/``; the JSON report to ``data/manifests/``.11See configs/data/books_sources.json for the verification status of each sub-source.12"""13from __future__ import annotations1415import argparse16import collections17import re18import sys19import time20from pathlib import Path2122ROOT = Path(__file__).resolve().parents[1]23sys.path.insert(0, str(ROOT / "src"))24from htr.ingest.books import (25    DEFAULT_LOCC_PREFIXES,26    discover_bookshelf,27    discover_gutenberg,28    discover_openstax,29    load_pg_catalog,30    make_client,31    pg_pool,32)33from htr.licenses.policy import TRAINING_CODES_DEFAULT34from htr.utils.io import write_json35from htr.utils.provenance import run_header363738def pool_estimate(out: Path, prefixes: tuple[str, ...]) -> dict:39    client = make_client()40    rows, info = load_pg_catalog(out / "_catalog", client)41    pool, by_class = pg_pool(rows, prefixes)42    by_letter = collections.Counter(code[0] for _, code in pool)43    authors_with_death = sum(1 for r, _ in pool if re.search(r"\d{3,4}\??\s*-\s*\d{3,4}", r.get("Authors") or ""))44    return {"catalog": info, "catalog_rows": len(rows), "text_en": sum(1 for r in rows if r.get("Type") == "Text" and (r.get("Language") or "").strip() == "en"), "eligible_pool": len(pool), "pool_by_letter": dict(by_letter.most_common()), "pool_by_class": dict(sorted(by_class.items(), key=lambda kv: -kv[1])), "pool_with_any_author_life_dates": authors_with_death}454647def main() -> None:48    ap = argparse.ArgumentParser()49    ap.add_argument("--source", choices=["gutenberg", "bookshelf", "openstax"], required=True)50    ap.add_argument("--max-docs", type=int, default=12)51    ap.add_argument("--max-attempts", type=int, default=None)52    ap.add_argument("--seed", type=int, default=1)53    ap.add_argument("--out", default=str(ROOT / "data/raw/books"))54    ap.add_argument("--codes", default=",".join(TRAINING_CODES_DEFAULT))55    ap.add_argument("--locc", default=",".join(DEFAULT_LOCC_PREFIXES), help="Gutenberg LoCC prefixes")56    ap.add_argument("--max-mb", type=float, default=25.0, help="bookshelf tarball / openstax PDF size cap")57    ap.add_argument("--pool-only", action="store_true", help="gutenberg: only estimate the eligible pool")58    ap.add_argument("--report", default=None)59    a = ap.parse_args()60    out = Path(a.out)61    codes = set(a.codes.split(","))62    prefixes = tuple(p.strip() for p in a.locc.split(",") if p.strip())63    t0 = time.time()64    if a.source == "gutenberg" and a.pool_only:65        stats = pool_estimate(out, prefixes)66    elif a.source == "gutenberg":67        stats = discover_gutenberg(out, a.max_docs, a.seed, prefixes, log_path=out / "_errors.jsonl", max_attempts=a.max_attempts)68    elif a.source == "bookshelf":69        stats = discover_bookshelf(out, a.max_docs, a.seed, codes, log_path=out / "_errors.jsonl", max_mb=a.max_mb, max_attempts=a.max_attempts)70    else:71        stats = discover_openstax(out, a.max_docs, a.seed, codes, log_path=out / "_errors.jsonl", max_pdf_mb=a.max_mb, max_attempts=a.max_attempts)72    stats["elapsed_s"] = round(time.time() - t0, 1)73    rep = {**run_header({"script": "discover_books", "args": vars(a)}), "stats": stats}74    suffix = "_pool" if a.pool_only else ""75    path = Path(a.report or (ROOT / "data/manifests" / f"discover_books_{a.source}{suffix}_seed{a.seed}.json"))76    write_json(path, rep)77    show = {k: v for k, v in stats.items() if k not in ("catalog", "http", "ftp", "pool_by_class")}78    print(show)79    print(f"report: {path}")808182if __name__ == "__main__":83    main()84