spb/wpai0002_uqo

1#!/usr/bin/env python32"""Parse raw documents → documents.jsonl / passages.jsonl / excluded.jsonl / build_summary.json."""3from __future__ import annotations45import argparse6import sys7from pathlib import Path89ROOT = Path(__file__).resolve().parents[1]10sys.path.insert(0, str(ROOT / "src"))11from htr.ingest.corpus import build_corpus  # noqa: E40212from htr.licenses.policy import TRAINING_CODES_DEFAULT  # noqa: E40213from htr.segment.passages import SegmentConfig  # noqa: E402141516def main() -> None:17    ap = argparse.ArgumentParser()18    ap.add_argument("--raw", nargs="+", default=[str(ROOT / "data/raw/pmc")])19    ap.add_argument("--out", default=str(ROOT / "data/parsed/pilot"))20    ap.add_argument("--codes", default=",".join(TRAINING_CODES_DEFAULT))21    ap.add_argument("--limit", type=int, default=None)22    ap.add_argument("--min-words", type=int, default=60)23    ap.add_argument("--max-words", type=int, default=400)24    ap.add_argument("--allow-post2018", action="store_true")25    ap.add_argument("--tag", default="corpus")26    a = ap.parse_args()27    st = build_corpus([Path(r) for r in a.raw], Path(a.out), set(a.codes.split(",")), SegmentConfig(min_words=a.min_words, max_words=a.max_words), a.limit, not a.allow_post2018, a.tag)28    print({k: v for k, v in st.__dict__.items() if k not in ("errors", "by_year")})29    print("by_year", dict(sorted(st.by_year.items())))30    if st.errors:31        print("first errors:", st.errors[:3])323334if __name__ == "__main__":35    main()36