spb/wpai0002_uqo

1"""Live arXiv test: OAI-PMH discovery of 3 CC-licensed q-fin records (2015) + e-print download + LaTeX parse.23Network-bound and slow (arXiv policy: 1 request / 3 s). Run with: pytest -m network tests/integration4"""5from __future__ import annotations67from pathlib import Path89import pytest1011from htr.ingest.arxiv_oai import discover_and_fetch_arxiv12from htr.ingest.corpus import parse_raw13from htr.utils.io import read_json141516@pytest.mark.network17@pytest.mark.slow18def test_discover_and_parse_qfin_2015(tmp_path: Path):19    stats = discover_and_fetch_arxiv(tmp_path, ["q-fin"], "2015-01-01", "2015-12-31", max_docs=3, seed=7, log_path=tmp_path / "_skipped.jsonl")20    assert stats["records_seen"] > 0 and stats["eligible"] > 021    assert stats["fetched"] + stats["pdf_only"] + stats["errors"] >= 122    stubs = sorted(tmp_path.glob("*.meta.json"))23    assert len(stubs) == stats["fetched"]24    parsed_ok = 025    for sp in stubs:26        stub = read_json(sp)27        assert stub["source"] == "arxiv" and stub["raw_format"] == "latex"28        assert stub["license"]["code"] in {"CC0", "CC-BY", "CC-BY-SA", "CC-BY-NC", "CC-BY-NC-SA"}29        assert stub["pub_date"] < "2018-01-01" and stub["pub_date"].startswith("2015")30        assert stub["subject_area"] in {"finance", "economics", "mathematics", "statistics", "computer_science", "physics", "other"}31        try:32            doc = parse_raw(stub)33        except ValueError:34            continue  # e.g. plain-TeX submission without \documentclass35        parsed_ok += 136        assert doc.pre2018_verified and doc.license_code == stub["license"]["code"]37        for p in doc.paragraphs:38            assert doc.body_text[p.char_start:p.char_end] == p.text39    assert parsed_ok >= min(1, len(stubs))40