spb/wpai0002_uqo

1"""Live test: fetch ONE Project Gutenberg book from the official mirror and parse it (network)."""2from __future__ import annotations34import pytest56from htr.ingest.books import make_client, pg_mirror_paths7from htr.parse.books import (8    detect_frontmatter_year,9    gutenberg_header_status,10    parse_book_text,11    strip_gutenberg_boilerplate,12)131415@pytest.mark.network16def test_gutenberg_one_book_live() -> None:17    client = make_client(timeout=120)18    url = pg_mirror_paths(1228)[0]  # On the Origin of Species (1859, first edition)19    r = client.get(url, check_robots=True)20    assert r.status_code == 200, r.status_code21    text = r.content.decode("utf-8", errors="replace").replace("\r\n", "\n")22    sp = strip_gutenberg_boilerplate(text)23    assert sp.start_found and sp.end_found24    assert gutenberg_header_status(sp.header) == "public_domain_us"25    fm = detect_frontmatter_year(sp.body)26    assert fm is not None and fm.year == 185927    doc = parse_book_text(text, {"source": "gutenberg", "raw_format": "txt", "language": "en"})28    assert len(doc.paragraphs) > 30029    roles = {p.section_role for p in doc.paragraphs}30    assert {"introduction", "chapter"} <= roles31    assert all(doc.body_text[p.char_start:p.char_end] == p.text for p in doc.paragraphs)32    client.close()33