Repository Public Research preview
1"""Unit tests for the books parser and the Gutenberg dating / filtering helpers (no network)."""2from __future__ import annotations34from pathlib import Path56from htr.ingest.books import (7 date_gutenberg,8 death_year_upper_bound,9 gutenberg_row_eligible,10 parse_pg_authors,11 pg_mirror_paths,12)13from htr.parse.books import (14 detect_frontmatter_year,15 gutenberg_header_status,16 parse_book_file,17 parse_book_html,18 roman_to_int,19 strip_gutenberg_boilerplate,20)2122SYNTHETIC = """The Project Gutenberg eBook of A Treatise on Synthetic Matters2324This eBook is for the use of anyone anywhere in the United States and25most other parts of the world at no cost and with almost no restrictions26whatsoever.2728Release date: May 1, 2004 [eBook #99999]2930*** START OF THE PROJECT GUTENBERG EBOOK A TREATISE ON SYNTHETIC MATTERS ***3132Produced by A. Volunteer and the Online Distributed Proofreading Team.333435A TREATISE36ON SYNTHETIC MATTERS3738BY JOHN EXAMPLE3940LONDON:41MACMILLAN AND CO.421898.434445CONTENTS4647 PREFACE v48 CHAPTER I. OF FIRST PRINCIPLES 149 CHAPTER II. OF LATER CONSEQUENCES 27505152PREFACE5354The present volume is the outcome of lectures delivered over several55years to students of natural philosophy, and it aims to set out the56first principles of the subject in a form that is at once rigorous and57readable. The author has avoided all needless technicality throughout.585960CHAPTER I6162OF FIRST PRINCIPLES6364Every inquiry into nature must begin with a careful statement of the65principles on which it rests, for without such a statement the66reasoning that follows can have no secure foundation.[1] We therefore67begin with the simplest cases and proceed by gradual steps toward the68more complex, testing each conclusion against well-established69observation.7071[Illustration: FIG. 1. A simple apparatus.]7273The second principle concerns the measurement of quantities. It is74well-75known that a measurement is only as good as the instrument, and we shall76see that 12 per cent of the error arises from the observer alone, which77is a large share indeed.7879[1] See the earlier memoir on this subject, p. 14.808182CHAPTER II8384OF LATER CONSEQUENCES8586From these principles certain consequences follow at once. The first87is that no single observation can settle a question of this kind; the88second is that the agreement of many observations is the surest test89of a theory that we possess.909192INDEX9394Apparatus, 395Principles, 1, 27969798*** END OF THE PROJECT GUTENBERG EBOOK A TREATISE ON SYNTHETIC MATTERS ***99100Section 5. General Information About Project Gutenberg electronic works101"""102103104def _parse(tmp_path: Path):105 p = tmp_path / "pg99999.txt"106 p.write_text(SYNTHETIC, encoding="utf-8")107 return parse_book_file(p, {"source": "gutenberg", "raw_format": "txt", "title": "A Treatise", "authors": ["John Example"], "language": "en"})108109110def test_header_footer_stripped(tmp_path: Path) -> None:111 sp = strip_gutenberg_boilerplate(SYNTHETIC)112 assert sp.start_found and sp.end_found113 assert "Release date" in sp.header and "General Information" in sp.footer114 assert gutenberg_header_status(sp.header) == "public_domain_us"115 doc = _parse(tmp_path)116 assert "Project Gutenberg" not in doc.body_text117 assert "Produced by" not in doc.body_text118 assert "MACMILLAN" not in doc.body_text # title page (front matter) dropped119120121def test_roles_sections_and_drops(tmp_path: Path) -> None:122 doc = _parse(tmp_path)123 roles = [(p.section_role, p.section_title) for p in doc.paragraphs]124 assert roles[0] == ("preface", "PREFACE")125 assert ("chapter", "CHAPTER I OF FIRST PRINCIPLES") in roles126 assert ("chapter", "CHAPTER II OF LATER CONSEQUENCES") in roles127 assert len(doc.paragraphs) == 4128 body = doc.body_text129 assert "Illustration" not in body and "apparatus" not in body.lower()130 assert "earlier memoir" not in body # footnote block dropped131 assert "[1]" not in body # dangling footnote anchor removed132 assert "Principles, 1, 27" not in body # index dropped133 assert "OF FIRST PRINCIPLES 1" not in body # table of contents dropped134 w = " ".join(doc.warnings)135 for key in ("dropped_illustration", "dropped_footnote", "dropped_toc", "dropped_index", "footnote_anchors_removed"):136 assert key in w, key137138139def test_reflow_and_offsets(tmp_path: Path) -> None:140 doc = _parse(tmp_path)141 for p in doc.paragraphs:142 assert "\n" not in p.text143 assert doc.body_text[p.char_start:p.char_end] == p.text144 assert doc.body_text == "".join(p.text + "\n\n" for p in doc.paragraphs)145 first_ch = next(p for p in doc.paragraphs if p.section_role == "chapter")146 assert first_ch.text.startswith("Every inquiry into nature must begin with a careful statement of the principles")147 second = doc.paragraphs[2]148 assert "well-known that a measurement" in second.text # hard hyphen at line end joined, hyphen kept149 assert "12" in second.numbers150 assert [p.para_index for p in doc.paragraphs] == list(range(len(doc.paragraphs)))151 assert doc.license_code == "UNKNOWN" and doc.pub_date is None # decided by the stub, not the parser152153154def test_frontmatter_year_rules() -> None:155 body = strip_gutenberg_boilerplate(SYNTHETIC).body156 fm = detect_frontmatter_year(body)157 assert fm is not None and fm.year == 1898 and fm.rule == "imprint"158 assert detect_frontmatter_year("Some Title\n\nCopyright, 1898, 1905,\nBY THE MACMILLAN COMPANY.\n").year == 1905159 assert detect_frontmatter_year("Entered according to Act of Congress, in the year 1872, by J. Smith").year == 1872160 assert detect_frontmatter_year("A TITLE\n\nLONDON\nMDCCCXC\n").year == 1890161 assert detect_frontmatter_year("Release date: May 1, 2004 [eBook #1]\nNothing else here.\n") is None162 assert roman_to_int("MDCCCXC") == 1890163164165def test_gutenberg_dating_and_filters() -> None:166 contribs = parse_pg_authors("Darwin, Charles, 1809-1882; Darwin, Francis, Sir, 1848-1925 [Editor]")167 assert contribs[0].name == "Charles Darwin"168 assert death_year_upper_bound(contribs) == 1925169 assert death_year_upper_bound(parse_pg_authors("Various")) is None170 row = {"Issued": "2000-02-01"}171 d = date_gutenberg("No dates in this front matter at all.\n\nCHAPTER I\n\nText.", row, contribs)172 assert d["pub_year"] == 1925 and d["date_rule"] == "death_year" and "upper bound" in d["pub_date_evidence"]173 d2 = date_gutenberg("No dates.\n", row, parse_pg_authors("Anonymous"))174 assert d2["pub_year"] is None and d2["date_rule"] == "none"175 base = {"Type": "Text", "Language": "en", "Subjects": "Evolution (Biology)", "Bookshelves": "Category: Science - Biology"}176 assert gutenberg_row_eligible({**base, "LoCC": "QH"}) == (True, "QH")177 assert gutenberg_row_eligible({**base, "LoCC": "QH; PZ"})[0] is False178 assert gutenberg_row_eligible({**base, "LoCC": "BX"})[0] is False179 assert gutenberg_row_eligible({**base, "LoCC": "QH", "Subjects": "Science fiction"})[0] is False180 assert gutenberg_row_eligible({**base, "LoCC": "QH", "Language": "fr"})[0] is False181 assert pg_mirror_paths(1228)[1].endswith("/1/2/2/1228/1228-0.txt")182 assert pg_mirror_paths(5)[1].endswith("/0/5/5-0.txt")183184185def test_html_sections() -> None:186 page = b"""<html><body><section id="pg-header"><p>The Project Gutenberg eBook header text that is long enough.</p></section>187 <h1>A Book</h1><p>Title page line.</p>188 <h2>PREFACE</h2><p>This preface explains the purpose of the work in two sentences. It is short.</p>189 <h2>CHAPTER I. ORIGINS</h2><p>The first chapter begins here<span class="pagenum">[Pg 1]</span> with a sentence. Then another one.</p>190 <div class="footnote"><p>[1] A footnote.</p></div>191 <h2>INDEX</h2><p>Origins, 1</p></body></html>"""192 doc = parse_book_html(page, {"source": "gutenberg", "raw_format": "html"})193 assert [p.section_role for p in doc.paragraphs] == ["preface", "chapter"]194 assert "Pg 1" not in doc.body_text and "footnote" not in doc.body_text and "Origins, 1" not in doc.body_text195 assert all(doc.body_text[p.char_start:p.char_end] == p.text for p in doc.paragraphs)196