Repository Public Research preview
1"""Unit tests for htr.parse.pdf on a synthetic two-page article generated with PyMuPDF."""2from __future__ import annotations34from pathlib import Path56import pytest78fitz = pytest.importorskip("fitz")910from htr.parse.pdf import parse_pdf # noqa: E4021112HEADER = "Journal of Test Economics 2015, 3"13STUB = {14 "title": "A Synthetic Article",15 "authors": ["Jane Doe"],16 "author_group": "doe_j",17 "venue": "Journal of Test Economics",18 "publisher": "Test Press",19 "pub_date": "2015-06-01",20 "pub_year": 2015,21 "pub_date_evidence": "crossref published-print date-parts",22 "license": {"code": "CC-BY", "uri": "https://creativecommons.org/licenses/by/4.0/", "evidence": "crossref license[].URL"},23 "source_ids": {"doi": "10.0000/test.1"},24 "doc_type": "article",25}2627PAGE1 = [28 # (text, x, fontsize, bold)29 ("A Synthetic Article on Prices", 72, 18, True),30 ("Jane Doe", 72, 11, False),31 ("Abstract", 72, 12, True),32 ("We study how economic shocks propagate through markets and why the long-run response of prices", 72, 10, False),33 ("differs from the short-run response. The analysis uses a simple model and a stylised dataset.", 72, 10, False),34 ("1. Introduction", 72, 12, True),35 ("Prices respond to shocks in ways that matter for policy and for the welfare of households.", 86, 10, False),36 ("A large literature documents that the econo-", 72, 10, False),37 ("mic effects of such shocks persist (Smith and Brown, 2010). We revisit this question with long-", 72, 10, False),38 ("term data and show that the response depends on market structure and on information.", 72, 10, False),39 ("Our contribution is twofold. First, we document the persistence of the term structure of", 86, 10, False),40 ("responses. Second, we show that long horizons matter for the economic interpretation.", 72, 10, False),41]42PAGE2 = [43 ("2. Data and Methods", 72, 12, True),44 ("We collect monthly prices for a panel of markets and estimate impulse responses with local", 86, 10, False),45 ("projections. The sample covers twenty years and the estimates are robust to the lag length.", 72, 10, False),46 ("Table 1. Summary statistics of the price panel.", 72, 9, False),47 ("Acknowledgments", 72, 12, True),48 ("We thank the anonymous funding agency for generous support of this project.", 72, 10, False),49 ("References", 72, 12, True),50 ("Smith, J., Brown, K., 2010. Persistent price responses. Journal of Prices 12, 1-20.", 72, 10, False),51 ("Taylor, M., 2008. Information and markets. Review of Markets 3, 44-60.", 72, 10, False),52]535455def _write_page(page, rows, page_no: int) -> None:56 page.insert_text((72, 40), HEADER, fontsize=9, fontname="helv")57 page.insert_text((300, 815), str(page_no), fontsize=9, fontname="helv")58 y = 90.059 for text, x, size, bold in rows:60 if bold:61 y += 862 page.insert_text((x, y), text, fontsize=size, fontname="hebo" if bold else "helv")63 y += size + 4 + (4 if bold else 0)646566@pytest.fixture()67def synthetic_pdf(tmp_path: Path) -> Path:68 doc = fitz.open()69 for i, rows in enumerate((PAGE1, PAGE2), start=1):70 page = doc.new_page(width=595, height=842)71 _write_page(page, rows, i)72 out = tmp_path / "synthetic.pdf"73 doc.save(str(out))74 doc.close()75 return out767778def test_headers_footers_removed(synthetic_pdf: Path) -> None:79 d = parse_pdf(synthetic_pdf, STUB)80 assert "Journal of Test Economics" not in d.body_text81 assert not any(p.text.strip() in {"1", "2"} for p in d.paragraphs)82 assert any(w.startswith("headers_footers_removed=") for w in d.warnings)838485def test_hyphenation_repair(synthetic_pdf: Path) -> None:86 d = parse_pdf(synthetic_pdf, STUB)87 assert "economic effects" in d.body_text88 assert "econo-" not in d.body_text and "econo- mic" not in d.body_text89 # "long-" + "term": both halves are words of the document → real compound, hyphen kept90 assert "long-term data" in d.body_text91 flags = {f for p in d.paragraphs for f in p.flags}92 assert "dehyphenated" in flags and "hyphen_kept" in flags939495def test_sections_roles_and_cutoff(synthetic_pdf: Path) -> None:96 d = parse_pdf(synthetic_pdf, STUB)97 roles = {p.section_role for p in d.paragraphs}98 assert {"abstract", "introduction", "methods"} <= roles99 intro = [p for p in d.paragraphs if p.section_role == "introduction"]100 assert intro and intro[0].section_title == "1. Introduction"101 assert len(intro) == 2 # indentation splits the two introduction paragraphs102 assert any(p.section_role == "caption" and p.text.startswith("Table 1.") for p in d.paragraphs)103 # References cut off, acknowledgements dropped, front matter (title/authors) dropped104 assert "Persistent price responses" not in d.body_text105 assert "funding agency" not in d.body_text106 assert "Jane Doe" not in d.body_text107 assert any(w.startswith("references_cut_at_page=2") for w in d.warnings)108 assert any(w.startswith("extraction_quality=") for w in d.warnings)109 # citation preserved verbatim and detected by protect() (note: protect.AUTHOR_YEAR_RE misses "et al., 2010")110 assert any("(Smith and Brown, 2010)" in c for p in d.paragraphs for c in p.citations)111112113def test_offsets_and_stub_fields(synthetic_pdf: Path) -> None:114 d = parse_pdf(synthetic_pdf, STUB)115 assert d.paragraphs116 for p in d.paragraphs:117 assert d.body_text[p.char_start:p.char_end] == p.text118 assert d.body_text.endswith("\n\n")119 assert [p.para_index for p in d.paragraphs] == list(range(len(d.paragraphs)))120 assert d.title == STUB["title"] and d.pub_year == 2015 and d.pre2018_verified121 assert d.license_code == "CC-BY" and d.license_evidence == "crossref license[].URL"122123124def test_no_text_layer_raises(tmp_path: Path) -> None:125 doc = fitz.open()126 for _ in range(2):127 page = doc.new_page(width=595, height=842)128 page.draw_rect(fitz.Rect(50, 50, 500, 700), color=(0, 0, 0), fill=(0.8, 0.8, 0.8))129 out = tmp_path / "scan.pdf"130 doc.save(str(out))131 doc.close()132 with pytest.raises(ValueError, match="no text layer"):133 parse_pdf(out, STUB)134