Repository Public Research preview
1"""PDF parser (PyMuPDF) for born-digital journal articles (Crossref / publisher PDFs).23Pipeline: spans → visual lines (fragments sharing a baseline are merged) → running header/footer and4page-number removal → small-print, equation and table-row filtering → heading / inline-label / caption5detection → paragraph reconstruction (indentation, short last line, vertical gap, page continuation) with6hyphenation repair → section roles via ``htr.parse.jats.section_role``. Everything from the first7``References | Bibliography | Works Cited`` heading onwards is cut; acknowledgement-like back-matter8sections are dropped. Bibliographic fields (title, authors, date, licence) come from the manifest stub,9which carries the provider evidence; the parser only cross-checks a licence URL printed in the PDF.1011The output ``ParsedDocument`` has the same contract as ``parse_jats``: ``body_text`` is the concatenation12of paragraph texts each followed by a blank line, and ``char_start``/``char_end`` index into it.13"""14from __future__ import annotations1516import datetime as _dt17import math18import re19from collections import Counter20from dataclasses import dataclass, field21from pathlib import Path2223from htr.licenses.policy import normalize_license24from htr.parse.jats import EXCLUDED_SECTION_TITLES, ParsedDocument, ParsedParagraph, section_role25from htr.parse.protect import protect26from htr.parse.quality import quality_score27from htr.utils.text import normalize_text2829# ---------------------------------------------------------------------------------------------------------30# Patterns31REFERENCES_RE = re.compile(r"^\s*(?:[\dIVX]+\.?\s*)?(?:references?(?: and notes)?|bibliography|works cited|literature cited|reference list|literature)\s*:?\s*$", re.I)32LOCAL_EXCLUDED_RE = re.compile(r"authors?['’]?s?['’]?\s+contributions?|^\s*(?:end)?notes\s*$|author details|publisher['’]?s note|additional information|^\s*keywords?\b|^\s*key words\b|^\s*jel\b|nomenclature|^\s*notation\s*$|^\s*open access\s*$|data availability|^\s*credit\b", re.I)33KNOWN_LABEL_RE = re.compile(34 r"^\s*(abstract|summary|keywords?|key words|jel(?:\s+(?:classification|codes?))?(?:\s+codes?)?|acknowledge?ments?|author contributions?|"35 r"conflicts? of interest|funding|abbreviations|supplementary materials?|data availability(?: statement)?|introduction)\s*[:.]",36 re.I,37)38KNOWN_HEADING_RE = re.compile(r"^\s*(?:[\dIVX]+\.?\s*)?(abstract|introduction|background|conclusions?|concluding remarks|discussion|results|methods?|methodology|data|references|bibliography|acknowledge?ments?|appendix(?:\s+[A-Z\d])?)\s*$", re.I)39NUMBERED_RE = re.compile(r"^\s*((?:\d{1,2}(?:\.\d{1,2}){0,3})\.?|[IVX]{1,5}\.|[A-H]\.(?:\d{1,2}\.?)*)\s+(\S.*)$")40CAPTION_RE = re.compile(r"^\s*(Table|Figure|Fig\.|Chart|Exhibit|Graph)\s*[A-Z]?\d+[a-z]?(?:\.\d+)?\s*(?:[.:|—–-]|$)", re.I)41CAPTION_LOOSE_RE = re.compile(r"^\s*(Table|Figure|Fig\.|Chart|Exhibit)\s*[A-Z]?\d+[a-z]?\s+[A-Z(]")42PAGE_NUM_RE = re.compile(r"^\s*(?:page\s*)?[\divxlc]{1,5}(?:\s*(?:of|/)\s*\d{1,4})?\s*$", re.I)43MARGIN_BOILER_RE = re.compile(r"\bdoi[:\s]|https?://|www\.|©|\bpage \d+ of \d+|\bvol(?:ume)?\.?\s*\d+", re.I)44FRONT_BOILER_RE = re.compile(r"^\s*(?:received|accepted|published(?: online)?|revised|academic editors?|correspondence|\*\s*correspondence|citation|copyright|©|e-?mail|tel\.|fax|issn|open access|article|review|research|original article|www\.|https?://)\b", re.I)45TERMINAL_RE = re.compile(r"[.!?][\"”’')\]]*$")46MATH_FONT_RE = re.compile(r"CM(?:EX|MI|SY|R|BX)\d|MSAM|MSBM|Math|Symbol|MT-?Extra|Euclid|STIX|rsfs|Cambria\s*Math|Mathematica|Pazo", re.I)47MATH_CHARS = set("=+−×÷∑∫∏√∞≤≥≈≠∂∇∈∉⊂⊆∪∩→←↔⇒⇔∀∃αβγδεζηθικλμνξπρστυφχψωΓΔΘΛΞΠΣΦΨΩ^_|")48WORD_RE = re.compile(r"[A-Za-z]+")49BOLD_FONT_RE = re.compile(r"bold|black|heavy|semibold|demi|[-,]b[do]?\b|\bbd\b", re.I)50ITALIC_FONT_RE = re.compile(r"ital|oblique|[-,]it\b", re.I)515253@dataclass54class _Line:55 page: int56 block: int57 x0: float58 y0: float59 x1: float60 y1: float61 size: float62 text: str63 font: str64 bold: bool65 italic: bool66 lead_bold: str # text of the leading run of bold spans67 math_frac: float68 n_chars: int69 kind: str = "text" # text | heading | label | caption | equation | table | dropped70 meta: dict = field(default_factory=dict)717273# ---------------------------------------------------------------------------------------------------------74# Extraction757677def _is_bold(span: dict) -> bool:78 return bool(span.get("flags", 0) & 16) or bool(BOLD_FONT_RE.search(span.get("font", "")))798081def _is_italic(span: dict) -> bool:82 return bool(span.get("flags", 0) & 2) or bool(ITALIC_FONT_RE.search(span.get("font", "")))838485def _line_from_spans(page: int, block: int, bbox, spans: list[dict]) -> _Line | None:86 spans = [s for s in spans if s.get("text")]87 if not spans:88 return None89 sizes = Counter()90 fonts = Counter()91 for s in spans:92 n = len(s["text"].strip())93 sizes[round(s["size"] * 2) / 2] += n94 fonts[s.get("font", "")] += n95 main_size = sizes.most_common(1)[0][0] if sizes else spans[0]["size"]96 # Drop raised small numeric spans (footnote markers such as "media1"); keep sub/superscripts in math.97 parts: list[str] = []98 prev_txt = ""99 baseline = max((s["bbox"][3] for s in spans if abs(s["size"] - main_size) < 0.6), default=bbox[3])100 prev_x1: float | None = None101 for s in spans:102 t = s["text"]103 raised = s["bbox"][3] < baseline - 0.25 * main_size104 if raised and s["size"] < 0.8 * main_size and re.fullmatch(r"\s*[\d,*†‡§¶–-]{1,7}\s*", t) and re.search(r"[A-Za-z.,;:)”’]$", prev_txt):105 continue106 if prev_x1 is not None and parts and not parts[-1].endswith(" ") and not t.startswith(" ") and s["bbox"][0] - prev_x1 > 0.2 * main_size:107 parts.append(" ")108 parts.append(t)109 prev_x1 = s["bbox"][2]110 prev_txt = (prev_txt + t)[-5:]111 text = "".join(parts)112 if not text.strip():113 return None114 n_chars = sum(1 for c in text if not c.isspace())115 math_chars = sum(len(s["text"].strip()) for s in spans if MATH_FONT_RE.search(s.get("font", "")))116 alpha_spans = [s for s in spans if re.search(r"[A-Za-z]", s["text"])]117 bold = bool(alpha_spans) and all(_is_bold(s) for s in alpha_spans)118 italic = bool(alpha_spans) and all(_is_italic(s) for s in alpha_spans)119 lead = []120 for s in spans:121 if _is_bold(s) or not s["text"].strip():122 lead.append(s["text"])123 else:124 break125 return _Line(page, block, bbox[0], bbox[1], bbox[2], bbox[3], float(main_size), text, fonts.most_common(1)[0][0], bold, italic, "".join(lead).strip(), math_chars / max(1, n_chars), n_chars)126127128def _merge_visual_lines(lines: list[_Line], two_col: bool) -> list[_Line]:129 """Merge consecutive fragments that share a baseline (PyMuPDF splits lines at font/spacing changes)."""130 out: list[_Line] = []131 for ln in lines:132 if out:133 p = out[-1]134 h = min(p.y1 - p.y0, ln.y1 - ln.y0) or 1.0135 overlap = min(p.y1, ln.y1) - max(p.y0, ln.y0)136 gap = ln.x0 - p.x1137 max_gap = (1.0 if two_col else 2.2) * max(p.size, ln.size)138 if overlap >= 0.4 * h and -1.0 <= gap <= max_gap and bool(p.meta.get("frag")) == bool(ln.meta.get("frag")):139 if gap > 0.9 * max(p.size, ln.size):140 p.meta["wide_gaps"] = p.meta.get("wide_gaps", 0) + 1 # cell-like spacing (table rows)141 sep = "" if (p.text.endswith(" ") or ln.text.startswith(" ")) else " "142 n = p.n_chars + ln.n_chars143 p.math_frac = (p.math_frac * p.n_chars + ln.math_frac * ln.n_chars) / max(1, n)144 p.text = p.text + sep + ln.text145 p.x1 = max(p.x1, ln.x1)146 p.y0, p.y1 = min(p.y0, ln.y0), max(p.y1, ln.y1)147 if ln.n_chars > p.n_chars:148 p.size, p.font = ln.size, ln.font149 p.n_chars = n150 p.bold = p.bold and ln.bold if re.search(r"[A-Za-z]", ln.text) else p.bold151 continue152 out.append(ln)153 return out154155156def _two_column(lines: list[_Line], width: float) -> bool:157 body = [ln for ln in lines if ln.n_chars >= 20]158 if len(body) < 12:159 return False160 mid = width / 2161 narrow = [ln for ln in body if (ln.x1 - ln.x0) < 0.48 * width]162 left = sum(1 for ln in narrow if ln.x1 <= mid + 5)163 right = sum(1 for ln in narrow if ln.x0 >= mid - 5)164 return len(narrow) >= 0.5 * len(body) and left >= 4 and right >= 4165166167def _order_two_column(lines: list[_Line], width: float) -> list[_Line]:168 """Reading order for a two-column page: full-width lines act as band separators; within a band, the169 left column is read before the right column."""170 mid = width / 2171 srt = sorted(lines, key=lambda ln: (ln.y0, ln.x0))172 out: list[_Line] = []173 band: list[_Line] = []174175 def flush() -> None:176 left = [ln for ln in band if ln.x0 < mid - 5]177 right = [ln for ln in band if ln.x0 >= mid - 5]178 out.extend(sorted(left, key=lambda ln: (ln.y0, ln.x0)))179 out.extend(sorted(right, key=lambda ln: (ln.y0, ln.x0)))180 band.clear()181182 for ln in srt:183 if (ln.x1 - ln.x0) > 0.6 * width or (ln.x0 < mid - 20 and ln.x1 > mid + 20):184 flush()185 out.append(ln)186 else:187 band.append(ln)188 flush()189 return out190191192def _mode(values: list[float], default: float) -> float:193 if not values:194 return default195 return Counter(values).most_common(1)[0][0]196197198# ---------------------------------------------------------------------------------------------------------199# Line classification helpers200201202def _is_terminal(text: str) -> bool:203 return bool(TERMINAL_RE.search(text.rstrip()))204205206def _starts_lower(text: str) -> bool:207 t = text.lstrip(" \"“‘'([")208 return bool(t) and t[0].islower()209210211def _is_table_line(text: str) -> bool:212 toks = text.split()213 if not toks:214 return False215 nonspace = sum(len(t) for t in toks)216 digits = sum(c.isdigit() for c in text)217 words = [w for w in WORD_RE.findall(text) if len(w) >= 4]218 numeric_toks = sum(1 for t in toks if re.fullmatch(r"[(\[]?[-−+]?[\d.,]+%?[)\]]?\**", t))219 if len(toks) >= 2 and digits / max(1, nonspace) > 0.4:220 return True221 if numeric_toks >= 3 and numeric_toks >= 0.5 * len(toks):222 return True223 if len(toks) >= 5 and nonspace / len(toks) <= 3.2 and len(words) <= 1:224 return True225 return False226227228def _is_equation_line(ln: _Line) -> bool:229 t = ln.text.strip()230 if not t:231 return False232 if ln.math_frac > 0.4:233 return True234 nonspace = [c for c in t if not c.isspace()]235 alpha = sum(c.isalpha() and c.isascii() for c in nonspace)236 words3 = [w for w in WORD_RE.findall(t) if len(w) >= 3]237 mathc = sum(c in MATH_CHARS for c in nonspace)238 if alpha / max(1, len(nonspace)) < 0.5 and len(words3) <= 2 and len(nonspace) >= 3:239 return True240 if mathc >= 2 and len(words3) <= 2:241 return True242 if re.search(r"\(\s*[A-Z]?\d{1,3}[a-z]?\s*\)\s*$", t) and len(words3) <= 3 and mathc >= 1:243 return True244 return False245246247def _heading_depth(text: str, size_level: int) -> int:248 m = NUMBERED_RE.match(text)249 if m:250 num = m.group(1).rstrip(".")251 if re.fullmatch(r"\d+(?:\.\d+)*", num):252 return num.count(".") + 1253 if re.fullmatch(r"[A-H](?:\.\d+)*", num):254 return num.count(".") + 1255 return 1256 return size_level257258259# ---------------------------------------------------------------------------------------------------------260# Main entry point261262263def parse_pdf(path: Path | str, stub: dict | None = None) -> ParsedDocument:264 import fitz # PyMuPDF265266 stub = stub or {}267 path = Path(path)268 doc = fitz.open(str(path))269 warnings: list[str] = []270 n_pages = doc.page_count271 raw_pages: list[list[_Line]] = []272 page_dims: list[tuple[float, float]] = []273 raw_text_chars = 0274 raw_text_all: list[str] = []275 two_col_pages = 0276 for pno in range(n_pages):277 page = doc[pno]278 w, h = page.rect.width, page.rect.height279 page_dims.append((w, h))280 d = page.get_text("dict")281 lines: list[_Line] = []282 for b in d.get("blocks", []):283 if b.get("type") != 0:284 continue285 for li in b.get("lines", []):286 ln = _line_from_spans(pno, b.get("number", 0), li["bbox"], li.get("spans", []))287 if ln is not None:288 lines.append(ln)289 raw_text_chars += ln.n_chars290 raw_text_all.append(ln.text)291 # Blocks made only of tiny glyph fragments are display equations typeset glyph by glyph292 by_block: dict[int, list[_Line]] = {}293 for ln in lines:294 by_block.setdefault(ln.block, []).append(ln)295 for blines in by_block.values():296 txt = " ".join(x.text for x in blines)297 if sum(x.n_chars for x in blines) / len(blines) < 6 and not any(len(wd) >= 4 for wd in WORD_RE.findall(txt)):298 for x in blines:299 x.meta["frag"] = True300 tc = _two_column(lines, w)301 if tc:302 two_col_pages += 1303 lines = _order_two_column(lines, w)304 lines = _merge_visual_lines(lines, tc)305 for ln in lines:306 ln.meta["two_col"] = tc307 raw_pages.append(lines)308 doc.close()309 if n_pages == 0 or raw_text_chars < 30 * max(1, n_pages) * 0.5:310 raise ValueError("no text layer")311 if raw_text_chars / max(1, n_pages) < 800:312 warnings.append(f"low_text_density={raw_text_chars / max(1, n_pages):.0f} chars/page")313 if two_col_pages:314 warnings.append(f"two_column_pages={two_col_pages}/{n_pages}")315 warnings.append(f"pdf_pages={n_pages}")316317 # Licence statement printed in the PDF (cross-check only; the stub's provider evidence is authoritative)318 joined_raw = " ".join(raw_text_all)319 m_lic = re.search(r"creativecommons\.org/(?:licenses|publicdomain)/[a-z\-]+(?:/\d\.\d)?", joined_raw, re.I)320 stub_lic = (stub.get("license") or {}).get("code", "UNKNOWN")321 if m_lic:322 pdf_code = normalize_license(None, m_lic.group(0))323 if pdf_code != stub_lic:324 warnings.append(f"license statement in PDF ({pdf_code}, {m_lic.group(0)}) differs from provider ({stub_lic})")325326 # Body font size / font (character-weighted modes)327 size_ctr: Counter = Counter()328 font_ctr: Counter = Counter()329 for lines in raw_pages:330 for ln in lines:331 size_ctr[ln.size] += ln.n_chars332 font_ctr[ln.font] += ln.n_chars333 body_size = size_ctr.most_common(1)[0][0] if size_ctr else 10.0334 body_font = font_ctr.most_common(1)[0][0] if font_ctr else ""335 total_chars = sum(font_ctr.values()) or 1336 rare_fonts = {f for f, c in font_ctr.items() if c / total_chars < 0.06}337338 # ---- running headers / footers / page numbers339 def _key(t: str) -> str:340 return re.sub(r"\d+", "#", re.sub(r"\s+", " ", t.strip().lower()))341342 key_pages: dict[str, set[int]] = {}343 for lines in raw_pages:344 for ln in lines:345 _, h = page_dims[ln.page]346 if ln.y1 < 0.13 * h or ln.y0 > 0.88 * h:347 key_pages.setdefault(_key(ln.text), set()).add(ln.page)348 threshold = max(2, math.ceil(0.4 * n_pages))349 n_hf = 0350 for lines in raw_pages:351 for ln in lines:352 _, h = page_dims[ln.page]353 in_margin = ln.y1 < 0.13 * h or ln.y0 > 0.88 * h354 if not in_margin:355 continue356 if ln.page == 0 and ln.y1 < 0.13 * h:357 ln.kind, ln.meta["why"] = "dropped", "first_page_masthead"358 n_hf += 1359 continue360 k = _key(ln.text)361 if len(key_pages.get(k, ())) >= threshold or PAGE_NUM_RE.match(ln.text) or (MARGIN_BOILER_RE.search(ln.text) and len(ln.text) < 160):362 ln.kind = "dropped"363 ln.meta["why"] = "header_footer"364 n_hf += 1365 if n_hf:366 warnings.append(f"headers_footers_removed={n_hf}")367368 # ---- caption blocks (kept even when set in small print)369 caption_blocks: set[tuple[int, int]] = set()370 for lines in raw_pages:371 seen_blocks: set[int] = set()372 for ln in lines:373 if ln.kind == "dropped" or ln.block in seen_blocks:374 continue375 seen_blocks.add(ln.block)376 if CAPTION_RE.match(ln.text) or (CAPTION_LOOSE_RE.match(ln.text) and (ln.lead_bold or ln.font != body_font)):377 caption_blocks.add((ln.page, ln.block))378379 # ---- per-line classification: small print, equations, tables, front-page boilerplate380 stream: list[_Line] = []381 n_small = n_eq = n_tab = 0382 for lines in raw_pages:383 for ln in lines:384 if ln.kind == "dropped":385 continue386 in_caption = (ln.page, ln.block) in caption_blocks387 if REFERENCES_RE.match(normalize_text(ln.text)):388 # detected before the small-print filter: some publishers set the back matter in small type389 ln.kind = "heading"390 ln.meta.update(depth=1, title=normalize_text(ln.text))391 stream.append(ln)392 continue393 if not in_caption and ln.size < 0.85 * body_size:394 ln.kind, ln.meta["why"] = "dropped", "small_font"395 n_small += 1396 continue397 if not in_caption and (ln.meta.get("frag") or _is_equation_line(ln)):398 ln.kind = "equation"399 n_eq += 1400 elif not in_caption and (_is_table_line(ln.text) or ln.meta.get("wide_gaps", 0) >= 2):401 ln.kind = "table"402 n_tab += 1403 elif ln.page <= 1 and FRONT_BOILER_RE.match(ln.text) and not KNOWN_LABEL_RE.match(ln.text) and len(ln.text) < 200 and not _is_terminal(ln.text[:0] or "x"):404 # Short boilerplate lines (received/accepted dates, editors, correspondence) on the first pages405 if len(WORD_RE.findall(ln.text)) <= 30:406 ln.kind, ln.meta["why"] = "dropped", "front_boilerplate"407 continue408 stream.append(ln)409 for name, n in (("dropped_small_font_lines", n_small), ("dropped_equation_lines", n_eq), ("dropped_table_lines", n_tab)):410 if n:411 warnings.append(f"{name}={n}")412413 # ---- geometry statistics414 margins: dict[int, float] = {}415 rights: dict[int, float] = {}416 for pno in range(n_pages):417 xs = [round(ln.x0) for ln in stream if ln.page == pno and ln.kind == "text" and abs(ln.size - body_size) < 1.0]418 x1s = sorted(ln.x1 for ln in stream if ln.page == pno and ln.kind == "text" and abs(ln.size - body_size) < 1.0)419 margins[pno] = _mode(xs, min(xs) if xs else 0.0)420 rights[pno] = x1s[int(0.8 * (len(x1s) - 1))] if x1s else page_dims[pno][0]421 pitches = [b.y0 - a.y0 for a, b in zip(stream, stream[1:]) if a.page == b.page and 0 < b.y0 - a.y0 < 3 * body_size and a.kind == b.kind == "text"]422 pitch = sorted(pitches)[len(pitches) // 2] if pitches else 1.2 * body_size423 block_min: dict[tuple[int, int], float] = {}424 block_n: Counter = Counter()425 for ln in stream:426 if ln.kind == "text":427 k = (ln.page, ln.block)428 block_min[k] = min(block_min.get(k, ln.x0), ln.x0)429 block_n[k] += 1430431 # ---- headings and inline labels432 def heading_style(ln: _Line) -> tuple[bool, int]:433 if ln.size >= body_size + 1.5:434 return True, 1435 if ln.size >= body_size + 0.4:436 return True, 1 if not ln.italic else 2437 if ln.bold:438 return True, 1 if not ln.italic else 2439 if ln.font != body_font and ln.font in rare_fonts and not MATH_FONT_RE.search(ln.font):440 return True, 2 if ln.italic else 1441 return False, 1442443 def standalone(i: int) -> bool:444 if i == 0:445 return True446 p = stream[i - 1]447 ln = stream[i]448 if p.kind in ("heading", "label", "equation", "table") or p.page != ln.page or p.block != ln.block:449 return True450 if _is_terminal(p.text):451 return True452 return (ln.y0 - p.y0) > 1.5 * pitch453454 for i, ln in enumerate(stream):455 if ln.kind != "text":456 continue457 t = normalize_text(ln.text)458 # Inline labels: "Abstract: ...", "Keywords: ...", "Acknowledgments: ..." (bold run-in or known word)459 mlab = KNOWN_LABEL_RE.match(t)460 lead = normalize_text(ln.lead_bold)461 if mlab and (standalone(i) or lead):462 label = mlab.group(1)463 rest = t[mlab.end():].strip()464 if rest:465 ln.kind = "label"466 ln.meta["label"] = label467 ln.meta["rest"] = rest468 continue469 if ((ln.page, ln.block) in caption_blocks and (CAPTION_RE.match(t) or CAPTION_LOOSE_RE.match(t))) or (CAPTION_RE.match(t) and standalone(i) and not _starts_lower(t)):470 ln.kind = "caption"471 continue472 n_words = len(t.split())473 if n_words == 0 or n_words > 14 or len(t) > 120 or re.search(r"[,;]$", t) or len(WORD_RE.findall(t)) == 0:474 continue475 if not (t[0].isupper() or t[0].isdigit()):476 continue477 if CAPTION_RE.match(t):478 continue479 body_t = NUMBERED_RE.match(t).group(2) if NUMBERED_RE.match(t) else t480 nonsp = [c for c in body_t if not c.isspace()]481 if not any(len(w) >= 3 for w in WORD_RE.findall(body_t)) or any(c in MATH_CHARS for c in body_t) or sum(c.isalpha() for c in nonsp) < 0.6 * max(1, len(nonsp)):482 continue483 styled, level = heading_style(ln)484 numbered = bool(NUMBERED_RE.match(t)) and not _is_terminal(t)485 known = bool(KNOWN_HEADING_RE.match(t))486 nxt = stream[i + 1] if i + 1 < len(stream) else None487 short_line = ln.x1 < rights.get(ln.page, ln.x1) - 2 * body_size or n_words <= 8488 if not short_line or not standalone(i):489 continue490 if REFERENCES_RE.match(t) or known or (styled and (numbered or not t.endswith("."))) or (numbered and n_words <= 10 and (nxt is None or nxt.y0 - ln.y0 > 1.15 * pitch or nxt.x0 > ln.x0 + 0.5 * body_size)):491 if t.endswith(".") and not numbered and not known and n_words > 6:492 continue493 ln.kind = "heading"494 ln.meta["depth"] = _heading_depth(t, level)495 ln.meta["title"] = t496497 # Merge wrapped multi-line headings498 for i in range(1, len(stream)):499 p, ln = stream[i - 1], stream[i]500 if p.kind == "heading" and ln.kind == "heading" and not NUMBERED_RE.match(ln.meta["title"]) and p.page == ln.page and (ln.y0 - p.y0) < 1.6 * max(pitch, p.size) and not REFERENCES_RE.match(ln.meta["title"]) and not KNOWN_HEADING_RE.match(ln.meta["title"]) and abs(p.size - ln.size) < 0.6:501 ln.meta["title"] = p.meta["title"] + " " + ln.meta["title"]502 ln.meta["depth"] = p.meta["depth"]503 p.kind = "dropped"504 p.meta["why"] = "merged_heading"505506 # ---- front matter: drop everything before the first heading/label if it occurs on the first two pages507 def _structural(ln: _Line) -> bool:508 if ln.kind == "label":509 return True510 if ln.kind != "heading":511 return False512 t = ln.meta["title"]513 return bool(KNOWN_HEADING_RE.match(t) or NUMBERED_RE.match(t) or REFERENCES_RE.match(t))514515 first_struct = next((i for i, ln in enumerate(stream) if _structural(ln) and ln.page <= 1), None)516 if first_struct is None:517 first_struct = next((i for i, ln in enumerate(stream) if ln.kind in ("heading", "label")), None)518 start = 0519 if first_struct is not None and stream[first_struct].page <= 1:520 start = first_struct521 n_front = sum(1 for ln in stream[:first_struct] if ln.kind != "dropped")522 if n_front:523 warnings.append(f"front_matter_dropped_lines={n_front}")524 else:525 warnings.append("no_heading_on_first_pages")526527 # ---- vocabulary for hyphenation repair (excluding the fragments around line-end hyphens)528 vocab: Counter = Counter()529 prev_hyph = False530 for ln in stream:531 if ln.kind not in ("text", "label", "caption"):532 continue533 toks = WORD_RE.findall(ln.text)534 hy = bool(re.search(r"[A-Za-z][-‐]\s*$", ln.text))535 if prev_hyph and toks:536 toks = toks[1:]537 if hy and toks:538 toks = toks[:-1]539 vocab.update(t.lower() for t in toks)540 # hyphenated compounds observed inside lines541 for comp in re.findall(r"[A-Za-z]+-[A-Za-z]+", ln.text):542 vocab["#" + comp.lower()] += 1543 prev_hyph = hy544545 # ---- paragraph assembly546 paragraphs: list[ParsedParagraph] = []547 body_parts: list[str] = []548 cursor = 0549 pidx = 0550 n_dehyph = n_kept = 0551 cur_lines: list[str] = []552 cur_flags: set[str] = set()553 last: _Line | None = None554 pending_math = False555 sec = {"id": "front", "title": None, "role": "other", "depth": 1}556 role_stack: list[tuple[int, str]] = []557 sec_counter = 0558 skipping = False559 cut_at: int | None = None560561 def join_line(prev_text: str, nxt: str) -> tuple[str, str | None]:562 """Join ``nxt`` to the accumulated ``prev_text`` repairing end-of-line hyphenation."""563 nonlocal n_dehyph, n_kept564 nxt = nxt.strip()565 if not prev_text:566 return nxt, None567 if prev_text.endswith(""):568 n_dehyph += 1569 return prev_text[:-1] + nxt, "dehyphenated"570 m = re.search(r"([A-Za-z]+)[-‐]$", prev_text)571 if m and _starts_lower(nxt) and not prev_text.endswith("--"):572 prefix = m.group(1)573 msuf = re.match(r"([a-z]+)", nxt)574 suffix = msuf.group(1) if msuf else ""575 joined = (prefix + suffix).lower()576 if suffix and (vocab.get(joined, 0) > 0 and not vocab.get("#" + prefix.lower() + "-" + suffix, 0)):577 n_dehyph += 1578 return prev_text[:-1] + nxt, "dehyphenated"579 if vocab.get("#" + prefix.lower() + "-" + suffix, 0) or (len(prefix) >= 2 and vocab.get(prefix.lower(), 0) > 0 and (vocab.get(suffix, 0) > 0 or len(suffix) <= 2)):580 n_kept += 1581 return prev_text + nxt, "hyphen_kept"582 n_dehyph += 1583 return prev_text[:-1] + nxt, "dehyphenated_guess"584 if m and nxt[:1].isupper():585 return prev_text + nxt, None # "Fin-" + "Tech" → "Fin-Tech"586 return prev_text + " " + nxt, None587588 captions: list[list[str]] = [] # captions seen while a body paragraph was open (emitted after it)589 cap_lines: list[str] | None = None590 cap_last: _Line | None = None591 table_region = False592 n_table_region = 0593594 def _emit(lines: list[str], flags: set[str], role: str) -> None:595 nonlocal cursor, pidx596 acc = ""597 for piece in lines:598 acc, fl = join_line(acc, piece)599 if fl:600 flags.add(fl)601 text = re.sub(r"\s+", " ", normalize_text(acc.replace("\n", " "))).strip()602 if not text or not WORD_RE.search(text):603 return604 pr = protect(text)605 start_c = cursor606 body_parts.append(text + "\n\n")607 cursor += len(text) + 2608 paragraphs.append(ParsedParagraph(sec["id"], sec["title"], role, sec["depth"], pidx, text, start_c, start_c + len(text), pr.citations, pr.math, pr.urls, pr.numbers, sorted(flags)))609 pidx += 1610611 def end_caption() -> None:612 nonlocal cap_lines, cap_last613 if cap_lines:614 captions.append(cap_lines)615 cap_lines, cap_last = None, None616617 def flush() -> None:618 nonlocal cur_lines, cur_flags619 end_caption()620 if cur_lines and not skipping:621 _emit(cur_lines, cur_flags, sec["role"])622 if not skipping:623 for c in captions:624 _emit(c, {"caption"}, "caption")625 captions.clear()626 cur_lines = []627 cur_flags = set()628629 def open_section(title: str, depth: int, forced_role: str | None = None) -> None:630 nonlocal sec_counter, skipping, sec631 flush()632 if EXCLUDED_SECTION_TITLES.search(title) or LOCAL_EXCLUDED_RE.search(title):633 skipping = True634 return635 skipping = False636 sec_counter += 1637 role = forced_role or section_role(title, depth)638 while role_stack and role_stack[-1][0] >= depth:639 role_stack.pop()640 if role == "other" and role_stack:641 role = role_stack[-1][1]642 role_stack.append((depth, role))643 sec = {"id": "abstract" if role == "abstract" else f"s{sec_counter}", "title": title, "role": role, "depth": depth}644645 def new_paragraph_needed(p: _Line, ln: _Line) -> bool:646 term = _is_terminal(p.text)647 size = ln.size or body_size648 if ln.page != p.page or (ln.meta.get("two_col") and ln.x0 > p.x1):649 # page or column break: continue unless the previous paragraph visibly ended650 margin = margins.get(ln.page, ln.x0)651 indent = 0.6 * size < (ln.x0 - margin) < 8 * size652 p_short = p.x1 < rights.get(p.page, p.x1) - 2.5 * size653 return term and (indent or p_short)654 bkey = (ln.page, ln.block)655 ref_left = block_min.get(bkey, margins.get(ln.page, ln.x0)) if block_n[bkey] >= 3 else margins.get(ln.page, ln.x0)656 indent = 0.6 * size < (ln.x0 - ref_left) < 8 * size657 p_short = p.x1 < rights.get(p.page, p.x1) - 2.5 * size658 big_gap = (ln.y0 - p.y0) > 1.6 * pitch659 if term and (indent or p_short or big_gap):660 return True661 if big_gap and not _starts_lower(ln.text) and p.block != ln.block and not p.text.rstrip().endswith(("-", ",")):662 return True663 return False664665 for i in range(start, len(stream)):666 ln = stream[i]667 if ln.kind == "dropped":668 continue669 if ln.kind == "heading":670 title = ln.meta["title"]671 if table_region and not (NUMBERED_RE.match(title) or KNOWN_HEADING_RE.match(title) or REFERENCES_RE.match(title)):672 n_table_region += 1 # bold column headers inside a table body are not section headings673 continue674 if REFERENCES_RE.match(title):675 cut_at = ln.page676 break677 open_section(title, ln.meta.get("depth", 1), "abstract" if re.match(r"^\s*abstract\s*$", title, re.I) else None)678 last, pending_math, table_region = None, False, False679 continue680 if ln.kind == "label":681 label = ln.meta["label"]682 low = label.lower()683 if low in ("abstract", "summary"):684 open_section("Abstract", 1, "abstract")685 elif low == "introduction":686 open_section("Introduction", 1)687 else:688 open_section(label, 1)689 last, pending_math = None, False690 if not skipping:691 cur_lines.append(ln.meta["rest"])692 last = ln693 continue694 if ln.kind == "equation":695 if cur_lines:696 pending_math = True697 continue698 if ln.kind == "table":699 continue700 if skipping:701 last = ln702 continue703 if ln.kind == "caption":704 # Captions are buffered so that a body paragraph interrupted by a float is not split.705 end_caption()706 cap_lines, cap_last = [ln.text], ln707 table_region = bool(re.match(r"\s*Table", ln.text, re.I))708 if not cur_lines:709 flush()710 continue711 if cap_lines is not None and cap_last is not None:712 same_block = (ln.page, ln.block) == (cap_last.page, cap_last.block)713 if same_block and not new_paragraph_needed(cap_last, ln):714 cap_lines.append(ln.text)715 cap_last = ln716 continue717 end_caption()718 if table_region:719 # Table bodies set at body size (common in MDPI PDFs) follow the "Table N." caption: drop lines720 # until the first prose-like line (long, full-width or sentence-final).721 n_w = len(ln.text.split())722 span = rights.get(ln.page, ln.x1) - margins.get(ln.page, ln.x0)723 prose_like = n_w >= 8 and not ln.meta.get("wide_gaps") and ((ln.x1 - ln.x0) >= 0.6 * span or _is_terminal(ln.text)) and not _is_table_line(ln.text)724 if not prose_like:725 n_table_region += 1726 continue727 table_region = False728 # plain text line729 if last is not None and cur_lines:730 if new_paragraph_needed(last, ln) and not (_starts_lower(ln.text) and not _is_terminal(last.text)):731 flush()732 pending_math = False733 elif pending_math:734 cur_lines.append("⟦MATH⟧")735 cur_flags.add("has_disp_formula")736 pending_math = False737 if cur_lines and last.page != ln.page:738 cur_flags.add("crosses_page")739 cur_lines.append(ln.text)740 last = ln741 flush()742743 if n_table_region:744 warnings.append(f"dropped_table_region_lines={n_table_region}")745 if cut_at is not None:746 warnings.append(f"references_cut_at_page={cut_at + 1}")747 else:748 warnings.append("no_references_heading")749 if n_dehyph or n_kept:750 warnings.append(f"dehyphenated={n_dehyph} hyphen_kept={n_kept}")751 if sec_counter == 0:752 warnings.append("no_headings_detected")753 if not paragraphs:754 warnings.append("no paragraphs extracted")755 good = sum(1 for p in paragraphs if p.section_role != "caption" and quality_score(p.text)[0] >= 0.6)756 tot = sum(1 for p in paragraphs if p.section_role != "caption")757 warnings.append(f"extraction_quality={good / tot if tot else 0.0:.2f} ({good}/{tot} paragraphs quality>=0.6)")758759 pub_date = stub.get("pub_date")760 pub_year = stub.get("pub_year") or (int(pub_date[:4]) if pub_date else None)761 pre2018 = False762 if pub_date:763 iso = pub_date + ("-01" if len(pub_date) == 7 else "-01-01" if len(pub_date) == 4 else "")764 try:765 pre2018 = _dt.date.fromisoformat(iso) < _dt.date(2018, 1, 1)766 except ValueError:767 pre2018 = False768 lic = stub.get("license") or {}769 ids = {"doi": stub["source_ids"]["doi"]} if stub.get("source_ids", {}).get("doi") else {}770 return ParsedDocument(771 title=stub.get("title"),772 authors=list(stub.get("authors") or []),773 author_group=stub.get("author_group"),774 venue=stub.get("venue"),775 publisher=stub.get("publisher"),776 doc_type=stub.get("doc_type") or "article",777 pub_date=pub_date,778 pub_year=pub_year,779 pub_date_evidence=stub.get("pub_date_evidence"),780 pre2018_verified=pre2018 and bool(stub.get("pub_date_evidence")),781 subjects=list(stub.get("subject_raw") or []),782 language=stub.get("language"),783 license_code=lic.get("code", "UNKNOWN"),784 license_uri=lic.get("uri"),785 license_evidence=lic.get("evidence"),786 body_text="".join(body_parts),787 paragraphs=paragraphs,788 warnings=warnings,789 article_type=stub.get("doc_type"),790 ids=ids,791 )792