Repository Public Research preview
1"""Paragraph → passage eligibility (the unit that is rewritten and reconstructed)."""2from __future__ import annotations34from dataclasses import dataclass56from htr.data.schema import Passage7from htr.parse.jats import ParsedDocument8from htr.parse.quality import detect_language, quality_score9from htr.utils.hashing import sha256_text, short_id10from htr.utils.text import split_sentences, word_count1112MAIN_TEXT_ROLES = {"abstract", "introduction", "background", "methods", "results", "discussion", "conclusion", "related_work", "theory", "data", "limitations", "other", "chapter", "preface"}131415@dataclass16class SegmentConfig:17 min_words: int = 6018 max_words: int = 40019 min_sentences: int = 220 min_quality: float = 0.621 language: str = "en"22 min_lang_prob: float = 0.923 include_abstract: bool = True24 include_captions: bool = False252627def passages_from_parsed(doc_id: str, parsed: ParsedDocument, cfg: SegmentConfig | None = None) -> list[Passage]:28 cfg = cfg or SegmentConfig()29 out: list[Passage] = []30 for p in parsed.paragraphs:31 n_words = word_count(p.text)32 n_sent = len(split_sentences(p.text))33 q, qflags = quality_score(p.text)34 flags = list(p.flags) + qflags35 lang, prob = detect_language(p.text) if n_words >= 20 else (None, 0.0)36 eligible = (37 p.section_role in MAIN_TEXT_ROLES38 and (cfg.include_abstract or p.section_role != "abstract")39 and (cfg.include_captions or p.section_role != "caption")40 and cfg.min_words <= n_words <= cfg.max_words41 and n_sent >= cfg.min_sentences42 and q >= cfg.min_quality43 and ((lang == cfg.language and prob >= cfg.min_lang_prob) or (lang is None and cfg.min_lang_prob <= 0))44 and "list_item" not in flags45 )46 if lang is not None and lang != cfg.language:47 flags.append(f"lang:{lang}")48 out.append(49 Passage(50 passage_id=short_id("psg", doc_id, str(p.para_index), sha256_text(p.text)[:16]),51 doc_id=doc_id,52 section_id=p.section_id,53 section_title=p.section_title,54 section_role=p.section_role, # type: ignore[arg-type]55 section_depth=p.section_depth,56 para_index=p.para_index,57 text=p.text,58 char_start=p.char_start,59 char_end=p.char_end,60 n_words=n_words,61 n_sentences=n_sent,62 language=lang,63 quality=q,64 flags=flags,65 protected={"citations": p.citations, "math": p.math, "urls": p.urls, "numbers": p.numbers},66 text_sha256=sha256_text(p.text),67 eligible=eligible,68 )69 )70 return out71