Repository Public
1"""JATS (and BITS, for NCBI Bookshelf books) parser: front matter, verified publication date, licence2evidence, body sections and paragraphs with citation/math preservation and reference-list exclusion.3"""4from __future__ import annotations56import datetime as _dt7import re8from dataclasses import dataclass, field9from pathlib import Path1011from lxml import etree1213from htr.licenses.policy import normalize_license14from htr.parse.protect import protect15from htr.utils.text import normalize_text, split_sentences, word_count1617XLINK = "{http://www.w3.org/1999/xlink}"18MML = "{http://www.w3.org/1998/Math/MathML}"1920SECTION_ROLE_PATTERNS: list[tuple[re.Pattern, str]] = [21 (re.compile(r"^(abstract|summary)$", re.I), "abstract"),22 (re.compile(r"introduction|^overview$|^purpose", re.I), "introduction"),23 (re.compile(r"background|motivation|preliminar", re.I), "background"),24 (re.compile(r"related work|prior work|literature", re.I), "related_work"),25 (re.compile(r"method|material|experimental|procedure|protocol|participants|study design|measures|statistical analysis|econometric|empirical strategy|identification|specification", re.I), "methods"),26 (re.compile(r"^data\b|data and|dataset|sample|descriptive statistics", re.I), "data"),27 (re.compile(r"theor|model\b|framework|hypothes", re.I), "theory"),28 (re.compile(r"result|finding|empirical evidence|estimates", re.I), "results"),29 (re.compile(r"discussion|interpretation|implication", re.I), "discussion"),30 (re.compile(r"limitation|caveat|threats to validity", re.I), "limitations"),31 (re.compile(r"conclu|concluding|final remarks|summary and", re.I), "conclusion"),32]33EXCLUDED_SECTION_TITLES = re.compile(r"acknowledg|funding|conflict|competing|author[s']{0,3}\s*contribution|supplement|abbreviation|ethic|consent|availability|disclosure|appendix|footnote|declaration|key ?words", re.I)343536def section_role(title: str | None, depth: int = 1, in_abstract: bool = False) -> str:37 if in_abstract:38 return "abstract"39 if not title:40 return "other"41 t = title.strip().rstrip(".:").strip()42 t = re.sub(r"^(?:\d+(?:\.\d+)*\.?|[IVX]+\.)\s+", "", t) # strip numbering ("3.1 Methods", "IV. Results")43 for pat, role in SECTION_ROLE_PATTERNS:44 if pat.search(t):45 return role46 return "other"474849@dataclass50class ParsedParagraph:51 section_id: str52 section_title: str | None53 section_role: str54 section_depth: int55 para_index: int56 text: str57 char_start: int58 char_end: int59 citations: list[str] = field(default_factory=list)60 math: list[str] = field(default_factory=list)61 urls: list[str] = field(default_factory=list)62 numbers: list[str] = field(default_factory=list)63 flags: list[str] = field(default_factory=list)646566@dataclass67class ParsedDocument:68 title: str | None69 authors: list[str]70 author_group: str | None71 venue: str | None72 publisher: str | None73 doc_type: str74 pub_date: str | None75 pub_year: int | None76 pub_date_evidence: str | None77 pre2018_verified: bool78 subjects: list[str]79 language: str | None80 license_code: str81 license_uri: str | None82 license_evidence: str | None83 body_text: str84 paragraphs: list[ParsedParagraph]85 warnings: list[str] = field(default_factory=list)86 article_type: str | None = None87 ids: dict = field(default_factory=dict)888990def _text_of(el: etree._Element) -> str:91 return normalize_text("".join(el.itertext()))929394def _first(root: etree._Element, *xpaths: str) -> etree._Element | None:95 """First element matching any of the xpaths, in order (never uses element truthiness)."""96 for xp in xpaths:97 r = root.xpath(xp)98 if r:99 return r[0]100 return None101102103def _parse_pub_dates(article_meta: etree._Element | None) -> tuple[str | None, int | None, str | None, list[tuple[str, str]]]:104 """Return (iso_date, year, evidence, all_dates). Preference order for *publication* date:105 collection/ppub (print) → epub → any pub-date with a year. Deposit/received/accepted dates are ignored."""106 if article_meta is None:107 return None, None, None, []108 found: list[tuple[str, str]] = []109 for pd in article_meta.xpath("./pub-date|./pub-history/event/date"):110 ptype = pd.get("pub-type") or pd.get("date-type") or pd.get("publication-format") or "unknown"111 y = pd.findtext("year")112 if not y or not re.fullmatch(r"\d{4}", y.strip()):113 continue114 m = pd.findtext("month")115 d = pd.findtext("day")116 iso = y.strip()117 if m and m.strip().isdigit():118 iso += f"-{int(m):02d}"119 if d and d.strip().isdigit():120 iso += f"-{int(d):02d}"121 found.append((ptype, iso))122 if not found:123 return None, None, None, []124 order = ["ppub", "collection", "epub", "epub-ppub", "pub", "published", "unknown"]125 for pref in order:126 for ptype, iso in found:127 if ptype == pref:128 return iso, int(iso[:4]), f"jats pub-date[pub-type={ptype}]", found129 ptype, iso = found[0]130 return iso, int(iso[:4]), f"jats pub-date[pub-type={ptype}]", found131132133def _license(article_meta: etree._Element | None, root: etree._Element) -> tuple[str, str | None, str | None]:134 nodes = root.xpath("//permissions/license") or root.xpath("//license")135 for lic in nodes:136 uri = lic.get(XLINK + "href") or lic.get("href")137 refs = [r.text for r in lic.xpath(".//*[local-name()='license_ref']") if r.text]138 uri = uri or (refs[0] if refs else None)139 txt = _text_of(lic)140 code = normalize_license(txt if "creative commons" in txt.lower() or "CC" in txt else None, uri)141 if code != "UNKNOWN":142 return code, uri, "jats permissions/license xlink:href" if uri else "jats permissions/license text"143 if uri:144 return normalize_license(None, uri), uri, "jats permissions/license xlink:href"145 # copyright statements that declare public domain146 cs = root.xpath("//permissions/copyright-statement")147 for c in cs:148 t = _text_of(c).lower()149 if "public domain" in t:150 return "PUBLIC-DOMAIN", None, "jats copyright-statement"151 return "UNKNOWN", None, None152153154def _inline_math(el: etree._Element) -> str:155 tex = el.xpath(".//tex-math")156 if tex and tex[0].text:157 t = tex[0].text.strip()158 t = re.sub(r"^\\documentclass.*?\\begin\{document\}", "", t, flags=re.S).replace("\\end{document}", "").strip()159 t = t.strip("$").strip()160 return f"${t}$" if t else "⟦MATH⟧"161 mml = el.xpath(".//*[local-name()='math']")162 if mml:163 alt = "".join(mml[0].itertext()).strip()164 alt = re.sub(r"\s+", "", alt)165 return f"${alt}$" if alt else "⟦MATH⟧"166 alt = (el.get("alt") or "").strip()167 return f"${alt}$" if alt else "⟦MATH⟧"168169170def _para_text(p: etree._Element) -> tuple[str, list[str]]:171 """Serialize a <p> into plain text, keeping xref text, converting formulas to $...$ and dropping172 figures/tables/footnotes embedded in the paragraph."""173 flags: list[str] = []174 parts: list[str] = []175176 def walk(node: etree._Element) -> None:177 tag = etree.QName(node).localname if isinstance(node.tag, str) else None178 if tag in ("fig", "table-wrap", "fn", "supplementary-material", "graphic", "media", "table", "boxed-text", "disp-quote") or tag is None:179 if tag is not None:180 flags.append(f"dropped:{tag}")181 if node.tail:182 parts.append(node.tail)183 return184 if tag in ("inline-formula", "disp-formula"):185 parts.append(" " + _inline_math(node) + " ")186 if tag == "disp-formula":187 flags.append("has_disp_formula")188 if node.tail:189 parts.append(node.tail)190 return191 if tag == "xref":192 rt = node.get("ref-type")193 txt = "".join(node.itertext())194 if rt == "bibr":195 flags.append("has_citation")196 # Normalise bare numeric citations to bracketed form so that they are detectable197 txt_s = txt.strip()198 if re.fullmatch(r"\d{1,4}([,–-]\d{1,4})*", txt_s):199 txt = f"[{txt_s}]"200 parts.append(txt)201 if node.tail:202 parts.append(node.tail)203 return204 if tag in ("sup", "sub"):205 parts.append("".join(node.itertext()))206 if node.tail:207 parts.append(node.tail)208 return209 if node.text:210 parts.append(node.text)211 for ch in node:212 walk(ch)213 if node.tail:214 parts.append(node.tail)215216 if p.text:217 parts.append(p.text)218 for ch in p:219 walk(ch)220 text = normalize_text("".join(parts))221 # Collapse adjacent bracketed citations produced from separate xrefs: "[1][2]" → "[1,2]"222 text = re.sub(r"\[\s*\[", "[", text)223 text = re.sub(r"\]\s*\]", "]", text)224 text = re.sub(r"\]\s*[,;]?\s*\[", ",", text)225 text = re.sub(r"\s+([.,;:])", r"\1", text)226 return text, sorted(set(flags))227228229def parse_jats(xml_bytes: bytes, source_hint: str = "pmc") -> ParsedDocument:230 parser = etree.XMLParser(recover=True, huge_tree=True, remove_comments=True, resolve_entities=False, no_network=True)231 root = etree.fromstring(xml_bytes, parser=parser)232 if root is None:233 raise ValueError("unparseable XML")234 if etree.QName(root).localname in ("pmc-articleset", "articleset"):235 arts = root.xpath("./article") or root.xpath(".//*[local-name()='article']")236 root = arts[0] if arts else root237 is_book = etree.QName(root).localname in ("book", "book-part", "book-part-wrapper")238 warnings: list[str] = []239 meta = _first(root, ".//front/article-meta", ".//book-meta", ".//book-part-meta")240 jmeta = _first(root, ".//front/journal-meta")241 title_el = _first(root, ".//front/article-meta/title-group/article-title", ".//book-title-group/book-title", ".//title-group/title")242 title = _text_of(title_el) if title_el is not None else None243 authors: list[str] = []244 for c in root.xpath(".//contrib-group/contrib[@contrib-type='author']|.//contrib-group/contrib[not(@contrib-type)]"):245 sn = c.findtext(".//surname") or ""246 gn = c.findtext(".//given-names") or ""247 nm = normalize_text(f"{gn} {sn}".strip()) or normalize_text(c.findtext(".//string-name") or c.findtext(".//collab") or "")248 if nm:249 authors.append(nm)250 author_group = None251 if authors:252 toks = authors[0].split()253 if toks:254 author_group = re.sub(r"[^a-z]", "", toks[-1].lower()) + ("_" + toks[0][0].lower() if len(toks) > 1 else "")255 venue = None256 publisher = None257 if jmeta is not None:258 venue = jmeta.findtext(".//journal-title") or jmeta.findtext(".//journal-id")259 publisher = jmeta.findtext(".//publisher-name")260 venue = normalize_text(venue) if venue else None261 if publisher is None and meta is not None:262 publisher = meta.findtext(".//publisher-name")263 pub_date, pub_year, evidence, _all = _parse_pub_dates(meta)264 pre2018 = bool(pub_date) and _dt.date.fromisoformat(pub_date + ("-01" if len(pub_date) == 7 else "-01-01" if len(pub_date) == 4 else "")) < _dt.date(2018, 1, 1)265 if pub_date and len(pub_date) == 4 and pub_year == 2017:266 warnings.append("year-only 2017 date: treated as pre-2018 (collection year)")267 subjects = [normalize_text(s) for s in root.xpath(".//article-categories//subject/text()") if s.strip()]268 lang = root.get("{http://www.w3.org/XML/1998/namespace}lang") or root.get("lang")269 lic_code, lic_uri, lic_ev = _license(meta, root)270 article_type = root.get("article-type") or ("book" if is_book else None)271 ids = {}272 for aid in root.xpath(".//article-meta/article-id|.//book-meta/book-id|.//book-part-meta/book-part-id"):273 t = aid.get("pub-id-type") or aid.get("book-id-type") or aid.get("book-part-id-type") or "id"274 if aid.text:275 ids[t] = aid.text.strip()276277 # Body: abstract + body sections; references, back matter, floats excluded278 paragraphs: list[ParsedParagraph] = []279 body_parts: list[str] = []280 cursor = 0281 pidx = 0282283 def emit(text: str, sec_id: str, sec_title: str | None, role: str, depth: int, flags: list[str]) -> None:284 nonlocal cursor, pidx285 if not text:286 return287 pr = protect(text)288 start = cursor289 body_parts.append(text + "\n\n")290 cursor += len(text) + 2291 paragraphs.append(ParsedParagraph(sec_id, sec_title, role, depth, pidx, text, start, start + len(text), pr.citations, pr.math, pr.urls, pr.numbers, flags))292 pidx += 1293294 abstract = _first(root, ".//front/article-meta/abstract[not(@abstract-type)]", ".//front/article-meta/abstract")295 if abstract is not None:296 for i, p in enumerate(abstract.xpath(".//p")):297 text, flags = _para_text(p)298 emit(text, "abstract", "Abstract", "abstract", 1, flags)299300 body = _first(root, ".//body")301 if body is None:302 warnings.append("no <body>")303 else:304 def walk_sec(sec: etree._Element, depth: int, parent_title: str | None, parent_role: str, sec_counter: list[int]) -> None:305 t_el = sec.find("title")306 stitle = _text_of(t_el) if t_el is not None else parent_title307 if stitle and EXCLUDED_SECTION_TITLES.search(stitle):308 return309 role = section_role(stitle, depth)310 if role == "other" and parent_role != "other":311 role = parent_role312 sec_counter[0] += 1313 sid = f"s{sec_counter[0]}"314 for ch in sec:315 tag = etree.QName(ch).localname if isinstance(ch.tag, str) else None316 if tag == "p":317 text, flags = _para_text(ch)318 emit(text, sid, stitle, role, depth, flags)319 elif tag == "sec":320 walk_sec(ch, depth + 1, stitle, role, sec_counter)321 elif tag in ("fig", "table-wrap"):322 cap = ch.find("caption")323 if cap is not None:324 text, flags = _para_text(cap)325 emit(text, sid, stitle, "caption", depth, flags + ["caption"])326 elif tag == "list":327 for li in ch.xpath(".//list-item/p"):328 text, flags = _para_text(li)329 emit(text, sid, stitle, role, depth, flags + ["list_item"])330 counter = [0]331 top_ps = [ch for ch in body if isinstance(ch.tag, str) and etree.QName(ch).localname == "p"]332 for p in top_ps:333 text, flags = _para_text(p)334 emit(text, "body", None, "other", 1, flags)335 for sec in body.xpath("./sec"):336 walk_sec(sec, 1, None, "other", counter)337 if not paragraphs:338 warnings.append("no paragraphs extracted")339 return ParsedDocument(title, authors, author_group, venue, publisher, "book_chapter" if is_book else ("review" if (article_type or "").startswith("review") else "article"), pub_date, pub_year, evidence, pre2018, subjects, lang, lic_code, lic_uri, lic_ev, "".join(body_parts), paragraphs, warnings, article_type, ids)340341342def parse_jats_file(path: Path | str) -> ParsedDocument:343 return parse_jats(Path(path).read_bytes())344345346def paragraph_stats(p: ParsedParagraph) -> dict:347 return {"n_words": word_count(p.text), "n_sentences": len(split_sentences(p.text))}348