Repository Public Research preview
1"""Detection of spans that must be preserved verbatim through rewriting and reconstruction:2citations, inline/display math, numbers with units, URLs/DOIs, and the placeholder protocol.34Placeholders are of the form ⟦C1⟧ (citation), ⟦M1⟧ (math), ⟦U1⟧ (url). Numbers are NOT replaced by5placeholders (they are part of the prose) but are extracted for invariance checks.6"""7from __future__ import annotations89import re10from dataclasses import dataclass, field1112CIT_MARK = "⟦C{}⟧"13MATH_MARK = "⟦M{}⟧"14URL_MARK = "⟦U{}⟧"15PLACEHOLDER_RE = re.compile(r"⟦([CMU])(\d+)⟧")1617# Numeric citations: [12], [3,4], [5–8], superscript style is lost in XML text; author-year: (Smith et al., 2015)18NUMERIC_CIT_RE = re.compile(r"\[(\d{1,4}(?:\s*[,–-]\s*\d{1,4})*)\]")19# BibTeX-key citations produced by the LaTeX parser from \cite{...}: [smith2010model, lee_2015], keys carry a year20KEY_CIT_RE = re.compile(r"\[[A-Za-z][\w:.\-/]*?(?:19|20)\d{2}[\w:.\-/]*(?:\s*,\s*[A-Za-z][\w:.\-/]*?(?:19|20)\d{2}[\w:.\-/]*)*\]")21PROTECT_VERSION = "1.1" # bump whenever a detection pattern changes; rewrites record the version they were produced with22# v1.0 pattern (kept for pairs produced before 2026-11-09 18:00 UTC; it missed "(Smith et al., 2010)")23AUTHOR_YEAR_RE_V10 = re.compile(r"\((?:[A-Z][A-Za-z'’\-]+(?:\s+(?:et al\.|and|&)\s+[A-Z][A-Za-z'’\-]+)*,?\s+(?:19|20)\d{2}[a-z]?(?:;\s*)?)+\)")24_NAME = r"(?:de |van |von |del |da |la |Mc|Mac|O')?[A-Z][A-Za-z'’\-]+"25AUTHOR_YEAR_RE = re.compile(26 r"\((?:(?:e\.g\.,?\s*|see\s+|cf\.\s*)?" + _NAME + r"(?:\s*,\s*" + _NAME + r")*(?:\s+(?:and|&)\s+" + _NAME + r")?(?:\s+et\s+al\.?)?,?\s+(?:1[6-9]|20)\d{2}[a-z]?(?:\s*,\s*(?:pp?\.|p\.|ch\.|chap\.)?\s*\d+(?:[–-]\d+)?)?(?:\s*;\s*)?)+\)"27)28URL_RE = re.compile(r"(?:https?://|www\.)[^\s)\]>]+|doi:\s*10\.\d{4,9}/[^\s)\]>]+", re.I)29MATH_RE = re.compile(r"\$\$[^$]+\$\$|\$[^$\n]+\$|\\\([^)]*?\\\)|\\\[[^\]]*?\\\]")30# Numbers may be glued to a unit ("0.13mm", "37°C"): the trailing lookahead only forbids further digits / decimals,31# so "0.13mm" and "0.13 mm" both yield "0.13"; thousands separators and scientific notation are kept.32NUMBER_RE = re.compile(r"(?<![\w.])[-−+]?\d{1,3}(?:[, ]\d{3})+(?:\.\d+)?(?:\s?[%‰]|\s?×\s?10\^?[-−]?\d+|e[-−+]?\d+)?(?![\d.])|(?<![\w.])[-−+]?\d+(?:\.\d+)?(?:\s?[%‰]|\s?×\s?10\^?[-−]?\d+|e[-−+]?\d+)?(?![\d.])")33UNIT_RE = re.compile(r"\b\d+(?:\.\d+)?\s?(?:mg|g|kg|µg|μg|ng|mL|ml|L|µL|μL|mm|cm|m|km|nm|µm|μm|s|ms|min|h|hr|days?|weeks?|months?|years?|°C|K|Hz|kHz|MHz|GHz|mol|mM|µM|μM|nM|M|kDa|Da|bp|kb|Mb|Gb|USD|EUR|\$|%|bps|pp)\b")34NEGATION_RE = re.compile(r"\b(?:not|no|never|neither|nor|none|cannot|can't|without|fail(?:s|ed)? to|lack(?:s|ed|ing)?|absence of|un(?:able|likely|clear))\b|n't\b", re.I)35HEDGE_RE = re.compile(r"\b(?:may|might|could|possibly|probably|likely|unlikely|suggest(?:s|ed)?|appear(?:s|ed)?|seem(?:s|ed)?|tend(?:s|ed)?|approximately|roughly|about|around|potentially|presumably|arguably|in part|to some extent|we hypothesi[sz]e|it is possible)\b", re.I)363738@dataclass39class Protected:40 text: str # text with placeholders41 citations: list[str] = field(default_factory=list)42 math: list[str] = field(default_factory=list)43 urls: list[str] = field(default_factory=list)44 numbers: list[str] = field(default_factory=list)4546 def restore(self, text: str) -> str:47 return restore_placeholders(text, self)484950def extract_numbers(text: str) -> list[str]:51 nums = []52 for m in NUMBER_RE.finditer(text):53 s = m.group(0).strip().replace("−", "-").replace(" ", "").replace(" ", "")54 if s in {"-", "+", ""}:55 continue56 nums.append(s)57 return nums585960def protect(text: str, protect_citations: bool = True, protect_math: bool = True, protect_urls: bool = True, version: str = PROTECT_VERSION) -> Protected:61 """Replace protected spans with indexed placeholders (order of appearance). Idempotent on already-protected text.62 ``version`` selects the detection patterns (pairs are validated with the version used when the rewrite was requested)."""63 p = Protected(text=text)64 author_year_re = AUTHOR_YEAR_RE_V10 if version == "1.0" else AUTHOR_YEAR_RE65 if protect_math:66 def _m(m: re.Match) -> str:67 p.math.append(m.group(0))68 return MATH_MARK.format(len(p.math))69 p.text = MATH_RE.sub(_m, p.text)70 if protect_urls:71 def _u(m: re.Match) -> str:72 p.urls.append(m.group(0))73 return URL_MARK.format(len(p.urls))74 p.text = URL_RE.sub(_u, p.text)75 if protect_citations:76 def _c(m: re.Match) -> str:77 p.citations.append(m.group(0))78 return CIT_MARK.format(len(p.citations))79 if version == "1.0": # legacy: two passes, numeric citations numbered before author–year ones80 p.text = NUMERIC_CIT_RE.sub(_c, p.text)81 p.text = author_year_re.sub(_c, p.text)82 else: # single pass → placeholders numbered in order of appearance; v1.1 also protects BibTeX-key citations83 combined = re.compile(f"(?:{NUMERIC_CIT_RE.pattern})|(?:{KEY_CIT_RE.pattern})|(?:{author_year_re.pattern})")84 p.text = combined.sub(_c, p.text)85 p.numbers = extract_numbers(p.text)86 return p878889def restore_placeholders(text: str, p: Protected) -> str:90 def _r(m: re.Match) -> str:91 kind, idx = m.group(1), int(m.group(2)) - 192 pool = {"C": p.citations, "M": p.math, "U": p.urls}[kind]93 return pool[idx] if 0 <= idx < len(pool) else m.group(0)94 return PLACEHOLDER_RE.sub(_r, text)959697def placeholder_multiset(text: str) -> dict[str, int]:98 out: dict[str, int] = {}99 for m in PLACEHOLDER_RE.finditer(text):100 k = m.group(0)101 out[k] = out.get(k, 0) + 1102 return out103