Repository Public
1"""Text normalisation and lightweight sentence splitting."""2from __future__ import annotations34import re5import unicodedata67_WS = re.compile(r"[ \t ]+")8_ABBR = {"e.g", "i.e", "et al", "cf", "vs", "fig", "figs", "eq", "eqs", "ref", "refs", "no", "nos", "approx", "ca", "dr", "prof", "mr", "mrs", "ms", "sec", "sect", "vol", "pp", "p", "chap", "resp", "inc", "ltd", "al", "st", "jr", "sr", "u.s", "u.k", "a.m", "p.m"}91011def normalize_text(s: str) -> str:12 s = unicodedata.normalize("NFKC", s)13 s = s.replace("\r\n", "\n").replace("\r", "\n")14 s = _WS.sub(" ", s)15 s = re.sub(r" *\n *", "\n", s)16 s = re.sub(r"\n{3,}", "\n\n", s)17 s = re.sub(r" ([,.;:!?)\]])", r"\1", s)18 s = re.sub(r"([(\[]) ", r"\1", s)19 return s.strip()202122def word_count(s: str) -> int:23 return len(re.findall(r"\S+", s))242526_SENT_END = re.compile(r"([.!?])(\s+)(?=[\"'“”‘’(\[]?[A-Z0-9])")272829def split_sentences(text: str) -> list[str]:30 """Conservative sentence splitter: splits on ., !, ? followed by whitespace and a capital/digit,31 unless the preceding token is a known abbreviation, a single initial, or a decimal number."""32 text = text.strip()33 if not text:34 return []35 out: list[str] = []36 start = 037 for m in _SENT_END.finditer(text):38 end = m.end(1)39 prev = text[start:end].rstrip(".!?")40 last_tok = prev.split()[-1].lower() if prev.split() else ""41 last_tok_clean = last_tok.strip("([\"'“”")42 if last_tok_clean in _ABBR or re.fullmatch(r"[a-z]", last_tok_clean) or re.fullmatch(r"[A-Z]\.?", prev.split()[-1] if prev.split() else "") or (m.group(1) == "." and re.search(r"\d$", prev) and re.match(r"\s+\d", text[end:end + 3] or "")):43 continue44 out.append(text[start:end].strip())45 start = m.end(2)46 tail = text[start:].strip()47 if tail:48 out.append(tail)49 return [s for s in out if s]50