spb/wpai0002_uqo

1"""Paragraph quality scoring and language identification (eligibility for rewriting)."""2from __future__ import annotations34import re56from htr.parse.protect import PLACEHOLDER_RE7from htr.utils.text import split_sentences, word_count89try:10    from langdetect import DetectorFactory, detect_langs1112    DetectorFactory.seed = 013    _HAS_LD = True14except Exception:  # noqa: BLE00115    _HAS_LD = False1617_ALPHA = re.compile(r"[A-Za-z]")18_GARBAGE = re.compile(r"[^\x09\x0a\x20-\x7e -ɏͰ-Ͽ‐-‧‰-⁞€℀-⅏←-⇿∀-⋿①-⓿■-➿⟦⟧×·°µ′″]")192021def detect_language(text: str) -> tuple[str | None, float]:22    if not _HAS_LD or word_count(text) < 5:23        return None, 0.024    try:25        langs = detect_langs(text)26        return (langs[0].lang, float(langs[0].prob)) if langs else (None, 0.0)27    except Exception:  # noqa: BLE00128        return None, 0.0293031def quality_score(text: str) -> tuple[float, list[str]]:32    """Heuristic quality in [0,1] with reasons. Penalises: too short/long, low alphabetic ratio, placeholder33    density, garbage characters, broken hyphenation, missing sentence punctuation, all-caps, list-like fragments."""34    flags: list[str] = []35    n_words = word_count(text)36    if n_words == 0:37        return 0.0, ["empty"]38    chars = len(text)39    alpha_ratio = len(_ALPHA.findall(text)) / max(1, chars)40    ph = len(PLACEHOLDER_RE.findall(text)) + text.count("$")41    ph_density = ph / n_words42    garbage = len(_GARBAGE.findall(text)) / max(1, chars)43    sents = split_sentences(text)44    n_sent = len(sents)45    score = 1.046    if n_words < 40:47        score -= 0.548        flags.append("short")49    if n_words > 450:50        score -= 0.351        flags.append("long")52    if alpha_ratio < 0.6:53        score -= 0.454        flags.append("low_alpha")55    if ph_density > 0.15:56        score -= 0.357        flags.append("placeholder_dense")58    if garbage > 0.01:59        score -= 0.460        flags.append("garbage_chars")61    if n_sent < 2:62        score -= 0.363        flags.append("single_sentence")64    if not re.search(r"[.!?]\s*$", text):65        score -= 0.266        flags.append("no_terminal_punct")67    caps_words = sum(1 for w in text.split() if len(w) > 3 and w.isupper())68    if caps_words / n_words > 0.2:69        score -= 0.370        flags.append("many_caps")71    if re.search(r"\w- \w", text) and len(re.findall(r"\w- \w", text)) / n_words > 0.02:72        score -= 0.273        flags.append("broken_hyphenation")74    if re.search(r"^(Table|Figure|Fig\.)\s*\d", text):75        score -= 0.576        flags.append("caption_like")77    avg_sent_words = n_words / max(1, n_sent)78    if avg_sent_words < 6:79        score -= 0.280        flags.append("fragmentary")81    return max(0.0, min(1.0, score)), flags82