Repository Public Research preview
1"""Automated pair-integrity checks (original ↔ rewrite). Each check returns (passed, score, reason).2Pairs that fail are quarantined, never silently dropped; acceptance rates are reported per variant."""3from __future__ import annotations45import re6from collections import Counter7from dataclasses import dataclass89from rapidfuzz.distance import Levenshtein1011from htr.data.schema import PairValidation12from htr.parse.protect import HEDGE_RE, NEGATION_RE, PLACEHOLDER_RE, extract_numbers, placeholder_multiset13from htr.parse.quality import detect_language14from htr.utils.hashing import normalize_for_hash15from htr.utils.text import split_sentences, word_count1617VALIDATOR_VERSION = "val-1.0"181920@dataclass21class ValidationConfig:22 min_len_ratio: float = 0.523 max_len_ratio: float = 1.824 concise_min_len_ratio: float = 0.425 max_norm_edit_similarity: float = 0.95 # above → near-verbatim copy (rewrite failed to rewrite)26 min_norm_edit_similarity: float = 0.05 # below → unrelated text27 min_semantic_sim: float = 0.7028 max_negation_delta: int = 229 max_hedge_delta: int = 430 require_language: str = "en"31 entity_min_recall: float = 0.632 use_embeddings: bool = True333435_EMB = None363738def _embed_sim(a: str, b: str) -> float | None:39 """Cosine similarity with a small sentence-embedding model if available locally (never downloads)."""40 global _EMB41 if _EMB is False:42 return None43 if _EMB is None:44 try:45 import os4647 os.environ.setdefault("HF_HUB_OFFLINE", "1")48 from sentence_transformers import SentenceTransformer4950 _EMB = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2", device="cpu")51 except Exception: # noqa: BLE00152 _EMB = False53 return None54 import numpy as np5556 v = _EMB.encode([a, b], normalize_embeddings=True)57 return float(np.dot(v[0], v[1]))585960def _norm_token(w: str) -> str:61 w = w.replace("’", "'").replace("‘", "'")62 w = w.strip("()[],.;:\"'“”‘’")63 w = re.sub(r"'s$", "", w) # possessive64 w = re.sub(r"(?<=[A-Za-z])\.?\d{1,2}$", "", w) # trailing footnote markers (extraction.3)65 return w666768def _entities(text: str) -> set[str]:69 """Proxy for named entities / technical terms: capitalised tokens not at sentence start, acronyms, tokens70 with digits+letters (e.g. CD4, p53, IL-6). Tokens are normalised (apostrophes, possessives, footnote marks);71 hyphenated compounds are kept whole and also split so that 'Bt-maize' ≈ 'Bt maize'."""72 ents: set[str] = set()73 for sent in split_sentences(PLACEHOLDER_RE.sub(" ", text)):74 toks = sent.split()75 for i, t in enumerate(toks):76 w = _norm_token(t)77 if len(w) < 2:78 continue79 parts = [w] + ([p for p in w.split("-") if len(p) >= 2] if "-" in w else [])80 for j, p in enumerate(parts):81 if (p.isupper() and len(p) >= 2 and p.isalpha()) or (any(c.isdigit() for c in p) and any(c.isalpha() for c in p) and not re.fullmatch(r"\d+(?:st|nd|rd|th|s)", p)):82 ents.add(p)83 elif i > 0 and j == 0 and p[0].isupper() and p[1:].islower() and len(p) > 2:84 ents.add(p)85 return ents868788def _entity_recall(src: str, rw: str) -> tuple[float, list[str]]:89 """An entity counts as preserved if its normalised form occurs (case-insensitively, hyphen/space-insensitively)90 anywhere in the rewrite."""91 e_src = _entities(src)92 if not e_src:93 return 1.0, []94 hay = re.sub(r"[\s\-]+", " ", rw.replace("’", "'")).lower()95 missing = [e for e in e_src if re.sub(r"[\s\-]+", " ", e).lower() not in hay]96 return 1 - len(missing) / len(e_src), sorted(missing)979899def validate_pair(original: str, rewrite: str | None, variant: str, cfg: ValidationConfig | None = None, original_protected: str | None = None) -> PairValidation:100 """``original_protected`` is the placeholder form sent to the LLM (so placeholder multisets are comparable)."""101 cfg = cfg or ValidationConfig()102 checks: dict[str, bool] = {}103 scores: dict[str, float] = {}104 reasons: list[str] = []105 if not rewrite or not rewrite.strip():106 return PairValidation(passed=False, checks={"non_empty": False}, scores={}, reasons=["empty rewrite"], validator_version=VALIDATOR_VERSION)107 checks["non_empty"] = True108 src = original_protected or original109 # length ratio110 lr = word_count(rewrite) / max(1, word_count(src))111 scores["len_ratio"] = round(lr, 3)112 lo = cfg.concise_min_len_ratio if variant == "C" else cfg.min_len_ratio113 checks["length_ratio"] = lo <= lr <= cfg.max_len_ratio114 if not checks["length_ratio"]:115 reasons.append(f"length ratio {lr:.2f}")116 # near-copy / unrelated117 sim = Levenshtein.normalized_similarity(normalize_for_hash(src), normalize_for_hash(rewrite))118 scores["norm_edit_similarity"] = round(sim, 4)119 checks["not_verbatim_copy"] = sim <= cfg.max_norm_edit_similarity120 checks["related"] = sim >= cfg.min_norm_edit_similarity121 if not checks["not_verbatim_copy"]:122 reasons.append("near-verbatim copy")123 if not checks["related"]:124 reasons.append("unrelated text")125 # language126 lang, prob = detect_language(rewrite)127 scores["lang_prob"] = round(prob, 3)128 checks["language"] = lang == cfg.require_language129 if not checks["language"]:130 reasons.append(f"language {lang}")131 # placeholders (citations / math / urls)132 pm_src, pm_rw = placeholder_multiset(src), placeholder_multiset(rewrite)133 checks["placeholders_preserved"] = pm_src == pm_rw134 if not checks["placeholders_preserved"]:135 missing = sum((Counter(pm_src) - Counter(pm_rw)).values())136 extra = sum((Counter(pm_rw) - Counter(pm_src)).values())137 reasons.append(f"placeholders missing={missing} extra={extra}")138 scores["placeholders_src"] = float(sum(pm_src.values()))139 # numbers140 n_src, n_rw = Counter(extract_numbers(PLACEHOLDER_RE.sub(" ", src))), Counter(extract_numbers(PLACEHOLDER_RE.sub(" ", rewrite)))141 missing_n = sum((n_src - n_rw).values())142 extra_n = sum((n_rw - n_src).values())143 scores["numbers_src"] = float(sum(n_src.values()))144 scores["numbers_missing"] = float(missing_n)145 scores["numbers_extra"] = float(extra_n)146 # Tolerate spelled-out small integers? No: numerals must match exactly (strict). Extra numbers tolerated if ≤1 (e.g. "70%" target in C is not in text).147 checks["numbers_preserved"] = missing_n == 0 and extra_n <= 1148 if not checks["numbers_preserved"]:149 reasons.append(f"numbers missing={missing_n} extra={extra_n}")150 # entities151 rec, missing_ents = _entity_recall(src, rewrite)152 scores["entity_recall"] = round(rec, 3)153 checks["entities_preserved"] = rec >= cfg.entity_min_recall154 if not checks["entities_preserved"]:155 reasons.append(f"entity recall {rec:.2f}; missing {missing_ents[:5]}")156 # negation / hedging157 neg_d = len(NEGATION_RE.findall(rewrite)) - len(NEGATION_RE.findall(src))158 hed_d = len(HEDGE_RE.findall(rewrite)) - len(HEDGE_RE.findall(src))159 scores["negation_delta"] = float(neg_d)160 scores["hedge_delta"] = float(hed_d)161 checks["negation_stable"] = abs(neg_d) <= cfg.max_negation_delta162 checks["hedging_stable"] = abs(hed_d) <= cfg.max_hedge_delta163 if not checks["negation_stable"]:164 reasons.append(f"negation delta {neg_d}")165 if not checks["hedging_stable"]:166 reasons.append(f"hedge delta {hed_d}")167 # structure (variant D: same sentence count ±1)168 ns, nr = len(split_sentences(src)), len(split_sentences(rewrite))169 scores["sent_src"], scores["sent_rw"] = float(ns), float(nr)170 if variant == "D":171 checks["structure_preserved"] = abs(ns - nr) <= 1172 if not checks["structure_preserved"]:173 reasons.append(f"sentence count {ns}→{nr}")174 # formatting artefacts175 bad = any(s in rewrite for s in ("<passage>", "</passage>", "Here is", "here is the", "Rewritten passage:", "\n- ", "\n* ", "**")) or bool(re.search(r"(?<!\w)\*[^*\n]{1,80}\*(?!\w)|(?<!\w)_[A-Za-z][^_\n]{0,60}_(?!\w)|^#+\s", rewrite, re.M))176 checks["no_artifacts"] = not bad177 if bad:178 reasons.append("formatting artefact / meta text")179 # semantic similarity (optional)180 if cfg.use_embeddings:181 s = _embed_sim(src, rewrite)182 if s is not None:183 scores["semantic_sim"] = round(s, 4)184 checks["semantic_similarity"] = s >= cfg.min_semantic_sim185 if not checks["semantic_similarity"]:186 reasons.append(f"semantic sim {s:.2f}")187 passed = all(checks.values())188 return PairValidation(passed=passed, checks=checks, scores=scores, reasons=reasons, validator_version=VALIDATOR_VERSION)189