Repository Public Research preview
1"""Deterministic mock rewriter for dry runs and tests. It applies rule-based perturbations (synonym swaps,2clause reordering, connective changes, light compression) so that the whole pipeline — validation, splits,3baselines, training smoke tests — can run with zero API spend. Outputs are clearly marked as mock."""4from __future__ import annotations56import random7import re89from htr.rewrite.provider import Provider, ProviderResult, Timer10from htr.utils.text import split_sentences1112_SYN = {13 "however": "nevertheless", "moreover": "furthermore", "therefore": "consequently", "thus": "hence", "show": "demonstrate", "showed": "demonstrated",14 "shown": "demonstrated", "use": "employ", "used": "employed", "using": "employing", "large": "substantial", "small": "modest", "important": "significant",15 "found": "observed", "find": "observe", "because": "since", "but": "yet", "also": "additionally", "very": "highly", "many": "numerous", "several": "multiple",16 "increase": "rise", "decrease": "decline", "increased": "elevated", "decreased": "reduced", "suggest": "indicate", "suggests": "indicates", "suggested": "indicated",17 "in order to": "to", "due to": "owing to", "as well as": "and", "a number of": "several", "in addition": "additionally", "on the other hand": "conversely",1819 "the results": "the findings", "this study": "the present investigation", "we observed": "we noted", "was performed": "was carried out", "were performed": "were carried out",20 "in this study": "in the present work", "previous studies": "earlier investigations", "it is": "it remains", "there is": "there exists", "there are": "there exist",21 "significant": "notable", "significantly": "markedly", "compared to": "relative to", "compared with": "relative to", "associated with": "linked to", "related to": "connected with",22 "whereas": "while", "although": "though", "while": "whereas", "indicate": "point to", "indicates": "points to", "demonstrate": "establish", "demonstrated": "established",23 "obtained": "acquired", "performed": "conducted", "analysis": "examination", "analyses": "examinations", "approach": "strategy", "effect": "impact", "effects": "impacts",24 "role": "function", "level": "degree", "levels": "degrees", "high": "elevated", "low": "reduced", "higher": "greater", "lower": "smaller", "various": "diverse", "different": "distinct",25 "mainly": "primarily", "often": "frequently", "rarely": "seldom", "recent": "contemporary", "recently": "lately", "currently": "at present", "finally": "lastly", "first": "initially",26 "may": "might", "can": "is able to", "could": "might", "should": "ought to", "require": "necessitate", "requires": "necessitates", "required": "necessitated", "provide": "offer", "provides": "offers",27 "provided": "offered", "include": "comprise", "includes": "comprises", "including": "comprising", "within": "inside", "among": "amongst", "between": "among", "about": "regarding",28 "estimate": "assess", "estimated": "assessed", "measure": "quantify", "measured": "quantified", "determine": "establish", "determined": "established", "report": "describe", "reported": "described",29 "possible": "feasible", "likely": "probable", "known": "recognized", "common": "widespread", "rapid": "fast", "slow": "gradual", "early": "initial", "late": "later", "overall": "in aggregate",30 "we": "we", "our": "our", "results": "findings", "study": "investigation", "method": "approach", "methods": "approaches", "patients": "patients", "data": "data",31}32_OPENERS = ["Notably, ", "In this context, ", "Importantly, ", "Overall, ", "In particular, ", ""]333435class MockProvider(Provider):36 name = "mock"37 model = "mock-rewriter-1"3839 def __init__(self, seed: int = 0):40 self.seed = seed4142 def rewrite(self, system: str, messages: list[dict], max_tokens: int, params: dict) -> ProviderResult:43 user = messages[-1]["content"]44 m = re.search(r"<passage>\n(.*)\n</passage>", user, re.S)45 src = m.group(1) if m else user46 variant = params.get("variant", "A")47 sample = int(params.get("sample_index", 0))48 rng = random.Random(f"{self.seed}:{variant}:{sample}:{hash(src) & 0xFFFF}")49 with Timer() as t:50 out = self._perturb(src, variant, rng)51 toks_in, toks_out = len(user) // 4, len(out) // 452 return ProviderResult(out, "ok", self.model, provider_request_id=f"mock-{rng.getrandbits(32):08x}", usage={"input_tokens": toks_in, "output_tokens": toks_out}, cost_usd=0.0, stop_reason="end_turn", latency_s=t.elapsed)5354 def _perturb(self, text: str, variant: str, rng: random.Random) -> str:55 sents = split_sentences(text)56 out_s: list[str] = []57 for i, s in enumerate(sents):58 s2 = s59 for k, v in _SYN.items():60 if rng.random() < (0.98 if variant in ("B", "C", "D") else 0.85):61 s2 = re.sub(rf"\b{re.escape(k)}\b", v, s2)62 if variant in ("A", "E") and rng.random() < 0.4 and "," in s2:63 a, b = s2.split(",", 1)64 if len(a.split()) > 3 and len(b.split()) > 3 and not re.search(r"⟦|\$|\d", a):65 s2 = b.strip().rstrip(".") + ", " + a[0].lower() + a[1:] + "."66 if variant == "B" and i == 0:67 s2 = rng.choice(_OPENERS) + s268 if variant == "C":69 s2 = re.sub(r"\b(?:very|quite|rather|relatively|clearly|indeed|in fact|it is worth noting that|it should be noted that)\s+", "", s2, flags=re.I)70 out_s.append(s2)71 if variant == "C" and len(out_s) > 3:72 # merge two short sentences73 j = rng.randrange(len(out_s) - 1)74 if not re.search(r"⟦|\$", out_s[j] + out_s[j + 1]):75 out_s[j : j + 2] = [out_s[j].rstrip(".") + "; " + out_s[j + 1][0].lower() + out_s[j + 1][1:]]76 if variant in ("A", "E") and len(out_s) >= 4 and rng.random() < 0.6:77 j = rng.randrange(1, len(out_s) - 1)78 if not re.match(r"^(However|Therefore|Thus|Hence|Moreover|Furthermore|Nevertheless|Consequently|Additionally|Conversely|In contrast|This|These|Here|It|They)\b", out_s[j + 1]):79 out_s[j], out_s[j + 1] = out_s[j + 1], out_s[j]80 text2 = " ".join(out_s)81 text2 = re.sub(r"\s+", " ", text2).strip()82 text2 = text2[0].upper() + text2[1:] if text2 else text283 return text284