Repository Public
1"""Non-identifiability and ambiguity tests.23Reconstruction from a paraphrase is a one-to-many inverse problem. We quantify this directly instead of treating4low lexical overlap as model failure:56* **Within-original agreement** — variant E produces two independent rewrites of the same original; a model's two7 reconstructions should agree with each other at least as much as either agrees with the original. We report8 chrF(recon_1, recon_2), chrF(recon_i, original) and the identity-baseline counterparts chrF(rewrite_1, rewrite_2).9* **Rewrite-level ambiguity** — for pairs of *distinct* originals whose rewrites are lexically close (nearest10 neighbours in the rewrite space by MinHash/5-gram Jaccard), we measure how far apart their originals are. The11 distribution of d(orig_a, orig_b) given small d(rw_a, rw_b) is an empirical lower bound on the irreducible error of12 any deterministic inverse map, computed from data only (no model involved).13* **Oracle ceiling** — the chrF between two human originals that an LLM mapped to near-identical rewrites bounds the14 expected chrF of a perfect-but-deterministic reconstructor on those items.15"""16from __future__ import annotations1718from collections import defaultdict1920import numpy as np2122from htr.dedup.minhash import minhash23from htr.evaluation.bootstrap import bootstrap_ci24from htr.evaluation.metrics import lexical252627def within_original_agreement(pairs: list[dict], outputs: dict[str, str] | None = None) -> dict:28 """Uses variant-E pairs (sample_index 0 and 1 of the same passage)."""29 by_passage: dict[str, dict[int, dict]] = defaultdict(dict)30 for p in pairs:31 if p["variant"] == "E":32 by_passage[p["passage_id"]][p["sample_index"]] = p33 rows = []34 for pid, d in by_passage.items():35 if 0 not in d or 1 not in d:36 continue37 a, b = d[0], d[1]38 rw_ab = lexical(a["source_text"], b["source_text"])["chrf"]39 rw_a_o = lexical(a["source_text"], a["target_text"])["chrf"]40 row = {"passage_id": pid, "doc_id": a["doc_id"], "chrf_rewrite1_rewrite2": rw_ab, "chrf_rewrite_original": rw_a_o}41 if outputs and a["pair_id"] in outputs and b["pair_id"] in outputs:42 r1, r2 = outputs[a["pair_id"]], outputs[b["pair_id"]]43 row["chrf_recon1_recon2"] = lexical(r1, r2)["chrf"]44 row["chrf_recon_original"] = 0.5 * (lexical(r1, a["target_text"])["chrf"] + lexical(r2, a["target_text"])["chrf"])45 rows.append(row)46 keys = [k for k in rows[0] if k.startswith("chrf")] if rows else []47 return {"n_passages": len(rows), **{k: bootstrap_ci([r[k] for r in rows if k in r], [r["doc_id"] for r in rows if k in r], n=1000) for k in keys}}484950def rewrite_ambiguity(pairs: list[dict], jaccard_threshold: float = 0.5, num_perm: int = 128, max_items: int = 5000) -> dict:51 """Nearest-neighbour rewrites from distinct originals: how different are their originals?"""52 items = [p for p in pairs if p["variant"] in ("A", "B", "E")][:max_items]53 mhs = [minhash(p["source_text"], num_perm, 5) for p in items]54 close = []55 for i in range(len(items)):56 for j in range(i + 1, len(items)):57 if items[i]["passage_id"] == items[j]["passage_id"]:58 continue59 jac = mhs[i].jaccard(mhs[j])60 if jac >= jaccard_threshold:61 lo = lexical(items[i]["target_text"], items[j]["target_text"])62 close.append({"jaccard_rewrites": float(jac), "chrf_originals": lo["chrf"], "edit_sim_originals": lo["edit_sim"], "same_doc": items[i]["doc_id"] == items[j]["doc_id"]})63 if not close:64 return {"n_close_pairs": 0, "jaccard_threshold": jaccard_threshold, "n_items": len(items)}65 chrf = np.array([c["chrf_originals"] for c in close])66 return {"n_close_pairs": len(close), "jaccard_threshold": jaccard_threshold, "n_items": len(items), "chrf_originals_mean": float(chrf.mean()), "chrf_originals_p10": float(np.quantile(chrf, 0.1)), "chrf_originals_median": float(np.median(chrf)), "frac_same_doc": float(np.mean([c["same_doc"] for c in close])), "examples": close[:5]}67