Repository Public Research preview
1from htr.evaluation.identifiability import rewrite_ambiguity, within_original_agreement2from htr.evaluation.memorization import audit_outputs, canary_leak_rate, make_canaries3from htr.evaluation.runner import holm_correction456def _pairs():7 base = "the quick brown fox jumps over the lazy dog near the river bank while the sun sets slowly"8 rows = []9 for i in range(6):10 t = base.replace("fox", f"fox{i}")11 for s in (0, 1):12 rows.append({"pair_id": f"p{i}_{s}", "passage_id": f"g{i}", "doc_id": f"d{i}", "variant": "E", "sample_index": s, "source_text": t.replace("quick", "fast" if s else "rapid"), "target_text": t, "metadata": {}})13 return rows141516def test_within_original_agreement_and_ambiguity():17 pairs = _pairs()18 outputs = {p["pair_id"]: p["target_text"] for p in pairs} # oracle outputs19 r = within_original_agreement(pairs, outputs)20 assert r["n_passages"] == 6 and r["chrf_recon1_recon2"]["mean"] == 100.021 amb = rewrite_ambiguity(pairs, jaccard_threshold=0.3)22 assert amb["n_close_pairs"] > 0 and amb["chrf_originals_mean"] < 100.0232425def test_memorization_audit_and_canaries():26 pairs = _pairs()27 train = {f"t{i}": f"completely different training sentence number {i} about economics and finance" for i in range(5)}28 train["leak"] = pairs[0]["target_text"]29 outputs = {p["pair_id"]: p["target_text"] for p in pairs}30 rep = audit_outputs(outputs, pairs, train)31 assert rep["n"] == 12 and rep["frac_jaccard_ge_0_8"] >= 1 / 1232 can = make_canaries(3)33 assert canary_leak_rate({c["pair_id"]: c["target_text"] for c in can}, can) == 1.034 assert canary_leak_rate({c["pair_id"]: c["source_text"] for c in can}, can) == 0.0353637def test_holm():38 adj = holm_correction({"a": 0.01, "b": 0.04, "c": 0.03})39 assert adj["a"] == 0.03 and adj["c"] == 0.06 and adj["b"] == 0.0640