SPB Git
spboucher.ai

spb/wpai0002_uqo

Public

Evaluation audits (identifiability, memorisation, Holm), MPS memory fixes (shape bucketing, cache release), Medium-stage acquisition scripts

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 9, 2026) parent c27a8da Browse files

7 changed files +487 −5

7 changed files
Unified Split
added data/manifests/discover_arxiv_q-fin-econ_2008-2017_metadata.json +242 −0
@@ -0,0 +1,242 @@
1 +{
2 + "created_at": "2026-11-09T17:57:10Z",
3 + "git": {
4 + "commit": "c27a8dad9b8875ec7e0b6eb6a79001b306b96ca4",
5 + "branch": "main",
6 + "dirty": true
7 + },
8 + "libraries": {
9 + "python": "3.14.4",
10 + "torch": "2.12.0",
11 + "numpy": "2.5.4",
12 + "anthropic": "2.3.0",
13 + "transformers": "5.8.1",
14 + "sentencepiece": "0.2.2",
15 + "lxml": "6.1.3",
16 + "pyarrow": "24.0.0",
17 + "sacrebleu": "2.6.0",
18 + "datasketch": "2.0.0",
19 + "rapidfuzz": "3.14.6"
20 + },
21 + "hardware": {
22 + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
23 + "machine": "arm64",
24 + "cpu_count": 18,
25 + "chip": "Apple M5 Max",
26 + "ram_gb": 48.0,
27 + "torch_mps": true,
28 + "torch_cuda": false
29 + },
30 + "script": "discover_arxiv",
31 + "args": {
32 + "sets": [
33 + "q-fin",
34 + "econ"
35 + ],
36 + "date_from": "2008-01-01",
37 + "date_until": "2017-12-31",
38 + "datestamp_until": null,
39 + "max_docs": 50,
40 + "seed": 1,
41 + "out": "data/raw/arxiv",
42 + "codes": "CC0,CC-BY,CC-BY-SA,CC-BY-NC,CC-BY-NC-SA",
43 + "metadata_only": true,
44 + "metadata_out": null,
45 + "report": null,
46 + "max_minutes": 25.0
47 + },
48 + "stats": {
49 + "records_seen": 25499,
50 + "duplicates": 0,
51 + "license_excluded": 18486,
52 + "no_versions": 0,
53 + "before_window": 2,
54 + "not_pre2018": 6849,
55 + "eligible": 162,
56 + "pdf_only": 0,
57 + "fetched": 0,
58 + "errors": 0,
59 + "by_license": {
60 + "UNKNOWN": 1416,
61 + "ARXIV-NONEXCLUSIVE": 15567,
62 + "CC-BY-NC-SA": 804,
63 + "CC-BY": 5676,
64 + "CC0": 220,
65 + "CC-BY-SA": 313,
66 + "CC-BY-NC-ND": 1503
67 + },
68 + "by_set": {
69 + "q-fin": 162
70 + },
71 + "by_v1_year_eligible": {
72 + "2008": 2,
73 + "2009": 7,
74 + "2010": 17,
75 + "2011": 5,
76 + "2012": 8,
77 + "2013": 21,
78 + "2014": 27,
79 + "2015": 17,
80 + "2016": 30,
81 + "2017": 28
82 + },
83 + "by_license_eligible": {
84 + "CC-BY-NC-SA": 55,
85 + "CC-BY": 96,
86 + "CC0": 9,
87 + "CC-BY-SA": 2
88 + },
89 + "windows_harvested": [
90 + {
91 + "set": "q-fin",
92 + "from": "2008-01-01",
93 + "until": "2008-12-31",
94 + "pages_total": 1,
95 + "records_seen_total": 946
96 + },
97 + {
98 + "set": "q-fin",
99 + "from": "2009-01-01",
100 + "until": "2009-12-31",
101 + "pages_total": 2,
102 + "records_seen_total": 1725
103 + },
104 + {
105 + "set": "q-fin",
106 + "from": "2010-01-01",
107 + "until": "2010-12-31",
108 + "pages_total": 3,
109 + "records_seen_total": 2129
110 + },
111 + {
112 + "set": "q-fin",
113 + "from": "2011-01-01",
114 + "until": "2011-12-31",
115 + "pages_total": 4,
116 + "records_seen_total": 2543
117 + },
118 + {
119 + "set": "q-fin",
120 + "from": "2012-01-01",
121 + "until": "2012-12-31",
122 + "pages_total": 5,
123 + "records_seen_total": 3011
124 + },
125 + {
126 + "set": "q-fin",
127 + "from": "2013-01-01",
128 + "until": "2013-12-31",
129 + "pages_total": 6,
130 + "records_seen_total": 3629
131 + },
132 + {
133 + "set": "q-fin",
134 + "from": "2014-01-01",
135 + "until": "2014-12-31",
136 + "pages_total": 7,
137 + "records_seen_total": 4302
138 + },
139 + {
140 + "set": "q-fin",
141 + "from": "2015-01-01",
142 + "until": "2015-12-31",
143 + "pages_total": 8,
144 + "records_seen_total": 5265
145 + },
146 + {
147 + "set": "q-fin",
148 + "from": "2016-01-01",
149 + "until": "2016-12-31",
150 + "pages_total": 9,
151 + "records_seen_total": 6062
152 + },
153 + {
154 + "set": "q-fin",
155 + "from": "2017-01-01",
156 + "until": "2017-12-31",
157 + "pages_total": 10,
158 + "records_seen_total": 6723
159 + },
160 + {
161 + "set": "q-fin",
162 + "from": "2018-01-01",
163 + "until": "2018-12-31",
164 + "pages_total": 11,
165 + "records_seen_total": 7644
166 + },
167 + {
168 + "set": "q-fin",
169 + "from": "2019-01-01",
170 + "until": "2019-12-31",
171 + "pages_total": 12,
172 + "records_seen_total": 8805
173 + },
174 + {
175 + "set": "q-fin",
176 + "from": "2020-01-01",
177 + "until": "2020-12-31",
178 + "pages_total": 14,
179 + "records_seen_total": 10476
180 + },
181 + {
182 + "set": "q-fin",
183 + "from": "2021-01-01",
184 + "until": "2021-12-31",
185 + "pages_total": 16,
186 + "records_seen_total": 12461
187 + },
188 + {
189 + "set": "q-fin",
190 + "from": "2022-01-01",
191 + "until": "2022-12-31",
192 + "pages_total": 18,
193 + "records_seen_total": 14229
194 + },
195 + {
196 + "set": "q-fin",
197 + "from": "2023-01-01",
198 + "until": "2023-12-31",
199 + "pages_total": 20,
200 + "records_seen_total": 16201
201 + },
202 + {
203 + "set": "q-fin",
204 + "from": "2024-01-01",
205 + "until": "2024-12-31",
206 + "pages_total": 22,
207 + "records_seen_total": 18596
208 + },
209 + {
210 + "set": "q-fin",
211 + "from": "2025-01-01",
212 + "until": "2025-12-31",
213 + "pages_total": 25,
214 + "records_seen_total": 21599
215 + }
216 + ],
217 + "truncated_at": {
218 + "set": "q-fin",
219 + "window": [
220 + "2026-01-01",
221 + "2026-11-09"
222 + ],
223 + "last_datestamp": "2026-09-15",
224 + "sets_not_started": [
225 + "econ"
226 + ]
227 + },
228 + "metadata_out": "/Users/simon-pierreboucher/Desktop/UQO/Recherche/Recherche-UQO/ia/wp-ai-0002/data/manifests/arxiv_eligible_q-fin-econ_2008-2017.jsonl",
229 + "datestamp_window": [
230 + "2008-01-01",
231 + "2026-11-09"
232 + ],
233 + "oai_endpoint": "https://oaipmh.arxiv.org/oai",
234 + "oai_pages": 29,
235 + "http": {
236 + "requests": 30,
237 + "retries": 0,
238 + "bytes": 64118631
239 + },
240 + "elapsed_s": 1506.1
241 + }
242 +}
added src/htr/evaluation/identifiability.py +66 −0
@@ -0,0 +1,66 @@
1 +"""Non-identifiability and ambiguity tests.
2 +
3 +Reconstruction from a paraphrase is a one-to-many inverse problem. We quantify this directly instead of treating
4 +low lexical overlap as model failure:
5 +
6 +* **Within-original agreement** — variant E produces two independent rewrites of the same original; a model's two
7 + reconstructions should agree with each other at least as much as either agrees with the original. We report
8 + chrF(recon_1, recon_2), chrF(recon_i, original) and the identity-baseline counterparts chrF(rewrite_1, rewrite_2).
9 +* **Rewrite-level ambiguity** — for pairs of *distinct* originals whose rewrites are lexically close (nearest
10 + neighbours in the rewrite space by MinHash/5-gram Jaccard), we measure how far apart their originals are. The
11 + distribution of d(orig_a, orig_b) given small d(rw_a, rw_b) is an empirical lower bound on the irreducible error of
12 + any deterministic inverse map, computed from data only (no model involved).
13 +* **Oracle ceiling** — the chrF between two human originals that an LLM mapped to near-identical rewrites bounds the
14 + expected chrF of a perfect-but-deterministic reconstructor on those items.
15 +"""
16 +from __future__ import annotations
17 +
18 +from collections import defaultdict
19 +
20 +import numpy as np
21 +
22 +from htr.dedup.minhash import minhash
23 +from htr.evaluation.bootstrap import bootstrap_ci
24 +from htr.evaluation.metrics import lexical
25 +
26 +
27 +def within_original_agreement(pairs: list[dict], outputs: dict[str, str] | None = None) -> dict:
28 + """Uses variant-E pairs (sample_index 0 and 1 of the same passage)."""
29 + by_passage: dict[str, dict[int, dict]] = defaultdict(dict)
30 + for p in pairs:
31 + if p["variant"] == "E":
32 + by_passage[p["passage_id"]][p["sample_index"]] = p
33 + rows = []
34 + for pid, d in by_passage.items():
35 + if 0 not in d or 1 not in d:
36 + continue
37 + a, b = d[0], d[1]
38 + rw_ab = lexical(a["source_text"], b["source_text"])["chrf"]
39 + rw_a_o = lexical(a["source_text"], a["target_text"])["chrf"]
40 + row = {"passage_id": pid, "doc_id": a["doc_id"], "chrf_rewrite1_rewrite2": rw_ab, "chrf_rewrite_original": rw_a_o}
41 + if outputs and a["pair_id"] in outputs and b["pair_id"] in outputs:
42 + r1, r2 = outputs[a["pair_id"]], outputs[b["pair_id"]]
43 + row["chrf_recon1_recon2"] = lexical(r1, r2)["chrf"]
44 + row["chrf_recon_original"] = 0.5 * (lexical(r1, a["target_text"])["chrf"] + lexical(r2, a["target_text"])["chrf"])
45 + rows.append(row)
46 + keys = [k for k in rows[0] if k.startswith("chrf")] if rows else []
47 + return {"n_passages": len(rows), **{k: bootstrap_ci([r[k] for r in rows if k in r], [r["doc_id"] for r in rows if k in r], n=1000) for k in keys}}
48 +
49 +
50 +def rewrite_ambiguity(pairs: list[dict], jaccard_threshold: float = 0.5, num_perm: int = 128, max_items: int = 5000) -> dict:
51 + """Nearest-neighbour rewrites from distinct originals: how different are their originals?"""
52 + items = [p for p in pairs if p["variant"] in ("A", "B", "E")][:max_items]
53 + mhs = [minhash(p["source_text"], num_perm, 5) for p in items]
54 + close = []
55 + for i in range(len(items)):
56 + for j in range(i + 1, len(items)):
57 + if items[i]["passage_id"] == items[j]["passage_id"]:
58 + continue
59 + jac = mhs[i].jaccard(mhs[j])
60 + if jac >= jaccard_threshold:
61 + lo = lexical(items[i]["target_text"], items[j]["target_text"])
62 + close.append({"jaccard_rewrites": float(jac), "chrf_originals": lo["chrf"], "edit_sim_originals": lo["edit_sim"], "same_doc": items[i]["doc_id"] == items[j]["doc_id"]})
63 + if not close:
64 + return {"n_close_pairs": 0, "jaccard_threshold": jaccard_threshold, "n_items": len(items)}
65 + chrf = np.array([c["chrf_originals"] for c in close])
66 + return {"n_close_pairs": len(close), "jaccard_threshold": jaccard_threshold, "n_items": len(items), "chrf_originals_mean": float(chrf.mean()), "chrf_originals_p10": float(np.quantile(chrf, 0.1)), "chrf_originals_median": float(np.median(chrf)), "frac_same_doc": float(np.mean([c["same_doc"] for c in close])), "examples": close[:5]}
added src/htr/evaluation/memorization.py +103 −0
@@ -0,0 +1,103 @@
1 +"""Memorisation and contamination audit.
2 +
3 +* **Nearest-neighbour overlap** — for each reconstruction on held-out items, the maximum 5-gram Jaccard (MinHash) and
4 + the longest common word n-gram against the TRAINING targets. A reconstruction that reproduces a training original
5 + (not its own test original) signals memorisation / regurgitation.
6 +* **Verbatim-copy rate** — fraction of outputs with normalised edit similarity ≥ 0.95 to any training target.
7 +* **Canaries (optional, lawful: synthetic text only)** — synthetic passages with unique markers inserted into the
8 + training set a fixed number of times; at test time we check whether the model emits the marker given an unrelated
9 + rewrite. Implemented as helpers; the canary set is generated locally and never contains real data.
10 +"""
11 +from __future__ import annotations
12 +
13 +import random
14 +import re
15 +
16 +from datasketch import MinHash, MinHashLSH
17 +
18 +from htr.dedup.minhash import minhash
19 +from htr.utils.hashing import normalize_for_hash
20 +
21 +_TOK = re.compile(r"\w+")
22 +
23 +
24 +def _ngrams(text: str, n: int) -> set[tuple[str, ...]]:
25 + t = _TOK.findall(normalize_for_hash(text))
26 + return {tuple(t[i : i + n]) for i in range(max(0, len(t) - n + 1))}
27 +
28 +
29 +class TrainingIndex:
30 + def __init__(self, train_targets: dict[str, str], threshold: float = 0.3, num_perm: int = 128):
31 + self.lsh = MinHashLSH(threshold=threshold, num_perm=num_perm)
32 + self.num_perm = num_perm
33 + self.targets = train_targets
34 + self.mh: dict[str, MinHash] = {}
35 + for pid, t in train_targets.items():
36 + m = minhash(t, num_perm, 5)
37 + self.mh[pid] = m
38 + self.lsh.insert(pid, m)
39 +
40 + def nearest(self, text: str) -> tuple[str | None, float]:
41 + m = minhash(text, self.num_perm, 5)
42 + best, best_j = None, 0.0
43 + for cand in self.lsh.query(m):
44 + j = float(m.jaccard(self.mh[cand]))
45 + if j > best_j:
46 + best, best_j = cand, j
47 + return best, best_j
48 +
49 +
50 +def longest_common_ngram(a: str, b: str, max_n: int = 40) -> int:
51 + ta, tb = _TOK.findall(normalize_for_hash(a)), _TOK.findall(normalize_for_hash(b))
52 + sb = set()
53 + best = 0
54 + for n in range(3, max_n + 1):
55 + grams_a = {tuple(ta[i : i + n]) for i in range(max(0, len(ta) - n + 1))}
56 + grams_b = {tuple(tb[i : i + n]) for i in range(max(0, len(tb) - n + 1))}
57 + if grams_a & grams_b:
58 + best = n
59 + else:
60 + break
61 + return best
62 +
63 +
64 +def audit_outputs(outputs: dict[str, str], eval_pairs: list[dict], train_targets: dict[str, str]) -> dict:
65 + idx = TrainingIndex(train_targets)
66 + rows = []
67 + for p in eval_pairs:
68 + out = outputs.get(p["pair_id"])
69 + if out is None:
70 + continue
71 + nn, jac = idx.nearest(out)
72 + lcn = longest_common_ngram(out, train_targets[nn]) if nn else 0
73 + rows.append({"pair_id": p["pair_id"], "nn_train_passage": nn, "jaccard_to_train": jac, "longest_common_ngram": lcn, "same_passage_leak": nn == p["passage_id"]})
74 + if not rows:
75 + return {"n": 0}
76 + jac = [r["jaccard_to_train"] for r in rows]
77 + return {
78 + "n": len(rows),
79 + "mean_max_jaccard_to_train": sum(jac) / len(jac),
80 + "frac_jaccard_ge_0_5": sum(j >= 0.5 for j in jac) / len(jac),
81 + "frac_jaccard_ge_0_8": sum(j >= 0.8 for j in jac) / len(jac),
82 + "max_longest_common_ngram": max(r["longest_common_ngram"] for r in rows),
83 + "frac_lcn_ge_12": sum(r["longest_common_ngram"] >= 12 for r in rows) / len(rows),
84 + "n_same_passage_leak": sum(r["same_passage_leak"] for r in rows), # must be 0 with correct splits
85 + "worst": sorted(rows, key=lambda r: -r["jaccard_to_train"])[:5],
86 + }
87 +
88 +
89 +def make_canaries(n: int = 5, seed: int = 0) -> list[dict]:
90 + """Synthetic canary pairs: an invented study with a unique marker token in the target only."""
91 + rng = random.Random(seed)
92 + out = []
93 + for i in range(n):
94 + marker = f"ZQX{rng.randrange(10**6):06d}"
95 + tgt = f"The specimen series labelled {marker} was examined in triplicate, and the recorded mass of 12.5 g did not vary between trials. No statistically significant difference emerged between the two preparations (p = 0.41)."
96 + src = "The specimen series was examined in triplicate; its recorded mass of 12.5 g remained constant across trials, and the two preparations did not differ significantly (p = 0.41)."
97 + out.append({"pair_id": f"canary_{i}", "passage_id": f"canary_{i}", "doc_id": f"canary_{i}", "variant": "A", "sample_index": 0, "source_text": src, "target_text": tgt, "marker": marker, "metadata": {"discipline": "other", "era": "2010-2017", "section_role": "methods", "source": "local"}})
98 + return out
99 +
100 +
101 +def canary_leak_rate(outputs: dict[str, str], canaries: list[dict]) -> float:
102 + hits = sum(1 for c in canaries if c["marker"] in outputs.get(c["pair_id"], ""))
103 + return hits / max(1, len(canaries))
modified src/htr/evaluation/runner.py +25 −4
@@ -34,11 +34,27 @@ def _agg(rows: list[dict], key: str | None, metrics: list[str]) -> dict:
34 34 groups[str(r[key]) if key else "all"].append(r)
35 35 out = {}
36 36 for g, rs in sorted(groups.items()):
37 − out[g] = {"n": len(rs), **{m: bootstrap_ci([r[m] for r in rs], [r["doc_id"] for r in rs], n=1000) for m in metrics}}
37 + out[g] = {"n": len(rs), **{m: bootstrap_ci([r[m] for r in rs], [r["doc_id"] for r in rs], n=_N_BOOT) for m in metrics}}
38 38 return out
39 39
40 40
41 −def evaluate_systems(pairs: list[dict], systems: dict[str, dict[str, str]], out_dir: Path, metrics: list[str] = PRIMARY, baseline: str = "B0_identity") -> dict:
41 +_N_BOOT = 2000
42 +
43 +
44 +def holm_correction(pvals: dict[str, float]) -> dict[str, float]:
45 + """Holm–Bonferroni step-down adjusted p-values (monotone, capped at 1)."""
46 + items = sorted(pvals.items(), key=lambda kv: kv[1])
47 + m = len(items)
48 + adj: dict[str, float] = {}
49 + running = 0.0
50 + for i, (k, p) in enumerate(items):
51 + val = min(1.0, (m - i) * p)
52 + running = max(running, val)
53 + adj[k] = running
54 + return adj
55 +
56 +
57 +def evaluate_systems(pairs: list[dict], systems: dict[str, dict[str, str]], out_dir: Path, metrics: list[str] = PRIMARY, baseline: str = "B0_identity", n_boot: int = 2000) -> dict:
42 58 out_dir.mkdir(parents=True, exist_ok=True)
43 59 report: dict = {**run_header({"n_pairs": len(pairs), "systems": list(systems)}), "systems": {}}
44 60 rows_by: dict[str, list[dict]] = {}
@@ -47,7 +63,7 @@ def evaluate_systems(pairs: list[dict], systems: dict[str, dict[str, str]], out_
47 63 rows_by[name] = rows
48 64 write_jsonl(out_dir / f"items_{name}.jsonl", rows)
49 65 report["systems"][name] = {"overall": _agg(rows, None, metrics)["all"], "by_variant": _agg(rows, "variant", metrics), "by_discipline": _agg(rows, "discipline", metrics), "by_era": _agg(rows, "era", metrics), "by_role": _agg(rows, "role", metrics)}
50 − # paired differences vs baseline
66 + # paired differences vs baseline, with Holm–Bonferroni correction over the primary metrics of each system
51 67 if baseline in rows_by:
52 68 base = {r["pair_id"]: r for r in rows_by[baseline]}
53 69 report["paired_vs_baseline"] = {}
@@ -55,7 +71,12 @@ def evaluate_systems(pairs: list[dict], systems: dict[str, dict[str, str]], out_
55 71 if name == baseline:
56 72 continue
57 73 common = [r for r in rows if r["pair_id"] in base]
58 − report["paired_vs_baseline"][name] = {m: paired_bootstrap_diff([r[m] for r in common], [base[r["pair_id"]][m] for r in common], [r["doc_id"] for r in common], n=1000) for m in metrics}
74 + diffs = {m: paired_bootstrap_diff([r[m] for r in common], [base[r["pair_id"]][m] for r in common], [r["doc_id"] for r in common], n=n_boot) for m in metrics}
75 + holm = holm_correction({m: d["p_value_two_sided"] for m, d in diffs.items()})
76 + for m in diffs:
77 + diffs[m]["p_holm"] = holm[m]
78 + diffs[m]["significant_holm_0_05"] = holm[m] < 0.05
79 + report["paired_vs_baseline"][name] = diffs
59 80 # stylometry: originals vs rewrites vs each system
60 81 originals = [p["target_text"] for p in pairs]
61 82 rewrites = [p["source_text"] for p in pairs]
modified src/htr/training/data.py +4 −0
@@ -80,8 +80,12 @@ class PairDataset(torch.utils.data.Dataset):
80 80 }
81 81
82 82
83 +PAD_MULTIPLE = 32 # pad lengths to a multiple → far fewer distinct tensor shapes (MPS/CUDA kernel caches stay small)
84 +
85 +
83 86 def _pad(seqs: list[list], pad_val, dtype=torch.long) -> torch.Tensor:
84 87 L = max(len(s) for s in seqs)
88 + L = ((L + PAD_MULTIPLE - 1) // PAD_MULTIPLE) * PAD_MULTIPLE
85 89 return torch.tensor([list(s) + [pad_val] * (L - len(s)) for s in seqs], dtype=dtype)
86 90
87 91
modified src/htr/training/trainer.py +8 −1
@@ -178,8 +178,15 @@ class Trainer:
178 178 torch.cuda.synchronize()
179 179 self.accel_seconds += time.perf_counter() - t0
180 180 self.step += 1
181 + if self.dev.type == "mps" and self.step % 20 == 0:
182 + torch.mps.empty_cache() # MPS keeps per-shape allocations; release them regularly
181 183 if self.step % self.cfg.log_every == 0:
182 − rec = {"step": self.step, "epoch": epoch, "lr": lr_at(self.step, self.cfg), "grad_norm": float(gn), "tokens_seen": self.tokens_seen, "elapsed_s": round(time.time() - t_start, 1), **logs}
184 + mem = None
185 + if self.dev.type == "mps":
186 + mem = round(torch.mps.driver_allocated_memory() / 2**30, 2)
187 + elif self.dev.type == "cuda":
188 + mem = round(torch.cuda.max_memory_allocated() / 2**30, 2)
189 + rec = {"step": self.step, "epoch": epoch, "lr": lr_at(self.step, self.cfg), "grad_norm": float(gn), "tokens_seen": self.tokens_seen, "elapsed_s": round(time.time() - t_start, 1), "mem_gb": mem, **logs}
183 190 append_jsonl(self.log_path, rec)
184 191 if not math.isfinite(logs["total"]):
185 192 raise RuntimeError(f"non-finite loss at step {self.step}: {logs}")
added tests/unit/test_eval_audits.py +39 −0
@@ -0,0 +1,39 @@
1 +from htr.evaluation.identifiability import rewrite_ambiguity, within_original_agreement
2 +from htr.evaluation.memorization import audit_outputs, canary_leak_rate, make_canaries
3 +from htr.evaluation.runner import holm_correction
4 +
5 +
6 +def _pairs():
7 + base = "the quick brown fox jumps over the lazy dog near the river bank while the sun sets slowly"
8 + rows = []
9 + for i in range(6):
10 + t = base.replace("fox", f"fox{i}")
11 + for s in (0, 1):
12 + rows.append({"pair_id": f"p{i}_{s}", "passage_id": f"g{i}", "doc_id": f"d{i}", "variant": "E", "sample_index": s, "source_text": t.replace("quick", "fast" if s else "rapid"), "target_text": t, "metadata": {}})
13 + return rows
14 +
15 +
16 +def test_within_original_agreement_and_ambiguity():
17 + pairs = _pairs()
18 + outputs = {p["pair_id"]: p["target_text"] for p in pairs} # oracle outputs
19 + r = within_original_agreement(pairs, outputs)
20 + assert r["n_passages"] == 6 and r["chrf_recon1_recon2"]["mean"] == 100.0
21 + amb = rewrite_ambiguity(pairs, jaccard_threshold=0.3)
22 + assert amb["n_close_pairs"] > 0 and amb["chrf_originals_mean"] < 100.0
23 +
24 +
25 +def test_memorization_audit_and_canaries():
26 + pairs = _pairs()
27 + train = {f"t{i}": f"completely different training sentence number {i} about economics and finance" for i in range(5)}
28 + train["leak"] = pairs[0]["target_text"]
29 + outputs = {p["pair_id"]: p["target_text"] for p in pairs}
30 + rep = audit_outputs(outputs, pairs, train)
31 + assert rep["n"] == 12 and rep["frac_jaccard_ge_0_8"] >= 1 / 12
32 + can = make_canaries(3)
33 + assert canary_leak_rate({c["pair_id"]: c["target_text"] for c in can}, can) == 1.0
34 + assert canary_leak_rate({c["pair_id"]: c["source_text"] for c in can}, can) == 0.0
35 +
36 +
37 +def test_holm():
38 + adj = holm_correction({"a": 0.01, "b": 0.04, "c": 0.03})
39 + assert adj["a"] == 0.03 and adj["c"] == 0.06 and adj["b"] == 0.06
40
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere