Repository Public Research preview
1#!/usr/bin/env python32"""RQ6 audits for one system's outputs on a split: memorisation (nearest training original), within-original agreement3across the two variant-E rewrites, and data-level rewrite ambiguity. Writes artifacts/metrics/<name>/audit_<system>.json."""4from __future__ import annotations56import argparse7import sys8from pathlib import Path910ROOT = Path(__file__).resolve().parents[1]11sys.path.insert(0, str(ROOT / "src"))12from htr.evaluation.identifiability import rewrite_ambiguity, within_original_agreement # noqa: E40213from htr.evaluation.memorization import audit_outputs # noqa: E40214from htr.utils.io import read_jsonl, write_json # noqa: E40215from htr.utils.provenance import run_header # noqa: E402161718def main() -> None:19 ap = argparse.ArgumentParser()20 ap.add_argument("--pairs", required=True)21 ap.add_argument("--splits", required=True)22 ap.add_argument("--split", default="val")23 ap.add_argument("--outputs", required=True, help="outputs_<system>.jsonl")24 ap.add_argument("--name", required=True)25 ap.add_argument("--max-train", type=int, default=60000)26 a = ap.parse_args()27 split_of = {s["doc_id"]: s["split"] for s in read_jsonl(a.splits)}28 pairs = list(read_jsonl(a.pairs))29 outputs = {r["pair_id"]: r["output"] for r in read_jsonl(a.outputs)}30 eval_pairs = [p for p in pairs if split_of.get(p["doc_id"]) == a.split and p["pair_id"] in outputs]31 train_targets: dict[str, str] = {}32 for p in pairs:33 if split_of.get(p["doc_id"]) == "train" and p["passage_id"] not in train_targets:34 train_targets[p["passage_id"]] = p["target_text"]35 if len(train_targets) >= a.max_train:36 break37 system = Path(a.outputs).stem[len("outputs_"):]38 rep = {39 **run_header({"system": system, "split": a.split, "n_eval_pairs": len(eval_pairs), "n_train_targets": len(train_targets)}),40 "memorization": audit_outputs(outputs, eval_pairs, train_targets),41 "within_original_agreement": within_original_agreement(eval_pairs, outputs),42 "rewrite_ambiguity": rewrite_ambiguity(eval_pairs, jaccard_threshold=0.5, max_items=3000),43 }44 out = ROOT / "artifacts/metrics" / a.name / f"audit_{system}.json"45 write_json(out, rep)46 m, w, amb = rep["memorization"], rep["within_original_agreement"], rep["rewrite_ambiguity"]47 print(f"{system}: memorisation mean max-Jaccard {m.get('mean_max_jaccard_to_train', 0):.3f}, frac≥0.5 {m.get('frac_jaccard_ge_0_5', 0):.3f}, max LCN {m.get('max_longest_common_ngram')}, same-passage leaks {m.get('n_same_passage_leak')}")48 print(f" agreement: recon1-recon2 chrF {w.get('chrf_recon1_recon2', {}).get('mean', float('nan')):.1f} vs rewrite1-rewrite2 {w.get('chrf_rewrite1_rewrite2', {}).get('mean', float('nan')):.1f}; recon-original {w.get('chrf_recon_original', {}).get('mean', float('nan')):.1f}")49 print(f" ambiguity: {amb.get('n_close_pairs')} close rewrite pairs (Jaccard≥0.5) among {amb.get('n_items')} items; originals chrF mean {amb.get('chrf_originals_mean', float('nan')):.1f}")505152if __name__ == "__main__":53 main()54