Repository Public
1"""Evaluation runner: per-item metrics for one or more systems on a set of pairs, aggregates by variant /2discipline / era / section role with document-clustered bootstrap CIs, paired differences vs the identity3baseline, stylometric distribution distances and the exploratory style probe."""4from __future__ import annotations56from collections import defaultdict7from pathlib import Path89import numpy as np1011from htr.evaluation.bootstrap import bootstrap_ci, paired_bootstrap_diff12from htr.evaluation.metrics import factual, lexical13from htr.evaluation.stylometry import FEATURES, distribution_distances, feature_matrix, style_probe14from htr.utils.io import write_json, write_jsonl15from htr.utils.provenance import run_header1617PRIMARY = ["chrf", "edit_sim", "rougeL", "exact_match", "number_recall", "placeholders_exact"]181920def score_items(pairs: list[dict], outputs: dict[str, str]) -> list[dict]:21 rows = []22 for p in pairs:23 hyp = outputs.get(p["pair_id"])24 if hyp is None:25 continue26 m = {**lexical(hyp, p["target_text"]), **factual(hyp, p["target_text"])}27 rows.append({"pair_id": p["pair_id"], "doc_id": p["doc_id"], "variant": p["variant"], "discipline": p["metadata"].get("discipline"), "era": p["metadata"].get("era"), "role": p["metadata"].get("section_role"), **m})28 return rows293031def _agg(rows: list[dict], key: str | None, metrics: list[str]) -> dict:32 groups: dict[str, list[dict]] = defaultdict(list)33 for r in rows:34 groups[str(r[key]) if key else "all"].append(r)35 out = {}36 for g, rs in sorted(groups.items()):37 # overall estimates use the full resample count (10 000 in confirmatory runs); subgroup breakdowns are38 # descriptive and capped at 2 000 resamples to keep the run tractable on 15 000+ items39 nb = _N_BOOT if key is None else min(_N_BOOT, 2000)40 out[g] = {"n": len(rs), **{m: bootstrap_ci([r[m] for r in rs], [r["doc_id"] for r in rs], n=nb) for m in metrics}}41 return out424344_N_BOOT = 2000454647def holm_correction(pvals: dict[str, float]) -> dict[str, float]:48 """Holm–Bonferroni step-down adjusted p-values (monotone, capped at 1)."""49 items = sorted(pvals.items(), key=lambda kv: kv[1])50 m = len(items)51 adj: dict[str, float] = {}52 running = 0.053 for i, (k, p) in enumerate(items):54 val = min(1.0, (m - i) * p)55 running = max(running, val)56 adj[k] = running57 return adj585960def evaluate_systems(pairs: list[dict], systems: dict[str, dict[str, str]], out_dir: Path, metrics: list[str] = PRIMARY, baseline: str = "B0_identity", n_boot: int = 2000) -> dict:61 global _N_BOOT62 _N_BOOT = n_boot63 out_dir.mkdir(parents=True, exist_ok=True)64 report: dict = {**run_header({"n_pairs": len(pairs), "systems": list(systems)}), "systems": {}}65 rows_by: dict[str, list[dict]] = {}66 for name, outputs in systems.items():67 rows = score_items(pairs, outputs)68 rows_by[name] = rows69 write_jsonl(out_dir / f"items_{name}.jsonl", rows)70 report["systems"][name] = {"overall": _agg(rows, None, metrics)["all"], "by_variant": _agg(rows, "variant", metrics), "by_discipline": _agg(rows, "discipline", metrics), "by_era": _agg(rows, "era", metrics), "by_role": _agg(rows, "role", metrics)}71 # paired differences vs baseline, with Holm–Bonferroni correction over the primary metrics of each system72 if baseline in rows_by:73 base = {r["pair_id"]: r for r in rows_by[baseline]}74 report["paired_vs_baseline"] = {}75 for name, rows in rows_by.items():76 if name == baseline:77 continue78 common = [r for r in rows if r["pair_id"] in base]79 diffs = {m: paired_bootstrap_diff([r[m] for r in common], [base[r["pair_id"]][m] for r in common], [r["doc_id"] for r in common], n=n_boot) for m in metrics}80 holm = holm_correction({m: d["p_value_two_sided"] for m, d in diffs.items()})81 for m in diffs:82 diffs[m]["p_holm"] = holm[m]83 diffs[m]["significant_holm_0_05"] = holm[m] < 0.0584 report["paired_vs_baseline"][name] = diffs85 # stylometry: originals vs rewrites vs each system86 originals = [p["target_text"] for p in pairs]87 rewrites = [p["source_text"] for p in pairs]88 X_o, X_r = feature_matrix(originals), feature_matrix(rewrites)89 X_scale = np.vstack([X_o, X_r]) # ONE common standardisation (originals ∪ rewrites) for every W1 comparison90 sty = {"rewrite_vs_original": distribution_distances(X_r, X_o, X_scale), "feature_means": {"original": dict(zip(FEATURES, X_o.mean(0).tolist())), "rewrite": dict(zip(FEATURES, X_r.mean(0).tolist()))}, "feature_sds": {"original": dict(zip(FEATURES, X_o.std(0).tolist())), "rewrite": dict(zip(FEATURES, X_r.std(0).tolist()))}}91 for name, outputs in systems.items():92 hyps = [outputs[p["pair_id"]] for p in pairs if p["pair_id"] in outputs]93 X_h = feature_matrix(hyps)94 sty[name] = {"vs_original": distribution_distances(X_h, X_o, X_scale), "feature_means": dict(zip(FEATURES, X_h.mean(0).tolist()))}95 if len(pairs) >= 40:96 sty[name]["probe"] = {k: v for k, v in style_probe(X_o, X_r, X_h).items() if k != "probe_weights"}97 report["stylometry"] = sty98 write_json(out_dir / "report.json", report)99 (out_dir / "summary.md").write_text(markdown_summary(report))100 return report101102103def markdown_summary(report: dict) -> str:104 lines = ["| system | n | " + " | ".join(PRIMARY) + " |", "|---|---|" + "---|" * len(PRIMARY)]105 for name, s in report["systems"].items():106 o = s["overall"]107 lines.append(f"| {name} | {o['n']} | " + " | ".join(f"{o[m]['mean']:.3f} [{o[m]['ci_low']:.3f}, {o[m]['ci_high']:.3f}]" for m in PRIMARY) + " |")108 lines.append("")109 lines.append("Stylometric W1 (mean over features, standardised; lower = closer to originals):")110 sty = report.get("stylometry", {})111 lines.append(f"- rewrite vs original: {sty.get('rewrite_vs_original', {}).get('W1_mean', float('nan')):.3f}")112 for name in report["systems"]:113 if name in sty:114 pr = sty[name].get("probe", {})115 lines.append(f"- {name} vs original: {sty[name]['vs_original']['W1_mean']:.3f}" + (f" · probe P(original) mean {pr.get('eval_mean_p_original', float('nan')):.2f}" if pr else ""))116 return "\n".join(lines) + "\n"117