Repository Public Research preview
1#!/usr/bin/env python32"""H4: document-clustered bootstrap of the stylometric distance difference W1(rewrite, original) − W1(system, original),3with ONE common standardisation (A1). Reports the relative reduction and its 95 % CI (pre-registration §2, H4 SESOI ≥ 10 %)."""4from __future__ import annotations56import argparse7import json8import sys9from pathlib import Path1011import numpy as np1213ROOT = Path(__file__).resolve().parents[1]14sys.path.insert(0, str(ROOT / "src"))15from htr.evaluation.stylometry import distribution_distances, feature_matrix # noqa: E40216from htr.utils.io import read_jsonl, write_json # noqa: E40217from htr.utils.provenance import run_header # noqa: E402181920def main() -> None:21 ap = argparse.ArgumentParser()22 ap.add_argument("--pairs", required=True)23 ap.add_argument("--outputs", required=True, help="outputs_<system>.jsonl")24 ap.add_argument("--name", required=True)25 ap.add_argument("--n-boot", type=int, default=2000)26 ap.add_argument("--seed", type=int, default=0)27 a = ap.parse_args()28 outs = {r["pair_id"]: r["output"] for r in read_jsonl(a.outputs)}29 pairs = [p for p in read_jsonl(a.pairs) if p["pair_id"] in outs]30 X_o = feature_matrix([p["target_text"] for p in pairs])31 X_r = feature_matrix([p["source_text"] for p in pairs])32 X_h = feature_matrix([outs[p["pair_id"]] for p in pairs])33 scale = np.vstack([X_o, X_r])34 docs = np.array([p["doc_id"] for p in pairs])35 uniq = np.unique(docs)36 idx_by = {u: np.where(docs == u)[0] for u in uniq}3738 def stat(sel):39 w_r = distribution_distances(X_r[sel], X_o[sel], scale)["W1_mean"]40 w_h = distribution_distances(X_h[sel], X_o[sel], scale)["W1_mean"]41 return w_r, w_h, (w_r - w_h) / w_r4243 w_r, w_h, rel = stat(np.arange(len(pairs)))44 rng = np.random.default_rng(a.seed)45 rels, diffs = [], []46 for _ in range(a.n_boot):47 pick = rng.choice(uniq, size=len(uniq), replace=True)48 sel = np.concatenate([idx_by[u] for u in pick])49 wr, wh, r = stat(sel)50 rels.append(r)51 diffs.append(wr - wh)52 rep = {**run_header({"name": a.name, "outputs": a.outputs, "n_pairs": len(pairs), "n_docs": int(len(uniq)), "n_boot": a.n_boot}),53 "W1_rewrite_vs_original": w_r, "W1_system_vs_original": w_h, "relative_reduction": rel,54 "relative_reduction_ci95": [float(np.quantile(rels, 0.025)), float(np.quantile(rels, 0.975))],55 "absolute_diff_ci95": [float(np.quantile(diffs, 0.025)), float(np.quantile(diffs, 0.975))],56 "H4_supported_SESOI_10pct": bool(np.quantile(rels, 0.025) > 0 and rel >= 0.10)}57 out = ROOT / "artifacts/metrics" / a.name / f"h4_w1_{Path(a.outputs).stem[len('outputs_'):]}.json"58 write_json(out, rep)59 print(f"{a.name}: W1 rewrite {w_r:.4f} → system {w_h:.4f}; relative reduction {rel:.1%} CI [{rep['relative_reduction_ci95'][0]:.1%}, {rep['relative_reduction_ci95'][1]:.1%}]; H4 {'SUPPORTED' if rep['H4_supported_SESOI_10pct'] else 'not supported'}")606162if __name__ == "__main__":63 main()64