spb/wpai0002_uqo

1#!/usr/bin/env python32"""Evaluate systems' outputs against originals → artifacts/metrics/<name>/report.json + summary.md."""3from __future__ import annotations45import argparse6import sys7from pathlib import Path89ROOT = Path(__file__).resolve().parents[1]10sys.path.insert(0, str(ROOT / "src"))11from htr.evaluation.runner import evaluate_systems  # noqa: E40212from htr.utils.io import read_jsonl  # noqa: E402131415def main() -> None:16    ap = argparse.ArgumentParser()17    ap.add_argument("--pairs", required=True)18    ap.add_argument("--outputs-dir", required=True, help="dir with outputs_<system>.jsonl files")19    ap.add_argument("--name", required=True)20    ap.add_argument("--baseline", default="B0_identity")21    ap.add_argument("--n-boot", type=int, default=2000, help="bootstrap resamples (pre-registration: 10000 for confirmatory runs)")22    ap.add_argument("--confirmatory", action="store_true", help="marks the report as a confirmatory run (requires the frozen pre-registration)")23    a = ap.parse_args()24    systems = {}25    for f in sorted(Path(a.outputs_dir).glob("outputs_*.jsonl")):26        systems[f.stem[len("outputs_"):]] = {r["pair_id"]: r["output"] for r in read_jsonl(f)}27    ids = set.intersection(*[set(v) for v in systems.values()]) if systems else set()28    pairs = [p for p in read_jsonl(a.pairs) if p["pair_id"] in ids]29    out = ROOT / "artifacts/metrics" / a.name30    rep = evaluate_systems(pairs, systems, out, baseline=a.baseline, n_boot=a.n_boot)31    if a.confirmatory:32        import json3334        prereg = (ROOT / "docs/PREREGISTRATION.md").read_text()35        rep["confirmatory"] = {"preregistration_frozen": "Commit hash: pending" not in prereg, "n_boot": a.n_boot}36        (out / "report.json").write_text(json.dumps(rep, indent=2, default=str))37    print(out.joinpath("summary.md").read_text())383940if __name__ == "__main__":41    main()42