Repository Public Research preview
1#!/usr/bin/env python32"""Blind human-evaluation sheets. For each sampled test item the sheet shows the LLM rewrite (context) and, in a3random order with hidden labels, the reconstructions of the compared systems plus the original. Raters score4fidelity (0–3), fluency (0–3), authenticity of pre-2018 scientific register (0–3) and factual safety (0/1 flag for5any changed number, citation, negation or claim). A Latin square assigns items to raters so that every system is6seen equally often in every position. The key file (system labels per slot) is written separately and must not be7shown to raters."""8from __future__ import annotations910import argparse11import json12import random13import sys14from pathlib import Path1516ROOT = Path(__file__).resolve().parents[1]17sys.path.insert(0, str(ROOT / "src"))18from htr.parse.protect import protect, restore_placeholders # noqa: E40219from htr.utils.io import read_jsonl, write_json # noqa: E402202122def main() -> None:23 ap = argparse.ArgumentParser()24 ap.add_argument("--pairs", required=True)25 ap.add_argument("--passages", required=True, help="corpus passages.jsonl (to restore placeholders for readability)")26 ap.add_argument("--outputs-dir", required=True)27 ap.add_argument("--systems", required=True, help="comma-separated system names (outputs_<name>.jsonl)")28 ap.add_argument("--n-items", type=int, default=40)29 ap.add_argument("--n-raters", type=int, default=3)30 ap.add_argument("--seed", type=int, default=7)31 ap.add_argument("--out", required=True)32 a = ap.parse_args()33 rng = random.Random(a.seed)34 systems = a.systems.split(",")35 outs = {s: {r["pair_id"]: r["output"] for r in read_jsonl(Path(a.outputs_dir) / f"outputs_{s}.jsonl")} for s in systems}36 passages = {p["passage_id"]: p for p in read_jsonl(a.passages)}37 pairs = [p for p in read_jsonl(a.pairs) if all(p["pair_id"] in o for o in outs.values())]38 rng.shuffle(pairs)39 items = pairs[: a.n_items]40 out_dir = Path(a.out)41 out_dir.mkdir(parents=True, exist_ok=True)42 key = []43 sheets = {r: [] for r in range(a.n_raters)}44 slots = systems + ["ORIGINAL"]45 for i, p in enumerate(items):46 psg = passages.get(p["passage_id"])47 prot = protect(psg["text"], version=p["metadata"].get("protect_version", "1.0")) if psg else None4849 def show(t: str) -> str:50 return restore_placeholders(t, prot) if prot else t5152 order = slots[:]53 rng.shuffle(order)54 texts = {s: show(outs[s][p["pair_id"]]) if s != "ORIGINAL" else show(p["target_text"]) for s in order}55 key.append({"item": i, "pair_id": p["pair_id"], "slots": {f"S{j+1}": s for j, s in enumerate(order)}})56 rater = i % a.n_raters # Latin-square style rotation; every rater sees every (position, system) combination over the sheet57 for r in range(a.n_raters):58 if (i + r) % a.n_raters == 0 or a.n_raters == 1:59 sheets[r].append({"item": i, "rewrite_context": show(p["source_text"]), "candidates": {f"S{j+1}": texts[s] for j, s in enumerate(order)}, "scores": {f"S{j+1}": {"fidelity_0_3": None, "fluency_0_3": None, "pre2018_register_0_3": None, "factual_issue_flag": None, "note": ""} for j in range(len(order))}})60 for r, rows in sheets.items():61 write_json(out_dir / f"sheet_rater{r+1}.json", {"instructions": INSTRUCTIONS, "items": rows})62 write_json(out_dir / "KEY_do_not_show_raters.json", {"systems": systems, "items": key, "seed": a.seed})63 print(f"wrote {a.n_raters} sheets with {sum(len(v) for v in sheets.values())} item-views over {len(items)} items → {out_dir}")646566INSTRUCTIONS = (67 "You see an LLM-rewritten scientific passage (context) and several candidate texts, one of which may be the original "68 "pre-2018 human passage and the others machine reconstructions. For each candidate: fidelity (0 = changes meaning, 3 = "69 "identical meaning), fluency (0 = ungrammatical, 3 = publishable), pre-2018 scientific register (0 = clearly modern LLM "70 "prose, 3 = indistinguishable from a 1990s–2010s journal paragraph), factual issue flag (1 if any number, unit, citation, "71 "negation or claim differs from the context). Do not try to guess which candidate is the original."72)737475if __name__ == "__main__":76 main()77