Repository Public Research preview
1#!/usr/bin/env python32"""Detector-sensitivity evaluation: score originals, LLM rewrites and system outputs with the commercial Pangram3AI-text detector → artifacts/metrics/<name>/pangram_report.json + pangram_summary.md + pangram_items.jsonl.45This MEASURES one detector's behaviour. A lower AI score is not evidence of human authorship; see6docs/experiments/PANGRAM_EVAL.md. Detector scores are never fed back into training.78Placeholders (⟦C1⟧/⟦M1⟧/⟦U1⟧) are restored before scoring by re-running ``protect`` on the source passage with the9pair's ``metadata.protect_version`` (the protocol the rewrite was requested with). Placeholders that cannot be10restored (no passage, or an index absent from the passage) become a neutral numeric citation marker "[k]".1112Spend is opt-in: without ``--live`` the script only prints the plan and the estimated cost (cache-aware).13"""14from __future__ import annotations1516import argparse17import hashlib18import json19import math20import re21import sys22import time23from collections import Counter, defaultdict24from pathlib import Path2526import numpy as np2728ROOT = Path(__file__).resolve().parents[1]29sys.path.insert(0, str(ROOT / "src"))30from htr.evaluation.bootstrap import bootstrap_ci, paired_bootstrap_diff # noqa: E40231from htr.evaluation.pangram import (USD_PER_UNIT_BULK, USD_PER_UNIT_SINGLE, PangramClient, # noqa: E40232 billable_units)33from htr.parse.protect import PLACEHOLDER_RE, protect, restore_placeholders # noqa: E40234from htr.utils.io import read_jsonl, write_json, write_jsonl # noqa: E4023536PID_RE = re.compile(r'"passage_id":\s*"([^"]+)"')373839def stratified_sample(pairs: list[dict], n: int, seed: int) -> list[dict]:40 """One pair per passage, then proportional allocation over metadata.discipline (largest remainder),41 deterministic order via sha256(seed, pair_id)."""42 def h(p):43 return hashlib.sha256(f"{seed}:{p['pair_id']}".encode()).hexdigest()44 by_psg: dict[str, dict] = {}45 for p in sorted(pairs, key=h):46 by_psg.setdefault(p.get("passage_id") or p["pair_id"], p)47 pool = sorted(by_psg.values(), key=h)48 strata: dict[str, list[dict]] = defaultdict(list)49 for p in pool:50 strata[(p.get("metadata") or {}).get("discipline") or "unknown"].append(p)51 n = min(n, len(pool))52 tot = len(pool)53 quota = {k: n * len(v) / tot for k, v in strata.items()}54 alloc = {k: min(len(strata[k]), int(math.floor(q))) for k, q in quota.items()}55 rem = n - sum(alloc.values())56 for k in sorted(quota, key=lambda k: (-(quota[k] - math.floor(quota[k])), k)):57 if rem <= 0:58 break59 if alloc[k] < len(strata[k]):60 alloc[k] += 161 rem -= 162 out = [p for k in sorted(strata) for p in strata[k][:alloc[k]]]63 return sorted(out, key=h)646566def load_passages(paths: list[str], needed: set[str]) -> dict[str, dict]:67 found: dict[str, dict] = {}68 for path in paths:69 with open(path) as f:70 for line in f:71 m = PID_RE.search(line[:200])72 if m and m.group(1) in needed and m.group(1) not in found:73 found[m.group(1)] = json.loads(line)74 if len(found) == len(needed):75 break76 return found777879def restore(text: str, prot, counter: Counter) -> tuple[str, int, int]:80 """Return (restored text, n restored, n replaced by neutral marker)."""81 n_ok = n_neutral = 082 if prot is not None:83 before = len(PLACEHOLDER_RE.findall(text))84 text = restore_placeholders(text, prot)85 after = len(PLACEHOLDER_RE.findall(text))86 n_ok = before - after87 def _neutral(m):88 nonlocal n_neutral89 n_neutral += 190 counter[m.group(1)] += 191 return f"[{m.group(2)}]"92 text = PLACEHOLDER_RE.sub(_neutral, text)93 return text, n_ok, n_neutral949596def ledger_cost(path: Path, run: str) -> dict:97 """Cumulative billable submissions recorded in the ledger for this run name (all invocations, resumes included)."""98 rows = [json.loads(x) for x in open(path)] if Path(path).exists() else []99 sub = [r for r in rows if r.get("run") == run and r.get("endpoint", "").startswith("POST") and r.get("status") == "submitted"]100 return {"ledger_run_texts": sum(r["n_texts"] for r in sub), "ledger_run_units": sum(r["est_units"] for r in sub),101 "ledger_run_usd": round(sum(r["est_usd"] for r in sub), 4), "ledger_run_requests": len(sub)}102103104def summarise(rows: list[dict], conditions: list[str], systems: list[str], n_boot: int, seed: int) -> dict:105 by = {(r["pair_id"], r["condition"]): r for r in rows}106 pair_doc = {r["pair_id"]: r["doc_id"] for r in rows}107 pids = sorted(pair_doc)108 rep: dict = {"conditions": {}, "paired_differences": {}}109 for c in conditions:110 ok = [by[(p, c)] for p in pids if (p, c) in by and by[(p, c)]["label"] != "ERROR"]111 if not ok:112 rep["conditions"][c] = {"n": 0}113 continue114 ai = [r["ai_likelihood"] for r in ok]115 flag = [1.0 if r["label"] == "AI" else 0.0 for r in ok]116 nonhuman = [1.0 if r["label"] != "Human" else 0.0 for r in ok]117 g = [r["doc_id"] for r in ok]118 labels = Counter(r["label"] for r in ok)119 rep["conditions"][c] = {120 "n": len(ok), "n_docs": len(set(g)), "n_errors": sum(1 for p in pids if (p, c) in by and by[(p, c)]["label"] == "ERROR"),121 "ai_likelihood_mean": bootstrap_ci(ai, g, n_boot, seed), "ai_likelihood_median": float(np.median(ai)),122 "frac_label_ai": bootstrap_ci(flag, g, n_boot, seed), "frac_label_not_human": bootstrap_ci(nonhuman, g, n_boot, seed),123 "labels": dict(labels), "frac_score_positive": float(np.mean([x > 0 for x in ai])),124 "fraction_ai_assisted_mean": float(np.mean([r.get("fraction_ai_assisted") or 0.0 for r in ok])),125 }126 for s in systems:127 for ref in ("rewrite", "original"):128 common = [p for p in pids if (p, s) in by and (p, ref) in by and "ERROR" not in (by[(p, s)]["label"], by[(p, ref)]["label"])]129 if not common:130 continue131 g = [pair_doc[p] for p in common]132 d_ai = paired_bootstrap_diff([by[(p, s)]["ai_likelihood"] for p in common], [by[(p, ref)]["ai_likelihood"] for p in common], g, n_boot, seed)133 d_fl = paired_bootstrap_diff([float(by[(p, s)]["label"] == "AI") for p in common], [float(by[(p, ref)]["label"] == "AI") for p in common], g, n_boot, seed)134 for d in (d_ai, d_fl):135 if d["ci_low"] == d["ci_high"] == d["mean"] == 0.0:136 d["p_value_two_sided"] = None # all paired differences are 0: the bootstrap p-value is undefined137 n_changed = sum(by[(p, s)]["label"] != by[(p, ref)]["label"] for p in common)138 rep["paired_differences"][f"{s} - {ref}"] = {"n": len(common), "ai_likelihood": d_ai, "frac_label_ai": d_fl,139 "n_label_changed": n_changed,140 "label_transitions": dict(Counter(f"{by[(p, ref)]['label']}->{by[(p, s)]['label']}" for p in common))}141 return rep142143144def fmt(ci: dict, pct: bool = False) -> str:145 k = 100 if pct else 1146 d = 1 if pct else 4147 return f"{ci['mean'] * k:.{d}f} [{ci['ci_low'] * k:.{d}f}, {ci['ci_high'] * k:.{d}f}]"148149150def markdown(rep: dict) -> str:151 L = [f"# Pangram detector-sensitivity evaluation — `{rep['name']}`", "",152 f"Model `{rep['detector']['model']}` (served version(s) {', '.join(rep['detector']['served_versions']) or 'n/a'}); "153 f"pairs `{rep['inputs']['pairs']}`; outputs `{rep['inputs']['outputs_dir']}`; n = {rep['sample']['n_pairs']} pairs "154 f"({rep['sample']['n_docs']} documents), stratified by discipline, seed {rep['sample']['seed']}.", "",155 "**This measures one commercial detector's behaviour. A low AI-likelihood is not evidence of human authorship.**", "",156 "`ai_likelihood` = word-weighted mean of Pangram window `ai_assistance_score` (0 = human … 1 = AI-generated). "157 "`% AI` = share labelled `AI` by the API (`prediction_short`); `% not Human` = `AI` or `Mixed`. "158 "95 % CIs: document-clustered bootstrap.", "",159 "| Condition | n | mean ai_likelihood [95 % CI] | median | % score > 0 | % AI [95 % CI] | % not Human [95 % CI] | labels |",160 "|---|---:|---|---:|---:|---|---|---|"]161 for c, m in rep["conditions"].items():162 if not m.get("n"):163 L.append(f"| {c} | 0 | – | – | – | – | – | – |")164 continue165 L.append(f"| {c} | {m['n']} | {fmt(m['ai_likelihood_mean'])} | {m['ai_likelihood_median']:.4f} | {100 * m['frac_score_positive']:.1f} | {fmt(m['frac_label_ai'], True)} | "166 f"{fmt(m['frac_label_not_human'], True)} | {', '.join(f'{k} {v}' for k, v in sorted(m['labels'].items()))} |")167 L += ["", "## Paired differences (system − reference)", "",168 "| Contrast | n | Δ ai_likelihood [95 % CI] | p | Δ % AI [95 % CI] | p | label transitions (ref→system) |",169 "|---|---:|---|---:|---|---:|---|"]170 def pv(x):171 return "n/a (all Δ = 0)" if x is None else f"{x:.3f}"172 for k, d in rep["paired_differences"].items():173 tr = ", ".join(f"{t} {n}" for t, n in sorted(d["label_transitions"].items()))174 L.append(f"| {k} | {d['n']} | {fmt(d['ai_likelihood'])} | {pv(d['ai_likelihood']['p_value_two_sided'])} | "175 f"{fmt(d['frac_label_ai'], True)} | {pv(d['frac_label_ai']['p_value_two_sided'])} | {tr} |")176 r = rep["restoration"]177 L += ["", "## Placeholder restoration", "",178 f"Passages found {r['passages_found']}/{rep['sample']['n_pairs']}; protect() re-run reproduces target_text: "179 f"{r['protect_reproduces_target']}/{r['passages_found']}; placeholders restored {r['restored']}, replaced by a neutral "180 f"marker {r['neutral']} (by kind {r['neutral_by_kind']}); texts scored with neutral markers: {r['texts_with_neutral']}.", "",181 "## Cost", "",182 f"Whole run (ledger, all invocations): {rep['cost']['ledger_run_texts']} texts billed in {rep['cost']['ledger_run_requests']} "183 f"request(s), ~{rep['cost']['ledger_run_units']} units, ~{rep['cost']['ledger_run_usd']:.2f} USD. This invocation: "184 f"{rep['cost']['texts_submitted']} submitted, {rep['cost']['cache_hits']} cache hits. Pangram does not return the billed "185 "amount per call; estimate = started 100-word blocks × list price (bulk 0.04 USD/unit).", ""]186 return "\n".join(L)187188189def main() -> None:190 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)191 ap.add_argument("--pairs", required=True)192 ap.add_argument("--outputs-dir", required=True)193 ap.add_argument("--systems", required=True, help="comma list; outputs_<system>.jsonl must exist")194 ap.add_argument("--passages", default="data/parsed/medium_v1/passages.jsonl", help="comma list of passages.jsonl")195 ap.add_argument("--n", type=int, default=200)196 ap.add_argument("--seed", type=int, default=20261110)197 ap.add_argument("--name", required=True)198 ap.add_argument("--max-calls", type=int, default=1200, help="cap on texts submitted for billing")199 ap.add_argument("--max-usd", type=float, default=20.0, help="abort before any call if the cache-aware estimate exceeds this")200 ap.add_argument("--no-bulk", action="store_true", help="use the single-task endpoint (no 20 %% bulk discount)")201 ap.add_argument("--n-boot", type=int, default=2000)202 ap.add_argument("--live", action="store_true", help="actually call the API (otherwise plan + estimate only)")203 a = ap.parse_args()204205 systems = [s.strip() for s in a.systems.split(",") if s.strip()]206 outs = {}207 for s in systems:208 outs[s] = {r["pair_id"]: r["output"] for r in read_jsonl(Path(a.outputs_dir) / f"outputs_{s}.jsonl")}209 ids = set.intersection(*[set(v) for v in outs.values()])210 pairs = [p for p in read_jsonl(a.pairs) if p["pair_id"] in ids]211 sample = stratified_sample(pairs, a.n, a.seed)212 needed = {p["passage_id"] for p in sample if p.get("passage_id")}213 t0 = time.time()214 psg = load_passages([x.strip() for x in a.passages.split(",") if x.strip()], needed)215 print(f"[pangram_eval] {len(pairs)} eligible pairs → sample {len(sample)}; passages found {len(psg)}/{len(needed)} ({time.time() - t0:.0f}s)")216217 neutral_kind: Counter = Counter()218 restoration = {"passages_found": 0, "protect_reproduces_target": 0, "restored": 0, "neutral": 0, "texts_with_neutral": 0}219 items = [] # (pair, condition, text, n_neutral)220 for p in sample:221 md = p.get("metadata") or {}222 P = psg.get(p.get("passage_id"))223 prot = None224 if P is not None:225 restoration["passages_found"] += 1226 prot = protect(P["text"], version=md.get("protect_version", "1.0"))227 if prot.text == p["target_text"]:228 restoration["protect_reproduces_target"] += 1229 else:230 prot = None # protocol mismatch → indices would be unreliable; fall back to neutral markers231 conds = [("original", p["target_text"]), ("rewrite", p["source_text"])] + [(s, outs[s][p["pair_id"]]) for s in systems]232 for c, t in conds:233 txt, n_ok, n_neu = restore(t or "", prot, neutral_kind)234 restoration["restored"] += n_ok235 restoration["neutral"] += n_neu236 restoration["texts_with_neutral"] += int(n_neu > 0)237 items.append((p, c, txt.strip(), n_neu))238 restoration["neutral_by_kind"] = dict(neutral_kind)239 print(f"[pangram_eval] restoration {restoration}")240241 client = PangramClient(max_calls=a.max_calls, run_tag=a.name)242 distinct = list(dict.fromkeys(t for _, _, t, _ in items if t))243 uncached = [t for t in distinct if client.cached(t) is None]244 units = sum(billable_units(t) for t in uncached)245 est = units * (USD_PER_UNIT_SINGLE if a.no_bulk else USD_PER_UNIT_BULK)246 print(f"[pangram_eval] {len(items)} scored items, {len(distinct)} distinct texts, {len(uncached)} uncached, ~{units} units, ~{est:.2f} USD "247 f"({'single' if a.no_bulk else 'bulk'} price); empty texts: {sum(1 for *_, t, _ in items if not t)}")248 if len(uncached) > a.max_calls or est > a.max_usd:249 sys.exit(f"[pangram_eval] ABORT: estimate exceeds the guard (max_calls={a.max_calls}, max_usd={a.max_usd}); lower --n")250 if not a.live:251 print("[pangram_eval] dry run (pass --live to call the API)")252 return253 res = client.score_many(distinct, bulk=not a.no_bulk, progress=lambda m: print(f"[pangram_eval] {m}", flush=True))254 client.close()255256 rows = []257 for p, c, t, n_neu in items:258 r = res.get(t) or {"ai_likelihood": float("nan"), "label": "ERROR"}259 rows.append({"pair_id": p["pair_id"], "doc_id": p["doc_id"], "passage_id": p.get("passage_id"), "condition": c,260 "discipline": (p.get("metadata") or {}).get("discipline"), "n_words": len(t.split()),261 "neutral_markers": n_neu, "ai_likelihood": r["ai_likelihood"], "label": r["label"],262 "fraction_ai": r.get("fraction_ai"), "fraction_ai_assisted": r.get("fraction_ai_assisted"),263 "version": r.get("version"), "text_sha256": hashlib.sha256(t.encode()).hexdigest()})264 conditions = ["original", "rewrite"] + systems265 rep = {"name": a.name, "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),266 "detector": {"vendor": "Pangram Labs", "model": client.model, "served_versions": sorted({r["version"] for r in rows if r.get("version")}),267 "ai_likelihood_definition": "word-weighted mean of windows[].ai_assistance_score",268 "flag_definition": "prediction_short == 'AI' (API's own label)"},269 "inputs": {"pairs": a.pairs, "outputs_dir": a.outputs_dir, "systems": systems, "passages": a.passages},270 "sample": {"n_pairs": len(sample), "n_docs": len({p["doc_id"] for p in sample}), "seed": a.seed, "requested_n": a.n,271 "by_discipline": dict(Counter((p.get("metadata") or {}).get("discipline") for p in sample)),272 "mean_words": {c: float(np.mean([r["n_words"] for r in rows if r["condition"] == c])) for c in conditions}},273 "restoration": restoration, "cost": {**client.stats(), **ledger_cost(client.ledger_path, a.name)}, "n_boot": a.n_boot,274 "caveat": "Detector scores measure one commercial classifier's behaviour; they are not evidence of authorship.",275 **summarise(rows, conditions, systems, a.n_boot, a.seed)}276 out = ROOT / "artifacts/metrics" / a.name277 out.mkdir(parents=True, exist_ok=True)278 write_json(out / "pangram_report.json", rep)279 write_jsonl(out / "pangram_items.jsonl", rows)280 md = markdown(rep)281 (out / "pangram_summary.md").write_text(md)282 print(md)283284285if __name__ == "__main__":286 main()287