1# Author: Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca2"""Faithfulness audit of debate transcripts (docs/architecture.md §3, docs/evaluation_protocol.md §7).34Inputs: `results/runs/<run_id>/transcripts/<tx_id>.json` (DebateTranscript) and `results/runs/<run_id>/packets/<tx_id>.json`5(EvidencePacket). Every check is deterministic and operates on the stored artefacts only.67Turn level (one row per estimate/revision turn):8 cited_share_valid share of `evidence_ids_used` that exist in the packet (NaN if nothing cited)9 adj_share_within_40 share of adjustments whose adjusted price is within ±40 % of the raw comparable price10 point_in_hull point inside [min adj, max adj] × [0.75, 1.25] (NaN if no adjustments)11 quantile_sane quantiles ordered, positive, point within [q05, q95], q95/q05 within [1.01, 4]12 summary_words word count of `reasoning_summary` (contract: ≤ 120)13Committee level (one row per transcript × round):14 iqr_log, convergence (iqr_r / iqr_0), anchoring_index (corr across roles of revision r−0 with the gap to the15 round-0 median), share_toward_median.16"""17from __future__ import annotations1819import json20from pathlib import Path21from typing import Any2223import numpy as np24import pandas as pd2526from ..schemas import AgentEstimate, DebateTranscript, EvidencePacket2728ADJ_BAND = 0.4029HULL_BAND = 0.2530MAX_SUMMARY_WORDS = 120313233def _word_count(text: str) -> int:34 return len([w for w in (text or "").split() if w.strip()])353637def quantile_sanity(est: AgentEstimate) -> bool:38 qs = [est.q05, est.q25, est.q50, est.q75, est.q95]39 if any(q <= 0 for q in qs) or est.point_estimate <= 0:40 return False41 if any(b < a for a, b in zip(qs, qs[1:])):42 return False43 if not (est.q05 <= est.point_estimate <= est.q95):44 return False45 ratio = est.q95 / est.q0546 return 1.01 <= ratio <= 4.0474849def audit_estimate(est: AgentEstimate, packet: EvidencePacket | None) -> dict[str, Any]:50 """Deterministic checks of one AgentEstimate against its packet (packet may be None → id checks are NaN)."""51 ids = packet.evidence_ids() if packet is not None else None52 comp_price = {c.evidence_id: c.price for c in packet.comparables} if packet is not None else {}53 cited = list(est.evidence_ids_used)54 cited_valid = float(np.mean([c in ids for c in cited])) if (cited and ids is not None) else np.nan55 n_invalid = int(sum(c not in ids for c in cited)) if ids is not None else 056 adj = est.adjustments57 within, adj_prices, n_adj_unknown = [], [], 058 for a in adj:59 raw = comp_price.get(a.comparable_id)60 if raw is None or raw <= 0:61 n_adj_unknown += 162 continue63 within.append(abs(a.adjusted_price / raw - 1.0) <= ADJ_BAND)64 adj_prices.append(a.adjusted_price)65 adj_share = float(np.mean(within)) if within else np.nan66 if adj_prices:67 lo, hi = min(adj_prices) * (1 - HULL_BAND), max(adj_prices) * (1 + HULL_BAND)68 in_hull = bool(lo <= est.point_estimate <= hi)69 hull_dev = 0.0 if in_hull else float(min(abs(np.log(est.point_estimate / lo)), abs(np.log(est.point_estimate / hi))))70 justified = (not in_hull) and len(est.key_assumptions) > 071 else:72 in_hull, hull_dev, justified = np.nan, np.nan, np.nan73 return {74 "n_cited": len(cited), "n_cited_invalid": n_invalid, "cited_share_valid": cited_valid,75 "n_adjustments": len(adj), "n_adjustments_unknown_comp": n_adj_unknown, "adj_share_within_40": adj_share,76 "point_in_hull": in_hull, "hull_log_deviation": hull_dev, "hull_deviation_justified": justified,77 "quantile_sane": quantile_sanity(est), "summary_words": _word_count(est.reasoning_summary),78 "summary_within_limit": _word_count(est.reasoning_summary) <= MAX_SUMMARY_WORDS,79 "n_missing_data": len(est.missing_data), "n_counterarguments": len(est.counterarguments),80 }818283def audit_transcript(tr: DebateTranscript, packet: EvidencePacket | None) -> tuple[pd.DataFrame, pd.DataFrame]:84 """(turn-level table, committee-level table) for one transcript."""85 turn_rows = []86 for t in tr.turns:87 if t.estimate is None:88 continue89 row = {"tx_id": tr.tx_id, "arm": tr.arm, "committee": tr.committee, "protocol": tr.protocol, "role": t.role,90 "round": t.round, "kind": t.kind, "model": t.model, "provider": t.provider,91 "validation_retries": t.validation_retries, "error": t.error is not None}92 row.update(audit_estimate(t.estimate, packet))93 row["packet_hash_match"] = (packet.content_hash() == tr.packet_hash) if packet is not None else np.nan94 turn_rows.append(row)95 turns = pd.DataFrame(turn_rows)9697 # committee dynamics: last estimate of each role per round98 est: dict[int, dict[str, float]] = {}99 for t in tr.turns:100 if t.estimate is not None:101 est.setdefault(t.round, {})[t.role] = float(np.log(t.estimate.point_estimate))102 comm_rows = []103 if est:104 r0 = est[min(est)]105 med0 = float(np.median(list(r0.values())))106 iqr0 = float(np.subtract(*np.percentile(list(r0.values()), [75, 25]))) if len(r0) >= 2 else np.nan107 for r in sorted(est):108 vals = est[r]109 v = np.array(list(vals.values()))110 iqr = float(np.subtract(*np.percentile(v, [75, 25]))) if len(v) >= 2 else np.nan111 roles = [k for k in vals if k in r0]112 if r > min(est) and len(roles) >= 3:113 rev = np.array([vals[k] - r0[k] for k in roles])114 gap = np.array([med0 - r0[k] for k in roles])115 anchoring = float(np.corrcoef(rev, gap)[0, 1]) if rev.std() > 0 and gap.std() > 0 else np.nan116 toward = float(np.mean([abs(vals[k] - med0) < abs(r0[k] - med0) for k in roles]))117 mean_abs_rev = float(np.mean(np.abs(rev)))118 else:119 anchoring, toward, mean_abs_rev = np.nan, np.nan, (0.0 if r == min(est) else np.nan)120 comm_rows.append({"tx_id": tr.tx_id, "arm": tr.arm, "committee": tr.committee, "protocol": tr.protocol, "round": r,121 "n_roles": len(v), "median_log": float(np.median(v)), "iqr_log": iqr,122 "convergence": (iqr / iqr0) if (iqr0 and np.isfinite(iqr0) and iqr0 > 0) else np.nan,123 "anchoring_index": anchoring, "share_toward_median": toward, "mean_abs_revision": mean_abs_rev,124 "total_cost_usd": tr.total_cost_usd, "total_tokens": tr.total_tokens})125 return turns, pd.DataFrame(comm_rows)126127128def load_packet(path: Path) -> EvidencePacket | None:129 if not path.exists():130 return None131 return EvidencePacket.model_validate_json(path.read_text(encoding="utf-8"))132133134def audit_run(run_dir: Path) -> tuple[pd.DataFrame, pd.DataFrame]:135 """Audit every transcript of a run directory. Returns (turn table, committee table), both with run_id."""136 run_dir = Path(run_dir)137 tdir, pdir = run_dir / "transcripts", run_dir / "packets"138 turns, comms = [], []139 if not tdir.exists():140 return pd.DataFrame(), pd.DataFrame()141 for f in sorted(tdir.glob("*.json")):142 tr = DebateTranscript.model_validate_json(f.read_text(encoding="utf-8"))143 packet = load_packet(pdir / f"{tr.tx_id}.json")144 t, c = audit_transcript(tr, packet)145 turns.append(t)146 comms.append(c)147 turns_df = pd.concat([t for t in turns if not t.empty], ignore_index=True) if any(not t.empty for t in turns) else pd.DataFrame()148 comm_df = pd.concat([c for c in comms if not c.empty], ignore_index=True) if any(not c.empty for c in comms) else pd.DataFrame()149 for df in (turns_df, comm_df):150 if not df.empty:151 df.insert(0, "run_id", run_dir.name)152 return turns_df, comm_df153154155def summarise_faithfulness(turns: pd.DataFrame, comms: pd.DataFrame, by: list[str] | None = None) -> pd.DataFrame:156 """Aggregate the turn and committee tables into one row per (run_id, arm, committee[, round])."""157 by = by or ["run_id", "arm", "committee"]158 if turns.empty:159 return pd.DataFrame()160 agg = turns.groupby(by, dropna=False).agg(161 n_turns=("tx_id", "size"), n_properties=("tx_id", "nunique"),162 cited_share_valid=("cited_share_valid", "mean"), share_turns_all_ids_valid=("cited_share_valid", lambda s: float(np.nanmean(s >= 1.0)) if s.notna().any() else np.nan),163 adj_share_within_40=("adj_share_within_40", "mean"), share_turns_adj_ok=("adj_share_within_40", lambda s: float(np.nanmean(s >= 1.0)) if s.notna().any() else np.nan),164 point_in_hull=("point_in_hull", "mean"), hull_deviation_justified=("hull_deviation_justified", "mean"),165 quantile_sane=("quantile_sane", "mean"), summary_words=("summary_words", "mean"), summary_within_limit=("summary_within_limit", "mean"),166 validation_retries=("validation_retries", "mean"), packet_hash_match=("packet_hash_match", "mean"),167 ).reset_index()168 if not comms.empty:169 last = comms.sort_values("round").groupby(["run_id", "tx_id"], as_index=False).tail(1)170 c = last.groupby([b for b in by if b in last.columns], dropna=False).agg(171 iqr_log_final=("iqr_log", "mean"), convergence=("convergence", "mean"),172 anchoring_index=("anchoring_index", "mean"), share_toward_median=("share_toward_median", "mean"),173 ).reset_index()174 first = comms[comms["round"] == comms.groupby(["run_id", "tx_id"])["round"].transform("min")]175 c0 = first.groupby([b for b in by if b in first.columns], dropna=False).agg(iqr_log_round0=("iqr_log", "mean")).reset_index()176 agg = agg.merge(c0, how="left").merge(c, how="left")177 return agg178