1"""Precision by tier from the annotated validation sample (LLM second annotator + operator review), with Wilson CIs,2plus the UPC-based precision/recall for tiers B/C. Writes data/processed/match_validation_summary.csv."""3import numpy as np, pandas as pd4from scipy.stats import beta5from src.common import path, get_logger6log = get_logger("validation")78def wilson(k, n, z=1.96):9 if n == 0: return (np.nan, np.nan)10 p = k / n; d = 1 + z**2 / n; c = p + z**2 / (2 * n); h = z * np.sqrt(p * (1 - p) / n + z**2 / (4 * n**2))11 return ((c - h) / d, (c + h) / d)1213def main():14 s = pd.read_csv(path("processed") / "match_validation_sample.csv")15 lab = s["label_operator"].where(s["label_operator"].fillna("") != "", s["label_llm"])16 s["label_final"] = lab17 rows = []18 for t, g in s.groupby("tier"):19 n = len(g); same = (g.label_final == "SAME").sum(); diff = (g.label_final == "DIFFERENT").sum(); uns = (g.label_final == "UNSURE").sum()20 lo, hi = wilson(same, n); lo2, hi2 = wilson(same, n - uns)21 rows.append(dict(tier=t, n=n, same=same, different=diff, unsure=uns, precision_strict=same / n, ci_lo=lo, ci_hi=hi,22 precision_excl_unsure=same / max(n - uns, 1), ci2_lo=lo2, ci2_hi=hi2,23 llm_operator_agreement=(g.label_llm == g.label_final).mean()))24 out = pd.DataFrame(rows)25 cal = pd.read_csv(path("processed") / "match_threshold_calibration.csv")26 upc_b = cal[(cal.scorer == "tokens_equal")].iloc[0]; upc_c = cal[(cal.scorer == "sort_ratio") & (cal.thr == 0.90)].iloc[0]27 out["upc_precision"] = out.tier.map({"A": 1.0, "B": upc_b.precision, "C": upc_c.precision}); out["upc_recall"] = out.tier.map({"A": np.nan, "B": upc_b.recall, "C": upc_c.recall})28 out.to_csv(path("processed") / "match_validation_summary.csv", index=False)29 log.info("\n%s", out.round(3).to_string(index=False))3031if __name__ == "__main__":32 main()33