1"""Phase 5 audit material (Agent I input): failure anatomy, critical violations, tool-policy compliance (denied calls), abstentions,2portal usage, A0 deliverable substitutions, judge-vs-deterministic disagreement. Writes analysis/tables/audit_*.csv and prints a summary.3Usage: .venv/bin/python analysis/robustness/audit_pilot.py --tag pilot"""4from __future__ import annotations5import argparse, json, os, sqlite3, collections6import pandas as pd7REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")); TAB = os.path.join(REPO, "analysis", "tables")8DB = os.path.join(REPO, "data", "raw", "wp42.sqlite")910def main():11 ap = argparse.ArgumentParser(); ap.add_argument("--tag", default="pilot"); a = ap.parse_args()12 con = sqlite3.connect(DB); con.row_factory = sqlite3.Row13 runs = {r["run_id"]: dict(r) for r in con.execute("SELECT * FROM runs WHERE stage=?", (a.tag,))}14 rec = {r["run_id"]: json.loads(r["receipt"]) for r in con.execute("SELECT c.run_id, receipt FROM receipts c JOIN runs r USING(run_id) WHERE r.stage=?", (a.tag,))}15 met = {r["run_id"]: json.loads(r["metrics"]) for r in con.execute("SELECT m.run_id, metrics FROM metrics m JOIN runs r USING(run_id) WHERE r.stage=?", (a.tag,))}16 # 1. failure anatomy: failed criteria and violations per run17 rows = []18 for rid, rc in rec.items():19 failed = [c["id"] for c in rc.get("required_criteria", []) if not c["passed"]]20 viol = [v["id"] for v in rc.get("critical_violations", [])]21 m = met.get(rid, {}); q = m.get("quality")22 rows.append({"run_id": rid, "task_id": rc["task_id"], "arm": runs[rid]["arm"], "rep": runs[rid]["rep"], "det": rc["det_completion"], "det_score": rc["det_score"], "failed_criteria": ";".join(failed), "violations": ";".join(viol),23 "quality": q, "accepted": bool(rc["det_completion"] and not viol and q is not None and q >= 70), "completion_status": m.get("completion_status"), "n_questions": m.get("n_questions"), "abstention_justified": (rc.get("abstention") or {}).get("justified"), "result": runs[rid]["result_subtype"], "turns": runs[rid]["num_turns"], "cost": m.get("total_cost_usd")})24 df = pd.DataFrame(rows).sort_values(["task_id", "arm", "rep"]); df.to_csv(os.path.join(TAB, f"audit_failures_{a.tag}.csv"), index=False)25 # 2. violations by type × arm26 vt = collections.Counter()27 for r in rows:28 for v in filter(None, r["violations"].split(";")): vt[(r["arm"], v)] += 129 pd.DataFrame([{"arm": k[0], "violation": k[1], "n": n} for k, n in sorted(vt.items())]).to_csv(os.path.join(TAB, f"audit_violations_{a.tag}.csv"), index=False)30 # 3. denied tool calls (policy enforcement) and tool families per arm from events31 ev = pd.DataFrame([dict(r) for r in con.execute("SELECT e.run_id, e.event_type, e.tool_name, e.summary, e.error_class FROM events e JOIN runs r USING(run_id) WHERE r.stage=?", (a.tag,))])32 ev["arm"] = ev["run_id"].map(lambda x: runs[x]["arm"])33 denied = ev[ev["summary"].str.contains("denied", na=False) & (ev["event_type"] == "user")]34 den = denied.groupby("arm").size().rename("denied_calls").reset_index(); den.to_csv(os.path.join(TAB, f"audit_denied_{a.tag}.csv"), index=False)35 denied_examples = denied.groupby("arm")["summary"].apply(lambda s: s.str[:160].head(3).tolist())36 fam = collections.Counter()37 for rid, m in met.items():38 for f, n in (m.get("by_family") or {}).items(): fam[(runs[rid]["arm"], f)] += n39 pd.DataFrame([{"arm": k[0], "family": k[1], "calls": n} for k, n in sorted(fam.items())]).to_csv(os.path.join(TAB, f"audit_families_{a.tag}.csv"), index=False)40 # 4. portal usage (A3/A4): submissions present41 portal = []42 for rid in runs:43 p = os.path.join(REPO, "data", "raw", "runs", rid, "out", "portal_submissions.json")44 portal.append({"run_id": rid, "arm": runs[rid]["arm"], "portal_submission": os.path.exists(p)})45 pdf = pd.DataFrame(portal); pu = pdf.groupby("arm")["portal_submission"].mean().reset_index(); pu.to_csv(os.path.join(TAB, f"audit_portal_{a.tag}.csv"), index=False)46 # 5. A0 substitutions: which files did A0 produce47 a0 = []48 for rid in [r for r in runs if runs[r]["arm"] == "A0"]:49 out = os.path.join(REPO, "data", "raw", "runs", rid, "out")50 a0.append({"run_id": rid, "files": ";".join(sorted(os.listdir(out))) if os.path.isdir(out) else ""})51 pd.DataFrame(a0).to_csv(os.path.join(TAB, f"audit_a0_outputs_{a.tag}.csv"), index=False)52 # 6. det vs judges53 jd = df.copy(); jd["judge_pass"] = jd["quality"] >= 7054 ct = pd.crosstab(jd["det"], jd["judge_pass"]); ct.to_csv(os.path.join(TAB, f"audit_det_vs_judge_{a.tag}.csv"))55 print("runs", len(df), "| accepted by arm:", df.groupby("arm")["accepted"].mean().round(2).to_dict())56 print("det fail by arm:", df.groupby("arm")["det"].apply(lambda s: int((~s.astype(bool)).sum())).to_dict())57 print("violations:", dict(vt)); print("denied calls by arm:", den.to_dict("records")); print("denied examples:", denied_examples.to_dict())58 print("portal submissions by arm:", pu.to_dict("records")); print("det vs judge>=70:\n", ct)59 print("failed runs:\n", df[~df["det"].astype(bool) | (df["violations"] != "")][["run_id", "det_score", "failed_criteria", "violations", "quality", "completion_status"]].to_string())6061if __name__ == "__main__":62 main()63