1"""Per-run measurement framework (mandate §11) computed from the normalised events and the receipts."""2from __future__ import annotations3import json, re45FAMILIES = [("browser", r"playwright|chromium|http://127\.0\.0\.1|curl |wget "), ("office", r"soffice|LibreOffice|textutil|cupsfilter|qlmanage"),6 ("applescript", r"osascript"), ("python", r"python|\.py\b|pip "), ("sql", r"sqlite3|duckdb|\bSELECT\b"), ("shell", r".")]78def family_of(tool: str, summary: str) -> str:9 if tool in ("Read", "Write", "Edit", "Glob", "Grep"): return "file"10 if tool == "Bash":11 for fam, pat in FAMILIES:12 if re.search(pat, summary or "", flags=re.I): return fam13 return tool or "other"1415def compute(events: list[dict], run: dict, receipt: dict | None, judgments: list[dict]) -> dict:16 tool_calls = 0; by_tool = {}; by_family = {}; fam_seq = []; errors = 0; retries = 0; last_err_tool = None17 tool_time = 0.0; pending_t = None; first_tool_t = None; last_t = 0.0; models = {}; tokens = {"input": 0, "output": 0, "cache_read": 0, "cache_create": 0}18 result = None; cc_version = None; model = None; init_tools = None19 for ev in events:20 t = ev.get("event_type"); el = float(ev.get("elapsed_s") or 0); last_t = max(last_t, el)21 if t == "system" and ev.get("usage"):22 u = json.loads(ev["usage"]) if isinstance(ev["usage"], str) else ev["usage"]23 if u.get("claude_code_version"): cc_version = u.get("claude_code_version"); model = u.get("model"); init_tools = u.get("tools") # only the init event carries the version (later system events are thinking_tokens/ui)24 elif t == "assistant":25 if ev.get("tool_name"):26 tool_calls += 1; by_tool[ev["tool_name"]] = by_tool.get(ev["tool_name"], 0) + 127 fam = family_of(ev["tool_name"], ev.get("summary")); by_family[fam] = by_family.get(fam, 0) + 128 if not fam_seq or fam_seq[-1] != fam: fam_seq.append(fam)29 pending_t = el30 if first_tool_t is None: first_tool_t = el31 if last_err_tool == ev["tool_name"]: retries += 132 last_err_tool = None33 u = json.loads(ev["usage"]) if isinstance(ev.get("usage"), str) else ev.get("usage")34 if isinstance(u, dict):35 tokens["input"] += u.get("input_tokens", 0) or 0; tokens["output"] += u.get("output_tokens", 0) or 036 tokens["cache_read"] += u.get("cache_read_input_tokens", 0) or 0; tokens["cache_create"] += u.get("cache_creation_input_tokens", 0) or 037 elif t == "user":38 if pending_t is not None: tool_time += max(0.0, el - pending_t); pending_t = None39 if ev.get("error_class") == "tool_error":40 errors += 141 m = re.match(r"tool_result:error:", ev.get("summary") or "")42 if m:43 # the tool that errored is the last tool_use44 last_err_tool = _last_tool(events, ev["seq"])45 elif t == "result":46 u = json.loads(ev["usage"]) if isinstance(ev.get("usage"), str) else ev.get("usage")47 result = u or {}48 models = (u or {}).get("modelUsage") or {}49 q = [j["score"] for j in judgments if j.get("score") is not None]50 quality = sum(q) / len(q) if q else None51 acc_votes = [json.loads(j["scores"]).get("acceptable") if isinstance(j.get("scores"), str) else None for j in judgments]52 det = bool(receipt and receipt.get("det_completion"))53 n_viol = len((receipt or {}).get("critical_violations") or [])54 cost = (run.get("cost_usd") or 0.0) + (run.get("recovery_cost_usd") or 0.0)55 return {56 "run_id": run["run_id"], "task_id": run["task_id"], "occupation": run.get("occupation"), "arm": run["arm"], "rep": run["rep"], "model": run.get("model"),57 "node": run.get("node"), "status": run.get("status"), "result_subtype": run.get("result_subtype"), "timed_out": bool(run.get("timed_out")),58 "wall_s": run.get("wall_s"), "wall_total_s": (run.get("wall_s") or 0) + (run.get("recovery_wall_s") or 0), "tool_time_s": round(tool_time, 1),59 "first_tool_s": first_tool_t, "num_turns": run.get("num_turns"), "n_events": len(events),60 "tool_calls": tool_calls, "by_tool": by_tool, "by_family": by_family, "family_sequence": fam_seq, "app_transitions": max(0, len(fam_seq) - 1),61 "tool_errors": errors, "retries": retries, "tokens": tokens, "model_usage": models, "claude_code_version": cc_version, "model_reported": model,62 "cost_usd": run.get("cost_usd"), "recovery_cost_usd": run.get("recovery_cost_usd"), "total_cost_usd": cost,63 "recovery_attempted": bool(run.get("recovery_attempted")), "recovery_success": run.get("recovery_success"),64 "selfcheck_pass_first": run.get("selfcheck_pass_first"), "selfcheck_pass_final": run.get("selfcheck_pass_final"),65 "det_completion": det, "det_score": (receipt or {}).get("det_score"), "n_critical_violations": n_viol, "critical_error": n_viol > 0,66 "abstention_status": ((receipt or {}).get("abstention") or {}).get("status"), "abstention_justified": ((receipt or {}).get("abstention") or {}).get("justified"),67 "quality": quality, "quality_scores": q, "judge_acceptable_votes": acc_votes, "n_judges": len(q),68 "interrupt_s": run.get("interrupt_s"), "human_intervention_min": 0.0, # updated from the interventions table by the exporter69 }7071def _last_tool(events, seq):72 for ev in reversed(events):73 if ev["seq"] < seq and ev.get("event_type") == "assistant" and ev.get("tool_name"): return ev["tool_name"]74 return None75