"""Per-run measurement framework (mandate ยง11) computed from the normalised events and the receipts.""" from __future__ import annotations import json, re FAMILIES = [("browser", r"playwright|chromium|http://127\.0\.0\.1|curl |wget "), ("office", r"soffice|LibreOffice|textutil|cupsfilter|qlmanage"), ("applescript", r"osascript"), ("python", r"python|\.py\b|pip "), ("sql", r"sqlite3|duckdb|\bSELECT\b"), ("shell", r".")] def family_of(tool: str, summary: str) -> str: if tool in ("Read", "Write", "Edit", "Glob", "Grep"): return "file" if tool == "Bash": for fam, pat in FAMILIES: if re.search(pat, summary or "", flags=re.I): return fam return tool or "other" def compute(events: list[dict], run: dict, receipt: dict | None, judgments: list[dict]) -> dict: tool_calls = 0; by_tool = {}; by_family = {}; fam_seq = []; errors = 0; retries = 0; last_err_tool = None tool_time = 0.0; pending_t = None; first_tool_t = None; last_t = 0.0; models = {}; tokens = {"input": 0, "output": 0, "cache_read": 0, "cache_create": 0} result = None; cc_version = None; model = None; init_tools = None for ev in events: t = ev.get("event_type"); el = float(ev.get("elapsed_s") or 0); last_t = max(last_t, el) if t == "system" and ev.get("usage"): u = json.loads(ev["usage"]) if isinstance(ev["usage"], str) else ev["usage"] if u.get("claude_code_version"): cc_version = u.get("claude_code_version"); model = u.get("model"); init_tools = u.get("tools") # only the init event carries the version (later system events are thinking_tokens/ui) elif t == "assistant": if ev.get("tool_name"): tool_calls += 1; by_tool[ev["tool_name"]] = by_tool.get(ev["tool_name"], 0) + 1 fam = family_of(ev["tool_name"], ev.get("summary")); by_family[fam] = by_family.get(fam, 0) + 1 if not fam_seq or fam_seq[-1] != fam: fam_seq.append(fam) pending_t = el if first_tool_t is None: first_tool_t = el if last_err_tool == ev["tool_name"]: retries += 1 last_err_tool = None u = json.loads(ev["usage"]) if isinstance(ev.get("usage"), str) else ev.get("usage") if isinstance(u, dict): tokens["input"] += u.get("input_tokens", 0) or 0; tokens["output"] += u.get("output_tokens", 0) or 0 tokens["cache_read"] += u.get("cache_read_input_tokens", 0) or 0; tokens["cache_create"] += u.get("cache_creation_input_tokens", 0) or 0 elif t == "user": if pending_t is not None: tool_time += max(0.0, el - pending_t); pending_t = None if ev.get("error_class") == "tool_error": errors += 1 m = re.match(r"tool_result:error:", ev.get("summary") or "") if m: # the tool that errored is the last tool_use last_err_tool = _last_tool(events, ev["seq"]) elif t == "result": u = json.loads(ev["usage"]) if isinstance(ev.get("usage"), str) else ev.get("usage") result = u or {} models = (u or {}).get("modelUsage") or {} q = [j["score"] for j in judgments if j.get("score") is not None] quality = sum(q) / len(q) if q else None acc_votes = [json.loads(j["scores"]).get("acceptable") if isinstance(j.get("scores"), str) else None for j in judgments] det = bool(receipt and receipt.get("det_completion")) n_viol = len((receipt or {}).get("critical_violations") or []) cost = (run.get("cost_usd") or 0.0) + (run.get("recovery_cost_usd") or 0.0) return { "run_id": run["run_id"], "task_id": run["task_id"], "occupation": run.get("occupation"), "arm": run["arm"], "rep": run["rep"], "model": run.get("model"), "node": run.get("node"), "status": run.get("status"), "result_subtype": run.get("result_subtype"), "timed_out": bool(run.get("timed_out")), "wall_s": run.get("wall_s"), "wall_total_s": (run.get("wall_s") or 0) + (run.get("recovery_wall_s") or 0), "tool_time_s": round(tool_time, 1), "first_tool_s": first_tool_t, "num_turns": run.get("num_turns"), "n_events": len(events), "tool_calls": tool_calls, "by_tool": by_tool, "by_family": by_family, "family_sequence": fam_seq, "app_transitions": max(0, len(fam_seq) - 1), "tool_errors": errors, "retries": retries, "tokens": tokens, "model_usage": models, "claude_code_version": cc_version, "model_reported": model, "cost_usd": run.get("cost_usd"), "recovery_cost_usd": run.get("recovery_cost_usd"), "total_cost_usd": cost, "recovery_attempted": bool(run.get("recovery_attempted")), "recovery_success": run.get("recovery_success"), "selfcheck_pass_first": run.get("selfcheck_pass_first"), "selfcheck_pass_final": run.get("selfcheck_pass_final"), "det_completion": det, "det_score": (receipt or {}).get("det_score"), "n_critical_violations": n_viol, "critical_error": n_viol > 0, "abstention_status": ((receipt or {}).get("abstention") or {}).get("status"), "abstention_justified": ((receipt or {}).get("abstention") or {}).get("justified"), "quality": quality, "quality_scores": q, "judge_acceptable_votes": acc_votes, "n_judges": len(q), "interrupt_s": run.get("interrupt_s"), "human_intervention_min": 0.0, # updated from the interventions table by the exporter } def _last_tool(events, seq): for ev in reversed(events): if ev["seq"] < seq and ev.get("event_type") == "assistant" and ev.get("tool_name"): return ev["tool_name"] return None