1"""Blind professional evaluators (Agent F): LLM judges run on the controller with `claude --bare -p --json-schema`.2They receive the rubric and the deliverables rendered as text — never the arm, the model, the node, the cost or the oracle values."""3from __future__ import annotations4import json, os, subprocess, sys, tempfile5from .config import ORACLE_ROOT, REPO, SECRETS_ENV, JUDGE_MODELS6sys.path.insert(0, os.path.join(REPO, "benchmark", "validators"))7import common # noqa: E40289MAX_CHARS_PER_DOC = 240001011def render_deliverables(task_id: str, out_dir: str, rubric: dict) -> str:12 parts = []13 paths = rubric.get("deliverables_to_read") or []14 if not paths:15 for dp, _, fns in os.walk(out_dir):16 for fn in sorted(fns): paths.append(os.path.relpath(os.path.join(dp, fn), out_dir))17 for rel in paths:18 p = os.path.join(out_dir, rel)19 if not os.path.isfile(p):20 # text substitutes (arm A0): same base name, other extension21 base = os.path.splitext(rel)[0]; alts = [x for x in os.listdir(out_dir) if os.path.splitext(x)[0] == os.path.basename(base)] if os.path.isdir(out_dir) else []22 if alts: p = os.path.join(out_dir, alts[0]); rel = alts[0]23 else: parts.append(f"=== {rel} ===\n[MISSING]\n"); continue24 txt = common.any_text(p) or "[empty or unreadable]"25 if len(txt) > MAX_CHARS_PER_DOC: txt = txt[:MAX_CHARS_PER_DOC] + f"\n[... truncated, {len(txt)} chars total]"26 parts.append(f"=== {rel} ({os.path.getsize(p)} bytes) ===\n{txt}\n")27 return "\n".join(parts)2829def judge_schema(rubric: dict) -> dict:30 props = {c["id"]: {"type": "integer", "minimum": 0, "maximum": 4} for c in rubric["criteria"]}31 return {"type": "object", "required": ["scores", "overall_comment", "professionally_acceptable"],32 "properties": {"scores": {"type": "object", "required": list(props), "properties": props},33 "overall_comment": {"type": "string"}, "professionally_acceptable": {"type": "boolean"}}}3435def build_prompt(task_id: str, brief: str, rubric: dict, rendered: str) -> str:36 def desc(c): return c.get("description") or c.get("name") or c.get("title") or c["id"]37 def anchors(c):38 a = c.get("anchors") or c.get("levels") or c.get("scale") or {}39 if isinstance(a, dict): return "; ".join(f"{k}={v}" for k, v in a.items())40 if isinstance(a, list): return "; ".join(f"{i}={v}" for i, v in enumerate(a))41 return str(a)42 crit = "\n".join(f"- {c['id']} (weight {c['weight']}): {desc(c)}\n anchors: {anchors(c)}" for c in rubric["criteria"])43 instr = rubric.get("instructions") or rubric.get("instructions_to_judge") or ""44 if isinstance(instr, (list, dict)): instr = json.dumps(instr)45 return f"""You are an experienced professional asked to grade a deliverable package produced by a junior colleague for the assignment below.46Grade strictly against the rubric; each criterion gets an integer 0-4 using the anchors. Judge only what is in the deliverables.47Do not reward length; penalise unsupported numbers, missing sections and instructions that were not followed. Then say whether a demanding48manager would accept this package as professionally acceptable work (true/false) and give a 3-6 sentence overall comment.4950ASSIGNMENT BRIEF51{brief}5253RUBRIC54{instr}55{crit}5657DELIVERABLES (rendered as text; spreadsheets as cell listings)58{rendered}59"""6061def run_judge(model: str, prompt: str, schema: dict, budget=0.80) -> dict:62 env = dict(os.environ)63 for line in open(SECRETS_ENV):64 if "=" in line and not line.startswith("#"):65 k, v = line.strip().split("=", 1); env[k] = v66 with tempfile.TemporaryDirectory() as td:67 pf = os.path.join(td, "prompt.md"); open(pf, "w").write(prompt)68 cmd = ["claude", "--bare", "-p", prompt, "--model", model, "--output-format", "json", "--max-turns", "1", "--max-budget-usd", f"{budget:.2f}",69 "--tools", "", "--permission-prompts", "none", "--no-chrome", "--disable-slash-commands", "--no-session-persistence",70 "--json-schema", json.dumps(schema)]71 r = subprocess.run(cmd, capture_output=True, text=True, env=env, cwd=td, timeout=600)72 try: d = json.loads(r.stdout)73 except Exception: return {"error": r.stdout[-500:] + r.stderr[-500:], "cost": 0.0} # noqa: BLE00174 out = d.get("structured_output")75 if out is None:76 try: out = json.loads(d.get("result") or "")77 except Exception: out = None # noqa: BLE00178 return {"structured": out, "cost": d.get("total_cost_usd") or 0.0, "raw": r.stdout[-20000:], "model_usage": d.get("modelUsage")}7980def judge_run(task_id: str, out_dir: str, store, run_id: str, models=None) -> list[dict]:81 rubric = json.load(open(os.path.join(ORACLE_ROOT, task_id, "rubric.json"), encoding="utf-8"))82 brief = open(os.path.join(REPO, "benchmark", "scenarios", task_id, "BRIEF.md"), encoding="utf-8").read()83 import shutil, tempfile84 from .office import recalc_tree85 tmp_out = tempfile.mkdtemp(prefix=f"wp42judge_{run_id}_")86 if os.path.isdir(out_dir): shutil.copytree(out_dir, tmp_out, dirs_exist_ok=True)87 recalc_tree(tmp_out)88 rendered = render_deliverables(task_id, tmp_out, rubric)89 shutil.rmtree(tmp_out, ignore_errors=True)90 prompt = build_prompt(task_id, brief, rubric, rendered); schema = judge_schema(rubric)91 weights = {c["id"]: float(c["weight"]) for c in rubric["criteria"]}; tw = sum(weights.values())92 results = []93 for i, m in enumerate(models or JUDGE_MODELS):94 r = run_judge(m, prompt, schema)95 s = r.get("structured") or {}96 scores = s.get("scores") or {}97 score = (sum(weights[k] * float(scores.get(k, 0)) for k in weights) / tw) * 25.0 if scores else None98 store.set_judgment(run_id, i, m, score, {"scores": scores, "acceptable": s.get("professionally_acceptable")}, s.get("overall_comment"), r.get("cost", 0.0), r.get("raw"))99 store.ledger_add(run_id, f"judge{i}", r.get("cost", 0.0))100 results.append({"judge": i, "model": m, "score": score, "acceptable": s.get("professionally_acceptable"), "cost": r.get("cost", 0.0), "error": r.get("error")})101 return results102