1"""Blind professional evaluators (Agent F): LLM judges run on the controller with `claude --bare -p --json-schema`.2They receive the rubric and the deliverables rendered as text — never the arm, the model, the node, the cost or the oracle values."""3from __future__ import annotations4import json, os, subprocess, sys, tempfile5from .config import ORACLE_ROOT, REPO, SECRETS_ENV, JUDGE_MODELS6sys.path.insert(0, os.path.join(REPO, "benchmark", "validators"))7import common # noqa: E40289MAX_CHARS_PER_DOC = 240001011def render_deliverables(task_id: str, out_dir: str, rubric: dict) -> str:12 parts = []13 paths = rubric.get("deliverables_to_read") or []14 if not paths:15 for dp, _, fns in os.walk(out_dir):16 for fn in sorted(fns): paths.append(os.path.relpath(os.path.join(dp, fn), out_dir))17 for rel in paths:18 p = os.path.join(out_dir, rel)19 if not os.path.isfile(p):20 # text substitutes (arm A0): same base name, other extension21 base = os.path.splitext(rel)[0]; alts = [x for x in os.listdir(out_dir) if os.path.splitext(x)[0] == os.path.basename(base)] if os.path.isdir(out_dir) else []22 if alts: p = os.path.join(out_dir, alts[0]); rel = alts[0]23 else: parts.append(f"=== {rel} ===\n[MISSING]\n"); continue24 txt = (common.any_text(p) or "[empty or unreadable]").replace("\x00", "")25 txt = "".join(ch for ch in txt if ch == "\n" or ch == "\t" or ord(ch) >= 32) # argv-safe: no NUL/control bytes26 if len(txt) > MAX_CHARS_PER_DOC: txt = txt[:MAX_CHARS_PER_DOC] + f"\n[... truncated, {len(txt)} chars total]"27 parts.append(f"=== {rel} ({os.path.getsize(p)} bytes) ===\n{txt}\n")28 return "\n".join(parts)2930def judge_schema(rubric: dict) -> dict:31 props = {c["id"]: {"type": "integer", "minimum": 0, "maximum": 4} for c in rubric["criteria"]}32 return {"type": "object", "required": ["scores", "overall_comment", "professionally_acceptable"],33 "properties": {"scores": {"type": "object", "required": list(props), "properties": props},34 "overall_comment": {"type": "string"}, "professionally_acceptable": {"type": "boolean"}}}3536def build_prompt(task_id: str, brief: str, rubric: dict, rendered: str) -> str:37 def desc(c): return c.get("description") or c.get("name") or c.get("title") or c["id"]38 def anchors(c):39 a = c.get("anchors") or c.get("levels") or c.get("scale") or {}40 if isinstance(a, dict): return "; ".join(f"{k}={v}" for k, v in a.items())41 if isinstance(a, list): return "; ".join(f"{i}={v}" for i, v in enumerate(a))42 return str(a)43 crit = "\n".join(f"- {c['id']} (weight {c['weight']}): {desc(c)}\n anchors: {anchors(c)}" for c in rubric["criteria"])44 instr = rubric.get("instructions") or rubric.get("instructions_to_judge") or ""45 if isinstance(instr, (list, dict)): instr = json.dumps(instr)46 return f"""You are an experienced professional asked to grade a deliverable package produced by a junior colleague for the assignment below.47Grade strictly against the rubric; each criterion gets an integer 0-4 using the anchors. Judge only what is in the deliverables.48Do not reward length; penalise unsupported numbers, missing sections and instructions that were not followed. Then say whether a demanding49manager would accept this package as professionally acceptable work (true/false) and give a 3-6 sentence overall comment.5051ASSIGNMENT BRIEF52{brief}5354RUBRIC55{instr}56{crit}5758DELIVERABLES (rendered as text; spreadsheets as cell listings)59{rendered}60"""6162def run_judge(model: str, prompt: str, schema: dict, budget=0.80) -> dict:63 env = dict(os.environ)64 for line in open(SECRETS_ENV):65 if "=" in line and not line.startswith("#"):66 k, v = line.strip().split("=", 1); env[k] = v67 prompt = prompt.replace("\x00", "")68 if len(prompt) > 400_000: prompt = prompt[:400_000] + "\n[... prompt truncated by the harness at 400,000 characters]"69 with tempfile.TemporaryDirectory() as td:70 pf = os.path.join(td, "prompt.md"); open(pf, "w").write(prompt)71 cmd = ["claude", "--bare", "-p", prompt, "--model", model, "--output-format", "json", "--max-turns", "1", "--max-budget-usd", f"{budget:.2f}",72 "--tools", "", "--permission-prompts", "none", "--no-chrome", "--disable-slash-commands", "--no-session-persistence",73 "--json-schema", json.dumps(schema)]74 try:75 r = subprocess.run(cmd, capture_output=True, text=True, env=env, cwd=td, timeout=900)76 except subprocess.TimeoutExpired:77 return {"error": "judge timeout (900 s)", "cost": 0.0}78 except Exception as e: # noqa: BLE00179 return {"error": f"judge launch failed: {e!r}", "cost": 0.0}80 try: d = json.loads(r.stdout)81 except Exception: return {"error": r.stdout[-500:] + r.stderr[-500:], "cost": 0.0} # noqa: BLE00182 out = d.get("structured_output")83 if out is None:84 try: out = json.loads(d.get("result") or "")85 except Exception: out = None # noqa: BLE00186 return {"structured": out, "cost": d.get("total_cost_usd") or 0.0, "raw": r.stdout[-20000:], "model_usage": d.get("modelUsage")}8788def judge_run(task_id: str, out_dir: str, store, run_id: str, models=None) -> list[dict]:89 rubric = json.load(open(os.path.join(ORACLE_ROOT, task_id, "rubric.json"), encoding="utf-8"))90 brief = open(os.path.join(REPO, "benchmark", "scenarios", task_id, "BRIEF.md"), encoding="utf-8").read()91 import shutil, tempfile92 from .office import recalc_tree93 tmp_out = tempfile.mkdtemp(prefix=f"wp42judge_{run_id}_")94 if os.path.isdir(out_dir): shutil.copytree(out_dir, tmp_out, dirs_exist_ok=True)95 recalc_tree(tmp_out)96 rendered = render_deliverables(task_id, tmp_out, rubric)97 shutil.rmtree(tmp_out, ignore_errors=True)98 prompt = build_prompt(task_id, brief, rubric, rendered); schema = judge_schema(rubric)99 weights = {c["id"]: float(c["weight"]) for c in rubric["criteria"]}; tw = sum(weights.values())100 results = []101 for i, m in enumerate(models or JUDGE_MODELS):102 r = run_judge(m, prompt, schema)103 s = r.get("structured") or {}104 scores = s.get("scores") or {}105 score = (sum(weights[k] * float(scores.get(k, 0)) for k in weights) / tw) * 25.0 if scores else None106 store.set_judgment(run_id, i, m, score, {"scores": scores, "acceptable": s.get("professionally_acceptable")}, s.get("overall_comment"), r.get("cost", 0.0), r.get("raw"))107 store.ledger_add(run_id, f"judge{i}", r.get("cost", 0.0))108 results.append({"judge": i, "model": m, "score": score, "acceptable": s.get("professionally_acceptable"), "cost": r.get("cost", 0.0), "error": r.get("error")})109 return results110