1"""Deterministic evaluation (Agent E) on the controller: loads the hidden oracle, runs its evaluate.py on the collected out/ inside a2composed workspace (public bundle + out/), validates the receipt against the public schema, stores it."""3from __future__ import annotations4import hashlib, importlib.util, json, os, shutil, tempfile5from .config import ORACLE_ROOT, REPO6from .bundle import scenario_dir, PUBLIC_EXCLUDES78def load_oracle(task_id: str):9 d = os.path.join(ORACLE_ROOT, task_id)10 oracle = json.load(open(os.path.join(d, "oracle.json"), encoding="utf-8"))11 spec = importlib.util.spec_from_file_location(f"oracle_{task_id.replace('-', '_')}", os.path.join(d, "evaluate.py"))12 mod = importlib.util.module_from_spec(spec); spec.loader.exec_module(mod)13 h = hashlib.sha256(open(os.path.join(d, "oracle.json"), "rb").read() + open(os.path.join(d, "evaluate.py"), "rb").read()).hexdigest()14 return oracle, mod, h1516def validate_receipt(receipt: dict) -> tuple[bool, str]:17 try:18 import jsonschema19 jsonschema.validate(receipt, json.load(open(os.path.join(REPO, "benchmark", "validators", "receipt_schema.json")))); return True, ""20 except Exception as e: # noqa: BLE00121 return False, str(e)[:300]2223def evaluate_out(task_id: str, out_dir: str, run_id: str, inputs_changed: list | None = None) -> dict:24 oracle, mod, ohash = load_oracle(task_id)25 ws = tempfile.mkdtemp(prefix=f"wp42eval_{run_id}_")26 try:27 src = scenario_dir(task_id)28 for name in os.listdir(src):29 if name in PUBLIC_EXCLUDES: continue30 p = os.path.join(src, name)31 (shutil.copytree if os.path.isdir(p) else shutil.copy2)(p, os.path.join(ws, name))32 if os.path.isdir(out_dir): shutil.copytree(out_dir, os.path.join(ws, "out"), dirs_exist_ok=True)33 else: os.makedirs(os.path.join(ws, "out"), exist_ok=True)34 from .office import recalc_tree; recalc_info = recalc_tree(os.path.join(ws, "out"))35 receipt = mod.evaluate(os.path.join(ws, "out"), oracle, ws, run_id=run_id)36 except Exception as e: # noqa: BLE00137 receipt = {"task_id": task_id, "run_id": run_id, "evaluator_version": "crashed", "required_criteria": [], "critical_violations": [],38 "det_completion": False, "det_score": 0.0, "abstention": {"status": None, "justified": None}, "notes": f"evaluator raised: {e!r}", "evaluator_crashed": True}39 finally:40 shutil.rmtree(ws, ignore_errors=True)41 if inputs_changed:42 receipt.setdefault("critical_violations", []).append({"id": "inputs_modified_harness", "description": "Input files were modified during the session (harness hash check)", "detail": ", ".join(inputs_changed[:10])})43 receipt["det_completion"] = False44 receipt["oracle_sha256"] = ohash45 try: receipt["xlsx_recalculated"] = recalc_info46 except NameError: pass47 ok, err = validate_receipt(receipt); receipt["schema_valid"] = ok; receipt["schema_error"] = err48 return receipt49