"""Deterministic evaluation (Agent E) on the controller: loads the hidden oracle, runs its evaluate.py on the collected out/ inside a composed workspace (public bundle + out/), validates the receipt against the public schema, stores it.""" from __future__ import annotations import hashlib, importlib.util, json, os, shutil, tempfile from .config import ORACLE_ROOT, REPO from .bundle import scenario_dir, PUBLIC_EXCLUDES def load_oracle(task_id: str): d = os.path.join(ORACLE_ROOT, task_id) oracle = json.load(open(os.path.join(d, "oracle.json"), encoding="utf-8")) spec = importlib.util.spec_from_file_location(f"oracle_{task_id.replace('-', '_')}", os.path.join(d, "evaluate.py")) mod = importlib.util.module_from_spec(spec); spec.loader.exec_module(mod) h = hashlib.sha256(open(os.path.join(d, "oracle.json"), "rb").read() + open(os.path.join(d, "evaluate.py"), "rb").read()).hexdigest() return oracle, mod, h def validate_receipt(receipt: dict) -> tuple[bool, str]: try: import jsonschema jsonschema.validate(receipt, json.load(open(os.path.join(REPO, "benchmark", "validators", "receipt_schema.json")))); return True, "" except Exception as e: # noqa: BLE001 return False, str(e)[:300] def evaluate_out(task_id: str, out_dir: str, run_id: str, inputs_changed: list | None = None) -> dict: oracle, mod, ohash = load_oracle(task_id) ws = tempfile.mkdtemp(prefix=f"wp42eval_{run_id}_") try: src = scenario_dir(task_id) for name in os.listdir(src): if name in PUBLIC_EXCLUDES: continue p = os.path.join(src, name) (shutil.copytree if os.path.isdir(p) else shutil.copy2)(p, os.path.join(ws, name)) if os.path.isdir(out_dir): shutil.copytree(out_dir, os.path.join(ws, "out"), dirs_exist_ok=True) else: os.makedirs(os.path.join(ws, "out"), exist_ok=True) from .office import recalc_tree; recalc_info = recalc_tree(os.path.join(ws, "out")) receipt = mod.evaluate(os.path.join(ws, "out"), oracle, ws, run_id=run_id) except Exception as e: # noqa: BLE001 receipt = {"task_id": task_id, "run_id": run_id, "evaluator_version": "crashed", "required_criteria": [], "critical_violations": [], "det_completion": False, "det_score": 0.0, "abstention": {"status": None, "justified": None}, "notes": f"evaluator raised: {e!r}", "evaluator_crashed": True} finally: shutil.rmtree(ws, ignore_errors=True) if inputs_changed: receipt.setdefault("critical_violations", []).append({"id": "inputs_modified_harness", "description": "Input files were modified during the session (harness hash check)", "detail": ", ".join(inputs_changed[:10])}) receipt["det_completion"] = False receipt["oracle_sha256"] = ohash try: receipt["xlsx_recalculated"] = recalc_info except NameError: pass ok, err = validate_receipt(receipt); receipt["schema_valid"] = ok; receipt["schema_error"] = err return receipt