1"""Final evaluation (evaluator host only): loads the hidden oracle for the task, runs its deterministic evaluate.py on the2collected deliverables, merges the harness-side reproducibility/contract result, validates the receipt against the public3schema, stores it and transitions the run to `evaluated`."""4from __future__ import annotations5import importlib.util, json, os, hashlib6from .contracts import utcnow78ORACLE_ROOT = os.path.expanduser("~/wp29-oracles")9WEIGHTS = {"correctness": 0.40, "integrity": 0.20, "reproducibility": 0.15, "reporting": 0.15, "baseline": 0.10}1011def load_oracle(task_id):12 d = os.path.join(ORACLE_ROOT, task_id)13 oracle = json.load(open(os.path.join(d, "oracle.json")))14 spec = importlib.util.spec_from_file_location(f"oracle_{task_id.replace('-', '_')}", os.path.join(d, "evaluate.py"))15 mod = importlib.util.module_from_spec(spec); spec.loader.exec_module(mod)16 return oracle, mod, hashlib.sha256(open(os.path.join(d, "oracle.json"), "rb").read()).hexdigest()1718def validate_receipt(receipt, schema_path):19 try:20 import jsonschema21 jsonschema.validate(receipt, json.load(open(schema_path))); return True, ""22 except Exception as e: # noqa: BLE00123 return False, str(e)[:300]2425def quality_score(q):26 return round(sum(WEIGHTS[k] * (q.get(k) or 0.0) for k in WEIGHTS), 4)2728def evaluate_run(repo, store, run_id, contract_receipt=None) -> dict:29 m = store.get_run(run_id)30 rd = os.path.join(repo, "runs", m["split"], run_id); out_dir = os.path.join(rd, "out")31 oracle, mod, ohash = load_oracle(m["task_id"])32 # Evaluators receive a composed workspace (public bundle + the run's collected out/), never the oracle directory.33 import shutil, tempfile34 ws = tempfile.mkdtemp(prefix=f"wp29eval_{run_id}_")35 try:36 shutil.copytree(os.path.join(repo, "tasks", "public_bundles", m["task_id"]), ws, dirs_exist_ok=True)37 if os.path.isdir(out_dir):38 shutil.copytree(out_dir, os.path.join(ws, "out"), dirs_exist_ok=True)39 receipt = mod.evaluate(os.path.join(ws, "out"), oracle, ws, run_id=run_id)40 except Exception as e: # noqa: BLE00141 receipt = {"task_id": m["task_id"], "run_id": run_id, "evaluator_version": "crashed", "evaluated_utc": utcnow(), "required_criteria": [],42 "critical_violations": [], "scientific_success": False, "quality_components": {"correctness": 0, "integrity": 0, "reproducibility": None, "reporting": 0, "baseline": 0},43 "notes": f"evaluator raised: {e}"}44 finally:45 shutil.rmtree(ws, ignore_errors=True)46 if contract_receipt is None:47 try: contract_receipt = json.load(open(os.path.join(rd, "contract_receipt.json")))48 except Exception: contract_receipt = None # noqa: BLE00149 repro = None50 if contract_receipt:51 rc = [c for c in contract_receipt.get("checks", []) if c["id"] == "reproducibility"]52 repro = (1.0 if rc[0]["passed"] else 0.0) if rc else None53 receipt["quality_components"]["reproducibility"] = repro54 receipt["quality_score"] = quality_score(receipt["quality_components"])55 receipt["oracle_sha256"] = ohash56 ok, err = validate_receipt(receipt, os.path.join(repo, "evaluators", "public_evaluation_interface", "receipt_schema.json"))57 receipt["schema_valid"] = ok; receipt["schema_error"] = err58 txt = json.dumps(receipt, indent=1, sort_keys=True, default=str)59 receipt["receipt_sha256"] = hashlib.sha256(txt.encode()).hexdigest()60 path = os.path.join(rd, "evaluation_receipt.json"); open(path, "w").write(json.dumps(receipt, indent=1, sort_keys=True, default=str))61 if contract_receipt:62 json.dump(contract_receipt, open(os.path.join(rd, "contract_receipt.json"), "w"), indent=1)63 store.add_evaluation(run_id, receipt, contract_receipt)64 if store.get_run(run_id)["status"] != "evaluated":65 store.transition(run_id, "evaluated", note="final evaluation stored", evaluation_receipt=os.path.relpath(path, repo))66 return receipt67