1"""Final evaluation (evaluator host only): loads the hidden oracle for the task, runs its deterministic evaluate.py on the2collected deliverables, merges the harness-side reproducibility/contract result, validates the receipt against the public3schema, stores it and transitions the run to `evaluated`."""4from __future__ import annotations5import importlib.util, json, os, hashlib6from .contracts import utcnow78ORACLE_ROOT = os.path.expanduser("~/wp29-oracles")9WEIGHTS = {"correctness": 0.40, "integrity": 0.20, "reproducibility": 0.15, "reporting": 0.15, "baseline": 0.10}1011def load_oracle(task_id):12 d = os.path.join(ORACLE_ROOT, task_id)13 oracle = json.load(open(os.path.join(d, "oracle.json")))14 spec = importlib.util.spec_from_file_location(f"oracle_{task_id.replace('-', '_')}", os.path.join(d, "evaluate.py"))15 mod = importlib.util.module_from_spec(spec); spec.loader.exec_module(mod)16 return oracle, mod, hashlib.sha256(open(os.path.join(d, "oracle.json"), "rb").read()).hexdigest()1718def validate_receipt(receipt, schema_path):19 try:20 import jsonschema21 jsonschema.validate(receipt, json.load(open(schema_path))); return True, ""22 except Exception as e: # noqa: BLE00123 return False, str(e)[:300]2425def quality_score(q):26 return round(sum(WEIGHTS[k] * (q.get(k) or 0.0) for k in WEIGHTS), 4)2728def evaluate_run(repo, store, run_id, contract_receipt=None) -> dict:29 m = store.get_run(run_id)30 rd = os.path.join(repo, "runs", m["split"], run_id); out_dir = os.path.join(rd, "out")31 oracle, mod, ohash = load_oracle(m["task_id"])32 try:33 receipt = mod.evaluate(out_dir, oracle, rd, run_id=run_id)34 except Exception as e: # noqa: BLE00135 receipt = {"task_id": m["task_id"], "run_id": run_id, "evaluator_version": "crashed", "evaluated_utc": utcnow(), "required_criteria": [],36 "critical_violations": [], "scientific_success": False, "quality_components": {"correctness": 0, "integrity": 0, "reproducibility": None, "reporting": 0, "baseline": 0},37 "notes": f"evaluator raised: {e}"}38 if contract_receipt is None:39 try: contract_receipt = json.load(open(os.path.join(rd, "contract_receipt.json")))40 except Exception: contract_receipt = None # noqa: BLE00141 repro = None42 if contract_receipt:43 rc = [c for c in contract_receipt.get("checks", []) if c["id"] == "reproducibility"]44 repro = (1.0 if rc[0]["passed"] else 0.0) if rc else None45 receipt["quality_components"]["reproducibility"] = repro46 receipt["quality_score"] = quality_score(receipt["quality_components"])47 receipt["oracle_sha256"] = ohash48 ok, err = validate_receipt(receipt, os.path.join(repo, "evaluators", "public_evaluation_interface", "receipt_schema.json"))49 receipt["schema_valid"] = ok; receipt["schema_error"] = err50 txt = json.dumps(receipt, indent=1, sort_keys=True, default=str)51 receipt["receipt_sha256"] = hashlib.sha256(txt.encode()).hexdigest()52 path = os.path.join(rd, "evaluation_receipt.json"); open(path, "w").write(json.dumps(receipt, indent=1, sort_keys=True, default=str))53 if contract_receipt:54 json.dump(contract_receipt, open(os.path.join(rd, "contract_receipt.json"), "w"), indent=1)55 store.add_evaluation(run_id, receipt, contract_receipt)56 if store.get_run(run_id)["status"] != "evaluated":57 store.transition(run_id, "evaluated", note="final evaluation stored", evaluation_receipt=os.path.relpath(path, repo))58 return receipt59