1"""Run every oracle's validation/validate.py (seeded correct/wrong artefacts) and write reports/evaluator_validation.md."""2import os, sys, subprocess, glob, json, datetime, re3REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")); PY = os.path.join(REPO, ".venv", "bin", "python")4rows = []5for d in sorted(glob.glob(os.path.expanduser("~/wp29-oracles/F*"))):6 tid = os.path.basename(d); v = os.path.join(d, "validation", "validate.py")7 if not os.path.exists(v):8 rows.append((tid, "no validate.py", "", 0, 0)); continue9 r = subprocess.run([PY, v], capture_output=True, text=True, timeout=900, cwd=d)10 out = (r.stdout + r.stderr)11 if r.returncode != 0: # some validators take the repository root as argument12 r = subprocess.run([PY, v, REPO], capture_output=True, text=True, timeout=900, cwd=d); out = (r.stdout + r.stderr)13 n_cases = len(re.findall(r"(?im)^\s*(?:\[?(?:ok|pass|PASS|OK)\]?|✓|✔)", out)) or out.lower().count("ok")14 status = "PASS" if r.returncode == 0 and re.search(r"ALL (OK|PASS)|VALIDATION PASSED|all cases ok|passed", out, re.I) else ("PASS(exit0)" if r.returncode == 0 else "FAIL")15 rows.append((tid, status, out.strip().splitlines()[-1][:160] if out.strip() else "", r.returncode, len(out.splitlines())))16 print(tid, status)17lines = ["# Evaluator validation (seeded artefacts)", "", f"Generated {datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')} by `evaluators/run_validations.py`. Each hidden oracle ships a `validation/` set with ≥1 correct_out (an independently written solution whose analysis.py is re-run) and ≥2 wrong_out artefacts (planted scientific errors, fabrication, false completion or leakage). The evaluator must accept the former and reject the latter; a PASS means every seeded case received the expected verdict (false acceptance = 0 and false rejection = 0 on the seeded set). The seeded set is small and designer-authored; it bounds gross evaluator errors, not the full space of valid solutions.", "", "| task_id | result | last line of validation output |", "|---|---|---|"]18for tid, st, last, rc, n in rows: lines.append(f"| {tid} | {st} | {last.replace('|', '/')} |")19lines += ["", f"Summary: {sum(1 for r in rows if r[1].startswith('PASS'))} PASS, {sum(1 for r in rows if r[1]=='FAIL')} FAIL, {sum(1 for r in rows if 'no validate' in r[1])} without validation set, of {len(rows)} oracles."]20open(os.path.join(REPO, "reports", "evaluator_validation.md"), "w").write("\n".join(lines) + "\n")21print(lines[-1])22