"""Rebuild benchmark/taxonomy/registry.csv deterministically from task.json files (+ validation status from the oracle validation reports).""" import csv, glob, json, os, re REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")); ORACLES = os.path.expanduser("~/wp42-oracles") rows = [] for tj in sorted(glob.glob(os.path.join(REPO, "benchmark", "scenarios", "*", "task.json"))): t = json.load(open(tj)); tid = t["task_id"] if tid.startswith("DUMMY"): continue n_inputs = sum(len(f) for _, _, f in os.walk(os.path.join(os.path.dirname(tj), "inputs"))) rep = os.path.join(ORACLES, tid, "validation", "report.md"); status = "generated" if os.path.exists(rep): txt = open(rep, encoding="utf-8", errors="replace").read()[:3000].upper() status = "invalid" if re.search(r"VERDICT[^\n]*INVALID", txt) else ("validated_with_fixes" if "WITH FIXES" in txt else ("validated" if "VALID" in txt else "report_unclear")) df = t.get("designed_features", {}) rows.append({"task_id": tid, "occupation": t["occupation"], "organisation": t["organisation"], "title": t["title"], "difficulty": t["difficulty"], "designed_dependency_depth": t["designed_dependency_depth"], "professional_minutes_estimate": t["professional_minutes_estimate"], "n_inputs": n_inputs, "deliverables": ";".join(d["path"] for d in t["required_deliverables"]), "planted_errors": df.get("planted_errors"), "prompt_injection": df.get("prompt_injection"), "requires_clarification": df.get("requires_clarification"), "generator_seed": t.get("generator_seed"), "time_limit_s": t.get("time_limit_s"), "budget_usd": t.get("budget_usd"), "status": status}) with open(os.path.join(REPO, "benchmark", "taxonomy", "registry.csv"), "w", newline="") as f: w = csv.DictWriter(f, fieldnames=list(rows[0].keys()), quoting=csv.QUOTE_MINIMAL); w.writeheader(); w.writerows(rows) print(f"{len(rows)} scenarios"); [print(r["task_id"], r["difficulty"], r["designed_dependency_depth"], r["status"]) for r in rows]