1"""Rebuild benchmark/taxonomy/registry.csv deterministically from task.json files (+ validation status from the oracle validation reports)."""2import csv, glob, json, os, re3REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")); ORACLES = os.path.expanduser("~/wp42-oracles")4rows = []5for tj in sorted(glob.glob(os.path.join(REPO, "benchmark", "scenarios", "*", "task.json"))):6 t = json.load(open(tj)); tid = t["task_id"]7 if tid.startswith("DUMMY"): continue8 n_inputs = sum(len(f) for _, _, f in os.walk(os.path.join(os.path.dirname(tj), "inputs")))9 rep = os.path.join(ORACLES, tid, "validation", "report.md"); status = "generated"10 if os.path.exists(rep):11 txt = open(rep, encoding="utf-8", errors="replace").read()[:3000].upper()12 status = "invalid" if re.search(r"VERDICT[^\n]*INVALID", txt) else ("validated_with_fixes" if "WITH FIXES" in txt else ("validated" if "VALID" in txt else "report_unclear"))13 df = t.get("designed_features", {})14 rows.append({"task_id": tid, "occupation": t["occupation"], "organisation": t["organisation"], "title": t["title"], "difficulty": t["difficulty"],15 "designed_dependency_depth": t["designed_dependency_depth"], "professional_minutes_estimate": t["professional_minutes_estimate"], "n_inputs": n_inputs,16 "deliverables": ";".join(d["path"] for d in t["required_deliverables"]), "planted_errors": df.get("planted_errors"), "prompt_injection": df.get("prompt_injection"),17 "requires_clarification": df.get("requires_clarification"), "generator_seed": t.get("generator_seed"), "time_limit_s": t.get("time_limit_s"), "budget_usd": t.get("budget_usd"), "status": status})18with open(os.path.join(REPO, "benchmark", "taxonomy", "registry.csv"), "w", newline="") as f:19 w = csv.DictWriter(f, fieldnames=list(rows[0].keys()), quoting=csv.QUOTE_MINIMAL); w.writeheader(); w.writerows(rows)20print(f"{len(rows)} scenarios"); [print(r["task_id"], r["difficulty"], r["designed_dependency_depth"], r["status"]) for r in rows]21