1"""Build datasets, splits, manifests. Agent-visible documents go to data/generated/docs/, answer keys and hidden tests2to data/generated/keys/ (never mounted into prompts by the harness; checked by tests/test_leakage.py)."""3from __future__ import annotations4import json, yaml, hashlib5from pathlib import Path6from ..util import ROOT, write_json, read_json, sha256_text, utc_now7from .synthetic import generate_document8from .exec_tasks import generate_exec_task9from .public import load_subset1011GEN = ROOT / "data" / "generated"12DOCS, KEYS, MAN = GEN / "docs", GEN / "keys", ROOT / "data" / "manifests"1314def _write_item(item: dict, family: str, split: str, extra_key: dict | None = None):15 did = item["document_id"]16 doc_pub = {"document_id": did, "family": family, "split": split, "text": item["text"], "char_len": item["char_len"],17 "document_hash": sha256_text(item["text"]),18 "questions": [{k: q[k] for k in ("question_id", "qtype", "question")} for q in item["questions"]]}19 key = {"document_id": did, "family": family, "split": split, "document_hash": doc_pub["document_hash"],20 "questions": item["questions"], "facts": item.get("facts", [])}21 if extra_key:22 key.update(extra_key)23 (DOCS / family).mkdir(parents=True, exist_ok=True); (KEYS / family).mkdir(parents=True, exist_ok=True)24 write_json(DOCS / family / f"{did}.json", doc_pub)25 write_json(KEYS / family / f"{did}.json", key)26 return {"document_id": did, "family": family, "split": split, "document_hash": doc_pub["document_hash"], "char_len": item["char_len"],27 "n_questions": len(item["questions"]), "cluster_id": did}2829def prepare(config_path: str) -> int:30 cfg = yaml.safe_load(open(config_path))31 manifest = {"created_utc": utc_now(), "config": cfg, "config_hash": sha256_text(json.dumps(cfg, sort_keys=True)), "items": []}32 # synthetic33 sc = cfg["synthetic"]; L = sc["lengths_tokens"]34 for split, r in sc["splits"].items():35 for i in range(r["count"]):36 seed = r["start"] + i; tl = L[i % len(L)]37 d = generate_document(seed, sc["n_projects"], tl)38 manifest["items"].append(_write_item(d, "synthetic", split) | {"target_tokens": tl, "seed": seed})39 ec = cfg["exec"]40 for split, r in ec["splits"].items():41 for i in range(r["count"]):42 seed = r["start"] + i43 t = generate_exec_task(seed, ec["target_tokens"])44 manifest["items"].append(_write_item(t, "exec", split, {"hidden_tests": t["hidden_tests"]}) | {"seed": seed})45 for sub in cfg["public"]["subsets"] + cfg["public"].get("extension_subsets", []):46 for it in load_subset(sub):47 h = it["document_hash"]48 split = "dev" if int(h[-1], 16) <= 3 else "test"49 fam = f"public_{sub}"50 manifest["items"].append(_write_item(it, fam, split) | {"length_words": it["length_words"], "extension": sub in cfg["public"].get("extension_subsets", [])})51 # duplicate check across splits (exact hash)52 seen = {}53 dups = []54 for it in manifest["items"]:55 if it["document_hash"] in seen and seen[it["document_hash"]] != it["split"]:56 dups.append(it["document_id"])57 seen[it["document_hash"]] = it["split"]58 manifest["cross_split_exact_duplicates"] = dups59 MAN.mkdir(parents=True, exist_ok=True)60 write_json(MAN / "splits.json", manifest)61 # license/provenance manifest62 write_json(MAN / "datasets.json", {63 "synthetic": {"source": "wp30.data.synthetic (this repository)", "license": "CC BY 4.0 (generated)", "note": "Fictional entities; a procedurally generated ledger is not a real filing."},64 "exec": {"source": "wp30.data.exec_tasks (this repository)", "license": "MIT (generated)"},65 "public_longbench": {"source": "https://huggingface.co/datasets/THUDM/LongBench data.zip", "paper": "Bai et al. 2024, ACL (10.18653/v1/2024.acl-long.172)",66 "subsets": {"qasper": "CC BY 4.0 (upstream)", "2wikimqa": "Apache-2.0 repo (data licence unverified)", "hotpotqa": "CC BY-SA 4.0 (upstream)"},67 "official_metric": "F1 (LongBench normalization)", "note": "LongBench dataset card has no licence field (unverified)."},68 })69 from collections import Counter70 c = Counter((it["family"], it["split"]) for it in manifest["items"])71 for k in sorted(c): print(k, c[k])72 print("cross-split exact duplicates:", len(dups))73 return 07475def validate(manifest_path: str) -> int:76 man = read_json(manifest_path)77 ok = True78 for it in man["items"]:79 p = DOCS / it["family"] / f"{it['document_id']}.json"80 if not p.exists():81 print("missing", p); ok = False; continue82 d = read_json(p)83 if sha256_text(d["text"]) != it["document_hash"]:84 print("hash mismatch", it["document_id"]); ok = False85 for q in d["questions"]:86 if "answer" in q or "answers" in q:87 print("LEAK: answer in public doc", it["document_id"]); ok = False88 if "facts" in d or "hidden_tests" in d:89 print("LEAK: facts/tests in public doc", it["document_id"]); ok = False90 print("validate:", "OK" if ok else "FAILED", len(man["items"]), "items")91 return 0 if ok else 19293def load_doc(document_id: str, family: str) -> dict:94 return read_json(DOCS / family / f"{document_id}.json")9596def load_key(document_id: str, family: str) -> dict:97 return read_json(KEYS / family / f"{document_id}.json")98