SPB Git

spb/wp30_uqo

Public
910 B · 20 lines python
1"""Evaluator isolation: agent-visible documents contain no answers, facts or hidden tests; prompts never include key paths."""2import json, glob, re3from pathlib import Path4from wp30.util import ROOT56def test_public_docs_have_no_keys():7    for p in glob.glob(str(ROOT / "data/generated/docs/*/*.json")):8        d = json.load(open(p))9        assert "facts" not in d and "hidden_tests" not in d10        for q in d["questions"]:11            assert "answer" not in q and "answers" not in q and "aliases" not in q1213def test_prompts_do_not_reference_keys():14    src = (ROOT / "src/wp30/prompts.py").read_text() + (ROOT / "src/wp30/methods/text.py").read_text()15    assert "keys/" not in src and "hidden_tests" not in src1617def test_synthetic_answers_not_in_filenames():18    for p in glob.glob(str(ROOT / "data/generated/docs/synthetic/*.json")):19        assert re.fullmatch(r"syn_\d{5}_\d+\.json", Path(p).name)20
Files · fea8e12394fac49c22b00e41baf681fd35f0576e
Menu

Theme