#!/usr/bin/env python3 # ============================================================================= # Project : modelmap # File : benchmarks/promptsets/make_promptsets_v2.py # Purpose : v2 corpora — structure-borne, token-balanced binary properties # Author : Simon-Pierre Boucher # Contact : contact@spboucher.ai # Website : https://modelmap.io # Created : 2026-08-12 # Modified : 2026-08-12 # Platform : macOS / Apple Silicon (arm64) # License : All rights reserved (research code) # ============================================================================= """Promptsets v2 (designed after expA run #1 failed its validity gate). Classes share vocabulary; the property lives in STRUCTURE: word_order : grammatical sentence vs seeded scramble of the same words agreement : subject-verb agreement correct vs violated (is/are balanced across classes so no single token predicts the label) arith_valid : correct vs off-by-delta equations (digits overlap) Each set carries a word-level class token-overlap certificate (Jaccard of class vocabularies) in the manifest — v1 sets sat near 0 overlap; v2 must sit near 1. Deterministic given SEED. """ from __future__ import annotations import hashlib import json import random from pathlib import Path SEED = 54321 N_PER_CLASS = 120 VERSION = "v2" OUT = Path(__file__).resolve().parent SUBJ = ["engineer", "archivist", "gardener", "pilot", "clerk", "surveyor", "tenant", "curator"] OBJ = ["ledger", "lantern", "blueprint", "manifest", "compass", "petition", "sample", "voucher"] VERB = ["inspected", "misplaced", "returned", "copied", "signed", "borrowed"] PLACE = ["before the audit", "after the storm", "during the recess", "near the depot", "without a permit", "under protest"] SUBJ2 = ["witness", "broker", "printer", "warden", "courier", "translator", "referee", "machinist"] OBJ2 = ["receipt", "diagram", "carton", "billet", "stencil", "docket", "spindle", "packet"] VERB2 = ["examined", "labelled", "shelved", "traced", "stamped", "weighed"] PLACE2 = ["behind the annex", "past the checkpoint", "beside the kiln", "within the hour", "against the rules", "along the quay"] NOUN_PAIRS = [("key", "keys"), ("crate", "crates"), ("report", "reports"), ("valve", "valves"), ("ticket", "tickets"), ("ladder", "ladders"), ("sample", "samples"), ("cable", "cables"), ("permit", "permits"), ("beacon", "beacons"), ("filter", "filters"), ("stamp", "stamps")] NEAR = ["near the entrance", "beside the counter", "under the shelf", "behind the gate", "next to the archive", "along the corridor", "opposite the office", "inside the vault"] TAIL = ["ready for review", "still unaccounted for", "marked for disposal", "listed in the registry", "covered in dust", "sealed since spring", "waiting for approval", "due back on Monday"] def scramble(words: list[str], rng: random.Random) -> list[str]: for _ in range(20): p = words[:] rng.shuffle(p) if p != words: return p return list(reversed(words)) def gen(rng: random.Random) -> dict[str, list[dict]]: sets: dict[str, list[dict]] = {} # ---- word_order: grammatical vs scramble of the SAME words def wo(subj, obj, verb, place): items = [] combos = [(s, v, o, p) for s in subj for v in verb for o in obj for p in place] rng.shuffle(combos) for s, v, o, p in combos[: 2 * N_PER_CLASS]: sent = f"The {s} {v} the {o} {p}." words = sent[:-1].split(" ") if len(items) % 2 == 0: items.append({"text": sent, "label": "grammatical"}) else: items.append({"text": " ".join(scramble(words, rng)) + ".", "label": "scrambled"}) return items sets["word_order_A"] = wo(SUBJ, OBJ, VERB, PLACE) sets["word_order_B"] = wo(SUBJ2, OBJ2, VERB2, PLACE2) # ---- agreement: correct vs violated, is/are balanced across classes def agree(pairs, near, tail): items = [] combos = [(n, loc, t) for n in pairs for loc in near for t in tail] rng.shuffle(combos) for i, ((sg, pl), loc, t) in enumerate(combos): if len(items) >= 4 * N_PER_CLASS: break mode = i % 4 if mode == 0: items.append({"text": f"The {sg} {loc} is {t}.", "label": "correct"}) elif mode == 1: items.append({"text": f"The {pl} {loc} are {t}.", "label": "correct"}) elif mode == 2: items.append({"text": f"The {sg} {loc} are {t}.", "label": "violated"}) else: items.append({"text": f"The {pl} {loc} is {t}.", "label": "violated"}) return items # A/B share location/tail banks but use disjoint agreement-bearing nouns sets["agreement_A"] = agree(NOUN_PAIRS[:6], NEAR, TAIL) sets["agreement_B"] = agree(NOUN_PAIRS[6:], NEAR, TAIL) # ---- arith_valid: correct vs off-by-delta (digits overlap across classes) def arith(op): items, seen = [], set() while len(items) < 4 * N_PER_CLASS: a, b = rng.randint(11, 89), rng.randint(11, 89) if op == "-" and a < b: a, b = b, a c = a + b if op == "+" else a - b wrong = c + rng.choice([-9, -7, -4, -3, 3, 4, 7, 9]) key = (a, b) if key in seen or wrong < 0: continue seen.add(key) if len(items) % 2 == 0: items.append({"text": f"{a} {op} {b} = {c}", "label": "valid"}) else: items.append({"text": f"{a} {op} {b} = {wrong}", "label": "invalid"}) return items sets["arith_valid_A"] = arith("+") sets["arith_valid_B"] = arith("-") # balance to N_PER_CLASS per label final = {} for name, items in sets.items(): by: dict[str, list] = {} for it in items: by.setdefault(it["label"], []).append(it) chosen = [] for label, pool in sorted(by.items()): seen, uniq = set(), [] for it in pool: if it["text"] not in seen: seen.add(it["text"]) uniq.append(it) if len(uniq) < N_PER_CLASS: raise SystemExit(f"{name}/{label}: only {len(uniq)} unique") chosen += uniq[:N_PER_CLASS] rng.shuffle(chosen) final[name] = chosen return final def class_token_overlap(items: list[dict]) -> float: """Word-level Jaccard between class vocabularies — difficulty certificate.""" vocab: dict[str, set] = {} for it in items: vocab.setdefault(it["label"], set()).update(it["text"].lower().split()) a, b = vocab.values() return len(a & b) / len(a | b) def main() -> int: rng = random.Random(SEED) sets = gen(rng) manifest = {"version": VERSION, "seed": SEED, "n_per_class": N_PER_CLASS, "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai", "website": "https://modelmap.io", "design": "structure-borne properties, token-balanced classes " "(response to expA run #1 validity-gate failure)", "files": {}} for name, items in sorted(sets.items()): path = OUT / f"{name}.jsonl" payload = "\n".join(json.dumps(it, ensure_ascii=False) for it in items) + "\n" path.write_text(payload) overlap = class_token_overlap(items) manifest["files"][path.name] = { "sha256": hashlib.sha256(payload.encode()).hexdigest(), "n": len(items), "class_token_overlap": round(overlap, 4), } print(f"{path.name:22s} n={len(items)} overlap={overlap:.3f} " f"sha256={manifest['files'][path.name]['sha256'][:12]}…") (OUT / "manifest_v2.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n") print("manifest_v2.json written") return 0 if __name__ == "__main__": raise SystemExit(main())