spb/modelmap Public License

1#!/usr/bin/env python32# =============================================================================3#  Project   : modelmap4#  File      : benchmarks/promptsets/make_promptsets_v2.py5#  Purpose   : v2 corpora — structure-borne, token-balanced binary properties6#  Author    : Simon-Pierre Boucher7#  Contact   : contact@spboucher.ai8#  Website   : https://modelmap.io9#  Created   : 2026-08-1210#  Modified  : 2026-08-1211#  Platform  : macOS / Apple Silicon (arm64)12#  License   : All rights reserved (research code)13# =============================================================================14"""Promptsets v2 (designed after expA run #1 failed its validity gate).1516Classes share vocabulary; the property lives in STRUCTURE:17  word_order  : grammatical sentence vs seeded scramble of the same words18  agreement   : subject-verb agreement correct vs violated (is/are balanced19                across classes so no single token predicts the label)20  arith_valid : correct vs off-by-delta equations (digits overlap)2122Each set carries a word-level class token-overlap certificate (Jaccard of23class vocabularies) in the manifest — v1 sets sat near 0 overlap; v2 must24sit near 1. Deterministic given SEED.25"""2627from __future__ import annotations2829import hashlib30import json31import random32from pathlib import Path3334SEED = 5432135N_PER_CLASS = 12036VERSION = "v2"37OUT = Path(__file__).resolve().parent3839SUBJ = ["engineer", "archivist", "gardener", "pilot", "clerk", "surveyor", "tenant", "curator"]40OBJ = ["ledger", "lantern", "blueprint", "manifest", "compass", "petition", "sample", "voucher"]41VERB = ["inspected", "misplaced", "returned", "copied", "signed", "borrowed"]42PLACE = ["before the audit", "after the storm", "during the recess", "near the depot",43         "without a permit", "under protest"]44SUBJ2 = ["witness", "broker", "printer", "warden", "courier", "translator", "referee", "machinist"]45OBJ2 = ["receipt", "diagram", "carton", "billet", "stencil", "docket", "spindle", "packet"]46VERB2 = ["examined", "labelled", "shelved", "traced", "stamped", "weighed"]47PLACE2 = ["behind the annex", "past the checkpoint", "beside the kiln", "within the hour",48          "against the rules", "along the quay"]49NOUN_PAIRS = [("key", "keys"), ("crate", "crates"), ("report", "reports"), ("valve", "valves"),50              ("ticket", "tickets"), ("ladder", "ladders"), ("sample", "samples"), ("cable", "cables"),51              ("permit", "permits"), ("beacon", "beacons"), ("filter", "filters"), ("stamp", "stamps")]52NEAR = ["near the entrance", "beside the counter", "under the shelf", "behind the gate",53        "next to the archive", "along the corridor", "opposite the office", "inside the vault"]54TAIL = ["ready for review", "still unaccounted for", "marked for disposal",55        "listed in the registry", "covered in dust", "sealed since spring",56        "waiting for approval", "due back on Monday"]575859def scramble(words: list[str], rng: random.Random) -> list[str]:60    for _ in range(20):61        p = words[:]62        rng.shuffle(p)63        if p != words:64            return p65    return list(reversed(words))666768def gen(rng: random.Random) -> dict[str, list[dict]]:69    sets: dict[str, list[dict]] = {}7071    # ---- word_order: grammatical vs scramble of the SAME words72    def wo(subj, obj, verb, place):73        items = []74        combos = [(s, v, o, p) for s in subj for v in verb for o in obj for p in place]75        rng.shuffle(combos)76        for s, v, o, p in combos[: 2 * N_PER_CLASS]:77            sent = f"The {s} {v} the {o} {p}."78            words = sent[:-1].split(" ")79            if len(items) % 2 == 0:80                items.append({"text": sent, "label": "grammatical"})81            else:82                items.append({"text": " ".join(scramble(words, rng)) + ".", "label": "scrambled"})83        return items84    sets["word_order_A"] = wo(SUBJ, OBJ, VERB, PLACE)85    sets["word_order_B"] = wo(SUBJ2, OBJ2, VERB2, PLACE2)8687    # ---- agreement: correct vs violated, is/are balanced across classes88    def agree(pairs, near, tail):89        items = []90        combos = [(n, loc, t) for n in pairs for loc in near for t in tail]91        rng.shuffle(combos)92        for i, ((sg, pl), loc, t) in enumerate(combos):93            if len(items) >= 4 * N_PER_CLASS:94                break95            mode = i % 496            if mode == 0:97                items.append({"text": f"The {sg} {loc} is {t}.", "label": "correct"})98            elif mode == 1:99                items.append({"text": f"The {pl} {loc} are {t}.", "label": "correct"})100            elif mode == 2:101                items.append({"text": f"The {sg} {loc} are {t}.", "label": "violated"})102            else:103                items.append({"text": f"The {pl} {loc} is {t}.", "label": "violated"})104        return items105    # A/B share location/tail banks but use disjoint agreement-bearing nouns106    sets["agreement_A"] = agree(NOUN_PAIRS[:6], NEAR, TAIL)107    sets["agreement_B"] = agree(NOUN_PAIRS[6:], NEAR, TAIL)108109    # ---- arith_valid: correct vs off-by-delta (digits overlap across classes)110    def arith(op):111        items, seen = [], set()112        while len(items) < 4 * N_PER_CLASS:113            a, b = rng.randint(11, 89), rng.randint(11, 89)114            if op == "-" and a < b:115                a, b = b, a116            c = a + b if op == "+" else a - b117            wrong = c + rng.choice([-9, -7, -4, -3, 3, 4, 7, 9])118            key = (a, b)119            if key in seen or wrong < 0:120                continue121            seen.add(key)122            if len(items) % 2 == 0:123                items.append({"text": f"{a} {op} {b} = {c}", "label": "valid"})124            else:125                items.append({"text": f"{a} {op} {b} = {wrong}", "label": "invalid"})126        return items127    sets["arith_valid_A"] = arith("+")128    sets["arith_valid_B"] = arith("-")129130    # balance to N_PER_CLASS per label131    final = {}132    for name, items in sets.items():133        by: dict[str, list] = {}134        for it in items:135            by.setdefault(it["label"], []).append(it)136        chosen = []137        for label, pool in sorted(by.items()):138            seen, uniq = set(), []139            for it in pool:140                if it["text"] not in seen:141                    seen.add(it["text"])142                    uniq.append(it)143            if len(uniq) < N_PER_CLASS:144                raise SystemExit(f"{name}/{label}: only {len(uniq)} unique")145            chosen += uniq[:N_PER_CLASS]146        rng.shuffle(chosen)147        final[name] = chosen148    return final149150151def class_token_overlap(items: list[dict]) -> float:152    """Word-level Jaccard between class vocabularies — difficulty certificate."""153    vocab: dict[str, set] = {}154    for it in items:155        vocab.setdefault(it["label"], set()).update(it["text"].lower().split())156    a, b = vocab.values()157    return len(a & b) / len(a | b)158159160def main() -> int:161    rng = random.Random(SEED)162    sets = gen(rng)163    manifest = {"version": VERSION, "seed": SEED, "n_per_class": N_PER_CLASS,164                "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai",165                "website": "https://modelmap.io",166                "design": "structure-borne properties, token-balanced classes "167                          "(response to expA run #1 validity-gate failure)",168                "files": {}}169    for name, items in sorted(sets.items()):170        path = OUT / f"{name}.jsonl"171        payload = "\n".join(json.dumps(it, ensure_ascii=False) for it in items) + "\n"172        path.write_text(payload)173        overlap = class_token_overlap(items)174        manifest["files"][path.name] = {175            "sha256": hashlib.sha256(payload.encode()).hexdigest(),176            "n": len(items),177            "class_token_overlap": round(overlap, 4),178        }179        print(f"{path.name:22s} n={len(items)} overlap={overlap:.3f} "180              f"sha256={manifest['files'][path.name]['sha256'][:12]}…")181    (OUT / "manifest_v2.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n")182    print("manifest_v2.json written")183    return 0184185186if __name__ == "__main__":187    raise SystemExit(main())188