spb/modelmap Public License

1#!/usr/bin/env python32# =============================================================================3#  Project   : modelmap4#  File      : benchmarks/promptsets/make_promptsets.py5#  Purpose   : Generate versioned, checksummed probing corpora (v1, template)6#  Author    : Simon-Pierre Boucher7#  Contact   : contact@spboucher.ai8#  Website   : https://modelmap.io9#  Created   : 2026-08-1210#  Modified  : 2026-08-1211#  Platform  : macOS / Apple Silicon (arm64)12#  License   : All rights reserved (research code)13# =============================================================================14"""Probing corpora v1 (charter §0.3: versioned artifacts with checksums).1516Three binary properties, TWO deliberately different template families per17property (set A / set B) so that dataset sensitivity is measurable:18  lang_id      : French vs English sentences19  code_prose   : code snippets vs English prose20  arith        : arithmetic-context vs non-arithmetic sentences2122Deterministic given SEED. v1 limitation, declared: template-generated text,23not natural corpora — lexical diversity is bounded; natural-corpus v2 is a24registered follow-up. Output: <name>.jsonl + manifest.json with sha256.25"""2627from __future__ import annotations2829import hashlib30import itertools31import json32import random33from pathlib import Path3435SEED = 1234536N_PER_CLASS = 12037VERSION = "v1"38OUT = Path(__file__).resolve().parent3940TOPICS_EN = ["the harbor", "the library", "the orchard", "the workshop", "the observatory",41             "the market", "the glacier", "the archive", "the vineyard", "the lighthouse"]42TOPICS_FR = ["le port", "la bibliothèque", "le verger", "l'atelier", "l'observatoire",43             "le marché", "le glacier", "les archives", "le vignoble", "le phare"]44ADJ_EN = ["quiet", "ancient", "crowded", "restored", "abandoned", "famous", "modest", "vast"]45ADJ_FR = ["calme", "ancien", "bondé", "restauré", "abandonné", "célèbre", "modeste", "vaste"]46VERB_EN = ["remained open despite the storm", "attracted visitors from the region",47           "was documented in the survey", "changed hands twice last century",48           "stood at the edge of town", "required constant maintenance"]49VERB_FR = ["est resté ouvert malgré la tempête", "a attiré des visiteurs de la région",50           "a été documenté dans l'enquête", "a changé de mains deux fois au siècle dernier",51           "se trouvait à la limite de la ville", "exigeait un entretien constant"]52Q_EN = ["Could you tell me whether", "Do you happen to know if", "I was wondering whether"]53Q_FR = ["Pourrais-tu me dire si", "Sais-tu par hasard si", "Je me demandais si"]54PY_FUNCS = ["total", "scale", "merge", "clip", "score", "rank", "fold", "trim"]55JS_VARS = ["items", "nodes", "queue", "cache", "rows", "edges", "bins", "keys"]56PROSE_SUBJ = ["The committee", "A local historian", "The lead engineer", "Her assistant",57              "The night watchman", "An early visitor", "The town council", "The apprentice"]58PROSE_TAIL = ["reviewed the plans before the meeting.", "kept detailed notes for years.",59              "questioned the original estimate.", "preferred the older method.",60              "described the process in a letter.", "returned before the first frost."]616263def gen(rng: random.Random):64    sets: dict[str, list[dict]] = {}6566    # -------- lang_id: set A = descriptive statements, set B = questions67    a, b = [], []68    for t, adj, v in itertools.product(TOPICS_EN, ADJ_EN, VERB_EN):69        a.append({"text": f"The {adj} site near {t} {v}.", "label": "en"})70    for t, adj, v in itertools.product(TOPICS_FR, ADJ_FR, VERB_FR):71        a.append({"text": f"Le site {adj} près de {t} {v}.", "label": "fr"})72    for q, t, v in itertools.product(Q_EN, TOPICS_EN, VERB_EN):73        b.append({"text": f"{q} the place near {t} {v}?", "label": "en"})74    for q, t, v in itertools.product(Q_FR, TOPICS_FR, VERB_FR):75        b.append({"text": f"{q} l'endroit près de {t} {v} ?", "label": "fr"})76    sets["lang_id_A"], sets["lang_id_B"] = a, b7778    # -------- code_prose: set A = python vs prose, set B = js vs prose79    a, b = [], []80    for f, op, k in itertools.product(PY_FUNCS, ["+", "-", "*"], [1, 2, 3, 5, 7]):81        a.append({"text": f"def {f}(xs):\n    return [x {op} {k} for x in xs if x > {k + 1}]",82                  "label": "code"})83    for s, t, adv in itertools.product(PROSE_SUBJ, PROSE_TAIL,84                                       ["eventually", "reluctantly", "quietly"]):85        a.append({"text": f"{s} {adv} {t.lower()}", "label": "prose"})86    for v, meth, k in itertools.product(JS_VARS, ["filter", "map", "find", "some"], [0, 1, 4, 9]):87        b.append({"text": f"const out = {v}.{meth}(x => x.size > {k}).length;",88                  "label": "code"})89    for s, t, adj in itertools.product(PROSE_SUBJ, PROSE_TAIL[:3], ADJ_EN[:5]):90        b.append({"text": f"{s}, though {adj}, {t.lower()}", "label": "prose"})91    sets["code_prose_A"], sets["code_prose_B"] = a, b9293    # -------- arith: set A = imperative computations, set B = embedded quantities94    a, b = [], []95    for _ in range(4 * N_PER_CLASS):96        x, y = rng.randint(11, 97), rng.randint(11, 97)97        a.append({"text": f"Calculate {x} + {y} and report the result.", "label": "arith"})98        s, t = rng.choice(PROSE_SUBJ), rng.choice(PROSE_TAIL)99        adv = rng.choice(["eventually", "reluctantly", "quietly", "finally", "later"])100        a.append({"text": f"{s} {adv} {t.lower()}", "label": "plain"})101        p, q = rng.randint(12, 89), rng.randint(12, 89)102        b.append({"text": f"If the crate holds {p} jars and {q} more arrive, how many jars are there in total?",103                  "label": "arith"})104        topic, verb, adj = rng.choice(TOPICS_EN), rng.choice(VERB_EN), rng.choice(ADJ_EN)105        b.append({"text": f"According to the {adj} report, {topic} {verb}.", "label": "plain"})106    sets["arith_A"], sets["arith_B"] = a, b107108    # balance + subsample every set to N_PER_CLASS per label, deterministic109    final = {}110    for name, items in sets.items():111        by = {}112        for it in items:113            by.setdefault(it["label"], []).append(it)114        chosen = []115        for label, pool in sorted(by.items()):116            rng.shuffle(pool)117            # dedupe by text before sampling118            seen, uniq = set(), []119            for it in pool:120                if it["text"] not in seen:121                    seen.add(it["text"])122                    uniq.append(it)123            if len(uniq) < N_PER_CLASS:124                raise SystemExit(f"{name}/{label}: only {len(uniq)} unique items")125            chosen += uniq[:N_PER_CLASS]126        rng.shuffle(chosen)127        final[name] = chosen128    return final129130131def main() -> int:132    rng = random.Random(SEED)133    sets = gen(rng)134    manifest = {"version": VERSION, "seed": SEED, "n_per_class": N_PER_CLASS,135                "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai",136                "website": "https://modelmap.io",137                "limitation": "template-generated v1; natural-corpus v2 registered",138                "files": {}}139    for name, items in sorted(sets.items()):140        path = OUT / f"{name}.jsonl"141        payload = "\n".join(json.dumps(it, ensure_ascii=False) for it in items) + "\n"142        path.write_text(payload)143        manifest["files"][path.name] = {144            "sha256": hashlib.sha256(payload.encode()).hexdigest(),145            "n": len(items),146        }147        print(f"{path.name:22s} n={len(items)} sha256={manifest['files'][path.name]['sha256'][:12]}…")148    (OUT / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n")149    print("manifest.json written")150    return 0151152153if __name__ == "__main__":154    raise SystemExit(main())155