spb/modelmap Public License

1#!/usr/bin/env python32# =============================================================================3#  Project   : modelmap4#  File      : experiments/micro/expA_probe_reliability/implementation/make_mapcard_v2.py5#  Purpose   : Build atlas/qwen3-0.6b-4bit/probes/v2 from expA run #2 results6#  Author    : Simon-Pierre Boucher7#  Contact   : contact@spboucher.ai8#  Website   : https://modelmap.io9#  Created   : 2026-08-1210#  Modified  : 2026-08-1211#  Platform  : macOS / Apple Silicon (arm64)12#  License   : All rights reserved (research code)13# =============================================================================14"""Atlas entry v2: differential probe maps (real − twin) on structure-borne,15token-balanced properties. Mixed outcome, published per property:16agreement + arith_valid carry trained-model signal above the architecture17prior (Level 1); word_order is flagged null-dominated."""1819from __future__ import annotations2021import hashlib22import json23import sys24import time25from pathlib import Path2627ROOT = Path(__file__).resolve().parents[4]28sys.path.insert(0, str(ROOT / "src"))2930from modelmap.atlas.mapcard import MapCard3132ENTRY = ROOT / "atlas" / "qwen3-0.6b-4bit" / "probes" / "v2"33MODEL_ID = "mlx-community/Qwen3-0.6B-4bit"34PROPERTIES = ("word_order", "agreement", "arith_valid")353637def newest_run2() -> Path:38    for d in sorted((ROOT / "results" / "expA_probe_reliability").iterdir(), reverse=True):39        doc = json.loads((d / "results.json").read_text())40        if doc.get("run") == 2:41            return d / "results.json"42    raise SystemExit("no run-2 results found")434445def model_hash() -> str:46    from mlx_lm.utils import hf_repo_to_path47    mp = Path(hf_repo_to_path(MODEL_ID))48    h = hashlib.sha256()49    for f in sorted(mp.glob("*.safetensors")):50        h.update(f.read_bytes())51    return h.hexdigest()525354def rows(doc, kind, name):55    keys = ("layer", "task_acc_mean", "task_acc_seed_sd", "selectivity_mean",56            "selectivity_ci", "fdr_significant")57    return [{k: r[k] for k in keys if k in r} for r in doc["results"][kind][name]["layers"]]585960def main() -> int:61    res_path = newest_run2()62    doc = json.loads(res_path.read_text())63    summary = doc["summary"]64    ENTRY.mkdir(parents=True, exist_ok=True)6566    map_doc = {67        "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai",68        "website": "https://modelmap.io",69        "map_type": "probes", "model_id": MODEL_ID,70        "design": "differential maps: real vs random-init twin, token-balanced classes",71        "properties": {},72        "source_results": str(res_path.relative_to(ROOT)),73    }74    verdicts = {75        "word_order": "null-dominated (twin acc 0.96; surface statistics explain the map)",76        "agreement": "trained-model signal (real−twin sel > 0.10 on 25/28 layers, max +0.38)",77        "arith_valid": "trained-model signal (real acc 0.86–0.90 vs twin 0.56–0.58)",78    }79    for prop in PROPERTIES:80        map_doc["properties"][prop] = {81            "verdict": verdicts[prop],82            "summary": {k: v for k, v in summary.items() if k.startswith(prop)},83            "per_layer": {s: rows(doc, "real", f"{prop}_{s}_mean") for s in ("A", "B")},84            "per_layer_last": {s: rows(doc, "real", f"{prop}_{s}_last") for s in ("A", "B")},85            "twin_null_per_layer_A": rows(doc, "twin", f"{prop}_A_mean"),86            "twin_null_per_layer_A_last": rows(doc, "twin", f"{prop}_A_last"),87        }88    (ENTRY / "map.json").write_text(json.dumps(map_doc, indent=2) + "\n")8990    mean_repl = float(sum(summary[f"{p}_mean"]["replication_topk_A"] for p in PROPERTIES) / 3)91    mhash = model_hash()92    created = time.strftime("%Y-%m-%d", time.gmtime())93    (ENTRY / "provenance.json").write_text(json.dumps({94        "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai",95        "website": "https://modelmap.io",96        "model_id": MODEL_ID, "map_type": "probes", "version": "v2",97        "commit": doc["commit"], "model_hash": mhash, "config": doc["config"],98        "seed": doc["config"]["seeds"], "hardware_manifest": doc["manifest"],99        "created": created, "source_results": str(res_path.relative_to(ROOT)),100    }, indent=2) + "\n")101102    promptsets = [f"{n}#{m['sha256'][:16]}" for n, m in103                  doc["config"]["promptsets"]["files"].items()]104    card = MapCard(105        map_id="atlas/qwen3-0.6b-4bit/probes/v2",106        map_type="probes", model_id=MODEL_ID, model_hash=mhash,107        quantization="q4 (mlx)", commit=doc["commit"],108        config=str(res_path.relative_to(ROOT)), created=created,109        hardware_manifest=doc["manifest"], confidence_level=1,110        regenerate_command=(111            ".venv/bin/python benchmarks/promptsets/make_promptsets_v2.py && "112            ".venv/bin/python experiments/micro/expA_probe_reliability/implementation/benchmark_v2.py && "113            ".venv/bin/python experiments/micro/expA_probe_reliability/implementation/make_mapcard_v2.py"),114        seeds=list(doc["config"]["seeds"]), prompt_sets=promptsets,115        controls=["shuffled-label (every probe)", "random-init architecture twin",116                  "BH-FDR q=0.05", "v1 positive control (ceiling check)",117                  "class token-overlap certificates in promptset manifest"],118        replication_rate=round(mean_repl, 4),119        featurizer_class="natural-basis (mean-pooled + last-token residual)",120        intervention_protocol="none (observational map — Level 1 by design)",121        negative_result=False,122        notes="DIFFERENTIAL map (real minus random-init twin), per the doctrine adopted "123              "after v1. Mixed outcome by property: agreement and arith_valid carry "124              "trained-model signal above the architecture prior; word_order is "125              "null-dominated and flagged as such. Strict twin gate (<0.05) still fails "126              "on word_order/agreement — only differential claims are published.",127    )128    (ENTRY / "mapcard.json").write_text(card.to_json())129130    lines = "\n".join(131        f"- {p}: {verdicts[p]}; maxAcc A (mean-pool) "132        f"{summary[f'{p}_mean']['max_task_acc_A']:.3f}, twin acc "133        f"{summary[f'{p}_mean']['twin_max_acc']:.3f}, signal layers "134        f"{summary[f'{p}_mean']['layers_real_minus_twin_gt_0.10']}/28"135        for p in PROPERTIES)136    (ENTRY / "confidence.md").write_text(f"""---137project: modelmap138document: qwen3-0.6b-4bit/probes/v2 — confidence139author: Simon-Pierre Boucher140contact: contact@spboucher.ai141website: https://modelmap.io142created: {created}143status: reviewed144---145146# Confidence — qwen3-0.6b-4bit / probes / v2147148```text149Level      : 1150Seeds      : {len(doc['config']['seeds'])}151Prompt sets: {len(promptsets)} (token-balanced, structure-borne; overlap certificates in manifest)152Methods in agreement : 1 (linear probes only — Level 2 requires a second method)153Causal verification  : none (observational; Level 3 requires intervention)154```155156Per-property verdicts (differential real−twin, mean pooling):157{lines}158159Published claims are DIFFERENTIAL only (real minus random-init twin), per the160doctrine adopted after v1's validity-gate failure. The strict twin gate161(selectivity < 0.05) still fails on word_order and agreement — the twin162extracts real surface signal from tokenization statistics — so raw probe163accuracies are never cited as evidence of learned structure. What survives:164agreement and arith_valid show layer-resolved trained-model signal165(Level 1, correlational; 5 seeds × 2 sets, controls listed above).166""")167    errs = card.validate()168    if errs:169        print("CARD INVALID:", errs)170        return 1171    print(f"atlas entry written: {ENTRY.relative_to(ROOT)} (Level 1, repl {mean_repl:.2f})")172    return 0173174175if __name__ == "__main__":176    sys.exit(main())177