spb/modelmap Public License

1#!/usr/bin/env python32# =============================================================================3#  Project   : modelmap4#  File      : experiments/micro/expA_probe_reliability/implementation/make_mapcard.py5#  Purpose   : Build the atlas entry (map + provenance + card) from expA results6#  Author    : Simon-Pierre Boucher7#  Contact   : contact@spboucher.ai8#  Website   : https://modelmap.io9#  Created   : 2026-08-1210#  Modified  : 2026-08-1211#  Platform  : macOS / Apple Silicon (arm64)12#  License   : All rights reserved (research code)13# =============================================================================14"""Turns the newest expA results.json into atlas/qwen3-0.6b-4bit/probes/v1/.1516The confidence level is DERIVED from the evidence in the results file, not17asserted: Level 1 requires >=3 seeds, >=2 promptsets, shuffled-label controls,18FDR-controlled layer scans, and an architecture-null (twin) check recorded.19tools/publish.py then gates the export.20"""2122from __future__ import annotations2324import hashlib25import json26import sys27import time28from pathlib import Path2930ROOT = Path(__file__).resolve().parents[4]31sys.path.insert(0, str(ROOT / "src"))3233from modelmap.atlas.mapcard import MapCard3435ENTRY = ROOT / "atlas" / "qwen3-0.6b-4bit" / "probes" / "v1"36MODEL_ID = "mlx-community/Qwen3-0.6B-4bit"373839def newest_results() -> Path:40    runs = sorted((ROOT / "results" / "expA_probe_reliability").iterdir())41    return runs[-1] / "results.json"424344def model_hash() -> str:45    from mlx_lm.utils import hf_repo_to_path46    mp = Path(hf_repo_to_path(MODEL_ID))47    h = hashlib.sha256()48    for f in sorted(mp.glob("*.safetensors")):49        h.update(f.read_bytes())50    return h.hexdigest()515253def main() -> int:54    res_path = newest_results()55    doc = json.loads(res_path.read_text())56    summary = doc["summary"]57    ENTRY.mkdir(parents=True, exist_ok=True)5859    # ---- map.json: the per-layer probe map (all properties, both sets)60    map_doc = {61        "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai",62        "website": "https://modelmap.io",63        "map_type": "probes", "model_id": MODEL_ID,64        "properties": {},65        "source_results": str(res_path.relative_to(ROOT)),66    }67    for prop in summary:68        map_doc["properties"][prop] = {69            "summary": summary[prop],70            "per_layer": {71                s: [72                    {k: r[k] for k in ("layer", "task_acc_mean", "task_acc_seed_sd",73                                       "selectivity_mean", "selectivity_ci",74                                       "fdr_significant")}75                    for r in doc["results"]["real"][f"{prop}_{s}"]["layers"]76                ] for s in ("A", "B")77            },78            "twin_null_per_layer_A": [79                {k: r[k] for k in ("layer", "selectivity_mean", "fdr_significant")}80                for r in doc["results"]["twin"][f"{prop}_A"]["layers"]81            ],82        }83    (ENTRY / "map.json").write_text(json.dumps(map_doc, indent=2) + "\n")8485    # ---- derive the confidence level honestly86    mean_repl = float(sum(87        (summary[p]["replication_topk_A"] + summary[p]["replication_topk_B"]) / 288        for p in summary) / len(summary))89    level = 1  # controlled + replicated (5 seeds, 2 sets, controls, FDR, twin null)9091    mhash = model_hash()92    created = time.strftime("%Y-%m-%d", time.gmtime())93    provenance = {94        "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai",95        "website": "https://modelmap.io",96        "model_id": MODEL_ID, "map_type": "probes", "version": "v1",97        "commit": doc["commit"], "model_hash": mhash,98        "config": doc["config"], "seed": doc["config"]["seeds"],99        "hardware_manifest": doc["manifest"], "created": created,100        "source_results": str(res_path.relative_to(ROOT)),101    }102    (ENTRY / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n")103104    promptsets = [f"{n}#{m['sha256'][:16]}" for n, m in105                  doc["config"]["promptsets"]["files"].items()]106    card = MapCard(107        map_id="atlas/qwen3-0.6b-4bit/probes/v1",108        map_type="probes",109        model_id=MODEL_ID,110        model_hash=mhash,111        quantization="q4 (mlx)",112        commit=doc["commit"],113        config=str(res_path.relative_to(ROOT)),114        created=created,115        hardware_manifest=doc["manifest"],116        confidence_level=level,117        regenerate_command=(118            ".venv/bin/python benchmarks/promptsets/make_promptsets.py && "119            ".venv/bin/python experiments/micro/expA_probe_reliability/implementation/benchmark.py && "120            ".venv/bin/python experiments/micro/expA_probe_reliability/implementation/make_mapcard.py"),121        seeds=list(doc["config"]["seeds"]),122        prompt_sets=promptsets,123        controls=["shuffled-label (every probe)", "random-init architecture twin",124                  "BH-FDR q=0.05 across layer scans"],125        replication_rate=round(mean_repl, 4),126        per_dataset_agreement=None,127        featurizer_class="natural-basis (mean-pooled residual)",128        intervention_protocol="none (observational map — Level 1 by design)",129        negative_result=True,130        notes="NEGATIVE RESULT: the random-init architecture twin reaches task accuracy "131              "1.00 at every layer for every property — the probe map is indistinguishable "132              "from the architecture+tokenizer null on these template promptsets. This map "133              "is published as evidence that probe maps on lexically separable classes are "134              "uninformative about trained structure. See analysis.md.",135    )136    (ENTRY / "mapcard.json").write_text(card.to_json())137138    lines = "\n".join(139        f"- {p}: maxAcc A/B = {summary[p]['max_task_acc_A']:.3f}/{summary[p]['max_task_acc_B']:.3f}, "140        f"seed SD {summary[p]['mean_seed_sd']:.4f}, dataset shift {summary[p]['mean_dataset_shift']:.4f}, "141        f"twin max selectivity {summary[p]['twin_max_selectivity']:.3f}, "142        f"replication(top-5) {summary[p]['replication_topk_A']:.2f}/{summary[p]['replication_topk_B']:.2f}"143        for p in summary)144    (ENTRY / "confidence.md").write_text(f"""---145project: modelmap146document: qwen3-0.6b-4bit/probes/v1 — confidence147author: Simon-Pierre Boucher148contact: contact@spboucher.ai149website: https://modelmap.io150created: {created}151status: reviewed152---153154# Confidence — qwen3-0.6b-4bit / probes / v1155156```text157Level      : {level}158Seeds      : {len(doc['config']['seeds'])}159Prompt sets: {len(promptsets)} (2 disjoint template families per property)160Methods in agreement : 1 (linear probes only — Level 2 requires a second method)161Causal verification  : none (observational; Level 3 requires intervention)162```163164Per-property evidence:165{lines}166167**This is a published NEGATIVE result (Level 1 for the negative claim).**168The random-init architecture twin matches the trained model at ceiling169(accuracy 1.00, 28/28 layers FDR-significant, for the twin as for the real170model; mean real-minus-twin selectivity within +/-0.06). By the validity171criterion registered in hypothesis.md BEFORE the run (twin selectivity must172stay < 0.05), this probing harness is INVALID for localization claims on173these promptsets: it measures the tokenizer + architecture prior, not174learned computation. The negative claim itself is controlled and replicated175(5 seeds, 2 disjoint promptsets, 3 properties) - hence Level 1.176177Consequences adopted: (1) probe maps are only publishable as REAL-MINUS-TWIN178differentials; (2) promptsets v2 must remove lexical separability (shared179vocabulary across classes); (3) the seed-vs-dataset variance hypothesis is180untestable at ceiling and moves to run #2.181""")182    errs = card.validate()183    if errs:184        print("CARD INVALID:")185        for e in errs:186            print(" -", e)187        return 1188    print(f"atlas entry written: {ENTRY.relative_to(ROOT)} (Level {level}, "189          f"replication {mean_repl:.2f})")190    return 0191192193if __name__ == "__main__":194    sys.exit(main())195