spb/modelmap Public License
Internal cartography of local LLMs on Apple Silicon — registered, gated, negative-first. Public atlas at modelmap.io.
Python 66.3%
JavaScript 24.5%
CSS 8.1%
Shell 0.7%
1#!/usr/bin/env python32# =============================================================================3# Project : modelmap4# File : experiments/micro/expC_causal_verification/implementation/make_interventions_mapcard.py5# Purpose : Publish the BAND-claim interventions map from expC run #46# Author : Simon-Pierre Boucher7# Contact : contact@spboucher.ai8# Website : https://modelmap.io9# Created : 2026-08-1210# Modified : 2026-08-1211# Platform : macOS / Apple Silicon (arm64)12# License : All rights reserved (research code)13# =============================================================================14"""Builds atlas/qwen3-0.6b-4bit/interventions/v1 from expC run #4 —15REFUSES unless the run's pre-registered band claim passed (early-band mean16specific damage >= bar in EVERY fresh source). Run #3's full-profile17version of this gate refused publication (documented in analysis.md);18this version publishes a BAND claim, the granularity that replicates."""1920from __future__ import annotations2122import hashlib23import json24import sys25import time26from pathlib import Path2728ROOT = Path(__file__).resolve().parents[4]29sys.path.insert(0, str(ROOT / "src"))3031from modelmap.atlas.mapcard import MapCard3233ENTRY = ROOT / "atlas" / "qwen3-0.6b-4bit" / "interventions" / "v1"34MODEL_ID = "mlx-community/Qwen3-0.6B-4bit"353637def newest_run4() -> Path:38 for d in sorted((ROOT / "results" / "expC_causal_verification").iterdir(), reverse=True):39 doc = json.loads((d / "results.json").read_text())40 if doc.get("run") == 4:41 return d / "results.json"42 raise SystemExit("no run-4 results found")434445def model_hash() -> str:46 from mlx_lm.utils import hf_repo_to_path47 mp = Path(hf_repo_to_path(MODEL_ID))48 h = hashlib.sha256()49 for f in sorted(mp.glob("*.safetensors")):50 h.update(f.read_bytes())51 return h.hexdigest()525354def main() -> int:55 res_path = newest_run4()56 doc = json.loads(res_path.read_text())57 claim = doc["band_claim"]58 if not claim["passes"]:59 print(f"REFUSED: band claim failed (min early mean {claim['min_early_mean']:+.3f} "60 f"< bar {claim['bar']}) — per the registered rule nothing is published.")61 return 16263 ENTRY.mkdir(parents=True, exist_ok=True)64 map_doc = {65 "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai",66 "website": "https://modelmap.io",67 "map_type": "interventions", "model_id": MODEL_ID,68 "claim": "BAND claim: erasing the diff-of-means agreement direction at any single "69 "layer in the early band (2-15) destroys most of the grammatical-agreement "70 "margin, replicated across six fresh direction estimates on a fresh "71 "behavioral bank. The late band (20-27) is reported but carries NO claim "72 "(declared estimator-unstable by run #3).",73 "behavior_metric": "logit margin correct-vs-incorrect verb, held-out minimal pairs",74 "baseline_margin": doc["baseline_margin"],75 "band": {"early_layers": doc["config"]["early_layers"],76 "late_layers": doc["config"]["late_layers"],77 "bar": claim["bar"], "early_means_per_source": claim["early_means"],78 "min_early_mean": claim["min_early_mean"]},79 "per_layer": {80 "mean": doc["profile_mean"],81 "min": doc["profile_min"],82 "random_direction_damage": doc["random_direction_damage"],83 },84 "profiles_per_source": doc["profiles"],85 "source_results": str(res_path.relative_to(ROOT)),86 }87 (ENTRY / "map.json").write_text(json.dumps(map_doc, indent=2) + "\n")8889 mhash = model_hash()90 created = time.strftime("%Y-%m-%d", time.gmtime())91 (ENTRY / "provenance.json").write_text(json.dumps({92 "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai",93 "website": "https://modelmap.io",94 "model_id": MODEL_ID, "map_type": "interventions", "version": "v1",95 "commit": doc["commit"], "model_hash": mhash, "config": doc["config"],96 "seed": doc["config"]["seed"], "hardware_manifest": doc["manifest"],97 "created": created, "source_results": str(res_path.relative_to(ROOT)),98 }, indent=2) + "\n")99100 card = MapCard(101 map_id="atlas/qwen3-0.6b-4bit/interventions/v1",102 map_type="interventions", model_id=MODEL_ID, model_hash=mhash,103 quantization="q4 (mlx)", commit=doc["commit"],104 config=str(res_path.relative_to(ROOT)), created=created,105 hardware_manifest=doc["manifest"], confidence_level=2,106 regenerate_command=(107 ".venv/bin/python experiments/micro/expC_causal_verification/implementation/benchmark_v4.py && "108 ".venv/bin/python experiments/micro/expC_causal_verification/implementation/make_interventions_mapcard.py"),109 seeds=[doc["config"]["seed"], *doc["config"]["sources"]],110 prompt_sets=["agreement_A halves (direction est.)", "agreement_B halves (direction est.)",111 "fresh held-out minimal-pair bank (8 unseen locations)"],112 controls=["random-direction erasure per layer (3 dirs, netted out)",113 "six-source replication with pre-registered bar (runs #3-#4)",114 "late band excluded as estimator-unstable (run #3 refusal)"],115 methods_in_agreement=["difference-in-means probing (direction exists, decodable)",116 "direction erasure (causally load-bearing)"],117 interventions=["rank-1 direction erasure at each layer's output, all positions"],118 replication_rate=round(claim["min_early_mean"] / doc["baseline_margin"], 4),119 featurizer_class="linear (difference-in-means direction)",120 intervention_protocol="erase h' = h − ⟨h−μ,u⟩u at layer ℓ; logit-margin metric; "121 "random-direction null netted out; BAND granularity",122 notes="Level 2, NOT 3: the two agreeing methods share the diff-of-means estimator; "123 "activation-addition steering is the registered Level-3 path. This entry "124 "exists because run #3's per-layer version was REFUSED by the gate — the "125 "band is the granularity that replicates. Anti-correlates with probes/v2 "126 "layer ranking (survival ledger 0/2): decodability peaks ≠ causal joints.",127 )128 (ENTRY / "mapcard.json").write_text(card.to_json())129130 per_src = "\n".join(f"- {n}: early-band mean {e:+.3f}"131 for n, e in zip(doc["config"]["sources"], claim["early_means"]))132 (ENTRY / "confidence.md").write_text(f"""---133project: modelmap134document: qwen3-0.6b-4bit/interventions/v1 — confidence135author: Simon-Pierre Boucher136contact: contact@spboucher.ai137website: https://modelmap.io138created: {created}139status: reviewed140---141142# Confidence — qwen3-0.6b-4bit / interventions / v1143144```text145Level : 2146Seeds : six fresh direction sources (disjoint halves of two promptsets147 + two fresh bootstraps); fresh behavioral bank148Prompt sets: 3 (two estimation sets + held-out behavioral bank)149Methods in agreement : 2 (diff-of-means probing; direction erasure) — shared150 estimator, hence Level 2 and not 3151Causal verification : YES — rank-1 erasure with random-direction nulls152```153154Pre-registered band claim (bar {claim['bar']} on a {doc['baseline_margin']:+.2f} baseline margin):155{per_src}156Minimum early-band mean across sources: {claim['min_early_mean']:+.3f} — claim PASSES.157158Granularity discipline: run #3's per-layer profile FAILED replication and was159refused by this very gate; the published object is the BAND (layers 2–15).160The late band (20–27) is displayed but carries no claim. This map is the161causal counterpart of probes/v2, whose decodability ranking it contradicts162(survival ledger 0/2) — both stay published, labeled by what they measure.163""")164 errs = card.validate()165 if errs:166 print("CARD INVALID:", errs)167 return 1168 print(f"atlas entry written: {ENTRY.relative_to(ROOT)} "169 f"(Level 2, band min {claim['min_early_mean']:+.3f} / bar {claim['bar']})")170 return 0171172173if __name__ == "__main__":174 sys.exit(main())175