#!/usr/bin/env python3 # ============================================================================= # Project : modelmap # File : experiments/micro/expC_causal_verification/implementation/make_interventions_mapcard.py # Purpose : Publish the BAND-claim interventions map from expC run #4 # Author : Simon-Pierre Boucher # Contact : contact@spboucher.ai # Website : https://modelmap.io # Created : 2026-08-12 # Modified : 2026-08-12 # Platform : macOS / Apple Silicon (arm64) # License : All rights reserved (research code) # ============================================================================= """Builds atlas/qwen3-0.6b-4bit/interventions/v1 from expC run #4 — REFUSES unless the run's pre-registered band claim passed (early-band mean specific damage >= bar in EVERY fresh source). Run #3's full-profile version of this gate refused publication (documented in analysis.md); this version publishes a BAND claim, the granularity that replicates.""" from __future__ import annotations import hashlib import json import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[4] sys.path.insert(0, str(ROOT / "src")) from modelmap.atlas.mapcard import MapCard ENTRY = ROOT / "atlas" / "qwen3-0.6b-4bit" / "interventions" / "v1" MODEL_ID = "mlx-community/Qwen3-0.6B-4bit" def newest_run4() -> Path: for d in sorted((ROOT / "results" / "expC_causal_verification").iterdir(), reverse=True): doc = json.loads((d / "results.json").read_text()) if doc.get("run") == 4: return d / "results.json" raise SystemExit("no run-4 results found") def model_hash() -> str: from mlx_lm.utils import hf_repo_to_path mp = Path(hf_repo_to_path(MODEL_ID)) h = hashlib.sha256() for f in sorted(mp.glob("*.safetensors")): h.update(f.read_bytes()) return h.hexdigest() def main() -> int: res_path = newest_run4() doc = json.loads(res_path.read_text()) claim = doc["band_claim"] if not claim["passes"]: print(f"REFUSED: band claim failed (min early mean {claim['min_early_mean']:+.3f} " f"< bar {claim['bar']}) — per the registered rule nothing is published.") return 1 ENTRY.mkdir(parents=True, exist_ok=True) map_doc = { "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai", "website": "https://modelmap.io", "map_type": "interventions", "model_id": MODEL_ID, "claim": "BAND claim: erasing the diff-of-means agreement direction at any single " "layer in the early band (2-15) destroys most of the grammatical-agreement " "margin, replicated across six fresh direction estimates on a fresh " "behavioral bank. The late band (20-27) is reported but carries NO claim " "(declared estimator-unstable by run #3).", "behavior_metric": "logit margin correct-vs-incorrect verb, held-out minimal pairs", "baseline_margin": doc["baseline_margin"], "band": {"early_layers": doc["config"]["early_layers"], "late_layers": doc["config"]["late_layers"], "bar": claim["bar"], "early_means_per_source": claim["early_means"], "min_early_mean": claim["min_early_mean"]}, "per_layer": { "mean": doc["profile_mean"], "min": doc["profile_min"], "random_direction_damage": doc["random_direction_damage"], }, "profiles_per_source": doc["profiles"], "source_results": str(res_path.relative_to(ROOT)), } (ENTRY / "map.json").write_text(json.dumps(map_doc, indent=2) + "\n") mhash = model_hash() created = time.strftime("%Y-%m-%d", time.gmtime()) (ENTRY / "provenance.json").write_text(json.dumps({ "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai", "website": "https://modelmap.io", "model_id": MODEL_ID, "map_type": "interventions", "version": "v1", "commit": doc["commit"], "model_hash": mhash, "config": doc["config"], "seed": doc["config"]["seed"], "hardware_manifest": doc["manifest"], "created": created, "source_results": str(res_path.relative_to(ROOT)), }, indent=2) + "\n") card = MapCard( map_id="atlas/qwen3-0.6b-4bit/interventions/v1", map_type="interventions", model_id=MODEL_ID, model_hash=mhash, quantization="q4 (mlx)", commit=doc["commit"], config=str(res_path.relative_to(ROOT)), created=created, hardware_manifest=doc["manifest"], confidence_level=2, regenerate_command=( ".venv/bin/python experiments/micro/expC_causal_verification/implementation/benchmark_v4.py && " ".venv/bin/python experiments/micro/expC_causal_verification/implementation/make_interventions_mapcard.py"), seeds=[doc["config"]["seed"], *doc["config"]["sources"]], prompt_sets=["agreement_A halves (direction est.)", "agreement_B halves (direction est.)", "fresh held-out minimal-pair bank (8 unseen locations)"], controls=["random-direction erasure per layer (3 dirs, netted out)", "six-source replication with pre-registered bar (runs #3-#4)", "late band excluded as estimator-unstable (run #3 refusal)"], methods_in_agreement=["difference-in-means probing (direction exists, decodable)", "direction erasure (causally load-bearing)"], interventions=["rank-1 direction erasure at each layer's output, all positions"], replication_rate=round(claim["min_early_mean"] / doc["baseline_margin"], 4), featurizer_class="linear (difference-in-means direction)", intervention_protocol="erase h' = h − ⟨h−μ,u⟩u at layer ℓ; logit-margin metric; " "random-direction null netted out; BAND granularity", notes="Level 2, NOT 3: the two agreeing methods share the diff-of-means estimator; " "activation-addition steering is the registered Level-3 path. This entry " "exists because run #3's per-layer version was REFUSED by the gate — the " "band is the granularity that replicates. Anti-correlates with probes/v2 " "layer ranking (survival ledger 0/2): decodability peaks ≠ causal joints.", ) (ENTRY / "mapcard.json").write_text(card.to_json()) per_src = "\n".join(f"- {n}: early-band mean {e:+.3f}" for n, e in zip(doc["config"]["sources"], claim["early_means"])) (ENTRY / "confidence.md").write_text(f"""--- project: modelmap document: qwen3-0.6b-4bit/interventions/v1 — confidence author: Simon-Pierre Boucher contact: contact@spboucher.ai website: https://modelmap.io created: {created} status: reviewed --- # Confidence — qwen3-0.6b-4bit / interventions / v1 ```text Level : 2 Seeds : six fresh direction sources (disjoint halves of two promptsets + two fresh bootstraps); fresh behavioral bank Prompt sets: 3 (two estimation sets + held-out behavioral bank) Methods in agreement : 2 (diff-of-means probing; direction erasure) — shared estimator, hence Level 2 and not 3 Causal verification : YES — rank-1 erasure with random-direction nulls ``` Pre-registered band claim (bar {claim['bar']} on a {doc['baseline_margin']:+.2f} baseline margin): {per_src} Minimum early-band mean across sources: {claim['min_early_mean']:+.3f} — claim PASSES. Granularity discipline: run #3's per-layer profile FAILED replication and was refused by this very gate; the published object is the BAND (layers 2–15). The late band (20–27) is displayed but carries no claim. This map is the causal counterpart of probes/v2, whose decodability ranking it contradicts (survival ledger 0/2) — both stay published, labeled by what they measure. """) errs = card.validate() if errs: print("CARD INVALID:", errs) return 1 print(f"atlas entry written: {ENTRY.relative_to(ROOT)} " f"(Level 2, band min {claim['min_early_mean']:+.3f} / bar {claim['bar']})") return 0 if __name__ == "__main__": sys.exit(main())