#!/usr/bin/env python3 # ============================================================================= # Project : modelmap # File : experiments/micro/expC_causal_verification/implementation/make_l3_mapcard.py # Purpose : Publish interventions/v2 at Level 3 from run #5 (steering) # Author : Simon-Pierre Boucher # Contact : contact@spboucher.ai # Website : https://modelmap.io # Created : 2026-08-12 # Modified : 2026-08-12 # Platform : macOS / Apple Silicon (arm64) # License : All rights reserved (research code) # ============================================================================= """Builds atlas/qwen3-0.6b-4bit/interventions/v2 (Level 3) — REFUSES unless run #5's pre-registered dose-response criterion passed AND the v1 band entry (Level 2) exists as its foundation. v1 stays published as the Level-2 record.""" from __future__ import annotations import hashlib import json import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[4] sys.path.insert(0, str(ROOT / "src")) from modelmap.atlas.mapcard import MapCard V1 = ROOT / "atlas" / "qwen3-0.6b-4bit" / "interventions" / "v1" ENTRY = ROOT / "atlas" / "qwen3-0.6b-4bit" / "interventions" / "v2" MODEL_ID = "mlx-community/Qwen3-0.6B-4bit" def newest_run(n: int) -> Path: for d in sorted((ROOT / "results" / "expC_causal_verification").iterdir(), reverse=True): doc = json.loads((d / "results.json").read_text()) if doc.get("run") == n: return d / "results.json" raise SystemExit(f"no run-{n} results found") def model_hash() -> str: from mlx_lm.utils import hf_repo_to_path mp = Path(hf_repo_to_path(MODEL_ID)) h = hashlib.sha256() for f in sorted(mp.glob("*.safetensors")): h.update(f.read_bytes()) return h.hexdigest() def main() -> int: if not (V1 / "mapcard.json").exists(): print("REFUSED: the Level-2 band entry (v1) must exist first.") return 1 res5 = newest_run(6) doc5 = json.loads(res5.read_text()) if not doc5["criterion"]["passes"]: print(f"REFUSED: run #6 criterion failed ({doc5['criterion']}) — v1 stays Level 2.") return 1 res4 = newest_run(4) doc4 = json.loads(res4.read_text()) ENTRY.mkdir(parents=True, exist_ok=True) v1_map = json.loads((V1 / "map.json").read_text()) map_doc = { "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai", "website": "https://modelmap.io", "map_type": "interventions", "model_id": MODEL_ID, "claim": "LEVEL-3 claim, scope-limited: (i) band necessity — erasing the " "diff-of-means agreement direction at any early-band layer (2-15) removes " "~73-75% of the grammatical margin (six fresh estimators, run #4, Level-2 " "record in v1); (ii) SINGLE-LAYER HANDLE at layer 12 — activation-addition " "steering moves the margin strictly monotonically with dose and halves it " "at -2 sigma for EVERY of four fresh direction sources on a third fresh " "behavioral bank, while random directions at the same doses move it by " "less than the registered 25% bound (run #6). Runs #3 and #5 record the " "gate refusals that scoped this claim.", "band": v1_map["band"], "per_layer": v1_map["per_layer"], "steering": {"layer": doc5["config"]["layer"], "per_source": doc5["per_source"], "specificity": doc5["specificity"], "baseline_margin": doc5["baseline_margin"]}, "source_results": {"erasure_band": str(res4.relative_to(ROOT)), "steering": str(res5.relative_to(ROOT))}, } (ENTRY / "map.json").write_text(json.dumps(map_doc, indent=2) + "\n") mhash = model_hash() created = time.strftime("%Y-%m-%d", time.gmtime()) (ENTRY / "provenance.json").write_text(json.dumps({ "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai", "website": "https://modelmap.io", "model_id": MODEL_ID, "map_type": "interventions", "version": "v2", "commit": doc5["commit"], "model_hash": mhash, "config": {"erasure": doc4["config"], "steering": doc5["config"]}, "seed": [doc4["config"]["seed"], doc5["config"]["seed"]], "hardware_manifest": doc5["manifest"], "created": created, "source_results": map_doc["source_results"], }, indent=2) + "\n") card = MapCard( map_id="atlas/qwen3-0.6b-4bit/interventions/v2", map_type="interventions", model_id=MODEL_ID, model_hash=mhash, quantization="q4 (mlx)", commit=doc5["commit"], config=json.dumps(map_doc["source_results"]), created=created, hardware_manifest=doc5["manifest"], confidence_level=3, regenerate_command=( ".venv/bin/python experiments/micro/expC_causal_verification/implementation/benchmark_v4.py && " ".venv/bin/python experiments/micro/expC_causal_verification/implementation/benchmark_v5.py && " ".venv/bin/python experiments/micro/expC_causal_verification/implementation/make_interventions_mapcard.py && " ".venv/bin/python experiments/micro/expC_causal_verification/implementation/make_l3_mapcard.py"), seeds=[doc4["config"]["seed"], doc5["config"]["seed"], *doc4["config"]["sources"]], prompt_sets=["agreement_A/B halves + full (direction est.)", "fresh held-out minimal-pair bank (behavior)"], controls=["random-direction erasure nulls (netted)", "random-direction steering specificity (< 25% bound)", "six-source direction replication (run #4)", "four fresh sources x 5 doses at L12, third fresh bank (run #6)", "late band excluded (run #3 refusal); L3 scope narrowed to L12 (run #5 refusal)"], methods_in_agreement=["difference-in-means probing (direction exists)", "rank-1 erasure (necessity, band-replicated)", "activation-addition steering (dose-controlled sufficiency)"], interventions=["rank-1 direction erasure per layer (run #4)", "activation-addition steering at L12, 5 doses x 4 sources (run #6)"], replication_rate=round(json.loads((V1 / "mapcard.json").read_text())["replication_rate"], 4), featurizer_class="linear (difference-in-means direction)", intervention_protocol="erasure h-⟨h−μ,u⟩u; steering h+ασℓu, α∈{−2..+2}; " "logit-margin metric; random-direction nulls both families", notes="Level 3: two independent intervention families (erasure necessity + steering " "dose-response) agree with probing on the same linear object. v1 remains " "published as the Level-2 erasure-only record. Claim scope: ONE model " "(Qwen3-0.6B-4bit), one behavior, band granularity — cross-model and " "cross-quantization generality are open (expG / candidate_02).", ) (ENTRY / "mapcard.json").write_text(card.to_json()) steer_lines = "\n".join( f"- {name}: " + " ".join(f"{a}σ→{sv['dose_margins'][a]:+.2f}" for a in ("-2.0", "-1.0", "0.0", "1.0", "2.0")) for name, sv in doc5["per_source"].items()) + ( f"\n- specificity: random-direction mean |Δ| " f"{doc5['specificity']['random_dir_mean_abs_change']:.2f} vs bound " f"{doc5['specificity']['bound']:.2f}") (ENTRY / "confidence.md").write_text(f"""--- project: modelmap document: qwen3-0.6b-4bit/interventions/v2 — confidence author: Simon-Pierre Boucher contact: contact@spboucher.ai website: https://modelmap.io created: {created} status: reviewed --- # Confidence — qwen3-0.6b-4bit / interventions / v2 ```text Level : 3 Seeds : erasure six fresh sources (run #4) + steering run (run #5) Prompt sets: estimation sets + fresh behavioral bank (never used for estimation) Methods in agreement : 3 (probing; erasure; steering) — two independent intervention families Causal verification : YES, both directions — necessity (erasure) and dose-controlled sufficiency (steering) ``` Steering dose-response at layer {doc5['config']['layer']} (baseline margin {doc5['baseline_margin']:+.2f}): {steer_lines} All pre-registered gates passed at L12 for every fresh source: strict monotonicity, halving at −2σ, random-direction specificity under the 25% bound. Scope honesty: the HANDLE claim is single-layer (L12) and the NECESSITY claim is band-level (v1); one model, one behavior. Runs #3 and #5 are the refusals that scoped this entry. Generality across sizes (expG) and quantization (candidate_02) is untested and registered next. """) errs = card.validate() if errs: print("CARD INVALID:", errs) return 1 print(f"atlas entry written: {ENTRY.relative_to(ROOT)} (LEVEL 3)") return 0 if __name__ == "__main__": sys.exit(main())