spb/modelmap Public License

1#!/usr/bin/env python32# =============================================================================3#  Project   : modelmap4#  File      : experiments/micro/expC_causal_verification/implementation/make_l3_mapcard.py5#  Purpose   : Publish interventions/v2 at Level 3 from run #5 (steering)6#  Author    : Simon-Pierre Boucher7#  Contact   : contact@spboucher.ai8#  Website   : https://modelmap.io9#  Created   : 2026-08-1210#  Modified  : 2026-08-1211#  Platform  : macOS / Apple Silicon (arm64)12#  License   : All rights reserved (research code)13# =============================================================================14"""Builds atlas/qwen3-0.6b-4bit/interventions/v2 (Level 3) — REFUSES unless15run #5's pre-registered dose-response criterion passed AND the v1 band entry16(Level 2) exists as its foundation. v1 stays published as the Level-2 record."""1718from __future__ import annotations1920import hashlib21import json22import sys23import time24from pathlib import Path2526ROOT = Path(__file__).resolve().parents[4]27sys.path.insert(0, str(ROOT / "src"))2829from modelmap.atlas.mapcard import MapCard3031V1 = ROOT / "atlas" / "qwen3-0.6b-4bit" / "interventions" / "v1"32ENTRY = ROOT / "atlas" / "qwen3-0.6b-4bit" / "interventions" / "v2"33MODEL_ID = "mlx-community/Qwen3-0.6B-4bit"343536def newest_run(n: int) -> Path:37    for d in sorted((ROOT / "results" / "expC_causal_verification").iterdir(), reverse=True):38        doc = json.loads((d / "results.json").read_text())39        if doc.get("run") == n:40            return d / "results.json"41    raise SystemExit(f"no run-{n} results found")424344def model_hash() -> str:45    from mlx_lm.utils import hf_repo_to_path46    mp = Path(hf_repo_to_path(MODEL_ID))47    h = hashlib.sha256()48    for f in sorted(mp.glob("*.safetensors")):49        h.update(f.read_bytes())50    return h.hexdigest()515253def main() -> int:54    if not (V1 / "mapcard.json").exists():55        print("REFUSED: the Level-2 band entry (v1) must exist first.")56        return 157    res5 = newest_run(6)58    doc5 = json.loads(res5.read_text())59    if not doc5["criterion"]["passes"]:60        print(f"REFUSED: run #6 criterion failed ({doc5['criterion']}) — v1 stays Level 2.")61        return 162    res4 = newest_run(4)63    doc4 = json.loads(res4.read_text())6465    ENTRY.mkdir(parents=True, exist_ok=True)66    v1_map = json.loads((V1 / "map.json").read_text())67    map_doc = {68        "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai",69        "website": "https://modelmap.io",70        "map_type": "interventions", "model_id": MODEL_ID,71        "claim": "LEVEL-3 claim, scope-limited: (i) band necessity — erasing the "72                 "diff-of-means agreement direction at any early-band layer (2-15) removes "73                 "~73-75% of the grammatical margin (six fresh estimators, run #4, Level-2 "74                 "record in v1); (ii) SINGLE-LAYER HANDLE at layer 12 — activation-addition "75                 "steering moves the margin strictly monotonically with dose and halves it "76                 "at -2 sigma for EVERY of four fresh direction sources on a third fresh "77                 "behavioral bank, while random directions at the same doses move it by "78                 "less than the registered 25% bound (run #6). Runs #3 and #5 record the "79                 "gate refusals that scoped this claim.",80        "band": v1_map["band"],81        "per_layer": v1_map["per_layer"],82        "steering": {"layer": doc5["config"]["layer"],83                     "per_source": doc5["per_source"],84                     "specificity": doc5["specificity"],85                     "baseline_margin": doc5["baseline_margin"]},86        "source_results": {"erasure_band": str(res4.relative_to(ROOT)),87                           "steering": str(res5.relative_to(ROOT))},88    }89    (ENTRY / "map.json").write_text(json.dumps(map_doc, indent=2) + "\n")9091    mhash = model_hash()92    created = time.strftime("%Y-%m-%d", time.gmtime())93    (ENTRY / "provenance.json").write_text(json.dumps({94        "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai",95        "website": "https://modelmap.io",96        "model_id": MODEL_ID, "map_type": "interventions", "version": "v2",97        "commit": doc5["commit"], "model_hash": mhash,98        "config": {"erasure": doc4["config"], "steering": doc5["config"]},99        "seed": [doc4["config"]["seed"], doc5["config"]["seed"]],100        "hardware_manifest": doc5["manifest"], "created": created,101        "source_results": map_doc["source_results"],102    }, indent=2) + "\n")103104    card = MapCard(105        map_id="atlas/qwen3-0.6b-4bit/interventions/v2",106        map_type="interventions", model_id=MODEL_ID, model_hash=mhash,107        quantization="q4 (mlx)", commit=doc5["commit"],108        config=json.dumps(map_doc["source_results"]), created=created,109        hardware_manifest=doc5["manifest"], confidence_level=3,110        regenerate_command=(111            ".venv/bin/python experiments/micro/expC_causal_verification/implementation/benchmark_v4.py && "112            ".venv/bin/python experiments/micro/expC_causal_verification/implementation/benchmark_v5.py && "113            ".venv/bin/python experiments/micro/expC_causal_verification/implementation/make_interventions_mapcard.py && "114            ".venv/bin/python experiments/micro/expC_causal_verification/implementation/make_l3_mapcard.py"),115        seeds=[doc4["config"]["seed"], doc5["config"]["seed"], *doc4["config"]["sources"]],116        prompt_sets=["agreement_A/B halves + full (direction est.)",117                     "fresh held-out minimal-pair bank (behavior)"],118        controls=["random-direction erasure nulls (netted)",119                  "random-direction steering specificity (< 25% bound)",120                  "six-source direction replication (run #4)",121                  "four fresh sources x 5 doses at L12, third fresh bank (run #6)",122                  "late band excluded (run #3 refusal); L3 scope narrowed to L12 (run #5 refusal)"],123        methods_in_agreement=["difference-in-means probing (direction exists)",124                              "rank-1 erasure (necessity, band-replicated)",125                              "activation-addition steering (dose-controlled sufficiency)"],126        interventions=["rank-1 direction erasure per layer (run #4)",127                       "activation-addition steering at L12, 5 doses x 4 sources (run #6)"],128        replication_rate=round(json.loads((V1 / "mapcard.json").read_text())["replication_rate"], 4),129        featurizer_class="linear (difference-in-means direction)",130        intervention_protocol="erasure h-⟨h−μ,u⟩u; steering h+ασℓu, α∈{−2..+2}; "131                              "logit-margin metric; random-direction nulls both families",132        notes="Level 3: two independent intervention families (erasure necessity + steering "133              "dose-response) agree with probing on the same linear object. v1 remains "134              "published as the Level-2 erasure-only record. Claim scope: ONE model "135              "(Qwen3-0.6B-4bit), one behavior, band granularity — cross-model and "136              "cross-quantization generality are open (expG / candidate_02).",137    )138    (ENTRY / "mapcard.json").write_text(card.to_json())139140    steer_lines = "\n".join(141        f"- {name}: " + " ".join(f"{a}σ→{sv['dose_margins'][a]:+.2f}"142                                 for a in ("-2.0", "-1.0", "0.0", "1.0", "2.0"))143        for name, sv in doc5["per_source"].items()) + (144        f"\n- specificity: random-direction mean |Δ| "145        f"{doc5['specificity']['random_dir_mean_abs_change']:.2f} vs bound "146        f"{doc5['specificity']['bound']:.2f}")147    (ENTRY / "confidence.md").write_text(f"""---148project: modelmap149document: qwen3-0.6b-4bit/interventions/v2 — confidence150author: Simon-Pierre Boucher151contact: contact@spboucher.ai152website: https://modelmap.io153created: {created}154status: reviewed155---156157# Confidence — qwen3-0.6b-4bit / interventions / v2158159```text160Level      : 3161Seeds      : erasure six fresh sources (run #4) + steering run (run #5)162Prompt sets: estimation sets + fresh behavioral bank (never used for estimation)163Methods in agreement : 3 (probing; erasure; steering) — two independent164                       intervention families165Causal verification  : YES, both directions — necessity (erasure) and166                       dose-controlled sufficiency (steering)167```168169Steering dose-response at layer {doc5['config']['layer']} (baseline margin {doc5['baseline_margin']:+.2f}):170{steer_lines}171172All pre-registered gates passed at L12 for every fresh source: strict173monotonicity, halving at −2σ, random-direction specificity under the 25%174bound. Scope honesty: the HANDLE claim is single-layer (L12) and the175NECESSITY claim is band-level (v1); one model, one behavior. Runs #3 and #5176are the refusals that scoped this entry. Generality across sizes (expG) and177quantization (candidate_02) is untested and registered next.178""")179    errs = card.validate()180    if errs:181        print("CARD INVALID:", errs)182        return 1183    print(f"atlas entry written: {ENTRY.relative_to(ROOT)} (LEVEL 3)")184    return 0185186187if __name__ == "__main__":188    sys.exit(main())189