#!/usr/bin/env python3 # ============================================================================= # Project : modelmap # File : experiments/micro/expA_probe_reliability/implementation/make_mapcard_v2.py # Purpose : Build atlas/qwen3-0.6b-4bit/probes/v2 from expA run #2 results # Author : Simon-Pierre Boucher # Contact : contact@spboucher.ai # Website : https://modelmap.io # Created : 2026-08-12 # Modified : 2026-08-12 # Platform : macOS / Apple Silicon (arm64) # License : All rights reserved (research code) # ============================================================================= """Atlas entry v2: differential probe maps (real − twin) on structure-borne, token-balanced properties. Mixed outcome, published per property: agreement + arith_valid carry trained-model signal above the architecture prior (Level 1); word_order is flagged null-dominated.""" from __future__ import annotations import hashlib import json import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[4] sys.path.insert(0, str(ROOT / "src")) from modelmap.atlas.mapcard import MapCard ENTRY = ROOT / "atlas" / "qwen3-0.6b-4bit" / "probes" / "v2" MODEL_ID = "mlx-community/Qwen3-0.6B-4bit" PROPERTIES = ("word_order", "agreement", "arith_valid") def newest_run2() -> Path: for d in sorted((ROOT / "results" / "expA_probe_reliability").iterdir(), reverse=True): doc = json.loads((d / "results.json").read_text()) if doc.get("run") == 2: return d / "results.json" raise SystemExit("no run-2 results found") def model_hash() -> str: from mlx_lm.utils import hf_repo_to_path mp = Path(hf_repo_to_path(MODEL_ID)) h = hashlib.sha256() for f in sorted(mp.glob("*.safetensors")): h.update(f.read_bytes()) return h.hexdigest() def rows(doc, kind, name): keys = ("layer", "task_acc_mean", "task_acc_seed_sd", "selectivity_mean", "selectivity_ci", "fdr_significant") return [{k: r[k] for k in keys if k in r} for r in doc["results"][kind][name]["layers"]] def main() -> int: res_path = newest_run2() doc = json.loads(res_path.read_text()) summary = doc["summary"] ENTRY.mkdir(parents=True, exist_ok=True) map_doc = { "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai", "website": "https://modelmap.io", "map_type": "probes", "model_id": MODEL_ID, "design": "differential maps: real vs random-init twin, token-balanced classes", "properties": {}, "source_results": str(res_path.relative_to(ROOT)), } verdicts = { "word_order": "null-dominated (twin acc 0.96; surface statistics explain the map)", "agreement": "trained-model signal (real−twin sel > 0.10 on 25/28 layers, max +0.38)", "arith_valid": "trained-model signal (real acc 0.86–0.90 vs twin 0.56–0.58)", } for prop in PROPERTIES: map_doc["properties"][prop] = { "verdict": verdicts[prop], "summary": {k: v for k, v in summary.items() if k.startswith(prop)}, "per_layer": {s: rows(doc, "real", f"{prop}_{s}_mean") for s in ("A", "B")}, "per_layer_last": {s: rows(doc, "real", f"{prop}_{s}_last") for s in ("A", "B")}, "twin_null_per_layer_A": rows(doc, "twin", f"{prop}_A_mean"), "twin_null_per_layer_A_last": rows(doc, "twin", f"{prop}_A_last"), } (ENTRY / "map.json").write_text(json.dumps(map_doc, indent=2) + "\n") mean_repl = float(sum(summary[f"{p}_mean"]["replication_topk_A"] for p in PROPERTIES) / 3) mhash = model_hash() created = time.strftime("%Y-%m-%d", time.gmtime()) (ENTRY / "provenance.json").write_text(json.dumps({ "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai", "website": "https://modelmap.io", "model_id": MODEL_ID, "map_type": "probes", "version": "v2", "commit": doc["commit"], "model_hash": mhash, "config": doc["config"], "seed": doc["config"]["seeds"], "hardware_manifest": doc["manifest"], "created": created, "source_results": str(res_path.relative_to(ROOT)), }, indent=2) + "\n") promptsets = [f"{n}#{m['sha256'][:16]}" for n, m in doc["config"]["promptsets"]["files"].items()] card = MapCard( map_id="atlas/qwen3-0.6b-4bit/probes/v2", map_type="probes", model_id=MODEL_ID, model_hash=mhash, quantization="q4 (mlx)", commit=doc["commit"], config=str(res_path.relative_to(ROOT)), created=created, hardware_manifest=doc["manifest"], confidence_level=1, regenerate_command=( ".venv/bin/python benchmarks/promptsets/make_promptsets_v2.py && " ".venv/bin/python experiments/micro/expA_probe_reliability/implementation/benchmark_v2.py && " ".venv/bin/python experiments/micro/expA_probe_reliability/implementation/make_mapcard_v2.py"), seeds=list(doc["config"]["seeds"]), prompt_sets=promptsets, controls=["shuffled-label (every probe)", "random-init architecture twin", "BH-FDR q=0.05", "v1 positive control (ceiling check)", "class token-overlap certificates in promptset manifest"], replication_rate=round(mean_repl, 4), featurizer_class="natural-basis (mean-pooled + last-token residual)", intervention_protocol="none (observational map — Level 1 by design)", negative_result=False, notes="DIFFERENTIAL map (real minus random-init twin), per the doctrine adopted " "after v1. Mixed outcome by property: agreement and arith_valid carry " "trained-model signal above the architecture prior; word_order is " "null-dominated and flagged as such. Strict twin gate (<0.05) still fails " "on word_order/agreement — only differential claims are published.", ) (ENTRY / "mapcard.json").write_text(card.to_json()) lines = "\n".join( f"- {p}: {verdicts[p]}; maxAcc A (mean-pool) " f"{summary[f'{p}_mean']['max_task_acc_A']:.3f}, twin acc " f"{summary[f'{p}_mean']['twin_max_acc']:.3f}, signal layers " f"{summary[f'{p}_mean']['layers_real_minus_twin_gt_0.10']}/28" for p in PROPERTIES) (ENTRY / "confidence.md").write_text(f"""--- project: modelmap document: qwen3-0.6b-4bit/probes/v2 — confidence author: Simon-Pierre Boucher contact: contact@spboucher.ai website: https://modelmap.io created: {created} status: reviewed --- # Confidence — qwen3-0.6b-4bit / probes / v2 ```text Level : 1 Seeds : {len(doc['config']['seeds'])} Prompt sets: {len(promptsets)} (token-balanced, structure-borne; overlap certificates in manifest) Methods in agreement : 1 (linear probes only — Level 2 requires a second method) Causal verification : none (observational; Level 3 requires intervention) ``` Per-property verdicts (differential real−twin, mean pooling): {lines} Published claims are DIFFERENTIAL only (real minus random-init twin), per the doctrine adopted after v1's validity-gate failure. The strict twin gate (selectivity < 0.05) still fails on word_order and agreement — the twin extracts real surface signal from tokenization statistics — so raw probe accuracies are never cited as evidence of learned structure. What survives: agreement and arith_valid show layer-resolved trained-model signal (Level 1, correlational; 5 seeds × 2 sets, controls listed above). """) errs = card.validate() if errs: print("CARD INVALID:", errs) return 1 print(f"atlas entry written: {ENTRY.relative_to(ROOT)} (Level 1, repl {mean_repl:.2f})") return 0 if __name__ == "__main__": sys.exit(main())