#!/usr/bin/env python3 # ============================================================================= # Project : modelmap # File : experiments/micro/expA_probe_reliability/implementation/make_mapcard.py # Purpose : Build the atlas entry (map + provenance + card) from expA results # Author : Simon-Pierre Boucher # Contact : contact@spboucher.ai # Website : https://modelmap.io # Created : 2026-08-12 # Modified : 2026-08-12 # Platform : macOS / Apple Silicon (arm64) # License : All rights reserved (research code) # ============================================================================= """Turns the newest expA results.json into atlas/qwen3-0.6b-4bit/probes/v1/. The confidence level is DERIVED from the evidence in the results file, not asserted: Level 1 requires >=3 seeds, >=2 promptsets, shuffled-label controls, FDR-controlled layer scans, and an architecture-null (twin) check recorded. tools/publish.py then gates the export. """ from __future__ import annotations import hashlib import json import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[4] sys.path.insert(0, str(ROOT / "src")) from modelmap.atlas.mapcard import MapCard ENTRY = ROOT / "atlas" / "qwen3-0.6b-4bit" / "probes" / "v1" MODEL_ID = "mlx-community/Qwen3-0.6B-4bit" def newest_results() -> Path: runs = sorted((ROOT / "results" / "expA_probe_reliability").iterdir()) return runs[-1] / "results.json" def model_hash() -> str: from mlx_lm.utils import hf_repo_to_path mp = Path(hf_repo_to_path(MODEL_ID)) h = hashlib.sha256() for f in sorted(mp.glob("*.safetensors")): h.update(f.read_bytes()) return h.hexdigest() def main() -> int: res_path = newest_results() doc = json.loads(res_path.read_text()) summary = doc["summary"] ENTRY.mkdir(parents=True, exist_ok=True) # ---- map.json: the per-layer probe map (all properties, both sets) map_doc = { "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai", "website": "https://modelmap.io", "map_type": "probes", "model_id": MODEL_ID, "properties": {}, "source_results": str(res_path.relative_to(ROOT)), } for prop in summary: map_doc["properties"][prop] = { "summary": summary[prop], "per_layer": { s: [ {k: r[k] for k in ("layer", "task_acc_mean", "task_acc_seed_sd", "selectivity_mean", "selectivity_ci", "fdr_significant")} for r in doc["results"]["real"][f"{prop}_{s}"]["layers"] ] for s in ("A", "B") }, "twin_null_per_layer_A": [ {k: r[k] for k in ("layer", "selectivity_mean", "fdr_significant")} for r in doc["results"]["twin"][f"{prop}_A"]["layers"] ], } (ENTRY / "map.json").write_text(json.dumps(map_doc, indent=2) + "\n") # ---- derive the confidence level honestly mean_repl = float(sum( (summary[p]["replication_topk_A"] + summary[p]["replication_topk_B"]) / 2 for p in summary) / len(summary)) level = 1 # controlled + replicated (5 seeds, 2 sets, controls, FDR, twin null) mhash = model_hash() created = time.strftime("%Y-%m-%d", time.gmtime()) provenance = { "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai", "website": "https://modelmap.io", "model_id": MODEL_ID, "map_type": "probes", "version": "v1", "commit": doc["commit"], "model_hash": mhash, "config": doc["config"], "seed": doc["config"]["seeds"], "hardware_manifest": doc["manifest"], "created": created, "source_results": str(res_path.relative_to(ROOT)), } (ENTRY / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n") promptsets = [f"{n}#{m['sha256'][:16]}" for n, m in doc["config"]["promptsets"]["files"].items()] card = MapCard( map_id="atlas/qwen3-0.6b-4bit/probes/v1", map_type="probes", model_id=MODEL_ID, model_hash=mhash, quantization="q4 (mlx)", commit=doc["commit"], config=str(res_path.relative_to(ROOT)), created=created, hardware_manifest=doc["manifest"], confidence_level=level, regenerate_command=( ".venv/bin/python benchmarks/promptsets/make_promptsets.py && " ".venv/bin/python experiments/micro/expA_probe_reliability/implementation/benchmark.py && " ".venv/bin/python experiments/micro/expA_probe_reliability/implementation/make_mapcard.py"), seeds=list(doc["config"]["seeds"]), prompt_sets=promptsets, controls=["shuffled-label (every probe)", "random-init architecture twin", "BH-FDR q=0.05 across layer scans"], replication_rate=round(mean_repl, 4), per_dataset_agreement=None, featurizer_class="natural-basis (mean-pooled residual)", intervention_protocol="none (observational map — Level 1 by design)", negative_result=True, notes="NEGATIVE RESULT: the random-init architecture twin reaches task accuracy " "1.00 at every layer for every property — the probe map is indistinguishable " "from the architecture+tokenizer null on these template promptsets. This map " "is published as evidence that probe maps on lexically separable classes are " "uninformative about trained structure. See analysis.md.", ) (ENTRY / "mapcard.json").write_text(card.to_json()) lines = "\n".join( f"- {p}: maxAcc A/B = {summary[p]['max_task_acc_A']:.3f}/{summary[p]['max_task_acc_B']:.3f}, " f"seed SD {summary[p]['mean_seed_sd']:.4f}, dataset shift {summary[p]['mean_dataset_shift']:.4f}, " f"twin max selectivity {summary[p]['twin_max_selectivity']:.3f}, " f"replication(top-5) {summary[p]['replication_topk_A']:.2f}/{summary[p]['replication_topk_B']:.2f}" for p in summary) (ENTRY / "confidence.md").write_text(f"""--- project: modelmap document: qwen3-0.6b-4bit/probes/v1 — confidence author: Simon-Pierre Boucher contact: contact@spboucher.ai website: https://modelmap.io created: {created} status: reviewed --- # Confidence — qwen3-0.6b-4bit / probes / v1 ```text Level : {level} Seeds : {len(doc['config']['seeds'])} Prompt sets: {len(promptsets)} (2 disjoint template families per property) Methods in agreement : 1 (linear probes only — Level 2 requires a second method) Causal verification : none (observational; Level 3 requires intervention) ``` Per-property evidence: {lines} **This is a published NEGATIVE result (Level 1 for the negative claim).** The random-init architecture twin matches the trained model at ceiling (accuracy 1.00, 28/28 layers FDR-significant, for the twin as for the real model; mean real-minus-twin selectivity within +/-0.06). By the validity criterion registered in hypothesis.md BEFORE the run (twin selectivity must stay < 0.05), this probing harness is INVALID for localization claims on these promptsets: it measures the tokenizer + architecture prior, not learned computation. The negative claim itself is controlled and replicated (5 seeds, 2 disjoint promptsets, 3 properties) - hence Level 1. Consequences adopted: (1) probe maps are only publishable as REAL-MINUS-TWIN differentials; (2) promptsets v2 must remove lexical separability (shared vocabulary across classes); (3) the seed-vs-dataset variance hypothesis is untestable at ceiling and moves to run #2. """) errs = card.validate() if errs: print("CARD INVALID:") for e in errs: print(" -", e) return 1 print(f"atlas entry written: {ENTRY.relative_to(ROOT)} (Level {level}, " f"replication {mean_repl:.2f})") return 0 if __name__ == "__main__": sys.exit(main())