spb/modelmap Public License

1# =============================================================================2#  Project   : modelmap3#  File      : tests/test_stats_and_probes.py4#  Purpose   : Correctness tests for stats + probe modules (tiny references)5#  Author    : Simon-Pierre Boucher6#  Contact   : contact@spboucher.ai7#  Website   : https://modelmap.io8#  Created   : 2026-08-129#  Modified  : 2026-08-1210#  Platform  : macOS / Apple Silicon (arm64)11#  License   : All rights reserved (research code)12# =============================================================================13"""Charter §0.3: numerical probes get correctness tests against tiny14references — a probe must separate real structure from noise, FDR must15control false discoveries on known nulls, map cards must gate publication."""1617import numpy as np1819from modelmap.atlas.mapcard import MapCard20from modelmap.probes.linear import probe_with_control21from modelmap.stats.replication import bh_fdr, bootstrap_ci, jaccard, replication_rate222324def test_bootstrap_ci_covers_mean():25    rng = np.random.default_rng(1)26    point, lo, hi = bootstrap_ci(rng.normal(5.0, 1.0, 500))27    assert lo < 5.0 < hi and abs(point - 5.0) < 0.2282930def test_bh_fdr_null_and_signal():31    rng = np.random.default_rng(2)32    null_p = rng.uniform(0, 1, 1000)33    assert bh_fdr(null_p, q=0.05).mean() < 0.01  # near-zero false discoveries34    mixed = np.concatenate([null_p, np.full(50, 1e-8)])35    disc = bh_fdr(mixed, q=0.05)36    assert disc[-50:].all()  # all planted signals found373839def test_jaccard_and_replication():40    assert jaccard({1, 2}, {1, 2}) == 1.041    assert jaccard({1}, {2}) == 0.042    point, lo, hi = replication_rate([{1, 2, 3}, {2, 3, 4}, {1, 2, 3}])43    assert 0 < lo <= point <= hi <= 1444546def test_probe_separates_structure_from_noise():47    """On separable data: high selectivity. On pure noise labels: ~0."""48    rng = np.random.default_rng(3)49    n, d = 600, 3250    y = rng.integers(0, 2, n)51    x = rng.normal(0, 1, (n, d))52    x[:, 0] += 3.0 * y  # planted signal in one dimension53    r = probe_with_control(x, y, seed=0)54    assert r.task_accuracy > 0.955    assert r.selectivity > 0.35657    y_noise = rng.integers(0, 2, n)58    x_noise = rng.normal(0, 1, (n, d))59    r0 = probe_with_control(x_noise, y_noise, seed=0)60    assert abs(r0.selectivity) < 0.15  # noise floor: no fake structure616263def test_mapcard_gates_publication():64    card = MapCard()65    assert card.validate()  # empty card blocked66    card = MapCard(67        map_id="atlas/demo/probes/v0", map_type="probes", model_id="demo",68        model_hash="abc", quantization="fp16", commit="deadbeef",69        config="{}", created="2026-08-12", hardware_manifest={"chip": "test"},70        confidence_level=1, regenerate_command="make demo",71        seeds=[0, 1, 2], prompt_sets=["a#sha", "b#sha"],72        controls=["shuffled-labels"], replication_rate=0.8,73    )74    assert card.validate() == []75    card.seeds = [0]  # Level 1 with 1 seed must fail76    assert any("3 seeds" in e for e in card.validate())77