spb/localvm-research Public License
Running LLMs larger than memory on a consumer Mac — falsification-driven research: margin-gated deferred refinement, out-of-core verification on Apple Silicon. TR-01 published.
Python 63.2%
JavaScript 23.5%
CSS 11.8%
Shell 0.9%
Makefile 0.5%
1#!/usr/bin/env python32# =============================================================================3# Project : localvm-research4# File : experiments/micro/expB_token_stability/benchmark.py5# Purpose : Temporal stability of per-token important FFN block sets6# (consumes expA's block-energy trace)7# Author : Simon-Pierre Boucher8# Contact : contact@spboucher.ai9# Created : 2026-08-1210# Modified : 2026-08-1211# Platform : macOS / Apple Silicon (arm64)12# License : All rights reserved (research code)13# =============================================================================14"""Experiment B — stability across consecutive tokens (charter §9.B).1516Usage:17 .venv/bin/python benchmark.py [--trace <path/to/block_energy_trace.npz>]18 [--target 0.95]19"""2021from __future__ import annotations2223import argparse24import glob25import json26import sys27from datetime import datetime, timezone28from pathlib import Path2930import numpy as np3132REPO_ROOT = Path(__file__).resolve().parents[3]33sys.path.insert(0, str(REPO_ROOT / "benchmarks"))34from hardware_manifest import collect_manifest # noqa: E4023536DELTAS = [1, 2, 4, 8, 16, 32]37WINDOWS = [8, 32, 128]383940def top_sets(energy: np.ndarray, target: float) -> list[np.ndarray]:41 """energy (T, B) → per-position boolean mask of the smallest block set42 covering `target` of the energy."""43 T, B = energy.shape44 order = np.argsort(energy, axis=1)[:, ::-1]45 srt = np.take_along_axis(energy, order, axis=1)46 csum = np.cumsum(srt, axis=1)47 total = csum[:, -1:] + 1e-1248 kneed = np.argmax(csum / total >= target, axis=1) + 149 masks = np.zeros((T, B), dtype=bool)50 for t in range(T):51 masks[t, order[t, : kneed[t]]] = True52 return masks535455def jaccard(a: np.ndarray, b: np.ndarray) -> float:56 inter = np.logical_and(a, b).sum()57 union = np.logical_or(a, b).sum()58 return float(inter / union) if union else 1.0596061def main() -> None:62 ap = argparse.ArgumentParser()63 ap.add_argument("--trace", default=None)64 ap.add_argument("--target", type=float, default=0.95)65 ap.add_argument("--agg-factor", type=int, default=4,66 help="aggregate trace blocks by this factor (16-neuron trace -> 64-neuron sets)")67 args = ap.parse_args()68 trace_path = args.trace or sorted(69 glob.glob(str(REPO_ROOT / "results/expA_weight_concentration/*/block_energy_trace.npz")))[-1]70 z = np.load(trace_path, allow_pickle=False)71 blocks = z["blocks"].astype(np.float32) # (P, L, B)72 if args.agg_factor > 1:73 P0, L0, B0 = blocks.shape74 blocks = blocks.reshape(P0, L0, B0 // args.agg_factor, args.agg_factor).sum(axis=-1)75 domains = z["domains"]76 traj_id = z["traj_id"]77 P, L, B = blocks.shape78 print(f"trace {trace_path}: {blocks.shape}", flush=True)7980 jac = {d: [] for d in DELTAS}81 jac_random = []82 union_frac = {w: [] for w in WINDOWS}83 per_layer_j1 = [[] for _ in range(L)]84 traj_masks_union = {} # (traj, layer) -> aggregate mask, for expC-lite85 traj_domain = {}8687 rng = np.random.default_rng(7)88 for ti in np.unique(traj_id):89 sel = traj_id == ti90 traj_domain[int(ti)] = str(domains[sel][0])91 for li in range(L):92 masks = top_sets(blocks[sel, li, :], args.target)93 T = masks.shape[0]94 for d in DELTAS:95 if T > d:96 vals = [jaccard(masks[t], masks[t + d]) for t in range(T - d)]97 jac[d].extend(vals)98 if d == 1:99 per_layer_j1[li].extend(vals)100 # random-set null at matched sizes (δ=1 pairs)101 sizes = masks.sum(axis=1)102 for t in range(min(T - 1, 8)):103 a = np.zeros(B, bool); a[rng.choice(B, sizes[t], replace=False)] = True104 b = np.zeros(B, bool); b[rng.choice(B, sizes[t + 1], replace=False)] = True105 jac_random.append(jaccard(a, b))106 for w in WINDOWS:107 for s in range(0, T - w + 1, w):108 union_frac[w].append(float(masks[s:s + w].any(axis=0).mean()))109 traj_masks_union[(int(ti), li)] = masks.any(axis=0)110111 # expC-lite: within- vs across-domain overlap of per-trajectory unions112 tids = sorted(traj_domain)113 within, across = [], []114 for i in range(len(tids)):115 for j in range(i + 1, len(tids)):116 v = np.mean([jaccard(traj_masks_union[(tids[i], li)], traj_masks_union[(tids[j], li)])117 for li in range(0, L, 4)])118 (within if traj_domain[tids[i]] == traj_domain[tids[j]] else across).append(v)119120 result = {121 "jaccard_by_delta": {str(d): {"mean": float(np.mean(v)), "std": float(np.std(v)), "n": len(v)}122 for d, v in jac.items()},123 "jaccard_random_null": {"mean": float(np.mean(jac_random)), "std": float(np.std(jac_random))},124 "union_working_set_frac_by_window": {str(w): {"mean": float(np.mean(v)), "std": float(np.std(v))}125 for w, v in union_frac.items()},126 "per_layer_jaccard1_mean": [float(np.mean(v)) for v in per_layer_j1],127 "expC_lite_domain_locality": {128 "within_domain_union_jaccard": float(np.mean(within)),129 "across_domain_union_jaccard": float(np.mean(across)),130 },131 "mean_topset_frac": float(np.mean([m.mean() for m in132 [top_sets(blocks[traj_id == t, li, :], args.target)133 for t in np.unique(traj_id)[:4] for li in (0, L // 2, L - 1)]])),134 }135 ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")136 out_dir = REPO_ROOT / "results" / "expB_token_stability" / ts137 out_dir.mkdir(parents=True)138 (out_dir / "results.json").write_text(json.dumps({139 "experiment": "expB_token_stability",140 "author": "Simon-Pierre Boucher",141 "contact": "contact@spboucher.ai",142 "manifest": collect_manifest(),143 "config": {"trace": str(trace_path), "target": args.target},144 **result,145 }, indent=2))146 print(json.dumps(result["jaccard_by_delta"], indent=1))147 print("random null:", result["jaccard_random_null"])148 print("union by window:", json.dumps(result["union_working_set_frac_by_window"]))149 print("domain locality:", result["expC_lite_domain_locality"])150 print(f"\nwrote {out_dir / 'results.json'}")151152153if __name__ == "__main__":154 main()155