expH: SSD feasibility benchmark implementation + hypothesis

Measures internal NVMe read throughput: block sizes 4KB-4MB, random vs
sequential, F_NOCACHE vs warm page cache, 1/4/8 threads, and key cells
under concurrent MLX Metal matmul load. Embeds hardware manifest.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Simon-Pierre Boucher committed 6 h ago (Aug 12, 2026) parent c709367

Showing 2 changed files with +258 and −14

modified experiments/micro/expH_ssd_feasibility/benchmark.py +232 −7

		`@@ -1,7 +1,9 @@`
	1	+#!/usr/bin/env python3
1	2	# =============================================================================
2	3	# Project : localvm-research
3	4	# File : experiments/micro/expH_ssd_feasibility/benchmark.py
4		−# Purpose : Benchmark runner: SSD feasibility on macOS/APFS: measured random/sequential reads, F_NOCACHE, concurrent Metal compute
	5	+# Purpose : Measure real SSD read behavior on macOS/APFS (F_NOCACHE, block
	6	+# sizes, random vs sequential, threads, concurrent Metal load)
5	7	# Author : Simon-Pierre Boucher
6	8	# Contact : contact@spboucher.ai
7	9	# Created : 2026-08-11
		`@@ -9,24 +11,247 @@`
9	11	# Platform : macOS / Apple Silicon (arm64)
10	12	# License : All rights reserved (research code)
11	13	# =============================================================================
	14	+"""Experiment H — SSD feasibility (charter §9.H).
12	15
13		−"""Benchmark entry point for expH_ssd_feasibility.
	16	+Measures actual (not theoretical) read throughput of the internal Apple NVMe
	17	+at block sizes 4 KB … 4 MB, sequential vs random, with and without the page
	18	+cache (F_NOCACHE — macOS has no O_DIRECT), 1/4/8 threads, and optionally
	19	+under concurrent Metal GPU compute (MLX matmul loop).
14	20
15		−Must embed the hardware manifest in all result output
16		−(see benchmarks/hardware_manifest.py) and write results to
17		−results/expH_ssd_feasibility/<timestamp>/.
	21	+Writes results JSON (with embedded hardware manifest) to
	22	+results/expH_ssd_feasibility/<timestamp>/results.json.
	23	+
	24	+Usage:
	25	+ python3 benchmark.py [--file-gib 8] [--repeats 3] [--quick]
18	26	"""
19	27
	28	+from __future__ import annotations
	29	+
	30	+import argparse
	31	+import fcntl
	32	+import json
	33	+import os
	34	+import statistics
20	35	import sys
	36	+import threading
	37	+import time
	38	+from datetime import datetime, timezone
21	39	from pathlib import Path
22	40
23		−sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "benchmarks"))
	41	+REPO_ROOT = Path(__file__).resolve().parents[3]
	42	+sys.path.insert(0, str(REPO_ROOT / "benchmarks"))
24	43	from hardware_manifest import collect_manifest # noqa: E402
25	44
	45	+F_NOCACHE = 48 # from <sys/fcntl.h> on macOS
	46	+BLOCK_SIZES = [4 << 10, 16 << 10, 64 << 10, 256 << 10, 1 << 20, 4 << 20]
	47	+THREAD_COUNTS = [1, 4, 8]
	48	+
	49	+
	50	+def create_test_file(path: Path, size_bytes: int) -> None:
	51	+ """Write an incompressible test file (APFS compresses nothing here, but
	52	+ avoid all-zero data so no transparent optimization can help)."""
	53	+ if path.exists() and path.stat().st_size == size_bytes:
	54	+ return
	55	+ import numpy as np
	56	+
	57	+ rng = np.random.default_rng(42)
	58	+ chunk = rng.integers(0, 256, size=64 << 20, dtype=np.uint8).tobytes()
	59	+ written = 0
	60	+ with open(path, "wb") as f:
	61	+ while written < size_bytes:
	62	+ n = min(len(chunk), size_bytes - written)
	63	+ f.write(chunk[:n])
	64	+ written += n
	65	+ f.flush()
	66	+ os.fsync(f.fileno())
	67	+
	68	+
	69	+def _reader(path: Path, offsets: list[int], block: int, nocache: bool,
	70	+ out: dict, idx: int) -> None:
	71	+ fd = os.open(path, os.O_RDONLY)
	72	+ try:
	73	+ if nocache:
	74	+ fcntl.fcntl(fd, F_NOCACHE, 1)
	75	+ total = 0
	76	+ t0 = time.perf_counter()
	77	+ for off in offsets:
	78	+ total += len(os.pread(fd, block, off))
	79	+ dt = time.perf_counter() - t0
	80	+ out[idx] = (total, dt)
	81	+ finally:
	82	+ os.close(fd)
	83	+
	84	+
	85	+def measure(path: Path, file_size: int, block: int, pattern: str,
	86	+ nocache: bool, threads: int, budget_bytes: int, seed: int) -> dict:
	87	+ """One measurement cell. Returns MB/s and IOPS."""
	88	+ import random as _random
	89	+
	90	+ n_blocks_total = max(1, budget_bytes // block)
	91	+ max_off = file_size - block
	92	+ rng = _random.Random(seed)
	93	+ if pattern == "random":
	94	+ all_offsets = [rng.randrange(0, max_off // block) * block for _ in range(n_blocks_total)]
	95	+ else:
	96	+ stride = max(block, (max_off // n_blocks_total) // block * block) if n_blocks_total > 1 else block
	97	+ all_offsets = [(i * block) % (max_off + 1) for i in range(n_blocks_total)]
	98	+ del stride
	99	+
	100	+ per_thread = [all_offsets[i::threads] for i in range(threads)]
	101	+ out: dict = {}
	102	+ ts = [
	103	+ threading.Thread(target=_reader, args=(path, per_thread[i], block, nocache, out, i))
	104	+ for i in range(threads)
	105	+ ]
	106	+ t0 = time.perf_counter()
	107	+ for t in ts:
	108	+ t.start()
	109	+ for t in ts:
	110	+ t.join()
	111	+ wall = time.perf_counter() - t0
	112	+ total_bytes = sum(v[0] for v in out.values())
	113	+ return {
	114	+ "mb_per_s": total_bytes / wall / 1e6,
	115	+ "iops": total_bytes / block / wall,
	116	+ "bytes": total_bytes,
	117	+ "wall_s": wall,
	118	+ }
	119	+
	120	+
	121	+def warm_cache(path: Path) -> None:
	122	+ with open(path, "rb") as f:
	123	+ while f.read(64 << 20):
	124	+ pass
	125	+
	126	+
	127	+class GpuLoad:
	128	+ """Background MLX matmul loop to contend for unified memory bandwidth."""
	129	+
	130	+ def __init__(self) -> None:
	131	+ self.stop = threading.Event()
	132	+ self.iterations = 0
	133	+ self.thread: threading.Thread \| None = None
	134	+ self.available = False
	135	+ try:
	136	+ import mlx.core as mx
	137	+
	138	+ self._mx = mx
	139	+ self.available = True
	140	+ except ImportError:
	141	+ pass
	142	+
	143	+ def _loop(self) -> None:
	144	+ mx = self._mx
	145	+ a = mx.random.normal((4096, 4096), dtype=mx.float16)
	146	+ b = mx.random.normal((4096, 4096), dtype=mx.float16)
	147	+ while not self.stop.is_set():
	148	+ c = mx.matmul(a, b)
	149	+ mx.eval(c)
	150	+ self.iterations += 1
	151	+
	152	+ def __enter__(self) -> "GpuLoad":
	153	+ if self.available:
	154	+ self.thread = threading.Thread(target=self._loop, daemon=True)
	155	+ self.thread.start()
	156	+ time.sleep(0.5) # let the GPU ramp
	157	+ return self
	158	+
	159	+ def __exit__(self, *exc: object) -> None:
	160	+ self.stop.set()
	161	+ if self.thread:
	162	+ self.thread.join(timeout=10)
	163	+
26	164
27	165	def main() -> None:
	166	+ ap = argparse.ArgumentParser()
	167	+ ap.add_argument("--file-gib", type=float, default=8.0)
	168	+ ap.add_argument("--repeats", type=int, default=3)
	169	+ ap.add_argument("--budget-mib", type=int, default=1024,
	170	+ help="bytes read per measurement cell")
	171	+ ap.add_argument("--quick", action="store_true",
	172	+ help="1 repeat, 512 MiB budget, threads {1,8}")
	173	+ args = ap.parse_args()
	174	+ if args.quick:
	175	+ args.repeats, args.budget_mib = 1, 512
	176	+ thread_counts = [1, 8]
	177	+ else:
	178	+ thread_counts = THREAD_COUNTS
	179	+
	180	+ file_size = int(args.file_gib * (1 << 30))
	181	+ test_file = Path(__file__).parent / "results" / "testfile.bin"
	182	+ test_file.parent.mkdir(exist_ok=True)
	183	+ print(f"creating {args.file_gib} GiB test file (once)…", flush=True)
	184	+ create_test_file(test_file, file_size)
	185	+
28	186	manifest = collect_manifest()
29		− raise NotImplementedError("experiment not yet implemented")
	187	+ results: list[dict] = []
	188	+ budget = args.budget_mib << 20
	189	+
	190	+ def run_cells(gpu_loaded: bool) -> None:
	191	+ for nocache in (True, False):
	192	+ if not nocache:
	193	+ warm_cache(test_file)
	194	+ for pattern in ("random", "sequential"):
	195	+ for block in BLOCK_SIZES:
	196	+ for threads in thread_counts:
	197	+ if gpu_loaded and (pattern != "random" or not nocache or threads != 8):
	198	+ continue # GPU-load condition: key cells only
	199	+ runs = [
	200	+ measure(test_file, file_size, block, pattern, nocache,
	201	+ threads, budget, seed=1000 + r)
	202	+ for r in range(args.repeats)
	203	+ ]
	204	+ mbps = [r["mb_per_s"] for r in runs]
	205	+ cell = {
	206	+ "block_bytes": block,
	207	+ "pattern": pattern,
	208	+ "nocache": nocache,
	209	+ "threads": threads,
	210	+ "gpu_load": gpu_loaded,
	211	+ "repeats": args.repeats,
	212	+ "mb_per_s_mean": statistics.mean(mbps),
	213	+ "mb_per_s_median": statistics.median(mbps),
	214	+ "mb_per_s_std": statistics.stdev(mbps) if len(mbps) > 1 else 0.0,
	215	+ "iops_mean": statistics.mean(r["iops"] for r in runs),
	216	+ }
	217	+ results.append(cell)
	218	+ print(
	219	+ f"{'GPU+' if gpu_loaded else ' '}"
	220	+ f"{pattern:>10} {block >> 10:>5} KiB "
	221	+ f"nocache={int(nocache)} t={threads}: "
	222	+ f"{cell['mb_per_s_mean']:9.1f} MB/s",
	223	+ flush=True,
	224	+ )
	225	+
	226	+ run_cells(gpu_loaded=False)
	227	+
	228	+ gpu = GpuLoad()
	229	+ gpu_iters = 0
	230	+ if gpu.available:
	231	+ print("re-running key cells under concurrent Metal (MLX) matmul load…", flush=True)
	232	+ with gpu:
	233	+ run_cells(gpu_loaded=True)
	234	+ gpu_iters = gpu.iterations
	235	+ else:
	236	+ print("MLX not available — skipping GPU-load condition", flush=True)
	237	+
	238	+ ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
	239	+ out_dir = REPO_ROOT / "results" / "expH_ssd_feasibility" / ts
	240	+ out_dir.mkdir(parents=True)
	241	+ payload = {
	242	+ "experiment": "expH_ssd_feasibility",
	243	+ "author": "Simon-Pierre Boucher",
	244	+ "contact": "contact@spboucher.ai",
	245	+ "manifest": manifest,
	246	+ "config": vars(args),
	247	+ "test_file_bytes": file_size,
	248	+ "gpu_load_matmul_iterations": gpu_iters,
	249	+ "thermal_level_after": collect_manifest()["thermal_level_at_collect"],
	250	+ "cells": results,
	251	+ }
	252	+ out_path = out_dir / "results.json"
	253	+ out_path.write_text(json.dumps(payload, indent=2))
	254	+ print(f"\nwrote {out_path}")
30	255
31	256
32	257	if __name__ == "__main__":

modified experiments/micro/expH_ssd_feasibility/hypothesis.md +26 −7

		`@@ -11,23 +11,42 @@ status: draft`
11	11
12	12	```text
13	13	Hypothesis
14		− <what we believe and why>
	14	+ The internal Apple NVMe SSD (AP2048Z, 2 TB) sustains enough random read
	15	+ bandwidth at weight-block-sized granularities (256 KB–4 MB) to stream a
	16	+ meaningful fraction of model weights per token: we expect ≥ 2 GB/s random
	17	+ reads at ≥ 1 MB blocks with F_NOCACHE (true storage path), degrading
	18	+ sharply below 64 KB, and only modest degradation under concurrent Metal
	19	+ GPU compute (unified memory contention is expected to be small relative
	20	+ to SSD ceiling).
15	21
16	22	Falsification criterion
17		− <the concrete measurable outcome that would prove this wrong>
	23	+ If uncached random reads at 1 MB blocks sustain < 500 MB/s, or collapse
	24	+ by > 50% under concurrent GPU matmul load, SSD weight streaming cannot
	25	+ supply even ~0.5 GB/token at interactive rates and candidate designs
	26	+ must not assume per-token SSD reads on this class of hardware.
18	27
19	28	Method
20		− <exact procedure, model(s), data, seeds, measurement points>
	29	+ Create an 8 GiB incompressible test file on the internal APFS volume.
	30	+ Measure read throughput at block sizes {4 KB, 16 KB, 64 KB, 256 KB,
	31	+ 1 MB, 4 MB} × {sequential, random} × {F_NOCACHE on, off/warm cache} ×
	32	+ {1, 4, 8 reader threads}. Cold-cache condition enforced via F_NOCACHE
	33	+ (macOS has no O_DIRECT); warm condition by pre-reading the file.
	34	+ Repeat each cell 3 times, report mean/median/std. Then repeat the key
	35	+ uncached random cells while an MLX fp16 4096×4096 matmul loop saturates
	36	+ the GPU. Record hardware manifest and thermal level before/after.
21	37
22	38	Baseline
23		− <what this is compared against — no straw men>
	39	+ Apple's advertised sequential throughput for this SSD class (~5-7 GB/s)
	40	+ and the sequential-read measurement from the same harness (internal
	41	+ baseline; no straw men — we compare random vs our own sequential).
24	42
25	43	Result
26		− <filled after the run: numbers, with mean/median/std and run count>
	44	+ <filled after the run>
27	45
28	46	Interpretation
29		− <what the numbers mean; alternative explanations considered>
	47	+ <filled after the run>
30	48
31	49	Next experiment
32		− <the most informative follow-up given this result>
	50	+ <filled after the run; likely expE (partial GEMM) or a prefetch-overlap
	51	+ probe depending on where the knee of the curve sits>
33	52	```
34	53