SPB Git

spb/localvm-research Public License

Running LLMs larger than memory on a consumer Mac — falsification-driven research: margin-gated deferred refinement, out-of-core verification on Apple Silicon. TR-01 published.

Python 63.2% JavaScript 23.5% CSS 11.8% Shell 0.9% Makefile 0.5%

expD: progressive residual-ladder benchmark + hypothesis; shared decision_stats lib

Residual ladders 3/3+3/3+3+3 and 4/4+4 bits (affine g64), teacher-forced
convergence, two-tier margin-gated policies, hidden-state error by depth.
Shared utilities extracted to src/localvm/quality/decision_stats.py.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Simon-Pierre Boucher committed 4 h ago (Aug 12, 2026) parent 42c7b3d

Showing 3 changed files with +362 and −19

modified experiments/micro/expD_progressive_reconstruction/benchmark.py +222 −11
@@ -1,32 +1,243 @@
1 +#!/usr/bin/env python3
1 2 # =============================================================================
2 3 # Project : localvm-research
3 4 # File : experiments/micro/expD_progressive_reconstruction/benchmark.py
4 −# Purpose : Benchmark runner: Progressive weight reconstruction: convergence of hidden-state/logit/decision error vs residual depth
5 +# Purpose : Residual-ladder progressive weight reconstruction — decision and
6 +# hidden-state convergence vs cumulative bits (candidate C1 math)
5 7 # Author : Simon-Pierre Boucher
6 8 # Contact : contact@spboucher.ai
7 −# Created : 2026-08-11
8 −# Modified : 2026-08-11
9 −# Platform : macOS / Apple Silicon (arm64)
9 +# Created : 2026-08-12
10 +# Modified : 2026-08-12
11 +# Platform : macOS / Apple Silicon (arm64) — MLX / Metal
10 12 # License : All rights reserved (research code)
11 13 # =============================================================================
14 +"""Experiment D — progressive weight reconstruction (charter §9.D).
12 15
13 −"""Benchmark entry point for expD_progressive_reconstruction.
16 +Builds base+residual affine-quantized ladders (3/3+3/3+3+3 and 4/4+4 bits),
17 +teacher-forces each cumulative stage over reference greedy trajectories, and
18 +measures decision convergence, two-tier margin-gated policies, and
19 +hidden-state error at several depths.
14 20
15 −Must embed the hardware manifest in all result output
16 −(see benchmarks/hardware_manifest.py) and write results to
17 −results/expD_progressive_reconstruction/<timestamp>/.
21 +Usage:
22 + .venv/bin/python benchmark.py [--model mlx-community/Qwen3-1.7B-bf16]
23 + [--gen-tokens 128] [--per-domain 8]
18 24 """
19 25
26 +from __future__ import annotations
27 +
28 +import argparse
29 +import json
20 30 import sys
31 +import time
32 +from datetime import datetime, timezone
21 33 from pathlib import Path
22 34
23 −sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "benchmarks"))
35 +import mlx.core as mx
36 +import mlx.nn as nn
37 +import numpy as np
38 +from mlx_lm import load
39 +
40 +REPO_ROOT = Path(__file__).resolve().parents[3]
41 +sys.path.insert(0, str(REPO_ROOT / "benchmarks"))
42 +sys.path.insert(0, str(REPO_ROOT / "src"))
24 43 from hardware_manifest import collect_manifest # noqa: E402
44 +from localvm.quality.decision_stats import ( # noqa: E402
45 + auroc, escalation_curve, greedy_generate, kl_ref_vs, teacher_forced_stats,
46 +)
47 +
48 +GROUP = 64
49 +
50 +
51 +def quantizable(m) -> bool:
52 + return isinstance(m, nn.Linear) and m.weight.shape[-1] % GROUP == 0
53 +
54 +
55 +def residual_ladder_weights(model, ladder: list[int]) -> list[dict[str, mx.array]]:
56 + """For each quantizable Linear, build cumulative dequantized weights for
57 + each stage of `ladder` (bits per stage). Returns a list (one per stage) of
58 + {param_path: bf16 weight} replacements. Memory: one bf16 copy per stage
59 + per layer is materialized lazily at apply time; here we keep the per-stage
60 + cumulative tensors (float32 accumulation, cast to bf16)."""
61 + stages = [dict() for _ in ladder]
62 + for path, module in model.named_modules():
63 + if not quantizable(module):
64 + continue
65 + w = module.weight.astype(mx.float32)
66 + acc = mx.zeros_like(w)
67 + err = w
68 + for k, bits in enumerate(ladder):
69 + qw, scales, biases = mx.quantize(err, group_size=GROUP, bits=bits)
70 + deq = mx.dequantize(qw, scales, biases, group_size=GROUP, bits=bits)
71 + acc = acc + deq
72 + err = w - acc
73 + stages[k][path] = acc.astype(mx.bfloat16)
74 + mx.eval(stages[k][path])
75 + return stages
76 +
77 +
78 +def apply_weights(model, replacement: dict[str, mx.array]) -> dict[str, mx.array]:
79 + """Swap Linear weights in place; returns the originals for restoration."""
80 + originals = {}
81 + for path, module in model.named_modules():
82 + if path in replacement:
83 + originals[path] = module.weight
84 + module.weight = replacement[path]
85 + return originals
86 +
87 +
88 +def hidden_state_errors(model, ref_hidden: dict, full_ids: list[int], start: int,
89 + depths: list[int]) -> dict[int, float]:
90 + """Relative L2 error of hidden states vs reference at given layer indices."""
91 + h = capture_hidden(model, full_ids, start, depths)
92 + out = {}
93 + for d in depths:
94 + r, q = ref_hidden[d], h[d]
95 + out[d] = float(np.linalg.norm(q - r) / (np.linalg.norm(r) + 1e-9))
96 + return out
97 +
98 +
99 +def capture_hidden(model, full_ids: list[int], start: int, depths: list[int]) -> dict:
100 + """Hidden states (post-layer) at selected depths for predicted positions.
101 + Replicates the inner transformer loop manually (instance-level __call__
102 + monkey-patching does not intercept Python's type-level dunder dispatch)."""
103 + from mlx_lm.models.base import create_attention_mask
104 +
105 + inner = model.model
106 + h = inner.embed_tokens(mx.array(full_ids)[None])
107 + mask = create_attention_mask(h, None)
108 + result = {}
109 + for i, layer in enumerate(inner.layers):
110 + h = layer(h, mask, cache=None)
111 + if i in depths:
112 + t = h[0, start - 1 : len(full_ids) - 1].astype(mx.float32)
113 + mx.eval(t)
114 + result[i] = np.array(t)
115 + return result
116 +
117 +
118 +def two_tier_policy(lo: dict, hi: dict, ref_next: np.ndarray, taus: list[float]) -> list[dict]:
119 + """Policy: take lo's decision when its margin >= tau, else hi's decision."""
120 + out = []
121 + for tau in taus:
122 + esc = lo["margin"] < tau
123 + decision = np.where(esc, hi["argmax"], lo["argmax"])
124 + out.append({
125 + "tau": tau,
126 + "escalated_frac": float(esc.mean()),
127 + "policy_agreement": float((decision == ref_next).mean()),
128 + })
129 + return out
25 130
26 131
27 132 def main() -> None:
28 − manifest = collect_manifest()
29 − raise NotImplementedError("experiment not yet implemented")
133 + ap = argparse.ArgumentParser()
134 + ap.add_argument("--model", default="mlx-community/Qwen3-1.7B-bf16")
135 + ap.add_argument("--gen-tokens", type=int, default=128)
136 + ap.add_argument("--per-domain", type=int, default=8)
137 + ap.add_argument("--hidden-trajectories", type=int, default=8)
138 + args = ap.parse_args()
139 +
140 + domains = json.loads((REPO_ROOT / "benchmarks/datasets/eval_prompts.json").read_text())["domains"]
141 +
142 + print(f"loading reference {args.model} …", flush=True)
143 + model, tokenizer = load(args.model)
144 + n_layers = len(model.model.layers)
145 + depths = [max(0, round(n_layers * f) - 1) for f in (0.25, 0.5, 0.75, 1.0)]
146 +
147 + trajectories = []
148 + t0 = time.time()
149 + for domain, plist in domains.items():
150 + for prompt in plist[: args.per_domain]:
151 + ids = tokenizer.apply_chat_template(
152 + [{"role": "user", "content": prompt}], add_generation_prompt=True)
153 + gen = greedy_generate(model, tokenizer, ids, args.gen_tokens)
154 + if len(gen) >= 8:
155 + trajectories.append({"domain": domain, "full_ids": list(ids) + gen, "start": len(ids)})
156 + print(f"{len(trajectories)} reference trajectories in {time.time()-t0:.0f}s", flush=True)
157 +
158 + ref_stats = [teacher_forced_stats(model, t["full_ids"], t["start"]) for t in trajectories]
159 + hidden_subset = trajectories[:: max(1, len(trajectories) // args.hidden_trajectories)][: args.hidden_trajectories]
160 + ref_hidden = [capture_hidden(model, t["full_ids"], t["start"], depths) for t in hidden_subset]
161 +
162 + ladders = {"A_base3": [3, 3, 3], "B_base4": [4, 4]}
163 + results: dict[str, list] = {}
164 + overhead_bits = 32 / GROUP * 2 # bf16 scales + biases per group per stage
165 +
166 + for name, ladder in ladders.items():
167 + print(f"building ladder {name} {ladder} …", flush=True)
168 + stages = residual_ladder_weights(model, ladder)
169 + stage_records = []
170 + for k, replacement in enumerate(stages):
171 + originals = apply_weights(model, replacement)
172 + rows_margin, rows_agree, rows_kl, rows_argmax, doms = [], [], [], [], []
173 + for t, ref in zip(trajectories, ref_stats):
174 + qs = teacher_forced_stats(model, t["full_ids"], t["start"])
175 + ref_next = np.array(t["full_ids"][t["start"]:])
176 + rows_margin.append(qs["margin"])
177 + rows_argmax.append(qs["argmax"])
178 + rows_agree.append((qs["argmax"] == ref_next).astype(np.int8))
179 + rows_kl.append(kl_ref_vs(qs["logprobs"], ref["logprobs"]))
180 + doms.append(t["domain"])
181 + hid = [hidden_state_errors(model, rh, t["full_ids"], t["start"], depths)
182 + for rh, t in zip(ref_hidden, hidden_subset)]
183 + apply_weights(model, originals)
184 +
185 + margins = np.concatenate(rows_margin)
186 + agrees = np.concatenate(rows_agree)
187 + cum_bits = sum(ladder[: k + 1]) + overhead_bits * (k + 1)
188 + rec = {
189 + "stage": k,
190 + "ladder_bits": ladder[: k + 1],
191 + "cumulative_bits_per_param": round(cum_bits, 2),
192 + "agreement_rate": float(agrees.mean()),
193 + "mean_kl": float(np.mean(np.concatenate(rows_kl))),
194 + "auroc": auroc(-margins, 1 - agrees),
195 + "escalation_curve": escalation_curve(margins, agrees),
196 + "hidden_rel_err_by_depth": {
197 + str(d): float(np.mean([h[d] for h in hid])) for d in depths
198 + },
199 + "_margins": margins, "_argmax": np.concatenate(rows_argmax),
200 + }
201 + for pt in rec["escalation_curve"]:
202 + if pt["residual_disagree"] <= 0.01:
203 + rec["escalation_frac_for_99pct"] = pt["escalated_frac"]
204 + break
205 + stage_records.append(rec)
206 + print(f" stage {k} ({rec['cumulative_bits_per_param']} bits): "
207 + f"agree={rec['agreement_rate']:.4f} KL={rec['mean_kl']:.4f} "
208 + f"auroc={rec['auroc']:.3f}", flush=True)
209 + results[name] = stage_records
210 +
211 + # two-tier margin-gated policies between consecutive stages
212 + ref_next_all = np.concatenate([np.array(t["full_ids"][t["start"]:]) for t in trajectories])
213 + taus = [0.25, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
214 + policies = {}
215 + for name, recs in results.items():
216 + for k in range(len(recs) - 1):
217 + lo = {"margin": recs[k]["_margins"], "argmax": recs[k]["_argmax"]}
218 + hi = {"argmax": recs[k + 1]["_argmax"]}
219 + policies[f"{name}_stage{k}_to_{k+1}"] = two_tier_policy(lo, hi, ref_next_all, taus)
220 + for recs in results.values():
221 + for r in recs:
222 + del r["_margins"], r["_argmax"]
223 +
224 + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
225 + out_dir = REPO_ROOT / "results" / "expD_progressive_reconstruction" / ts
226 + out_dir.mkdir(parents=True)
227 + (out_dir / "results.json").write_text(json.dumps({
228 + "experiment": "expD_progressive_reconstruction",
229 + "author": "Simon-Pierre Boucher",
230 + "contact": "contact@spboucher.ai",
231 + "manifest": collect_manifest(),
232 + "config": vars(args),
233 + "group_size": GROUP,
234 + "scale_overhead_bits_per_param_per_stage": overhead_bits,
235 + "n_trajectories": len(trajectories),
236 + "hidden_depth_layers": depths,
237 + "ladders": {k: v for k, v in results.items()},
238 + "two_tier_policies": policies,
239 + }, indent=2))
240 + print(f"\nwrote {out_dir / 'results.json'}")
30 241
31 242
32 243 if __name__ == "__main__":
modified experiments/micro/expD_progressive_reconstruction/hypothesis.md +38 −8
@@ -3,31 +3,61 @@ project: localvm-research
3 3 document: expD_progressive_reconstruction/hypothesis
4 4 author: Simon-Pierre Boucher
5 5 contact: contact@spboucher.ai
6 −created: 2026-08-11
6 +created: 2026-08-12
7 7 status: draft
8 8 ---
9 9
10 10 # Hypothesis — expD_progressive_reconstruction
11 11
12 +Follows expG (margin gating promoted; naive affine 2-bit base dead; 4-bit
13 +escalation need 36.6%). This experiment measures how rapidly the token
14 +decision and distribution converge as residual quantization stages are added
15 +— the quality-vs-cumulative-bits curve that, combined with expG's escalation
16 +rates and expH's byte budget, decides candidate C1's arithmetic.
17 +
12 18 ```text
13 19 Hypothesis
14 − <what we believe and why>
20 + Weights represented as base + residual stages (each stage an affine
21 + group-quantization of the previous stage's error) converge rapidly:
22 + one residual stage over a 3-bit base (≈6.4 cumulative bits/param)
23 + reaches ≥95% greedy agreement, and a margin-gated two-tier policy
24 + (stage-k decision when margin ≥ τ, stage-(k+1) decision otherwise)
25 + attains ≥97% agreement while consulting the residual for ≤40% of
26 + tokens. Hidden-state error shrinks monotonically with each stage.
15 27
16 28 Falsification criterion
17 − <the concrete measurable outcome that would prove this wrong>
29 + If base3+1 residual (≈6.4 bits) stays below 90% agreement, or the
30 + two-tier margin policy cannot beat the flat next-stage agreement while
31 + escalating <50% of tokens, or hidden-state error does NOT decrease
32 + monotonically with stages (residual coding unstable), then progressive
33 + residual representations lose to simply shipping a flat higher-bit
34 + model, and C1 must pivot to expert/sparsity paging (C3) or amortized
35 + verification (C2).
18 36
19 37 Method
20 − <exact procedure, model(s), data, seeds, measurement points>
38 + Model: Qwen3-1.7B bf16 reference (as expG). Residual ladders, affine
39 + group-64 quantization at every stage, applied to all divisible Linear
40 + layers: A) 3 → 3+3 → 3+3+3 bits; B) 4 → 4+4 bits.
41 + Same 48 trajectories × 128 tokens protocol as expG (teacher-forced).
42 + Per cumulative stage: agreement, KL(ref||stage), margin stats, AUROC,
43 + escalation curve. Two-tier policy simulation from recorded per-stage
44 + argmax/margins across a τ grid. Hidden-state relative L2 error vs
45 + reference at layer depths {25%, 50%, 75%, 100%} on 8 trajectories.
46 + Bits accounting includes scale/bias overhead (group 64 → +0.5 bits/param
47 + per stage at bf16 scales+biases).
21 48
22 49 Baseline
23 − <what this is compared against — no straw men>
50 + Flat MLX affine quantization at matched cumulative bit-widths from expG
51 + (3, 4, 8-bit rows) — the "just ship a bigger flat model" alternative.
52 + No straw men: residual ladders must beat or match flat models at equal
53 + bytes to be interesting.
24 54
25 55 Result
26 − <filled after the run: numbers, with mean/median/std and run count>
56 + <filled after the run>
27 57
28 58 Interpretation
29 − <what the numbers mean; alternative explanations considered>
59 + <filled after the run>
30 60
31 61 Next experiment
32 − <the most informative follow-up given this result>
62 + <filled after the run>
33 63 ```
added src/localvm/quality/decision_stats.py +102 −0
@@ -0,0 +1,102 @@
1 +# =============================================================================
2 +# Project : localvm-research
3 +# File : src/localvm/quality/decision_stats.py
4 +# Purpose : Shared decision-stability measurement utilities (greedy
5 +# trajectories, teacher-forced margins/agreement, AUROC, curves)
6 +# Author : Simon-Pierre Boucher
7 +# Contact : contact@spboucher.ai
8 +# Created : 2026-08-12
9 +# Modified : 2026-08-12
10 +# Platform : macOS / Apple Silicon (arm64) — MLX / Metal
11 +# License : All rights reserved (research code)
12 +# =============================================================================
13 +"""Decision-stability measurement utilities shared by expG/expD and successors.
14 +
15 +First used (inlined) by experiments/micro/expG_decision_stability/benchmark.py
16 +(commit 42c7b3d); extracted here unchanged so later experiments reuse one
17 +implementation. expG's committed copy is kept as-is for reproducibility.
18 +"""
19 +
20 +from __future__ import annotations
21 +
22 +import mlx.core as mx
23 +import numpy as np
24 +
25 +
26 +def greedy_generate(model, tokenizer, prompt_ids: list[int], n_tokens: int) -> list[int]:
27 + """Deterministic greedy generation; returns generated token ids."""
28 + from mlx_lm.models.cache import make_prompt_cache
29 +
30 + cache = make_prompt_cache(model)
31 + generated: list[int] = []
32 + inp = mx.array(list(prompt_ids))[None]
33 + for _ in range(n_tokens):
34 + logits = model(inp, cache=cache)
35 + nxt = int(mx.argmax(logits[0, -1]).item())
36 + if nxt == tokenizer.eos_token_id:
37 + break
38 + generated.append(nxt)
39 + inp = mx.array([[nxt]])
40 + return generated
41 +
42 +
43 +def teacher_forced_stats(model, full_ids: list[int], start: int) -> dict:
44 + """Forward full_ids once; per-position stats for predictions of tokens
45 + [start, len). Returns margin (top-1 minus top-2 logit gap), argmax, and
46 + float16 logprobs (float16 keeps 48 x (128, ~152k) around 2 GB)."""
47 + logits = model(mx.array(full_ids)[None])[0]
48 + sel = logits[start - 1 : len(full_ids) - 1].astype(mx.float32)
49 + top2 = mx.topk(sel, 2, axis=-1)
50 + argmax = mx.argmax(sel, axis=-1)
51 + logprobs = sel - mx.logsumexp(sel, axis=-1, keepdims=True)
52 + mx.eval(top2, argmax, logprobs)
53 + v = np.array(top2)
54 + return {
55 + "margin": np.abs(v[:, 1] - v[:, 0]),
56 + "argmax": np.array(argmax),
57 + "logprobs": np.array(logprobs).astype(np.float16),
58 + }
59 +
60 +
61 +def auroc(scores: np.ndarray, labels: np.ndarray) -> float:
62 + """Rank-based AUROC with tie handling (scores: higher = predicted 1)."""
63 + pos, neg = scores[labels == 1], scores[labels == 0]
64 + if len(pos) == 0 or len(neg) == 0:
65 + return float("nan")
66 + allv = np.concatenate([pos, neg])
67 + order = np.argsort(allv, kind="mergesort")
68 + ranks = np.empty(len(order))
69 + ranks[order] = np.arange(1, len(order) + 1)
70 + sorted_v = allv[order]
71 + i = 0
72 + while i < len(sorted_v):
73 + j = i
74 + while j + 1 < len(sorted_v) and sorted_v[j + 1] == sorted_v[i]:
75 + j += 1
76 + if j > i:
77 + ranks[order[i : j + 1]] = ranks[order[i : j + 1]].mean()
78 + i = j + 1
79 + r_pos = ranks[: len(pos)].sum()
80 + return float((r_pos - len(pos) * (len(pos) + 1) / 2) / (len(pos) * len(neg)))
81 +
82 +
83 +def escalation_curve(margins: np.ndarray, agree: np.ndarray, points: int = 200) -> list[dict]:
84 + """Escalate tokens with margin < tau (escalated decision assumed exact);
85 + report escalated fraction vs residual disagreement."""
86 + qs = np.quantile(margins, np.linspace(0, 1, points))
87 + out, n = [], len(margins)
88 + for tau in qs:
89 + esc = margins < tau
90 + out.append({
91 + "tau": float(tau),
92 + "escalated_frac": float(esc.mean()),
93 + "residual_disagree": float(np.sum((~esc) & (agree == 0)) / n),
94 + })
95 + return out
96 +
97 +
98 +def kl_ref_vs(model_logprobs_f16: np.ndarray, ref_logprobs_f16: np.ndarray) -> np.ndarray:
99 + """Per-position KL(ref || model), computed in float32."""
100 + ref = ref_logprobs_f16.astype(np.float32)
101 + q = model_logprobs_f16.astype(np.float32)
102 + return np.sum(np.exp(ref) * (ref - q), axis=-1)
103