SPB Git

spb/localvm-research Public License

Running LLMs larger than memory on a consumer Mac — falsification-driven research: margin-gated deferred refinement, out-of-core verification on Apple Silicon. TR-01 published.

Python 63.2% JavaScript 23.5% CSS 11.8% Shell 0.9% Makefile 0.5%

candidate_01 v2: incremental q8 verify-cache (O(window) sweeps) + bf16 quality judge

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Simon-Pierre Boucher committed 4 h ago (Aug 12, 2026) parent 93bcd68

Showing 2 changed files with +57 and −3

modified experiments/candidate_01/benchmark.py +39 −0
@@ -77,6 +77,35 @@ def greedy_baseline(model, tokenizer, prompt_ids, max_tokens):
77 77 return tokens, time.perf_counter() - t0
78 78
79 79
80 +def judge_outputs(outputs_by_config: dict, prompts: list[dict]) -> dict:
81 + """Quality-level metric: mean per-token logprob of each config's generated
82 + continuation under the bf16 reference model (higher = better). Token-exact
83 + fidelity is incoherent on Metal (1.56%/token prefill/decode flips), so the
84 + judge scores usefulness of the text the system actually produced."""
85 + import gc
86 +
87 + gc.collect(); mx.clear_cache()
88 + judge, _ = load(HF_MODEL)
89 + scores = {}
90 + for name, outs in outputs_by_config.items():
91 + vals = []
92 + for p, toks in zip(prompts, outs):
93 + if len(toks) < 2:
94 + continue
95 + full = p["ids"] + list(toks)
96 + logits = judge(mx.array(full)[None])[0]
97 + sel = logits[len(p["ids"]) - 1 : len(full) - 1].astype(mx.float32)
98 + logprobs = sel - mx.logsumexp(sel, axis=-1, keepdims=True)
99 + idx = mx.array(toks)
100 + tok_lp = mx.take_along_axis(logprobs, idx[:, None], axis=-1)
101 + mx.eval(tok_lp)
102 + vals.append(float(mx.mean(tok_lp).item()))
103 + scores[name] = {"mean_logprob_bf16": sum(vals) / len(vals), "n": len(vals)}
104 + del judge
105 + gc.collect(); mx.clear_cache()
106 + return scores
107 +
108 +
80 109 def fidelity(a: list[int], b: list[int]) -> float:
81 110 """Similarity of two token sequences (difflib ratio — robust to length
82 111 drift after divergence)."""
@@ -134,15 +163,18 @@ def main() -> None:
134 163 for tau in ([float(x) for x in args.taus.split(",")] if mode == "margin" else [2.0]):
135 164 configs.append({"mode": mode, "tau": tau})
136 165
166 + outputs_by_config = {"pure_q4": q4_out, "pure_q8": q8_out}
137 167 results = []
138 168 for cfg in configs:
139 169 print(f"runtime: mode={cfg['mode']} tau={cfg['tau']} W={args.window} …", flush=True)
140 170 fid, agg = [], {"tokens": 0, "deferred": 0, "sweeps": 0, "rollbacks": 0,
141 171 "sweep_s": 0.0, "gen_s": 0.0, "logical_bytes": 0}
172 + cfg_outputs = []
142 173 for p, ref in zip(prompts, q8_out):
143 174 toks, st = generate_deferred(
144 175 base_model, verify_model, tokenizer, p["ids"],
145 176 args.max_tokens, cfg["tau"], args.window, cfg["mode"], q8_bytes)
177 + cfg_outputs.append(toks)
146 178 fid.append(fidelity(toks, ref))
147 179 agg["tokens"] += st.tokens_out; agg["deferred"] += st.deferred
148 180 agg["sweeps"] += st.sweeps; agg["rollbacks"] += st.rollbacks
@@ -160,11 +192,17 @@ def main() -> None:
160 192 "logical_verify_bytes_per_token": agg["logical_bytes"] / n,
161 193 "raw": agg,
162 194 })
195 + outputs_by_config[f"{cfg['mode']}_tau{cfg['tau']}"] = cfg_outputs
163 196 r = results[-1]
164 197 print(f" fidelity={r['fidelity_vs_q8_mean']:.4f} tok/s={r['tokens_per_s']:.1f} "
165 198 f"defer={r['deferral_rate']:.2f} rollback={r['rollback_rate']:.3f} "
166 199 f"MB/token(logical)={r['logical_verify_bytes_per_token']/1e6:.0f}", flush=True)
167 200
201 + print("judging outputs with bf16 reference …", flush=True)
202 + quality = judge_outputs(outputs_by_config, prompts)
203 + for name, s in quality.items():
204 + print(f" {name:>18}: mean logprob (bf16 judge) = {s['mean_logprob_bf16']:.4f}", flush=True)
205 +
168 206 payload = {
169 207 "experiment": "candidate_01_deferred_refinement",
170 208 "author": "Simon-Pierre Boucher",
@@ -179,6 +217,7 @@ def main() -> None:
179 217 "pure_q8": {"tokens_per_s": toks_per_s(q8_times), "fidelity_vs_q8_mean": 1.0},
180 218 },
181 219 "runs": results,
220 + "quality_bf16_judge": quality,
182 221 }
183 222 ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
184 223 out_dir = REPO_ROOT / "results" / "candidate_01" / ts
modified experiments/candidate_01/implementation/runtime.py +18 −3
@@ -62,13 +62,18 @@ def generate_deferred(base_model, verify_model, tokenizer, prompt_ids: list[int]
62 62 verified_upto = 0 # tokens before this index are settled
63 63
64 64 cache = make_prompt_cache(base_model)
65 + # Incremental verify cache: sweeps feed only the unseen suffix, cutting
66 + # sweep compute from O(context) to O(window). v_pos = tokens of the
67 + # current sequence already ingested; invariant at sweep: v_pos <= base-1.
68 + vcache = make_prompt_cache(verify_model)
69 + v_pos = 0
65 70 t0 = time.perf_counter()
66 71 inp = mx.array(prompt_ids)[None]
67 72
68 73 def sweep() -> bool:
69 74 """Verify tokens[verified_upto:] with q8; rollback on first flip.
70 75 Returns True if a rollback happened."""
71 − nonlocal verified_upto, tokens, flags, inp, cache
76 + nonlocal verified_upto, tokens, flags, inp, cache, v_pos
72 77 pending = len(tokens) - verified_upto
73 78 if pending <= 0:
74 79 return False
@@ -76,9 +81,13 @@ def generate_deferred(base_model, verify_model, tokenizer, prompt_ids: list[int]
76 81 stats.sweeps += 1
77 82 stats.sweep_logical_bytes += q8_bytes
78 83 full = prompt_ids + tokens
79 − logits = verify_model(mx.array(full)[None])[0]
80 84 base_of_window = len(prompt_ids) + verified_upto
81 − sel = logits[base_of_window - 1 : len(full) - 1].astype(mx.float32)
85 + # feed full[v_pos : end-1]; logits rows are absolute v_pos..end-2,
86 + # predictions for tokens v_pos+1..end-1 ⊇ the pending window
87 + chunk = full[v_pos : len(full) - 1]
88 + logits = verify_model(mx.array(chunk)[None], cache=vcache)[0]
89 + sel = logits[base_of_window - 1 - v_pos :].astype(mx.float32)
90 + v_pos = len(full) - 1
82 91 q8_argmax = mx.argmax(sel, axis=-1)
83 92 mx.eval(q8_argmax)
84 93 q8_next = q8_argmax.tolist()
@@ -99,6 +108,12 @@ def generate_deferred(base_model, verify_model, tokenizer, prompt_ids: list[int]
99 108 # newest token was appended but not yet fed). To leave the cache at
100 109 # prompt + tokens[:i], trim (len(tokens)-1 - i) entries.
101 110 trim_prompt_cache(cache, len(tokens) - 1 - i)
111 + # verify cache must hold a prefix of the corrected sequence:
112 + # keep exactly prompt + tokens[:i] (= base_next - 1, invariant holds)
113 + keep = len(prompt_ids) + i
114 + if v_pos > keep:
115 + trim_prompt_cache(vcache, v_pos - keep)
116 + v_pos = keep
102 117 tokens = tokens[:i] + [corrected]
103 118 flags = flags[:i] + [False]
104 119 verified_upto = len(tokens)
105 120