candidate_01: margin-gated deferred-refinement runtime + benchmark + hypothesis
q4 resident base generates with margin recording; low-margin tokens deferred; windowed q8 sweeps verify (margin mode) or verify-all (QSpec-style exact); rollback via KV trim. Baselines: pure q4/q8. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Showing 7 changed files with +420 and −0
modified
.gitignore
+3 −0
@@ -48,3 +48,6 @@ results/**/*.fsusage | ||
| 48 | 48 | # web platform: generated content snapshot + deps |
| 49 | 49 | web/content/ |
| 50 | 50 | web/node_modules/ |
| 51 | + | |
| 52 | +# candidate model artifacts (rebuilt by benchmark --build) | |
| 53 | +experiments/candidate_01/implementation/models/ | |
added
experiments/candidate_01/README.md
+14 −0
@@ -0,0 +1,14 @@ | ||
| 1 | +--- | |
| 2 | +project: localvm-research | |
| 3 | +document: candidate_01/README | |
| 4 | +author: Simon-Pierre Boucher | |
| 5 | +contact: contact@spboucher.ai | |
| 6 | +created: 2026-08-12 | |
| 7 | +status: draft | |
| 8 | +--- | |
| 9 | + | |
| 10 | +# candidate_01 | |
| 11 | + | |
| 12 | +Margin-gated deferred-refinement runtime: 4-bit resident base + windowed q8 verification sweeps with rollback | |
| 13 | + | |
| 14 | +Status: scaffolded 2026-08-12, not yet run. | |
added
experiments/candidate_01/analysis.md
+12 −0
@@ -0,0 +1,12 @@ | ||
| 1 | +--- | |
| 2 | +project: localvm-research | |
| 3 | +document: candidate_01/analysis | |
| 4 | +author: Simon-Pierre Boucher | |
| 5 | +contact: contact@spboucher.ai | |
| 6 | +created: 2026-08-12 | |
| 7 | +status: draft | |
| 8 | +--- | |
| 9 | + | |
| 10 | +# Analysis — candidate_01 | |
| 11 | + | |
| 12 | +*To be written after results exist. Must include the seven-field block and the evidence standard of CLAUDE.md §10.* | |
added
experiments/candidate_01/benchmark.py
+192 −0
@@ -0,0 +1,192 @@ | ||
| 1 | +#!/usr/bin/env python3 | |
| 2 | +# ============================================================================= | |
| 3 | +# Project : localvm-research | |
| 4 | +# File : experiments/candidate_01/benchmark.py | |
| 5 | +# Purpose : End-to-end evaluation of the margin-gated deferred-refinement | |
| 6 | +# runtime vs pure-q4 / pure-q8 baselines | |
| 7 | +# Author : Simon-Pierre Boucher | |
| 8 | +# Contact : contact@spboucher.ai | |
| 9 | +# Created : 2026-08-12 | |
| 10 | +# Modified : 2026-08-12 | |
| 11 | +# Platform : macOS / Apple Silicon (arm64) — MLX / Metal | |
| 12 | +# License : All rights reserved (research code) | |
| 13 | +# ============================================================================= | |
| 14 | +"""Candidate-01 benchmark. | |
| 15 | + | |
| 16 | +Build once (downloads + quantizes): | |
| 17 | + .venv/bin/python benchmark.py --build | |
| 18 | +Run: | |
| 19 | + .venv/bin/python benchmark.py [--per-domain 4] [--max-tokens 128] | |
| 20 | + [--window 32] [--taus 1.0,2.0] | |
| 21 | +""" | |
| 22 | + | |
| 23 | +from __future__ import annotations | |
| 24 | + | |
| 25 | +import argparse | |
| 26 | +import difflib | |
| 27 | +import json | |
| 28 | +import sys | |
| 29 | +import time | |
| 30 | +from datetime import datetime, timezone | |
| 31 | +from pathlib import Path | |
| 32 | + | |
| 33 | +import mlx.core as mx | |
| 34 | +from mlx_lm import load | |
| 35 | + | |
| 36 | +REPO_ROOT = Path(__file__).resolve().parents[2] | |
| 37 | +sys.path.insert(0, str(REPO_ROOT / "benchmarks")) | |
| 38 | +sys.path.insert(0, str(Path(__file__).parent / "implementation")) | |
| 39 | +from hardware_manifest import collect_manifest # noqa: E402 | |
| 40 | +from runtime import generate_deferred # noqa: E402 | |
| 41 | + | |
| 42 | +MODELS_DIR = Path(__file__).parent / "implementation" / "models" | |
| 43 | +HF_MODEL = "mlx-community/Qwen3-1.7B-bf16" | |
| 44 | + | |
| 45 | + | |
| 46 | +def build() -> None: | |
| 47 | + from mlx_lm import convert | |
| 48 | + | |
| 49 | + for bits in (4, 8): | |
| 50 | + out = MODELS_DIR / f"q{bits}" | |
| 51 | + if out.exists(): | |
| 52 | + print(f"{out} exists, skipping") | |
| 53 | + continue | |
| 54 | + print(f"converting {HF_MODEL} → q{bits} …", flush=True) | |
| 55 | + convert(HF_MODEL, mlx_path=str(out), quantize=True, q_bits=bits, q_group_size=64) | |
| 56 | + print("build done") | |
| 57 | + | |
| 58 | + | |
| 59 | +def dir_weight_bytes(d: Path) -> int: | |
| 60 | + return sum(f.stat().st_size for f in d.glob("*.safetensors")) | |
| 61 | + | |
| 62 | + | |
| 63 | +def greedy_baseline(model, tokenizer, prompt_ids, max_tokens): | |
| 64 | + from mlx_lm.models.cache import make_prompt_cache | |
| 65 | + | |
| 66 | + cache = make_prompt_cache(model) | |
| 67 | + tokens = [] | |
| 68 | + inp = mx.array(list(prompt_ids))[None] | |
| 69 | + t0 = time.perf_counter() | |
| 70 | + for _ in range(max_tokens): | |
| 71 | + logits = model(inp, cache=cache) | |
| 72 | + nxt = int(mx.argmax(logits[0, -1]).item()) | |
| 73 | + if nxt == tokenizer.eos_token_id: | |
| 74 | + break | |
| 75 | + tokens.append(nxt) | |
| 76 | + inp = mx.array([[nxt]]) | |
| 77 | + return tokens, time.perf_counter() - t0 | |
| 78 | + | |
| 79 | + | |
| 80 | +def fidelity(a: list[int], b: list[int]) -> float: | |
| 81 | + """Similarity of two token sequences (difflib ratio — robust to length | |
| 82 | + drift after divergence).""" | |
| 83 | + if not a and not b: | |
| 84 | + return 1.0 | |
| 85 | + return difflib.SequenceMatcher(None, a, b).ratio() | |
| 86 | + | |
| 87 | + | |
| 88 | +def main() -> None: | |
| 89 | + ap = argparse.ArgumentParser() | |
| 90 | + ap.add_argument("--build", action="store_true") | |
| 91 | + ap.add_argument("--per-domain", type=int, default=4) | |
| 92 | + ap.add_argument("--max-tokens", type=int, default=128) | |
| 93 | + ap.add_argument("--window", type=int, default=32) | |
| 94 | + ap.add_argument("--taus", default="1.0,2.0") | |
| 95 | + args = ap.parse_args() | |
| 96 | + if args.build: | |
| 97 | + build() | |
| 98 | + return | |
| 99 | + | |
| 100 | + domains = json.loads((REPO_ROOT / "benchmarks/datasets/eval_prompts.json").read_text())["domains"] | |
| 101 | + q4_dir, q8_dir = MODELS_DIR / "q4", MODELS_DIR / "q8" | |
| 102 | + q8_bytes = dir_weight_bytes(q8_dir) | |
| 103 | + q4_bytes = dir_weight_bytes(q4_dir) | |
| 104 | + print(f"resident q4: {q4_bytes/1e9:.2f} GB · streamed q8: {q8_bytes/1e9:.2f} GB", flush=True) | |
| 105 | + | |
| 106 | + base_model, tokenizer = load(str(q4_dir)) | |
| 107 | + verify_model, _ = load(str(q8_dir)) | |
| 108 | + | |
| 109 | + prompts = [] | |
| 110 | + for domain, plist in domains.items(): | |
| 111 | + for prompt in plist[: args.per_domain]: | |
| 112 | + ids = tokenizer.apply_chat_template( | |
| 113 | + [{"role": "user", "content": prompt}], add_generation_prompt=True) | |
| 114 | + prompts.append({"domain": domain, "ids": list(ids)}) | |
| 115 | + | |
| 116 | + # baselines | |
| 117 | + print("baseline: pure q8 greedy …", flush=True) | |
| 118 | + q8_out, q8_times = [], [] | |
| 119 | + for p in prompts: | |
| 120 | + toks, dt = greedy_baseline(verify_model, tokenizer, p["ids"], args.max_tokens) | |
| 121 | + q8_out.append(toks); q8_times.append((len(toks), dt)) | |
| 122 | + print("baseline: pure q4 greedy …", flush=True) | |
| 123 | + q4_out, q4_times = [], [] | |
| 124 | + for p in prompts: | |
| 125 | + toks, dt = greedy_baseline(base_model, tokenizer, p["ids"], args.max_tokens) | |
| 126 | + q4_out.append(toks); q4_times.append((len(toks), dt)) | |
| 127 | + | |
| 128 | + def toks_per_s(times): | |
| 129 | + n = sum(t for t, _ in times); s = sum(d for _, d in times) | |
| 130 | + return n / s if s else 0.0 | |
| 131 | + | |
| 132 | + configs = [] | |
| 133 | + for mode in ("margin", "verify-all"): | |
| 134 | + for tau in ([float(x) for x in args.taus.split(",")] if mode == "margin" else [2.0]): | |
| 135 | + configs.append({"mode": mode, "tau": tau}) | |
| 136 | + | |
| 137 | + results = [] | |
| 138 | + for cfg in configs: | |
| 139 | + print(f"runtime: mode={cfg['mode']} tau={cfg['tau']} W={args.window} …", flush=True) | |
| 140 | + fid, agg = [], {"tokens": 0, "deferred": 0, "sweeps": 0, "rollbacks": 0, | |
| 141 | + "sweep_s": 0.0, "gen_s": 0.0, "logical_bytes": 0} | |
| 142 | + for p, ref in zip(prompts, q8_out): | |
| 143 | + toks, st = generate_deferred( | |
| 144 | + base_model, verify_model, tokenizer, p["ids"], | |
| 145 | + args.max_tokens, cfg["tau"], args.window, cfg["mode"], q8_bytes) | |
| 146 | + fid.append(fidelity(toks, ref)) | |
| 147 | + agg["tokens"] += st.tokens_out; agg["deferred"] += st.deferred | |
| 148 | + agg["sweeps"] += st.sweeps; agg["rollbacks"] += st.rollbacks | |
| 149 | + agg["sweep_s"] += st.sweep_time_s; agg["gen_s"] += st.gen_time_s | |
| 150 | + agg["logical_bytes"] += st.sweep_logical_bytes | |
| 151 | + n = max(agg["tokens"], 1) | |
| 152 | + results.append({ | |
| 153 | + **cfg, "window": args.window, | |
| 154 | + "fidelity_vs_q8_mean": sum(fid) / len(fid), | |
| 155 | + "tokens_per_s": n / (agg["gen_s"] + agg["sweep_s"]), | |
| 156 | + "deferral_rate": agg["deferred"] / n, | |
| 157 | + "rollback_rate": agg["rollbacks"] / n, | |
| 158 | + "sweeps_per_100tok": 100 * agg["sweeps"] / n, | |
| 159 | + "sweep_latency_s_mean": agg["sweep_s"] / max(agg["sweeps"], 1), | |
| 160 | + "logical_verify_bytes_per_token": agg["logical_bytes"] / n, | |
| 161 | + "raw": agg, | |
| 162 | + }) | |
| 163 | + r = results[-1] | |
| 164 | + print(f" fidelity={r['fidelity_vs_q8_mean']:.4f} tok/s={r['tokens_per_s']:.1f} " | |
| 165 | + f"defer={r['deferral_rate']:.2f} rollback={r['rollback_rate']:.3f} " | |
| 166 | + f"MB/token(logical)={r['logical_verify_bytes_per_token']/1e6:.0f}", flush=True) | |
| 167 | + | |
| 168 | + payload = { | |
| 169 | + "experiment": "candidate_01_deferred_refinement", | |
| 170 | + "author": "Simon-Pierre Boucher", | |
| 171 | + "contact": "contact@spboucher.ai", | |
| 172 | + "manifest": collect_manifest(), | |
| 173 | + "config": vars(args), | |
| 174 | + "model": HF_MODEL, | |
| 175 | + "q4_resident_bytes": q4_bytes, "q8_stream_bytes": q8_bytes, | |
| 176 | + "baselines": { | |
| 177 | + "pure_q4": {"tokens_per_s": toks_per_s(q4_times), | |
| 178 | + "fidelity_vs_q8_mean": sum(fidelity(a, b) for a, b in zip(q4_out, q8_out)) / len(q8_out)}, | |
| 179 | + "pure_q8": {"tokens_per_s": toks_per_s(q8_times), "fidelity_vs_q8_mean": 1.0}, | |
| 180 | + }, | |
| 181 | + "runs": results, | |
| 182 | + } | |
| 183 | + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") | |
| 184 | + out_dir = REPO_ROOT / "results" / "candidate_01" / ts | |
| 185 | + out_dir.mkdir(parents=True) | |
| 186 | + (out_dir / "results.json").write_text(json.dumps(payload, indent=2)) | |
| 187 | + print(f"\nwrote {out_dir / 'results.json'}") | |
| 188 | + print("baselines:", json.dumps(payload["baselines"], indent=1)) | |
| 189 | + | |
| 190 | + | |
| 191 | +if __name__ == "__main__": | |
| 192 | + main() | |
added
experiments/candidate_01/hypothesis.md
+66 −0
@@ -0,0 +1,66 @@ | ||
| 1 | +--- | |
| 2 | +project: localvm-research | |
| 3 | +document: candidate_01/hypothesis | |
| 4 | +author: Simon-Pierre Boucher | |
| 5 | +contact: contact@spboucher.ai | |
| 6 | +created: 2026-08-12 | |
| 7 | +status: draft | |
| 8 | +--- | |
| 9 | + | |
| 10 | +# Hypothesis — candidate_01 (margin-gated deferred-refinement runtime) | |
| 11 | + | |
| 12 | +The architecture the micro-experiment campaign selected: expG (margins predict | |
| 13 | +disagreement, AUROC 0.9), expD (residual refinement converges; two-tier policy | |
| 14 | +works), expF/expA/expB (fine-grained escalation routes all dead → amortize), | |
| 15 | +expH (sequential residual sweeps are nearly free: ~13 GB/s). | |
| 16 | + | |
| 17 | +```text | |
| 18 | +Hypothesis | |
| 19 | + A runtime holding only a 4-bit-class base model resident, generating | |
| 20 | + optimistically while deferring low-margin tokens, and verifying windows | |
| 21 | + with a periodically streamed 8-bit-class refinement (rollback on flips) | |
| 22 | + achieves: (a) ≥95% token fidelity to the pure-q8 greedy output in | |
| 23 | + margin-gated mode and 100% in verify-all mode; (b) residual/verify | |
| 24 | + bytes-per-token an order of magnitude below the q8 checkpoint size per | |
| 25 | + token (sweep bytes ÷ window); (c) throughput within 2× of pure-q4 | |
| 26 | + generation. In verify-all mode the system IS precision-tiered | |
| 27 | + speculative decoding (QSpec-family); margin-gated mode is the novel | |
| 28 | + relaxation trading exactness for fewer rollbacks and sweeps. | |
| 29 | + | |
| 30 | +Falsification criterion | |
| 31 | + If rollback overhead + sweep cost push throughput below 1/3 of pure-q4, | |
| 32 | + or margin-gated fidelity falls below 90%, or measured bytes/token shows | |
| 33 | + no order-of-magnitude advantage over resident-q8 execution, the | |
| 34 | + architecture is not competitive on Apple Silicon and the project pivots | |
| 35 | + to C3 (MoE residency) as primary candidate. | |
| 36 | + | |
| 37 | +Method | |
| 38 | + Qwen3-1.7B. Build (offline): q4 and q8 MLX conversions (group 64). | |
| 39 | + Run: resident q4 generates greedily with KV cache, recording margins. | |
| 40 | + Tokens with margin < τ are deferred. Every W generated tokens (or at | |
| 41 | + EOS), a sweep loads the q8 model weights, teacher-forces the window in | |
| 42 | + one pass, and (mode=margin) checks deferred positions only / | |
| 43 | + (mode=verify-all) checks all positions; on first flip: rollback (trim | |
| 44 | + base KV), accept the q8 token, resume. Sweep bytes counted = q8 weight | |
| 45 | + bytes actually (re)loaded; at 1.7B the page cache hides re-reads, so | |
| 46 | + logical bytes are also reported for scale extrapolation (documented). | |
| 47 | + Configs: τ ∈ {1.0, 2.0}, W ∈ {32}, both modes; 24 prompts × 128 tokens. | |
| 48 | + Metrics: tok/s end-to-end, sweep latency, rollback rate, deferral rate, | |
| 49 | + fidelity vs pure-q8 greedy, bytes/token (physical and logical). | |
| 50 | + | |
| 51 | +Baseline | |
| 52 | + Pure-q4 greedy and pure-q8 greedy (both fully resident) on the same | |
| 53 | + prompts — the honest brackets for speed and quality. No straw men: | |
| 54 | + resident-q8 is what a 48 GB Mac would actually do at 1.7B; the value | |
| 55 | + proposition targets models where q8 does NOT fit, so the reported | |
| 56 | + advantage is bytes-per-token structure, not wall-clock at 1.7B. | |
| 57 | + | |
| 58 | +Result | |
| 59 | + <filled after the run> | |
| 60 | + | |
| 61 | +Interpretation | |
| 62 | + <filled after the run> | |
| 63 | + | |
| 64 | +Next experiment | |
| 65 | + <filled after the run> | |
| 66 | +``` | |
added
experiments/candidate_01/implementation/runtime.py
+129 −0
@@ -0,0 +1,129 @@ | ||
| 1 | +# ============================================================================= | |
| 2 | +# Project : localvm-research | |
| 3 | +# File : experiments/candidate_01/implementation/runtime.py | |
| 4 | +# Purpose : Margin-gated deferred-refinement generation loop (q4 resident | |
| 5 | +# base + windowed q8 verification sweeps with rollback) | |
| 6 | +# Author : Simon-Pierre Boucher | |
| 7 | +# Contact : contact@spboucher.ai | |
| 8 | +# Created : 2026-08-12 | |
| 9 | +# Modified : 2026-08-12 | |
| 10 | +# Platform : macOS / Apple Silicon (arm64) — MLX / Metal | |
| 11 | +# License : All rights reserved (research code) | |
| 12 | +# ============================================================================= | |
| 13 | +"""Candidate-01 runtime. | |
| 14 | + | |
| 15 | +Two modes: | |
| 16 | + * ``verify-all`` — every window position is checked against the q8 sweep | |
| 17 | + (exact q8-greedy output; precision-tiered speculative decoding). | |
| 18 | + * ``margin`` — only positions whose base margin < tau are checked | |
| 19 | + (approximate; fewer rollbacks — the novel relaxation under test). | |
| 20 | +""" | |
| 21 | + | |
| 22 | +from __future__ import annotations | |
| 23 | + | |
| 24 | +import time | |
| 25 | +from dataclasses import dataclass, field | |
| 26 | + | |
| 27 | +import mlx.core as mx | |
| 28 | +from mlx_lm.models.cache import make_prompt_cache, trim_prompt_cache | |
| 29 | + | |
| 30 | + | |
| 31 | +@dataclass | |
| 32 | +class RunStats: | |
| 33 | + tokens_out: int = 0 | |
| 34 | + deferred: int = 0 | |
| 35 | + sweeps: int = 0 | |
| 36 | + rollbacks: int = 0 | |
| 37 | + sweep_time_s: float = 0.0 | |
| 38 | + gen_time_s: float = 0.0 | |
| 39 | + sweep_logical_bytes: int = 0 | |
| 40 | + events: list = field(default_factory=list) | |
| 41 | + | |
| 42 | + | |
| 43 | +def _greedy_step(model, inp, cache): | |
| 44 | + logits = model(inp, cache=cache)[0, -1].astype(mx.float32) | |
| 45 | + top2 = mx.topk(logits, 2) | |
| 46 | + nxt = mx.argmax(logits) | |
| 47 | + mx.eval(top2, nxt) | |
| 48 | + v = top2.tolist() | |
| 49 | + return int(nxt.item()), abs(v[1] - v[0]) | |
| 50 | + | |
| 51 | + | |
| 52 | +def generate_deferred(base_model, verify_model, tokenizer, prompt_ids: list[int], | |
| 53 | + max_tokens: int, tau: float, window: int, | |
| 54 | + mode: str, q8_bytes: int) -> tuple[list[int], RunStats]: | |
| 55 | + """Generate with the resident base model, verifying windows with the q8 | |
| 56 | + model. Returns (tokens, stats). `verify_model` weights are assumed | |
| 57 | + mmap-backed; each sweep accounts `q8_bytes` logical bytes.""" | |
| 58 | + assert mode in ("verify-all", "margin") | |
| 59 | + stats = RunStats() | |
| 60 | + tokens: list[int] = [] # accepted generated tokens | |
| 61 | + flags: list[bool] = [] # deferred flag per generated token | |
| 62 | + verified_upto = 0 # tokens before this index are settled | |
| 63 | + | |
| 64 | + cache = make_prompt_cache(base_model) | |
| 65 | + t0 = time.perf_counter() | |
| 66 | + inp = mx.array(prompt_ids)[None] | |
| 67 | + | |
| 68 | + def sweep() -> bool: | |
| 69 | + """Verify tokens[verified_upto:] with q8; rollback on first flip. | |
| 70 | + Returns True if a rollback happened.""" | |
| 71 | + nonlocal verified_upto, tokens, flags, inp, cache | |
| 72 | + pending = len(tokens) - verified_upto | |
| 73 | + if pending <= 0: | |
| 74 | + return False | |
| 75 | + s0 = time.perf_counter() | |
| 76 | + stats.sweeps += 1 | |
| 77 | + stats.sweep_logical_bytes += q8_bytes | |
| 78 | + full = prompt_ids + tokens | |
| 79 | + logits = verify_model(mx.array(full)[None])[0] | |
| 80 | + base_of_window = len(prompt_ids) + verified_upto | |
| 81 | + sel = logits[base_of_window - 1 : len(full) - 1].astype(mx.float32) | |
| 82 | + q8_argmax = mx.argmax(sel, axis=-1) | |
| 83 | + mx.eval(q8_argmax) | |
| 84 | + q8_next = q8_argmax.tolist() | |
| 85 | + flip_at = None | |
| 86 | + for j in range(pending): | |
| 87 | + i = verified_upto + j | |
| 88 | + check = mode == "verify-all" or flags[i] | |
| 89 | + if check and q8_next[j] != tokens[i]: | |
| 90 | + flip_at = (i, q8_next[j]) | |
| 91 | + break | |
| 92 | + if flip_at is None: | |
| 93 | + verified_upto = len(tokens) | |
| 94 | + stats.sweep_time_s += time.perf_counter() - s0 | |
| 95 | + return False | |
| 96 | + i, corrected = flip_at | |
| 97 | + stats.rollbacks += 1 | |
| 98 | + # Cache invariant at sweep time: cache = prompt + tokens[:-1] (the | |
| 99 | + # newest token was appended but not yet fed). To leave the cache at | |
| 100 | + # prompt + tokens[:i], trim (len(tokens)-1 - i) entries. | |
| 101 | + trim_prompt_cache(cache, len(tokens) - 1 - i) | |
| 102 | + tokens = tokens[:i] + [corrected] | |
| 103 | + flags = flags[:i] + [False] | |
| 104 | + verified_upto = len(tokens) | |
| 105 | + # do NOT feed `corrected` here — the main loop's next _greedy_step | |
| 106 | + # feeds tokens[-1], preserving the invariant | |
| 107 | + stats.sweep_time_s += time.perf_counter() - s0 | |
| 108 | + stats.events.append({"rollback_at": i}) | |
| 109 | + return True | |
| 110 | + | |
| 111 | + while len(tokens) < max_tokens: | |
| 112 | + nxt, margin = _greedy_step(base_model, inp, cache) | |
| 113 | + if nxt == tokenizer.eos_token_id: | |
| 114 | + sweep() | |
| 115 | + break | |
| 116 | + tokens.append(nxt) | |
| 117 | + flags.append(margin < tau) | |
| 118 | + if margin < tau: | |
| 119 | + stats.deferred += 1 | |
| 120 | + inp = mx.array([[nxt]]) | |
| 121 | + if len(tokens) - verified_upto >= window: | |
| 122 | + if sweep(): | |
| 123 | + inp = mx.array([[tokens[-1]]]) # corrected token, not yet fed | |
| 124 | + else: | |
| 125 | + sweep() | |
| 126 | + | |
| 127 | + stats.tokens_out = len(tokens) | |
| 128 | + stats.gen_time_s = time.perf_counter() - t0 - stats.sweep_time_s | |
| 129 | + return tokens, stats | |
added
experiments/micro/expG_decision_stability/expG_8B.log
+4 −0
@@ -0,0 +1,4 @@ | ||
| 1 | +loading reference mlx-community/Qwen3-8B-bf16 … | |
| 2 | + Fetching 12 files: 0%| | 0/12 [00:00<?, ?it/s]Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. | |
| 3 | + Fetching 12 files: 17%|█▋ | 2/12 [00:00<00:00, 15.49it/s] Fetching 12 files: 42%|████▏ | 5/12 [00:00<00:00, 18.46it/s] Fetching 12 files: 58%|█████▊ | 7/12 [00:00<00:00, 17.87it/s] Fetching 12 files: 67%|██████▋ | 8/12 [00:20<00:00, 17.87it/s] Fetching 12 files: 75%|███████▌ | 9/12 [00:38<00:20, 6.86s/it] Fetching 12 files: 83%|████████▎ | 10/12 [01:43<00:37, 18.88s/it] Fetching 12 files: 92%|█████████▏| 11/12 [01:44<00:14, 14.68s/it] Fetching 12 files: 100%|██████████| 12/12 [01:50<00:00, 9.21s/it] | |
| 4 | + generated code | |
| 5 | ||