Harvester: prefer the Apple Silicon runtime, MoE-aware KV fallback, size-constrained starter slots; README perf table

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 13 days ago (Sep 11, 2026) parent 7625424

2 changed files +43 −21

modified README.md +18 −10

		`@@ -52,16 +52,24 @@ curl https://www.llm-api.io/v1/chat/completions \`
52	52	-d '{"model": "qwen3.8-27b-4bit", "messages": [{"role": "user", "content": "Explain monetary policy."}], "stream": true}'
53	53	```
54	54
55		−## Verified on the M1 Max (2026-09-10)
56		−
57		−\| Step \| Result \|
58		−\|---\|---\|
59		−\| Cold load Qwen3-4B-Instruct-2507 (MLX 4-bit) \| 1.7 s, ~100 tok/s generation \|
60		−\| Cold load Llama-3.2-3B (MLX 4-bit) \| 1.4 s, ~170 tok/s \|
61		−\| Gemma-3-1B GGUF via llama.cpp \| 1.1 s, ~150 tok/s, streaming with `timings` \|
62		−\| Switch A → B → A \| previous worker killed, memory returned to the OS before the next load \|
63		−\| OpenAI Python SDK \| models / chat / stream / embeddings OK \|
64		−\| Memory rejection \| budget lowered → model refused with `MODEL_TOO_LARGE`, compatibility re-evaluated \|
	55	+## Verified on the M1 Max (2026-09-10, all through the API, `temperature=0`)
	56	+
	57	+\| Model \| Runtime \| Cold load \| Generation \| Memory measured \|
	58	+\|---\|---\|---\|---\|---\|
	59	+\| Qwen3-4B-Instruct-2507 4-bit \| MLX \| 1.7 s \| ~100 tok/s \| 2.4 GB \|
	60	+\| Llama-3.2-3B-Instruct 4-bit \| MLX \| 1.4 s \| ~170 tok/s \| 2.0 GB \|
	61	+\| gemma-3-1b-it Q4_K_M \| llama.cpp \| 1.1 s \| ~150 tok/s \| 0.7 GB \|
	62	+\| Qwen3.5-9B 4-bit (VLM) \| MLX (mlx-vlm) \| 3.5 s \| 77 tok/s \| 5.9 GB \|
	63	+\| gpt-oss-20b MXFP4 (reasoning → `reasoning_content`) \| MLX \| 4.9 s \| 73 tok/s \| 12.0 GB \|
	64	+\| Devstral-Small-2-24B 4-bit \| MLX \| 4.7 s \| 22 tok/s \| 14.0 GB \|
	65	+\| gemma-4-26B-A4B 4-bit (MoE) \| MLX \| 6.0 s \| 80 tok/s \| 15.0 GB \|
	66	+\| Qwen3.8-27B 4-bit \| MLX \| ~5 s \| 20 tok/s \| 16.3 GB \|
	67	+\| Qwen3.6-35B-A3B 4-bit (MoE) \| MLX \| ~5 s \| 66 tok/s \| 20.5 GB \|
	68	+\| Qwen3-Coder-30B-A3B 4-bit \| MLX \| ~5 s \| 68 tok/s \| 17.2 GB \|
	69	+\| Llama-3.3-70B-Instruct 4-bit (XL, ctx 8K) \| MLX \| 9.3 s \| 8.6 tok/s \| 37.7 GB worker · 47 GB system · no swap \|
	70	+\| Qwen3-Embedding-0.6B 8-bit · Qwen3-Reranker-0.6B 4-bit · embeddinggemma-300M (GGUF) \| MLX / llama.cpp \| < 2 s \| — \| < 1 GB, can stay resident \|
	71	+
	72	+Also verified: A → B → A switching releases the previous worker's memory before the next load (e.g. 20.9 GB back after evicting Qwen3.6-35B); OpenAI Python SDK (models, chat, streaming, embeddings); memory rejection (`MODEL_TOO_LARGE`) when the budget is lowered; Harvester scan of 5 Hugging Face authors → 361 repos listed, 307 candidates, 205 unique base models in ~90 s.
65	73
66	74	## Tests
67	75

modified server/llm_api/harvester.py +25 −11

		`@@ -58,6 +58,7 @@ class Harvester:`
58	58	self.registry = registry
59	59	self.jobs = jobs
60	60	self.downloader = downloader
	61	+ self._preferred_runtime = "mlx"
61	62
62	63	def _api(self):
63	64	from huggingface_hub import HfApi
		`@@ -84,6 +85,7 @@ class Harvester:`
84	85	async def _scan(self, job: Job, opt: dict) -> dict:
85	86	api = self._api()
86	87	budget, absolute = await self.registry.budgets()
	88	+ self._preferred_runtime = str(await self.db.get_setting("preferred_runtime", "mlx") or "mlx")
87	89	max_ram = opt["max_ram_gb"] or budget
88	90	installed = {r["repository"] for r in await self.db.fetchall("SELECT repository FROM models WHERE installed=1 AND repository IS NOT NULL")}
89	91	installed_keys = set()
		`@@ -264,7 +266,11 @@ class Harvester:`
264	266	kv = kv_bytes_per_token(cfg.get("n_layers"), cfg.get("n_kv_heads"), cfg.get("head_dim"), 16,
265	267	cfg.get("full_attention_layers"), cfg.get("sliding_window"))
266	268	if not kv and param_count:
267		− kv = int(130_000 * (param_count / 8e9) ** 0.6)
	269	+ # No config in the listing: ~130 KB/token for an 8B dense model, scaling gently; MoE models
	270	+ # (e.g. 30B-A3B) have KV sized like their active parameters, not their total.
	271	+ _, active = formats.parse_param_count_from_name(name)
	272	+ ref = active or param_count
	273	+ kv = int(130_000 * max(0.3, (ref / 8e9)) ** 0.5)
268	274	vision = flags.get("vision", False) or bool(cfg.get("vision"))
269	275	embedding = flags.get("embedding", False)
270	276	reranker = flags.get("reranker", False)
		`@@ -282,7 +288,7 @@ class Harvester:`
282	288	return None
283	289	# quantization quality policy
284	290	pol = self._quant_policy(param_count, bits)
285		− score = self._score(c, comp, bits, param_count, pol, task)
	291	+ score = self._score(c, comp, bits, param_count, pol, task, self._preferred_runtime)
286	292	return {
287	293	"repo_id": c["repo"], "runtime": c["runtime"], "family": family, "base_model": _base_key(c["repo"], c["tags"]),
288	294	"name": name, "task": task, "quantization": quant, "parameter_count": param_count, "download_bytes": download_bytes,
		`@@ -309,9 +315,11 @@ class Harvester:`
309	315	return "ok" if bits >= 3 else "too_low"
310	316
311	317	@staticmethod
312		− def _score(c: dict, comp, bits, params, pol: str, task: str) -> float:
	318	+ def _score(c: dict, comp, bits, params, pol: str, task: str, preferred_runtime: str = "mlx") -> float:
313	319	import math
314	320	s = math.log10(max(10, c["downloads"])) * 10
	321	+ if c["runtime"] == preferred_runtime:
	322	+ s += 14 # Apple Silicon: the preferred runtime wins ties against more-downloaded GGUF mirrors
315	323	s += math.log10(max(1, c["likes"])) * 3
316	324	s += {"ok": 15, "low": 5, "too_low": -20, "unknown": 0}[pol]
317	325	s += {compat.COMPATIBLE: 10, compat.RESTRICTED: 4, compat.EXPERIMENTAL: -5}.get(comp.status, 0)
		`@@ -379,15 +387,21 @@ class Harvester:`
379	387	"""A curated slot list (small/medium/large general, coding, reasoning, vision, embedding, reranker)
380	388	filled from the latest harvest, best score first."""
381	389	rows = await self.candidates(limit=1000)
	390	+ def p(r):
	391	+ return (r["parameter_count"] or 0) / 1e9
	392	+
	393	+ def g(r):
	394	+ return r["estimated_ram_gb"] or 0
	395	+
382	396	slots = {
383		− "small general": lambda r: r["task"] == "general" and (r["estimated_ram_gb"] or 0) < 8,
384		− "small coding": lambda r: r["task"] == "coding" and (r["estimated_ram_gb"] or 0) < 12,
385		− "small reasoning": lambda r: r["task"] == "reasoning" and (r["estimated_ram_gb"] or 0) < 12,
386		− "medium general": lambda r: r["task"] == "general" and 8 <= (r["estimated_ram_gb"] or 0) < 22,
387		− "medium coding": lambda r: r["task"] == "coding" and 12 <= (r["estimated_ram_gb"] or 0) < 25,
388		− "large general": lambda r: r["task"] == "general" and 22 <= (r["estimated_ram_gb"] or 0) <= 45,
389		− "large reasoning": lambda r: r["task"] == "reasoning" and 12 <= (r["estimated_ram_gb"] or 0) <= 45,
390		− "vision": lambda r: r["task"] == "vision",
	397	+ "small general": lambda r: r["task"] in ("general", "vision") and 3 <= p(r) <= 12 and g(r) < 12,
	398	+ "small coding": lambda r: r["task"] == "coding" and 3 <= p(r) <= 12,
	399	+ "small reasoning": lambda r: r["task"] == "reasoning" and 3 <= p(r) <= 12,
	400	+ "medium general": lambda r: r["task"] in ("general", "vision") and 12 < p(r) <= 32 and g(r) < 26,
	401	+ "medium coding": lambda r: r["task"] == "coding" and 12 < p(r) <= 40,
	402	+ "large general": lambda r: r["task"] in ("general", "vision") and p(r) > 24 and 20 <= g(r) <= 45,
	403	+ "large reasoning": lambda r: r["task"] == "reasoning" and p(r) > 12 and g(r) <= 45,
	404	+ "vision": lambda r: r["task"] == "vision" and p(r) >= 7,
391	405	"embedding": lambda r: r["task"] == "embedding",
392	406	"reranker": lambda r: r["task"] == "reranker",
393	407	}
394	408