| 58 | self.registry = registry |
58 | self.registry = registry |
| 59 | self.jobs = jobs |
59 | self.jobs = jobs |
| 60 | self.downloader = downloader |
60 | self.downloader = downloader |
| |
61 | + self._preferred_runtime = "mlx" |
| 61 | |
62 | |
| 62 | def _api(self): |
63 | def _api(self): |
| 63 | from huggingface_hub import HfApi |
64 | from huggingface_hub import HfApi |
| 84 | async def _scan(self, job: Job, opt: dict) -> dict: |
85 | async def _scan(self, job: Job, opt: dict) -> dict: |
| 85 | api = self._api() |
86 | api = self._api() |
| 86 | budget, absolute = await self.registry.budgets() |
87 | budget, absolute = await self.registry.budgets() |
| |
88 | + self._preferred_runtime = str(await self.db.get_setting("preferred_runtime", "mlx") or "mlx") |
| 87 | max_ram = opt["max_ram_gb"] or budget |
89 | max_ram = opt["max_ram_gb"] or budget |
| 88 | installed = {r["repository"] for r in await self.db.fetchall("SELECT repository FROM models WHERE installed=1 AND repository IS NOT NULL")} |
90 | installed = {r["repository"] for r in await self.db.fetchall("SELECT repository FROM models WHERE installed=1 AND repository IS NOT NULL")} |
| 89 | installed_keys = set() |
91 | installed_keys = set() |
| 264 | kv = kv_bytes_per_token(cfg.get("n_layers"), cfg.get("n_kv_heads"), cfg.get("head_dim"), 16, |
266 | kv = kv_bytes_per_token(cfg.get("n_layers"), cfg.get("n_kv_heads"), cfg.get("head_dim"), 16, |
| 265 | cfg.get("full_attention_layers"), cfg.get("sliding_window")) |
267 | cfg.get("full_attention_layers"), cfg.get("sliding_window")) |
| 266 | if not kv and param_count: |
268 | if not kv and param_count: |
| 267 | − kv = int(130_000 * (param_count / 8e9) ** 0.6) |
269 | + # No config in the listing: ~130 KB/token for an 8B dense model, scaling gently; MoE models |
| |
270 | + # (e.g. 30B-A3B) have KV sized like their active parameters, not their total. |
| |
271 | + _, active = formats.parse_param_count_from_name(name) |
| |
272 | + ref = active or param_count |
| |
273 | + kv = int(130_000 * max(0.3, (ref / 8e9)) ** 0.5) |
| 268 | vision = flags.get("vision", False) or bool(cfg.get("vision")) |
274 | vision = flags.get("vision", False) or bool(cfg.get("vision")) |
| 269 | embedding = flags.get("embedding", False) |
275 | embedding = flags.get("embedding", False) |
| 270 | reranker = flags.get("reranker", False) |
276 | reranker = flags.get("reranker", False) |
| 282 | return None |
288 | return None |
| 283 | # quantization quality policy |
289 | # quantization quality policy |
| 284 | pol = self._quant_policy(param_count, bits) |
290 | pol = self._quant_policy(param_count, bits) |
| 285 | − score = self._score(c, comp, bits, param_count, pol, task) |
291 | + score = self._score(c, comp, bits, param_count, pol, task, self._preferred_runtime) |
| 286 | return { |
292 | return { |
| 287 | "repo_id": c["repo"], "runtime": c["runtime"], "family": family, "base_model": _base_key(c["repo"], c["tags"]), |
293 | "repo_id": c["repo"], "runtime": c["runtime"], "family": family, "base_model": _base_key(c["repo"], c["tags"]), |
| 288 | "name": name, "task": task, "quantization": quant, "parameter_count": param_count, "download_bytes": download_bytes, |
294 | "name": name, "task": task, "quantization": quant, "parameter_count": param_count, "download_bytes": download_bytes, |
| 309 | return "ok" if bits >= 3 else "too_low" |
315 | return "ok" if bits >= 3 else "too_low" |
| 310 | |
316 | |
| 311 | @staticmethod |
317 | @staticmethod |
| 312 | − def _score(c: dict, comp, bits, params, pol: str, task: str) -> float: |
318 | + def _score(c: dict, comp, bits, params, pol: str, task: str, preferred_runtime: str = "mlx") -> float: |
| 313 | import math |
319 | import math |
| 314 | s = math.log10(max(10, c["downloads"])) * 10 |
320 | s = math.log10(max(10, c["downloads"])) * 10 |
| |
321 | + if c["runtime"] == preferred_runtime: |
| |
322 | + s += 14 # Apple Silicon: the preferred runtime wins ties against more-downloaded GGUF mirrors |
| 315 | s += math.log10(max(1, c["likes"])) * 3 |
323 | s += math.log10(max(1, c["likes"])) * 3 |
| 316 | s += {"ok": 15, "low": 5, "too_low": -20, "unknown": 0}[pol] |
324 | s += {"ok": 15, "low": 5, "too_low": -20, "unknown": 0}[pol] |
| 317 | s += {compat.COMPATIBLE: 10, compat.RESTRICTED: 4, compat.EXPERIMENTAL: -5}.get(comp.status, 0) |
325 | s += {compat.COMPATIBLE: 10, compat.RESTRICTED: 4, compat.EXPERIMENTAL: -5}.get(comp.status, 0) |
| 379 | """A curated slot list (small/medium/large general, coding, reasoning, vision, embedding, reranker) |
387 | """A curated slot list (small/medium/large general, coding, reasoning, vision, embedding, reranker) |
| 380 | filled from the latest harvest, best score first.""" |
388 | filled from the latest harvest, best score first.""" |
| 381 | rows = await self.candidates(limit=1000) |
389 | rows = await self.candidates(limit=1000) |
| |
390 | + def p(r): |
| |
391 | + return (r["parameter_count"] or 0) / 1e9 |
| |
392 | + |
| |
393 | + def g(r): |
| |
394 | + return r["estimated_ram_gb"] or 0 |
| |
395 | + |
| 382 | slots = { |
396 | slots = { |
| 383 | − "small general": lambda r: r["task"] == "general" and (r["estimated_ram_gb"] or 0) < 8, |
397 | + "small general": lambda r: r["task"] in ("general", "vision") and 3 <= p(r) <= 12 and g(r) < 12, |
| 384 | − "small coding": lambda r: r["task"] == "coding" and (r["estimated_ram_gb"] or 0) < 12, |
398 | + "small coding": lambda r: r["task"] == "coding" and 3 <= p(r) <= 12, |
| 385 | − "small reasoning": lambda r: r["task"] == "reasoning" and (r["estimated_ram_gb"] or 0) < 12, |
399 | + "small reasoning": lambda r: r["task"] == "reasoning" and 3 <= p(r) <= 12, |
| 386 | − "medium general": lambda r: r["task"] == "general" and 8 <= (r["estimated_ram_gb"] or 0) < 22, |
400 | + "medium general": lambda r: r["task"] in ("general", "vision") and 12 < p(r) <= 32 and g(r) < 26, |
| 387 | − "medium coding": lambda r: r["task"] == "coding" and 12 <= (r["estimated_ram_gb"] or 0) < 25, |
401 | + "medium coding": lambda r: r["task"] == "coding" and 12 < p(r) <= 40, |
| 388 | − "large general": lambda r: r["task"] == "general" and 22 <= (r["estimated_ram_gb"] or 0) <= 45, |
402 | + "large general": lambda r: r["task"] in ("general", "vision") and p(r) > 24 and 20 <= g(r) <= 45, |
| 389 | − "large reasoning": lambda r: r["task"] == "reasoning" and 12 <= (r["estimated_ram_gb"] or 0) <= 45, |
403 | + "large reasoning": lambda r: r["task"] == "reasoning" and p(r) > 12 and g(r) <= 45, |
| 390 | − "vision": lambda r: r["task"] == "vision", |
404 | + "vision": lambda r: r["task"] == "vision" and p(r) >= 7, |
| 391 | "embedding": lambda r: r["task"] == "embedding", |
405 | "embedding": lambda r: r["task"] == "embedding", |
| 392 | "reranker": lambda r: r["task"] == "reranker", |
406 | "reranker": lambda r: r["task"] == "reranker", |
| 393 | } |
407 | } |
| 394 | |
408 | |