1"""llama.cpp runtime: spawns `llama-server` (Metal) for GGUF models."""23from __future__ import annotations45import os6import shutil78import httpx910from .base import RuntimeAdapter, WorkerHandle111213class LlamaCppAdapter(RuntimeAdapter):14 name = "llamacpp"1516 def binary(self) -> str | None:17 b = self.settings.llama_server_bin18 if os.path.isabs(b) and os.path.exists(b):19 return b20 found = shutil.which(b)21 if found:22 return found23 for cand in ("/opt/homebrew/bin/llama-server", "/usr/local/bin/llama-server"):24 if os.path.exists(cand):25 return cand26 return None2728 def available(self) -> bool:29 return self.binary() is not None3031 def build_command(self, model: dict, port: int, context: int) -> list[str]:32 b = self.binary()33 assert b, "llama-server not found"34 threads = max(4, (os.cpu_count() or 8) - 2)35 cmd = [b, "-m", model["weights_file"], "--host", "127.0.0.1", "--port", str(port), "-ngl", "999",36 "-c", str(context), "--alias", model["id"], "--no-webui", "-np", "1", "-fa", "auto",37 "-t", str(threads), "--cache-reuse", "256", "--metrics", "--slots", "--jinja"]38 if model.get("embedding"):39 cmd += ["--embeddings", "-ub", "2048", "-b", "2048"]40 # Pooling: model default unless overridden41 pooling = (model.get("overrides") or {}).get("pooling")42 if pooling:43 cmd += ["--pooling", pooling]44 elif model.get("reranker"):45 cmd += ["--reranking"]46 else:47 if model.get("thinking"):48 cmd += ["--reasoning-format", "deepseek"]49 if model.get("mmproj_file"):50 cmd += ["--mmproj", model["mmproj_file"]]51 overrides = model.get("overrides") or {}52 if overrides.get("kv_bits") in (4, 8):53 t = "q4_0" if overrides["kv_bits"] == 4 else "q8_0"54 cmd += ["-ctk", t, "-ctv", t]55 for extra in overrides.get("llama_args") or []:56 if isinstance(extra, str) and not extra.startswith(("|", ";", "&", "$", "`")):57 cmd.append(extra)58 return cmd5960 async def is_ready(self, handle: WorkerHandle, client: httpx.AsyncClient) -> tuple[str, str | None]:61 try:62 r = await client.get(f"{handle.base_url}/health", timeout=3)63 except Exception:64 return "loading", None65 if r.status_code == 200:66 return "ready", None67 if r.status_code == 503:68 return "loading", None69 return "loading", None70