"""llama.cpp runtime: spawns `llama-server` (Metal) for GGUF models.""" from __future__ import annotations import os import shutil import httpx from .base import RuntimeAdapter, WorkerHandle class LlamaCppAdapter(RuntimeAdapter): name = "llamacpp" def binary(self) -> str | None: b = self.settings.llama_server_bin if os.path.isabs(b) and os.path.exists(b): return b found = shutil.which(b) if found: return found for cand in ("/opt/homebrew/bin/llama-server", "/usr/local/bin/llama-server"): if os.path.exists(cand): return cand return None def available(self) -> bool: return self.binary() is not None def build_command(self, model: dict, port: int, context: int) -> list[str]: b = self.binary() assert b, "llama-server not found" threads = max(4, (os.cpu_count() or 8) - 2) cmd = [b, "-m", model["weights_file"], "--host", "127.0.0.1", "--port", str(port), "-ngl", "999", "-c", str(context), "--alias", model["id"], "--no-webui", "-np", "1", "-fa", "auto", "-t", str(threads), "--cache-reuse", "256", "--metrics", "--slots", "--jinja"] if model.get("embedding"): cmd += ["--embeddings", "-ub", "2048", "-b", "2048"] # Pooling: model default unless overridden pooling = (model.get("overrides") or {}).get("pooling") if pooling: cmd += ["--pooling", pooling] elif model.get("reranker"): cmd += ["--reranking"] else: if model.get("thinking"): cmd += ["--reasoning-format", "deepseek"] if model.get("mmproj_file"): cmd += ["--mmproj", model["mmproj_file"]] overrides = model.get("overrides") or {} if overrides.get("kv_bits") in (4, 8): t = "q4_0" if overrides["kv_bits"] == 4 else "q8_0" cmd += ["-ctk", t, "-ctv", t] for extra in overrides.get("llama_args") or []: if isinstance(extra, str) and not extra.startswith(("|", ";", "&", "$", "`")): cmd.append(extra) return cmd async def is_ready(self, handle: WorkerHandle, client: httpx.AsyncClient) -> tuple[str, str | None]: try: r = await client.get(f"{handle.base_url}/health", timeout=3) except Exception: return "loading", None if r.status_code == 200: return "ready", None if r.status_code == 503: return "loading", None return "loading", None