Harvester: correct MLX weight estimates from HF metadata, flag installed mirrors; docs: managed-firewall WireGuard relay
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
3 changed files +14 −6
modified
docs/deployment.md
+5 −1
@@ -24,7 +24,11 @@ mld logs llm-api | |||||
| 24 | 24 | ||||
| 25 | Manifest: `M1M32:~/dispatch/apps/llm-api.json` (dir `~/apps/llm-api`, port 8300, health `/health`, tunnel `www.llm-api.io` on BHS64, pinned to `M1M64`). Secrets (`ADMIN_PASSWORD`, `HF_TOKEN`) live in the manifest's `env` on M1M32 only and are written into the PM2 environment; the app also reads `~/llm-api/.env`. | 25 | Manifest: `M1M32:~/dispatch/apps/llm-api.json` (dir `~/apps/llm-api`, port 8300, health `/health`, tunnel `www.llm-api.io` on BHS64, pinned to `M1M64`). Secrets (`ADMIN_PASSWORD`, `HF_TOKEN`) live in the manifest's `env` on M1M32 only and are written into the PM2 environment; the app also reads `~/llm-api/.env`. | ||
| 26 | 26 | ||||
| 27 | −M1M64 is a *reserved* node: always deploy with `--node M1M64`. It has no graphical session guaranteed at boot, so PM2 runs as a system LaunchDaemon (`mld prepare` does this). | 27 | +M1M64 is a *reserved* node: always deploy with `--node M1M64`. | ||
| 28 | + | ||||
| 29 | +### WireGuard relay on M1M64 (managed firewall) | ||||
| 30 | + | ||||
| 31 | +The API binds `127.0.0.1:8300`, but Caddy on BHS64 reaches the node at its WireGuard address `10.67.0.40:8300`. On this rented Macly Mac the macOS application firewall is **MDM-managed** (`socketfilterfw` refuses changes) and blocks Homebrew binaries such as `socat` from accepting connections on the WireGuard interface (the cluster's usual `wg-forward-setup.sh` relay therefore fails silently: it listens, but every connection is dropped). The relay is instead an Apple-signed `/usr/bin/ssh -L 10.67.0.40:8300:127.0.0.1:8300 simon@127.0.0.1` (key `~/.ssh/id_ed25519_wgfwd`, authorized on the node itself), run as the PM2 process `llm-api-wgfwd` and declared as the third process of the manifest. Verify with `ssh BHS64 curl -s http://10.67.0.40:8300/health`. It has no graphical session guaranteed at boot, so PM2 runs as a system LaunchDaemon (`mld prepare` does this). | ||||
| 28 | 32 | ||||
| 29 | ## Manual (without mld) | 33 | ## Manual (without mld) | ||
| 30 | 34 | ||||
modified
docs/troubleshooting.md
+1 −0
@@ -15,6 +15,7 @@ | |||||
| 15 | | Download fails 429 | Downloads history | HF rate limit: retry later | | 15 | | Download fails 429 | Downloads history | HF rate limit: retry later | | ||
| 16 | | `INSUFFICIENT_DISK` | System → Storage | `MIN_FREE_DISK_GB` reserve; delete unused models | | 16 | | `INSUFFICIENT_DISK` | System → Storage | `MIN_FREE_DISK_GB` reserve; delete unused models | | ||
| 17 | | Dashboard 503 "not reachable" | `pm2 logs llm-api-web`, port 8301 | Next not running / build failed (`pnpm build`) | | 17 | | Dashboard 503 "not reachable" | `pm2 logs llm-api-web`, port 8301 | Next not running / build failed (`pnpm build`) | | ||
| 18 | +| Public site 502 while `localhost:8300` is fine | `ssh BHS64 curl http://10.67.0.40:8300/health`, `pm2 ls` (`llm-api-wgfwd`) | WireGuard relay down: restart `llm-api-wgfwd` (managed firewall blocks non-Apple listeners on the wg address, see deployment.md) | | ||||
| 18 | | Login loop | cookies | behind HTTPS set `SECURE_COOKIES=true`; ensure the proxy passes `X-Forwarded-Proto` | | 19 | | Login loop | cookies | behind HTTPS set `SECURE_COOKIES=true`; ensure the proxy passes `X-Forwarded-Proto` | | ||
| 19 | | Model shows loaded after a crash | restart server | stale state is cleared at startup (`workers.json`) | | 20 | | Model shows loaded after a crash | restart server | stale state is cleared at startup (`workers.json`) | | ||
| 20 | | GPU shows `—` | `ioreg -r -c IOAccelerator` | metric unavailable on this macOS; degrades gracefully | | 21 | | GPU shows `—` | `ioreg -r -c IOAccelerator` | metric unavailable on this macOS; degrades gracefully | | ||
modified
server/llm_api/harvester.py
+8 −5
@@ -44,7 +44,7 @@ def _base_key(repo: str, tags: list[str]) -> str: | |||||
| 44 | if not bm: | 44 | if not bm: | ||
| 45 | bm = next((t.split(":", 1)[1] for t in tags if t.startswith("base_model:") and "finetune:" not in t), None) | 45 | bm = next((t.split(":", 1)[1] for t in tags if t.startswith("base_model:") and "finetune:" not in t), None) | ||
| 46 | if bm: | 46 | if bm: | ||
| 47 | − return bm.lower() | 47 | + return bm.split("/")[-1].lower() | ||
| 48 | name = repo.split("/")[-1].lower() | 48 | name = repo.split("/")[-1].lower() | ||
| 49 | name = re.sub(r"-(\d+bit|q\d[_a-z0-9]*|iq\d[_a-z0-9]*|mxfp\d|nvfp4|bf16|fp16|f16|fp8|dwq|optiq|gguf|mlx|4bit|8bit)+$", "", name) | 49 | name = re.sub(r"-(\d+bit|q\d[_a-z0-9]*|iq\d[_a-z0-9]*|mxfp\d|nvfp4|bf16|fp16|f16|fp8|dwq|optiq|gguf|mlx|4bit|8bit)+$", "", name) | ||
| 50 | name = re.sub(r"-(gguf|mlx)$", "", name) | 50 | name = re.sub(r"-(gguf|mlx)$", "", name) | ||
@@ -89,8 +89,10 @@ class Harvester: | |||||
| 89 | max_ram = opt["max_ram_gb"] or budget | 89 | max_ram = opt["max_ram_gb"] or budget | ||
| 90 | installed = {r["repository"] for r in await self.db.fetchall("SELECT repository FROM models WHERE installed=1 AND repository IS NOT NULL")} | 90 | installed = {r["repository"] for r in await self.db.fetchall("SELECT repository FROM models WHERE installed=1 AND repository IS NOT NULL")} | ||
| 91 | installed_keys = set() | 91 | installed_keys = set() | ||
| 92 | − for r in await self.db.fetchall("SELECT name, tags FROM models WHERE installed=1"): | 92 | + for r in await self.db.fetchall("SELECT name, tags, repository FROM models WHERE installed=1"): | ||
| 93 | installed_keys.add(_base_key(r["name"], json.loads(r["tags"] or "[]"))) | 93 | installed_keys.add(_base_key(r["name"], json.loads(r["tags"] or "[]"))) | ||
| 94 | + if r["repository"]: | ||||
| 95 | + installed_keys.add(_base_key(r["repository"], [])) | ||||
| 94 | sources: list[tuple[str, str]] = [] | 96 | sources: list[tuple[str, str]] = [] | ||
| 95 | for rt in opt["runtimes"]: | 97 | for rt in opt["runtimes"]: | ||
| 96 | authors = opt["authors"] or DEFAULT_SOURCES.get(rt, []) | 98 | authors = opt["authors"] or DEFAULT_SOURCES.get(rt, []) | ||
@@ -220,10 +222,11 @@ class Harvester: | |||||
| 220 | total_params = getattr(st, "total", None) or (st.get("total") if isinstance(st, dict) else None) | 222 | total_params = getattr(st, "total", None) or (st.get("total") if isinstance(st, dict) else None) | ||
| 221 | params_by_dtype = getattr(st, "parameters", None) or (st.get("parameters") if isinstance(st, dict) else None) | 223 | params_by_dtype = getattr(st, "parameters", None) or (st.get("parameters") if isinstance(st, dict) else None) | ||
| 222 | if params_by_dtype and isinstance(params_by_dtype, dict): | 224 | if params_by_dtype and isinstance(params_by_dtype, dict): | ||
| 223 | − # packed uint32 quantized weights: count * 32 / bits | 225 | + # Hugging Face reports quantized uint32 tensors by *unpacked* parameter count | ||
| 226 | + # (an 8B 4-bit repo shows U32 ≈ 8.0e9): bytes = params × bits / 8, plus ~12 % for scales/biases. | ||||
| 224 | for dt, cnt in params_by_dtype.items(): | 227 | for dt, cnt in params_by_dtype.items(): | ||
| 225 | if dt in ("U32", "I32") and bits: | 228 | if dt in ("U32", "I32") and bits: | ||
| 226 | − weights_bytes += int(cnt * 4) | 229 | + weights_bytes += int(cnt * bits / 8 * 1.12) | ||
| 227 | elif dt in ("F16", "BF16"): | 230 | elif dt in ("F16", "BF16"): | ||
| 228 | weights_bytes += int(cnt * 2) | 231 | weights_bytes += int(cnt * 2) | ||
| 229 | elif dt in ("F32",): | 232 | elif dt in ("F32",): | ||
@@ -242,7 +245,7 @@ class Harvester: | |||||
| 242 | if bits and total_params and bits < 16 and weights_bytes: | 245 | if bits and total_params and bits < 16 and weights_bytes: | ||
| 243 | # 'total' counts packed uint32 elements as 1 param; approximate real param count | 246 | # 'total' counts packed uint32 elements as 1 param; approximate real param count | ||
| 244 | pass | 247 | pass | ||
| 245 | − param_count = pc or (int(weights_bytes * 8 / bits) if bits and weights_bytes else None) | 248 | + param_count = pc or (int(total_params) if total_params else None) or (int(weights_bytes * 8 / bits) if bits and weights_bytes else None) | ||
| 246 | download_bytes = int(weights_bytes * 1.02) | 249 | download_bytes = int(weights_bytes * 1.02) | ||
| 247 | if not weights_bytes: | 250 | if not weights_bytes: | ||
| 248 | return None | 251 | return None | ||
| 249 | 252 | ||||