Harvester: correct MLX weight estimates from HF metadata, flag installed mirrors; docs: managed-firewall WireGuard relay
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
3 changed files +14 −6
modified
docs/deployment.md
+5 −1
@@ -24,7 +24,11 @@ mld logs llm-api | ||
| 24 | 24 | |
| 25 | 25 | Manifest: `M1M32:~/dispatch/apps/llm-api.json` (dir `~/apps/llm-api`, port 8300, health `/health`, tunnel `www.llm-api.io` on BHS64, pinned to `M1M64`). Secrets (`ADMIN_PASSWORD`, `HF_TOKEN`) live in the manifest's `env` on M1M32 only and are written into the PM2 environment; the app also reads `~/llm-api/.env`. |
| 26 | 26 | |
| 27 | −M1M64 is a *reserved* node: always deploy with `--node M1M64`. It has no graphical session guaranteed at boot, so PM2 runs as a system LaunchDaemon (`mld prepare` does this). | |
| 27 | +M1M64 is a *reserved* node: always deploy with `--node M1M64`. | |
| 28 | + | |
| 29 | +### WireGuard relay on M1M64 (managed firewall) | |
| 30 | + | |
| 31 | +The API binds `127.0.0.1:8300`, but Caddy on BHS64 reaches the node at its WireGuard address `10.67.0.40:8300`. On this rented Macly Mac the macOS application firewall is **MDM-managed** (`socketfilterfw` refuses changes) and blocks Homebrew binaries such as `socat` from accepting connections on the WireGuard interface (the cluster's usual `wg-forward-setup.sh` relay therefore fails silently: it listens, but every connection is dropped). The relay is instead an Apple-signed `/usr/bin/ssh -L 10.67.0.40:8300:127.0.0.1:8300 simon@127.0.0.1` (key `~/.ssh/id_ed25519_wgfwd`, authorized on the node itself), run as the PM2 process `llm-api-wgfwd` and declared as the third process of the manifest. Verify with `ssh BHS64 curl -s http://10.67.0.40:8300/health`. It has no graphical session guaranteed at boot, so PM2 runs as a system LaunchDaemon (`mld prepare` does this). | |
| 28 | 32 | |
| 29 | 33 | ## Manual (without mld) |
| 30 | 34 | |
modified
docs/troubleshooting.md
+1 −0
@@ -15,6 +15,7 @@ | ||
| 15 | 15 | | Download fails 429 | Downloads history | HF rate limit: retry later | |
| 16 | 16 | | `INSUFFICIENT_DISK` | System → Storage | `MIN_FREE_DISK_GB` reserve; delete unused models | |
| 17 | 17 | | Dashboard 503 "not reachable" | `pm2 logs llm-api-web`, port 8301 | Next not running / build failed (`pnpm build`) | |
| 18 | +| Public site 502 while `localhost:8300` is fine | `ssh BHS64 curl http://10.67.0.40:8300/health`, `pm2 ls` (`llm-api-wgfwd`) | WireGuard relay down: restart `llm-api-wgfwd` (managed firewall blocks non-Apple listeners on the wg address, see deployment.md) | | |
| 18 | 19 | | Login loop | cookies | behind HTTPS set `SECURE_COOKIES=true`; ensure the proxy passes `X-Forwarded-Proto` | |
| 19 | 20 | | Model shows loaded after a crash | restart server | stale state is cleared at startup (`workers.json`) | |
| 20 | 21 | | GPU shows `—` | `ioreg -r -c IOAccelerator` | metric unavailable on this macOS; degrades gracefully | |
modified
server/llm_api/harvester.py
+8 −5
@@ -44,7 +44,7 @@ def _base_key(repo: str, tags: list[str]) -> str: | ||
| 44 | 44 | if not bm: |
| 45 | 45 | bm = next((t.split(":", 1)[1] for t in tags if t.startswith("base_model:") and "finetune:" not in t), None) |
| 46 | 46 | if bm: |
| 47 | − return bm.lower() | |
| 47 | + return bm.split("/")[-1].lower() | |
| 48 | 48 | name = repo.split("/")[-1].lower() |
| 49 | 49 | name = re.sub(r"-(\d+bit|q\d[_a-z0-9]*|iq\d[_a-z0-9]*|mxfp\d|nvfp4|bf16|fp16|f16|fp8|dwq|optiq|gguf|mlx|4bit|8bit)+$", "", name) |
| 50 | 50 | name = re.sub(r"-(gguf|mlx)$", "", name) |
@@ -89,8 +89,10 @@ class Harvester: | ||
| 89 | 89 | max_ram = opt["max_ram_gb"] or budget |
| 90 | 90 | installed = {r["repository"] for r in await self.db.fetchall("SELECT repository FROM models WHERE installed=1 AND repository IS NOT NULL")} |
| 91 | 91 | installed_keys = set() |
| 92 | − for r in await self.db.fetchall("SELECT name, tags FROM models WHERE installed=1"): | |
| 92 | + for r in await self.db.fetchall("SELECT name, tags, repository FROM models WHERE installed=1"): | |
| 93 | 93 | installed_keys.add(_base_key(r["name"], json.loads(r["tags"] or "[]"))) |
| 94 | + if r["repository"]: | |
| 95 | + installed_keys.add(_base_key(r["repository"], [])) | |
| 94 | 96 | sources: list[tuple[str, str]] = [] |
| 95 | 97 | for rt in opt["runtimes"]: |
| 96 | 98 | authors = opt["authors"] or DEFAULT_SOURCES.get(rt, []) |
@@ -220,10 +222,11 @@ class Harvester: | ||
| 220 | 222 | total_params = getattr(st, "total", None) or (st.get("total") if isinstance(st, dict) else None) |
| 221 | 223 | params_by_dtype = getattr(st, "parameters", None) or (st.get("parameters") if isinstance(st, dict) else None) |
| 222 | 224 | if params_by_dtype and isinstance(params_by_dtype, dict): |
| 223 | − # packed uint32 quantized weights: count * 32 / bits | |
| 225 | + # Hugging Face reports quantized uint32 tensors by *unpacked* parameter count | |
| 226 | + # (an 8B 4-bit repo shows U32 ≈ 8.0e9): bytes = params × bits / 8, plus ~12 % for scales/biases. | |
| 224 | 227 | for dt, cnt in params_by_dtype.items(): |
| 225 | 228 | if dt in ("U32", "I32") and bits: |
| 226 | − weights_bytes += int(cnt * 4) | |
| 229 | + weights_bytes += int(cnt * bits / 8 * 1.12) | |
| 227 | 230 | elif dt in ("F16", "BF16"): |
| 228 | 231 | weights_bytes += int(cnt * 2) |
| 229 | 232 | elif dt in ("F32",): |
@@ -242,7 +245,7 @@ class Harvester: | ||
| 242 | 245 | if bits and total_params and bits < 16 and weights_bytes: |
| 243 | 246 | # 'total' counts packed uint32 elements as 1 param; approximate real param count |
| 244 | 247 | pass |
| 245 | − param_count = pc or (int(weights_bytes * 8 / bits) if bits and weights_bytes else None) | |
| 248 | + param_count = pc or (int(total_params) if total_params else None) or (int(weights_bytes * 8 / bits) if bits and weights_bytes else None) | |
| 246 | 249 | download_bytes = int(weights_bytes * 1.02) |
| 247 | 250 | if not weights_bytes: |
| 248 | 251 | return None |
| 249 | 252 | |