SPB Git
spboucher.ai

spb/maclustr-dispatch

Public

ngrok → MacLustr Tunnel intégré à mld ; mld heal (auto-réparation) ; mld harden (résilience coupure/MAJ)

- tunnel.py : routes Caddy et pairs WireGuard via tunnelctl (BHS64/R9128) ; deploy/move repointent la route publique
  vers le nœud courant, raccordement wg1 automatique ; retire supprime la route ; mld tunnel status|peer|route|rm
- manifest : bloc ngrok obsolète converti en tunnel au chargement, runtime ngrok retiré, legacy_pm2 nettoyé au start
- heal.py : sonde 1 SSH/nœud (PM2, launchd, wg1, HTTP), répare (resurrect/start/restart/kickstart/route), garde-fous,
  LaunchDaemon io.maclustr.mld-heal toutes les 5 min (mld heal --install)
- harden.py : autorestart/restartfreeze/sleep 0/womp, MAJ macOS+App Store off, Tailscale auto-update off,
  auto-login kcpassword+autoLoginUser (sauf FileVault / nœuds loués), vérif pm2 startup + wg1 ; appelé par prepare
- registry.update_many (une seule diffusion pour status --live / heal)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 10, 2026) parent 415501a Browse files

12 changed files +943 −65

12 changed files
Unified Split
added assets/wg-node-setup.sh +70 −0
@@ -0,0 +1,70 @@
1+#!/bin/bash
2+# wg-node-setup.sh — installe WireGuard (wireguard-go + wg-quick) sur un Mac du cluster et le raccorde au hub R9128.
3+# Usage : bash wg-node-setup.sh <ip-wg du nœud> <pubkey-serveur> <mot-de-passe-sudo> [iface=$IF] [endpoint=51.255.75.61:51820]
4+# $IF = hub R9128 (France, 10.66.0.x) ; wg1 = hub BHS64 (Québec, 10.67.0.x). Un LaunchDaemon par interface.
5+# Imprime « PUBKEY <clé> » à la fin (à donner à tunnelctl peer add sur la passerelle).
6+set -euo pipefail
7+ADDR=${1:?ip wg}; SERVER_PUB=${2:?pubkey serveur}; PW=${3:?sudo}; IF=${4:-wg0}; ENDPOINT=${5:-51.255.75.61:51820}
8+SUBNET=${ADDR%.*} # ex. 10.67.0
9+BREW=/opt/homebrew/bin/brew
10+# NB : sudo -S lit le mot de passe sur stdin → ne jamais combiner sudo_ avec un heredoc/pipe de contenu.
11+sudo_(){ echo "$PW" | sudo -S -p '' "$@"; }
12+export PATH=/opt/homebrew/bin:/opt/homebrew/sbin:$PATH
13+[ -x "$BREW" ] || { echo "Homebrew absent"; exit 2; }
14+if ! [ -x /opt/homebrew/bin/wg-quick ]; then HOMEBREW_NO_AUTO_UPDATE=1 "$BREW" install -q wireguard-tools >/dev/null 2>&1 || HOMEBREW_NO_AUTO_UPDATE=1 "$BREW" install wireguard-tools; fi
15+[ -x /opt/homebrew/bin/wireguard-go ] || HOMEBREW_NO_AUTO_UPDATE=1 "$BREW" install -q wireguard-go >/dev/null 2>&1 || true
16+ETC=/opt/homebrew/etc/wireguard
17+sudo_ mkdir -p "$ETC"
18+if ! sudo_ test -f "$ETC/$IF.key"; then
19+ K=$(/opt/homebrew/bin/wg genkey); T=$(mktemp); chmod 600 "$T"; printf '%s\n' "$K" > "$T"
20+ sudo_ install -m 600 -o root "$T" "$ETC/$IF.key"; rm -f "$T"
21+fi
22+PRIV=$(sudo_ cat "$ETC/$IF.key")
23+PUB=$(printf '%s' "$PRIV" | /opt/homebrew/bin/wg pubkey)
24+TMPC=$(mktemp); chmod 600 "$TMPC"
25+{
26+ echo "[Interface]"
27+ echo "PrivateKey = $PRIV"
28+ echo "Address = $ADDR/24"
29+ echo "MTU = 1380"
30+ echo
31+ echo "[Peer]"
32+ echo "# hub R9128 (OVH)"
33+ echo "PublicKey = $SERVER_PUB"
34+ echo "Endpoint = $ENDPOINT"
35+ echo "AllowedIPs = $SUBNET.0/24"
36+ echo "PersistentKeepalive = 25"
37+} > "$TMPC"
38+sudo_ install -m 600 -o root "$TMPC" "$ETC/$IF.conf"; rm -f "$TMPC"
39+
40+# LaunchDaemon : monte $IF au boot (wireguard-go reste en arrière-plan) et le remonte s'il tombe.
41+TMPP=$(mktemp)
42+cat > "$TMPP" <<PLIST
43+<?xml version="1.0" encoding="UTF-8"?>
44+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
45+<plist version="1.0"><dict>
46+ <key>Label</key><string>io.maclustr.wireguard-$IF</string>
47+ <key>ProgramArguments</key><array>
48+ <string>/bin/bash</string><string>-c</string>
49+ <string>export PATH=/opt/homebrew/bin:/opt/homebrew/sbin:/usr/bin:/bin:/usr/sbin:/sbin; up(){ n=\$(cat /var/run/wireguard/$IF.name 2>/dev/null) &amp;&amp; [ -n "\$n" ] &amp;&amp; /opt/homebrew/bin/wg show "\$n" >/dev/null 2>&amp;1; }; while true; do if ! up; then /opt/homebrew/bin/wg-quick down $IF >/dev/null 2>&amp;1; /opt/homebrew/bin/wg-quick up $IF || true; fi; sleep 30; done</string>
50+ </array>
51+ <key>RunAtLoad</key><true/>
52+ <key>KeepAlive</key><true/>
53+ <key>StandardOutPath</key><string>/var/log/maclustr-wireguard-$IF.log</string>
54+ <key>StandardErrorPath</key><string>/var/log/maclustr-wireguard-$IF.log</string>
55+</dict></plist>
56+PLIST
57+sudo_ install -m 644 -o root -g wheel "$TMPP" /Library/LaunchDaemons/io.maclustr.wireguard-$IF.plist; rm -f "$TMPP"
58+sudo_ launchctl bootout system/io.maclustr.wireguard-$IF >/dev/null 2>&1 || true
59+sudo_ /opt/homebrew/bin/wg-quick down $IF >/dev/null 2>&1 || true
60+sleep 2
61+ok=0; for i in 1 2 3 4 5 6; do
62+ if sudo_ launchctl bootstrap system /Library/LaunchDaemons/io.maclustr.wireguard-$IF.plist 2>/tmp/wg-bootstrap.err; then ok=1; break; fi
63+ grep -q -i "already" /tmp/wg-bootstrap.err && { ok=1; break; }
64+ sleep 3
65+done
66+[ $ok = 1 ] || { cat /tmp/wg-bootstrap.err; echo "bootstrap KO → démarrage direct"; sudo_ /opt/homebrew/bin/wg-quick up $IF; }
67+# NB : `wg show $IF` ne résout pas le nom sur macOS (wireguard-tools 1.0.2026) → passer par /var/run/wireguard/$IF.name.
68+for i in $(seq 1 30); do N=$(sudo_ cat /var/run/wireguard/$IF.name 2>/dev/null || true); [ -n "$N" ] && sudo_ /opt/homebrew/bin/wg show "$N" >/dev/null 2>&1 && break; sleep 1; done
69+sudo_ /opt/homebrew/bin/wg show "$N" | head -3
70+echo "NODE $(cat ~/.maclustr-node 2>/dev/null) IF $N PUBKEY $PUB"
modified mld/cli.py +57 −2
@@ -129,6 +129,58 @@ def cmd_retire(a):
129 deploy.retire(a.app, a.node or registry.node_of(a.app), remove_dir=not a.keep_dir) 129 deploy.retire(a.app, a.node or registry.node_of(a.app), remove_dir=not a.keep_dir)
130 if not a.keep_registry: 130 if not a.keep_registry:
131 registry.remove_app(a.app, note=a.note or "") 131 registry.remove_app(a.app, note=a.note or "")
132+ # app retirée du cluster : sa route publique tombe (page 404 MacLustr au lieu d'un 502)
133+ from . import tunnel
134+ try:
135+ tunnel.remove_route(manifest.load(a.app))
136+ except tunnel.TunnelError as e:
137+ print(" tunnel :", e)
138+
139+
140+def cmd_tunnel(a):
141+ from . import tunnel
142+ try:
143+ if a.sub == "status":
144+ tunnel.print_status(a.gateway)
145+ elif a.sub == "peer":
146+ for n in a.args:
147+ tunnel.ensure_peer(n, a.gateway, log=print) or tunnel.peer(n, a.gateway, log=print)
148+ elif a.sub == "route":
149+ for app in a.args:
150+ node = registry.node_of(app)
151+ if not node:
152+ print("%s : pas dans le registre" % app)
153+ continue
154+ m = manifest.load(app)
155+ s = tunnel.ensure_route(m, node, log=print)
156+ if s:
157+ registry.set_app(app, tunnel={"gateway": s["gateway"], "domain": s["domain"], "upstream": s["upstreams"][0], "url": "https://%s" % s["domain"]})
158+ else:
159+ print("%s : pas de domaine → rien à exposer" % app)
160+ elif a.sub == "rm":
161+ for app in a.args:
162+ tunnel.remove_route(manifest.load(app))
163+ except tunnel.TunnelError as e:
164+ raise SystemExit("ÉCHEC : %s" % e)
165+
166+
167+def cmd_heal(a):
168+ from . import heal
169+ if a.install:
170+ heal.install(interval=a.interval)
171+ return
172+ s = heal.heal(apps_filter=a.apps or None, nodes_filter=a.nodes or None, dry_run=a.dry_run, quiet=a.quiet, public=not a.no_public)
173+ sys.exit(0 if not s["ko"] else 1)
174+
175+
176+def cmd_harden(a):
177+ from . import harden
178+ autologin = True if a.autologin else (False if a.no_autologin else None)
179+ if a.all or not a.nodes:
180+ harden.harden_all(aliases=a.nodes or None, autologin=autologin)
181+ else:
182+ for n in a.nodes:
183+ harden.harden(n, autologin=autologin)
132 184
133 185
134 def cmd_status(a): 186 def cmd_status(a):
@@ -186,7 +238,7 @@ def cmd_import(a):
186 238
187 def cmd_prepare(a): 239 def cmd_prepare(a):
188 from . import prepare 240 from . import prepare
189− prepare.prepare(a.node, runtimes=a.runtime or None, ka_helpers=a.ka_helpers) 241+ prepare.prepare(a.node, runtimes=a.runtime or None, ka_helpers=a.ka_helpers, tunnel_peer=not a.no_tunnel, harden=not a.no_harden)
190 242
191 243
192 def cmd_bootstrap(a): 244 def cmd_bootstrap(a):
@@ -226,8 +278,11 @@ def main(argv=None):
226 s = sp.add_parser("registry", help="registre JSON complet (--push : rediffuse aux abonnés)"); s.add_argument("--push", action="store_true"); s.set_defaults(f=cmd_registry) 278 s = sp.add_parser("registry", help="registre JSON complet (--push : rediffuse aux abonnés)"); s.add_argument("--push", action="store_true"); s.set_defaults(f=cmd_registry)
227 s = sp.add_parser("subscribers", help="abonnés du registre (copie poussée à chaque sauvegarde)"); s.add_argument("--add", nargs=3, metavar=("app|node", "NOM", "CHEMIN")); s.add_argument("--remove", nargs=2, metavar=("app|node", "NOM")); s.set_defaults(f=cmd_subscribers) 279 s = sp.add_parser("subscribers", help="abonnés du registre (copie poussée à chaque sauvegarde)"); s.add_argument("--add", nargs=3, metavar=("app|node", "NOM", "CHEMIN")); s.add_argument("--remove", nargs=2, metavar=("app|node", "NOM")); s.set_defaults(f=cmd_subscribers)
228 s = sp.add_parser("import", help="brouillons de manifestes depuis les PM2 d'un nœud"); s.add_argument("node"); s.add_argument("apps", nargs="*"); s.set_defaults(f=cmd_import) 280 s = sp.add_parser("import", help="brouillons de manifestes depuis les PM2 d'un nœud"); s.add_argument("node"); s.add_argument("apps", nargs="*"); s.set_defaults(f=cmd_import)
229− s = sp.add_parser("prepare", help="installe runtimes/ngrok/pm2-startup sur un nœud"); s.add_argument("node"); s.add_argument("--runtime", action="append"); s.add_argument("--ka-helpers", action="store_true"); s.set_defaults(f=cmd_prepare) 281+ s = sp.add_parser("prepare", help="runtimes Homebrew + pm2 startup + raccordement tunnel + harden sur un nœud"); s.add_argument("node"); s.add_argument("--runtime", action="append"); s.add_argument("--ka-helpers", action="store_true"); s.add_argument("--no-tunnel", action="store_true"); s.add_argument("--no-harden", action="store_true"); s.set_defaults(f=cmd_prepare)
230 s = sp.add_parser("bootstrap", help="Xcode CLT + Homebrew sur un nœud vierge (long)"); s.add_argument("node"); s.set_defaults(f=cmd_bootstrap) 282 s = sp.add_parser("bootstrap", help="Xcode CLT + Homebrew sur un nœud vierge (long)"); s.add_argument("node"); s.set_defaults(f=cmd_bootstrap)
283+ s = sp.add_parser("tunnel", help="MacLustr Tunnel : status | peer <nœud>… | route <app>… | rm <app>…"); s.add_argument("sub", choices=["status", "peer", "route", "rm"]); s.add_argument("args", nargs="*"); s.add_argument("-g", "--gateway", help="BHS64 (défaut) ou R9128"); s.set_defaults(f=cmd_tunnel)
284+ s = sp.add_parser("heal", help="auto-réparation : PM2/launchd/wg1/routes de chaque app du registre (--install = LaunchDaemon 5 min)"); s.add_argument("--apps", nargs="*"); s.add_argument("--nodes", nargs="*"); s.add_argument("--dry-run", action="store_true"); s.add_argument("--quiet", action="store_true"); s.add_argument("--no-public", action="store_true"); s.add_argument("--install", action="store_true"); s.add_argument("--interval", type=int, help="secondes entre deux passages (avec --install)"); s.set_defaults(f=cmd_heal)
285+ s = sp.add_parser("harden", help="résilience d'un nœud : auto-login, pas de veille, redémarrage après coupure/gel, MAJ macOS + Tailscale off"); s.add_argument("nodes", nargs="*"); s.add_argument("--all", action="store_true"); s.add_argument("--autologin", action="store_true", help="force l'auto-login même sur un nœud loué"); s.add_argument("--no-autologin", action="store_true"); s.set_defaults(f=cmd_harden)
231 a = p.parse_args(argv) 286 a = p.parse_args(argv)
232 a.f(a) 287 a.f(a)
233 288
modified mld/config.py +13 −3
@@ -17,9 +17,18 @@ LOG_DIR = os.path.join(STATE, "logs")
17 REGISTRY = os.path.join(STATE, "registry.json") 17 REGISTRY = os.path.join(STATE, "registry.json")
18 LAN_CACHE = os.path.join(STATE, "lan.json") 18 LAN_CACHE = os.path.join(STATE, "lan.json")
19 SCAN_CACHE = os.path.join(STATE, "scan.json") 19 SCAN_CACHE = os.path.join(STATE, "scan.json")
20−NGROK_CFG = os.path.join(HOME, "Library", "Application Support", "ngrok", "ngrok.yml")
21 RUN_DIR = "~/dispatch-run" # sur les nœuds : ecosystem PM2 générés 20 RUN_DIR = "~/dispatch-run" # sur les nœuds : ecosystem PM2 générés
22 SUDO_PW_FILE = os.path.join(STATE, ".sudo") # mot de passe sudo des nœuds (0600), pour `prepare` 21 SUDO_PW_FILE = os.path.join(STATE, ".sudo") # mot de passe sudo des nœuds (0600), pour `prepare`
22+HEAL_STATE = os.path.join(STATE, "heal-state.json") # mémoire de `mld heal` (actions récentes, nœuds en panne)
23+
24+# MacLustr Tunnel (remplace ngrok depuis le 2026-09-10) : passerelles OVH = hub WireGuard + Caddy (TLS auto), pilotées
25+# par `tunnelctl` (sudo sans mot de passe pour ubuntu). La clé de la passerelle M1M32 est autorisée pour `ubuntu`.
26+# Chaque Mac porte une interface par passerelle (wg1 → BHS64, wg0 → R9128), LaunchDaemon io.maclustr.wireguard-<iface>.
27+TUNNEL_GATEWAYS = {
28+ "BHS64": {"host": "51.161.112.61", "user": "ubuntu", "subnet": "10.67.0", "iface": "wg1", "endpoint": "51.161.112.61:51820", "place": "OVH Beauharnois (Québec)"},
29+ "R9128": {"host": "51.255.75.61", "user": "ubuntu", "subnet": "10.66.0", "iface": "wg0", "endpoint": "51.255.75.61:51820", "place": "OVH Gravelines (France)"},
30+}
31+TUNNEL_DEFAULT = "BHS64"
23 32
24 # Inventaire statique (alias -> cœurs, RAM Go, modèle). Les IP LAN sont découvertes en live. 33 # Inventaire statique (alias -> cœurs, RAM Go, modèle). Les IP LAN sont découvertes en live.
25 NODES = { 34 NODES = {
@@ -125,7 +134,8 @@ RUNTIME_CHECKS = {
125 "node": "test -x /opt/homebrew/bin/node", 134 "node": "test -x /opt/homebrew/bin/node",
126 "pnpm": "test -x /opt/homebrew/bin/pnpm", 135 "pnpm": "test -x /opt/homebrew/bin/pnpm",
127 "pm2": "test -x /opt/homebrew/bin/pm2", 136 "pm2": "test -x /opt/homebrew/bin/pm2",
128− "ngrok": "(test -x /opt/homebrew/bin/ngrok || test -x $HOME/bin/ngrok) && test -f \"$HOME/Library/Application Support/ngrok/ngrok.yml\"", 137+ # raccordé au MacLustr Tunnel (wg1 → BHS64 monté) : informatif, `mld deploy` raccorde lui-même un nœud qui ne l'est pas
138+ "tunnel": "test -f /Library/LaunchDaemons/io.maclustr.wireguard-wg1.plist && ifconfig 2>/dev/null | grep -q 'inet 10\\.67\\.0\\.'",
129 "python@3.14": "test -x /opt/homebrew/opt/python@3.14/bin/python3.14", 139 "python@3.14": "test -x /opt/homebrew/opt/python@3.14/bin/python3.14",
130 "python@3.13": "test -x /opt/homebrew/opt/python@3.13/bin/python3.13", 140 "python@3.13": "test -x /opt/homebrew/opt/python@3.13/bin/python3.13",
131 "python@3.12": "test -x /opt/homebrew/opt/python@3.12/bin/python3.12", 141 "python@3.12": "test -x /opt/homebrew/opt/python@3.12/bin/python3.12",
@@ -141,7 +151,7 @@ BREW_FORMULAE = {
141 "node": "node", "pnpm": "pnpm", "pm2": "pm2", "python@3.14": "python@3.14", "python@3.13": "python@3.13", 151 "node": "node", "pnpm": "pnpm", "pm2": "pm2", "python@3.14": "python@3.14", "python@3.13": "python@3.13",
142 "python@3.12": "python@3.12", "uv": "uv", "redis": "redis", "docker": "docker", "postgresql@17": "postgresql@17", "git": "git", 152 "python@3.12": "python@3.12", "uv": "uv", "redis": "redis", "docker": "docker", "postgresql@17": "postgresql@17", "git": "git",
143 } 153 }
144−BREW_CASKS = {"ngrok": "ngrok"} 154+BREW_CASKS = {} # ngrok (cask) retiré le 2026-09-10 : l'exposition publique passe par TUNNEL_GATEWAYS
145 155
146 os.makedirs(APPS_DIR, exist_ok=True) 156 os.makedirs(APPS_DIR, exist_ok=True)
147 os.makedirs(STAGE_DIR, exist_ok=True) 157 os.makedirs(STAGE_DIR, exist_ok=True)
modified mld/deploy.py +75 −30
@@ -1,9 +1,12 @@
1−"""Cycle de vie d'une app : fetch (nœud -> staging), deploy (staging -> nœud), stop, start, retire, move, health.""" 1+"""Cycle de vie d'une app : fetch (nœud -> staging), deploy (staging -> nœud), stop, start, retire, move, health.
2+
3+Exposition publique = MacLustr Tunnel (module tunnel) : au deploy, la route Caddy https://<domaine> de la passerelle est
4+pointée vers le nœud choisi (après le healthcheck local), donc elle suit automatiquement les `mld move`."""
2 import json 5 import json
3 import os 6 import os
4 import shlex 7 import shlex
5 import time 8 import time
6−from . import config, manifest, nodes, registry, render, ssh 9+from . import config, manifest, nodes, registry, render, ssh, tunnel
7 10
8 SYNC_BASE_EXCLUDES = [".DS_Store", "*.log", "__pycache__/", ".pytest_cache/", ".mypy_cache/"] 11 SYNC_BASE_EXCLUDES = [".DS_Store", "*.log", "__pycache__/", ".pytest_cache/", ".mypy_cache/"]
9 12
@@ -89,21 +92,17 @@ def stop(app, alias, keep_dir=True):
89 m = manifest.load(app) 92 m = manifest.load(app)
90 ip = _ip(alias) 93 ip = _ip(alias)
91 _log(app, "stop sur %s" % alias) 94 _log(app, "stop sur %s" % alias)
92− names = manifest.pm2_names(m) 95+ names = manifest.pm2_names(m) + manifest.legacy_pm2_names(m)
93 script = "" 96 script = ""
94 if names: 97 if names:
95 script += "for n in %s; do pm2 delete $n >/dev/null 2>&1 && echo \" pm2 delete $n\"; done; pm2 save --force >/dev/null 2>&1\n" % " ".join(names) 98 script += "for n in %s; do pm2 delete $n >/dev/null 2>&1 && echo \" pm2 delete $n\"; done; pm2 save --force >/dev/null 2>&1\n" % " ".join(names)
96 for lbl in manifest.launchd_labels(m): 99 for lbl in manifest.launchd_labels(m):
97 script += "launchctl bootout gui/$(id -u)/%s >/dev/null 2>&1 && echo ' bootout %s'; rm -f ~/Library/LaunchAgents/%s.plist\n" % (lbl, lbl, lbl) 100 script += "launchctl bootout gui/$(id -u)/%s >/dev/null 2>&1 && echo ' bootout %s'; rm -f ~/Library/LaunchAgents/%s.plist\n" % (lbl, lbl, lbl)
98− if m.get("ngrok"):
99− script += "pkill -f -- 'url=%s' 2>/dev/null; pkill -f -- 'url=https://%s' 2>/dev/null; pkill -f -- 'domain=%s' 2>/dev/null; true\n" % (m["ngrok"]["url"], m["ngrok"]["url"], m["ngrok"]["url"])
100 for hook in (m.get("hooks") or {}).get("pre_stop") or []: 101 for hook in (m.get("hooks") or {}).get("pre_stop") or []:
101 script += hook + "\n" 102 script += hook + "\n"
102 # tue ce qui écoute encore sur les ports de l'app (orphelins) 103 # tue ce qui écoute encore sur les ports de l'app (orphelins)
103 for p in m["requires"].get("ports") or []: 104 for p in m["requires"].get("ports") or []:
104 script += "sleep 1; for pid in $(lsof -nP -iTCP:%s -sTCP:LISTEN -t 2>/dev/null); do kill $pid 2>/dev/null && echo ' kill port %s pid '$pid; done\n" % (p, p) 105 script += "sleep 1; for pid in $(lsof -nP -iTCP:%s -sTCP:LISTEN -t 2>/dev/null); do kill $pid 2>/dev/null && echo ' kill port %s pid '$pid; done\n" % (p, p)
105− if m.get("ngrok"):
106− script += "sleep 4\n" # laisse ngrok libérer le domaine réservé côté service
107 rc, out, err = ssh.run(ip, script, timeout=180) 106 rc, out, err = ssh.run(ip, script, timeout=180)
108 for l in out.splitlines(): 107 for l in out.splitlines():
109 _log(app, l) 108 _log(app, l)
@@ -177,12 +176,16 @@ def deploy(app, alias, reinstall=False, skip_sync=False):
177 raise DeployError("post-sync a échoué (%s): %s" % (rc, (err or out).strip()[-400:])) 176 raise DeployError("post-sync a échoué (%s): %s" % (rc, (err or out).strip()[-400:]))
178 # 4. lancement 177 # 4. lancement
179 start(app, alias, m=m, ctx=ctx, ip=ip) 178 start(app, alias, m=m, ctx=ctx, ip=ip)
180− # 5. santé 179+ # 5. santé locale, puis route publique (MacLustr Tunnel) pointée vers CE nœud, puis santé publique
181− healthy, detail = health(app, alias, ip=ip, m=m) 180+ local_ok, code = _health_local(ip, m)
181+ tun = None
182+ if local_ok and tunnel.spec(m, alias):
183+ tun = _publish(app, alias, m, ip)
184+ healthy, detail = health(app, alias, ip=ip, m=m, timeout=30, public=bool(tun))
182 registry.set_app(app, node=alias, ip=ip, port=m.get("port"), domain=m.get("domain"), dir=m["dir"], label=m.get("label") or app, 185 registry.set_app(app, node=alias, ip=ip, port=m.get("port"), domain=m.get("domain"), dir=m["dir"], label=m.get("label") or app,
183 health_path=m.get("health_path") or "/", 186 health_path=m.get("health_path") or "/",
184 processes=manifest.pm2_names(m), launchd=manifest.launchd_labels(m), deployed=time.strftime("%Y-%m-%d %H:%M"), 187 processes=manifest.pm2_names(m), launchd=manifest.launchd_labels(m), deployed=time.strftime("%Y-%m-%d %H:%M"),
185− status="online" if healthy else "unhealthy", health=detail) 188+ tunnel=tun, status="online" if healthy else "unhealthy", health=detail)
186 registry.log(app, "deployed", node=alias, healthy=healthy) 189 registry.log(app, "deployed", node=alias, healthy=healthy)
187 _log(app, " santé : %s" % detail) 190 _log(app, " santé : %s" % detail)
188 if not healthy: 191 if not healthy:
@@ -205,8 +208,9 @@ def start(app, alias, m=None, ctx=None, ip=None):
205 ssh.write_remote_file(ip, eco_path, eco, mode="600") 208 ssh.write_remote_file(ip, eco_path, eco, mode="600")
206 script = "mkdir -p %s\n" % run_dir 209 script = "mkdir -p %s\n" % run_dir
207 names = manifest.pm2_names(m) 210 names = manifest.pm2_names(m)
211+ if names or manifest.legacy_pm2_names(m):
212+ script += "for n in %s; do pm2 delete $n >/dev/null 2>&1; done\n" % " ".join(names + manifest.legacy_pm2_names(m))
208 if names: 213 if names:
209− script += "for n in %s; do pm2 delete $n >/dev/null 2>&1; done\n" % " ".join(names)
210 script += "pm2 start %s --update-env 2>&1 | grep -E 'error|Error|\\[PM2\\]' | head -5; pm2 save --force >/dev/null 2>&1; pm2 jlist | python3 -c \"import json,sys; L=json.load(sys.stdin); print(' pm2: '+', '.join(p['name']+'='+p['pm2_env']['status'] for p in L if p['name'] in %s))\" 2>/dev/null || pm2 ls | grep -E '%s'\n" % (shlex.quote(eco_path), json.dumps(names), "|".join(names)) 214 script += "pm2 start %s --update-env 2>&1 | grep -E 'error|Error|\\[PM2\\]' | head -5; pm2 save --force >/dev/null 2>&1; pm2 jlist | python3 -c \"import json,sys; L=json.load(sys.stdin); print(' pm2: '+', '.join(p['name']+'='+p['pm2_env']['status'] for p in L if p['name'] in %s))\" 2>/dev/null || pm2 ls | grep -E '%s'\n" % (shlex.quote(eco_path), json.dumps(names), "|".join(names))
211 for item in m.get("launchd") or []: 215 for item in m.get("launchd") or []:
212 pl = render.plist(item, ctx) 216 pl = render.plist(item, ctx)
@@ -223,11 +227,29 @@ def start(app, alias, m=None, ctx=None, ip=None):
223 return rc == 0 227 return rc == 0
224 228
225 229
226−def health(app, alias, ip=None, m=None, timeout=90): 230+def _publish(app, alias, m, ip):
227− m = m or manifest.load(app) 231+ """Raccorde le nœud au tunnel si besoin et pointe la route publique de l'app dessus. Retourne le spec de route ou None."""
228− ip = ip or _ip(alias) 232+ log = lambda s: _log(app, s)
233+ try:
234+ st = None
235+ try:
236+ st = tunnel.state((m.get("tunnel") or {}).get("gateway"))
237+ except tunnel.TunnelError:
238+ pass
239+ if not tunnel.ensure_peer(alias, (m.get("tunnel") or {}).get("gateway"), log=log, st=st):
240+ _log(app, " tunnel : %s n'a pas de handshake avec la passerelle — route pointée quand même" % alias)
241+ s = tunnel.ensure_route(m, alias, log=log)
242+ if not s:
243+ return None
244+ return {"gateway": s["gateway"], "domain": s["domain"], "upstream": s["upstreams"][0], "url": "https://%s" % s["domain"]}
245+ except tunnel.TunnelError as e:
246+ _log(app, " tunnel : %s" % e)
247+ return None
248+
249+
250+def _health_local(ip, m, timeout=90):
251+ """(ok, code|texte) : HTTP local sur le port de l'app, ou processus PM2 online si l'app n'a pas de port."""
229 if not m.get("port"): 252 if not m.get("port"):
230− # pas de port HTTP : vérifie que les processus PM2 sont online
231 names = manifest.pm2_names(m) 253 names = manifest.pm2_names(m)
232 if not names: 254 if not names:
233 return True, "aucun port/processus à vérifier" 255 return True, "aucun port/processus à vérifier"
@@ -237,22 +259,40 @@ def health(app, alias, ip=None, m=None, timeout=90):
237 url = "http://127.0.0.1:%s%s" % (m["port"], m.get("health_path") or "/") 259 url = "http://127.0.0.1:%s%s" % (m["port"], m.get("health_path") or "/")
238 deadline = time.time() + timeout 260 deadline = time.time() + timeout
239 code = "000" 261 code = "000"
240− while time.time() < deadline: 262+ while True:
241 rc, out, _ = ssh.run(ip, "curl -s -o /dev/null -m 8 -w '%%{http_code}' %s" % shlex.quote(url), timeout=20) 263 rc, out, _ = ssh.run(ip, "curl -s -o /dev/null -m 8 -w '%%{http_code}' %s" % shlex.quote(url), timeout=20)
242 code = out.strip()[-3:] if out.strip() else "000" 264 code = out.strip()[-3:] if out.strip() else "000"
243 if code.isdigit() and code not in ("000",) and int(code) < 500: 265 if code.isdigit() and code not in ("000",) and int(code) < 500:
244 break 266 break
267+ if time.time() >= deadline:
268+ break
245 time.sleep(4) 269 time.sleep(4)
246− local_ok = code.isdigit() and code != "000" and int(code) < 500 270+ return code.isdigit() and code != "000" and int(code) < 500, code
271+
272+
273+def _health_public(m, tries=20):
274+ """Code HTTP de https://<domaine><health_path> à travers la passerelle (tries × 5 s max)."""
275+ import subprocess
276+ pub = ""
277+ for _ in range(tries):
278+ p = subprocess.run(["curl", "-s", "-o", "/dev/null", "-m", "12", "-w", "%{http_code}", "https://%s%s" % (m["domain"], m.get("health_path") or "/")], capture_output=True, text=True)
279+ pub = p.stdout.strip()
280+ if pub.isdigit() and int(pub) < 500 and pub != "404":
281+ break
282+ time.sleep(5)
283+ return pub
284+
285+
286+def health(app, alias, ip=None, m=None, timeout=90, public=True):
287+ """Santé locale (port/PM2) puis publique (via MacLustr Tunnel) si l'app a un domaine. Retourne (ok, détail)."""
288+ m = m or manifest.load(app)
289+ ip = ip or _ip(alias)
290+ local_ok, code = _health_local(ip, m, timeout=timeout)
291+ if not m.get("port"):
292+ return local_ok, code
247 pub = "" 293 pub = ""
248− if m.get("domain") and local_ok and m.get("ngrok"): 294+ if public and m.get("domain") and local_ok and tunnel.spec(m, alias):
249− import subprocess 295+ pub = _health_public(m, tries=max(2, min(20, timeout // 5)))
250− for _ in range(20):
251− p = subprocess.run(["curl", "-s", "-o", "/dev/null", "-m", "12", "-w", "%{http_code}", "https://%s%s" % (m["domain"], m.get("health_path") or "/")], capture_output=True, text=True)
252− pub = p.stdout.strip()
253− if pub.isdigit() and int(pub) < 500 and pub != "404":
254− break
255− time.sleep(5)
256 detail = "local %s" % code + (" / public https://%s %s" % (m["domain"], pub) if pub else "") 296 detail = "local %s" % code + (" / public https://%s %s" % (m["domain"], pub) if pub else "")
257 ok = local_ok and (not pub or (pub.isdigit() and int(pub) < 500 and pub != "404")) 297 ok = local_ok and (not pub or (pub.isdigit() and int(pub) < 500 and pub != "404"))
258 return ok, detail 298 return ok, detail
@@ -296,8 +336,11 @@ def move(app, to_alias, from_alias=None, keep_source=False, reinstall=False):
296 except Exception: 336 except Exception:
297 pass 337 pass
298 start(app, from_alias) 338 start(app, from_alias)
299− ok, det = health(app, from_alias) 339+ m = manifest.load(app)
300− registry.set_app(app, node=from_alias, status="online" if ok else "unhealthy", health=det) 340+ ip0 = _ip(from_alias)
341+ tun = _publish(app, from_alias, m, ip0) if tunnel.spec(m, from_alias) else None # la route publique revient sur la source
342+ ok, det = health(app, from_alias, ip=ip0, m=m)
343+ registry.set_app(app, node=from_alias, ip=ip0, tunnel=tun, status="online" if ok else "unhealthy", health=det)
301 raise 344 raise
302 if not keep_source: 345 if not keep_source:
303 retire(app, from_alias) # 5. nettoyage source 346 retire(app, from_alias) # 5. nettoyage source
@@ -320,10 +363,11 @@ def status(app=None):
320 363
321 364
322 def live_status(app=None, public=True): 365 def live_status(app=None, public=True):
323− """Vérifie en live (PM2 + port + domaine) chaque app du registre.""" 366+ """Vérifie en live (PM2 + port + domaine via le tunnel) chaque app du registre. Une seule sauvegarde du registre à la fin."""
324 r = registry.load() 367 r = registry.load()
325 apps = [app] if app else sorted(r["apps"]) 368 apps = [app] if app else sorted(r["apps"])
326 out = [] 369 out = []
370+ changes = {}
327 for a in apps: 371 for a in apps:
328 v = r["apps"].get(a) 372 v = r["apps"].get(a)
329 if not v: 373 if not v:
@@ -331,9 +375,10 @@ def live_status(app=None, public=True):
331 m = manifest.load(a) 375 m = manifest.load(a)
332 try: 376 try:
333 ip = _ip(v["node"]) 377 ip = _ip(v["node"])
334− ok, det = health(a, v["node"], ip=ip, m=m, timeout=10) if public else (None, "") 378+ ok, det = health(a, v["node"], ip=ip, m=m, timeout=10, public=public)
335 except DeployError as e: 379 except DeployError as e:
336 ok, det = False, str(e) 380 ok, det = False, str(e)
337− registry.set_app(a, status="online" if ok else "unhealthy", health=det) 381+ changes[a] = {"status": "online" if ok else "unhealthy", "health": det}
338 out.append((a, v["node"], v.get("domain") or "", "OK" if ok else "KO", det)) 382 out.append((a, v["node"], v.get("domain") or "", "OK" if ok else "KO", det))
383+ registry.update_many(changes)
339 return out 384 return out
added mld/harden.py +124 −0
@@ -0,0 +1,124 @@
1+"""`mld harden <nœud>|--all` : un Mac du cluster doit revenir TOUT SEUL après une coupure de courant ou une mise à jour.
2+
3+Ce que ça règle (idempotent, sudo via ~/dispatch/.sudo*) :
4+ 1. alimentation : redémarrage automatique après coupure (autorestart) et après gel (restartfreeze), jamais de veille
5+ (sleep/disksleep 0), réveil réseau (womp) ;
6+ 2. mises à jour macOS + App Store : aucune installation automatique (un redémarrage surprise couperait les apps) ;
7+ 3. Tailscale : `tailscale set --auto-update=false` + Sparkle désactivé — une MAJ relance tailscaled et le nœud disparaît
8+ de *.maclustr.io (les noms résolvent vers l'IP Tailscale) même si le LAN et le tunnel vont bien ;
9+ 4. ouverture de session automatique (kcpassword + autoLoginUser) : sans session graphique, les LaunchAgents (pm2 startup,
10+ gardiens Ka, pousseur, admin-ka) ne démarrent jamais — c'est ce qui a laissé les apps mortes le 2026-09-10.
11+ Impossible si FileVault est actif (le disque reste chiffré jusqu'à la saisie du mot de passe) : signalé ;
12+ 5. vérifie que pm2 startup (LaunchAgent ou LaunchDaemon) et le LaunchDaemon WireGuard wg1 sont en place.
13+Les nœuds loués (REMOTE_NODES) reçoivent 1, 2, 5 mais pas l'auto-login (session du fournisseur) sauf --autologin.
14+"""
15+import os
16+import shlex
17+from concurrent.futures import ThreadPoolExecutor
18+from . import config, nodes, ssh
19+
20+KC_KEY = [0x7D, 0x89, 0x52, 0x23, 0xD2, 0xBC, 0xDD, 0xEA, 0xA3, 0xB9, 0x1F]
21+
22+
23+def kcpassword(pw):
24+ """Encodage /etc/kcpassword (XOR avec la clé Apple, complété à un multiple de 12 octets)."""
25+ b = list(pw.encode("utf-8")) + [0]
26+ while len(b) % 12:
27+ b.append(0)
28+ return bytes(x ^ KC_KEY[i % len(KC_KEY)] for i, x in enumerate(b))
29+
30+
31+def _sudo_pw(alias):
32+ pwf = config.sudo_pw_file(alias)
33+ return open(pwf).read().strip() if os.path.exists(pwf) else None
34+
35+
36+def script_for(alias, autologin):
37+ pw = _sudo_pw(alias)
38+ if not pw:
39+ return None
40+ user = config.user_of(alias)
41+ kc = "".join("\\x%02x" % x for x in kcpassword(pw)) # même mot de passe pour sudo et l'ouverture de session
42+ s = r'''
43+S(){ echo %(pw)s | sudo -S -p '' "$@"; }
44+echo "== alimentation"
45+S pmset -a sleep 0 disksleep 0 womp 1 >/dev/null 2>&1
46+S pmset -a autorestart 1 >/dev/null 2>&1 || echo " autorestart non supporté (portable)"
47+S systemsetup -setrestartfreeze on >/dev/null 2>&1
48+S systemsetup -setrestartpowerfailure on >/dev/null 2>&1
49+echo " $(pmset -g | grep -E '^ *(autorestart|sleep|disksleep|womp) ' | awk '{print $1"="$2}' | tr '\n' ' ')"
50+echo "== mises à jour macOS / App Store : désactivées"
51+for k in AutomaticCheckEnabled AutomaticDownload AutomaticallyInstallMacOSUpdates ConfigDataInstall CriticalUpdateInstall; do S defaults write /Library/Preferences/com.apple.SoftwareUpdate $k -bool false; done
52+S defaults write /Library/Preferences/com.apple.commerce AutoUpdate -bool false
53+S defaults write /Library/Preferences/com.apple.commerce AutoUpdateRestartRequired -bool false
54+echo " SoftwareUpdate: $(defaults read /Library/Preferences/com.apple.SoftwareUpdate 2>/dev/null | grep -E 'Automatic|Install' | tr -d ' ;' | tr '\n' ' ')"
55+T=/Applications/Tailscale.app/Contents/MacOS/Tailscale
56+if [ -x "$T" ]; then
57+ echo "== Tailscale $($T version 2>/dev/null | head -1) : pas de mise à jour automatique"
58+ "$T" set --auto-update=false >/dev/null 2>&1 && echo " auto-update off" || echo " ! tailscale set a échoué (session ?)"
59+ defaults write io.tailscale.ipn.macsys SUEnableAutomaticChecks -bool false 2>/dev/null
60+ defaults write io.tailscale.ipn.macsys SUAutomaticallyUpdate -bool false 2>/dev/null
61+ defaults write io.tailscale.ipn.macos SUEnableAutomaticChecks -bool false 2>/dev/null
62+else
63+ echo "== Tailscale : absent"
64+fi
65+echo "== ouverture de session automatique (%(user)s)"
66+if fdesetup status 2>/dev/null | grep -q 'On'; then
67+ echo " ! FILEVAULT ACTIF : auto-login impossible, le Mac attend un mot de passe au démarrage (fdesetup disable pour lever ça)"
68+elif [ "%(autologin)s" = "1" ]; then
69+ printf '%(kc)s' > /tmp/.kc.$$ && S install -m 600 -o root -g wheel /tmp/.kc.$$ /etc/kcpassword; rm -f /tmp/.kc.$$
70+ S defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser -string %(user)s
71+ echo " autoLoginUser=$(defaults read /Library/Preferences/com.apple.loginwindow autoLoginUser 2>/dev/null) kcpassword=$(S stat -f %%z /etc/kcpassword 2>/dev/null) octets"
72+else
73+ echo " non touché (nœud loué : session du fournisseur) — autoLoginUser=$(defaults read /Library/Preferences/com.apple.loginwindow autoLoginUser 2>/dev/null || echo aucun)"
74+fi
75+echo "== démarrage des services"
76+P=$(ls ~/Library/LaunchAgents/pm2.*.plist /Library/LaunchDaemons/pm2.*.plist 2>/dev/null | head -1)
77+[ -n "$P" ] && echo " pm2 startup : $P" || echo " ! pm2 startup ABSENT → mld prepare %(alias)s"
78+if [ -f /Library/LaunchDaemons/io.maclustr.wireguard-wg1.plist ]; then
79+ ifconfig 2>/dev/null | grep -q 'inet 10\.67\.0\.' && echo " wg1 (tunnel BHS64) : monté" || { echo " wg1 : plist présent mais interface absente → relance"; S launchctl kickstart -k system/io.maclustr.wireguard-wg1; }
80+else
81+ echo " wg1 : non raccordé au MacLustr Tunnel (mld tunnel peer %(alias)s si le nœud doit exposer un site)"
82+fi
83+echo " session console : $(who | awk '/console/{print $1" depuis "$3" "$4}' | head -1)"
84+''' % {"pw": shlex.quote(pw), "user": user, "kc": kc, "autologin": "1" if autologin else "0", "alias": alias}
85+ return s
86+
87+
88+def harden(alias, autologin=None, verbose=True):
89+ ip = nodes.ip_of(alias, rediscover=False) or nodes.ip_of(alias)
90+ if not ip:
91+ return alias, False, "injoignable"
92+ if autologin is None:
93+ autologin = alias not in config.REMOTE_NODES
94+ s = script_for(alias, autologin)
95+ if not s:
96+ return alias, False, "mot de passe sudo absent (%s)" % config.sudo_pw_file(alias)
97+ rc, out, err = ssh.run(ip, s, timeout=240)
98+ if verbose:
99+ print("### %s (%s)" % (alias, ip))
100+ print(out.rstrip())
101+ if err.strip():
102+ print(" stderr: " + err.strip()[-200:])
103+ flags = []
104+ if "FILEVAULT" in out:
105+ flags.append("FileVault")
106+ if "pm2 startup ABSENT" in out:
107+ flags.append("pm2-startup")
108+ if "tailscale set a échoué" in out:
109+ flags.append("tailscale")
110+ return alias, rc == 0, ", ".join(flags) if flags else "ok"
111+
112+
113+def harden_all(aliases=None, autologin=None):
114+ lan = nodes.load_lan()["ips"] or nodes.discover(verbose=False)
115+ targets = [a for a in config.NODES if a in lan and (not aliases or a in aliases)]
116+ with ThreadPoolExecutor(max_workers=12) as ex:
117+ results = list(ex.map(lambda a: harden(a, autologin=autologin, verbose=False), targets))
118+ print("%-8s %-6s %s" % ("nœud", "état", "à surveiller"))
119+ for a, ok, note in results:
120+ print("%-8s %-6s %s" % (a, "ok" if ok else "ÉCHEC", note))
121+ missing = [a for a in config.NODES if a not in lan]
122+ if missing:
123+ print("non joignables (éteints ?) : %s" % ", ".join(missing))
124+ return results
added mld/heal.py +295 −0
@@ -0,0 +1,295 @@
1+"""`mld heal` : auto-réparation du cluster depuis la passerelle (toutes les 5 min via le LaunchDaemon io.maclustr.mld-heal).
2+
3+Pour chaque nœud qui héberge des apps du registre, UN seul SSH ramène : uptime, démon PM2, statut de chaque processus,
4+LaunchAgents chargés, interface WireGuard wg1, code HTTP local de chaque app. Puis, seulement si quelque chose manque :
5+ - démon PM2 mort ou processus absents → `pm2 resurrect` puis `deploy.start(app)` (régénère l'ecosystem, idempotent)
6+ - processus errored/stopped (hors cron) → `pm2 restart`
7+ - LaunchAgent absent → re-bootstrap du plist (ou start complet s'il n'existe plus)
8+ - wg1 tombé → kickstart du LaunchDaemon (sinon raccordement complet)
9+ - route Caddy absente / mauvais nœud → `tunnelctl add` (via tunnel.ensure_route)
10+Garde-fous : nœud redémarré depuis < 3 min ignoré (pm2 resurrect fait son travail), au plus 6 actions par app et par heure,
11+registre sauvegardé UNE fois (diffusion aux abonnés) et seulement si un état change. Journal : ~/dispatch/logs/heal.log.
12+"""
13+import json
14+import os
15+import plistlib
16+import shlex
17+import time
18+from . import config, deploy, manifest, nodes, registry, ssh, tunnel
19+
20+GRACE_S = 180
21+MAX_ACTIONS_PER_HOUR = 6
22+HEAL_LABEL = "io.maclustr.mld-heal"
23+HEAL_INTERVAL_S = 300
24+
25+
26+def _now():
27+ return time.strftime("%Y-%m-%d %H:%M:%S")
28+
29+
30+def _log(msg, quiet=False):
31+ line = "%s %s" % (_now(), msg)
32+ if not quiet:
33+ print(msg, flush=True)
34+ with open(os.path.join(config.LOG_DIR, "heal.log"), "a") as f:
35+ f.write(line + "\n")
36+
37+
38+def _state():
39+ if os.path.exists(config.HEAL_STATE):
40+ try:
41+ return json.load(open(config.HEAL_STATE))
42+ except Exception:
43+ pass
44+ return {"actions": {}, "down": {}}
45+
46+
47+def _save_state(st):
48+ tmp = config.HEAL_STATE + ".tmp"
49+ json.dump(st, open(tmp, "w"), indent=1)
50+ os.replace(tmp, config.HEAL_STATE)
51+
52+
53+def _allowed(st, app):
54+ cut = time.time() - 3600
55+ acts = [t for t in st["actions"].get(app, []) if t > cut]
56+ st["actions"][app] = acts
57+ return len(acts) < MAX_ACTIONS_PER_HOUR
58+
59+
60+def _record(st, app):
61+ st["actions"].setdefault(app, []).append(time.time())
62+
63+
64+def _node_report(ip, apps):
65+ """Un SSH : BOOT/NOW, PM2 up|down, PROC nom statut restarts, LA label, WG1 up|down, HUB ok|ko, HTTP app code."""
66+ script = r'''
67+echo "BOOT $(sysctl -n kern.boottime | sed -E 's/.*sec = ([0-9]+).*/\1/')"
68+echo "NOW $(date +%s)"
69+if pm2 ping >/dev/null 2>&1; then echo "PM2 up"; else echo "PM2 down"; fi
70+pm2 jlist 2>/dev/null | python3 -c "
71+import json,sys
72+try: L=json.load(sys.stdin)
73+except Exception: L=[]
74+for p in L: print('PROC', p['name'], p['pm2_env'].get('status'), p['pm2_env'].get('restart_time',0))" 2>/dev/null
75+launchctl list 2>/dev/null | awk 'NR>1{print "LA", $3}'
76+ifconfig 2>/dev/null | grep -q 'inet 10\.67\.0\.' && echo "WG1 up" || echo "WG1 down"
77+ping -c 1 -W 2000 10.67.0.1 >/dev/null 2>&1 && echo "HUB ok" || echo "HUB ko"
78+'''
79+ for app, m in apps.items():
80+ if m.get("port"):
81+ script += "echo \"HTTP %s $(curl -s -o /dev/null -m 6 -w '%%{http_code}' http://127.0.0.1:%s%s)\"\n" % (app, m["port"], m.get("health_path") or "/")
82+ rc, out, err = ssh.run(ip, script, timeout=90)
83+ if rc != 0 and "BOOT" not in out:
84+ return None
85+ rep = {"pm2": None, "procs": {}, "la": set(), "wg1": None, "hub": None, "http": {}, "boot": 0, "now": 0}
86+ for line in out.splitlines():
87+ p = line.split()
88+ if not p:
89+ continue
90+ if p[0] == "BOOT" and len(p) > 1 and p[1].isdigit():
91+ rep["boot"] = int(p[1])
92+ elif p[0] == "NOW" and len(p) > 1:
93+ rep["now"] = int(p[1])
94+ elif p[0] == "PM2":
95+ rep["pm2"] = p[1] == "up"
96+ elif p[0] == "PROC" and len(p) >= 3:
97+ rep["procs"][p[1]] = (p[2], int(p[3]) if len(p) > 3 and p[3].isdigit() else 0)
98+ elif p[0] == "LA" and len(p) > 1:
99+ rep["la"].add(p[1])
100+ elif p[0] == "WG1":
101+ rep["wg1"] = p[1] == "up"
102+ elif p[0] == "HUB":
103+ rep["hub"] = p[1] == "ok"
104+ elif p[0] == "HTTP" and len(p) >= 2:
105+ rep["http"][p[1]] = p[2] if len(p) > 2 else "000"
106+ rep["uptime"] = (rep["now"] - rep["boot"]) if rep["boot"] and rep["now"] else 10 ** 6
107+ return rep
108+
109+
110+def _http_ok(code):
111+ return code.isdigit() and code != "000" and int(code) < 500
112+
113+
114+def heal(apps_filter=None, nodes_filter=None, dry_run=False, quiet=False, public=True):
115+ t0 = time.time()
116+ st = _state()
117+ reg = registry.load()
118+ by_node = {}
119+ for app, v in reg["apps"].items():
120+ if apps_filter and app not in apps_filter:
121+ continue
122+ if not v.get("node"):
123+ continue
124+ if nodes_filter and v["node"] not in nodes_filter:
125+ continue
126+ by_node.setdefault(v["node"], {})[app] = manifest.load(app)
127+ changes = {}
128+ summary = {"ok": 0, "fixed": 0, "ko": 0, "skipped": 0}
129+ tun_state = None
130+ try:
131+ tun_state = tunnel.state()
132+ except tunnel.TunnelError as e:
133+ _log("tunnel : %s" % e, quiet)
134+ for alias in sorted(by_node):
135+ apps = by_node[alias]
136+ ip = nodes.ip_of(alias, rediscover=False)
137+ if not ip:
138+ # une seule redécouverte par passage
139+ ip = nodes.ip_of(alias, rediscover=True)
140+ if not ip:
141+ since = st["down"].setdefault(alias, _now())
142+ _log("%s : INJOIGNABLE (depuis %s) — %d app(s) : %s" % (alias, since, len(apps), ", ".join(sorted(apps))), quiet)
143+ for app in apps:
144+ if reg["apps"][app].get("status") != "node-down":
145+ changes[app] = {"status": "node-down", "health": "nœud %s injoignable" % alias}
146+ summary["ko"] += len(apps)
147+ continue
148+ if alias in st["down"]:
149+ _log("%s : de retour (était injoignable depuis %s)" % (alias, st["down"].pop(alias)), quiet)
150+ rep = _node_report(ip, apps)
151+ if rep is None:
152+ _log("%s : sonde impossible" % alias, quiet)
153+ summary["skipped"] += len(apps)
154+ continue
155+ if rep["uptime"] < GRACE_S:
156+ _log("%s : redémarré il y a %ds — délai de grâce, on repasse plus tard" % (alias, rep["uptime"]), quiet)
157+ summary["skipped"] += len(apps)
158+ continue
159+ # --- WireGuard vers la passerelle (seulement si le nœud expose un site)
160+ exposes = any(tunnel.spec(m, alias) for m in apps.values())
161+ if exposes and (rep["wg1"] is False or rep["hub"] is False):
162+ _log("%s : wg1 %s / hub %s" % (alias, "up" if rep["wg1"] else "DOWN", "ok" if rep["hub"] else "KO"), quiet)
163+ if not dry_run:
164+ try:
165+ tunnel.ensure_peer(alias, log=lambda s: _log(s, quiet), st=tun_state)
166+ except tunnel.TunnelError as e:
167+ _log(" tunnel : %s" % e, quiet)
168+ resurrected = False
169+ for app in sorted(apps):
170+ m = apps[app]
171+ names = manifest.pm2_names(m)
172+ labels = manifest.launchd_labels(m)
173+ missing = [n for n in names if n not in rep["procs"]] if rep["pm2"] else list(names)
174+ bad = [n for n in names if n in rep["procs"] and rep["procs"][n][0] in ("errored", "stopped")
175+ and next((p for p in m["processes"] if p["name"] == n), {}).get("autorestart", True)
176+ and not next((p for p in m["processes"] if p["name"] == n), {}).get("cron_restart")]
177+ la_missing = [l for l in labels if l not in rep["la"]]
178+ code = rep["http"].get(app)
179+ http_ok = _http_ok(code) if m.get("port") else True
180+ problems = []
181+ if missing:
182+ problems.append("PM2 absent: %s" % ",".join(missing))
183+ if bad:
184+ problems.append("PM2 %s: %s" % (rep["procs"][bad[0]][0], ",".join(bad)))
185+ if la_missing:
186+ problems.append("launchd absent: %s" % ",".join(la_missing))
187+ if not http_ok and not missing and not bad:
188+ problems.append("HTTP local %s" % code)
189+ if not problems:
190+ summary["ok"] += 1
191+ if reg["apps"][app].get("status") != "online":
192+ changes[app] = {"status": "online", "health": "local %s" % code if m.get("port") else "processus online"}
193+ # route publique : suit-elle bien ce nœud ?
194+ if tun_state and public and tunnel.spec(m, alias):
195+ s = tunnel.spec(m, alias)
196+ r = tunnel.routes(s["gateway"], tun_state).get(s["domain"])
197+ if not tunnel._route_matches(r, s["upstreams"]):
198+ _log("%s/%s : route https://%s %s" % (alias, app, s["domain"], "absente" if not r else "→ %s au lieu de %s" % (", ".join(u["addr"] for u in r.get("upstreams", [])), s["upstreams"][0])), quiet)
199+ if not dry_run and _allowed(st, app):
200+ try:
201+ tunnel.ensure_route(m, alias, log=lambda x: _log(x, quiet), st=tun_state)
202+ _record(st, app)
203+ changes[app] = dict(changes.get(app, {}), tunnel={"gateway": s["gateway"], "domain": s["domain"], "upstream": s["upstreams"][0], "url": "https://%s" % s["domain"]})
204+ summary["fixed"] += 1
205+ except tunnel.TunnelError as e:
206+ _log(" tunnel : %s" % e, quiet)
207+ continue
208+ _log("%s/%s : %s" % (alias, app, " ; ".join(problems)), quiet)
209+ if dry_run:
210+ summary["ko"] += 1
211+ continue
212+ if not _allowed(st, app):
213+ _log(" %s : trop d'actions dans l'heure (%d), on laisse tranquille" % (app, MAX_ACTIONS_PER_HOUR), quiet)
214+ summary["ko"] += 1
215+ changes[app] = {"status": "unhealthy", "health": " ; ".join(problems)}
216+ continue
217+ _record(st, app)
218+ try:
219+ if rep["pm2"] is False and not resurrected:
220+ rc, out, _ = ssh.run(ip, "pm2 resurrect 2>&1 | tail -1", timeout=120)
221+ _log(" pm2 resurrect sur %s : %s" % (alias, out.strip()[-100:]), quiet)
222+ resurrected = True
223+ if missing or la_missing:
224+ deploy.start(app, alias, m=m, ip=ip)
225+ elif bad:
226+ rc, out, _ = ssh.run(ip, "pm2 restart %s --update-env >/dev/null 2>&1; pm2 save --force >/dev/null 2>&1; echo ok" % " ".join(bad), timeout=120)
227+ _log(" pm2 restart %s : %s" % (",".join(bad), out.strip()[-40:]), quiet)
228+ elif not http_ok:
229+ rc, out, _ = ssh.run(ip, "pm2 restart %s --update-env >/dev/null 2>&1; echo ok" % " ".join(names), timeout=120) if names else (0, "", "")
230+ _log(" HTTP local %s → pm2 restart %s" % (code, ",".join(names)), quiet)
231+ ok, det = deploy.health(app, alias, ip=ip, m=m, timeout=45, public=False)
232+ tun = None
233+ if ok and tunnel.spec(m, alias):
234+ tun = deploy._publish(app, alias, m, ip)
235+ ok, det = deploy.health(app, alias, ip=ip, m=m, timeout=20, public=public and bool(tun))
236+ _log(" %s : %s (%s)" % (app, "RÉPARÉE" if ok else "toujours KO", det), quiet)
237+ fields = {"status": "online" if ok else "unhealthy", "health": det, "healed": _now()}
238+ if tun:
239+ fields["tunnel"] = tun
240+ changes[app] = fields
241+ summary["fixed" if ok else "ko"] += 1
242+ except (deploy.DeployError, tunnel.TunnelError) as e:
243+ _log(" %s : ÉCHEC de réparation — %s" % (app, e), quiet)
244+ changes[app] = {"status": "unhealthy", "health": str(e)[:200]}
245+ summary["ko"] += 1
246+ if changes and not dry_run:
247+ registry.update_many(changes)
248+ _save_state(st)
249+ _log("heal terminé en %.0fs : %d ok, %d réparées, %d KO, %d ignorées (grâce/sonde)%s" % (
250+ time.time() - t0, summary["ok"], summary["fixed"], summary["ko"], summary["skipped"], " [dry-run]" if dry_run else ""), quiet)
251+ return summary
252+
253+
254+# ------------------------------------------------------------ LaunchDaemon sur la passerelle ---
255+
256+def plist_content():
257+ mld_bin = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "bin", "mld")
258+ d = {
259+ "Label": HEAL_LABEL,
260+ "ProgramArguments": [mld_bin, "heal", "--quiet"],
261+ "UserName": config.USER,
262+ "RunAtLoad": True,
263+ "StartInterval": HEAL_INTERVAL_S,
264+ "EnvironmentVariables": {"HOME": config.HOME, "PATH": "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", "LANG": "fr_CA.UTF-8"},
265+ "WorkingDirectory": config.HOME,
266+ "StandardOutPath": os.path.join(config.LOG_DIR, "heal.out"),
267+ "StandardErrorPath": os.path.join(config.LOG_DIR, "heal.err"),
268+ "ProcessType": "Background",
269+ "LowPriorityIO": True,
270+ }
271+ return plistlib.dumps(d).decode()
272+
273+
274+def install(interval=None):
275+ """Installe (ou met à jour) le LaunchDaemon io.maclustr.mld-heal sur la passerelle (sudo via ~/dispatch/.sudo)."""
276+ import subprocess
277+ global HEAL_INTERVAL_S
278+ if interval:
279+ HEAL_INTERVAL_S = int(interval)
280+ pwf = config.SUDO_PW_FILE
281+ pw = open(pwf).read().strip() if os.path.exists(pwf) else None
282+ if not pw:
283+ raise SystemExit("mot de passe sudo absent : %s" % pwf)
284+ tmp = "/tmp/%s.plist" % HEAL_LABEL
285+ open(tmp, "w").write(plist_content())
286+ dest = "/Library/LaunchDaemons/%s.plist" % HEAL_LABEL
287+ sh = ("echo %s | sudo -S -p '' sh -c 'install -m 644 -o root -g wheel %s %s && launchctl bootout system/%s 2>/dev/null; "
288+ "launchctl bootstrap system %s && launchctl print system/%s | grep -E \"state|interval\" | head -2'") % (
289+ shlex.quote(pw), tmp, dest, HEAL_LABEL, dest, HEAL_LABEL)
290+ p = subprocess.run(["bash", "-c", sh], capture_output=True, text=True)
291+ print(p.stdout.strip() or p.stderr.strip()[-300:])
292+ os.remove(tmp)
293+ ok = p.returncode == 0
294+ print("LaunchDaemon %s : %s (toutes les %d s, journal %s/heal.log)" % (HEAL_LABEL, "installé" if ok else "ÉCHEC", HEAL_INTERVAL_S, config.LOG_DIR))
295+ return ok
modified mld/importer.py +8 −3
@@ -1,6 +1,7 @@
1 """`mld import <nœud>` : brouillons de manifestes à partir des processus PM2 / launchd vivants d'un nœud. 1 """`mld import <nœud>` : brouillons de manifestes à partir des processus PM2 / launchd vivants d'un nœud.
2 2
3−Regroupe par préfixe de nom (lou-ka-web, lou-ka-sync, lou-ka-ngrok → lou-ka). Les brouillons vont dans 3+Regroupe par préfixe de nom (lou-ka-web, lou-ka-sync, lou-ka-ngrok → lou-ka ; un vieux tunnel ngrok devient une route
4+MacLustr Tunnel `tunnel.domain`). Les brouillons vont dans
4 ~/dispatch/apps/_drafts/<app>.json ; on les relit, on ajuste, puis on les déplace dans ~/dispatch/apps/. 5 ~/dispatch/apps/_drafts/<app>.json ; on les relit, on ajuste, puis on les déplace dans ~/dispatch/apps/.
5 """ 6 """
6 import json 7 import json
@@ -70,8 +71,12 @@ def import_node(alias, apps_filter=None):
70 m = json.loads(json.dumps(manifest.DEFAULTS)) 71 m = json.loads(json.dumps(manifest.DEFAULTS))
71 mo = re.match(r"(/Users/[^/]+/(?:apps/)?[^/]+|/opt/[^/]+)", cwd) 72 mo = re.match(r"(/Users/[^/]+/(?:apps/)?[^/]+|/opt/[^/]+)", cwd)
72 d = mo.group(1) if mo else cwd 73 d = mo.group(1) if mo else cwd
73− m.update({"app": app, "label": app, "dir": d.replace(config.home_of(alias), "~"), "processes": g["processes"], "ngrok": g["ngrok"], 74+ # un tunnel ngrok encore vivant sur le nœud devient une route MacLustr Tunnel (le processus ngrok sera supprimé au 1er start)
74− "domain": (g["ngrok"] or {}).get("url"), "port": (g["ngrok"] or {}).get("port"), "ram_mb_observed": max(g["ram"], 128)}) 75+ ng = g["ngrok"] or {}
76+ m.update({"app": app, "label": app, "dir": d.replace(config.home_of(alias), "~"), "processes": g["processes"], "ngrok": None,
77+ "tunnel": ({"domain": ng.get("url"), "gateway": config.TUNNEL_DEFAULT} if ng.get("url") else None),
78+ "legacy_pm2": [ng["name"]] if ng.get("name") else [],
79+ "domain": ng.get("url"), "port": ng.get("port"), "ram_mb_observed": max(g["ram"], 128)})
75 m["requires"]["ports"] = [m["port"]] if m["port"] else [] 80 m["requires"]["ports"] = [m["port"]] if m["port"] else []
76 m["placement"]["reason"] = "importé de %s" % alias 81 m["placement"]["reason"] = "importé de %s" % alias
77 manifest.detect_requirements(m) 82 manifest.detect_requirements(m)
modified mld/manifest.py +40 −9
@@ -6,12 +6,15 @@ Schéma :
6 "dir": "~/apps/lou-ka", # répertoire de l'app (même chemin absolu sur tous les nœuds) 6 "dir": "~/apps/lou-ka", # répertoire de l'app (même chemin absolu sur tous les nœuds)
7 "extra_paths": ["~/.ssh/trouveka_tunnel"], # autres fichiers/dossiers à copier avec l'app 7 "extra_paths": ["~/.ssh/trouveka_tunnel"], # autres fichiers/dossiers à copier avec l'app
8 "sync_excludes": ["data/backups/"], # exclusions rsync (relatives à dir) 8 "sync_excludes": ["data/backups/"], # exclusions rsync (relatives à dir)
9− "requires": {"runtimes": ["python@3.14", "pm2", "ngrok"], "ram_gb": 2, "ports": [8095]}, 9+ "requires": {"runtimes": ["python@3.14", "pm2"], "ram_gb": 2, "ports": [8095]},
10 "ram_mb_observed": 400, "size_mb": 4359, 10 "ram_mb_observed": 400, "size_mb": 4359,
11 "placement": {"pin": null, "prefer": null, "avoid": [], "reason": ""}, 11 "placement": {"pin": null, "prefer": null, "avoid": [], "reason": ""},
12 "processes": [{"name": "lou-ka-web", "manager": "pm2", "script": "...", "args": [...], "interpreter": null, 12 "processes": [{"name": "lou-ka-web", "manager": "pm2", "script": "...", "args": [...], "interpreter": null,
13 "cwd": "...", "env": {...}, "cron_restart": null, "autorestart": true, "max_memory_restart": null}], 13 "cwd": "...", "env": {...}, "cron_restart": null, "autorestart": true, "max_memory_restart": null}],
14− "ngrok": {"name": "lou-ka-ngrok", "url": "www.lou-ka.com", "port": 8095} | null, 14+ "tunnel": {"domain": "www.lou-ka.com", "gateway": "BHS64", "redirects": ["lou-ka.com"], "websocket": false} | null,
15+ # exposition publique via MacLustr Tunnel : la route Caddy https://domain →
16+ # <nœud courant>:<port> est (re)pointée par `mld deploy`/`move`. `domain` seul suffit.
17+ "ngrok": null, # OBSOLÈTE (ngrok retiré le 2026-09-10) : un vieux bloc est converti en `tunnel` au chargement
15 "launchd": [{"label": "...", "program_arguments": [...], "working_directory": "...", "env": {...}, "keep_alive": true}], 18 "launchd": [{"label": "...", "program_arguments": [...], "working_directory": "...", "env": {...}, "keep_alive": true}],
16 "env_overrides": {".env": {"DATABASE_URL": "postgresql://...@{{IP:M2M32}}:5432/x"}}, # réécritures de fichiers .env 19 "env_overrides": {".env": {"DATABASE_URL": "postgresql://...@{{IP:M2M32}}:5432/x"}}, # réécritures de fichiers .env
17 "hooks": {"post_sync": ["bash ..."], "post_start": []}, 20 "hooks": {"post_sync": ["bash ..."], "post_start": []},
@@ -28,7 +31,7 @@ from . import config
28 DEFAULTS = { 31 DEFAULTS = {
29 "label": None, "domain": None, "port": None, "health_path": "/", "extra_paths": [], "sync_excludes": [], 32 "label": None, "domain": None, "port": None, "health_path": "/", "extra_paths": [], "sync_excludes": [],
30 "requires": {"runtimes": ["pm2"], "ram_gb": 1, "ports": []}, "ram_mb_observed": 512, "size_mb": 0, 33 "requires": {"runtimes": ["pm2"], "ram_gb": 1, "ports": []}, "ram_mb_observed": 512, "size_mb": 0,
31− "placement": {"pin": None, "prefer": None, "avoid": [], "reason": ""}, "processes": [], "ngrok": None, 34+ "placement": {"pin": None, "prefer": None, "avoid": [], "reason": ""}, "processes": [], "tunnel": None, "ngrok": None,
32 "launchd": [], "env_overrides": {}, "hooks": {"post_sync": [], "post_start": []}, "ka_repo": False, 35 "launchd": [], "env_overrides": {}, "hooks": {"post_sync": [], "post_start": []}, "ka_repo": False,
33 } 36 }
34 37
@@ -47,6 +50,33 @@ def load(app):
47 m["requires"].setdefault("runtimes", ["pm2"]) 50 m["requires"].setdefault("runtimes", ["pm2"])
48 m["requires"].setdefault("ram_gb", 1) 51 m["requires"].setdefault("ram_gb", 1)
49 m["requires"].setdefault("ports", [m["port"]] if m.get("port") else []) 52 m["requires"].setdefault("ports", [m["port"]] if m.get("port") else [])
53+ return modernize(m)
54+
55+
56+def modernize(m):
57+ """ngrok → MacLustr Tunnel (2026-09-10). Un ancien bloc `ngrok` devient une route `tunnel` ; le runtime « ngrok » disparaît
58+ des prérequis (plus installé nulle part) ; le nom du vieux processus PM2 `<app>-ngrok` est gardé pour le nettoyer au start."""
59+ n = m.get("ngrok")
60+ if n:
61+ if not m.get("tunnel"):
62+ m["tunnel"] = {"domain": n.get("url") or m.get("domain"), "gateway": config.TUNNEL_DEFAULT,
63+ "note": "converti automatiquement depuis l'ancien bloc ngrok"}
64+ if not m.get("domain"):
65+ m["domain"] = n.get("url")
66+ if not m.get("port") and n.get("port"):
67+ m["port"] = n["port"]
68+ if n.get("name"):
69+ m["legacy_pm2"] = sorted(set((m.get("legacy_pm2") or []) + [n["name"]]))
70+ m["ngrok"] = None
71+ if m.get("domain") and not m.get("tunnel"):
72+ m["tunnel"] = {"domain": m["domain"], "gateway": config.TUNNEL_DEFAULT}
73+ if m.get("tunnel") and not m["tunnel"].get("domain"):
74+ m["tunnel"]["domain"] = m.get("domain")
75+ if m.get("tunnel") and not m.get("domain"):
76+ m["domain"] = m["tunnel"]["domain"]
77+ m["requires"]["runtimes"] = [r for r in m["requires"]["runtimes"] if r != "ngrok"]
78+ # les vieux processus `<app>-ngrok` (PM2) sont supprimés à chaque start/stop s'ils traînent encore
79+ m["legacy_pm2"] = sorted(set((m.get("legacy_pm2") or []) + ["%s-ngrok" % m["app"]]))
50 return m 80 return m
51 81
52 82
@@ -80,10 +110,12 @@ def render(value, ctx):
80 110
81 111
82 def pm2_names(m): 112 def pm2_names(m):
83− names = [p["name"] for p in m["processes"] if p.get("manager", "pm2") == "pm2"] 113+ return [p["name"] for p in m["processes"] if p.get("manager", "pm2") == "pm2"]
84− if m.get("ngrok"): 114+
85− names.append(m["ngrok"]["name"]) 115+
86− return names 116+def legacy_pm2_names(m):
117+ """Processus PM2 d'anciennes versions du manifeste (tunnels ngrok) à supprimer s'ils existent encore sur le nœud."""
118+ return [n for n in (m.get("legacy_pm2") or []) if n not in pm2_names(m)]
87 119
88 120
89 def launchd_labels(m): 121 def launchd_labels(m):
@@ -94,8 +126,7 @@ def detect_requirements(m):
94 """Déduit les runtimes requis à partir des processus (complète requires.runtimes).""" 126 """Déduit les runtimes requis à partir des processus (complète requires.runtimes)."""
95 req = set(m["requires"].get("runtimes") or []) 127 req = set(m["requires"].get("runtimes") or [])
96 req.add("pm2") 128 req.add("pm2")
97− if m.get("ngrok"): 129+ req.discard("ngrok")
98− req.add("ngrok")
99 blob = json.dumps(m["processes"]) + json.dumps(m.get("launchd", [])) 130 blob = json.dumps(m["processes"]) + json.dumps(m.get("launchd", []))
100 if "/opt/homebrew/bin/node" in blob or "next" in blob or "npm" in blob or ".mjs" in blob or ".js" in blob: 131 if "/opt/homebrew/bin/node" in blob or "next" in blob or "npm" in blob or ".mjs" in blob or ".js" in blob:
101 req.add("node") 132 req.add("node")
modified mld/prepare.py +16 −9
@@ -1,4 +1,5 @@
1−"""`mld prepare <nœud>` : installe les runtimes manquants (Homebrew), copie la config ngrok, active pm2 au démarrage, pose le marqueur.""" 1+"""`mld prepare <nœud>` : installe les runtimes manquants (Homebrew), active pm2 au démarrage, pose le marqueur, raccorde le
2+nœud au MacLustr Tunnel (wg1 → BHS64) et applique `mld harden` (résilience coupure/MAJ)."""
2 import os 3 import os
3 import shlex 4 import shlex
4 from . import config, nodes, ssh 5 from . import config, nodes, ssh
@@ -11,13 +12,13 @@ def _sudo_pw(alias=None):
11 return None 12 return None
12 13
13 14
14−def prepare(alias, runtimes=None, ka_helpers=False): 15+def prepare(alias, runtimes=None, ka_helpers=False, tunnel_peer=True, harden=True):
15 ip = nodes.ip_of(alias) 16 ip = nodes.ip_of(alias)
16 if not ip: 17 if not ip:
17 raise SystemExit("nœud %s injoignable" % alias) 18 raise SystemExit("nœud %s injoignable" % alias)
18 info = nodes.probe(alias, ip) 19 info = nodes.probe(alias, ip)
19 have = info.get("runtimes", {}) 20 have = info.get("runtimes", {})
20− want = runtimes or ["node", "pnpm", "pm2", "ngrok", "python@3.14", "python@3.13", "uv", "git"] 21+ want = runtimes or ["node", "pnpm", "pm2", "python@3.14", "python@3.13", "uv", "git"]
21 print("prepare %s (%s) — présents : %s" % (alias, ip, ", ".join(k for k, v in have.items() if v))) 22 print("prepare %s (%s) — présents : %s" % (alias, ip, ", ".join(k for k, v in have.items() if v)))
22 if not have.get("brew"): 23 if not have.get("brew"):
23 print(" ✗ Homebrew absent : bootstrap requis (Xcode CLT). Lancer d'abord : mld bootstrap %s" % alias) 24 print(" ✗ Homebrew absent : bootstrap requis (Xcode CLT). Lancer d'abord : mld bootstrap %s" % alias)
@@ -29,12 +30,7 @@ def prepare(alias, runtimes=None, ka_helpers=False):
29 fallback = " || npm install -g pm2 >/dev/null 2>&1" if r == "pm2" else "" 30 fallback = " || npm install -g pm2 >/dev/null 2>&1" if r == "pm2" else ""
30 script += "echo ' brew install %s'; brew install -q %s >/dev/null 2>&1%s || echo ' ! échec %s'\n" % (r, config.BREW_FORMULAE[r], fallback, r) 31 script += "echo ' brew install %s'; brew install -q %s >/dev/null 2>&1%s || echo ' ! échec %s'\n" % (r, config.BREW_FORMULAE[r], fallback, r)
31 elif r in config.BREW_CASKS: 32 elif r in config.BREW_CASKS:
32− script += "echo ' brew install --cask ngrok'; brew install -q --cask ngrok >/dev/null 2>&1 || brew install -q ngrok/ngrok/ngrok >/dev/null 2>&1 || echo ' ! échec ngrok'\n" 33+ script += "echo ' brew install --cask %s'; brew install -q --cask %s >/dev/null 2>&1 || echo ' ! échec %s'\n" % (r, config.BREW_CASKS[r], r)
33− # config ngrok (authtoken) depuis la passerelle
34− if "ngrok" in missing or not have.get("ngrok"):
35− if os.path.exists(config.NGROK_CFG):
36− rc0, _, _ = ssh.write_remote_file(ip, "%s/Library/Application Support/ngrok/ngrok.yml" % config.home_of(alias), open(config.NGROK_CFG).read(), mode="600")
37− script += "echo ' ngrok.yml %s'\n" % ("copié" if rc0 == 0 else "ÉCHEC de copie")
38 # marqueur 34 # marqueur
39 script += "[ -f ~/.maclustr-node ] || echo %s > ~/.maclustr-node\n" % alias 35 script += "[ -f ~/.maclustr-node ] || echo %s > ~/.maclustr-node\n" % alias
40 # pm2 startup (launchd) — nécessite sudo 36 # pm2 startup (launchd) — nécessite sudo
@@ -62,6 +58,17 @@ def prepare(alias, runtimes=None, ka_helpers=False):
62 print(out) 58 print(out)
63 if err.strip(): 59 if err.strip():
64 print(" stderr:", err.strip()[-300:]) 60 print(" stderr:", err.strip()[-300:])
61+ # MacLustr Tunnel : raccorde le nœud au hub BHS64 (wg1) s'il ne l'est pas — nécessaire pour exposer un site
62+ if tunnel_peer:
63+ from . import tunnel
64+ try:
65+ tunnel.ensure_peer(alias, log=print)
66+ except tunnel.TunnelError as e:
67+ print(" tunnel : %s" % e)
68+ # résilience (auto-login, pas de veille, redémarrage après coupure, MAJ macOS/Tailscale off)
69+ if harden:
70+ from . import harden as _harden
71+ _harden.harden(alias)
65 info = nodes.probe(alias, ip) 72 info = nodes.probe(alias, ip)
66 still = [r for r in want if not info.get("runtimes", {}).get(r)] 73 still = [r for r in want if not info.get("runtimes", {}).get(r)]
67 print(" résultat : %s" % ("tout est là" if not still else "manquent encore " + ", ".join(still))) 74 print(" résultat : %s" % ("tout est là" if not still else "manquent encore " + ", ".join(still)))
modified mld/registry.py +14 −0
@@ -59,6 +59,20 @@ def set_app(app, **fields):
59 save(r) 59 save(r)
60 60
61 61
62+def update_many(changes, push=True):
63+ """Met à jour plusieurs apps ({app: {champs}}) en une seule sauvegarde/diffusion (status --live, heal)."""
64+ if not changes:
65+ return
66+ r = load()
67+ now = time.strftime("%Y-%m-%dT%H:%M:%S")
68+ for app, fields in changes.items():
69+ cur = r["apps"].get(app, {})
70+ cur.update(fields)
71+ cur["updated"] = now
72+ r["apps"][app] = cur
73+ save(r, push=push)
74+
75+
62 def remove_app(app, note=""): 76 def remove_app(app, note=""):
63 r = load() 77 r = load()
64 if app in r["apps"]: 78 if app in r["apps"]:
modified mld/render.py +2 −9
@@ -20,7 +20,8 @@ def _home(path, ctx):
20 20
21 21
22 def ecosystem(m, ctx): 22 def ecosystem(m, ctx):
23− """Retourne le contenu JS d'un ecosystem.config.cjs pour les processus PM2 + tunnel ngrok.""" 23+ """Retourne le contenu JS d'un ecosystem.config.cjs pour les processus PM2 (l'exposition publique est une route
24+ Caddy sur la passerelle du MacLustr Tunnel, plus aucun processus ngrok local)."""
24 from .manifest import render 25 from .manifest import render
25 apps = [] 26 apps = []
26 for p in m["processes"]: 27 for p in m["processes"]:
@@ -54,14 +55,6 @@ def ecosystem(m, ctx):
54 if p.get("node_args"): 55 if p.get("node_args"):
55 a["node_args"] = p["node_args"] 56 a["node_args"] = p["node_args"]
56 apps.append(a) 57 apps.append(a)
57− if m.get("ngrok"):
58− n = m["ngrok"]
59− args = ["http", "--url=%s" % n["url"], str(n["port"]), "--log=stdout"] + list(n.get("extra_args") or [])
60− apps.append({
61− "name": n["name"], "script": n.get("binary") or "/opt/homebrew/bin/ngrok", "args": " ".join(args),
62− "interpreter": "none", "cwd": _home(render(m["dir"], ctx), ctx), "exec_mode": "fork", "autorestart": True,
63− "restart_delay": 5000, "merge_logs": True, "time": True,
64− })
65 head = "// généré par maclustr-dispatch — ne pas éditer à la main (app: %s, nœud: %s)\n" % (m["app"], ctx.get("NODE")) 58 head = "// généré par maclustr-dispatch — ne pas éditer à la main (app: %s, nœud: %s)\n" % (m["app"], ctx.get("NODE"))
66 return head + "module.exports = { apps: %s };\n" % json.dumps(apps, indent=2, ensure_ascii=False) 59 return head + "module.exports = { apps: %s };\n" % json.dumps(apps, indent=2, ensure_ascii=False)
67 60
added mld/tunnel.py +229 −0
@@ -0,0 +1,229 @@
1+"""MacLustr Tunnel — le « ngrok maison » du cluster, vu depuis mld.
2+
3+Une passerelle OVH (BHS64 par défaut) porte un hub WireGuard et Caddy (TLS Let's Encrypt). Chaque Mac y est raccordé
4+par une interface WireGuard (LaunchDaemon io.maclustr.wireguard-wg1) et chaque site public est une route Caddy
5+`https://<domaine> → <ip-wg du nœud>:<port>`. Tout se pilote sur la passerelle avec `tunnelctl` :
6+
7+ tunnelctl json état machine-lisible (pairs + routes)
8+ tunnelctl add <dom> <alias>:<port> route (idempotent : réécrit sites/<dom>.caddy + reload)
9+ tunnelctl redirect <dom> <cible> 308 (apex → www)
10+ tunnelctl rm <dom>
11+ tunnelctl peer add <alias> <pubkey> pair WireGuard (IP fixe tirée de /etc/maclustr-tunnel/ipmap)
12+
13+Ici : `ensure_peer(alias)` raccorde un nœud qui ne l'est pas (script assets/wg-node-setup.sh, même recette que `mlt peer`),
14+`ensure_route(m, alias)` fait suivre la route publique d'une app au nœud où mld vient de la démarrer. Le manifeste ne porte
15+que `tunnel.domain` (+ `gateway`, `redirects`, `websocket`) : l'upstream est TOUJOURS le nœud courant du registre.
16+"""
17+import json
18+import os
19+import shlex
20+import subprocess
21+import time
22+from . import config, ssh
23+
24+ASSETS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "assets")
25+HANDSHAKE_OK_S = 200 # keepalive 25 s : au-delà de ~3 min sans handshake, le pair est considéré tombé
26+
27+
28+class TunnelError(Exception):
29+ pass
30+
31+
32+def gateway(name=None):
33+ name = name or config.TUNNEL_DEFAULT
34+ if name not in config.TUNNEL_GATEWAYS:
35+ raise TunnelError("passerelle inconnue « %s » (connues : %s)" % (name, ", ".join(config.TUNNEL_GATEWAYS)))
36+ return name, config.TUNNEL_GATEWAYS[name]
37+
38+
39+def gw_run(name, cmd, timeout=60):
40+ """Commande sur la passerelle (ssh direct ubuntu@ip avec la clé de M1M32)."""
41+ name, g = gateway(name)
42+ try:
43+ p = subprocess.run(["ssh"] + ssh.SSH_OPTS + ["%s@%s" % (g["user"], g["host"]), cmd], capture_output=True, text=True, timeout=timeout)
44+ except subprocess.TimeoutExpired:
45+ return 124, "", "timeout après %ss" % timeout
46+ return p.returncode, p.stdout, p.stderr
47+
48+
49+def state(name=None):
50+ """Sortie de `tunnelctl json` : {"gateway", "ip", "wg": {"peers": [...]}, "caddy": {"active", "routes": [...]}}."""
51+ name, g = gateway(name)
52+ rc, out, err = gw_run(name, "sudo -n tunnelctl json", timeout=40)
53+ if rc != 0 or not out.strip():
54+ raise TunnelError("passerelle %s (%s) injoignable ou tunnelctl KO : %s" % (name, g["host"], (err or out).strip()[-160:]))
55+ return json.loads(out.strip().splitlines()[-1])
56+
57+
58+def routes(name=None, st=None):
59+ st = st or state(name)
60+ return {r["domain"]: r for r in st["caddy"]["routes"]}
61+
62+
63+def peers(name=None, st=None):
64+ st = st or state(name)
65+ return {p["alias"]: p for p in st["wg"]["peers"]}
66+
67+
68+# ------------------------------------------------------------------ routes ---
69+
70+def spec(m, alias):
71+ """Route attendue pour l'app `m` tournant sur `alias` → dict ou None si l'app n'est pas publique."""
72+ t = m.get("tunnel") or {}
73+ domain = t.get("domain") or m.get("domain")
74+ if not domain or not m.get("port"):
75+ return None
76+ return {
77+ "gateway": t.get("gateway") or config.TUNNEL_DEFAULT,
78+ "domain": domain,
79+ "upstreams": ["%s:%s" % (alias, m["port"])],
80+ "redirects": list(t.get("redirects") or []),
81+ "websocket": bool(t.get("websocket")),
82+ }
83+
84+
85+def _route_matches(route, want):
86+ """La route Caddy actuelle pointe-t-elle déjà vers les upstreams voulus (comparaison par alias:port) ?"""
87+ if not route or route.get("kind") != "proxy":
88+ return False
89+ cur = sorted("%s:%s" % (u.get("alias") or u["addr"].split(":")[0], u["addr"].split(":")[-1]) for u in route.get("upstreams", []))
90+ return cur == sorted(want)
91+
92+
93+def ensure_route(m, alias, log=print, st=None):
94+ """Fait pointer https://<domaine> vers <alias>:<port> (et pose les redirections). Retourne le spec, ou None si l'app n'est pas publique."""
95+ s = spec(m, alias)
96+ if not s:
97+ return None
98+ name, g = gateway(s["gateway"])
99+ cur = routes(name, st)
100+ r = cur.get(s["domain"])
101+ if _route_matches(r, s["upstreams"]):
102+ log(" tunnel %s : route https://%s → %s déjà en place" % (name, s["domain"], ", ".join(s["upstreams"])))
103+ else:
104+ args = " ".join(shlex.quote(x) for x in s["upstreams"]) + (" --websocket" if s["websocket"] else "")
105+ rc, out, err = gw_run(name, "sudo -n tunnelctl add %s %s" % (shlex.quote(s["domain"]), args), timeout=90)
106+ if rc != 0:
107+ raise TunnelError("route %s → %s refusée par %s : %s" % (s["domain"], args, name, (err or out).strip()[-200:]))
108+ log(" tunnel %s : route https://%s → %s%s" % (name, s["domain"], ", ".join(s["upstreams"]), " (remplace %s)" % ", ".join(u["addr"] for u in r.get("upstreams", [])) if r else ""))
109+ for red in s["redirects"]:
110+ rr = cur.get(red)
111+ if rr and rr.get("kind") == "redirect" and rr.get("target", "").rstrip("/").endswith(s["domain"]):
112+ continue
113+ rc, out, err = gw_run(name, "sudo -n tunnelctl redirect %s %s" % (shlex.quote(red), shlex.quote(s["domain"])), timeout=90)
114+ log(" tunnel %s : redirection %s → %s %s" % (name, red, s["domain"], "ok" if rc == 0 else "ÉCHEC " + (err or out).strip()[-120:]))
115+ return s
116+
117+
118+def remove_route(m, log=print):
119+ """Retire la route (et ses redirections) d'une app retirée du cluster : le domaine tombe sur la page 404 MacLustr."""
120+ t = m.get("tunnel") or {}
121+ domain = t.get("domain") or m.get("domain")
122+ if not domain:
123+ return False
124+ name, g = gateway(t.get("gateway"))
125+ for d in [domain] + list(t.get("redirects") or []):
126+ rc, out, err = gw_run(name, "sudo -n tunnelctl rm %s" % shlex.quote(d), timeout=60)
127+ log(" tunnel %s : route %s %s" % (name, d, "retirée" if rc == 0 else "ÉCHEC " + (err or out).strip()[-120:]))
128+ return True
129+
130+
131+# ------------------------------------------------------------------- pairs ---
132+
133+def peer_state(alias, name=None, st=None):
134+ """(présent, handshake_s) — handshake None = jamais."""
135+ p = peers(name, st).get(alias)
136+ if not p:
137+ return False, None
138+ return True, p.get("handshakeS")
139+
140+
141+def peer(alias, name=None, log=print):
142+ """Raccorde un Mac au hub : installe WireGuard (Homebrew) + LaunchDaemon, enregistre la clé sur la passerelle (= `mlt peer`)."""
143+ from . import nodes
144+ name, g = gateway(name)
145+ ip = nodes.ip_of(alias)
146+ if not ip:
147+ raise TunnelError("nœud %s injoignable" % alias)
148+ rc, out, err = gw_run(name, "awk -v a=%s '$1==a{print $2}' /etc/maclustr-tunnel/ipmap; sudo -n wg show wg0 public-key" % shlex.quote(alias))
149+ parts = out.split()
150+ if rc != 0 or len(parts) < 2:
151+ raise TunnelError("alias %s absent de /etc/maclustr-tunnel/ipmap sur %s (l'ajouter : « %s %s.<n> »)" % (alias, name, alias, g["subnet"]))
152+ wg_ip, hub_pub = parts[0], parts[1]
153+ pwf = config.sudo_pw_file(alias)
154+ pw = open(pwf).read().strip() if os.path.exists(pwf) else None
155+ if not pw:
156+ raise TunnelError("mot de passe sudo de %s absent (%s)" % (alias, pwf))
157+ script = open(os.path.join(ASSETS, "wg-node-setup.sh")).read()
158+ ssh.write_remote_file(ip, "/tmp/mld-wg-node-setup.sh", script, mode="755")
159+ log(" tunnel %s : raccordement de %s (%s, %s) …" % (name, alias, wg_ip, g["iface"]))
160+ rc, out, err = ssh.run(ip, "bash /tmp/mld-wg-node-setup.sh %s %s %s %s %s; rm -f /tmp/mld-wg-node-setup.sh" % (wg_ip, hub_pub, shlex.quote(pw), g["iface"], g["endpoint"]), timeout=1200)
161+ pub = [l.split()[-1] for l in out.splitlines() if "PUBKEY" in l]
162+ if not pub:
163+ raise TunnelError("wg-node-setup.sh sur %s n'a pas renvoyé de clé publique : %s" % (alias, (err or out).strip()[-300:]))
164+ rc, out, err = gw_run(name, "sudo -n tunnelctl peer add %s %s" % (shlex.quote(alias), shlex.quote(pub[-1])), timeout=60)
165+ if rc != 0:
166+ raise TunnelError("tunnelctl peer add %s : %s" % (alias, (err or out).strip()[-200:]))
167+ for _ in range(12):
168+ time.sleep(4)
169+ present, hs = peer_state(alias, name)
170+ if present and hs is not None and hs < 60:
171+ log(" tunnel %s : %s raccordé (%s, handshake %ss)" % (name, alias, wg_ip, hs))
172+ return True
173+ log(" tunnel %s : %s enregistré (%s), handshake pas encore vu — keepalive 25 s" % (name, alias, wg_ip))
174+ return True
175+
176+
177+def kick_node_iface(alias, ip, name=None, log=print):
178+ """Relance le LaunchDaemon WireGuard du nœud (interface tombée après un réveil réseau, par ex.)."""
179+ name, g = gateway(name)
180+ pwf = config.sudo_pw_file(alias)
181+ pw = open(pwf).read().strip() if os.path.exists(pwf) else None
182+ if not pw:
183+ return False
184+ lbl = "io.maclustr.wireguard-%s" % g["iface"]
185+ rc, out, err = ssh.run(ip, "test -f /Library/LaunchDaemons/%s.plist || exit 9; echo %s | sudo -S -p '' launchctl kickstart -k system/%s && echo kicked" % (lbl, shlex.quote(pw), lbl), timeout=60)
186+ if rc == 9:
187+ return False
188+ log(" tunnel %s : %s %s" % (name, lbl, "relancé sur " + alias if "kicked" in out else "ÉCHEC " + (err or out).strip()[-100:]))
189+ return "kicked" in out
190+
191+
192+def ensure_peer(alias, name=None, log=print, st=None):
193+ """Nœud raccordé et vivant sur le hub ; sinon relance son interface, sinon le raccorde. Retourne True si OK."""
194+ from . import nodes
195+ name, g = gateway(name)
196+ present, hs = peer_state(alias, name, st)
197+ if present and hs is not None and hs < HANDSHAKE_OK_S:
198+ return True
199+ if present:
200+ ip = nodes.ip_of(alias, rediscover=False)
201+ if ip and kick_node_iface(alias, ip, name, log):
202+ for _ in range(8):
203+ time.sleep(4)
204+ present, hs = peer_state(alias, name)
205+ if hs is not None and hs < 60:
206+ return True
207+ if ip and ssh.ok(ip, "test -f /Library/LaunchDaemons/io.maclustr.wireguard-%s.plist" % g["iface"]):
208+ log(" tunnel %s : %s enregistré mais sans handshake récent (%s)" % (name, alias, "jamais" if hs is None else "%ss" % hs))
209+ return False
210+ return peer(alias, name, log)
211+
212+
213+# ------------------------------------------------------------------ affichage ---
214+
215+def print_status(name=None):
216+ name, g = gateway(name)
217+ st = state(name)
218+ now = st.get("ts") or int(time.time())
219+ print("== %s (%s, %s) — WireGuard udp %s, %d pairs" % (name, g["host"], g["place"], st["wg"].get("listenPort"), len(st["wg"]["peers"])))
220+ for p in sorted(st["wg"]["peers"], key=lambda x: x["ip"]):
221+ hs = p.get("handshakeS")
222+ flag = "" if hs is not None and hs < HANDSHAKE_OK_S else " <<< pas de handshake récent"
223+ print(" %-8s %-12s handshake %-8s rx %8.1f Mo tx %8.1f Mo%s" % (p["alias"], p["ip"], "jamais" if hs is None else "%ss" % hs, p["rxBytes"] / 1048576.0, p["txBytes"] / 1048576.0, flag))
224+ print("== Caddy : %s — %d routes" % ("actif" if st["caddy"]["active"] else "ARRÊTÉ", len(st["caddy"]["routes"])))
225+ for r in st["caddy"]["routes"]:
226+ if r["kind"] == "redirect":
227+ print(" %-36s → redirection %s" % (r["domain"], r.get("target")))
228+ else:
229+ print(" %-36s → %s" % (r["domain"], ", ".join("%s (%s)" % (u["addr"], u.get("alias") or "?") for u in r["upstreams"])))
230
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere