ngrok → MacLustr Tunnel intégré à mld ; mld heal (auto-réparation) ; mld harden (résilience coupure/MAJ)
- tunnel.py : routes Caddy et pairs WireGuard via tunnelctl (BHS64/R9128) ; deploy/move repointent la route publique vers le nœud courant, raccordement wg1 automatique ; retire supprime la route ; mld tunnel status|peer|route|rm - manifest : bloc ngrok obsolète converti en tunnel au chargement, runtime ngrok retiré, legacy_pm2 nettoyé au start - heal.py : sonde 1 SSH/nœud (PM2, launchd, wg1, HTTP), répare (resurrect/start/restart/kickstart/route), garde-fous, LaunchDaemon io.maclustr.mld-heal toutes les 5 min (mld heal --install) - harden.py : autorestart/restartfreeze/sleep 0/womp, MAJ macOS+App Store off, Tailscale auto-update off, auto-login kcpassword+autoLoginUser (sauf FileVault / nœuds loués), vérif pm2 startup + wg1 ; appelé par prepare - registry.update_many (une seule diffusion pour status --live / heal) Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
12 changed files +943 −65
added
assets/wg-node-setup.sh
+70 −0
@@ -0,0 +1,70 @@ | ||
| 1 | +#!/bin/bash | |
| 2 | +# wg-node-setup.sh — installe WireGuard (wireguard-go + wg-quick) sur un Mac du cluster et le raccorde au hub R9128. | |
| 3 | +# Usage : bash wg-node-setup.sh <ip-wg du nœud> <pubkey-serveur> <mot-de-passe-sudo> [iface=$IF] [endpoint=51.255.75.61:51820] | |
| 4 | +# $IF = hub R9128 (France, 10.66.0.x) ; wg1 = hub BHS64 (Québec, 10.67.0.x). Un LaunchDaemon par interface. | |
| 5 | +# Imprime « PUBKEY <clé> » à la fin (à donner à tunnelctl peer add sur la passerelle). | |
| 6 | +set -euo pipefail | |
| 7 | +ADDR=${1:?ip wg}; SERVER_PUB=${2:?pubkey serveur}; PW=${3:?sudo}; IF=${4:-wg0}; ENDPOINT=${5:-51.255.75.61:51820} | |
| 8 | +SUBNET=${ADDR%.*} # ex. 10.67.0 | |
| 9 | +BREW=/opt/homebrew/bin/brew | |
| 10 | +# NB : sudo -S lit le mot de passe sur stdin → ne jamais combiner sudo_ avec un heredoc/pipe de contenu. | |
| 11 | +sudo_(){ echo "$PW" | sudo -S -p '' "$@"; } | |
| 12 | +export PATH=/opt/homebrew/bin:/opt/homebrew/sbin:$PATH | |
| 13 | +[ -x "$BREW" ] || { echo "Homebrew absent"; exit 2; } | |
| 14 | +if ! [ -x /opt/homebrew/bin/wg-quick ]; then HOMEBREW_NO_AUTO_UPDATE=1 "$BREW" install -q wireguard-tools >/dev/null 2>&1 || HOMEBREW_NO_AUTO_UPDATE=1 "$BREW" install wireguard-tools; fi | |
| 15 | +[ -x /opt/homebrew/bin/wireguard-go ] || HOMEBREW_NO_AUTO_UPDATE=1 "$BREW" install -q wireguard-go >/dev/null 2>&1 || true | |
| 16 | +ETC=/opt/homebrew/etc/wireguard | |
| 17 | +sudo_ mkdir -p "$ETC" | |
| 18 | +if ! sudo_ test -f "$ETC/$IF.key"; then | |
| 19 | + K=$(/opt/homebrew/bin/wg genkey); T=$(mktemp); chmod 600 "$T"; printf '%s\n' "$K" > "$T" | |
| 20 | + sudo_ install -m 600 -o root "$T" "$ETC/$IF.key"; rm -f "$T" | |
| 21 | +fi | |
| 22 | +PRIV=$(sudo_ cat "$ETC/$IF.key") | |
| 23 | +PUB=$(printf '%s' "$PRIV" | /opt/homebrew/bin/wg pubkey) | |
| 24 | +TMPC=$(mktemp); chmod 600 "$TMPC" | |
| 25 | +{ | |
| 26 | + echo "[Interface]" | |
| 27 | + echo "PrivateKey = $PRIV" | |
| 28 | + echo "Address = $ADDR/24" | |
| 29 | + echo "MTU = 1380" | |
| 30 | + echo | |
| 31 | + echo "[Peer]" | |
| 32 | + echo "# hub R9128 (OVH)" | |
| 33 | + echo "PublicKey = $SERVER_PUB" | |
| 34 | + echo "Endpoint = $ENDPOINT" | |
| 35 | + echo "AllowedIPs = $SUBNET.0/24" | |
| 36 | + echo "PersistentKeepalive = 25" | |
| 37 | +} > "$TMPC" | |
| 38 | +sudo_ install -m 600 -o root "$TMPC" "$ETC/$IF.conf"; rm -f "$TMPC" | |
| 39 | + | |
| 40 | +# LaunchDaemon : monte $IF au boot (wireguard-go reste en arrière-plan) et le remonte s'il tombe. | |
| 41 | +TMPP=$(mktemp) | |
| 42 | +cat > "$TMPP" <<PLIST | |
| 43 | +<?xml version="1.0" encoding="UTF-8"?> | |
| 44 | +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> | |
| 45 | +<plist version="1.0"><dict> | |
| 46 | + <key>Label</key><string>io.maclustr.wireguard-$IF</string> | |
| 47 | + <key>ProgramArguments</key><array> | |
| 48 | + <string>/bin/bash</string><string>-c</string> | |
| 49 | + <string>export PATH=/opt/homebrew/bin:/opt/homebrew/sbin:/usr/bin:/bin:/usr/sbin:/sbin; up(){ n=\$(cat /var/run/wireguard/$IF.name 2>/dev/null) && [ -n "\$n" ] && /opt/homebrew/bin/wg show "\$n" >/dev/null 2>&1; }; while true; do if ! up; then /opt/homebrew/bin/wg-quick down $IF >/dev/null 2>&1; /opt/homebrew/bin/wg-quick up $IF || true; fi; sleep 30; done</string> | |
| 50 | + </array> | |
| 51 | + <key>RunAtLoad</key><true/> | |
| 52 | + <key>KeepAlive</key><true/> | |
| 53 | + <key>StandardOutPath</key><string>/var/log/maclustr-wireguard-$IF.log</string> | |
| 54 | + <key>StandardErrorPath</key><string>/var/log/maclustr-wireguard-$IF.log</string> | |
| 55 | +</dict></plist> | |
| 56 | +PLIST | |
| 57 | +sudo_ install -m 644 -o root -g wheel "$TMPP" /Library/LaunchDaemons/io.maclustr.wireguard-$IF.plist; rm -f "$TMPP" | |
| 58 | +sudo_ launchctl bootout system/io.maclustr.wireguard-$IF >/dev/null 2>&1 || true | |
| 59 | +sudo_ /opt/homebrew/bin/wg-quick down $IF >/dev/null 2>&1 || true | |
| 60 | +sleep 2 | |
| 61 | +ok=0; for i in 1 2 3 4 5 6; do | |
| 62 | + if sudo_ launchctl bootstrap system /Library/LaunchDaemons/io.maclustr.wireguard-$IF.plist 2>/tmp/wg-bootstrap.err; then ok=1; break; fi | |
| 63 | + grep -q -i "already" /tmp/wg-bootstrap.err && { ok=1; break; } | |
| 64 | + sleep 3 | |
| 65 | +done | |
| 66 | +[ $ok = 1 ] || { cat /tmp/wg-bootstrap.err; echo "bootstrap KO → démarrage direct"; sudo_ /opt/homebrew/bin/wg-quick up $IF; } | |
| 67 | +# NB : `wg show $IF` ne résout pas le nom sur macOS (wireguard-tools 1.0.2026) → passer par /var/run/wireguard/$IF.name. | |
| 68 | +for i in $(seq 1 30); do N=$(sudo_ cat /var/run/wireguard/$IF.name 2>/dev/null || true); [ -n "$N" ] && sudo_ /opt/homebrew/bin/wg show "$N" >/dev/null 2>&1 && break; sleep 1; done | |
| 69 | +sudo_ /opt/homebrew/bin/wg show "$N" | head -3 | |
| 70 | +echo "NODE $(cat ~/.maclustr-node 2>/dev/null) IF $N PUBKEY $PUB" | |
modified
mld/cli.py
+57 −2
@@ -129,6 +129,58 @@ def cmd_retire(a): | ||
| 129 | 129 | deploy.retire(a.app, a.node or registry.node_of(a.app), remove_dir=not a.keep_dir) |
| 130 | 130 | if not a.keep_registry: |
| 131 | 131 | registry.remove_app(a.app, note=a.note or "") |
| 132 | + # app retirée du cluster : sa route publique tombe (page 404 MacLustr au lieu d'un 502) | |
| 133 | + from . import tunnel | |
| 134 | + try: | |
| 135 | + tunnel.remove_route(manifest.load(a.app)) | |
| 136 | + except tunnel.TunnelError as e: | |
| 137 | + print(" tunnel :", e) | |
| 138 | + | |
| 139 | + | |
| 140 | +def cmd_tunnel(a): | |
| 141 | + from . import tunnel | |
| 142 | + try: | |
| 143 | + if a.sub == "status": | |
| 144 | + tunnel.print_status(a.gateway) | |
| 145 | + elif a.sub == "peer": | |
| 146 | + for n in a.args: | |
| 147 | + tunnel.ensure_peer(n, a.gateway, log=print) or tunnel.peer(n, a.gateway, log=print) | |
| 148 | + elif a.sub == "route": | |
| 149 | + for app in a.args: | |
| 150 | + node = registry.node_of(app) | |
| 151 | + if not node: | |
| 152 | + print("%s : pas dans le registre" % app) | |
| 153 | + continue | |
| 154 | + m = manifest.load(app) | |
| 155 | + s = tunnel.ensure_route(m, node, log=print) | |
| 156 | + if s: | |
| 157 | + registry.set_app(app, tunnel={"gateway": s["gateway"], "domain": s["domain"], "upstream": s["upstreams"][0], "url": "https://%s" % s["domain"]}) | |
| 158 | + else: | |
| 159 | + print("%s : pas de domaine → rien à exposer" % app) | |
| 160 | + elif a.sub == "rm": | |
| 161 | + for app in a.args: | |
| 162 | + tunnel.remove_route(manifest.load(app)) | |
| 163 | + except tunnel.TunnelError as e: | |
| 164 | + raise SystemExit("ÉCHEC : %s" % e) | |
| 165 | + | |
| 166 | + | |
| 167 | +def cmd_heal(a): | |
| 168 | + from . import heal | |
| 169 | + if a.install: | |
| 170 | + heal.install(interval=a.interval) | |
| 171 | + return | |
| 172 | + s = heal.heal(apps_filter=a.apps or None, nodes_filter=a.nodes or None, dry_run=a.dry_run, quiet=a.quiet, public=not a.no_public) | |
| 173 | + sys.exit(0 if not s["ko"] else 1) | |
| 174 | + | |
| 175 | + | |
| 176 | +def cmd_harden(a): | |
| 177 | + from . import harden | |
| 178 | + autologin = True if a.autologin else (False if a.no_autologin else None) | |
| 179 | + if a.all or not a.nodes: | |
| 180 | + harden.harden_all(aliases=a.nodes or None, autologin=autologin) | |
| 181 | + else: | |
| 182 | + for n in a.nodes: | |
| 183 | + harden.harden(n, autologin=autologin) | |
| 132 | 184 | |
| 133 | 185 | |
| 134 | 186 | def cmd_status(a): |
@@ -186,7 +238,7 @@ def cmd_import(a): | ||
| 186 | 238 | |
| 187 | 239 | def cmd_prepare(a): |
| 188 | 240 | from . import prepare |
| 189 | − prepare.prepare(a.node, runtimes=a.runtime or None, ka_helpers=a.ka_helpers) | |
| 241 | + prepare.prepare(a.node, runtimes=a.runtime or None, ka_helpers=a.ka_helpers, tunnel_peer=not a.no_tunnel, harden=not a.no_harden) | |
| 190 | 242 | |
| 191 | 243 | |
| 192 | 244 | def cmd_bootstrap(a): |
@@ -226,8 +278,11 @@ def main(argv=None): | ||
| 226 | 278 | s = sp.add_parser("registry", help="registre JSON complet (--push : rediffuse aux abonnés)"); s.add_argument("--push", action="store_true"); s.set_defaults(f=cmd_registry) |
| 227 | 279 | s = sp.add_parser("subscribers", help="abonnés du registre (copie poussée à chaque sauvegarde)"); s.add_argument("--add", nargs=3, metavar=("app|node", "NOM", "CHEMIN")); s.add_argument("--remove", nargs=2, metavar=("app|node", "NOM")); s.set_defaults(f=cmd_subscribers) |
| 228 | 280 | s = sp.add_parser("import", help="brouillons de manifestes depuis les PM2 d'un nœud"); s.add_argument("node"); s.add_argument("apps", nargs="*"); s.set_defaults(f=cmd_import) |
| 229 | − s = sp.add_parser("prepare", help="installe runtimes/ngrok/pm2-startup sur un nœud"); s.add_argument("node"); s.add_argument("--runtime", action="append"); s.add_argument("--ka-helpers", action="store_true"); s.set_defaults(f=cmd_prepare) | |
| 281 | + s = sp.add_parser("prepare", help="runtimes Homebrew + pm2 startup + raccordement tunnel + harden sur un nœud"); s.add_argument("node"); s.add_argument("--runtime", action="append"); s.add_argument("--ka-helpers", action="store_true"); s.add_argument("--no-tunnel", action="store_true"); s.add_argument("--no-harden", action="store_true"); s.set_defaults(f=cmd_prepare) | |
| 230 | 282 | s = sp.add_parser("bootstrap", help="Xcode CLT + Homebrew sur un nœud vierge (long)"); s.add_argument("node"); s.set_defaults(f=cmd_bootstrap) |
| 283 | + s = sp.add_parser("tunnel", help="MacLustr Tunnel : status | peer <nœud>… | route <app>… | rm <app>…"); s.add_argument("sub", choices=["status", "peer", "route", "rm"]); s.add_argument("args", nargs="*"); s.add_argument("-g", "--gateway", help="BHS64 (défaut) ou R9128"); s.set_defaults(f=cmd_tunnel) | |
| 284 | + s = sp.add_parser("heal", help="auto-réparation : PM2/launchd/wg1/routes de chaque app du registre (--install = LaunchDaemon 5 min)"); s.add_argument("--apps", nargs="*"); s.add_argument("--nodes", nargs="*"); s.add_argument("--dry-run", action="store_true"); s.add_argument("--quiet", action="store_true"); s.add_argument("--no-public", action="store_true"); s.add_argument("--install", action="store_true"); s.add_argument("--interval", type=int, help="secondes entre deux passages (avec --install)"); s.set_defaults(f=cmd_heal) | |
| 285 | + s = sp.add_parser("harden", help="résilience d'un nœud : auto-login, pas de veille, redémarrage après coupure/gel, MAJ macOS + Tailscale off"); s.add_argument("nodes", nargs="*"); s.add_argument("--all", action="store_true"); s.add_argument("--autologin", action="store_true", help="force l'auto-login même sur un nœud loué"); s.add_argument("--no-autologin", action="store_true"); s.set_defaults(f=cmd_harden) | |
| 231 | 286 | a = p.parse_args(argv) |
| 232 | 287 | a.f(a) |
| 233 | 288 | |
modified
mld/config.py
+13 −3
@@ -17,9 +17,18 @@ LOG_DIR = os.path.join(STATE, "logs") | ||
| 17 | 17 | REGISTRY = os.path.join(STATE, "registry.json") |
| 18 | 18 | LAN_CACHE = os.path.join(STATE, "lan.json") |
| 19 | 19 | SCAN_CACHE = os.path.join(STATE, "scan.json") |
| 20 | −NGROK_CFG = os.path.join(HOME, "Library", "Application Support", "ngrok", "ngrok.yml") | |
| 21 | 20 | RUN_DIR = "~/dispatch-run" # sur les nœuds : ecosystem PM2 générés |
| 22 | 21 | SUDO_PW_FILE = os.path.join(STATE, ".sudo") # mot de passe sudo des nœuds (0600), pour `prepare` |
| 22 | +HEAL_STATE = os.path.join(STATE, "heal-state.json") # mémoire de `mld heal` (actions récentes, nœuds en panne) | |
| 23 | + | |
| 24 | +# MacLustr Tunnel (remplace ngrok depuis le 2026-09-10) : passerelles OVH = hub WireGuard + Caddy (TLS auto), pilotées | |
| 25 | +# par `tunnelctl` (sudo sans mot de passe pour ubuntu). La clé de la passerelle M1M32 est autorisée pour `ubuntu`. | |
| 26 | +# Chaque Mac porte une interface par passerelle (wg1 → BHS64, wg0 → R9128), LaunchDaemon io.maclustr.wireguard-<iface>. | |
| 27 | +TUNNEL_GATEWAYS = { | |
| 28 | + "BHS64": {"host": "51.161.112.61", "user": "ubuntu", "subnet": "10.67.0", "iface": "wg1", "endpoint": "51.161.112.61:51820", "place": "OVH Beauharnois (Québec)"}, | |
| 29 | + "R9128": {"host": "51.255.75.61", "user": "ubuntu", "subnet": "10.66.0", "iface": "wg0", "endpoint": "51.255.75.61:51820", "place": "OVH Gravelines (France)"}, | |
| 30 | +} | |
| 31 | +TUNNEL_DEFAULT = "BHS64" | |
| 23 | 32 | |
| 24 | 33 | # Inventaire statique (alias -> cœurs, RAM Go, modèle). Les IP LAN sont découvertes en live. |
| 25 | 34 | NODES = { |
@@ -125,7 +134,8 @@ RUNTIME_CHECKS = { | ||
| 125 | 134 | "node": "test -x /opt/homebrew/bin/node", |
| 126 | 135 | "pnpm": "test -x /opt/homebrew/bin/pnpm", |
| 127 | 136 | "pm2": "test -x /opt/homebrew/bin/pm2", |
| 128 | − "ngrok": "(test -x /opt/homebrew/bin/ngrok || test -x $HOME/bin/ngrok) && test -f \"$HOME/Library/Application Support/ngrok/ngrok.yml\"", | |
| 137 | + # raccordé au MacLustr Tunnel (wg1 → BHS64 monté) : informatif, `mld deploy` raccorde lui-même un nœud qui ne l'est pas | |
| 138 | + "tunnel": "test -f /Library/LaunchDaemons/io.maclustr.wireguard-wg1.plist && ifconfig 2>/dev/null | grep -q 'inet 10\\.67\\.0\\.'", | |
| 129 | 139 | "python@3.14": "test -x /opt/homebrew/opt/python@3.14/bin/python3.14", |
| 130 | 140 | "python@3.13": "test -x /opt/homebrew/opt/python@3.13/bin/python3.13", |
| 131 | 141 | "python@3.12": "test -x /opt/homebrew/opt/python@3.12/bin/python3.12", |
@@ -141,7 +151,7 @@ BREW_FORMULAE = { | ||
| 141 | 151 | "node": "node", "pnpm": "pnpm", "pm2": "pm2", "python@3.14": "python@3.14", "python@3.13": "python@3.13", |
| 142 | 152 | "python@3.12": "python@3.12", "uv": "uv", "redis": "redis", "docker": "docker", "postgresql@17": "postgresql@17", "git": "git", |
| 143 | 153 | } |
| 144 | −BREW_CASKS = {"ngrok": "ngrok"} | |
| 154 | +BREW_CASKS = {} # ngrok (cask) retiré le 2026-09-10 : l'exposition publique passe par TUNNEL_GATEWAYS | |
| 145 | 155 | |
| 146 | 156 | os.makedirs(APPS_DIR, exist_ok=True) |
| 147 | 157 | os.makedirs(STAGE_DIR, exist_ok=True) |
modified
mld/deploy.py
+75 −30
@@ -1,9 +1,12 @@ | ||
| 1 | −"""Cycle de vie d'une app : fetch (nœud -> staging), deploy (staging -> nœud), stop, start, retire, move, health.""" | |
| 1 | +"""Cycle de vie d'une app : fetch (nœud -> staging), deploy (staging -> nœud), stop, start, retire, move, health. | |
| 2 | + | |
| 3 | +Exposition publique = MacLustr Tunnel (module tunnel) : au deploy, la route Caddy https://<domaine> de la passerelle est | |
| 4 | +pointée vers le nœud choisi (après le healthcheck local), donc elle suit automatiquement les `mld move`.""" | |
| 2 | 5 | import json |
| 3 | 6 | import os |
| 4 | 7 | import shlex |
| 5 | 8 | import time |
| 6 | −from . import config, manifest, nodes, registry, render, ssh | |
| 9 | +from . import config, manifest, nodes, registry, render, ssh, tunnel | |
| 7 | 10 | |
| 8 | 11 | SYNC_BASE_EXCLUDES = [".DS_Store", "*.log", "__pycache__/", ".pytest_cache/", ".mypy_cache/"] |
| 9 | 12 | |
@@ -89,21 +92,17 @@ def stop(app, alias, keep_dir=True): | ||
| 89 | 92 | m = manifest.load(app) |
| 90 | 93 | ip = _ip(alias) |
| 91 | 94 | _log(app, "stop sur %s" % alias) |
| 92 | − names = manifest.pm2_names(m) | |
| 95 | + names = manifest.pm2_names(m) + manifest.legacy_pm2_names(m) | |
| 93 | 96 | script = "" |
| 94 | 97 | if names: |
| 95 | 98 | script += "for n in %s; do pm2 delete $n >/dev/null 2>&1 && echo \" pm2 delete $n\"; done; pm2 save --force >/dev/null 2>&1\n" % " ".join(names) |
| 96 | 99 | for lbl in manifest.launchd_labels(m): |
| 97 | 100 | script += "launchctl bootout gui/$(id -u)/%s >/dev/null 2>&1 && echo ' bootout %s'; rm -f ~/Library/LaunchAgents/%s.plist\n" % (lbl, lbl, lbl) |
| 98 | − if m.get("ngrok"): | |
| 99 | − script += "pkill -f -- 'url=%s' 2>/dev/null; pkill -f -- 'url=https://%s' 2>/dev/null; pkill -f -- 'domain=%s' 2>/dev/null; true\n" % (m["ngrok"]["url"], m["ngrok"]["url"], m["ngrok"]["url"]) | |
| 100 | 101 | for hook in (m.get("hooks") or {}).get("pre_stop") or []: |
| 101 | 102 | script += hook + "\n" |
| 102 | 103 | # tue ce qui écoute encore sur les ports de l'app (orphelins) |
| 103 | 104 | for p in m["requires"].get("ports") or []: |
| 104 | 105 | script += "sleep 1; for pid in $(lsof -nP -iTCP:%s -sTCP:LISTEN -t 2>/dev/null); do kill $pid 2>/dev/null && echo ' kill port %s pid '$pid; done\n" % (p, p) |
| 105 | − if m.get("ngrok"): | |
| 106 | − script += "sleep 4\n" # laisse ngrok libérer le domaine réservé côté service | |
| 107 | 106 | rc, out, err = ssh.run(ip, script, timeout=180) |
| 108 | 107 | for l in out.splitlines(): |
| 109 | 108 | _log(app, l) |
@@ -177,12 +176,16 @@ def deploy(app, alias, reinstall=False, skip_sync=False): | ||
| 177 | 176 | raise DeployError("post-sync a échoué (%s): %s" % (rc, (err or out).strip()[-400:])) |
| 178 | 177 | # 4. lancement |
| 179 | 178 | start(app, alias, m=m, ctx=ctx, ip=ip) |
| 180 | − # 5. santé | |
| 181 | − healthy, detail = health(app, alias, ip=ip, m=m) | |
| 179 | + # 5. santé locale, puis route publique (MacLustr Tunnel) pointée vers CE nœud, puis santé publique | |
| 180 | + local_ok, code = _health_local(ip, m) | |
| 181 | + tun = None | |
| 182 | + if local_ok and tunnel.spec(m, alias): | |
| 183 | + tun = _publish(app, alias, m, ip) | |
| 184 | + healthy, detail = health(app, alias, ip=ip, m=m, timeout=30, public=bool(tun)) | |
| 182 | 185 | registry.set_app(app, node=alias, ip=ip, port=m.get("port"), domain=m.get("domain"), dir=m["dir"], label=m.get("label") or app, |
| 183 | 186 | health_path=m.get("health_path") or "/", |
| 184 | 187 | processes=manifest.pm2_names(m), launchd=manifest.launchd_labels(m), deployed=time.strftime("%Y-%m-%d %H:%M"), |
| 185 | − status="online" if healthy else "unhealthy", health=detail) | |
| 188 | + tunnel=tun, status="online" if healthy else "unhealthy", health=detail) | |
| 186 | 189 | registry.log(app, "deployed", node=alias, healthy=healthy) |
| 187 | 190 | _log(app, " santé : %s" % detail) |
| 188 | 191 | if not healthy: |
@@ -205,8 +208,9 @@ def start(app, alias, m=None, ctx=None, ip=None): | ||
| 205 | 208 | ssh.write_remote_file(ip, eco_path, eco, mode="600") |
| 206 | 209 | script = "mkdir -p %s\n" % run_dir |
| 207 | 210 | names = manifest.pm2_names(m) |
| 211 | + if names or manifest.legacy_pm2_names(m): | |
| 212 | + script += "for n in %s; do pm2 delete $n >/dev/null 2>&1; done\n" % " ".join(names + manifest.legacy_pm2_names(m)) | |
| 208 | 213 | if names: |
| 209 | − script += "for n in %s; do pm2 delete $n >/dev/null 2>&1; done\n" % " ".join(names) | |
| 210 | 214 | script += "pm2 start %s --update-env 2>&1 | grep -E 'error|Error|\\[PM2\\]' | head -5; pm2 save --force >/dev/null 2>&1; pm2 jlist | python3 -c \"import json,sys; L=json.load(sys.stdin); print(' pm2: '+', '.join(p['name']+'='+p['pm2_env']['status'] for p in L if p['name'] in %s))\" 2>/dev/null || pm2 ls | grep -E '%s'\n" % (shlex.quote(eco_path), json.dumps(names), "|".join(names)) |
| 211 | 215 | for item in m.get("launchd") or []: |
| 212 | 216 | pl = render.plist(item, ctx) |
@@ -223,11 +227,29 @@ def start(app, alias, m=None, ctx=None, ip=None): | ||
| 223 | 227 | return rc == 0 |
| 224 | 228 | |
| 225 | 229 | |
| 226 | −def health(app, alias, ip=None, m=None, timeout=90): | |
| 227 | − m = m or manifest.load(app) | |
| 228 | − ip = ip or _ip(alias) | |
| 230 | +def _publish(app, alias, m, ip): | |
| 231 | + """Raccorde le nœud au tunnel si besoin et pointe la route publique de l'app dessus. Retourne le spec de route ou None.""" | |
| 232 | + log = lambda s: _log(app, s) | |
| 233 | + try: | |
| 234 | + st = None | |
| 235 | + try: | |
| 236 | + st = tunnel.state((m.get("tunnel") or {}).get("gateway")) | |
| 237 | + except tunnel.TunnelError: | |
| 238 | + pass | |
| 239 | + if not tunnel.ensure_peer(alias, (m.get("tunnel") or {}).get("gateway"), log=log, st=st): | |
| 240 | + _log(app, " tunnel : %s n'a pas de handshake avec la passerelle — route pointée quand même" % alias) | |
| 241 | + s = tunnel.ensure_route(m, alias, log=log) | |
| 242 | + if not s: | |
| 243 | + return None | |
| 244 | + return {"gateway": s["gateway"], "domain": s["domain"], "upstream": s["upstreams"][0], "url": "https://%s" % s["domain"]} | |
| 245 | + except tunnel.TunnelError as e: | |
| 246 | + _log(app, " tunnel : %s" % e) | |
| 247 | + return None | |
| 248 | + | |
| 249 | + | |
| 250 | +def _health_local(ip, m, timeout=90): | |
| 251 | + """(ok, code|texte) : HTTP local sur le port de l'app, ou processus PM2 online si l'app n'a pas de port.""" | |
| 229 | 252 | if not m.get("port"): |
| 230 | − # pas de port HTTP : vérifie que les processus PM2 sont online | |
| 231 | 253 | names = manifest.pm2_names(m) |
| 232 | 254 | if not names: |
| 233 | 255 | return True, "aucun port/processus à vérifier" |
@@ -237,22 +259,40 @@ def health(app, alias, ip=None, m=None, timeout=90): | ||
| 237 | 259 | url = "http://127.0.0.1:%s%s" % (m["port"], m.get("health_path") or "/") |
| 238 | 260 | deadline = time.time() + timeout |
| 239 | 261 | code = "000" |
| 240 | − while time.time() < deadline: | |
| 262 | + while True: | |
| 241 | 263 | rc, out, _ = ssh.run(ip, "curl -s -o /dev/null -m 8 -w '%%{http_code}' %s" % shlex.quote(url), timeout=20) |
| 242 | 264 | code = out.strip()[-3:] if out.strip() else "000" |
| 243 | 265 | if code.isdigit() and code not in ("000",) and int(code) < 500: |
| 244 | 266 | break |
| 267 | + if time.time() >= deadline: | |
| 268 | + break | |
| 245 | 269 | time.sleep(4) |
| 246 | − local_ok = code.isdigit() and code != "000" and int(code) < 500 | |
| 270 | + return code.isdigit() and code != "000" and int(code) < 500, code | |
| 271 | + | |
| 272 | + | |
| 273 | +def _health_public(m, tries=20): | |
| 274 | + """Code HTTP de https://<domaine><health_path> à travers la passerelle (tries × 5 s max).""" | |
| 275 | + import subprocess | |
| 276 | + pub = "" | |
| 277 | + for _ in range(tries): | |
| 278 | + p = subprocess.run(["curl", "-s", "-o", "/dev/null", "-m", "12", "-w", "%{http_code}", "https://%s%s" % (m["domain"], m.get("health_path") or "/")], capture_output=True, text=True) | |
| 279 | + pub = p.stdout.strip() | |
| 280 | + if pub.isdigit() and int(pub) < 500 and pub != "404": | |
| 281 | + break | |
| 282 | + time.sleep(5) | |
| 283 | + return pub | |
| 284 | + | |
| 285 | + | |
| 286 | +def health(app, alias, ip=None, m=None, timeout=90, public=True): | |
| 287 | + """Santé locale (port/PM2) puis publique (via MacLustr Tunnel) si l'app a un domaine. Retourne (ok, détail).""" | |
| 288 | + m = m or manifest.load(app) | |
| 289 | + ip = ip or _ip(alias) | |
| 290 | + local_ok, code = _health_local(ip, m, timeout=timeout) | |
| 291 | + if not m.get("port"): | |
| 292 | + return local_ok, code | |
| 247 | 293 | pub = "" |
| 248 | − if m.get("domain") and local_ok and m.get("ngrok"): | |
| 249 | − import subprocess | |
| 250 | − for _ in range(20): | |
| 251 | − p = subprocess.run(["curl", "-s", "-o", "/dev/null", "-m", "12", "-w", "%{http_code}", "https://%s%s" % (m["domain"], m.get("health_path") or "/")], capture_output=True, text=True) | |
| 252 | − pub = p.stdout.strip() | |
| 253 | − if pub.isdigit() and int(pub) < 500 and pub != "404": | |
| 254 | − break | |
| 255 | − time.sleep(5) | |
| 294 | + if public and m.get("domain") and local_ok and tunnel.spec(m, alias): | |
| 295 | + pub = _health_public(m, tries=max(2, min(20, timeout // 5))) | |
| 256 | 296 | detail = "local %s" % code + (" / public https://%s %s" % (m["domain"], pub) if pub else "") |
| 257 | 297 | ok = local_ok and (not pub or (pub.isdigit() and int(pub) < 500 and pub != "404")) |
| 258 | 298 | return ok, detail |
@@ -296,8 +336,11 @@ def move(app, to_alias, from_alias=None, keep_source=False, reinstall=False): | ||
| 296 | 336 | except Exception: |
| 297 | 337 | pass |
| 298 | 338 | start(app, from_alias) |
| 299 | − ok, det = health(app, from_alias) | |
| 300 | − registry.set_app(app, node=from_alias, status="online" if ok else "unhealthy", health=det) | |
| 339 | + m = manifest.load(app) | |
| 340 | + ip0 = _ip(from_alias) | |
| 341 | + tun = _publish(app, from_alias, m, ip0) if tunnel.spec(m, from_alias) else None # la route publique revient sur la source | |
| 342 | + ok, det = health(app, from_alias, ip=ip0, m=m) | |
| 343 | + registry.set_app(app, node=from_alias, ip=ip0, tunnel=tun, status="online" if ok else "unhealthy", health=det) | |
| 301 | 344 | raise |
| 302 | 345 | if not keep_source: |
| 303 | 346 | retire(app, from_alias) # 5. nettoyage source |
@@ -320,10 +363,11 @@ def status(app=None): | ||
| 320 | 363 | |
| 321 | 364 | |
| 322 | 365 | def live_status(app=None, public=True): |
| 323 | − """Vérifie en live (PM2 + port + domaine) chaque app du registre.""" | |
| 366 | + """Vérifie en live (PM2 + port + domaine via le tunnel) chaque app du registre. Une seule sauvegarde du registre à la fin.""" | |
| 324 | 367 | r = registry.load() |
| 325 | 368 | apps = [app] if app else sorted(r["apps"]) |
| 326 | 369 | out = [] |
| 370 | + changes = {} | |
| 327 | 371 | for a in apps: |
| 328 | 372 | v = r["apps"].get(a) |
| 329 | 373 | if not v: |
@@ -331,9 +375,10 @@ def live_status(app=None, public=True): | ||
| 331 | 375 | m = manifest.load(a) |
| 332 | 376 | try: |
| 333 | 377 | ip = _ip(v["node"]) |
| 334 | − ok, det = health(a, v["node"], ip=ip, m=m, timeout=10) if public else (None, "") | |
| 378 | + ok, det = health(a, v["node"], ip=ip, m=m, timeout=10, public=public) | |
| 335 | 379 | except DeployError as e: |
| 336 | 380 | ok, det = False, str(e) |
| 337 | − registry.set_app(a, status="online" if ok else "unhealthy", health=det) | |
| 381 | + changes[a] = {"status": "online" if ok else "unhealthy", "health": det} | |
| 338 | 382 | out.append((a, v["node"], v.get("domain") or "", "OK" if ok else "KO", det)) |
| 383 | + registry.update_many(changes) | |
| 339 | 384 | return out |
added
mld/harden.py
+124 −0
@@ -0,0 +1,124 @@ | ||
| 1 | +"""`mld harden <nœud>|--all` : un Mac du cluster doit revenir TOUT SEUL après une coupure de courant ou une mise à jour. | |
| 2 | + | |
| 3 | +Ce que ça règle (idempotent, sudo via ~/dispatch/.sudo*) : | |
| 4 | + 1. alimentation : redémarrage automatique après coupure (autorestart) et après gel (restartfreeze), jamais de veille | |
| 5 | + (sleep/disksleep 0), réveil réseau (womp) ; | |
| 6 | + 2. mises à jour macOS + App Store : aucune installation automatique (un redémarrage surprise couperait les apps) ; | |
| 7 | + 3. Tailscale : `tailscale set --auto-update=false` + Sparkle désactivé — une MAJ relance tailscaled et le nœud disparaît | |
| 8 | + de *.maclustr.io (les noms résolvent vers l'IP Tailscale) même si le LAN et le tunnel vont bien ; | |
| 9 | + 4. ouverture de session automatique (kcpassword + autoLoginUser) : sans session graphique, les LaunchAgents (pm2 startup, | |
| 10 | + gardiens Ka, pousseur, admin-ka) ne démarrent jamais — c'est ce qui a laissé les apps mortes le 2026-09-10. | |
| 11 | + Impossible si FileVault est actif (le disque reste chiffré jusqu'à la saisie du mot de passe) : signalé ; | |
| 12 | + 5. vérifie que pm2 startup (LaunchAgent ou LaunchDaemon) et le LaunchDaemon WireGuard wg1 sont en place. | |
| 13 | +Les nœuds loués (REMOTE_NODES) reçoivent 1, 2, 5 mais pas l'auto-login (session du fournisseur) sauf --autologin. | |
| 14 | +""" | |
| 15 | +import os | |
| 16 | +import shlex | |
| 17 | +from concurrent.futures import ThreadPoolExecutor | |
| 18 | +from . import config, nodes, ssh | |
| 19 | + | |
| 20 | +KC_KEY = [0x7D, 0x89, 0x52, 0x23, 0xD2, 0xBC, 0xDD, 0xEA, 0xA3, 0xB9, 0x1F] | |
| 21 | + | |
| 22 | + | |
| 23 | +def kcpassword(pw): | |
| 24 | + """Encodage /etc/kcpassword (XOR avec la clé Apple, complété à un multiple de 12 octets).""" | |
| 25 | + b = list(pw.encode("utf-8")) + [0] | |
| 26 | + while len(b) % 12: | |
| 27 | + b.append(0) | |
| 28 | + return bytes(x ^ KC_KEY[i % len(KC_KEY)] for i, x in enumerate(b)) | |
| 29 | + | |
| 30 | + | |
| 31 | +def _sudo_pw(alias): | |
| 32 | + pwf = config.sudo_pw_file(alias) | |
| 33 | + return open(pwf).read().strip() if os.path.exists(pwf) else None | |
| 34 | + | |
| 35 | + | |
| 36 | +def script_for(alias, autologin): | |
| 37 | + pw = _sudo_pw(alias) | |
| 38 | + if not pw: | |
| 39 | + return None | |
| 40 | + user = config.user_of(alias) | |
| 41 | + kc = "".join("\\x%02x" % x for x in kcpassword(pw)) # même mot de passe pour sudo et l'ouverture de session | |
| 42 | + s = r''' | |
| 43 | +S(){ echo %(pw)s | sudo -S -p '' "$@"; } | |
| 44 | +echo "== alimentation" | |
| 45 | +S pmset -a sleep 0 disksleep 0 womp 1 >/dev/null 2>&1 | |
| 46 | +S pmset -a autorestart 1 >/dev/null 2>&1 || echo " autorestart non supporté (portable)" | |
| 47 | +S systemsetup -setrestartfreeze on >/dev/null 2>&1 | |
| 48 | +S systemsetup -setrestartpowerfailure on >/dev/null 2>&1 | |
| 49 | +echo " $(pmset -g | grep -E '^ *(autorestart|sleep|disksleep|womp) ' | awk '{print $1"="$2}' | tr '\n' ' ')" | |
| 50 | +echo "== mises à jour macOS / App Store : désactivées" | |
| 51 | +for k in AutomaticCheckEnabled AutomaticDownload AutomaticallyInstallMacOSUpdates ConfigDataInstall CriticalUpdateInstall; do S defaults write /Library/Preferences/com.apple.SoftwareUpdate $k -bool false; done | |
| 52 | +S defaults write /Library/Preferences/com.apple.commerce AutoUpdate -bool false | |
| 53 | +S defaults write /Library/Preferences/com.apple.commerce AutoUpdateRestartRequired -bool false | |
| 54 | +echo " SoftwareUpdate: $(defaults read /Library/Preferences/com.apple.SoftwareUpdate 2>/dev/null | grep -E 'Automatic|Install' | tr -d ' ;' | tr '\n' ' ')" | |
| 55 | +T=/Applications/Tailscale.app/Contents/MacOS/Tailscale | |
| 56 | +if [ -x "$T" ]; then | |
| 57 | + echo "== Tailscale $($T version 2>/dev/null | head -1) : pas de mise à jour automatique" | |
| 58 | + "$T" set --auto-update=false >/dev/null 2>&1 && echo " auto-update off" || echo " ! tailscale set a échoué (session ?)" | |
| 59 | + defaults write io.tailscale.ipn.macsys SUEnableAutomaticChecks -bool false 2>/dev/null | |
| 60 | + defaults write io.tailscale.ipn.macsys SUAutomaticallyUpdate -bool false 2>/dev/null | |
| 61 | + defaults write io.tailscale.ipn.macos SUEnableAutomaticChecks -bool false 2>/dev/null | |
| 62 | +else | |
| 63 | + echo "== Tailscale : absent" | |
| 64 | +fi | |
| 65 | +echo "== ouverture de session automatique (%(user)s)" | |
| 66 | +if fdesetup status 2>/dev/null | grep -q 'On'; then | |
| 67 | + echo " ! FILEVAULT ACTIF : auto-login impossible, le Mac attend un mot de passe au démarrage (fdesetup disable pour lever ça)" | |
| 68 | +elif [ "%(autologin)s" = "1" ]; then | |
| 69 | + printf '%(kc)s' > /tmp/.kc.$$ && S install -m 600 -o root -g wheel /tmp/.kc.$$ /etc/kcpassword; rm -f /tmp/.kc.$$ | |
| 70 | + S defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser -string %(user)s | |
| 71 | + echo " autoLoginUser=$(defaults read /Library/Preferences/com.apple.loginwindow autoLoginUser 2>/dev/null) kcpassword=$(S stat -f %%z /etc/kcpassword 2>/dev/null) octets" | |
| 72 | +else | |
| 73 | + echo " non touché (nœud loué : session du fournisseur) — autoLoginUser=$(defaults read /Library/Preferences/com.apple.loginwindow autoLoginUser 2>/dev/null || echo aucun)" | |
| 74 | +fi | |
| 75 | +echo "== démarrage des services" | |
| 76 | +P=$(ls ~/Library/LaunchAgents/pm2.*.plist /Library/LaunchDaemons/pm2.*.plist 2>/dev/null | head -1) | |
| 77 | +[ -n "$P" ] && echo " pm2 startup : $P" || echo " ! pm2 startup ABSENT → mld prepare %(alias)s" | |
| 78 | +if [ -f /Library/LaunchDaemons/io.maclustr.wireguard-wg1.plist ]; then | |
| 79 | + ifconfig 2>/dev/null | grep -q 'inet 10\.67\.0\.' && echo " wg1 (tunnel BHS64) : monté" || { echo " wg1 : plist présent mais interface absente → relance"; S launchctl kickstart -k system/io.maclustr.wireguard-wg1; } | |
| 80 | +else | |
| 81 | + echo " wg1 : non raccordé au MacLustr Tunnel (mld tunnel peer %(alias)s si le nœud doit exposer un site)" | |
| 82 | +fi | |
| 83 | +echo " session console : $(who | awk '/console/{print $1" depuis "$3" "$4}' | head -1)" | |
| 84 | +''' % {"pw": shlex.quote(pw), "user": user, "kc": kc, "autologin": "1" if autologin else "0", "alias": alias} | |
| 85 | + return s | |
| 86 | + | |
| 87 | + | |
| 88 | +def harden(alias, autologin=None, verbose=True): | |
| 89 | + ip = nodes.ip_of(alias, rediscover=False) or nodes.ip_of(alias) | |
| 90 | + if not ip: | |
| 91 | + return alias, False, "injoignable" | |
| 92 | + if autologin is None: | |
| 93 | + autologin = alias not in config.REMOTE_NODES | |
| 94 | + s = script_for(alias, autologin) | |
| 95 | + if not s: | |
| 96 | + return alias, False, "mot de passe sudo absent (%s)" % config.sudo_pw_file(alias) | |
| 97 | + rc, out, err = ssh.run(ip, s, timeout=240) | |
| 98 | + if verbose: | |
| 99 | + print("### %s (%s)" % (alias, ip)) | |
| 100 | + print(out.rstrip()) | |
| 101 | + if err.strip(): | |
| 102 | + print(" stderr: " + err.strip()[-200:]) | |
| 103 | + flags = [] | |
| 104 | + if "FILEVAULT" in out: | |
| 105 | + flags.append("FileVault") | |
| 106 | + if "pm2 startup ABSENT" in out: | |
| 107 | + flags.append("pm2-startup") | |
| 108 | + if "tailscale set a échoué" in out: | |
| 109 | + flags.append("tailscale") | |
| 110 | + return alias, rc == 0, ", ".join(flags) if flags else "ok" | |
| 111 | + | |
| 112 | + | |
| 113 | +def harden_all(aliases=None, autologin=None): | |
| 114 | + lan = nodes.load_lan()["ips"] or nodes.discover(verbose=False) | |
| 115 | + targets = [a for a in config.NODES if a in lan and (not aliases or a in aliases)] | |
| 116 | + with ThreadPoolExecutor(max_workers=12) as ex: | |
| 117 | + results = list(ex.map(lambda a: harden(a, autologin=autologin, verbose=False), targets)) | |
| 118 | + print("%-8s %-6s %s" % ("nœud", "état", "à surveiller")) | |
| 119 | + for a, ok, note in results: | |
| 120 | + print("%-8s %-6s %s" % (a, "ok" if ok else "ÉCHEC", note)) | |
| 121 | + missing = [a for a in config.NODES if a not in lan] | |
| 122 | + if missing: | |
| 123 | + print("non joignables (éteints ?) : %s" % ", ".join(missing)) | |
| 124 | + return results | |
added
mld/heal.py
+295 −0
@@ -0,0 +1,295 @@ | ||
| 1 | +"""`mld heal` : auto-réparation du cluster depuis la passerelle (toutes les 5 min via le LaunchDaemon io.maclustr.mld-heal). | |
| 2 | + | |
| 3 | +Pour chaque nœud qui héberge des apps du registre, UN seul SSH ramène : uptime, démon PM2, statut de chaque processus, | |
| 4 | +LaunchAgents chargés, interface WireGuard wg1, code HTTP local de chaque app. Puis, seulement si quelque chose manque : | |
| 5 | + - démon PM2 mort ou processus absents → `pm2 resurrect` puis `deploy.start(app)` (régénère l'ecosystem, idempotent) | |
| 6 | + - processus errored/stopped (hors cron) → `pm2 restart` | |
| 7 | + - LaunchAgent absent → re-bootstrap du plist (ou start complet s'il n'existe plus) | |
| 8 | + - wg1 tombé → kickstart du LaunchDaemon (sinon raccordement complet) | |
| 9 | + - route Caddy absente / mauvais nœud → `tunnelctl add` (via tunnel.ensure_route) | |
| 10 | +Garde-fous : nœud redémarré depuis < 3 min ignoré (pm2 resurrect fait son travail), au plus 6 actions par app et par heure, | |
| 11 | +registre sauvegardé UNE fois (diffusion aux abonnés) et seulement si un état change. Journal : ~/dispatch/logs/heal.log. | |
| 12 | +""" | |
| 13 | +import json | |
| 14 | +import os | |
| 15 | +import plistlib | |
| 16 | +import shlex | |
| 17 | +import time | |
| 18 | +from . import config, deploy, manifest, nodes, registry, ssh, tunnel | |
| 19 | + | |
| 20 | +GRACE_S = 180 | |
| 21 | +MAX_ACTIONS_PER_HOUR = 6 | |
| 22 | +HEAL_LABEL = "io.maclustr.mld-heal" | |
| 23 | +HEAL_INTERVAL_S = 300 | |
| 24 | + | |
| 25 | + | |
| 26 | +def _now(): | |
| 27 | + return time.strftime("%Y-%m-%d %H:%M:%S") | |
| 28 | + | |
| 29 | + | |
| 30 | +def _log(msg, quiet=False): | |
| 31 | + line = "%s %s" % (_now(), msg) | |
| 32 | + if not quiet: | |
| 33 | + print(msg, flush=True) | |
| 34 | + with open(os.path.join(config.LOG_DIR, "heal.log"), "a") as f: | |
| 35 | + f.write(line + "\n") | |
| 36 | + | |
| 37 | + | |
| 38 | +def _state(): | |
| 39 | + if os.path.exists(config.HEAL_STATE): | |
| 40 | + try: | |
| 41 | + return json.load(open(config.HEAL_STATE)) | |
| 42 | + except Exception: | |
| 43 | + pass | |
| 44 | + return {"actions": {}, "down": {}} | |
| 45 | + | |
| 46 | + | |
| 47 | +def _save_state(st): | |
| 48 | + tmp = config.HEAL_STATE + ".tmp" | |
| 49 | + json.dump(st, open(tmp, "w"), indent=1) | |
| 50 | + os.replace(tmp, config.HEAL_STATE) | |
| 51 | + | |
| 52 | + | |
| 53 | +def _allowed(st, app): | |
| 54 | + cut = time.time() - 3600 | |
| 55 | + acts = [t for t in st["actions"].get(app, []) if t > cut] | |
| 56 | + st["actions"][app] = acts | |
| 57 | + return len(acts) < MAX_ACTIONS_PER_HOUR | |
| 58 | + | |
| 59 | + | |
| 60 | +def _record(st, app): | |
| 61 | + st["actions"].setdefault(app, []).append(time.time()) | |
| 62 | + | |
| 63 | + | |
| 64 | +def _node_report(ip, apps): | |
| 65 | + """Un SSH : BOOT/NOW, PM2 up|down, PROC nom statut restarts, LA label, WG1 up|down, HUB ok|ko, HTTP app code.""" | |
| 66 | + script = r''' | |
| 67 | +echo "BOOT $(sysctl -n kern.boottime | sed -E 's/.*sec = ([0-9]+).*/\1/')" | |
| 68 | +echo "NOW $(date +%s)" | |
| 69 | +if pm2 ping >/dev/null 2>&1; then echo "PM2 up"; else echo "PM2 down"; fi | |
| 70 | +pm2 jlist 2>/dev/null | python3 -c " | |
| 71 | +import json,sys | |
| 72 | +try: L=json.load(sys.stdin) | |
| 73 | +except Exception: L=[] | |
| 74 | +for p in L: print('PROC', p['name'], p['pm2_env'].get('status'), p['pm2_env'].get('restart_time',0))" 2>/dev/null | |
| 75 | +launchctl list 2>/dev/null | awk 'NR>1{print "LA", $3}' | |
| 76 | +ifconfig 2>/dev/null | grep -q 'inet 10\.67\.0\.' && echo "WG1 up" || echo "WG1 down" | |
| 77 | +ping -c 1 -W 2000 10.67.0.1 >/dev/null 2>&1 && echo "HUB ok" || echo "HUB ko" | |
| 78 | +''' | |
| 79 | + for app, m in apps.items(): | |
| 80 | + if m.get("port"): | |
| 81 | + script += "echo \"HTTP %s $(curl -s -o /dev/null -m 6 -w '%%{http_code}' http://127.0.0.1:%s%s)\"\n" % (app, m["port"], m.get("health_path") or "/") | |
| 82 | + rc, out, err = ssh.run(ip, script, timeout=90) | |
| 83 | + if rc != 0 and "BOOT" not in out: | |
| 84 | + return None | |
| 85 | + rep = {"pm2": None, "procs": {}, "la": set(), "wg1": None, "hub": None, "http": {}, "boot": 0, "now": 0} | |
| 86 | + for line in out.splitlines(): | |
| 87 | + p = line.split() | |
| 88 | + if not p: | |
| 89 | + continue | |
| 90 | + if p[0] == "BOOT" and len(p) > 1 and p[1].isdigit(): | |
| 91 | + rep["boot"] = int(p[1]) | |
| 92 | + elif p[0] == "NOW" and len(p) > 1: | |
| 93 | + rep["now"] = int(p[1]) | |
| 94 | + elif p[0] == "PM2": | |
| 95 | + rep["pm2"] = p[1] == "up" | |
| 96 | + elif p[0] == "PROC" and len(p) >= 3: | |
| 97 | + rep["procs"][p[1]] = (p[2], int(p[3]) if len(p) > 3 and p[3].isdigit() else 0) | |
| 98 | + elif p[0] == "LA" and len(p) > 1: | |
| 99 | + rep["la"].add(p[1]) | |
| 100 | + elif p[0] == "WG1": | |
| 101 | + rep["wg1"] = p[1] == "up" | |
| 102 | + elif p[0] == "HUB": | |
| 103 | + rep["hub"] = p[1] == "ok" | |
| 104 | + elif p[0] == "HTTP" and len(p) >= 2: | |
| 105 | + rep["http"][p[1]] = p[2] if len(p) > 2 else "000" | |
| 106 | + rep["uptime"] = (rep["now"] - rep["boot"]) if rep["boot"] and rep["now"] else 10 ** 6 | |
| 107 | + return rep | |
| 108 | + | |
| 109 | + | |
| 110 | +def _http_ok(code): | |
| 111 | + return code.isdigit() and code != "000" and int(code) < 500 | |
| 112 | + | |
| 113 | + | |
| 114 | +def heal(apps_filter=None, nodes_filter=None, dry_run=False, quiet=False, public=True): | |
| 115 | + t0 = time.time() | |
| 116 | + st = _state() | |
| 117 | + reg = registry.load() | |
| 118 | + by_node = {} | |
| 119 | + for app, v in reg["apps"].items(): | |
| 120 | + if apps_filter and app not in apps_filter: | |
| 121 | + continue | |
| 122 | + if not v.get("node"): | |
| 123 | + continue | |
| 124 | + if nodes_filter and v["node"] not in nodes_filter: | |
| 125 | + continue | |
| 126 | + by_node.setdefault(v["node"], {})[app] = manifest.load(app) | |
| 127 | + changes = {} | |
| 128 | + summary = {"ok": 0, "fixed": 0, "ko": 0, "skipped": 0} | |
| 129 | + tun_state = None | |
| 130 | + try: | |
| 131 | + tun_state = tunnel.state() | |
| 132 | + except tunnel.TunnelError as e: | |
| 133 | + _log("tunnel : %s" % e, quiet) | |
| 134 | + for alias in sorted(by_node): | |
| 135 | + apps = by_node[alias] | |
| 136 | + ip = nodes.ip_of(alias, rediscover=False) | |
| 137 | + if not ip: | |
| 138 | + # une seule redécouverte par passage | |
| 139 | + ip = nodes.ip_of(alias, rediscover=True) | |
| 140 | + if not ip: | |
| 141 | + since = st["down"].setdefault(alias, _now()) | |
| 142 | + _log("%s : INJOIGNABLE (depuis %s) — %d app(s) : %s" % (alias, since, len(apps), ", ".join(sorted(apps))), quiet) | |
| 143 | + for app in apps: | |
| 144 | + if reg["apps"][app].get("status") != "node-down": | |
| 145 | + changes[app] = {"status": "node-down", "health": "nœud %s injoignable" % alias} | |
| 146 | + summary["ko"] += len(apps) | |
| 147 | + continue | |
| 148 | + if alias in st["down"]: | |
| 149 | + _log("%s : de retour (était injoignable depuis %s)" % (alias, st["down"].pop(alias)), quiet) | |
| 150 | + rep = _node_report(ip, apps) | |
| 151 | + if rep is None: | |
| 152 | + _log("%s : sonde impossible" % alias, quiet) | |
| 153 | + summary["skipped"] += len(apps) | |
| 154 | + continue | |
| 155 | + if rep["uptime"] < GRACE_S: | |
| 156 | + _log("%s : redémarré il y a %ds — délai de grâce, on repasse plus tard" % (alias, rep["uptime"]), quiet) | |
| 157 | + summary["skipped"] += len(apps) | |
| 158 | + continue | |
| 159 | + # --- WireGuard vers la passerelle (seulement si le nœud expose un site) | |
| 160 | + exposes = any(tunnel.spec(m, alias) for m in apps.values()) | |
| 161 | + if exposes and (rep["wg1"] is False or rep["hub"] is False): | |
| 162 | + _log("%s : wg1 %s / hub %s" % (alias, "up" if rep["wg1"] else "DOWN", "ok" if rep["hub"] else "KO"), quiet) | |
| 163 | + if not dry_run: | |
| 164 | + try: | |
| 165 | + tunnel.ensure_peer(alias, log=lambda s: _log(s, quiet), st=tun_state) | |
| 166 | + except tunnel.TunnelError as e: | |
| 167 | + _log(" tunnel : %s" % e, quiet) | |
| 168 | + resurrected = False | |
| 169 | + for app in sorted(apps): | |
| 170 | + m = apps[app] | |
| 171 | + names = manifest.pm2_names(m) | |
| 172 | + labels = manifest.launchd_labels(m) | |
| 173 | + missing = [n for n in names if n not in rep["procs"]] if rep["pm2"] else list(names) | |
| 174 | + bad = [n for n in names if n in rep["procs"] and rep["procs"][n][0] in ("errored", "stopped") | |
| 175 | + and next((p for p in m["processes"] if p["name"] == n), {}).get("autorestart", True) | |
| 176 | + and not next((p for p in m["processes"] if p["name"] == n), {}).get("cron_restart")] | |
| 177 | + la_missing = [l for l in labels if l not in rep["la"]] | |
| 178 | + code = rep["http"].get(app) | |
| 179 | + http_ok = _http_ok(code) if m.get("port") else True | |
| 180 | + problems = [] | |
| 181 | + if missing: | |
| 182 | + problems.append("PM2 absent: %s" % ",".join(missing)) | |
| 183 | + if bad: | |
| 184 | + problems.append("PM2 %s: %s" % (rep["procs"][bad[0]][0], ",".join(bad))) | |
| 185 | + if la_missing: | |
| 186 | + problems.append("launchd absent: %s" % ",".join(la_missing)) | |
| 187 | + if not http_ok and not missing and not bad: | |
| 188 | + problems.append("HTTP local %s" % code) | |
| 189 | + if not problems: | |
| 190 | + summary["ok"] += 1 | |
| 191 | + if reg["apps"][app].get("status") != "online": | |
| 192 | + changes[app] = {"status": "online", "health": "local %s" % code if m.get("port") else "processus online"} | |
| 193 | + # route publique : suit-elle bien ce nœud ? | |
| 194 | + if tun_state and public and tunnel.spec(m, alias): | |
| 195 | + s = tunnel.spec(m, alias) | |
| 196 | + r = tunnel.routes(s["gateway"], tun_state).get(s["domain"]) | |
| 197 | + if not tunnel._route_matches(r, s["upstreams"]): | |
| 198 | + _log("%s/%s : route https://%s %s" % (alias, app, s["domain"], "absente" if not r else "→ %s au lieu de %s" % (", ".join(u["addr"] for u in r.get("upstreams", [])), s["upstreams"][0])), quiet) | |
| 199 | + if not dry_run and _allowed(st, app): | |
| 200 | + try: | |
| 201 | + tunnel.ensure_route(m, alias, log=lambda x: _log(x, quiet), st=tun_state) | |
| 202 | + _record(st, app) | |
| 203 | + changes[app] = dict(changes.get(app, {}), tunnel={"gateway": s["gateway"], "domain": s["domain"], "upstream": s["upstreams"][0], "url": "https://%s" % s["domain"]}) | |
| 204 | + summary["fixed"] += 1 | |
| 205 | + except tunnel.TunnelError as e: | |
| 206 | + _log(" tunnel : %s" % e, quiet) | |
| 207 | + continue | |
| 208 | + _log("%s/%s : %s" % (alias, app, " ; ".join(problems)), quiet) | |
| 209 | + if dry_run: | |
| 210 | + summary["ko"] += 1 | |
| 211 | + continue | |
| 212 | + if not _allowed(st, app): | |
| 213 | + _log(" %s : trop d'actions dans l'heure (%d), on laisse tranquille" % (app, MAX_ACTIONS_PER_HOUR), quiet) | |
| 214 | + summary["ko"] += 1 | |
| 215 | + changes[app] = {"status": "unhealthy", "health": " ; ".join(problems)} | |
| 216 | + continue | |
| 217 | + _record(st, app) | |
| 218 | + try: | |
| 219 | + if rep["pm2"] is False and not resurrected: | |
| 220 | + rc, out, _ = ssh.run(ip, "pm2 resurrect 2>&1 | tail -1", timeout=120) | |
| 221 | + _log(" pm2 resurrect sur %s : %s" % (alias, out.strip()[-100:]), quiet) | |
| 222 | + resurrected = True | |
| 223 | + if missing or la_missing: | |
| 224 | + deploy.start(app, alias, m=m, ip=ip) | |
| 225 | + elif bad: | |
| 226 | + rc, out, _ = ssh.run(ip, "pm2 restart %s --update-env >/dev/null 2>&1; pm2 save --force >/dev/null 2>&1; echo ok" % " ".join(bad), timeout=120) | |
| 227 | + _log(" pm2 restart %s : %s" % (",".join(bad), out.strip()[-40:]), quiet) | |
| 228 | + elif not http_ok: | |
| 229 | + rc, out, _ = ssh.run(ip, "pm2 restart %s --update-env >/dev/null 2>&1; echo ok" % " ".join(names), timeout=120) if names else (0, "", "") | |
| 230 | + _log(" HTTP local %s → pm2 restart %s" % (code, ",".join(names)), quiet) | |
| 231 | + ok, det = deploy.health(app, alias, ip=ip, m=m, timeout=45, public=False) | |
| 232 | + tun = None | |
| 233 | + if ok and tunnel.spec(m, alias): | |
| 234 | + tun = deploy._publish(app, alias, m, ip) | |
| 235 | + ok, det = deploy.health(app, alias, ip=ip, m=m, timeout=20, public=public and bool(tun)) | |
| 236 | + _log(" %s : %s (%s)" % (app, "RÉPARÉE" if ok else "toujours KO", det), quiet) | |
| 237 | + fields = {"status": "online" if ok else "unhealthy", "health": det, "healed": _now()} | |
| 238 | + if tun: | |
| 239 | + fields["tunnel"] = tun | |
| 240 | + changes[app] = fields | |
| 241 | + summary["fixed" if ok else "ko"] += 1 | |
| 242 | + except (deploy.DeployError, tunnel.TunnelError) as e: | |
| 243 | + _log(" %s : ÉCHEC de réparation — %s" % (app, e), quiet) | |
| 244 | + changes[app] = {"status": "unhealthy", "health": str(e)[:200]} | |
| 245 | + summary["ko"] += 1 | |
| 246 | + if changes and not dry_run: | |
| 247 | + registry.update_many(changes) | |
| 248 | + _save_state(st) | |
| 249 | + _log("heal terminé en %.0fs : %d ok, %d réparées, %d KO, %d ignorées (grâce/sonde)%s" % ( | |
| 250 | + time.time() - t0, summary["ok"], summary["fixed"], summary["ko"], summary["skipped"], " [dry-run]" if dry_run else ""), quiet) | |
| 251 | + return summary | |
| 252 | + | |
| 253 | + | |
| 254 | +# ------------------------------------------------------------ LaunchDaemon sur la passerelle --- | |
| 255 | + | |
| 256 | +def plist_content(): | |
| 257 | + mld_bin = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "bin", "mld") | |
| 258 | + d = { | |
| 259 | + "Label": HEAL_LABEL, | |
| 260 | + "ProgramArguments": [mld_bin, "heal", "--quiet"], | |
| 261 | + "UserName": config.USER, | |
| 262 | + "RunAtLoad": True, | |
| 263 | + "StartInterval": HEAL_INTERVAL_S, | |
| 264 | + "EnvironmentVariables": {"HOME": config.HOME, "PATH": "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", "LANG": "fr_CA.UTF-8"}, | |
| 265 | + "WorkingDirectory": config.HOME, | |
| 266 | + "StandardOutPath": os.path.join(config.LOG_DIR, "heal.out"), | |
| 267 | + "StandardErrorPath": os.path.join(config.LOG_DIR, "heal.err"), | |
| 268 | + "ProcessType": "Background", | |
| 269 | + "LowPriorityIO": True, | |
| 270 | + } | |
| 271 | + return plistlib.dumps(d).decode() | |
| 272 | + | |
| 273 | + | |
| 274 | +def install(interval=None): | |
| 275 | + """Installe (ou met à jour) le LaunchDaemon io.maclustr.mld-heal sur la passerelle (sudo via ~/dispatch/.sudo).""" | |
| 276 | + import subprocess | |
| 277 | + global HEAL_INTERVAL_S | |
| 278 | + if interval: | |
| 279 | + HEAL_INTERVAL_S = int(interval) | |
| 280 | + pwf = config.SUDO_PW_FILE | |
| 281 | + pw = open(pwf).read().strip() if os.path.exists(pwf) else None | |
| 282 | + if not pw: | |
| 283 | + raise SystemExit("mot de passe sudo absent : %s" % pwf) | |
| 284 | + tmp = "/tmp/%s.plist" % HEAL_LABEL | |
| 285 | + open(tmp, "w").write(plist_content()) | |
| 286 | + dest = "/Library/LaunchDaemons/%s.plist" % HEAL_LABEL | |
| 287 | + sh = ("echo %s | sudo -S -p '' sh -c 'install -m 644 -o root -g wheel %s %s && launchctl bootout system/%s 2>/dev/null; " | |
| 288 | + "launchctl bootstrap system %s && launchctl print system/%s | grep -E \"state|interval\" | head -2'") % ( | |
| 289 | + shlex.quote(pw), tmp, dest, HEAL_LABEL, dest, HEAL_LABEL) | |
| 290 | + p = subprocess.run(["bash", "-c", sh], capture_output=True, text=True) | |
| 291 | + print(p.stdout.strip() or p.stderr.strip()[-300:]) | |
| 292 | + os.remove(tmp) | |
| 293 | + ok = p.returncode == 0 | |
| 294 | + print("LaunchDaemon %s : %s (toutes les %d s, journal %s/heal.log)" % (HEAL_LABEL, "installé" if ok else "ÉCHEC", HEAL_INTERVAL_S, config.LOG_DIR)) | |
| 295 | + return ok | |
modified
mld/importer.py
+8 −3
@@ -1,6 +1,7 @@ | ||
| 1 | 1 | """`mld import <nœud>` : brouillons de manifestes à partir des processus PM2 / launchd vivants d'un nœud. |
| 2 | 2 | |
| 3 | −Regroupe par préfixe de nom (lou-ka-web, lou-ka-sync, lou-ka-ngrok → lou-ka). Les brouillons vont dans | |
| 3 | +Regroupe par préfixe de nom (lou-ka-web, lou-ka-sync, lou-ka-ngrok → lou-ka ; un vieux tunnel ngrok devient une route | |
| 4 | +MacLustr Tunnel `tunnel.domain`). Les brouillons vont dans | |
| 4 | 5 | ~/dispatch/apps/_drafts/<app>.json ; on les relit, on ajuste, puis on les déplace dans ~/dispatch/apps/. |
| 5 | 6 | """ |
| 6 | 7 | import json |
@@ -70,8 +71,12 @@ def import_node(alias, apps_filter=None): | ||
| 70 | 71 | m = json.loads(json.dumps(manifest.DEFAULTS)) |
| 71 | 72 | mo = re.match(r"(/Users/[^/]+/(?:apps/)?[^/]+|/opt/[^/]+)", cwd) |
| 72 | 73 | d = mo.group(1) if mo else cwd |
| 73 | − m.update({"app": app, "label": app, "dir": d.replace(config.home_of(alias), "~"), "processes": g["processes"], "ngrok": g["ngrok"], | |
| 74 | − "domain": (g["ngrok"] or {}).get("url"), "port": (g["ngrok"] or {}).get("port"), "ram_mb_observed": max(g["ram"], 128)}) | |
| 74 | + # un tunnel ngrok encore vivant sur le nœud devient une route MacLustr Tunnel (le processus ngrok sera supprimé au 1er start) | |
| 75 | + ng = g["ngrok"] or {} | |
| 76 | + m.update({"app": app, "label": app, "dir": d.replace(config.home_of(alias), "~"), "processes": g["processes"], "ngrok": None, | |
| 77 | + "tunnel": ({"domain": ng.get("url"), "gateway": config.TUNNEL_DEFAULT} if ng.get("url") else None), | |
| 78 | + "legacy_pm2": [ng["name"]] if ng.get("name") else [], | |
| 79 | + "domain": ng.get("url"), "port": ng.get("port"), "ram_mb_observed": max(g["ram"], 128)}) | |
| 75 | 80 | m["requires"]["ports"] = [m["port"]] if m["port"] else [] |
| 76 | 81 | m["placement"]["reason"] = "importé de %s" % alias |
| 77 | 82 | manifest.detect_requirements(m) |
modified
mld/manifest.py
+40 −9
@@ -6,12 +6,15 @@ Schéma : | ||
| 6 | 6 | "dir": "~/apps/lou-ka", # répertoire de l'app (même chemin absolu sur tous les nœuds) |
| 7 | 7 | "extra_paths": ["~/.ssh/trouveka_tunnel"], # autres fichiers/dossiers à copier avec l'app |
| 8 | 8 | "sync_excludes": ["data/backups/"], # exclusions rsync (relatives à dir) |
| 9 | − "requires": {"runtimes": ["python@3.14", "pm2", "ngrok"], "ram_gb": 2, "ports": [8095]}, | |
| 9 | + "requires": {"runtimes": ["python@3.14", "pm2"], "ram_gb": 2, "ports": [8095]}, | |
| 10 | 10 | "ram_mb_observed": 400, "size_mb": 4359, |
| 11 | 11 | "placement": {"pin": null, "prefer": null, "avoid": [], "reason": ""}, |
| 12 | 12 | "processes": [{"name": "lou-ka-web", "manager": "pm2", "script": "...", "args": [...], "interpreter": null, |
| 13 | 13 | "cwd": "...", "env": {...}, "cron_restart": null, "autorestart": true, "max_memory_restart": null}], |
| 14 | − "ngrok": {"name": "lou-ka-ngrok", "url": "www.lou-ka.com", "port": 8095} | null, | |
| 14 | + "tunnel": {"domain": "www.lou-ka.com", "gateway": "BHS64", "redirects": ["lou-ka.com"], "websocket": false} | null, | |
| 15 | + # exposition publique via MacLustr Tunnel : la route Caddy https://domain → | |
| 16 | + # <nœud courant>:<port> est (re)pointée par `mld deploy`/`move`. `domain` seul suffit. | |
| 17 | + "ngrok": null, # OBSOLÈTE (ngrok retiré le 2026-09-10) : un vieux bloc est converti en `tunnel` au chargement | |
| 15 | 18 | "launchd": [{"label": "...", "program_arguments": [...], "working_directory": "...", "env": {...}, "keep_alive": true}], |
| 16 | 19 | "env_overrides": {".env": {"DATABASE_URL": "postgresql://...@{{IP:M2M32}}:5432/x"}}, # réécritures de fichiers .env |
| 17 | 20 | "hooks": {"post_sync": ["bash ..."], "post_start": []}, |
@@ -28,7 +31,7 @@ from . import config | ||
| 28 | 31 | DEFAULTS = { |
| 29 | 32 | "label": None, "domain": None, "port": None, "health_path": "/", "extra_paths": [], "sync_excludes": [], |
| 30 | 33 | "requires": {"runtimes": ["pm2"], "ram_gb": 1, "ports": []}, "ram_mb_observed": 512, "size_mb": 0, |
| 31 | − "placement": {"pin": None, "prefer": None, "avoid": [], "reason": ""}, "processes": [], "ngrok": None, | |
| 34 | + "placement": {"pin": None, "prefer": None, "avoid": [], "reason": ""}, "processes": [], "tunnel": None, "ngrok": None, | |
| 32 | 35 | "launchd": [], "env_overrides": {}, "hooks": {"post_sync": [], "post_start": []}, "ka_repo": False, |
| 33 | 36 | } |
| 34 | 37 | |
@@ -47,6 +50,33 @@ def load(app): | ||
| 47 | 50 | m["requires"].setdefault("runtimes", ["pm2"]) |
| 48 | 51 | m["requires"].setdefault("ram_gb", 1) |
| 49 | 52 | m["requires"].setdefault("ports", [m["port"]] if m.get("port") else []) |
| 53 | + return modernize(m) | |
| 54 | + | |
| 55 | + | |
| 56 | +def modernize(m): | |
| 57 | + """ngrok → MacLustr Tunnel (2026-09-10). Un ancien bloc `ngrok` devient une route `tunnel` ; le runtime « ngrok » disparaît | |
| 58 | + des prérequis (plus installé nulle part) ; le nom du vieux processus PM2 `<app>-ngrok` est gardé pour le nettoyer au start.""" | |
| 59 | + n = m.get("ngrok") | |
| 60 | + if n: | |
| 61 | + if not m.get("tunnel"): | |
| 62 | + m["tunnel"] = {"domain": n.get("url") or m.get("domain"), "gateway": config.TUNNEL_DEFAULT, | |
| 63 | + "note": "converti automatiquement depuis l'ancien bloc ngrok"} | |
| 64 | + if not m.get("domain"): | |
| 65 | + m["domain"] = n.get("url") | |
| 66 | + if not m.get("port") and n.get("port"): | |
| 67 | + m["port"] = n["port"] | |
| 68 | + if n.get("name"): | |
| 69 | + m["legacy_pm2"] = sorted(set((m.get("legacy_pm2") or []) + [n["name"]])) | |
| 70 | + m["ngrok"] = None | |
| 71 | + if m.get("domain") and not m.get("tunnel"): | |
| 72 | + m["tunnel"] = {"domain": m["domain"], "gateway": config.TUNNEL_DEFAULT} | |
| 73 | + if m.get("tunnel") and not m["tunnel"].get("domain"): | |
| 74 | + m["tunnel"]["domain"] = m.get("domain") | |
| 75 | + if m.get("tunnel") and not m.get("domain"): | |
| 76 | + m["domain"] = m["tunnel"]["domain"] | |
| 77 | + m["requires"]["runtimes"] = [r for r in m["requires"]["runtimes"] if r != "ngrok"] | |
| 78 | + # les vieux processus `<app>-ngrok` (PM2) sont supprimés à chaque start/stop s'ils traînent encore | |
| 79 | + m["legacy_pm2"] = sorted(set((m.get("legacy_pm2") or []) + ["%s-ngrok" % m["app"]])) | |
| 50 | 80 | return m |
| 51 | 81 | |
| 52 | 82 | |
@@ -80,10 +110,12 @@ def render(value, ctx): | ||
| 80 | 110 | |
| 81 | 111 | |
| 82 | 112 | def pm2_names(m): |
| 83 | − names = [p["name"] for p in m["processes"] if p.get("manager", "pm2") == "pm2"] | |
| 84 | − if m.get("ngrok"): | |
| 85 | − names.append(m["ngrok"]["name"]) | |
| 86 | − return names | |
| 113 | + return [p["name"] for p in m["processes"] if p.get("manager", "pm2") == "pm2"] | |
| 114 | + | |
| 115 | + | |
| 116 | +def legacy_pm2_names(m): | |
| 117 | + """Processus PM2 d'anciennes versions du manifeste (tunnels ngrok) à supprimer s'ils existent encore sur le nœud.""" | |
| 118 | + return [n for n in (m.get("legacy_pm2") or []) if n not in pm2_names(m)] | |
| 87 | 119 | |
| 88 | 120 | |
| 89 | 121 | def launchd_labels(m): |
@@ -94,8 +126,7 @@ def detect_requirements(m): | ||
| 94 | 126 | """Déduit les runtimes requis à partir des processus (complète requires.runtimes).""" |
| 95 | 127 | req = set(m["requires"].get("runtimes") or []) |
| 96 | 128 | req.add("pm2") |
| 97 | − if m.get("ngrok"): | |
| 98 | − req.add("ngrok") | |
| 129 | + req.discard("ngrok") | |
| 99 | 130 | blob = json.dumps(m["processes"]) + json.dumps(m.get("launchd", [])) |
| 100 | 131 | if "/opt/homebrew/bin/node" in blob or "next" in blob or "npm" in blob or ".mjs" in blob or ".js" in blob: |
| 101 | 132 | req.add("node") |
modified
mld/prepare.py
+16 −9
@@ -1,4 +1,5 @@ | ||
| 1 | −"""`mld prepare <nœud>` : installe les runtimes manquants (Homebrew), copie la config ngrok, active pm2 au démarrage, pose le marqueur.""" | |
| 1 | +"""`mld prepare <nœud>` : installe les runtimes manquants (Homebrew), active pm2 au démarrage, pose le marqueur, raccorde le | |
| 2 | +nœud au MacLustr Tunnel (wg1 → BHS64) et applique `mld harden` (résilience coupure/MAJ).""" | |
| 2 | 3 | import os |
| 3 | 4 | import shlex |
| 4 | 5 | from . import config, nodes, ssh |
@@ -11,13 +12,13 @@ def _sudo_pw(alias=None): | ||
| 11 | 12 | return None |
| 12 | 13 | |
| 13 | 14 | |
| 14 | −def prepare(alias, runtimes=None, ka_helpers=False): | |
| 15 | +def prepare(alias, runtimes=None, ka_helpers=False, tunnel_peer=True, harden=True): | |
| 15 | 16 | ip = nodes.ip_of(alias) |
| 16 | 17 | if not ip: |
| 17 | 18 | raise SystemExit("nœud %s injoignable" % alias) |
| 18 | 19 | info = nodes.probe(alias, ip) |
| 19 | 20 | have = info.get("runtimes", {}) |
| 20 | − want = runtimes or ["node", "pnpm", "pm2", "ngrok", "python@3.14", "python@3.13", "uv", "git"] | |
| 21 | + want = runtimes or ["node", "pnpm", "pm2", "python@3.14", "python@3.13", "uv", "git"] | |
| 21 | 22 | print("prepare %s (%s) — présents : %s" % (alias, ip, ", ".join(k for k, v in have.items() if v))) |
| 22 | 23 | if not have.get("brew"): |
| 23 | 24 | print(" ✗ Homebrew absent : bootstrap requis (Xcode CLT). Lancer d'abord : mld bootstrap %s" % alias) |
@@ -29,12 +30,7 @@ def prepare(alias, runtimes=None, ka_helpers=False): | ||
| 29 | 30 | fallback = " || npm install -g pm2 >/dev/null 2>&1" if r == "pm2" else "" |
| 30 | 31 | script += "echo ' brew install %s'; brew install -q %s >/dev/null 2>&1%s || echo ' ! échec %s'\n" % (r, config.BREW_FORMULAE[r], fallback, r) |
| 31 | 32 | elif r in config.BREW_CASKS: |
| 32 | − script += "echo ' brew install --cask ngrok'; brew install -q --cask ngrok >/dev/null 2>&1 || brew install -q ngrok/ngrok/ngrok >/dev/null 2>&1 || echo ' ! échec ngrok'\n" | |
| 33 | − # config ngrok (authtoken) depuis la passerelle | |
| 34 | − if "ngrok" in missing or not have.get("ngrok"): | |
| 35 | − if os.path.exists(config.NGROK_CFG): | |
| 36 | − rc0, _, _ = ssh.write_remote_file(ip, "%s/Library/Application Support/ngrok/ngrok.yml" % config.home_of(alias), open(config.NGROK_CFG).read(), mode="600") | |
| 37 | − script += "echo ' ngrok.yml %s'\n" % ("copié" if rc0 == 0 else "ÉCHEC de copie") | |
| 33 | + script += "echo ' brew install --cask %s'; brew install -q --cask %s >/dev/null 2>&1 || echo ' ! échec %s'\n" % (r, config.BREW_CASKS[r], r) | |
| 38 | 34 | # marqueur |
| 39 | 35 | script += "[ -f ~/.maclustr-node ] || echo %s > ~/.maclustr-node\n" % alias |
| 40 | 36 | # pm2 startup (launchd) — nécessite sudo |
@@ -62,6 +58,17 @@ def prepare(alias, runtimes=None, ka_helpers=False): | ||
| 62 | 58 | print(out) |
| 63 | 59 | if err.strip(): |
| 64 | 60 | print(" stderr:", err.strip()[-300:]) |
| 61 | + # MacLustr Tunnel : raccorde le nœud au hub BHS64 (wg1) s'il ne l'est pas — nécessaire pour exposer un site | |
| 62 | + if tunnel_peer: | |
| 63 | + from . import tunnel | |
| 64 | + try: | |
| 65 | + tunnel.ensure_peer(alias, log=print) | |
| 66 | + except tunnel.TunnelError as e: | |
| 67 | + print(" tunnel : %s" % e) | |
| 68 | + # résilience (auto-login, pas de veille, redémarrage après coupure, MAJ macOS/Tailscale off) | |
| 69 | + if harden: | |
| 70 | + from . import harden as _harden | |
| 71 | + _harden.harden(alias) | |
| 65 | 72 | info = nodes.probe(alias, ip) |
| 66 | 73 | still = [r for r in want if not info.get("runtimes", {}).get(r)] |
| 67 | 74 | print(" résultat : %s" % ("tout est là" if not still else "manquent encore " + ", ".join(still))) |
modified
mld/registry.py
+14 −0
@@ -59,6 +59,20 @@ def set_app(app, **fields): | ||
| 59 | 59 | save(r) |
| 60 | 60 | |
| 61 | 61 | |
| 62 | +def update_many(changes, push=True): | |
| 63 | + """Met à jour plusieurs apps ({app: {champs}}) en une seule sauvegarde/diffusion (status --live, heal).""" | |
| 64 | + if not changes: | |
| 65 | + return | |
| 66 | + r = load() | |
| 67 | + now = time.strftime("%Y-%m-%dT%H:%M:%S") | |
| 68 | + for app, fields in changes.items(): | |
| 69 | + cur = r["apps"].get(app, {}) | |
| 70 | + cur.update(fields) | |
| 71 | + cur["updated"] = now | |
| 72 | + r["apps"][app] = cur | |
| 73 | + save(r, push=push) | |
| 74 | + | |
| 75 | + | |
| 62 | 76 | def remove_app(app, note=""): |
| 63 | 77 | r = load() |
| 64 | 78 | if app in r["apps"]: |
modified
mld/render.py
+2 −9
@@ -20,7 +20,8 @@ def _home(path, ctx): | ||
| 20 | 20 | |
| 21 | 21 | |
| 22 | 22 | def ecosystem(m, ctx): |
| 23 | − """Retourne le contenu JS d'un ecosystem.config.cjs pour les processus PM2 + tunnel ngrok.""" | |
| 23 | + """Retourne le contenu JS d'un ecosystem.config.cjs pour les processus PM2 (l'exposition publique est une route | |
| 24 | + Caddy sur la passerelle du MacLustr Tunnel, plus aucun processus ngrok local).""" | |
| 24 | 25 | from .manifest import render |
| 25 | 26 | apps = [] |
| 26 | 27 | for p in m["processes"]: |
@@ -54,14 +55,6 @@ def ecosystem(m, ctx): | ||
| 54 | 55 | if p.get("node_args"): |
| 55 | 56 | a["node_args"] = p["node_args"] |
| 56 | 57 | apps.append(a) |
| 57 | − if m.get("ngrok"): | |
| 58 | − n = m["ngrok"] | |
| 59 | − args = ["http", "--url=%s" % n["url"], str(n["port"]), "--log=stdout"] + list(n.get("extra_args") or []) | |
| 60 | − apps.append({ | |
| 61 | − "name": n["name"], "script": n.get("binary") or "/opt/homebrew/bin/ngrok", "args": " ".join(args), | |
| 62 | − "interpreter": "none", "cwd": _home(render(m["dir"], ctx), ctx), "exec_mode": "fork", "autorestart": True, | |
| 63 | − "restart_delay": 5000, "merge_logs": True, "time": True, | |
| 64 | − }) | |
| 65 | 58 | head = "// généré par maclustr-dispatch — ne pas éditer à la main (app: %s, nœud: %s)\n" % (m["app"], ctx.get("NODE")) |
| 66 | 59 | return head + "module.exports = { apps: %s };\n" % json.dumps(apps, indent=2, ensure_ascii=False) |
| 67 | 60 | |
added
mld/tunnel.py
+229 −0
@@ -0,0 +1,229 @@ | ||
| 1 | +"""MacLustr Tunnel — le « ngrok maison » du cluster, vu depuis mld. | |
| 2 | + | |
| 3 | +Une passerelle OVH (BHS64 par défaut) porte un hub WireGuard et Caddy (TLS Let's Encrypt). Chaque Mac y est raccordé | |
| 4 | +par une interface WireGuard (LaunchDaemon io.maclustr.wireguard-wg1) et chaque site public est une route Caddy | |
| 5 | +`https://<domaine> → <ip-wg du nœud>:<port>`. Tout se pilote sur la passerelle avec `tunnelctl` : | |
| 6 | + | |
| 7 | + tunnelctl json état machine-lisible (pairs + routes) | |
| 8 | + tunnelctl add <dom> <alias>:<port> route (idempotent : réécrit sites/<dom>.caddy + reload) | |
| 9 | + tunnelctl redirect <dom> <cible> 308 (apex → www) | |
| 10 | + tunnelctl rm <dom> | |
| 11 | + tunnelctl peer add <alias> <pubkey> pair WireGuard (IP fixe tirée de /etc/maclustr-tunnel/ipmap) | |
| 12 | + | |
| 13 | +Ici : `ensure_peer(alias)` raccorde un nœud qui ne l'est pas (script assets/wg-node-setup.sh, même recette que `mlt peer`), | |
| 14 | +`ensure_route(m, alias)` fait suivre la route publique d'une app au nœud où mld vient de la démarrer. Le manifeste ne porte | |
| 15 | +que `tunnel.domain` (+ `gateway`, `redirects`, `websocket`) : l'upstream est TOUJOURS le nœud courant du registre. | |
| 16 | +""" | |
| 17 | +import json | |
| 18 | +import os | |
| 19 | +import shlex | |
| 20 | +import subprocess | |
| 21 | +import time | |
| 22 | +from . import config, ssh | |
| 23 | + | |
| 24 | +ASSETS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "assets") | |
| 25 | +HANDSHAKE_OK_S = 200 # keepalive 25 s : au-delà de ~3 min sans handshake, le pair est considéré tombé | |
| 26 | + | |
| 27 | + | |
| 28 | +class TunnelError(Exception): | |
| 29 | + pass | |
| 30 | + | |
| 31 | + | |
| 32 | +def gateway(name=None): | |
| 33 | + name = name or config.TUNNEL_DEFAULT | |
| 34 | + if name not in config.TUNNEL_GATEWAYS: | |
| 35 | + raise TunnelError("passerelle inconnue « %s » (connues : %s)" % (name, ", ".join(config.TUNNEL_GATEWAYS))) | |
| 36 | + return name, config.TUNNEL_GATEWAYS[name] | |
| 37 | + | |
| 38 | + | |
| 39 | +def gw_run(name, cmd, timeout=60): | |
| 40 | + """Commande sur la passerelle (ssh direct ubuntu@ip avec la clé de M1M32).""" | |
| 41 | + name, g = gateway(name) | |
| 42 | + try: | |
| 43 | + p = subprocess.run(["ssh"] + ssh.SSH_OPTS + ["%s@%s" % (g["user"], g["host"]), cmd], capture_output=True, text=True, timeout=timeout) | |
| 44 | + except subprocess.TimeoutExpired: | |
| 45 | + return 124, "", "timeout après %ss" % timeout | |
| 46 | + return p.returncode, p.stdout, p.stderr | |
| 47 | + | |
| 48 | + | |
| 49 | +def state(name=None): | |
| 50 | + """Sortie de `tunnelctl json` : {"gateway", "ip", "wg": {"peers": [...]}, "caddy": {"active", "routes": [...]}}.""" | |
| 51 | + name, g = gateway(name) | |
| 52 | + rc, out, err = gw_run(name, "sudo -n tunnelctl json", timeout=40) | |
| 53 | + if rc != 0 or not out.strip(): | |
| 54 | + raise TunnelError("passerelle %s (%s) injoignable ou tunnelctl KO : %s" % (name, g["host"], (err or out).strip()[-160:])) | |
| 55 | + return json.loads(out.strip().splitlines()[-1]) | |
| 56 | + | |
| 57 | + | |
| 58 | +def routes(name=None, st=None): | |
| 59 | + st = st or state(name) | |
| 60 | + return {r["domain"]: r for r in st["caddy"]["routes"]} | |
| 61 | + | |
| 62 | + | |
| 63 | +def peers(name=None, st=None): | |
| 64 | + st = st or state(name) | |
| 65 | + return {p["alias"]: p for p in st["wg"]["peers"]} | |
| 66 | + | |
| 67 | + | |
| 68 | +# ------------------------------------------------------------------ routes --- | |
| 69 | + | |
| 70 | +def spec(m, alias): | |
| 71 | + """Route attendue pour l'app `m` tournant sur `alias` → dict ou None si l'app n'est pas publique.""" | |
| 72 | + t = m.get("tunnel") or {} | |
| 73 | + domain = t.get("domain") or m.get("domain") | |
| 74 | + if not domain or not m.get("port"): | |
| 75 | + return None | |
| 76 | + return { | |
| 77 | + "gateway": t.get("gateway") or config.TUNNEL_DEFAULT, | |
| 78 | + "domain": domain, | |
| 79 | + "upstreams": ["%s:%s" % (alias, m["port"])], | |
| 80 | + "redirects": list(t.get("redirects") or []), | |
| 81 | + "websocket": bool(t.get("websocket")), | |
| 82 | + } | |
| 83 | + | |
| 84 | + | |
| 85 | +def _route_matches(route, want): | |
| 86 | + """La route Caddy actuelle pointe-t-elle déjà vers les upstreams voulus (comparaison par alias:port) ?""" | |
| 87 | + if not route or route.get("kind") != "proxy": | |
| 88 | + return False | |
| 89 | + cur = sorted("%s:%s" % (u.get("alias") or u["addr"].split(":")[0], u["addr"].split(":")[-1]) for u in route.get("upstreams", [])) | |
| 90 | + return cur == sorted(want) | |
| 91 | + | |
| 92 | + | |
| 93 | +def ensure_route(m, alias, log=print, st=None): | |
| 94 | + """Fait pointer https://<domaine> vers <alias>:<port> (et pose les redirections). Retourne le spec, ou None si l'app n'est pas publique.""" | |
| 95 | + s = spec(m, alias) | |
| 96 | + if not s: | |
| 97 | + return None | |
| 98 | + name, g = gateway(s["gateway"]) | |
| 99 | + cur = routes(name, st) | |
| 100 | + r = cur.get(s["domain"]) | |
| 101 | + if _route_matches(r, s["upstreams"]): | |
| 102 | + log(" tunnel %s : route https://%s → %s déjà en place" % (name, s["domain"], ", ".join(s["upstreams"]))) | |
| 103 | + else: | |
| 104 | + args = " ".join(shlex.quote(x) for x in s["upstreams"]) + (" --websocket" if s["websocket"] else "") | |
| 105 | + rc, out, err = gw_run(name, "sudo -n tunnelctl add %s %s" % (shlex.quote(s["domain"]), args), timeout=90) | |
| 106 | + if rc != 0: | |
| 107 | + raise TunnelError("route %s → %s refusée par %s : %s" % (s["domain"], args, name, (err or out).strip()[-200:])) | |
| 108 | + log(" tunnel %s : route https://%s → %s%s" % (name, s["domain"], ", ".join(s["upstreams"]), " (remplace %s)" % ", ".join(u["addr"] for u in r.get("upstreams", [])) if r else "")) | |
| 109 | + for red in s["redirects"]: | |
| 110 | + rr = cur.get(red) | |
| 111 | + if rr and rr.get("kind") == "redirect" and rr.get("target", "").rstrip("/").endswith(s["domain"]): | |
| 112 | + continue | |
| 113 | + rc, out, err = gw_run(name, "sudo -n tunnelctl redirect %s %s" % (shlex.quote(red), shlex.quote(s["domain"])), timeout=90) | |
| 114 | + log(" tunnel %s : redirection %s → %s %s" % (name, red, s["domain"], "ok" if rc == 0 else "ÉCHEC " + (err or out).strip()[-120:])) | |
| 115 | + return s | |
| 116 | + | |
| 117 | + | |
| 118 | +def remove_route(m, log=print): | |
| 119 | + """Retire la route (et ses redirections) d'une app retirée du cluster : le domaine tombe sur la page 404 MacLustr.""" | |
| 120 | + t = m.get("tunnel") or {} | |
| 121 | + domain = t.get("domain") or m.get("domain") | |
| 122 | + if not domain: | |
| 123 | + return False | |
| 124 | + name, g = gateway(t.get("gateway")) | |
| 125 | + for d in [domain] + list(t.get("redirects") or []): | |
| 126 | + rc, out, err = gw_run(name, "sudo -n tunnelctl rm %s" % shlex.quote(d), timeout=60) | |
| 127 | + log(" tunnel %s : route %s %s" % (name, d, "retirée" if rc == 0 else "ÉCHEC " + (err or out).strip()[-120:])) | |
| 128 | + return True | |
| 129 | + | |
| 130 | + | |
| 131 | +# ------------------------------------------------------------------- pairs --- | |
| 132 | + | |
| 133 | +def peer_state(alias, name=None, st=None): | |
| 134 | + """(présent, handshake_s) — handshake None = jamais.""" | |
| 135 | + p = peers(name, st).get(alias) | |
| 136 | + if not p: | |
| 137 | + return False, None | |
| 138 | + return True, p.get("handshakeS") | |
| 139 | + | |
| 140 | + | |
| 141 | +def peer(alias, name=None, log=print): | |
| 142 | + """Raccorde un Mac au hub : installe WireGuard (Homebrew) + LaunchDaemon, enregistre la clé sur la passerelle (= `mlt peer`).""" | |
| 143 | + from . import nodes | |
| 144 | + name, g = gateway(name) | |
| 145 | + ip = nodes.ip_of(alias) | |
| 146 | + if not ip: | |
| 147 | + raise TunnelError("nœud %s injoignable" % alias) | |
| 148 | + rc, out, err = gw_run(name, "awk -v a=%s '$1==a{print $2}' /etc/maclustr-tunnel/ipmap; sudo -n wg show wg0 public-key" % shlex.quote(alias)) | |
| 149 | + parts = out.split() | |
| 150 | + if rc != 0 or len(parts) < 2: | |
| 151 | + raise TunnelError("alias %s absent de /etc/maclustr-tunnel/ipmap sur %s (l'ajouter : « %s %s.<n> »)" % (alias, name, alias, g["subnet"])) | |
| 152 | + wg_ip, hub_pub = parts[0], parts[1] | |
| 153 | + pwf = config.sudo_pw_file(alias) | |
| 154 | + pw = open(pwf).read().strip() if os.path.exists(pwf) else None | |
| 155 | + if not pw: | |
| 156 | + raise TunnelError("mot de passe sudo de %s absent (%s)" % (alias, pwf)) | |
| 157 | + script = open(os.path.join(ASSETS, "wg-node-setup.sh")).read() | |
| 158 | + ssh.write_remote_file(ip, "/tmp/mld-wg-node-setup.sh", script, mode="755") | |
| 159 | + log(" tunnel %s : raccordement de %s (%s, %s) …" % (name, alias, wg_ip, g["iface"])) | |
| 160 | + rc, out, err = ssh.run(ip, "bash /tmp/mld-wg-node-setup.sh %s %s %s %s %s; rm -f /tmp/mld-wg-node-setup.sh" % (wg_ip, hub_pub, shlex.quote(pw), g["iface"], g["endpoint"]), timeout=1200) | |
| 161 | + pub = [l.split()[-1] for l in out.splitlines() if "PUBKEY" in l] | |
| 162 | + if not pub: | |
| 163 | + raise TunnelError("wg-node-setup.sh sur %s n'a pas renvoyé de clé publique : %s" % (alias, (err or out).strip()[-300:])) | |
| 164 | + rc, out, err = gw_run(name, "sudo -n tunnelctl peer add %s %s" % (shlex.quote(alias), shlex.quote(pub[-1])), timeout=60) | |
| 165 | + if rc != 0: | |
| 166 | + raise TunnelError("tunnelctl peer add %s : %s" % (alias, (err or out).strip()[-200:])) | |
| 167 | + for _ in range(12): | |
| 168 | + time.sleep(4) | |
| 169 | + present, hs = peer_state(alias, name) | |
| 170 | + if present and hs is not None and hs < 60: | |
| 171 | + log(" tunnel %s : %s raccordé (%s, handshake %ss)" % (name, alias, wg_ip, hs)) | |
| 172 | + return True | |
| 173 | + log(" tunnel %s : %s enregistré (%s), handshake pas encore vu — keepalive 25 s" % (name, alias, wg_ip)) | |
| 174 | + return True | |
| 175 | + | |
| 176 | + | |
| 177 | +def kick_node_iface(alias, ip, name=None, log=print): | |
| 178 | + """Relance le LaunchDaemon WireGuard du nœud (interface tombée après un réveil réseau, par ex.).""" | |
| 179 | + name, g = gateway(name) | |
| 180 | + pwf = config.sudo_pw_file(alias) | |
| 181 | + pw = open(pwf).read().strip() if os.path.exists(pwf) else None | |
| 182 | + if not pw: | |
| 183 | + return False | |
| 184 | + lbl = "io.maclustr.wireguard-%s" % g["iface"] | |
| 185 | + rc, out, err = ssh.run(ip, "test -f /Library/LaunchDaemons/%s.plist || exit 9; echo %s | sudo -S -p '' launchctl kickstart -k system/%s && echo kicked" % (lbl, shlex.quote(pw), lbl), timeout=60) | |
| 186 | + if rc == 9: | |
| 187 | + return False | |
| 188 | + log(" tunnel %s : %s %s" % (name, lbl, "relancé sur " + alias if "kicked" in out else "ÉCHEC " + (err or out).strip()[-100:])) | |
| 189 | + return "kicked" in out | |
| 190 | + | |
| 191 | + | |
| 192 | +def ensure_peer(alias, name=None, log=print, st=None): | |
| 193 | + """Nœud raccordé et vivant sur le hub ; sinon relance son interface, sinon le raccorde. Retourne True si OK.""" | |
| 194 | + from . import nodes | |
| 195 | + name, g = gateway(name) | |
| 196 | + present, hs = peer_state(alias, name, st) | |
| 197 | + if present and hs is not None and hs < HANDSHAKE_OK_S: | |
| 198 | + return True | |
| 199 | + if present: | |
| 200 | + ip = nodes.ip_of(alias, rediscover=False) | |
| 201 | + if ip and kick_node_iface(alias, ip, name, log): | |
| 202 | + for _ in range(8): | |
| 203 | + time.sleep(4) | |
| 204 | + present, hs = peer_state(alias, name) | |
| 205 | + if hs is not None and hs < 60: | |
| 206 | + return True | |
| 207 | + if ip and ssh.ok(ip, "test -f /Library/LaunchDaemons/io.maclustr.wireguard-%s.plist" % g["iface"]): | |
| 208 | + log(" tunnel %s : %s enregistré mais sans handshake récent (%s)" % (name, alias, "jamais" if hs is None else "%ss" % hs)) | |
| 209 | + return False | |
| 210 | + return peer(alias, name, log) | |
| 211 | + | |
| 212 | + | |
| 213 | +# ------------------------------------------------------------------ affichage --- | |
| 214 | + | |
| 215 | +def print_status(name=None): | |
| 216 | + name, g = gateway(name) | |
| 217 | + st = state(name) | |
| 218 | + now = st.get("ts") or int(time.time()) | |
| 219 | + print("== %s (%s, %s) — WireGuard udp %s, %d pairs" % (name, g["host"], g["place"], st["wg"].get("listenPort"), len(st["wg"]["peers"]))) | |
| 220 | + for p in sorted(st["wg"]["peers"], key=lambda x: x["ip"]): | |
| 221 | + hs = p.get("handshakeS") | |
| 222 | + flag = "" if hs is not None and hs < HANDSHAKE_OK_S else " <<< pas de handshake récent" | |
| 223 | + print(" %-8s %-12s handshake %-8s rx %8.1f Mo tx %8.1f Mo%s" % (p["alias"], p["ip"], "jamais" if hs is None else "%ss" % hs, p["rxBytes"] / 1048576.0, p["txBytes"] / 1048576.0, flag)) | |
| 224 | + print("== Caddy : %s — %d routes" % ("actif" if st["caddy"]["active"] else "ARRÊTÉ", len(st["caddy"]["routes"]))) | |
| 225 | + for r in st["caddy"]["routes"]: | |
| 226 | + if r["kind"] == "redirect": | |
| 227 | + print(" %-36s → redirection %s" % (r["domain"], r.get("target"))) | |
| 228 | + else: | |
| 229 | + print(" %-36s → %s" % (r["domain"], ", ".join("%s (%s)" % (u["addr"], u.get("alias") or "?") for u in r["upstreams"]))) | |
| 230 | ||