"""Cycle de vie d'une app : fetch (nœud -> staging), deploy (staging -> nœud), stop, start, retire, move, health. Exposition publique = MacLustr Tunnel (module tunnel) : au deploy, la route Caddy https:// de la passerelle est pointée vers le nœud choisi (après le healthcheck local), donc elle suit automatiquement les `mld move`.""" import json import os import shlex import time from . import config, manifest, nodes, registry, render, ssh, tunnel SYNC_BASE_EXCLUDES = [".DS_Store", "*.log", "__pycache__/", ".pytest_cache/", ".mypy_cache/"] class DeployError(Exception): pass def _ctx(alias, ip): lan = nodes.load_lan()["ips"] return {"HOME": config.home_of(ip), "NODE": alias, "LAN_IP": ip, "ips": lan, "USER": config.user_of(ip)} def _abs(p, ip=None): """~ → home du nœud (dépend de l'utilisateur : simon-pierreboucher, ou celui d'un REMOTE_NODE).""" return p.replace("~", config.home_of(ip), 1) if p.startswith("~") else p def _log(app, msg): line = "%s %s" % (time.strftime("%H:%M:%S"), msg) print(line, flush=True) with open(os.path.join(config.LOG_DIR, app + ".log"), "a") as f: f.write(time.strftime("%Y-%m-%d ") + line + "\n") def _ip(alias): ip = nodes.ip_of(alias) if not ip: raise DeployError("nœud %s injoignable sur le LAN" % alias) return ip def _ensure_dir(ip, d): """Crée le répertoire cible ; sous /opt (racine root) passe par sudo -S avec le mot de passe de ~/dispatch/.sudo.""" if ssh.ok(ip, "mkdir -p %s 2>/dev/null && test -w %s" % (shlex.quote(d), shlex.quote(d))): return pwf = config.sudo_pw_file(ip) pw = open(pwf).read().strip() if os.path.exists(pwf) else None if not pw: raise DeployError("impossible de créer %s (sudo requis, %s absent)" % (d, pwf)) rc, out, err = ssh.run(ip, "echo %s | sudo -S -p '' mkdir -p %s && echo %s | sudo -S -p '' chown %s %s" % (shlex.quote(pw), shlex.quote(d), shlex.quote(pw), config.user_of(ip), shlex.quote(d))) if rc != 0: raise DeployError("création de %s impossible : %s" % (d, err.strip()[-200:])) def stage_dir(app): return os.path.join(config.STAGE_DIR, app) # ---------------------------------------------------------------- fetch : nœud -> staging def fetch(app, from_alias, final=False): m = manifest.load(app) ip = _ip(from_alias) _log(app, "fetch %s depuis %s (%s)%s" % (m["dir"], from_alias, ip, " [final]" if final else "")) excl = SYNC_BASE_EXCLUDES + list(m.get("sync_excludes") or []) rc, out, err = ssh.rsync_pull(ip, _abs(m["dir"], ip), os.path.join(stage_dir(app), "dir"), excludes=excl) if rc == 23 and not final: _log(app, " avertissement pré-copie à chaud (fichiers en cours d'écriture, repris au delta final) : %s" % err.strip().splitlines()[0][:120]) elif rc not in (0, 24): raise DeployError("rsync fetch a échoué (%s): %s" % (rc, err.strip()[-300:])) for extra in m.get("extra_paths") or []: dest = os.path.join(stage_dir(app), "extra", extra.strip("~/").replace("/", "__")) rc2, out2, err2 = ssh.run(ip, "test -e %s" % shlex.quote(_abs(extra, ip))) if rc2 != 0: _log(app, " extra absent sur la source, ignoré : %s" % extra) continue isdir = ssh.ok(ip, "test -d %s" % shlex.quote(_abs(extra, ip))) if isdir: rc2, out2, err2 = ssh.rsync_pull(ip, _abs(extra, ip), dest, delete=True) else: os.makedirs(dest, exist_ok=True) import subprocess subprocess.run(["scp", "-p"] + ssh.SSH_OPTS + ["%s:%s" % (ssh.target(ip), _abs(extra, ip)), os.path.join(dest, os.path.basename(extra))], capture_output=True) json.dump({"app": app, "from": from_alias, "ip": ip, "ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "final": final}, open(os.path.join(stage_dir(app), "fetch.json"), "w"), indent=2) stat = [l for l in out.splitlines() if "Total transferred file size" in l or "Number of regular files transferred" in l or "Number of files transferred" in l or "Total bytes sent" in l] _log(app, " ok — " + " ; ".join(s.strip() for s in stat)) return True # ---------------------------------------------------------------- stop / start / status sur un nœud def stop(app, alias, keep_dir=True): m = manifest.load(app) ip = _ip(alias) _log(app, "stop sur %s" % alias) names = manifest.pm2_names(m) + manifest.legacy_pm2_names(m) script = "" if names: script += "for n in %s; do pm2 delete $n >/dev/null 2>&1 && echo \" pm2 delete $n\"; done; pm2 save --force >/dev/null 2>&1\n" % " ".join(names) for lbl in manifest.launchd_labels(m): script += "launchctl bootout gui/$(id -u)/%s >/dev/null 2>&1 && echo ' bootout %s'; rm -f ~/Library/LaunchAgents/%s.plist\n" % (lbl, lbl, lbl) for hook in (m.get("hooks") or {}).get("pre_stop") or []: script += hook + "\n" # tue ce qui écoute encore sur les ports de l'app (orphelins) for p in m["requires"].get("ports") or []: script += "sleep 1; for pid in $(lsof -nP -iTCP:%s -sTCP:LISTEN -t 2>/dev/null); do kill $pid 2>/dev/null && echo ' kill port %s pid '$pid; done\n" % (p, p) rc, out, err = ssh.run(ip, script, timeout=180) for l in out.splitlines(): _log(app, l) registry.log(app, "stopped", node=alias) return rc == 0 def _install_hooks(m, ip, ctx): """Commandes post-sync : réécritures .env, hooks, inscription pousseur.""" script = "" d = _abs(m["dir"], ip) for fname, kv in (m.get("env_overrides") or {}).items(): target = os.path.join(d, fname) kv = manifest.render(kv, ctx) py = "import re,sys,os\np=%r\nkv=%r\ns=open(p).read() if os.path.exists(p) else ''\nfor k,v in kv.items():\n line=k+'='+v\n if re.search(r'^'+re.escape(k)+r'=.*$', s, re.M): s=re.sub(r'^'+re.escape(k)+r'=.*$', line.replace('\\\\','\\\\\\\\'), s, flags=re.M)\n else: s=s.rstrip('\\n')+'\\n'+line+'\\n'\nopen(p,'w').write(s)\nprint(' .env réécrit: '+', '.join(kv))" % (target, kv) script += "python3 - <<'PYEOF'\n%s\nPYEOF\n" % py if m.get("ka_repo"): script += "grep -qx %s ~/.ka-pousseur-repos 2>/dev/null || echo %s >> ~/.ka-pousseur-repos; echo ' pousseur: %s inscrit'\n" % (shlex.quote(m["dir"]), shlex.quote(m["dir"]), m["dir"]) for hook in (m.get("hooks") or {}).get("post_sync") or []: script += "cd %s && { %s ; }\n" % (shlex.quote(d), manifest.render(hook, ctx)) return script def deploy(app, alias, reinstall=False, skip_sync=False): """staging -> nœud, puis démarrage (PM2 + launchd) et healthcheck.""" m = manifest.load(app) ip = _ip(alias) ctx = _ctx(alias, ip) sd = os.path.join(stage_dir(app), "dir") if not skip_sync and not os.path.isdir(sd): raise DeployError("staging vide pour %s — faire `mld fetch %s --from ` ou `mld stage`" % (app, app)) _log(app, "deploy → %s (%s)" % (alias, ip)) # 1. runtimes scan = nodes.load_scan().get(alias) or nodes.probe(alias, ip) missing = [r for r in m["requires"]["runtimes"] if not scan.get("runtimes", {}).get(r)] if missing: raise DeployError("runtimes manquants sur %s : %s (→ `mld prepare %s`)" % (alias, ", ".join(missing), alias)) # 2. sync d = _abs(m["dir"], ip) if not skip_sync: _ensure_dir(ip, d) excl = SYNC_BASE_EXCLUDES + list(m.get("sync_excludes") or []) rc, out, err = ssh.rsync_push(sd, ip, d, excludes=excl, delete=True) if rc not in (0, 24): raise DeployError("rsync deploy a échoué (%s): %s" % (rc, err.strip()[-300:])) stat = [l.strip() for l in out.splitlines() if "Total transferred file size" in l or "Total bytes sent" in l] _log(app, " sync ok — %s" % (stat[0] if stat else "")) for extra in m.get("extra_paths") or []: src = os.path.join(stage_dir(app), "extra", extra.strip("~/").replace("/", "__")) if not os.path.exists(src): continue if os.path.isdir(src) and not os.path.exists(os.path.join(src, os.path.basename(extra))): ssh.rsync_push(src, ip, _abs(extra, ip), delete=False) else: import subprocess ssh.run(ip, "mkdir -p %s" % shlex.quote(os.path.dirname(_abs(extra, ip)))) subprocess.run(["scp", "-p"] + ssh.SSH_OPTS + [os.path.join(src, os.path.basename(extra)), "%s:%s" % (ssh.target(ip), _abs(extra, ip))], capture_output=True) _log(app, " extra: %s" % extra) # 3. install (optionnel) + hooks script = "cd %s || exit 3\n" % shlex.quote(d) if reinstall: script += "if [ -f pnpm-lock.yaml ]; then pnpm install --frozen-lockfile; elif [ -f package-lock.json ]; then npm ci; elif [ -f package.json ]; then npm install; fi\n" script += "if [ -f requirements.txt ] && [ -x .venv/bin/pip ]; then .venv/bin/pip install -q -r requirements.txt; fi\n" elif m.get("rebuild", True) and os.path.exists(os.path.join(sd, "package.json")) and os.path.isdir(os.path.join(sd, "node_modules")): script += "npm rebuild >/dev/null 2>&1 && echo ' npm rebuild ok' || echo ' npm rebuild: avertissement'\n" script += _install_hooks(m, ip, ctx) rc, out, err = ssh.run(ip, script, timeout=1800) for l in out.splitlines()[-15:]: _log(app, l) if rc != 0: raise DeployError("post-sync a échoué (%s): %s" % (rc, (err or out).strip()[-400:])) # 4. lancement start(app, alias, m=m, ctx=ctx, ip=ip) # 5. santé locale, puis route publique (MacLustr Tunnel) pointée vers CE nœud, puis santé publique local_ok, code = _health_local(ip, m) tun = None if local_ok and tunnel.spec(m, alias): tun = _publish(app, alias, m, ip) healthy, detail = health(app, alias, ip=ip, m=m, timeout=30, public=bool(tun)) registry.set_app(app, node=alias, ip=ip, port=m.get("port"), domain=m.get("domain"), dir=m["dir"], label=m.get("label") or app, health_path=m.get("health_path") or "/", processes=manifest.pm2_names(m), launchd=manifest.launchd_labels(m), deployed=time.strftime("%Y-%m-%d %H:%M"), tunnel=tun, status="online" if healthy else "unhealthy", health=detail) registry.log(app, "deployed", node=alias, healthy=healthy) _log(app, " santé : %s" % detail) if not healthy: names = manifest.pm2_names(m) rc, out, _ = ssh.run(ip, "pm2 logs %s --nostream --lines 6 2>/dev/null | grep -v '^\\[TAILING' | tail -12" % " ".join(names), timeout=40) for l in out.splitlines(): _log(app, " log: " + l[:160]) raise DeployError("healthcheck KO sur %s : %s" % (alias, detail)) return True def start(app, alias, m=None, ctx=None, ip=None): m = m or manifest.load(app) ip = ip or _ip(alias) ctx = ctx or _ctx(alias, ip) _log(app, "start sur %s" % alias) run_dir = _abs(config.RUN_DIR, ip) eco = render.ecosystem(m, ctx) eco_path = "%s/%s.config.cjs" % (run_dir, app) ssh.write_remote_file(ip, eco_path, eco, mode="600") script = "mkdir -p %s\n" % run_dir names = manifest.pm2_names(m) if names or manifest.legacy_pm2_names(m): script += "for n in %s; do pm2 delete $n >/dev/null 2>&1; done\n" % " ".join(names + manifest.legacy_pm2_names(m)) if names: script += "pm2 start %s --update-env 2>&1 | grep -E 'error|Error|\\[PM2\\]' | head -5; pm2 save --force >/dev/null 2>&1; pm2 jlist | python3 -c \"import json,sys; L=json.load(sys.stdin); print(' pm2: '+', '.join(p['name']+'='+p['pm2_env']['status'] for p in L if p['name'] in %s))\" 2>/dev/null || pm2 ls | grep -E '%s'\n" % (shlex.quote(eco_path), json.dumps(names), "|".join(names)) for item in m.get("launchd") or []: pl = render.plist(item, ctx) ppath = "%s/Library/LaunchAgents/%s.plist" % (ctx["HOME"], item["label"]) ssh.write_remote_file(ip, ppath, pl, mode="644") script += "launchctl bootout gui/$(id -u)/%s >/dev/null 2>&1; launchctl bootstrap gui/$(id -u) %s && echo ' launchd: %s chargé' || echo ' launchd: %s ÉCHEC'\n" % (item["label"], shlex.quote(ppath), item["label"], item["label"]) for hook in (m.get("hooks") or {}).get("post_start") or []: script += "cd %s && { %s ; }\n" % (shlex.quote(_abs(m["dir"], ip)), manifest.render(hook, ctx)) rc, out, err = ssh.run(ip, script, timeout=300) for l in out.splitlines(): _log(app, l) if err.strip(): _log(app, " stderr: " + err.strip()[-200:]) return rc == 0 def _publish(app, alias, m, ip): """Raccorde le nœud au tunnel si besoin et pointe la route publique de l'app dessus. Retourne le spec de route ou None.""" log = lambda s: _log(app, s) try: st = None try: st = tunnel.state((m.get("tunnel") or {}).get("gateway")) except tunnel.TunnelError: pass if not tunnel.ensure_peer(alias, (m.get("tunnel") or {}).get("gateway"), log=log, st=st): _log(app, " tunnel : %s n'a pas de handshake avec la passerelle — route pointée quand même" % alias) s = tunnel.ensure_route(m, alias, log=log) if not s: return None return {"gateway": s["gateway"], "domain": s["domain"], "upstream": s["upstreams"][0], "url": "https://%s" % s["domain"]} except tunnel.TunnelError as e: _log(app, " tunnel : %s" % e) return None def _health_local(ip, m, timeout=90): """(ok, code|texte) : HTTP local sur le port de l'app, ou processus PM2 online si l'app n'a pas de port.""" if not m.get("port"): names = manifest.pm2_names(m) if not names: return True, "aucun port/processus à vérifier" rc, out, _ = ssh.run(ip, "pm2 jlist | python3 -c \"import json,sys; L=json.load(sys.stdin); print(sum(1 for p in L if p['name'] in %s and p['pm2_env']['status']=='online'))\"" % json.dumps(names)) n = int((out.strip() or "0").splitlines()[-1] or 0) return n == len(names), "%d/%d processus PM2 online" % (n, len(names)) url = "http://127.0.0.1:%s%s" % (m["port"], m.get("health_path") or "/") deadline = time.time() + timeout code = "000" while True: rc, out, _ = ssh.run(ip, "curl -s -o /dev/null -m 8 -w '%%{http_code}' %s" % shlex.quote(url), timeout=20) code = out.strip()[-3:] if out.strip() else "000" if code.isdigit() and code not in ("000",) and int(code) < 500: break if time.time() >= deadline: break time.sleep(4) return code.isdigit() and code != "000" and int(code) < 500, code def _health_public(m, tries=20): """Code HTTP de https:// à travers la passerelle (tries × 5 s max).""" import subprocess pub = "" for _ in range(tries): p = subprocess.run(["curl", "-s", "-o", "/dev/null", "-m", "12", "-w", "%{http_code}", "https://%s%s" % (m["domain"], m.get("health_path") or "/")], capture_output=True, text=True) pub = p.stdout.strip() if pub.isdigit() and int(pub) < 500 and pub != "404": break time.sleep(5) return pub def health(app, alias, ip=None, m=None, timeout=90, public=True): """Santé locale (port/PM2) puis publique (via MacLustr Tunnel) si l'app a un domaine. Retourne (ok, détail).""" m = m or manifest.load(app) ip = ip or _ip(alias) local_ok, code = _health_local(ip, m, timeout=timeout) if not m.get("port"): return local_ok, code pub = "" if public and m.get("domain") and local_ok and tunnel.spec(m, alias): pub = _health_public(m, tries=max(2, min(20, timeout // 5))) detail = "local %s" % code + (" / public https://%s %s" % (m["domain"], pub) if pub else "") ok = local_ok and (not pub or (pub.isdigit() and int(pub) < 500 and pub != "404")) return ok, detail def retire(app, alias, remove_dir=True): """Arrête et efface la copie d'un nœud (après migration réussie).""" m = manifest.load(app) ip = _ip(alias) stop(app, alias) if remove_dir: d = _abs(m["dir"], ip) rc, out, err = ssh.run(ip, "rm -rf %s && echo ' rm -rf %s'; rm -f %s/%s.config.cjs; grep -v -x %s ~/.ka-pousseur-repos > ~/.ka-pousseur-repos.tmp 2>/dev/null && mv ~/.ka-pousseur-repos.tmp ~/.ka-pousseur-repos; true" % (shlex.quote(d), d, _abs(config.RUN_DIR, ip), app, shlex.quote(m["dir"])), timeout=600) for l in out.splitlines(): _log(app, l) registry.log(app, "retired", node=alias) return True def move(app, to_alias, from_alias=None, keep_source=False, reinstall=False): """Migration complète : pré-sync à chaud, arrêt, sync final, déploiement, santé, retrait de la source (ou rollback).""" from_alias = from_alias or registry.node_of(app) or manifest.load(app).get("source_node") if not from_alias: raise DeployError("nœud source inconnu pour %s (préciser --from)" % app) if from_alias == to_alias: _log(app, "déjà sur %s → redéploiement en place" % to_alias) fetch(app, from_alias) stop(app, from_alias) fetch(app, from_alias, final=True) return deploy(app, to_alias, reinstall=reinstall) _log(app, "MOVE %s : %s → %s" % (app, from_alias, to_alias)) fetch(app, from_alias) # 1. copie à chaud (gros volume) stop(app, from_alias) # 2. arrêt source fetch(app, from_alias, final=True) # 3. delta final (cohérence des bases) try: deploy(app, to_alias, reinstall=reinstall) # 4. cible except DeployError as e: _log(app, "ÉCHEC sur %s : %s — rollback sur %s" % (to_alias, e, from_alias)) try: stop(app, to_alias) except Exception: pass start(app, from_alias) m = manifest.load(app) ip0 = _ip(from_alias) tun = _publish(app, from_alias, m, ip0) if tunnel.spec(m, from_alias) else None # la route publique revient sur la source ok, det = health(app, from_alias, ip=ip0, m=m) registry.set_app(app, node=from_alias, ip=ip0, tunnel=tun, status="online" if ok else "unhealthy", health=det) raise if not keep_source: retire(app, from_alias) # 5. nettoyage source registry.log(app, "moved", **{"from": from_alias, "to": to_alias}) _log(app, "MOVE terminé : %s tourne sur %s" % (app, to_alias)) return True def status(app=None): r = registry.load() apps = [app] if app else sorted(r["apps"]) rows = [] for a in apps: v = r["apps"].get(a) if not v: rows.append((a, "—", "", "", "non déployée", "")) continue rows.append((a, v.get("node"), v.get("port"), v.get("domain") or "", v.get("status"), v.get("deployed"))) return rows def live_status(app=None, public=True): """Vérifie en live (PM2 + port + domaine via le tunnel) chaque app du registre. Une seule sauvegarde du registre à la fin.""" r = registry.load() apps = [app] if app else sorted(r["apps"]) out = [] changes = {} for a in apps: v = r["apps"].get(a) if not v: continue m = manifest.load(a) try: ip = _ip(v["node"]) ok, det = health(a, v["node"], ip=ip, m=m, timeout=10, public=public) except DeployError as e: ok, det = False, str(e) changes[a] = {"status": "online" if ok else "unhealthy", "health": det} out.append((a, v["node"], v.get("domain") or "", "OK" if ok else "KO", det)) registry.update_many(changes) return out