spb/maclustr-dispatch

1"""Cycle de vie d'une app : fetch (nœud -> staging), deploy (staging -> nœud), stop, start, retire, move, health.23Exposition publique = MacLustr Tunnel (module tunnel) : au deploy, la route Caddy https://<domaine> de la passerelle est4pointée vers le nœud choisi (après le healthcheck local), donc elle suit automatiquement les `mld move`."""5import json6import os7import shlex8import time9from . import config, manifest, nodes, registry, render, ssh, tunnel1011SYNC_BASE_EXCLUDES = [".DS_Store", "*.log", "__pycache__/", ".pytest_cache/", ".mypy_cache/"]121314class DeployError(Exception):15    pass161718def _ctx(alias, ip):19    lan = nodes.load_lan()["ips"]20    return {"HOME": config.home_of(ip), "NODE": alias, "LAN_IP": ip, "ips": lan, "USER": config.user_of(ip)}212223def _abs(p, ip=None):24    """~ → home du nœud (dépend de l'utilisateur : simon-pierreboucher, ou celui d'un REMOTE_NODE)."""25    return p.replace("~", config.home_of(ip), 1) if p.startswith("~") else p262728def _log(app, msg):29    line = "%s %s" % (time.strftime("%H:%M:%S"), msg)30    print(line, flush=True)31    with open(os.path.join(config.LOG_DIR, app + ".log"), "a") as f:32        f.write(time.strftime("%Y-%m-%d ") + line + "\n")333435def _ip(alias):36    ip = nodes.ip_of(alias)37    if not ip:38        raise DeployError("nœud %s injoignable sur le LAN" % alias)39    return ip404142def _ensure_dir(ip, d):43    """Crée le répertoire cible ; sous /opt (racine root) passe par sudo -S avec le mot de passe de ~/dispatch/.sudo."""44    if ssh.ok(ip, "mkdir -p %s 2>/dev/null && test -w %s" % (shlex.quote(d), shlex.quote(d))):45        return46    pwf = config.sudo_pw_file(ip)47    pw = open(pwf).read().strip() if os.path.exists(pwf) else None48    if not pw:49        raise DeployError("impossible de créer %s (sudo requis, %s absent)" % (d, pwf))50    rc, out, err = ssh.run(ip, "echo %s | sudo -S -p '' mkdir -p %s && echo %s | sudo -S -p '' chown %s %s" % (shlex.quote(pw), shlex.quote(d), shlex.quote(pw), config.user_of(ip), shlex.quote(d)))51    if rc != 0:52        raise DeployError("création de %s impossible : %s" % (d, err.strip()[-200:]))535455def stage_dir(app):56    return os.path.join(config.STAGE_DIR, app)575859# ---------------------------------------------------------------- fetch : nœud -> staging60def fetch(app, from_alias, final=False):61    m = manifest.load(app)62    ip = _ip(from_alias)63    _log(app, "fetch %s depuis %s (%s)%s" % (m["dir"], from_alias, ip, " [final]" if final else ""))64    excl = SYNC_BASE_EXCLUDES + list(m.get("sync_excludes") or [])65    rc, out, err = ssh.rsync_pull(ip, _abs(m["dir"], ip), os.path.join(stage_dir(app), "dir"), excludes=excl)66    if rc == 23 and not final:67        _log(app, "  avertissement pré-copie à chaud (fichiers en cours d'écriture, repris au delta final) : %s" % err.strip().splitlines()[0][:120])68    elif rc not in (0, 24):69        raise DeployError("rsync fetch a échoué (%s): %s" % (rc, err.strip()[-300:]))70    for extra in m.get("extra_paths") or []:71        dest = os.path.join(stage_dir(app), "extra", extra.strip("~/").replace("/", "__"))72        rc2, out2, err2 = ssh.run(ip, "test -e %s" % shlex.quote(_abs(extra, ip)))73        if rc2 != 0:74            _log(app, "  extra absent sur la source, ignoré : %s" % extra)75            continue76        isdir = ssh.ok(ip, "test -d %s" % shlex.quote(_abs(extra, ip)))77        if isdir:78            rc2, out2, err2 = ssh.rsync_pull(ip, _abs(extra, ip), dest, delete=True)79        else:80            os.makedirs(dest, exist_ok=True)81            import subprocess82            subprocess.run(["scp", "-p"] + ssh.SSH_OPTS + ["%s:%s" % (ssh.target(ip), _abs(extra, ip)), os.path.join(dest, os.path.basename(extra))], capture_output=True)83    json.dump({"app": app, "from": from_alias, "ip": ip, "ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "final": final},84              open(os.path.join(stage_dir(app), "fetch.json"), "w"), indent=2)85    stat = [l for l in out.splitlines() if "Total transferred file size" in l or "Number of regular files transferred" in l or "Number of files transferred" in l or "Total bytes sent" in l]86    _log(app, "  ok — " + " ; ".join(s.strip() for s in stat))87    return True888990# ---------------------------------------------------------------- stop / start / status sur un nœud91def stop(app, alias, keep_dir=True):92    m = manifest.load(app)93    ip = _ip(alias)94    _log(app, "stop sur %s" % alias)95    names = manifest.pm2_names(m) + manifest.legacy_pm2_names(m)96    script = ""97    if names:98        script += "for n in %s; do pm2 delete $n >/dev/null 2>&1 && echo \"  pm2 delete $n\"; done; pm2 save --force >/dev/null 2>&1\n" % " ".join(names)99    for lbl in manifest.launchd_labels(m):100        script += "launchctl bootout gui/$(id -u)/%s >/dev/null 2>&1 && echo '  bootout %s'; rm -f ~/Library/LaunchAgents/%s.plist\n" % (lbl, lbl, lbl)101    for hook in (m.get("hooks") or {}).get("pre_stop") or []:102        script += hook + "\n"103    # tue ce qui écoute encore sur les ports de l'app (orphelins)104    for p in m["requires"].get("ports") or []:105        script += "sleep 1; for pid in $(lsof -nP -iTCP:%s -sTCP:LISTEN -t 2>/dev/null); do kill $pid 2>/dev/null && echo '  kill port %s pid '$pid; done\n" % (p, p)106    rc, out, err = ssh.run(ip, script, timeout=180)107    for l in out.splitlines():108        _log(app, l)109    registry.log(app, "stopped", node=alias)110    return rc == 0111112113def _install_hooks(m, ip, ctx):114    """Commandes post-sync : réécritures .env, hooks, inscription pousseur."""115    script = ""116    d = _abs(m["dir"], ip)117    for fname, kv in (m.get("env_overrides") or {}).items():118        target = os.path.join(d, fname)119        kv = manifest.render(kv, ctx)120        py = "import re,sys,os\np=%r\nkv=%r\ns=open(p).read() if os.path.exists(p) else ''\nfor k,v in kv.items():\n    line=k+'='+v\n    if re.search(r'^'+re.escape(k)+r'=.*$', s, re.M): s=re.sub(r'^'+re.escape(k)+r'=.*$', line.replace('\\\\','\\\\\\\\'), s, flags=re.M)\n    else: s=s.rstrip('\\n')+'\\n'+line+'\\n'\nopen(p,'w').write(s)\nprint('  .env réécrit: '+', '.join(kv))" % (target, kv)121        script += "python3 - <<'PYEOF'\n%s\nPYEOF\n" % py122    if m.get("ka_repo"):123        script += "grep -qx %s ~/.ka-pousseur-repos 2>/dev/null || echo %s >> ~/.ka-pousseur-repos; echo '  pousseur: %s inscrit'\n" % (shlex.quote(m["dir"]), shlex.quote(m["dir"]), m["dir"])124    for hook in (m.get("hooks") or {}).get("post_sync") or []:125        script += "cd %s && { %s ; }\n" % (shlex.quote(d), manifest.render(hook, ctx))126    return script127128129def deploy(app, alias, reinstall=False, skip_sync=False):130    """staging -> nœud, puis démarrage (PM2 + launchd) et healthcheck."""131    m = manifest.load(app)132    ip = _ip(alias)133    ctx = _ctx(alias, ip)134    sd = os.path.join(stage_dir(app), "dir")135    if not skip_sync and not os.path.isdir(sd):136        raise DeployError("staging vide pour %s — faire `mld fetch %s --from <nœud>` ou `mld stage`" % (app, app))137    _log(app, "deploy → %s (%s)" % (alias, ip))138    # 1. runtimes139    scan = nodes.load_scan().get(alias) or nodes.probe(alias, ip)140    missing = [r for r in m["requires"]["runtimes"] if not scan.get("runtimes", {}).get(r)]141    if missing:142        raise DeployError("runtimes manquants sur %s : %s (→ `mld prepare %s`)" % (alias, ", ".join(missing), alias))143    # 2. sync144    d = _abs(m["dir"], ip)145    if not skip_sync:146        _ensure_dir(ip, d)147        excl = SYNC_BASE_EXCLUDES + list(m.get("sync_excludes") or [])148        rc, out, err = ssh.rsync_push(sd, ip, d, excludes=excl, delete=True)149        if rc not in (0, 24):150            raise DeployError("rsync deploy a échoué (%s): %s" % (rc, err.strip()[-300:]))151        stat = [l.strip() for l in out.splitlines() if "Total transferred file size" in l or "Total bytes sent" in l]152        _log(app, "  sync ok — %s" % (stat[0] if stat else ""))153        for extra in m.get("extra_paths") or []:154            src = os.path.join(stage_dir(app), "extra", extra.strip("~/").replace("/", "__"))155            if not os.path.exists(src):156                continue157            if os.path.isdir(src) and not os.path.exists(os.path.join(src, os.path.basename(extra))):158                ssh.rsync_push(src, ip, _abs(extra, ip), delete=False)159            else:160                import subprocess161                ssh.run(ip, "mkdir -p %s" % shlex.quote(os.path.dirname(_abs(extra, ip))))162                subprocess.run(["scp", "-p"] + ssh.SSH_OPTS + [os.path.join(src, os.path.basename(extra)), "%s:%s" % (ssh.target(ip), _abs(extra, ip))], capture_output=True)163            _log(app, "  extra: %s" % extra)164    # 3. install (optionnel) + hooks165    script = "cd %s || exit 3\n" % shlex.quote(d)166    if reinstall:167        script += "if [ -f pnpm-lock.yaml ]; then pnpm install --frozen-lockfile; elif [ -f package-lock.json ]; then npm ci; elif [ -f package.json ]; then npm install; fi\n"168        script += "if [ -f requirements.txt ] && [ -x .venv/bin/pip ]; then .venv/bin/pip install -q -r requirements.txt; fi\n"169    elif m.get("rebuild", True) and os.path.exists(os.path.join(sd, "package.json")) and os.path.isdir(os.path.join(sd, "node_modules")):170        script += "npm rebuild >/dev/null 2>&1 && echo '  npm rebuild ok' || echo '  npm rebuild: avertissement'\n"171    script += _install_hooks(m, ip, ctx)172    rc, out, err = ssh.run(ip, script, timeout=1800)173    for l in out.splitlines()[-15:]:174        _log(app, l)175    if rc != 0:176        raise DeployError("post-sync a échoué (%s): %s" % (rc, (err or out).strip()[-400:]))177    # 4. lancement178    start(app, alias, m=m, ctx=ctx, ip=ip)179    # 5. santé locale, puis route publique (MacLustr Tunnel) pointée vers CE nœud, puis santé publique180    local_ok, code = _health_local(ip, m)181    tun = None182    if local_ok and tunnel.spec(m, alias):183        tun = _publish(app, alias, m, ip)184    healthy, detail = health(app, alias, ip=ip, m=m, timeout=30, public=bool(tun))185    registry.set_app(app, node=alias, ip=ip, port=m.get("port"), domain=m.get("domain"), dir=m["dir"], label=m.get("label") or app,186                     health_path=m.get("health_path") or "/",187                     processes=manifest.pm2_names(m), launchd=manifest.launchd_labels(m), deployed=time.strftime("%Y-%m-%d %H:%M"),188                     tunnel=tun, status="online" if healthy else "unhealthy", health=detail)189    registry.log(app, "deployed", node=alias, healthy=healthy)190    _log(app, "  santé : %s" % detail)191    if not healthy:192        names = manifest.pm2_names(m)193        rc, out, _ = ssh.run(ip, "pm2 logs %s --nostream --lines 6 2>/dev/null | grep -v '^\\[TAILING' | tail -12" % " ".join(names), timeout=40)194        for l in out.splitlines():195            _log(app, "  log: " + l[:160])196        raise DeployError("healthcheck KO sur %s : %s" % (alias, detail))197    return True198199200def start(app, alias, m=None, ctx=None, ip=None):201    m = m or manifest.load(app)202    ip = ip or _ip(alias)203    ctx = ctx or _ctx(alias, ip)204    _log(app, "start sur %s" % alias)205    run_dir = _abs(config.RUN_DIR, ip)206    eco = render.ecosystem(m, ctx)207    eco_path = "%s/%s.config.cjs" % (run_dir, app)208    ssh.write_remote_file(ip, eco_path, eco, mode="600")209    script = "mkdir -p %s\n" % run_dir210    names = manifest.pm2_names(m)211    if names or manifest.legacy_pm2_names(m):212        script += "for n in %s; do pm2 delete $n >/dev/null 2>&1; done\n" % " ".join(names + manifest.legacy_pm2_names(m))213    if names:214        script += "pm2 start %s --update-env 2>&1 | grep -E 'error|Error|\\[PM2\\]' | head -5; pm2 save --force >/dev/null 2>&1; pm2 jlist | python3 -c \"import json,sys; L=json.load(sys.stdin); print('  pm2: '+', '.join(p['name']+'='+p['pm2_env']['status'] for p in L if p['name'] in %s))\" 2>/dev/null || pm2 ls | grep -E '%s'\n" % (shlex.quote(eco_path), json.dumps(names), "|".join(names))215    for item in m.get("launchd") or []:216        pl = render.plist(item, ctx)217        ppath = "%s/Library/LaunchAgents/%s.plist" % (ctx["HOME"], item["label"])218        ssh.write_remote_file(ip, ppath, pl, mode="644")219        script += "launchctl bootout gui/$(id -u)/%s >/dev/null 2>&1; launchctl bootstrap gui/$(id -u) %s && echo '  launchd: %s chargé' || echo '  launchd: %s ÉCHEC'\n" % (item["label"], shlex.quote(ppath), item["label"], item["label"])220    for hook in (m.get("hooks") or {}).get("post_start") or []:221        script += "cd %s && { %s ; }\n" % (shlex.quote(_abs(m["dir"], ip)), manifest.render(hook, ctx))222    rc, out, err = ssh.run(ip, script, timeout=300)223    for l in out.splitlines():224        _log(app, l)225    if err.strip():226        _log(app, "  stderr: " + err.strip()[-200:])227    return rc == 0228229230def _publish(app, alias, m, ip):231    """Raccorde le nœud au tunnel si besoin et pointe la route publique de l'app dessus. Retourne le spec de route ou None."""232    log = lambda s: _log(app, s)233    try:234        st = None235        try:236            st = tunnel.state((m.get("tunnel") or {}).get("gateway"))237        except tunnel.TunnelError:238            pass239        if not tunnel.ensure_peer(alias, (m.get("tunnel") or {}).get("gateway"), log=log, st=st):240            _log(app, "  tunnel : %s n'a pas de handshake avec la passerelle — route pointée quand même" % alias)241        s = tunnel.ensure_route(m, alias, log=log)242        if not s:243            return None244        return {"gateway": s["gateway"], "domain": s["domain"], "upstream": s["upstreams"][0], "url": "https://%s" % s["domain"]}245    except tunnel.TunnelError as e:246        _log(app, "  tunnel : %s" % e)247        return None248249250def _health_local(ip, m, timeout=90):251    """(ok, code|texte) : HTTP local sur le port de l'app, ou processus PM2 online si l'app n'a pas de port."""252    if not m.get("port"):253        names = manifest.pm2_names(m)254        if not names:255            return True, "aucun port/processus à vérifier"256        rc, out, _ = ssh.run(ip, "pm2 jlist | python3 -c \"import json,sys; L=json.load(sys.stdin); print(sum(1 for p in L if p['name'] in %s and p['pm2_env']['status']=='online'))\"" % json.dumps(names))257        n = int((out.strip() or "0").splitlines()[-1] or 0)258        return n == len(names), "%d/%d processus PM2 online" % (n, len(names))259    url = "http://127.0.0.1:%s%s" % (m["port"], m.get("health_path") or "/")260    deadline = time.time() + timeout261    code = "000"262    while True:263        rc, out, _ = ssh.run(ip, "curl -s -o /dev/null -m 8 -w '%%{http_code}' %s" % shlex.quote(url), timeout=20)264        code = out.strip()[-3:] if out.strip() else "000"265        if code.isdigit() and code not in ("000",) and int(code) < 500:266            break267        if time.time() >= deadline:268            break269        time.sleep(4)270    return code.isdigit() and code != "000" and int(code) < 500, code271272273def _health_public(m, tries=20):274    """Code HTTP de https://<domaine><health_path> à travers la passerelle (tries × 5 s max)."""275    import subprocess276    pub = ""277    for _ in range(tries):278        p = subprocess.run(["curl", "-s", "-o", "/dev/null", "-m", "12", "-w", "%{http_code}", "https://%s%s" % (m["domain"], m.get("health_path") or "/")], capture_output=True, text=True)279        pub = p.stdout.strip()280        if pub.isdigit() and int(pub) < 500 and pub != "404":281            break282        time.sleep(5)283    return pub284285286def health(app, alias, ip=None, m=None, timeout=90, public=True):287    """Santé locale (port/PM2) puis publique (via MacLustr Tunnel) si l'app a un domaine. Retourne (ok, détail)."""288    m = m or manifest.load(app)289    ip = ip or _ip(alias)290    local_ok, code = _health_local(ip, m, timeout=timeout)291    if not m.get("port"):292        return local_ok, code293    pub = ""294    if public and m.get("domain") and local_ok and tunnel.spec(m, alias):295        pub = _health_public(m, tries=max(2, min(20, timeout // 5)))296    detail = "local %s" % code + (" / public https://%s %s" % (m["domain"], pub) if pub else "")297    ok = local_ok and (not pub or (pub.isdigit() and int(pub) < 500 and pub != "404"))298    return ok, detail299300301def retire(app, alias, remove_dir=True):302    """Arrête et efface la copie d'un nœud (après migration réussie)."""303    m = manifest.load(app)304    ip = _ip(alias)305    stop(app, alias)306    if remove_dir:307        d = _abs(m["dir"], ip)308        rc, out, err = ssh.run(ip, "rm -rf %s && echo '  rm -rf %s'; rm -f %s/%s.config.cjs; grep -v -x %s ~/.ka-pousseur-repos > ~/.ka-pousseur-repos.tmp 2>/dev/null && mv ~/.ka-pousseur-repos.tmp ~/.ka-pousseur-repos; true" % (shlex.quote(d), d, _abs(config.RUN_DIR, ip), app, shlex.quote(m["dir"])), timeout=600)309        for l in out.splitlines():310            _log(app, l)311    registry.log(app, "retired", node=alias)312    return True313314315def move(app, to_alias, from_alias=None, keep_source=False, reinstall=False):316    """Migration complète : pré-sync à chaud, arrêt, sync final, déploiement, santé, retrait de la source (ou rollback)."""317    from_alias = from_alias or registry.node_of(app) or manifest.load(app).get("source_node")318    if not from_alias:319        raise DeployError("nœud source inconnu pour %s (préciser --from)" % app)320    if from_alias == to_alias:321        _log(app, "déjà sur %s → redéploiement en place" % to_alias)322        fetch(app, from_alias)323        stop(app, from_alias)324        fetch(app, from_alias, final=True)325        return deploy(app, to_alias, reinstall=reinstall)326    _log(app, "MOVE %s : %s → %s" % (app, from_alias, to_alias))327    fetch(app, from_alias)                 # 1. copie à chaud (gros volume)328    stop(app, from_alias)                  # 2. arrêt source329    fetch(app, from_alias, final=True)     # 3. delta final (cohérence des bases)330    try:331        deploy(app, to_alias, reinstall=reinstall)   # 4. cible332    except DeployError as e:333        _log(app, "ÉCHEC sur %s : %s — rollback sur %s" % (to_alias, e, from_alias))334        try:335            stop(app, to_alias)336        except Exception:337            pass338        start(app, from_alias)339        m = manifest.load(app)340        ip0 = _ip(from_alias)341        tun = _publish(app, from_alias, m, ip0) if tunnel.spec(m, from_alias) else None   # la route publique revient sur la source342        ok, det = health(app, from_alias, ip=ip0, m=m)343        registry.set_app(app, node=from_alias, ip=ip0, tunnel=tun, status="online" if ok else "unhealthy", health=det)344        raise345    if not keep_source:346        retire(app, from_alias)            # 5. nettoyage source347    registry.log(app, "moved", **{"from": from_alias, "to": to_alias})348    _log(app, "MOVE terminé : %s tourne sur %s" % (app, to_alias))349    return True350351352def status(app=None):353    r = registry.load()354    apps = [app] if app else sorted(r["apps"])355    rows = []356    for a in apps:357        v = r["apps"].get(a)358        if not v:359            rows.append((a, "—", "", "", "non déployée", ""))360            continue361        rows.append((a, v.get("node"), v.get("port"), v.get("domain") or "", v.get("status"), v.get("deployed")))362    return rows363364365def live_status(app=None, public=True):366    """Vérifie en live (PM2 + port + domaine via le tunnel) chaque app du registre. Une seule sauvegarde du registre à la fin."""367    r = registry.load()368    apps = [app] if app else sorted(r["apps"])369    out = []370    changes = {}371    for a in apps:372        v = r["apps"].get(a)373        if not v:374            continue375        m = manifest.load(a)376        try:377            ip = _ip(v["node"])378            ok, det = health(a, v["node"], ip=ip, m=m, timeout=10, public=public)379        except DeployError as e:380            ok, det = False, str(e)381        changes[a] = {"status": "online" if ok else "unhealthy", "health": det}382        out.append((a, v["node"], v.get("domain") or "", "OK" if ok else "KO", det))383    registry.update_many(changes)384    return out385