spb/maclustr-dispatch

1"""Cycle de vie d'une app : fetch (nœud -> staging), deploy (staging -> nœud), stop, start, retire, move, health."""2import json3import os4import shlex5import time6from . import config, manifest, nodes, registry, render, ssh78SYNC_BASE_EXCLUDES = [".DS_Store", "*.log", "__pycache__/", ".pytest_cache/", ".mypy_cache/"]91011class DeployError(Exception):12    pass131415def _ctx(alias, ip):16    lan = nodes.load_lan()["ips"]17    return {"HOME": config.home_of(ip), "NODE": alias, "LAN_IP": ip, "ips": lan, "USER": config.user_of(ip)}181920def _abs(p, ip=None):21    """~ → home du nœud (dépend de l'utilisateur : simon-pierreboucher, ou celui d'un REMOTE_NODE)."""22    return p.replace("~", config.home_of(ip), 1) if p.startswith("~") else p232425def _log(app, msg):26    line = "%s %s" % (time.strftime("%H:%M:%S"), msg)27    print(line, flush=True)28    with open(os.path.join(config.LOG_DIR, app + ".log"), "a") as f:29        f.write(time.strftime("%Y-%m-%d ") + line + "\n")303132def _ip(alias):33    ip = nodes.ip_of(alias)34    if not ip:35        raise DeployError("nœud %s injoignable sur le LAN" % alias)36    return ip373839def _ensure_dir(ip, d):40    """Crée le répertoire cible ; sous /opt (racine root) passe par sudo -S avec le mot de passe de ~/dispatch/.sudo."""41    if ssh.ok(ip, "mkdir -p %s 2>/dev/null && test -w %s" % (shlex.quote(d), shlex.quote(d))):42        return43    pwf = config.sudo_pw_file(ip)44    pw = open(pwf).read().strip() if os.path.exists(pwf) else None45    if not pw:46        raise DeployError("impossible de créer %s (sudo requis, %s absent)" % (d, pwf))47    rc, out, err = ssh.run(ip, "echo %s | sudo -S -p '' mkdir -p %s && echo %s | sudo -S -p '' chown %s %s" % (shlex.quote(pw), shlex.quote(d), shlex.quote(pw), config.user_of(ip), shlex.quote(d)))48    if rc != 0:49        raise DeployError("création de %s impossible : %s" % (d, err.strip()[-200:]))505152def stage_dir(app):53    return os.path.join(config.STAGE_DIR, app)545556# ---------------------------------------------------------------- fetch : nœud -> staging57def fetch(app, from_alias, final=False):58    m = manifest.load(app)59    ip = _ip(from_alias)60    _log(app, "fetch %s depuis %s (%s)%s" % (m["dir"], from_alias, ip, " [final]" if final else ""))61    excl = SYNC_BASE_EXCLUDES + list(m.get("sync_excludes") or [])62    rc, out, err = ssh.rsync_pull(ip, _abs(m["dir"], ip), os.path.join(stage_dir(app), "dir"), excludes=excl)63    if rc == 23 and not final:64        _log(app, "  avertissement pré-copie à chaud (fichiers en cours d'écriture, repris au delta final) : %s" % err.strip().splitlines()[0][:120])65    elif rc not in (0, 24):66        raise DeployError("rsync fetch a échoué (%s): %s" % (rc, err.strip()[-300:]))67    for extra in m.get("extra_paths") or []:68        dest = os.path.join(stage_dir(app), "extra", extra.strip("~/").replace("/", "__"))69        rc2, out2, err2 = ssh.run(ip, "test -e %s" % shlex.quote(_abs(extra, ip)))70        if rc2 != 0:71            _log(app, "  extra absent sur la source, ignoré : %s" % extra)72            continue73        isdir = ssh.ok(ip, "test -d %s" % shlex.quote(_abs(extra, ip)))74        if isdir:75            rc2, out2, err2 = ssh.rsync_pull(ip, _abs(extra, ip), dest, delete=True)76        else:77            os.makedirs(dest, exist_ok=True)78            import subprocess79            subprocess.run(["scp", "-p"] + ssh.SSH_OPTS + ["%s:%s" % (ssh.target(ip), _abs(extra, ip)), os.path.join(dest, os.path.basename(extra))], capture_output=True)80    json.dump({"app": app, "from": from_alias, "ip": ip, "ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "final": final},81              open(os.path.join(stage_dir(app), "fetch.json"), "w"), indent=2)82    stat = [l for l in out.splitlines() if "Total transferred file size" in l or "Number of regular files transferred" in l or "Number of files transferred" in l or "Total bytes sent" in l]83    _log(app, "  ok — " + " ; ".join(s.strip() for s in stat))84    return True858687# ---------------------------------------------------------------- stop / start / status sur un nœud88def stop(app, alias, keep_dir=True):89    m = manifest.load(app)90    ip = _ip(alias)91    _log(app, "stop sur %s" % alias)92    names = manifest.pm2_names(m)93    script = ""94    if names:95        script += "for n in %s; do pm2 delete $n >/dev/null 2>&1 && echo \"  pm2 delete $n\"; done; pm2 save --force >/dev/null 2>&1\n" % " ".join(names)96    for lbl in manifest.launchd_labels(m):97        script += "launchctl bootout gui/$(id -u)/%s >/dev/null 2>&1 && echo '  bootout %s'; rm -f ~/Library/LaunchAgents/%s.plist\n" % (lbl, lbl, lbl)98    if m.get("ngrok"):99        script += "pkill -f -- 'url=%s' 2>/dev/null; pkill -f -- 'url=https://%s' 2>/dev/null; pkill -f -- 'domain=%s' 2>/dev/null; true\n" % (m["ngrok"]["url"], m["ngrok"]["url"], m["ngrok"]["url"])100    for hook in (m.get("hooks") or {}).get("pre_stop") or []:101        script += hook + "\n"102    # tue ce qui écoute encore sur les ports de l'app (orphelins)103    for p in m["requires"].get("ports") or []:104        script += "sleep 1; for pid in $(lsof -nP -iTCP:%s -sTCP:LISTEN -t 2>/dev/null); do kill $pid 2>/dev/null && echo '  kill port %s pid '$pid; done\n" % (p, p)105    if m.get("ngrok"):106        script += "sleep 4\n"   # laisse ngrok libérer le domaine réservé côté service107    rc, out, err = ssh.run(ip, script, timeout=180)108    for l in out.splitlines():109        _log(app, l)110    registry.log(app, "stopped", node=alias)111    return rc == 0112113114def _install_hooks(m, ip, ctx):115    """Commandes post-sync : réécritures .env, hooks, inscription pousseur."""116    script = ""117    d = _abs(m["dir"], ip)118    for fname, kv in (m.get("env_overrides") or {}).items():119        target = os.path.join(d, fname)120        kv = manifest.render(kv, ctx)121        py = "import re,sys,os\np=%r\nkv=%r\ns=open(p).read() if os.path.exists(p) else ''\nfor k,v in kv.items():\n    line=k+'='+v\n    if re.search(r'^'+re.escape(k)+r'=.*$', s, re.M): s=re.sub(r'^'+re.escape(k)+r'=.*$', line.replace('\\\\','\\\\\\\\'), s, flags=re.M)\n    else: s=s.rstrip('\\n')+'\\n'+line+'\\n'\nopen(p,'w').write(s)\nprint('  .env réécrit: '+', '.join(kv))" % (target, kv)122        script += "python3 - <<'PYEOF'\n%s\nPYEOF\n" % py123    if m.get("ka_repo"):124        script += "grep -qx %s ~/.ka-pousseur-repos 2>/dev/null || echo %s >> ~/.ka-pousseur-repos; echo '  pousseur: %s inscrit'\n" % (shlex.quote(m["dir"]), shlex.quote(m["dir"]), m["dir"])125    for hook in (m.get("hooks") or {}).get("post_sync") or []:126        script += "cd %s && { %s ; }\n" % (shlex.quote(d), manifest.render(hook, ctx))127    return script128129130def deploy(app, alias, reinstall=False, skip_sync=False):131    """staging -> nœud, puis démarrage (PM2 + launchd) et healthcheck."""132    m = manifest.load(app)133    ip = _ip(alias)134    ctx = _ctx(alias, ip)135    sd = os.path.join(stage_dir(app), "dir")136    if not skip_sync and not os.path.isdir(sd):137        raise DeployError("staging vide pour %s — faire `mld fetch %s --from <nœud>` ou `mld stage`" % (app, app))138    _log(app, "deploy → %s (%s)" % (alias, ip))139    # 1. runtimes140    scan = nodes.load_scan().get(alias) or nodes.probe(alias, ip)141    missing = [r for r in m["requires"]["runtimes"] if not scan.get("runtimes", {}).get(r)]142    if missing:143        raise DeployError("runtimes manquants sur %s : %s (→ `mld prepare %s`)" % (alias, ", ".join(missing), alias))144    # 2. sync145    d = _abs(m["dir"], ip)146    if not skip_sync:147        _ensure_dir(ip, d)148        excl = SYNC_BASE_EXCLUDES + list(m.get("sync_excludes") or [])149        rc, out, err = ssh.rsync_push(sd, ip, d, excludes=excl, delete=True)150        if rc not in (0, 24):151            raise DeployError("rsync deploy a échoué (%s): %s" % (rc, err.strip()[-300:]))152        stat = [l.strip() for l in out.splitlines() if "Total transferred file size" in l or "Total bytes sent" in l]153        _log(app, "  sync ok — %s" % (stat[0] if stat else ""))154        for extra in m.get("extra_paths") or []:155            src = os.path.join(stage_dir(app), "extra", extra.strip("~/").replace("/", "__"))156            if not os.path.exists(src):157                continue158            if os.path.isdir(src) and not os.path.exists(os.path.join(src, os.path.basename(extra))):159                ssh.rsync_push(src, ip, _abs(extra, ip), delete=False)160            else:161                import subprocess162                ssh.run(ip, "mkdir -p %s" % shlex.quote(os.path.dirname(_abs(extra, ip))))163                subprocess.run(["scp", "-p"] + ssh.SSH_OPTS + [os.path.join(src, os.path.basename(extra)), "%s:%s" % (ssh.target(ip), _abs(extra, ip))], capture_output=True)164            _log(app, "  extra: %s" % extra)165    # 3. install (optionnel) + hooks166    script = "cd %s || exit 3\n" % shlex.quote(d)167    if reinstall:168        script += "if [ -f pnpm-lock.yaml ]; then pnpm install --frozen-lockfile; elif [ -f package-lock.json ]; then npm ci; elif [ -f package.json ]; then npm install; fi\n"169        script += "if [ -f requirements.txt ] && [ -x .venv/bin/pip ]; then .venv/bin/pip install -q -r requirements.txt; fi\n"170    elif m.get("rebuild", True) and os.path.exists(os.path.join(sd, "package.json")) and os.path.isdir(os.path.join(sd, "node_modules")):171        script += "npm rebuild >/dev/null 2>&1 && echo '  npm rebuild ok' || echo '  npm rebuild: avertissement'\n"172    script += _install_hooks(m, ip, ctx)173    rc, out, err = ssh.run(ip, script, timeout=1800)174    for l in out.splitlines()[-15:]:175        _log(app, l)176    if rc != 0:177        raise DeployError("post-sync a échoué (%s): %s" % (rc, (err or out).strip()[-400:]))178    # 4. lancement179    start(app, alias, m=m, ctx=ctx, ip=ip)180    # 5. santé181    healthy, detail = health(app, alias, ip=ip, m=m)182    registry.set_app(app, node=alias, ip=ip, port=m.get("port"), domain=m.get("domain"), dir=m["dir"], label=m.get("label") or app,183                     health_path=m.get("health_path") or "/",184                     processes=manifest.pm2_names(m), launchd=manifest.launchd_labels(m), deployed=time.strftime("%Y-%m-%d %H:%M"),185                     status="online" if healthy else "unhealthy", health=detail)186    registry.log(app, "deployed", node=alias, healthy=healthy)187    _log(app, "  santé : %s" % detail)188    if not healthy:189        names = manifest.pm2_names(m)190        rc, out, _ = ssh.run(ip, "pm2 logs %s --nostream --lines 6 2>/dev/null | grep -v '^\\[TAILING' | tail -12" % " ".join(names), timeout=40)191        for l in out.splitlines():192            _log(app, "  log: " + l[:160])193        raise DeployError("healthcheck KO sur %s : %s" % (alias, detail))194    return True195196197def start(app, alias, m=None, ctx=None, ip=None):198    m = m or manifest.load(app)199    ip = ip or _ip(alias)200    ctx = ctx or _ctx(alias, ip)201    _log(app, "start sur %s" % alias)202    run_dir = _abs(config.RUN_DIR, ip)203    eco = render.ecosystem(m, ctx)204    eco_path = "%s/%s.config.cjs" % (run_dir, app)205    ssh.write_remote_file(ip, eco_path, eco, mode="600")206    script = "mkdir -p %s\n" % run_dir207    names = manifest.pm2_names(m)208    if names:209        script += "for n in %s; do pm2 delete $n >/dev/null 2>&1; done\n" % " ".join(names)210        script += "pm2 start %s --update-env 2>&1 | grep -E 'error|Error|\\[PM2\\]' | head -5; pm2 save --force >/dev/null 2>&1; pm2 jlist | python3 -c \"import json,sys; L=json.load(sys.stdin); print('  pm2: '+', '.join(p['name']+'='+p['pm2_env']['status'] for p in L if p['name'] in %s))\" 2>/dev/null || pm2 ls | grep -E '%s'\n" % (shlex.quote(eco_path), json.dumps(names), "|".join(names))211    for item in m.get("launchd") or []:212        pl = render.plist(item, ctx)213        ppath = "%s/Library/LaunchAgents/%s.plist" % (ctx["HOME"], item["label"])214        ssh.write_remote_file(ip, ppath, pl, mode="644")215        script += "launchctl bootout gui/$(id -u)/%s >/dev/null 2>&1; launchctl bootstrap gui/$(id -u) %s && echo '  launchd: %s chargé' || echo '  launchd: %s ÉCHEC'\n" % (item["label"], shlex.quote(ppath), item["label"], item["label"])216    for hook in (m.get("hooks") or {}).get("post_start") or []:217        script += "cd %s && { %s ; }\n" % (shlex.quote(_abs(m["dir"], ip)), manifest.render(hook, ctx))218    rc, out, err = ssh.run(ip, script, timeout=300)219    for l in out.splitlines():220        _log(app, l)221    if err.strip():222        _log(app, "  stderr: " + err.strip()[-200:])223    return rc == 0224225226def health(app, alias, ip=None, m=None, timeout=90):227    m = m or manifest.load(app)228    ip = ip or _ip(alias)229    if not m.get("port"):230        # pas de port HTTP : vérifie que les processus PM2 sont online231        names = manifest.pm2_names(m)232        if not names:233            return True, "aucun port/processus à vérifier"234        rc, out, _ = ssh.run(ip, "pm2 jlist | python3 -c \"import json,sys; L=json.load(sys.stdin); print(sum(1 for p in L if p['name'] in %s and p['pm2_env']['status']=='online'))\"" % json.dumps(names))235        n = int((out.strip() or "0").splitlines()[-1] or 0)236        return n == len(names), "%d/%d processus PM2 online" % (n, len(names))237    url = "http://127.0.0.1:%s%s" % (m["port"], m.get("health_path") or "/")238    deadline = time.time() + timeout239    code = "000"240    while time.time() < deadline:241        rc, out, _ = ssh.run(ip, "curl -s -o /dev/null -m 8 -w '%%{http_code}' %s" % shlex.quote(url), timeout=20)242        code = out.strip()[-3:] if out.strip() else "000"243        if code.isdigit() and code not in ("000",) and int(code) < 500:244            break245        time.sleep(4)246    local_ok = code.isdigit() and code != "000" and int(code) < 500247    pub = ""248    if m.get("domain") and local_ok and m.get("ngrok"):249        import subprocess250        for _ in range(20):251            p = subprocess.run(["curl", "-s", "-o", "/dev/null", "-m", "12", "-w", "%{http_code}", "https://%s%s" % (m["domain"], m.get("health_path") or "/")], capture_output=True, text=True)252            pub = p.stdout.strip()253            if pub.isdigit() and int(pub) < 500 and pub != "404":254                break255            time.sleep(5)256    detail = "local %s" % code + (" / public https://%s %s" % (m["domain"], pub) if pub else "")257    ok = local_ok and (not pub or (pub.isdigit() and int(pub) < 500 and pub != "404"))258    return ok, detail259260261def retire(app, alias, remove_dir=True):262    """Arrête et efface la copie d'un nœud (après migration réussie)."""263    m = manifest.load(app)264    ip = _ip(alias)265    stop(app, alias)266    if remove_dir:267        d = _abs(m["dir"], ip)268        rc, out, err = ssh.run(ip, "rm -rf %s && echo '  rm -rf %s'; rm -f %s/%s.config.cjs; grep -v -x %s ~/.ka-pousseur-repos > ~/.ka-pousseur-repos.tmp 2>/dev/null && mv ~/.ka-pousseur-repos.tmp ~/.ka-pousseur-repos; true" % (shlex.quote(d), d, _abs(config.RUN_DIR, ip), app, shlex.quote(m["dir"])), timeout=600)269        for l in out.splitlines():270            _log(app, l)271    registry.log(app, "retired", node=alias)272    return True273274275def move(app, to_alias, from_alias=None, keep_source=False, reinstall=False):276    """Migration complète : pré-sync à chaud, arrêt, sync final, déploiement, santé, retrait de la source (ou rollback)."""277    from_alias = from_alias or registry.node_of(app) or manifest.load(app).get("source_node")278    if not from_alias:279        raise DeployError("nœud source inconnu pour %s (préciser --from)" % app)280    if from_alias == to_alias:281        _log(app, "déjà sur %s → redéploiement en place" % to_alias)282        fetch(app, from_alias)283        stop(app, from_alias)284        fetch(app, from_alias, final=True)285        return deploy(app, to_alias, reinstall=reinstall)286    _log(app, "MOVE %s : %s → %s" % (app, from_alias, to_alias))287    fetch(app, from_alias)                 # 1. copie à chaud (gros volume)288    stop(app, from_alias)                  # 2. arrêt source289    fetch(app, from_alias, final=True)     # 3. delta final (cohérence des bases)290    try:291        deploy(app, to_alias, reinstall=reinstall)   # 4. cible292    except DeployError as e:293        _log(app, "ÉCHEC sur %s : %s — rollback sur %s" % (to_alias, e, from_alias))294        try:295            stop(app, to_alias)296        except Exception:297            pass298        start(app, from_alias)299        ok, det = health(app, from_alias)300        registry.set_app(app, node=from_alias, status="online" if ok else "unhealthy", health=det)301        raise302    if not keep_source:303        retire(app, from_alias)            # 5. nettoyage source304    registry.log(app, "moved", **{"from": from_alias, "to": to_alias})305    _log(app, "MOVE terminé : %s tourne sur %s" % (app, to_alias))306    return True307308309def status(app=None):310    r = registry.load()311    apps = [app] if app else sorted(r["apps"])312    rows = []313    for a in apps:314        v = r["apps"].get(a)315        if not v:316            rows.append((a, "—", "", "", "non déployée", ""))317            continue318        rows.append((a, v.get("node"), v.get("port"), v.get("domain") or "", v.get("status"), v.get("deployed")))319    return rows320321322def live_status(app=None, public=True):323    """Vérifie en live (PM2 + port + domaine) chaque app du registre."""324    r = registry.load()325    apps = [app] if app else sorted(r["apps"])326    out = []327    for a in apps:328        v = r["apps"].get(a)329        if not v:330            continue331        m = manifest.load(a)332        try:333            ip = _ip(v["node"])334            ok, det = health(a, v["node"], ip=ip, m=m, timeout=10) if public else (None, "")335        except DeployError as e:336            ok, det = False, str(e)337        registry.set_app(a, status="online" if ok else "unhealthy", health=det)338        out.append((a, v["node"], v.get("domain") or "", "OK" if ok else "KO", det))339    return out340