spb/maclustr-dispatch

1"""Cycle de vie d'une app : fetch (nœud -> staging), deploy (staging -> nœud), stop, start, retire, move, health."""2import json3import os4import shlex5import time6from . import config, manifest, nodes, registry, render, ssh78SYNC_BASE_EXCLUDES = [".DS_Store", "*.log", "__pycache__/", ".pytest_cache/", ".mypy_cache/"]91011class DeployError(Exception):12    pass131415def _ctx(alias, ip):16    lan = nodes.load_lan()["ips"]17    return {"HOME": "/Users/%s" % config.USER, "NODE": alias, "LAN_IP": ip, "ips": lan, "USER": config.USER}181920def _abs(p):21    return p.replace("~", "/Users/%s" % config.USER, 1) if p.startswith("~") else p222324def _log(app, msg):25    line = "%s %s" % (time.strftime("%H:%M:%S"), msg)26    print(line, flush=True)27    with open(os.path.join(config.LOG_DIR, app + ".log"), "a") as f:28        f.write(time.strftime("%Y-%m-%d ") + line + "\n")293031def _ip(alias):32    ip = nodes.ip_of(alias)33    if not ip:34        raise DeployError("nœud %s injoignable sur le LAN" % alias)35    return ip363738def _ensure_dir(ip, d):39    """Crée le répertoire cible ; sous /opt (racine root) passe par sudo -S avec le mot de passe de ~/dispatch/.sudo."""40    if ssh.ok(ip, "mkdir -p %s 2>/dev/null && test -w %s" % (shlex.quote(d), shlex.quote(d))):41        return42    pw = open(config.SUDO_PW_FILE).read().strip() if os.path.exists(config.SUDO_PW_FILE) else None43    if not pw:44        raise DeployError("impossible de créer %s (sudo requis, %s absent)" % (d, config.SUDO_PW_FILE))45    rc, out, err = ssh.run(ip, "echo %s | sudo -S -p '' mkdir -p %s && echo %s | sudo -S -p '' chown %s %s" % (shlex.quote(pw), shlex.quote(d), shlex.quote(pw), config.USER, shlex.quote(d)))46    if rc != 0:47        raise DeployError("création de %s impossible : %s" % (d, err.strip()[-200:]))484950def stage_dir(app):51    return os.path.join(config.STAGE_DIR, app)525354# ---------------------------------------------------------------- fetch : nœud -> staging55def fetch(app, from_alias, final=False):56    m = manifest.load(app)57    ip = _ip(from_alias)58    _log(app, "fetch %s depuis %s (%s)%s" % (m["dir"], from_alias, ip, " [final]" if final else ""))59    excl = SYNC_BASE_EXCLUDES + list(m.get("sync_excludes") or [])60    rc, out, err = ssh.rsync_pull(ip, _abs(m["dir"]), os.path.join(stage_dir(app), "dir"), excludes=excl)61    if rc not in (0, 24):62        raise DeployError("rsync fetch a échoué (%s): %s" % (rc, err.strip()[-300:]))63    for extra in m.get("extra_paths") or []:64        dest = os.path.join(stage_dir(app), "extra", extra.strip("~/").replace("/", "__"))65        rc2, out2, err2 = ssh.run(ip, "test -e %s" % shlex.quote(_abs(extra)))66        if rc2 != 0:67            _log(app, "  extra absent sur la source, ignoré : %s" % extra)68            continue69        isdir = ssh.ok(ip, "test -d %s" % shlex.quote(_abs(extra)))70        if isdir:71            rc2, out2, err2 = ssh.rsync_pull(ip, _abs(extra), dest, delete=True)72        else:73            os.makedirs(dest, exist_ok=True)74            import subprocess75            subprocess.run(["scp", "-p"] + ssh.SSH_OPTS + ["%s:%s" % (ssh.target(ip), _abs(extra)), os.path.join(dest, os.path.basename(extra))], capture_output=True)76    json.dump({"app": app, "from": from_alias, "ip": ip, "ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "final": final},77              open(os.path.join(stage_dir(app), "fetch.json"), "w"), indent=2)78    stat = [l for l in out.splitlines() if "Total transferred file size" in l or "Number of regular files transferred" in l or "Number of files transferred" in l or "Total bytes sent" in l]79    _log(app, "  ok — " + " ; ".join(s.strip() for s in stat))80    return True818283# ---------------------------------------------------------------- stop / start / status sur un nœud84def stop(app, alias, keep_dir=True):85    m = manifest.load(app)86    ip = _ip(alias)87    _log(app, "stop sur %s" % alias)88    names = manifest.pm2_names(m)89    script = ""90    if names:91        script += "for n in %s; do pm2 delete $n >/dev/null 2>&1 && echo \"  pm2 delete $n\"; done; pm2 save --force >/dev/null 2>&1\n" % " ".join(names)92    for lbl in manifest.launchd_labels(m):93        script += "launchctl bootout gui/$(id -u)/%s >/dev/null 2>&1 && echo '  bootout %s'; rm -f ~/Library/LaunchAgents/%s.plist\n" % (lbl, lbl, lbl)94    if m.get("ngrok"):95        script += "pkill -f -- 'url=%s' 2>/dev/null; pkill -f -- 'url=https://%s' 2>/dev/null; pkill -f -- 'domain=%s' 2>/dev/null; true\n" % (m["ngrok"]["url"], m["ngrok"]["url"], m["ngrok"]["url"])96    for hook in (m.get("hooks") or {}).get("pre_stop") or []:97        script += hook + "\n"98    # tue ce qui écoute encore sur les ports de l'app (orphelins)99    for p in m["requires"].get("ports") or []:100        script += "sleep 1; for pid in $(lsof -nP -iTCP:%s -sTCP:LISTEN -t 2>/dev/null); do kill $pid 2>/dev/null && echo '  kill port %s pid '$pid; done\n" % (p, p)101    if m.get("ngrok"):102        script += "sleep 4\n"   # laisse ngrok libérer le domaine réservé côté service103    rc, out, err = ssh.run(ip, script, timeout=180)104    for l in out.splitlines():105        _log(app, l)106    registry.log(app, "stopped", node=alias)107    return rc == 0108109110def _install_hooks(m, ip, ctx):111    """Commandes post-sync : réécritures .env, hooks, inscription pousseur."""112    script = ""113    d = _abs(m["dir"])114    for fname, kv in (m.get("env_overrides") or {}).items():115        target = os.path.join(d, fname)116        kv = manifest.render(kv, ctx)117        py = "import re,sys,os\np=%r\nkv=%r\ns=open(p).read() if os.path.exists(p) else ''\nfor k,v in kv.items():\n    line=k+'='+v\n    if re.search(r'^'+re.escape(k)+r'=.*$', s, re.M): s=re.sub(r'^'+re.escape(k)+r'=.*$', line.replace('\\\\','\\\\\\\\'), s, flags=re.M)\n    else: s=s.rstrip('\\n')+'\\n'+line+'\\n'\nopen(p,'w').write(s)\nprint('  .env réécrit: '+', '.join(kv))" % (target, kv)118        script += "python3 - <<'PYEOF'\n%s\nPYEOF\n" % py119    if m.get("ka_repo"):120        script += "grep -qx %s ~/.ka-pousseur-repos 2>/dev/null || echo %s >> ~/.ka-pousseur-repos; echo '  pousseur: %s inscrit'\n" % (shlex.quote(m["dir"]), shlex.quote(m["dir"]), m["dir"])121    for hook in (m.get("hooks") or {}).get("post_sync") or []:122        script += "cd %s && (%s)\n" % (shlex.quote(d), manifest.render(hook, ctx))123    return script124125126def deploy(app, alias, reinstall=False, skip_sync=False):127    """staging -> nœud, puis démarrage (PM2 + launchd) et healthcheck."""128    m = manifest.load(app)129    ip = _ip(alias)130    ctx = _ctx(alias, ip)131    sd = os.path.join(stage_dir(app), "dir")132    if not skip_sync and not os.path.isdir(sd):133        raise DeployError("staging vide pour %s — faire `mld fetch %s --from <nœud>` ou `mld stage`" % (app, app))134    _log(app, "deploy → %s (%s)" % (alias, ip))135    # 1. runtimes136    scan = nodes.load_scan().get(alias) or nodes.probe(alias, ip)137    missing = [r for r in m["requires"]["runtimes"] if not scan.get("runtimes", {}).get(r)]138    if missing:139        raise DeployError("runtimes manquants sur %s : %s (→ `mld prepare %s`)" % (alias, ", ".join(missing), alias))140    # 2. sync141    d = _abs(m["dir"])142    if not skip_sync:143        _ensure_dir(ip, d)144        excl = SYNC_BASE_EXCLUDES + list(m.get("sync_excludes") or [])145        rc, out, err = ssh.rsync_push(sd, ip, d, excludes=excl, delete=True)146        if rc not in (0, 24):147            raise DeployError("rsync deploy a échoué (%s): %s" % (rc, err.strip()[-300:]))148        stat = [l.strip() for l in out.splitlines() if "Total transferred file size" in l or "Total bytes sent" in l]149        _log(app, "  sync ok — %s" % (stat[0] if stat else ""))150        for extra in m.get("extra_paths") or []:151            src = os.path.join(stage_dir(app), "extra", extra.strip("~/").replace("/", "__"))152            if not os.path.exists(src):153                continue154            if os.path.isdir(src) and not os.path.exists(os.path.join(src, os.path.basename(extra))):155                ssh.rsync_push(src, ip, _abs(extra), delete=False)156            else:157                import subprocess158                ssh.run(ip, "mkdir -p %s" % shlex.quote(os.path.dirname(_abs(extra))))159                subprocess.run(["scp", "-p"] + ssh.SSH_OPTS + [os.path.join(src, os.path.basename(extra)), "%s:%s" % (ssh.target(ip), _abs(extra))], capture_output=True)160            _log(app, "  extra: %s" % extra)161    # 3. install (optionnel) + hooks162    script = "cd %s || exit 3\n" % shlex.quote(d)163    if reinstall:164        script += "if [ -f pnpm-lock.yaml ]; then pnpm install --frozen-lockfile; elif [ -f package-lock.json ]; then npm ci; elif [ -f package.json ]; then npm install; fi\n"165        script += "if [ -f requirements.txt ] && [ -x .venv/bin/pip ]; then .venv/bin/pip install -q -r requirements.txt; fi\n"166    elif os.path.exists(os.path.join(sd, "package.json")) and os.path.isdir(os.path.join(sd, "node_modules")):167        script += "npm rebuild >/dev/null 2>&1 && echo '  npm rebuild ok' || echo '  npm rebuild: avertissement'\n"168    script += _install_hooks(m, ip, ctx)169    rc, out, err = ssh.run(ip, script, timeout=1800)170    for l in out.splitlines()[-15:]:171        _log(app, l)172    if rc != 0:173        raise DeployError("post-sync a échoué (%s): %s" % (rc, (err or out).strip()[-400:]))174    # 4. lancement175    start(app, alias, m=m, ctx=ctx, ip=ip)176    # 5. santé177    healthy, detail = health(app, alias, ip=ip, m=m)178    registry.set_app(app, node=alias, ip=ip, port=m.get("port"), domain=m.get("domain"), dir=m["dir"], label=m.get("label") or app,179                     processes=manifest.pm2_names(m), launchd=manifest.launchd_labels(m), deployed=time.strftime("%Y-%m-%d %H:%M"),180                     status="online" if healthy else "unhealthy", health=detail)181    registry.log(app, "deployed", node=alias, healthy=healthy)182    _log(app, "  santé : %s" % detail)183    if not healthy:184        names = manifest.pm2_names(m)185        rc, out, _ = ssh.run(ip, "pm2 logs %s --nostream --lines 6 2>/dev/null | grep -v '^\\[TAILING' | tail -12" % " ".join(names), timeout=40)186        for l in out.splitlines():187            _log(app, "  log: " + l[:160])188        raise DeployError("healthcheck KO sur %s : %s" % (alias, detail))189    return True190191192def start(app, alias, m=None, ctx=None, ip=None):193    m = m or manifest.load(app)194    ip = ip or _ip(alias)195    ctx = ctx or _ctx(alias, ip)196    _log(app, "start sur %s" % alias)197    run_dir = _abs(config.RUN_DIR)198    eco = render.ecosystem(m, ctx)199    eco_path = "%s/%s.config.cjs" % (run_dir, app)200    ssh.write_remote_file(ip, eco_path, eco, mode="600")201    script = "mkdir -p %s\n" % run_dir202    names = manifest.pm2_names(m)203    if names:204        script += "for n in %s; do pm2 delete $n >/dev/null 2>&1; done\n" % " ".join(names)205        script += "pm2 start %s --update-env 2>&1 | grep -E 'error|Error|\\[PM2\\]' | head -5; pm2 save --force >/dev/null 2>&1; pm2 jlist | python3 -c \"import json,sys; L=json.load(sys.stdin); print('  pm2: '+', '.join(p['name']+'='+p['pm2_env']['status'] for p in L if p['name'] in %s))\" 2>/dev/null || pm2 ls | grep -E '%s'\n" % (shlex.quote(eco_path), json.dumps(names), "|".join(names))206    for item in m.get("launchd") or []:207        pl = render.plist(item, ctx)208        ppath = "%s/Library/LaunchAgents/%s.plist" % (ctx["HOME"], item["label"])209        ssh.write_remote_file(ip, ppath, pl, mode="644")210        script += "launchctl bootout gui/$(id -u)/%s >/dev/null 2>&1; launchctl bootstrap gui/$(id -u) %s && echo '  launchd: %s chargé' || echo '  launchd: %s ÉCHEC'\n" % (item["label"], shlex.quote(ppath), item["label"], item["label"])211    for hook in (m.get("hooks") or {}).get("post_start") or []:212        script += "cd %s && (%s)\n" % (shlex.quote(_abs(m["dir"])), manifest.render(hook, ctx))213    rc, out, err = ssh.run(ip, script, timeout=300)214    for l in out.splitlines():215        _log(app, l)216    if err.strip():217        _log(app, "  stderr: " + err.strip()[-200:])218    return rc == 0219220221def health(app, alias, ip=None, m=None, timeout=90):222    m = m or manifest.load(app)223    ip = ip or _ip(alias)224    if not m.get("port"):225        # pas de port HTTP : vérifie que les processus PM2 sont online226        names = manifest.pm2_names(m)227        if not names:228            return True, "aucun port/processus à vérifier"229        rc, out, _ = ssh.run(ip, "pm2 jlist | python3 -c \"import json,sys; L=json.load(sys.stdin); print(sum(1 for p in L if p['name'] in %s and p['pm2_env']['status']=='online'))\"" % json.dumps(names))230        n = int((out.strip() or "0").splitlines()[-1] or 0)231        return n == len(names), "%d/%d processus PM2 online" % (n, len(names))232    url = "http://127.0.0.1:%s%s" % (m["port"], m.get("health_path") or "/")233    deadline = time.time() + timeout234    code = "000"235    while time.time() < deadline:236        rc, out, _ = ssh.run(ip, "curl -s -o /dev/null -m 8 -w '%%{http_code}' %s" % shlex.quote(url), timeout=20)237        code = out.strip()[-3:] if out.strip() else "000"238        if code.isdigit() and code not in ("000",) and int(code) < 500:239            break240        time.sleep(4)241    local_ok = code.isdigit() and code != "000" and int(code) < 500242    pub = ""243    if m.get("domain") and local_ok and m.get("ngrok"):244        import subprocess245        for _ in range(20):246            p = subprocess.run(["curl", "-s", "-o", "/dev/null", "-m", "12", "-w", "%{http_code}", "https://%s%s" % (m["domain"], m.get("health_path") or "/")], capture_output=True, text=True)247            pub = p.stdout.strip()248            if pub.isdigit() and int(pub) < 500 and pub != "404":249                break250            time.sleep(5)251    detail = "local %s" % code + (" / public https://%s %s" % (m["domain"], pub) if pub else "")252    ok = local_ok and (not pub or (pub.isdigit() and int(pub) < 500 and pub != "404"))253    return ok, detail254255256def retire(app, alias, remove_dir=True):257    """Arrête et efface la copie d'un nœud (après migration réussie)."""258    m = manifest.load(app)259    ip = _ip(alias)260    stop(app, alias)261    if remove_dir:262        d = _abs(m["dir"])263        rc, out, err = ssh.run(ip, "rm -rf %s && echo '  rm -rf %s'; rm -f %s/%s.config.cjs; grep -v -x %s ~/.ka-pousseur-repos > ~/.ka-pousseur-repos.tmp 2>/dev/null && mv ~/.ka-pousseur-repos.tmp ~/.ka-pousseur-repos; true" % (shlex.quote(d), d, _abs(config.RUN_DIR), app, shlex.quote(m["dir"])), timeout=600)264        for l in out.splitlines():265            _log(app, l)266    registry.log(app, "retired", node=alias)267    return True268269270def move(app, to_alias, from_alias=None, keep_source=False, reinstall=False):271    """Migration complète : pré-sync à chaud, arrêt, sync final, déploiement, santé, retrait de la source (ou rollback)."""272    from_alias = from_alias or registry.node_of(app) or manifest.load(app).get("source_node")273    if not from_alias:274        raise DeployError("nœud source inconnu pour %s (préciser --from)" % app)275    if from_alias == to_alias:276        _log(app, "déjà sur %s → redéploiement en place" % to_alias)277        fetch(app, from_alias)278        stop(app, from_alias)279        fetch(app, from_alias, final=True)280        return deploy(app, to_alias, reinstall=reinstall)281    _log(app, "MOVE %s : %s → %s" % (app, from_alias, to_alias))282    fetch(app, from_alias)                 # 1. copie à chaud (gros volume)283    stop(app, from_alias)                  # 2. arrêt source284    fetch(app, from_alias, final=True)     # 3. delta final (cohérence des bases)285    try:286        deploy(app, to_alias, reinstall=reinstall)   # 4. cible287    except DeployError as e:288        _log(app, "ÉCHEC sur %s : %s — rollback sur %s" % (to_alias, e, from_alias))289        try:290            stop(app, to_alias)291        except Exception:292            pass293        start(app, from_alias)294        ok, det = health(app, from_alias)295        registry.set_app(app, node=from_alias, status="online" if ok else "unhealthy", health=det)296        raise297    if not keep_source:298        retire(app, from_alias)            # 5. nettoyage source299    registry.log(app, "moved", **{"from": from_alias, "to": to_alias})300    _log(app, "MOVE terminé : %s tourne sur %s" % (app, to_alias))301    return True302303304def status(app=None):305    r = registry.load()306    apps = [app] if app else sorted(r["apps"])307    rows = []308    for a in apps:309        v = r["apps"].get(a)310        if not v:311            rows.append((a, "—", "", "", "non déployée", ""))312            continue313        rows.append((a, v.get("node"), v.get("port"), v.get("domain") or "", v.get("status"), v.get("deployed")))314    return rows315316317def live_status(app=None, public=True):318    """Vérifie en live (PM2 + port + domaine) chaque app du registre."""319    r = registry.load()320    apps = [app] if app else sorted(r["apps"])321    out = []322    for a in apps:323        v = r["apps"].get(a)324        if not v:325            continue326        m = manifest.load(a)327        try:328            ip = _ip(v["node"])329            ok, det = health(a, v["node"], ip=ip, m=m, timeout=10) if public else (None, "")330        except DeployError as e:331            ok, det = False, str(e)332        registry.set_app(a, status="online" if ok else "unhealthy", health=det)333        out.append((a, v["node"], v.get("domain") or "", "OK" if ok else "KO", det))334    return out335