spb/trouve-ka

1#!/bin/sh2# Trouve-KA — watchdog auto-guérisseur (tourne sur le hub M2M32 toutes les 2 min)3# Author: Simon-Pierre Boucher4# Contact: contact@spboucher.ai5#6# Vérifie chaque maillon et répare sans intervention humaine :7#   - API (health + search_ok), web, Postgres, Redis (localhost)8#   - tunnels OpenSearch (9210) et embeddings (8191)9#   - tunnel public ngrok (vue de l'extérieur)10#   - toutes les 30 min : passe de guérison sur toute la flotte satellite11#     (clé à commande forcée → exécute uniquement boot.sh distant)12# Journal : ~/trouveka-watchdog.log (tronqué automatiquement).1314PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"15export PATH16LOG="$HOME/trouveka-watchdog.log"17touch "$LOG"18STATE="$HOME/.trouveka-watchdog-count"19HEAL_KEY="$HOME/.ssh/trouveka_heal"20# Une IP LAN par ligne — générée à l'installation (scripts/ops/README)21SATELLITES=$(cat "$HOME/.trouveka-fleet" 2>/dev/null)2223log() { echo "$(date '+%F %T') $*" >> "$LOG"; }2425heal_local() {26  log "HEAL local : $1"27  /bin/sh "$HOME/trouve-ka/scripts/ops/boot.sh"28}2930heal_remote() { # $1 = ip31  log "HEAL distant : $1"32  /usr/bin/ssh -i "$HEAL_KEY" -o ControlMaster=no -o ControlPath=none -o IdentitiesOnly=yes -o IdentityAgent=none -o ForwardAgent=no -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new \33    "simon-pierreboucher@$1" heal >> "$LOG" 2>&1 || log "HEAL $1 : injoignable"34}3536FAILURES=03738# --- Postgres / Redis (docker local)39nc -z -w 3 127.0.0.1 5432 || { heal_local "postgres injoignable"; FAILURES=$((FAILURES+1)); }40nc -z -w 3 127.0.0.1 6379 || { heal_local "redis injoignable"; FAILURES=$((FAILURES+1)); }4142# --- Tunnels43if ! curl -s -m 5 http://127.0.0.1:9210/_cluster/health | grep -q '"status"'; then44  heal_local "tunnel/OpenSearch 9210 muet"45  sleep 346  curl -s -m 5 http://127.0.0.1:9210/_cluster/health | grep -q '"status"' || heal_remote 192.168.2.7747  FAILURES=$((FAILURES+1))48fi49if ! curl -s -m 5 http://127.0.0.1:8191/health | grep -q '"ok"'; then50  heal_local "tunnel/embeddings 8191 muet"51  sleep 352  curl -s -m 5 http://127.0.0.1:8191/health | grep -q '"ok"' || heal_remote 192.168.2.7553  FAILURES=$((FAILURES+1))54fi5556# --- API : santé + moteur de recherche réellement joignable57HEALTH=$(curl -s -m 8 http://127.0.0.1:8080/api/health)58case "$HEALTH" in59  *'"ok":true'*'"search_ok":true'*) : ;;60  *'"search_ok":false'*)61    log "API up mais search_ok=false"62    pm2 restart tk-api >> "$LOG" 2>&163    FAILURES=$((FAILURES+1)) ;;64  *)65    log "API muette"66    pm2 restart tk-api >> "$LOG" 2>&1 || heal_local "API morte"67    FAILURES=$((FAILURES+1)) ;;68esac6970# --- Web71WEB=$(curl -s -m 8 -o /dev/null -w "%{http_code}" http://127.0.0.1:3000)72if [ "$WEB" != "200" ]; then73  log "web local $WEB"74  pm2 restart tk-web >> "$LOG" 2>&175  FAILURES=$((FAILURES+1))76fi7778# --- Public (ngrok) : vue extérieure79PUB=$(curl -s -m 12 -o /dev/null -w "%{http_code}" https://www.trouve-ka.com/)80if [ "$PUB" = "000" ]; then81  log "public 000 : redémarrage ngrok"82  pkill -f "ngrok http" 2>/dev/null83  sleep 284  nohup ngrok http --url=www.trouve-ka.com 3000 >> "$HOME/trouve-ka-ngrok.log" 2>&1 &85  FAILURES=$((FAILURES+1))86fi8788# --- pm2 en erreur89ERRORED=$(pm2 jlist 2>/dev/null | grep -o '"name":"[^"]*","pm2_env":{"status":"errored"' | grep -o 'tk-[a-z0-9-]*')90for name in $ERRORED; do91  log "pm2 $name errored : restart"92  pm2 restart "$name" >> "$LOG" 2>&193  FAILURES=$((FAILURES+1))94done9596# --- Passe de flotte toutes les 15 exécutions (~30 min) : boot.sh partout97COUNT=$(cat "$STATE" 2>/dev/null || echo 0)98COUNT=$((COUNT+1))99echo "$COUNT" > "$STATE"100if [ $((COUNT % 15)) -eq 0 ]; then101  log "passe de flotte (#$COUNT)"102  for ip in $SATELLITES; do heal_remote "$ip"; done103fi104105[ "$FAILURES" -eq 0 ] || log "cycle terminé : $FAILURES réparation(s)"106107# --- Rotation du journal (garder ~5000 lignes)108if [ "$(wc -l < "$LOG" 2>/dev/null || echo 0)" -gt 10000 ]; then109  tail -5000 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"110fi111exit 0112