#!/bin/sh
# Trouve-KA — watchdog auto-guérisseur (tourne sur le hub M2M32 toutes les 2 min)
# Author: Simon-Pierre Boucher
# Contact: contact@spboucher.ai
#
# Vérifie chaque maillon et répare sans intervention humaine :
#   - API (health + search_ok), web, Postgres, Redis (localhost)
#   - tunnels OpenSearch (9210) et embeddings (8191)
#   - tunnel public ngrok (vue de l'extérieur)
#   - toutes les 30 min : passe de guérison sur toute la flotte satellite
#     (clé à commande forcée → exécute uniquement boot.sh distant)
# Journal : ~/trouveka-watchdog.log (tronqué automatiquement).

PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
export PATH
LOG="$HOME/trouveka-watchdog.log"
touch "$LOG"
STATE="$HOME/.trouveka-watchdog-count"
HEAL_KEY="$HOME/.ssh/trouveka_heal"
# Une IP LAN par ligne — générée à l'installation (scripts/ops/README)
SATELLITES=$(cat "$HOME/.trouveka-fleet" 2>/dev/null)

log() { echo "$(date '+%F %T') $*" >> "$LOG"; }

heal_local() {
  log "HEAL local : $1"
  /bin/sh "$HOME/trouve-ka/scripts/ops/boot.sh"
}

heal_remote() { # $1 = ip
  log "HEAL distant : $1"
  /usr/bin/ssh -i "$HEAL_KEY" -o ControlMaster=no -o ControlPath=none -o IdentitiesOnly=yes -o IdentityAgent=none -o ForwardAgent=no -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new \
    "simon-pierreboucher@$1" heal >> "$LOG" 2>&1 || log "HEAL $1 : injoignable"
}

FAILURES=0

# --- Postgres / Redis (docker local)
nc -z -w 3 127.0.0.1 5432 || { heal_local "postgres injoignable"; FAILURES=$((FAILURES+1)); }
nc -z -w 3 127.0.0.1 6379 || { heal_local "redis injoignable"; FAILURES=$((FAILURES+1)); }

# --- Tunnels
if ! curl -s -m 5 http://127.0.0.1:9210/_cluster/health | grep -q '"status"'; then
  heal_local "tunnel/OpenSearch 9210 muet"
  sleep 3
  curl -s -m 5 http://127.0.0.1:9210/_cluster/health | grep -q '"status"' || heal_remote 192.168.2.77
  FAILURES=$((FAILURES+1))
fi
if ! curl -s -m 5 http://127.0.0.1:8191/health | grep -q '"ok"'; then
  heal_local "tunnel/embeddings 8191 muet"
  sleep 3
  curl -s -m 5 http://127.0.0.1:8191/health | grep -q '"ok"' || heal_remote 192.168.2.75
  FAILURES=$((FAILURES+1))
fi

# --- API : santé + moteur de recherche réellement joignable
HEALTH=$(curl -s -m 8 http://127.0.0.1:8080/api/health)
case "$HEALTH" in
  *'"ok":true'*'"search_ok":true'*) : ;;
  *'"search_ok":false'*)
    log "API up mais search_ok=false"
    pm2 restart tk-api >> "$LOG" 2>&1
    FAILURES=$((FAILURES+1)) ;;
  *)
    log "API muette"
    pm2 restart tk-api >> "$LOG" 2>&1 || heal_local "API morte"
    FAILURES=$((FAILURES+1)) ;;
esac

# --- Web
WEB=$(curl -s -m 8 -o /dev/null -w "%{http_code}" http://127.0.0.1:3000)
if [ "$WEB" != "200" ]; then
  log "web local $WEB"
  pm2 restart tk-web >> "$LOG" 2>&1
  FAILURES=$((FAILURES+1))
fi

# --- Public (ngrok) : vue extérieure
PUB=$(curl -s -m 12 -o /dev/null -w "%{http_code}" https://www.trouve-ka.com/)
if [ "$PUB" = "000" ]; then
  log "public 000 : redémarrage ngrok"
  pkill -f "ngrok http" 2>/dev/null
  sleep 2
  nohup ngrok http --url=www.trouve-ka.com 3000 >> "$HOME/trouve-ka-ngrok.log" 2>&1 &
  FAILURES=$((FAILURES+1))
fi

# --- pm2 en erreur
ERRORED=$(pm2 jlist 2>/dev/null | grep -o '"name":"[^"]*","pm2_env":{"status":"errored"' | grep -o 'tk-[a-z0-9-]*')
for name in $ERRORED; do
  log "pm2 $name errored : restart"
  pm2 restart "$name" >> "$LOG" 2>&1
  FAILURES=$((FAILURES+1))
done

# --- Passe de flotte toutes les 15 exécutions (~30 min) : boot.sh partout
COUNT=$(cat "$STATE" 2>/dev/null || echo 0)
COUNT=$((COUNT+1))
echo "$COUNT" > "$STATE"
if [ $((COUNT % 15)) -eq 0 ]; then
  log "passe de flotte (#$COUNT)"
  for ip in $SATELLITES; do heal_remote "$ip"; done
fi

[ "$FAILURES" -eq 0 ] || log "cycle terminé : $FAILURES réparation(s)"

# --- Rotation du journal (garder ~5000 lignes)
if [ "$(wc -l < "$LOG" 2>/dev/null || echo 0)" -gt 10000 ]; then
  tail -5000 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"
fi
exit 0