spb/datacenterindex

1#!/usr/bin/env bash2# deploy.sh data|crawl [--no-build] [--skip-migrate] [--no-backup-setup]3#4# data  : rsync → build (web/api/worker images) → up stores → run migrate (migrations + seed + ClickHouse DDL)5#         → up everything → install the dci-backup systemd timer → wait for http://10.67.0.60:8300/api/health6# crawl : rsync → build worker image → up → wait for http://10.68.0.2:8320/healthz7#8# Zero-downtime-ish: images are built before `up -d`, so containers are only recreated when their image or9# config changed; edge/web/api restart in a few seconds each and Caddy retries upstreams (lb_try_duration).10source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"1112target="${1:-}"; shift || true13check_target "$target"14build=1; migrate=1; backup_setup=115for a in "$@"; do16  case "$a" in17    --no-build) build=0 ;;18    --skip-migrate) migrate=0 ;;19    --no-backup-setup) backup_setup=0 ;;20    *) die "unknown flag $a" ;;21  esac22done23host="$(host_for "$target")"2425# ── preflight ────────────────────────────────────────────────────────────────────────────────────────26log "preflight ($target → $host)"27require_env_file "$target"28remote "$target" "command -v docker >/dev/null && docker compose version >/dev/null" || die "docker compose missing on $host"29if [[ $target == data ]]; then30  remote "$target" "ip -4 -o addr | grep -q ' $DATA_WG_IP/' && ip -4 -o addr | grep -q ' $DATA_PRIVATE_IP/'" \31    || warn "$DATA_WG_IP (wg1) and/or $DATA_PRIVATE_IP (dci0) not present on $host — port binds will fail unless ip_nonlocal_bind is set"32  # Docker must be able to bind the WireGuard IPs even if wg comes up after dockerd (reboot ordering).33  remote "$target" "printf 'net.ipv4.ip_nonlocal_bind = 1\n' | sudo tee /etc/sysctl.d/90-dci.conf >/dev/null && sudo sysctl -q -p /etc/sysctl.d/90-dci.conf"34else35  remote "$target" "ip -4 -o addr | grep -q ' $CRAWL_PRIVATE_IP/'" || warn "$CRAWL_PRIVATE_IP (dci0) not present on $host"36  remote "$target" "printf 'net.ipv4.ip_nonlocal_bind = 1\n' | sudo tee /etc/sysctl.d/90-dci.conf >/dev/null && sudo sysctl -q -p /etc/sysctl.d/90-dci.conf"37  remote "$target" "timeout 3 bash -c '</dev/tcp/$DATA_PRIVATE_IP/5432'" 2>/dev/null \38    || warn "cannot reach $DATA_PRIVATE_IP:5432 from $host — deploy the data node first / check dci0"39fi40ensure_remote_layout "$target"4142# ── sync + build ──────────────────────────────────────────────────────────────────────────────────────43sync_repo "$target"44if [[ $build == 1 ]]; then45  log "building images on $host"46  compose "$target" "build"47fi4849# ── up ────────────────────────────────────────────────────────────────────────────────────────────────50if [[ $target == data ]]; then51  log "starting stores"52  compose data "up -d postgres clickhouse redis minio"53  compose data "up -d minio-init"   # one-shot: bucket + app user54  if [[ $migrate == 1 ]]; then55    log "migrate (postgres migrations → seed → clickhouse ddl)"56    compose data "run --rm migrate"57  fi58  log "starting application + monitoring"59  compose data "up -d --remove-orphans"60  if [[ $backup_setup == 1 ]]; then61    log "backup: systemd timer + off-node key to $(host_for crawl)"62    remote data "sudo install -m 644 $REMOTE_DIR/deploy/systemd/dci-backup.service $REMOTE_DIR/deploy/systemd/dci-backup.timer /etc/systemd/system/ \63      && sudo systemctl daemon-reload && sudo systemctl enable --now dci-backup.timer"64    # key for the private-link rsync (ubuntu@BHS128 → ubuntu@10.68.0.2)65    remote data "test -f ~/.ssh/dci-backup || ssh-keygen -q -t ed25519 -N '' -C dci-backup@bhs128 -f ~/.ssh/dci-backup"66    pub="$(remote data "cat ~/.ssh/dci-backup.pub")"67    remote crawl "mkdir -p ~/.ssh && chmod 700 ~/.ssh && touch ~/.ssh/authorized_keys && grep -qF '$pub' ~/.ssh/authorized_keys || echo '$pub' >> ~/.ssh/authorized_keys; sudo mkdir -p $OFFSITE_DIR && sudo chown \$USER:\$USER $OFFSITE_DIR"68    remote data "ssh-keygen -F $CRAWL_PRIVATE_IP >/dev/null 2>&1 || ssh-keyscan -T 5 $CRAWL_PRIVATE_IP >> ~/.ssh/known_hosts 2>/dev/null" || warn "known_hosts for $CRAWL_PRIVATE_IP not populated"69    ok "backup timer installed: $(remote data 'systemctl list-timers dci-backup.timer --no-pager | sed -n 2p')"70  fi71  log "health"72  wait_http data "http://$DATA_WG_IP:8300/api/health" 60 5 || { compose data "ps"; compose data "logs --tail=50 api web edge"; die "edge/api not healthy"; }73  if curl -fsS -m 10 -o /dev/null "$PUBLIC_URL/api/health" 2>/dev/null; then ok "public: $PUBLIC_URL/api/health"; else warn "public route not answering yet — on BHS64: tunnelctl add www.datacenterindex.io BHS128:8300"; fi74else75  log "starting workers"76  compose crawl "up -d --remove-orphans"77  log "health"78  wait_http crawl "http://$CRAWL_PRIVATE_IP:8320/healthz" 36 5 || { compose crawl "ps"; compose crawl "logs --tail=80 worker"; die "worker not healthy"; }79fi8081compose "$target" "ps --format 'table {{.Service}}\t{{.Status}}\t{{.Ports}}'"82ok "deploy $target done"83