#!/usr/bin/env bash # deploy.sh data|crawl [--no-build] [--skip-migrate] [--no-backup-setup] # # data : rsync → build (web/api/worker images) → up stores → run migrate (migrations + seed + ClickHouse DDL) # → up everything → install the dci-backup systemd timer → wait for http://10.67.0.60:8300/api/health # crawl : rsync → build worker image → up → wait for http://10.68.0.2:8320/healthz # # Zero-downtime-ish: images are built before `up -d`, so containers are only recreated when their image or # config changed; edge/web/api restart in a few seconds each and Caddy retries upstreams (lb_try_duration). source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" target="${1:-}"; shift || true check_target "$target" build=1; migrate=1; backup_setup=1 for a in "$@"; do case "$a" in --no-build) build=0 ;; --skip-migrate) migrate=0 ;; --no-backup-setup) backup_setup=0 ;; *) die "unknown flag $a" ;; esac done host="$(host_for "$target")" # ── preflight ──────────────────────────────────────────────────────────────────────────────────────── log "preflight ($target → $host)" require_env_file "$target" remote "$target" "command -v docker >/dev/null && docker compose version >/dev/null" || die "docker compose missing on $host" if [[ $target == data ]]; then remote "$target" "ip -4 -o addr | grep -q ' $DATA_WG_IP/' && ip -4 -o addr | grep -q ' $DATA_PRIVATE_IP/'" \ || warn "$DATA_WG_IP (wg1) and/or $DATA_PRIVATE_IP (dci0) not present on $host — port binds will fail unless ip_nonlocal_bind is set" # Docker must be able to bind the WireGuard IPs even if wg comes up after dockerd (reboot ordering). remote "$target" "printf 'net.ipv4.ip_nonlocal_bind = 1\n' | sudo tee /etc/sysctl.d/90-dci.conf >/dev/null && sudo sysctl -q -p /etc/sysctl.d/90-dci.conf" else remote "$target" "ip -4 -o addr | grep -q ' $CRAWL_PRIVATE_IP/'" || warn "$CRAWL_PRIVATE_IP (dci0) not present on $host" remote "$target" "printf 'net.ipv4.ip_nonlocal_bind = 1\n' | sudo tee /etc/sysctl.d/90-dci.conf >/dev/null && sudo sysctl -q -p /etc/sysctl.d/90-dci.conf" remote "$target" "timeout 3 bash -c '/dev/null \ || warn "cannot reach $DATA_PRIVATE_IP:5432 from $host — deploy the data node first / check dci0" fi ensure_remote_layout "$target" # ── sync + build ────────────────────────────────────────────────────────────────────────────────────── sync_repo "$target" if [[ $build == 1 ]]; then log "building images on $host" compose "$target" "build" fi # ── up ──────────────────────────────────────────────────────────────────────────────────────────────── if [[ $target == data ]]; then log "starting stores" compose data "up -d postgres clickhouse redis minio" compose data "up -d minio-init" # one-shot: bucket + app user if [[ $migrate == 1 ]]; then log "migrate (postgres migrations → seed → clickhouse ddl)" compose data "run --rm migrate" fi log "starting application + monitoring" compose data "up -d --remove-orphans" if [[ $backup_setup == 1 ]]; then log "backup: systemd timer + off-node key to $(host_for crawl)" remote data "sudo install -m 644 $REMOTE_DIR/deploy/systemd/dci-backup.service $REMOTE_DIR/deploy/systemd/dci-backup.timer /etc/systemd/system/ \ && sudo systemctl daemon-reload && sudo systemctl enable --now dci-backup.timer" # key for the private-link rsync (ubuntu@BHS128 → ubuntu@10.68.0.2) remote data "test -f ~/.ssh/dci-backup || ssh-keygen -q -t ed25519 -N '' -C dci-backup@bhs128 -f ~/.ssh/dci-backup" pub="$(remote data "cat ~/.ssh/dci-backup.pub")" remote crawl "mkdir -p ~/.ssh && chmod 700 ~/.ssh && touch ~/.ssh/authorized_keys && grep -qF '$pub' ~/.ssh/authorized_keys || echo '$pub' >> ~/.ssh/authorized_keys; sudo mkdir -p $OFFSITE_DIR && sudo chown \$USER:\$USER $OFFSITE_DIR" remote data "ssh-keygen -F $CRAWL_PRIVATE_IP >/dev/null 2>&1 || ssh-keyscan -T 5 $CRAWL_PRIVATE_IP >> ~/.ssh/known_hosts 2>/dev/null" || warn "known_hosts for $CRAWL_PRIVATE_IP not populated" ok "backup timer installed: $(remote data 'systemctl list-timers dci-backup.timer --no-pager | sed -n 2p')" fi log "health" wait_http data "http://$DATA_WG_IP:8300/api/health" 60 5 || { compose data "ps"; compose data "logs --tail=50 api web edge"; die "edge/api not healthy"; } if curl -fsS -m 10 -o /dev/null "$PUBLIC_URL/api/health" 2>/dev/null; then ok "public: $PUBLIC_URL/api/health"; else warn "public route not answering yet — on BHS64: tunnelctl add www.datacenterindex.io BHS128:8300"; fi else log "starting workers" compose crawl "up -d --remove-orphans" log "health" wait_http crawl "http://$CRAWL_PRIVATE_IP:8320/healthz" 36 5 || { compose crawl "ps"; compose crawl "logs --tail=80 worker"; die "worker not healthy"; } fi compose "$target" "ps --format 'table {{.Service}}\t{{.Status}}\t{{.Ports}}'" ok "deploy $target done"