#!/usr/bin/env bash # Nightly backup, runs ON BHS64b (installed as a cron by deploy/bin/backup.sh install from the laptop): # - Postgres: pg_dump (registry, config, events, annotations) # - ClickHouse: the irreplaceable tables (pressure_history, bgp_stats_10s, signal_features last 7 d) as Native+zstd # - copies to BHS128:/srv/backups/internetpressure (off-node, same DC), keeps 14 days locally # # deploy/bin/backup.sh install (from the laptop) copy this script to the server + cron 04:10 UTC # deploy/bin/backup.sh run (on the server) perform a backup now set -euo pipefail if [[ "${1:-}" == "install" ]]; then source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" scp -q "$REPO_ROOT/deploy/bin/backup.sh" "$IP_SERVER:/opt/internetpressure/deploy/bin/backup.sh" rssh 'chmod +x /opt/internetpressure/deploy/bin/backup.sh; sudo mkdir -p /var/backups/internetpressure && sudo chown $(id -un) /var/backups/internetpressure; (crontab -l 2>/dev/null | grep -v internetpressure/deploy/bin/backup.sh; echo "10 4 * * * /opt/internetpressure/deploy/bin/backup.sh run >> /var/backups/internetpressure/backup.log 2>&1") | crontab -; crontab -l | grep backup' ssh -n BHS128 'sudo mkdir -p /srv/backups/internetpressure && sudo chown ubuntu /srv/backups/internetpressure' ok "backup cron installed" exit 0 fi [[ "${1:-}" == "run" ]] || { echo "usage: backup.sh install|run" >&2; exit 2; } cd /opt/internetpressure DEST=/var/backups/internetpressure STAMP=$(date -u +%Y%m%d) mkdir -p "$DEST/$STAMP" COMPOSE="docker compose -f infra/compose.yml --env-file deploy/.env" CHPW=$(grep '^CLICKHOUSE_PASSWORD=' deploy/.env | cut -d= -f2) echo "[$(date -u +%FT%TZ)] postgres" $COMPOSE exec -T postgres pg_dump -U ip -d ip --no-owner | gzip -6 > "$DEST/$STAMP/postgres.sql.gz" for t in pressure_history bgp_stats_10s bgp_origin_1m events_placeholder; do [[ "$t" == "events_placeholder" ]] && continue echo "[$(date -u +%FT%TZ)] clickhouse $t" $COMPOSE exec -T clickhouse clickhouse-client -u ip --password "$CHPW" -d ip \ -q "SELECT * FROM $t FORMAT Native" | zstd -q -3 -o "$DEST/$STAMP/$t.native.zst" -f done echo "[$(date -u +%FT%TZ)] clickhouse signal_features (7 d)" $COMPOSE exec -T clickhouse clickhouse-client -u ip --password "$CHPW" -d ip \ -q "SELECT * FROM signal_features WHERE ts >= now() - INTERVAL 7 DAY FORMAT Native" | zstd -q -3 -o "$DEST/$STAMP/signal_features_7d.native.zst" -f du -sh "$DEST/$STAMP" rsync -a --delete-after "$DEST/" BHS128:/srv/backups/internetpressure/ 2>/dev/null \ || rsync -a "$DEST/" ubuntu@51.161.112.69:/srv/backups/internetpressure/ || echo "off-node copy failed" find "$DEST" -maxdepth 1 -type d -name '20*' -mtime +14 -exec rm -rf {} + echo "[$(date -u +%FT%TZ)] done"