spb/internetpressure

1#!/usr/bin/env bash2# Nightly backup, runs ON BHS64b (installed as a cron by deploy/bin/backup.sh install from the laptop):3#   - Postgres: pg_dump (registry, config, events, annotations)4#   - ClickHouse: the irreplaceable tables (pressure_history, bgp_stats_10s, signal_features last 7 d) as Native+zstd5#   - copies to BHS128:/srv/backups/internetpressure (off-node, same DC), keeps 14 days locally6#7#   deploy/bin/backup.sh install     (from the laptop) copy this script to the server + cron 04:10 UTC8#   deploy/bin/backup.sh run         (on the server)  perform a backup now9set -euo pipefail1011if [[ "${1:-}" == "install" ]]; then12  source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"13  scp -q "$REPO_ROOT/deploy/bin/backup.sh" "$IP_SERVER:/opt/internetpressure/deploy/bin/backup.sh"14  rssh 'chmod +x /opt/internetpressure/deploy/bin/backup.sh; sudo mkdir -p /var/backups/internetpressure && sudo chown $(id -un) /var/backups/internetpressure;15        (crontab -l 2>/dev/null | grep -v internetpressure/deploy/bin/backup.sh; echo "10 4 * * * /opt/internetpressure/deploy/bin/backup.sh run >> /var/backups/internetpressure/backup.log 2>&1") | crontab -; crontab -l | grep backup'16  ssh -n BHS128 'sudo mkdir -p /srv/backups/internetpressure && sudo chown ubuntu /srv/backups/internetpressure'17  ok "backup cron installed"18  exit 019fi2021[[ "${1:-}" == "run" ]] || { echo "usage: backup.sh install|run" >&2; exit 2; }22cd /opt/internetpressure23DEST=/var/backups/internetpressure24STAMP=$(date -u +%Y%m%d)25mkdir -p "$DEST/$STAMP"26COMPOSE="docker compose -f infra/compose.yml --env-file deploy/.env"27CHPW=$(grep '^CLICKHOUSE_PASSWORD=' deploy/.env | cut -d= -f2)2829echo "[$(date -u +%FT%TZ)] postgres"30$COMPOSE exec -T postgres pg_dump -U ip -d ip --no-owner | gzip -6 > "$DEST/$STAMP/postgres.sql.gz"3132for t in pressure_history bgp_stats_10s bgp_origin_1m events_placeholder; do33  [[ "$t" == "events_placeholder" ]] && continue34  echo "[$(date -u +%FT%TZ)] clickhouse $t"35  $COMPOSE exec -T clickhouse clickhouse-client -u ip --password "$CHPW" -d ip \36    -q "SELECT * FROM $t FORMAT Native" | zstd -q -3 -o "$DEST/$STAMP/$t.native.zst" -f37done38echo "[$(date -u +%FT%TZ)] clickhouse signal_features (7 d)"39$COMPOSE exec -T clickhouse clickhouse-client -u ip --password "$CHPW" -d ip \40  -q "SELECT * FROM signal_features WHERE ts >= now() - INTERVAL 7 DAY FORMAT Native" | zstd -q -3 -o "$DEST/$STAMP/signal_features_7d.native.zst" -f4142du -sh "$DEST/$STAMP"43rsync -a --delete-after "$DEST/" BHS128:/srv/backups/internetpressure/ 2>/dev/null \44  || rsync -a "$DEST/" ubuntu@51.161.112.69:/srv/backups/internetpressure/ || echo "off-node copy failed"45find "$DEST" -maxdepth 1 -type d -name '20*' -mtime +14 -exec rm -rf {} +46echo "[$(date -u +%FT%TZ)] done"47