spb/datacenterindex

1#!/usr/bin/env bash2# Runs ON the data node (BHS128) — by the dci-backup systemd timer (daily 03:30) or deploy/bin/backup.sh.3#   1. pg_dump -Fc dci                 → $BACKUP_DIR/pg/dci-<stamp>.dump4#   2. connector configs + parsers     → $BACKUP_DIR/config/dci-config-<stamp>.tar.zst (+ git rev)5#   3. ClickHouse BACKUP DATABASE dci  → $BACKUP_DIR/clickhouse/dci-ch-<stamp>.zip (best effort)6#   4. mc mirror dci-raw               → $BACKUP_DIR/minio/dci-raw (incremental mirror, not versioned)7#   5. retention: keep 14 daily + 8 weekly (Sunday) for 1–38#   6. rsync $BACKUP_DIR → ubuntu@10.68.0.2:/srv/dci-backups/ over the private link (key ~/.ssh/dci-backup)9set -euo pipefail10APP_DIR="${DCI_APP_DIR:-/srv/dci/app}"11BACKUP_DIR="${DCI_BACKUP_DIR:-/srv/dci/backups}"12OFFSITE_HOST="${DCI_OFFSITE_HOST:-10.68.0.2}"13OFFSITE_DIR="${DCI_OFFSITE_DIR:-/srv/dci-backups}"14OFFSITE="${DCI_OFFSITE:-1}"15KEEP_DAILY="${DCI_KEEP_DAILY:-14}"16KEEP_WEEKLY="${DCI_KEEP_WEEKLY:-8}"17STAMP="$(date +%Y%m%d-%H%M%S)"18LOCK=/tmp/dci-backup.lock1920exec 9>"$LOCK"; flock -n 9 || { echo "backup already running"; exit 0; }21say() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"; }2223cd "$APP_DIR"24if docker info >/dev/null 2>&1; then DOCKER=docker; else DOCKER="sudo docker"; fi25dc() { $DOCKER compose -f deploy/compose.data.yml --env-file deploy/.env.data "$@"; }26mkdir -p "$BACKUP_DIR"/{pg,config,clickhouse,minio}2728# 1. Postgres29say "pg_dump → pg/dci-$STAMP.dump"30# streamed to the host (written as the invoking user; the container's postgres uid cannot write the bind mount)31dc exec -T postgres pg_dump -U dci -d dci -Fc --no-owner --compress=zstd > "$BACKUP_DIR/pg/dci-$STAMP.dump.part"32mv "$BACKUP_DIR/pg/dci-$STAMP.dump.part" "$BACKUP_DIR/pg/dci-$STAMP.dump"33say "  $(du -h "$BACKUP_DIR/pg/dci-$STAMP.dump" | cut -f1)"3435# 2. Config + parser sources36say "config → config/dci-config-$STAMP.tar.zst"37{ git -C "$APP_DIR" rev-parse HEAD 2>/dev/null || echo "no-git ($(date -Iseconds))"; } > "$BACKUP_DIR/config/GIT_REV-$STAMP.txt"38tar --zstd -cf "$BACKUP_DIR/config/dci-config-$STAMP.tar.zst" -C "$APP_DIR" \39  --ignore-failed-read \40  config/connectors packages/connectors/src apps/worker/src/parsers apps/worker/src/connectors packages/db/migrations \41  deploy/compose.data.yml deploy/compose.crawl.yml deploy/edge deploy/clickhouse deploy/monitoring \42  -C "$BACKUP_DIR/config" "GIT_REV-$STAMP.txt" 2>/dev/null || true43rm -f "$BACKUP_DIR/config/GIT_REV-$STAMP.txt"4445# 3. ClickHouse (best effort — analytics are derived)46say "clickhouse BACKUP DATABASE dci → clickhouse/dci-ch-$STAMP.zip"47CH_PW="$(sed -nE 's/^CLICKHOUSE_PASSWORD=(.*)$/\1/p' deploy/.env.data | tr -d '"')"48CH_DB="$(sed -nE 's/^CLICKHOUSE_DB=(.*)$/\1/p' deploy/.env.data | tr -d '"')"; CH_DB="${CH_DB:-dci}"49if ! dc exec -T clickhouse clickhouse-client --user dci --password "$CH_PW" \50     --query "BACKUP DATABASE $CH_DB TO File('/backups/dci-ch-$STAMP.zip') SETTINGS compression_method='zstd'" >/dev/null; then51  say "  clickhouse backup failed (non-fatal)"52fi53# the zip is written by the clickhouse container user (uid 101, mode 640) → make it readable for the offsite rsync54sudo chown -R "$(id -u):$(id -g)" "$BACKUP_DIR/clickhouse" 2>/dev/null || true55chmod -R u+rwX,go+rX "$BACKUP_DIR/clickhouse" 2>/dev/null || true5657# 4. MinIO raw archive mirror58say "mc mirror dci-raw → minio/dci-raw"59dc run --rm --no-deps --entrypoint /bin/sh minio-init -c \60  'mc alias set local http://minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null && mc mirror --overwrite --quiet local/'"${S3_BUCKET:-dci-raw}"' /backup/dci-raw' \61  || say "  mc mirror failed (non-fatal)"6263# 5. Retention (14 daily + 8 weekly on Sundays)64prune() {65  local dir="$1" f d age dow now; now=$(date +%s)66  for f in "$dir"/dci-*; do67    [[ -e $f ]] || continue68    d="$(basename "$f" | grep -oE '[0-9]{8}' | head -1)" || continue69    [[ -n $d ]] || continue70    age=$(( (now - $(date -d "$d" +%s)) / 86400 ))71    dow=$(date -d "$d" +%u)   # 7 = Sunday72    if (( age <= KEEP_DAILY )); then continue; fi73    if (( dow == 7 && age <= KEEP_DAILY + 7 * KEEP_WEEKLY )); then continue; fi74    say "  prune $(basename "$f")"; rm -rf -- "$f"75  done76}77say "retention: $KEEP_DAILY daily + $KEEP_WEEKLY weekly"78prune "$BACKUP_DIR/pg"; prune "$BACKUP_DIR/config"; prune "$BACKUP_DIR/clickhouse"7980# 6. Off-node copy over the private link81if [[ $OFFSITE == 1 ]]; then82  say "rsync → ubuntu@$OFFSITE_HOST:$OFFSITE_DIR"83  rsync -a --delete --info=stats1 \84    -e "ssh -i $HOME/.ssh/dci-backup -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new" \85    "$BACKUP_DIR/" "ubuntu@$OFFSITE_HOST:$OFFSITE_DIR/" \86    || say "  offsite rsync failed (check ~/.ssh/dci-backup is authorized on $OFFSITE_HOST)"87fi8889say "done — $(du -sh "$BACKUP_DIR" | cut -f1) in $BACKUP_DIR; latest: $(ls -1t "$BACKUP_DIR/pg" | head -1)"90