#!/usr/bin/env bash # CancerIndex — PostgreSQL backup (CLAUDE.md §172). # # pg_dump -Fc → $BACKUP_DIR/cancerindex-YYYYMMDD-HHMM.dump, verified with pg_restore --list, # retention: last 14 daily dumps + 8 weekly (the newest dump of each ISO week), everything else pruned. # Logs to $LOG_DIR/backup.log; exits non-zero on any failure (PM2 shows the process as errored). # # Usage (on the node): cd ~/apps/cancerindex && bash deploy/backup.sh # DATABASE_URL postgres://localhost:5432/cancerindex (default) # BACKUP_DIR ~/apps/cancerindex/backups (default) # LOG_DIR /logs (default) # KEEP_DAILY=14 KEEP_WEEKLY=8 # # Scheduled by PM2 (`cancerindex-backup`, cron_restart "20 5 * * *", autorestart false) — see the mld manifest. # The raw data lake (data/raw) is append-only and NOT part of this dump: rsync it separately (deploy/README.md). set -uo pipefail cd "$(dirname "$0")/.." || exit 1 DATABASE_URL="${DATABASE_URL:-postgres://localhost:5432/cancerindex}" BACKUP_DIR="${BACKUP_DIR:-$HOME/apps/cancerindex/backups}" LOG_DIR="${LOG_DIR:-$PWD/logs}" KEEP_DAILY="${KEEP_DAILY:-14}" KEEP_WEEKLY="${KEEP_WEEKLY:-8}" PREFIX="cancerindex-" mkdir -p "$BACKUP_DIR" "$LOG_DIR" || { echo "cannot create $BACKUP_DIR / $LOG_DIR" >&2; exit 1; } LOG="$LOG_DIR/backup.log" ts() { date -u +%Y-%m-%dT%H:%M:%SZ; } log() { echo "[$(ts)] $*" | tee -a "$LOG"; } fail() { log "ERROR: $*"; exit 1; } command -v pg_dump >/dev/null || fail "pg_dump not found in PATH" command -v pg_restore >/dev/null || fail "pg_restore not found in PATH" STAMP="$(date +%Y%m%d-%H%M)" OUT="$BACKUP_DIR/${PREFIX}${STAMP}.dump" TMP="$OUT.partial" DB_LABEL="$(printf '%s' "$DATABASE_URL" | sed -E 's#//([^:@/]+):[^@/]+@#//\1:***@#')" log "== backup start db=$DB_LABEL → $OUT" t0=$(date +%s) if ! pg_dump -Fc --no-owner --no-acl -d "$DATABASE_URL" -f "$TMP" 2>>"$LOG"; then rm -f "$TMP" fail "pg_dump failed (see $LOG)" fi mv "$TMP" "$OUT" || fail "cannot move $TMP → $OUT" # Verification: the archive must be a readable custom-format dump listing our core tables. if ! LIST="$(pg_restore --list "$OUT" 2>>"$LOG")"; then fail "pg_restore --list failed on $OUT — dump is corrupt" fi TABLES="$(printf '%s\n' "$LIST" | grep -c 'TABLE DATA' || true)" for t in cancers provenance ingest_runs; do printf '%s\n' "$LIST" | grep -q "TABLE DATA public $t " || fail "table $t missing from dump listing" done SIZE="$(du -h "$OUT" | cut -f1)" log "dump ok: $SIZE, $TABLES tables with data, $(( $(date +%s) - t0 )) s" # Retention: keep the newest $KEEP_DAILY dumps, plus the newest dump of each of the last $KEEP_WEEKLY ISO weeks. KEEP_FILE="$(mktemp)" # shellcheck disable=SC2012 ls -1 "$BACKUP_DIR"/${PREFIX}*.dump 2>/dev/null | sort -r | head -n "$KEEP_DAILY" >"$KEEP_FILE" # (no associative arrays: macOS /bin/bash is 3.2) SEEN_WEEKS=" " weeks=0 for f in $(ls -1 "$BACKUP_DIR"/${PREFIX}*.dump 2>/dev/null | sort -r); do base="$(basename "$f" .dump)"; d="${base#"$PREFIX"}"; d="${d%%-*}" # YYYYMMDD if date -j -f %Y%m%d "$d" +%G-%V >/dev/null 2>&1; then wk="$(date -j -f %Y%m%d "$d" +%G-%V)"; else wk="$(date -d "$d" +%G-%V 2>/dev/null || echo "$d")"; fi case "$SEEN_WEEKS" in *" $wk "*) continue ;; esac if [ "$weeks" -lt "$KEEP_WEEKLY" ]; then SEEN_WEEKS="$SEEN_WEEKS$wk "; weeks=$((weeks + 1)); echo "$f" >>"$KEEP_FILE" fi done pruned=0 for f in "$BACKUP_DIR"/${PREFIX}*.dump; do [ -e "$f" ] || continue if ! grep -qxF "$f" "$KEEP_FILE"; then rm -f "$f" && pruned=$((pruned + 1)) && log "pruned $(basename "$f")"; fi done rm -f "$KEEP_FILE" "$BACKUP_DIR"/${PREFIX}*.dump.partial TOTAL="$(du -sh "$BACKUP_DIR" | cut -f1)" log "retention: kept $(ls -1 "$BACKUP_DIR"/${PREFIX}*.dump | wc -l | tr -d ' ') dumps ($TOTAL), pruned $pruned" log "== backup done"