spb/cancerindex

1#!/usr/bin/env bash2# CancerIndex — ordered first ingestion (idempotent; safe to re-run).3#4# Runs connectors in the recommended order (terminology → genes → evidence/genomics/variants →5# trials → literature → epidemiology) with per-connector time budgets, then rebuilds counters and6# rankings. Connectors are restartable (cursor) and idempotent (payload hash), so re-running this7# script only fetches what is missing. A connector that is not registered yet, paused, awaiting8# credentials or under license review is skipped with a message — never treated as an error.9#10# Usage: cd ~/apps/cancerindex && bash deploy/first-run.sh [--only id,id] [--skip id,id] [--dry-run]11set -uo pipefail1213cd "$(dirname "$0")/.." || exit 114export DATABASE_URL="${DATABASE_URL:-postgres://localhost:5432/cancerindex}"15export CI_DATA_DIR="${CI_DATA_DIR:-$PWD/data}"16export CI_SERVICE=first-run17LOG_DIR="${LOG_DIR:-$PWD/logs}"18mkdir -p "$LOG_DIR" data/raw data/cache1920# connector id : time budget (minutes) : max attempts within this script21ORDER=(22  "ncit-evs:45:2"23  "oncotree:10:2"24  "hgnc:15:2"25  "civic:30:2"26  "gdc:30:2"27  "clinvar:45:2"28  "clinicaltrials:45:3"29  "pubmed:45:3"30  "cdc-wonder:30:2"31)3233ONLY=""; SKIP=""; DRY=""34while [ $# -gt 0 ]; do35  case "$1" in36    --only) ONLY="$2"; shift 2 ;;37    --skip) SKIP="$2"; shift 2 ;;38    --dry-run) DRY="--mode dry_run --max-records 100"; shift ;;39    *) echo "unknown option $1"; exit 2 ;;40  esac41done4243ts() { date -u +%Y-%m-%dT%H:%M:%SZ; }44log() { echo "[$(ts)] $*" | tee -a "$LOG_DIR/first-run.log"; }45in_list() { [ -z "$2" ] && return 1; echo ",$2," | grep -q ",$1,"; }4647log "== CancerIndex first run (db=$DATABASE_URL, data=$CI_DATA_DIR) =="48pnpm cix sources:sync >>"$LOG_DIR/first-run.log" 2>&1 || log "warning: sources:sync failed (continuing)"49REGISTERED="$(pnpm --silent cix connectors 2>/dev/null | awk '{print $1}' | tr '\n' ',')"5051last_status() {52  psql "$DATABASE_URL" -Atc "SELECT status FROM ingest_runs WHERE connector_id = '$1' ORDER BY started_at DESC LIMIT 1" 2>/dev/null53}54cursor_health() {55  psql "$DATABASE_URL" -Atc "SELECT coalesce(health,'unknown') || '|' || coalesce(paused::text,'false') FROM connector_cursors WHERE connector_id = '$1'" 2>/dev/null56}5758for entry in "${ORDER[@]}"; do59  IFS=: read -r id minutes attempts <<<"$entry"60  if [ -n "$ONLY" ] && ! in_list "$id" "$ONLY"; then continue; fi61  if in_list "$id" "$SKIP"; then log "-- $id: skipped (--skip)"; continue; fi62  if ! echo ",$REGISTERED" | grep -q ",$id,"; then log "-- $id: not registered in packages/connectors/src/registry.ts — skipped"; continue; fi63  hp="$(cursor_health "$id")"64  case "$hp" in65    awaiting_credentials*) log "-- $id: awaiting credentials — skipped"; continue ;;66    *"|true") log "-- $id: paused — skipped"; continue ;;67  esac68  if [ -z "$DRY" ] && [ "$(last_status "$id")" = "succeeded" ] && [ "${FORCE:-}" != "1" ]; then69    log "-- $id: last run already succeeded — running incremental refresh (FORCE=1 to force full)"70  fi71  n=072  while [ "$n" -lt "$attempts" ]; do73    n=$((n + 1))74    log "-> $id attempt $n/$attempts (budget ${minutes} min)"75    # shellcheck disable=SC208676    if pnpm cix run "$id" --max-minutes "$minutes" $DRY >>"$LOG_DIR/first-run.$id.log" 2>&1; then77      st="$(last_status "$id")"78      log "   $id: $st"79      # partial = time budget hit with cursor saved → run again while attempts remain80      [ "$st" = "partial" ] && [ "$n" -lt "$attempts" ] && continue81      break82    else83      st="$(last_status "$id")"84      log "   $id: FAILED (status=${st:-none}); see $LOG_DIR/first-run.$id.log"85      case "$st" in aborted) break ;; esac   # license/credentials gate: do not retry86    fi87  done88done8990if [ -z "$DRY" ]; then91  log "-> counters"92  pnpm cix counters >>"$LOG_DIR/first-run.log" 2>&1 && log "   counters ok" || log "   counters FAILED"93  log "-> rank"94  pnpm cix rank >>"$LOG_DIR/first-run.log" 2>&1 && log "   rank ok" || log "   rank FAILED"95fi9697log "== summary =="98pnpm --silent cix connectors 2>/dev/null | tee -a "$LOG_DIR/first-run.log"99pnpm --silent cix stats 2>/dev/null | tee -a "$LOG_DIR/first-run.log"100log "== done =="101