# DataCenterIndex — DATA node (BHS128). Public traffic arrives from the BHS64 gateway over WireGuard wg1
# (edge :8300 bound to 10.67.0.60). Stores are published ONLY on the private link dci0 (10.68.0.1) for the
# crawl node (BHS64b, 10.68.0.2). Nothing is bound to 0.0.0.0 / the public IP.
#
#   docker compose -f deploy/compose.data.yml --env-file deploy/.env.data up -d --remove-orphans
#   docker compose -f deploy/compose.data.yml --env-file deploy/.env.data run --rm migrate
#   docker compose -f deploy/compose.data.yml --env-file deploy/.env.data run --rm cli run <connector> --dry-run --limit 5
#
# Profiles: (none) = production set (incl. scheduler + worker-maint, DCI_QUEUES=maintenance) · ops = migrate/cli one-shots
name: dci

x-logging: &logging
  driver: json-file
  options: { max-size: "50m", max-file: "5" }

x-restart: &restart
  restart: unless-stopped

# Environment shared by every process built from the repo (api, scheduler, migrate, cli, worker-maint).
x-app-env: &app-env
  NODE_ENV: production
  TZ: ${TZ:-America/Toronto}
  DATABASE_URL: postgres://dci:${POSTGRES_PASSWORD}@postgres:5432/dci
  PG_POOL_MAX: ${PG_POOL_MAX:-16}
  REDIS_URL: redis://:${REDIS_PASSWORD}@redis:6379/0
  CLICKHOUSE_URL: http://clickhouse:8123
  CLICKHOUSE_DB: ${CLICKHOUSE_DB:-dci}
  CLICKHOUSE_USER: dci
  CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
  S3_ENDPOINT: http://minio:9000
  S3_BUCKET: ${S3_BUCKET:-dci-raw}
  S3_ACCESS_KEY: ${S3_ACCESS_KEY}
  S3_SECRET_KEY: ${S3_SECRET_KEY}
  S3_REGION: us-east-1
  DCI_ADMIN_TOKEN: ${DCI_ADMIN_TOKEN}
  DCI_USER_AGENT: ${DCI_USER_AGENT}
  DCI_LOG_LEVEL: ${DCI_LOG_LEVEL:-info}
  DCI_CONFIG_DIR: /app/config/connectors
  SCRAPFLY_API_KEY: ${SCRAPFLY_API_KEY:-}
  FIRECRAWL_API_KEY: ${FIRECRAWL_API_KEY:-}
  DCI_SCRAPFLY_DAILY_BUDGET: ${DCI_SCRAPFLY_DAILY_BUDGET:-400}
  DCI_FIRECRAWL_DAILY_BUDGET: ${DCI_FIRECRAWL_DAILY_BUDGET:-200}
  NEXT_PUBLIC_SITE_URL: ${NEXT_PUBLIC_SITE_URL:-https://www.datacenterindex.io}

x-worker-image: &worker-image
  image: dci/worker:${DCI_TAG:-latest}
  build:
    context: ..
    dockerfile: deploy/docker/Dockerfile.worker
    args:
      TSX_VERSION: ${TSX_VERSION:-4.23.13}

services:
  # ───────────────────────────────────────────── edge (the only thing the gateway talks to)
  edge:
    image: caddy:2.10-alpine
    <<: *restart
    logging: *logging
    ports:
      - "${DATA_WG_IP:-10.67.0.60}:8300:8300"
    volumes:
      - ./edge/Caddyfile:/etc/caddy/Caddyfile:ro
      - caddy-data:/data
      - caddy-config:/config
    depends_on:
      web: { condition: service_started }
      api: { condition: service_started }
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8300/api/health"]
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 20s
    mem_limit: 512m
    cpus: 2

  # ───────────────────────────────────────────── application
  web:
    image: dci/web:${DCI_TAG:-latest}
    build:
      context: ..
      dockerfile: deploy/docker/Dockerfile.web
      args:
        NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-https://www.datacenterindex.io}
        NEXT_PUBLIC_SITE_URL: ${NEXT_PUBLIC_SITE_URL:-https://www.datacenterindex.io}
    <<: *restart
    logging: *logging
    environment:
      NODE_ENV: production
      TZ: ${TZ:-America/Toronto}
      PORT: "8310"
      HOSTNAME: 0.0.0.0
      NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-https://www.datacenterindex.io}
      NEXT_PUBLIC_SITE_URL: ${NEXT_PUBLIC_SITE_URL:-https://www.datacenterindex.io}
      API_URL_INTERNAL: http://api:8311
    depends_on:
      api: { condition: service_healthy }
    mem_limit: 4g
    cpus: 4

  api:
    image: dci/api:${DCI_TAG:-latest}
    build:
      context: ..
      dockerfile: deploy/docker/Dockerfile.api
      args:
        TSX_VERSION: ${TSX_VERSION:-4.23.13}
    <<: *restart
    logging: *logging
    environment:
      <<: *app-env
      API_PORT: "8311"
      HOST: 0.0.0.0
      NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-https://www.datacenterindex.io}
      DCI_API_CACHE_ENTRIES: ${DCI_API_CACHE_ENTRIES:-5000}
      DCI_TRUST_PROXY: "1"
      # internal worker HTTP (extraction debugger /trace, /data-gaps) — proxied behind the admin token
      DCI_WORKER_URL: http://worker-maint:8320
    depends_on:
      postgres: { condition: service_healthy }
      redis: { condition: service_healthy }
      clickhouse: { condition: service_healthy }
      minio: { condition: service_healthy }
    mem_limit: 4g
    cpus: 4

  # Scheduler: enqueues connector runs (BullMQ) — co-located with Redis. The crawl workers run on BHS64b.
  # /healthz + /metrics on 8321 (container network only; scraped by Prometheus as scheduler:8321). The image's
  # HEALTHCHECK probes http://127.0.0.1:$WORKER_PORT/healthz.
  scheduler:
    <<: [*worker-image, *restart]
    command: ["scheduler"]
    logging: *logging
    environment:
      <<: *app-env
      DCI_SCHEDULER_INTERVAL_MS: ${DCI_SCHEDULER_INTERVAL_MS:-60000}
      WORKER_PORT: "8321"
    healthcheck:
      test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:8321/healthz').then(r=>process.exit(r.ok?0:1),()=>process.exit(1))"]
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 40s
    depends_on:
      postgres: { condition: service_healthy }
      redis: { condition: service_healthy }
    mem_limit: 2g
    cpus: 2

  # Light worker for the maintenance queue only (rankings, metrics, refresh-stats, cleanup) — the worker honours
  # DCI_QUEUES, so this node never crawls. Part of the default set since 2026-09-11 (was profile `maintenance`).
  worker-maint:
    <<: [*worker-image, *restart]
    command: ["worker"]
    logging: *logging
    environment:
      <<: *app-env
      DCI_QUEUES: maintenance
      DCI_CRAWL_CONCURRENCY: "1"
      DCI_MAINTENANCE_CONCURRENCY: ${DCI_MAINTENANCE_CONCURRENCY:-2}
      WORKER_PORT: "8320"
    depends_on:
      postgres: { condition: service_healthy }
      redis: { condition: service_healthy }
    mem_limit: 4g
    cpus: 2

  # One-shots (profile ops): `compose run --rm migrate`, `compose run --rm cli <args>`
  migrate:
    <<: *worker-image
    profiles: ["ops"]
    restart: "no"
    command: ["migrate"]
    environment:
      <<: *app-env
    depends_on:
      postgres: { condition: service_healthy }
      clickhouse: { condition: service_healthy }
    mem_limit: 2g

  cli:
    <<: *worker-image
    profiles: ["ops"]
    restart: "no"
    command: ["cli", "--help"]
    environment:
      <<: *app-env
    volumes:
      - /srv/dci/out:/app/data
    depends_on:
      postgres: { condition: service_healthy }
      redis: { condition: service_healthy }
    mem_limit: 4g

  # ───────────────────────────────────────────── stores (published on the private link only)
  postgres:
    image: postgres:17
    <<: *restart
    logging: *logging
    command:
      - postgres
      - -c
      - shared_buffers=4GB
      - -c
      - effective_cache_size=32GB
      - -c
      - work_mem=64MB
      - -c
      - maintenance_work_mem=1GB
      - -c
      - max_connections=200
      - -c
      - max_wal_size=4GB
      - -c
      - min_wal_size=1GB
      - -c
      - checkpoint_completion_target=0.9
      - -c
      - wal_compression=zstd
      - -c
      - random_page_cost=1.1
      - -c
      - effective_io_concurrency=200
      - -c
      - max_worker_processes=16
      - -c
      - max_parallel_workers=8
      - -c
      - max_parallel_workers_per_gather=4
      - -c
      - shared_preload_libraries=pg_stat_statements
      - -c
      - log_min_duration_statement=500
      - -c
      - log_checkpoints=on
      - -c
      - timezone=UTC
    environment:
      POSTGRES_USER: dci
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      POSTGRES_DB: dci
      POSTGRES_INITDB_ARGS: "--data-checksums --encoding=UTF8 --locale=C.UTF-8"
      TZ: UTC
    ports:
      - "${DATA_PRIVATE_IP:-10.68.0.1}:5432:5432"
    volumes:
      - pgdata:/var/lib/postgresql/data
      - ./postgres/initdb:/docker-entrypoint-initdb.d:ro
      - /srv/dci/backups/pg:/backups
    shm_size: 2g
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U dci -d dci -h 127.0.0.1"]
      interval: 10s
      timeout: 5s
      retries: 10
      start_period: 30s
    mem_limit: 40g
    cpus: 8
    stop_grace_period: 60s

  clickhouse:
    image: clickhouse/clickhouse-server:25.8
    <<: *restart
    logging: *logging
    environment:
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
      CLICKHOUSE_SKIP_USER_SETUP: "1"
      TZ: UTC
    ports:
      - "${DATA_PRIVATE_IP:-10.68.0.1}:8123:8123"
      # native protocol: host port 9009 (9000 on 10.68.0.1 belongs to MinIO S3)
      - "${DATA_PRIVATE_IP:-10.68.0.1}:9009:9000"
    volumes:
      - chdata:/var/lib/clickhouse
      - ./clickhouse/config.d/dci.xml:/etc/clickhouse-server/config.d/dci.xml:ro
      - ./clickhouse/users.d/dci.xml:/etc/clickhouse-server/users.d/dci.xml:ro
      - /srv/dci/backups/clickhouse:/backups
    ulimits:
      nofile: { soft: 262144, hard: 262144 }
    cap_add: [SYS_NICE, NET_ADMIN, IPC_LOCK]
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8123/ping"]
      interval: 10s
      timeout: 5s
      retries: 10
      start_period: 30s
    mem_limit: 24g
    cpus: 8

  redis:
    image: redis:7.4-alpine
    <<: *restart
    logging: *logging
    command:
      - redis-server
      - --appendonly
      - "yes"
      - --appendfsync
      - everysec
      - --requirepass
      - ${REDIS_PASSWORD}
      - --maxmemory
      - 4gb
      - --maxmemory-policy
      - noeviction
      - --save
      - "900 1"
      - --tcp-keepalive
      - "60"
    ports:
      - "${DATA_PRIVATE_IP:-10.68.0.1}:6379:6379"
    volumes:
      - redisdata:/data
    healthcheck:
      test: ["CMD-SHELL", "redis-cli -a \"$$REDIS_PASSWORD\" --no-auth-warning ping | grep -q PONG"]
      interval: 10s
      timeout: 5s
      retries: 10
    environment:
      REDIS_PASSWORD: ${REDIS_PASSWORD}
    mem_limit: 6g
    cpus: 2

  minio:
    image: minio/minio:RELEASE.2025-09-07T16-13-09Z
    <<: *restart
    logging: *logging
    command: ["server", "/data", "--address", ":9000", "--console-address", ":9001"]
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
      MINIO_BROWSER_REDIRECT_URL: http://${DATA_PRIVATE_IP:-10.68.0.1}:9001
      MINIO_PROMETHEUS_AUTH_TYPE: public
      TZ: UTC
    ports:
      - "${DATA_PRIVATE_IP:-10.68.0.1}:9000:9000"
      - "${DATA_PRIVATE_IP:-10.68.0.1}:9001:9001"
    volumes:
      - miniodata:/data
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://127.0.0.1:9000/minio/health/live"]
      interval: 15s
      timeout: 5s
      retries: 10
      start_period: 20s
    mem_limit: 8g
    cpus: 4

  minio-init:
    image: minio/mc:RELEASE.2025-08-13T08-35-41Z
    restart: "no"
    entrypoint: ["/bin/sh", "/init/init.sh"]
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
      S3_ACCESS_KEY: ${S3_ACCESS_KEY}
      S3_SECRET_KEY: ${S3_SECRET_KEY}
      S3_BUCKET: ${S3_BUCKET:-dci-raw}
    volumes:
      - ./minio/init.sh:/init/init.sh:ro
      - ./minio/policy-dci-raw.json:/policy/policy-dci-raw.json:ro
      - /srv/dci/backups/minio:/backup
    depends_on:
      minio: { condition: service_healthy }

  # ───────────────────────────────────────────── monitoring (internal; Grafana on the private link)
  prometheus:
    image: prom/prometheus:v3.5.0
    <<: *restart
    logging: *logging
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --storage.tsdb.retention.time=90d
      - --storage.tsdb.retention.size=20GB
      - --web.enable-lifecycle
    ports:
      - "${DATA_PRIVATE_IP:-10.68.0.1}:9090:9090"
    volumes:
      - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - ./monitoring/alerts.yml:/etc/prometheus/alerts.yml:ro
      - promdata:/prometheus
    mem_limit: 4g
    cpus: 2

  node-exporter:
    image: prom/node-exporter:v1.9.1
    <<: *restart
    logging: *logging
    command:
      - --path.rootfs=/host
      - --path.procfs=/host/proc
      - --path.sysfs=/host/sys
      - --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc|run|var/lib/docker/.+)($$|/)
    pid: host
    volumes:
      - /:/host:ro,rslave
    mem_limit: 256m

  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.52.1
    <<: *restart
    logging: *logging
    command:
      - --housekeeping_interval=30s
      - --docker_only=true
      - --store_container_labels=false
      - --disable_metrics=advtcp,cpu_topology,cpuset,hugetlb,memory_numa,percpu,process,referenced_memory,resctrl,sched,tcp,udp
    privileged: true
    devices: ["/dev/kmsg"]
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker:/var/lib/docker:ro
    mem_limit: 1g

  postgres-exporter:
    image: prometheuscommunity/postgres-exporter:v0.17.1
    <<: *restart
    logging: *logging
    environment:
      DATA_SOURCE_URI: postgres:5432/dci?sslmode=disable
      DATA_SOURCE_USER: dci
      DATA_SOURCE_PASS: ${POSTGRES_PASSWORD}
    depends_on:
      postgres: { condition: service_healthy }
    mem_limit: 256m

  redis-exporter:
    image: oliver006/redis_exporter:v1.74.0
    <<: *restart
    logging: *logging
    environment:
      REDIS_ADDR: redis://redis:6379
      REDIS_PASSWORD: ${REDIS_PASSWORD}
    depends_on:
      redis: { condition: service_healthy }
    mem_limit: 128m

  grafana:
    image: grafana/grafana:12.1.1
    <<: *restart
    logging: *logging
    ports:
      - "${DATA_PRIVATE_IP:-10.68.0.1}:3000:3000"
    environment:
      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
      GF_USERS_ALLOW_SIGN_UP: "false"
      GF_AUTH_ANONYMOUS_ENABLED: "false"
      GF_ANALYTICS_REPORTING_ENABLED: "false"
      GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
      GF_SERVER_ROOT_URL: http://localhost:3000
      GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/dci-overview.json
      GF_LOG_LEVEL: warn
    volumes:
      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro
      - grafanadata:/var/lib/grafana
    depends_on:
      prometheus: { condition: service_started }
    mem_limit: 1g

volumes:
  pgdata: {}
  chdata: {}
  redisdata: {}
  miniodata: {}
  promdata: {}
  grafanadata: {}
  caddy-data: {}
  caddy-config: {}