spb/datacenterindex
Public
HTML 53.9%
TypeScript 44.5%
JavaScript 0.6%
SQL 0.5%
1# DataCenterIndex — DATA node (BHS128). Public traffic arrives from the BHS64 gateway over WireGuard wg12# (edge :8300 bound to 10.67.0.60). Stores are published ONLY on the private link dci0 (10.68.0.1) for the3# crawl node (BHS64b, 10.68.0.2). Nothing is bound to 0.0.0.0 / the public IP.4#5# docker compose -f deploy/compose.data.yml --env-file deploy/.env.data up -d --remove-orphans6# docker compose -f deploy/compose.data.yml --env-file deploy/.env.data run --rm migrate7# docker compose -f deploy/compose.data.yml --env-file deploy/.env.data run --rm cli run <connector> --dry-run --limit 58#9# Profiles: (none) = production set (incl. scheduler + worker-maint, DCI_QUEUES=maintenance) · ops = migrate/cli one-shots10name: dci1112x-logging: &logging13 driver: json-file14 options: { max-size: "50m", max-file: "5" }1516x-restart: &restart17 restart: unless-stopped1819# Environment shared by every process built from the repo (api, scheduler, migrate, cli, worker-maint).20x-app-env: &app-env21 NODE_ENV: production22 TZ: ${TZ:-America/Toronto}23 DATABASE_URL: postgres://dci:${POSTGRES_PASSWORD}@postgres:5432/dci24 PG_POOL_MAX: ${PG_POOL_MAX:-16}25 REDIS_URL: redis://:${REDIS_PASSWORD}@redis:6379/026 CLICKHOUSE_URL: http://clickhouse:812327 CLICKHOUSE_DB: ${CLICKHOUSE_DB:-dci}28 CLICKHOUSE_USER: dci29 CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}30 S3_ENDPOINT: http://minio:900031 S3_BUCKET: ${S3_BUCKET:-dci-raw}32 S3_ACCESS_KEY: ${S3_ACCESS_KEY}33 S3_SECRET_KEY: ${S3_SECRET_KEY}34 S3_REGION: us-east-135 DCI_ADMIN_TOKEN: ${DCI_ADMIN_TOKEN}36 DCI_USER_AGENT: ${DCI_USER_AGENT}37 DCI_LOG_LEVEL: ${DCI_LOG_LEVEL:-info}38 DCI_CONFIG_DIR: /app/config/connectors39 SCRAPFLY_API_KEY: ${SCRAPFLY_API_KEY:-}40 FIRECRAWL_API_KEY: ${FIRECRAWL_API_KEY:-}41 DCI_SCRAPFLY_DAILY_BUDGET: ${DCI_SCRAPFLY_DAILY_BUDGET:-400}42 DCI_FIRECRAWL_DAILY_BUDGET: ${DCI_FIRECRAWL_DAILY_BUDGET:-200}43 NEXT_PUBLIC_SITE_URL: ${NEXT_PUBLIC_SITE_URL:-https://www.datacenterindex.io}4445x-worker-image: &worker-image46 image: dci/worker:${DCI_TAG:-latest}47 build:48 context: ..49 dockerfile: deploy/docker/Dockerfile.worker50 args:51 TSX_VERSION: ${TSX_VERSION:-4.23.13}5253services:54 # ───────────────────────────────────────────── edge (the only thing the gateway talks to)55 edge:56 image: caddy:2.10-alpine57 <<: *restart58 logging: *logging59 ports:60 - "${DATA_WG_IP:-10.67.0.60}:8300:8300"61 volumes:62 - ./edge/Caddyfile:/etc/caddy/Caddyfile:ro63 - caddy-data:/data64 - caddy-config:/config65 depends_on:66 web: { condition: service_started }67 api: { condition: service_started }68 healthcheck:69 test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8300/api/health"]70 interval: 30s71 timeout: 5s72 retries: 373 start_period: 20s74 mem_limit: 512m75 cpus: 27677 # ───────────────────────────────────────────── application78 web:79 image: dci/web:${DCI_TAG:-latest}80 build:81 context: ..82 dockerfile: deploy/docker/Dockerfile.web83 args:84 NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-https://www.datacenterindex.io}85 NEXT_PUBLIC_SITE_URL: ${NEXT_PUBLIC_SITE_URL:-https://www.datacenterindex.io}86 <<: *restart87 logging: *logging88 environment:89 NODE_ENV: production90 TZ: ${TZ:-America/Toronto}91 PORT: "8310"92 HOSTNAME: 0.0.0.093 NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-https://www.datacenterindex.io}94 NEXT_PUBLIC_SITE_URL: ${NEXT_PUBLIC_SITE_URL:-https://www.datacenterindex.io}95 API_URL_INTERNAL: http://api:831196 depends_on:97 api: { condition: service_healthy }98 mem_limit: 4g99 cpus: 4100101 api:102 image: dci/api:${DCI_TAG:-latest}103 build:104 context: ..105 dockerfile: deploy/docker/Dockerfile.api106 args:107 TSX_VERSION: ${TSX_VERSION:-4.23.13}108 <<: *restart109 logging: *logging110 environment:111 <<: *app-env112 API_PORT: "8311"113 HOST: 0.0.0.0114 NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-https://www.datacenterindex.io}115 DCI_API_CACHE_ENTRIES: ${DCI_API_CACHE_ENTRIES:-5000}116 DCI_TRUST_PROXY: "1"117 # internal worker HTTP (extraction debugger /trace, /data-gaps) — proxied behind the admin token118 DCI_WORKER_URL: http://worker-maint:8320119 depends_on:120 postgres: { condition: service_healthy }121 redis: { condition: service_healthy }122 clickhouse: { condition: service_healthy }123 minio: { condition: service_healthy }124 mem_limit: 4g125 cpus: 4126127 # Scheduler: enqueues connector runs (BullMQ) — co-located with Redis. The crawl workers run on BHS64b.128 # /healthz + /metrics on 8321 (container network only; scraped by Prometheus as scheduler:8321). The image's129 # HEALTHCHECK probes http://127.0.0.1:$WORKER_PORT/healthz.130 scheduler:131 <<: [*worker-image, *restart]132 command: ["scheduler"]133 logging: *logging134 environment:135 <<: *app-env136 DCI_SCHEDULER_INTERVAL_MS: ${DCI_SCHEDULER_INTERVAL_MS:-60000}137 WORKER_PORT: "8321"138 healthcheck:139 test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:8321/healthz').then(r=>process.exit(r.ok?0:1),()=>process.exit(1))"]140 interval: 30s141 timeout: 5s142 retries: 3143 start_period: 40s144 depends_on:145 postgres: { condition: service_healthy }146 redis: { condition: service_healthy }147 mem_limit: 2g148 cpus: 2149150 # Light worker for the maintenance queue only (rankings, metrics, refresh-stats, cleanup) — the worker honours151 # DCI_QUEUES, so this node never crawls. Part of the default set since 2026-09-11 (was profile `maintenance`).152 worker-maint:153 <<: [*worker-image, *restart]154 command: ["worker"]155 logging: *logging156 environment:157 <<: *app-env158 DCI_QUEUES: maintenance159 DCI_CRAWL_CONCURRENCY: "1"160 DCI_MAINTENANCE_CONCURRENCY: ${DCI_MAINTENANCE_CONCURRENCY:-2}161 WORKER_PORT: "8320"162 depends_on:163 postgres: { condition: service_healthy }164 redis: { condition: service_healthy }165 mem_limit: 4g166 cpus: 2167168 # One-shots (profile ops): `compose run --rm migrate`, `compose run --rm cli <args>`169 migrate:170 <<: *worker-image171 profiles: ["ops"]172 restart: "no"173 command: ["migrate"]174 environment:175 <<: *app-env176 depends_on:177 postgres: { condition: service_healthy }178 clickhouse: { condition: service_healthy }179 mem_limit: 2g180181 cli:182 <<: *worker-image183 profiles: ["ops"]184 restart: "no"185 command: ["cli", "--help"]186 environment:187 <<: *app-env188 volumes:189 - /srv/dci/out:/app/data190 depends_on:191 postgres: { condition: service_healthy }192 redis: { condition: service_healthy }193 mem_limit: 4g194195 # ───────────────────────────────────────────── stores (published on the private link only)196 postgres:197 image: postgres:17198 <<: *restart199 logging: *logging200 command:201 - postgres202 - -c203 - shared_buffers=4GB204 - -c205 - effective_cache_size=32GB206 - -c207 - work_mem=64MB208 - -c209 - maintenance_work_mem=1GB210 - -c211 - max_connections=200212 - -c213 - max_wal_size=4GB214 - -c215 - min_wal_size=1GB216 - -c217 - checkpoint_completion_target=0.9218 - -c219 - wal_compression=zstd220 - -c221 - random_page_cost=1.1222 - -c223 - effective_io_concurrency=200224 - -c225 - max_worker_processes=16226 - -c227 - max_parallel_workers=8228 - -c229 - max_parallel_workers_per_gather=4230 - -c231 - shared_preload_libraries=pg_stat_statements232 - -c233 - log_min_duration_statement=500234 - -c235 - log_checkpoints=on236 - -c237 - timezone=UTC238 environment:239 POSTGRES_USER: dci240 POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}241 POSTGRES_DB: dci242 POSTGRES_INITDB_ARGS: "--data-checksums --encoding=UTF8 --locale=C.UTF-8"243 TZ: UTC244 ports:245 - "${DATA_PRIVATE_IP:-10.68.0.1}:5432:5432"246 volumes:247 - pgdata:/var/lib/postgresql/data248 - ./postgres/initdb:/docker-entrypoint-initdb.d:ro249 - /srv/dci/backups/pg:/backups250 shm_size: 2g251 healthcheck:252 test: ["CMD-SHELL", "pg_isready -U dci -d dci -h 127.0.0.1"]253 interval: 10s254 timeout: 5s255 retries: 10256 start_period: 30s257 mem_limit: 40g258 cpus: 8259 stop_grace_period: 60s260261 clickhouse:262 image: clickhouse/clickhouse-server:25.8263 <<: *restart264 logging: *logging265 environment:266 CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}267 CLICKHOUSE_SKIP_USER_SETUP: "1"268 TZ: UTC269 ports:270 - "${DATA_PRIVATE_IP:-10.68.0.1}:8123:8123"271 # native protocol: host port 9009 (9000 on 10.68.0.1 belongs to MinIO S3)272 - "${DATA_PRIVATE_IP:-10.68.0.1}:9009:9000"273 volumes:274 - chdata:/var/lib/clickhouse275 - ./clickhouse/config.d/dci.xml:/etc/clickhouse-server/config.d/dci.xml:ro276 - ./clickhouse/users.d/dci.xml:/etc/clickhouse-server/users.d/dci.xml:ro277 - /srv/dci/backups/clickhouse:/backups278 ulimits:279 nofile: { soft: 262144, hard: 262144 }280 cap_add: [SYS_NICE, NET_ADMIN, IPC_LOCK]281 healthcheck:282 test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8123/ping"]283 interval: 10s284 timeout: 5s285 retries: 10286 start_period: 30s287 mem_limit: 24g288 cpus: 8289290 redis:291 image: redis:7.4-alpine292 <<: *restart293 logging: *logging294 command:295 - redis-server296 - --appendonly297 - "yes"298 - --appendfsync299 - everysec300 - --requirepass301 - ${REDIS_PASSWORD}302 - --maxmemory303 - 4gb304 - --maxmemory-policy305 - noeviction306 - --save307 - "900 1"308 - --tcp-keepalive309 - "60"310 ports:311 - "${DATA_PRIVATE_IP:-10.68.0.1}:6379:6379"312 volumes:313 - redisdata:/data314 healthcheck:315 test: ["CMD-SHELL", "redis-cli -a \"$$REDIS_PASSWORD\" --no-auth-warning ping | grep -q PONG"]316 interval: 10s317 timeout: 5s318 retries: 10319 environment:320 REDIS_PASSWORD: ${REDIS_PASSWORD}321 mem_limit: 6g322 cpus: 2323324 minio:325 image: minio/minio:RELEASE.2025-09-07T16-13-09Z326 <<: *restart327 logging: *logging328 command: ["server", "/data", "--address", ":9000", "--console-address", ":9001"]329 environment:330 MINIO_ROOT_USER: ${MINIO_ROOT_USER}331 MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}332 MINIO_BROWSER_REDIRECT_URL: http://${DATA_PRIVATE_IP:-10.68.0.1}:9001333 MINIO_PROMETHEUS_AUTH_TYPE: public334 TZ: UTC335 ports:336 - "${DATA_PRIVATE_IP:-10.68.0.1}:9000:9000"337 - "${DATA_PRIVATE_IP:-10.68.0.1}:9001:9001"338 volumes:339 - miniodata:/data340 healthcheck:341 test: ["CMD", "curl", "-sf", "http://127.0.0.1:9000/minio/health/live"]342 interval: 15s343 timeout: 5s344 retries: 10345 start_period: 20s346 mem_limit: 8g347 cpus: 4348349 minio-init:350 image: minio/mc:RELEASE.2025-08-13T08-35-41Z351 restart: "no"352 entrypoint: ["/bin/sh", "/init/init.sh"]353 environment:354 MINIO_ROOT_USER: ${MINIO_ROOT_USER}355 MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}356 S3_ACCESS_KEY: ${S3_ACCESS_KEY}357 S3_SECRET_KEY: ${S3_SECRET_KEY}358 S3_BUCKET: ${S3_BUCKET:-dci-raw}359 volumes:360 - ./minio/init.sh:/init/init.sh:ro361 - ./minio/policy-dci-raw.json:/policy/policy-dci-raw.json:ro362 - /srv/dci/backups/minio:/backup363 depends_on:364 minio: { condition: service_healthy }365366 # ───────────────────────────────────────────── monitoring (internal; Grafana on the private link)367 prometheus:368 image: prom/prometheus:v3.5.0369 <<: *restart370 logging: *logging371 command:372 - --config.file=/etc/prometheus/prometheus.yml373 - --storage.tsdb.path=/prometheus374 - --storage.tsdb.retention.time=90d375 - --storage.tsdb.retention.size=20GB376 - --web.enable-lifecycle377 ports:378 - "${DATA_PRIVATE_IP:-10.68.0.1}:9090:9090"379 volumes:380 - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro381 - ./monitoring/alerts.yml:/etc/prometheus/alerts.yml:ro382 - promdata:/prometheus383 mem_limit: 4g384 cpus: 2385386 node-exporter:387 image: prom/node-exporter:v1.9.1388 <<: *restart389 logging: *logging390 command:391 - --path.rootfs=/host392 - --path.procfs=/host/proc393 - --path.sysfs=/host/sys394 - --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc|run|var/lib/docker/.+)($$|/)395 pid: host396 volumes:397 - /:/host:ro,rslave398 mem_limit: 256m399400 cadvisor:401 image: gcr.io/cadvisor/cadvisor:v0.52.1402 <<: *restart403 logging: *logging404 command:405 - --housekeeping_interval=30s406 - --docker_only=true407 - --store_container_labels=false408 - --disable_metrics=advtcp,cpu_topology,cpuset,hugetlb,memory_numa,percpu,process,referenced_memory,resctrl,sched,tcp,udp409 privileged: true410 devices: ["/dev/kmsg"]411 volumes:412 - /:/rootfs:ro413 - /var/run:/var/run:ro414 - /sys:/sys:ro415 - /var/lib/docker:/var/lib/docker:ro416 mem_limit: 1g417418 postgres-exporter:419 image: prometheuscommunity/postgres-exporter:v0.17.1420 <<: *restart421 logging: *logging422 environment:423 DATA_SOURCE_URI: postgres:5432/dci?sslmode=disable424 DATA_SOURCE_USER: dci425 DATA_SOURCE_PASS: ${POSTGRES_PASSWORD}426 depends_on:427 postgres: { condition: service_healthy }428 mem_limit: 256m429430 redis-exporter:431 image: oliver006/redis_exporter:v1.74.0432 <<: *restart433 logging: *logging434 environment:435 REDIS_ADDR: redis://redis:6379436 REDIS_PASSWORD: ${REDIS_PASSWORD}437 depends_on:438 redis: { condition: service_healthy }439 mem_limit: 128m440441 grafana:442 image: grafana/grafana:12.1.1443 <<: *restart444 logging: *logging445 ports:446 - "${DATA_PRIVATE_IP:-10.68.0.1}:3000:3000"447 environment:448 GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}449 GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}450 GF_USERS_ALLOW_SIGN_UP: "false"451 GF_AUTH_ANONYMOUS_ENABLED: "false"452 GF_ANALYTICS_REPORTING_ENABLED: "false"453 GF_ANALYTICS_CHECK_FOR_UPDATES: "false"454 GF_SERVER_ROOT_URL: http://localhost:3000455 GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/dci-overview.json456 GF_LOG_LEVEL: warn457 volumes:458 - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro459 - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro460 - grafanadata:/var/lib/grafana461 depends_on:462 prometheus: { condition: service_started }463 mem_limit: 1g464465volumes:466 pgdata: {}467 chdata: {}468 redisdata: {}469 miniodata: {}470 promdata: {}471 grafanadata: {}472 caddy-data: {}473 caddy-config: {}474