spb/websensor

1import { closeDb, migrate } from "@websensor/db";2import { closeDispatcher } from "@websensor/connectors";3import { loadRecent, refreshClusterStates } from "./cluster";4import { config, log } from "./config";5import { startMetricsServer } from "./metrics";6import { closeRedis } from "./redis";7import { runDiscovery, syncRegistry } from "./registry";8import { pruneNotifications, pruneRawSnapshots } from "./retention";9import { pruneOldRuns, rollupConnectorHealth, Scheduler } from "./scheduler";1011async function main(): Promise<void> {12  log.info({ env: config.env, version: config.version, llm: Boolean(config.llm.apiKey) }, "websensor engine starting");13  const applied = await migrate(config.databaseUrl);14  if (applied.length) log.info({ applied }, "migrations applied");15  await syncRegistry();16  await loadRecent();17  await startMetricsServer();1819  const scheduler = new Scheduler();20  await scheduler.start();2122  const timers: NodeJS.Timeout[] = [];23  timers.push(setInterval(() => rollupConnectorHealth().catch((e) => log.warn({ err: (e as Error).message }, "health rollup failed")), 60_000));24  timers.push(setInterval(() => pruneOldRuns().catch(() => undefined), 6 * 3600e3));25  timers.push(setInterval(() => refreshClusterStates().catch((e) => log.warn({ err: (e as Error).message }, "cluster state refresh failed")), 5 * 60_000));26  if (config.retention.enabled) {27    timers.push(setInterval(() => pruneRawSnapshots().catch((e) => log.warn({ err: (e as Error).message }, "retention failed")), 30 * 60_000));28    timers.push(setInterval(() => pruneNotifications().catch(() => undefined), 24 * 3600e3));29    setTimeout(() => pruneRawSnapshots().catch(() => undefined), 60_000);30  }31  await rollupConnectorHealth().catch(() => undefined);3233  if (config.discovery.enabled) {34    // Background: validate feeds/sitemaps/status pages for sources not discovered recently, then weekly.35    setTimeout(() => runDiscovery({ onlyMissing: true }).catch((e) => log.warn({ err: (e as Error).message }, "discovery failed")), 5_000);36    timers.push(setInterval(() => runDiscovery({ onlyMissing: true }).catch(() => undefined), 24 * 3600e3));37  }3839  let shuttingDown = false;40  const shutdown = async (signal: string): Promise<void> => {41    if (shuttingDown) return;42    shuttingDown = true;43    log.info({ signal }, "shutting down");44    for (const t of timers) clearInterval(t);45    await scheduler.stop();46    await closeDispatcher();47    await closeRedis();48    await closeDb();49    process.exit(0);50  };51  process.on("SIGINT", () => void shutdown("SIGINT"));52  process.on("SIGTERM", () => void shutdown("SIGTERM"));53  process.on("unhandledRejection", (e) => log.error({ err: e instanceof Error ? e.stack : String(e) }, "unhandled rejection"));54}5556main().catch((e) => {57  log.fatal({ err: (e as Error).stack ?? String(e) }, "engine failed to start");58  process.exit(1);59});60