import pino from "pino"; const env = process.env; /** `WS_PER_HOST_OVERRIDES="github.com=8,data.sec.gov=6"` */ function parseHostOverrides(v: string | undefined): Record { const out: Record = {}; for (const part of (v ?? "").split(",")) { const [h, n] = part.split("="); if (h && n && Number(n) > 0) out[h.trim().toLowerCase()] = Number(n); } return out; } export const config = { env: env.NODE_ENV ?? "development", logLevel: env.LOG_LEVEL ?? "info", databaseUrl: env.DATABASE_URL ?? "postgres://localhost:5432/websensor", redisUrl: env.REDIS_URL ?? "redis://127.0.0.1:6379", sourcesFile: env.WS_SOURCES_FILE ?? "./config/sources.yaml", /** fragments merged on top of sourcesFile (sorted by file name; `extend: true` entries add sensors to existing sources) */ sourcesDir: env.WS_SOURCES_DIR ?? "./config/sources.d", entitiesFile: env.WS_ENTITIES_FILE ?? "./config/entities.yaml", fetchConcurrency: Number(env.WS_FETCH_CONCURRENCY ?? 16), perHostConcurrency: Number(env.WS_PER_HOST_CONCURRENCY ?? 2), /** hosts that tolerate more parallel requests than the default (large public APIs) */ perHostOverrides: { "github.com": 6, "api.github.com": 4, "data.sec.gov": 4, "huggingface.co": 4, "www.federalregister.gov": 4, "services.nvd.nist.gov": 1, "export.arxiv.org": 1, "rss.arxiv.org": 2, "api.crossref.org": 2, "api.openalex.org": 2, "clinicaltrials.gov": 3, "www.gov.uk": 4, "api.io.canada.ca": 4, "www.nature.com": 1, ...parseHostOverrides(env.WS_PER_HOST_OVERRIDES) } as Record, metricsPort: Number(env.ENGINE_METRICS_PORT ?? 8262), metricsHost: env.ENGINE_METRICS_HOST ?? "127.0.0.1", processingVersion: "event-pipeline-v2", /** heuristic signal threshold to promote a raw change to an event */ meaningfulSignal: Number(env.WS_MEANINGFUL_SIGNAL ?? 0.32), llm: { apiKey: env.ANTHROPIC_API_KEY ?? "", modelFast: env.WS_LLM_MODEL_FAST ?? "claude-haiku-4-5", modelDeep: env.WS_LLM_MODEL_DEEP ?? "claude-opus-5", dailyCallBudget: Number(env.WS_LLM_DAILY_CALL_BUDGET ?? 600), minImportance: Number(env.WS_LLM_MIN_IMPORTANCE ?? 35), deepMinImportance: Number(env.WS_LLM_DEEP_MIN_IMPORTANCE ?? 78), }, discovery: { enabled: (env.WS_DISCOVERY ?? "1") !== "0", /** re-run discovery for a source after this many days */ intervalDays: Number(env.WS_DISCOVERY_INTERVAL_DAYS ?? 7), }, deletion: { confirmations: Number(env.WS_DELETE_CONFIRMATIONS ?? 3), minSeparationMin: Number(env.WS_DELETE_SEPARATION_MIN ?? 60) }, /** storage lifecycle (spec §58): raw bodies of unchanged snapshots are dropped after N days; canonical + hashes kept */ retention: { rawDays: Number(env.WS_RETENTION_RAW_DAYS ?? 21), changedRawDays: Number(env.WS_RETENTION_CHANGED_RAW_DAYS ?? 60), enabled: (env.WS_RETENTION ?? "1") !== "0" }, /** silent-change bar: only eligible types with importance ≥ this become "silent" */ silentMinImportance: Number(env.WS_SILENT_MIN_IMPORTANCE ?? 45), version: "0.3.0", }; export const log = pino({ level: config.logLevel, ...(config.env !== "production" ? { transport: { target: "pino-pretty", options: { colorize: true, translateTime: "HH:MM:ss" } } } : {}), base: { service: "websensor-engine" }, }); /** Source Factory (apps/engine/src/factory) — organization expansion, shadow monitoring, acceptance. */ export const factoryConfig = { enabled: (env.WS_FACTORY ?? "1") !== "0", /** seeds discovered in parallel */ concurrency: Number(env.WS_FACTORY_CONCURRENCY ?? 10), /** HTTP requests per organization during discovery */ budget: Number(env.WS_FACTORY_BUDGET ?? 90), /** wall-clock deadline per organization (ms) */ deadlineMs: Number(env.WS_FACTORY_DEADLINE_MS ?? 120_000), /** minimum candidate score to enter shadow monitoring */ minScore: Number(env.WS_FACTORY_MIN_SCORE ?? 0.35), /** shadow evaluation: checks needed, minimum and maximum shadow duration */ shadowMinChecks: Number(env.WS_FACTORY_SHADOW_MIN_CHECKS ?? 5), shadowMinHours: Number(env.WS_FACTORY_SHADOW_MIN_HOURS ?? 24), shadowMaxHours: Number(env.WS_FACTORY_SHADOW_MAX_HOURS ?? 120), /** per-organization promotion caps */ caps: { feeds: 4, pages: 6, releases: 4, sitemap: 1, status: 1, models: 1, filings: 1, api: 1, posture: 3, legal: 2, total: 18 }, /** seconds between factory ticks */ tickSeconds: Number(env.WS_FACTORY_TICK_SECONDS ?? 20), coverageDir: env.WS_COVERAGE_DIR ?? "./config/coverage", seedsDir: env.WS_FACTORY_SEEDS_DIR ?? "./config/factory/seeds", };