TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import pino from "pino";23const env = process.env;45/** `WS_PER_HOST_OVERRIDES="github.com=8,data.sec.gov=6"` */6function parseHostOverrides(v: string | undefined): Record<string, number> {7 const out: Record<string, number> = {};8 for (const part of (v ?? "").split(",")) {9 const [h, n] = part.split("=");10 if (h && n && Number(n) > 0) out[h.trim().toLowerCase()] = Number(n);11 }12 return out;13}1415export const config = {16 env: env.NODE_ENV ?? "development",17 logLevel: env.LOG_LEVEL ?? "info",18 databaseUrl: env.DATABASE_URL ?? "postgres://localhost:5432/websensor",19 redisUrl: env.REDIS_URL ?? "redis://127.0.0.1:6379",20 sourcesFile: env.WS_SOURCES_FILE ?? "./config/sources.yaml",21 /** fragments merged on top of sourcesFile (sorted by file name; `extend: true` entries add sensors to existing sources) */22 sourcesDir: env.WS_SOURCES_DIR ?? "./config/sources.d",23 entitiesFile: env.WS_ENTITIES_FILE ?? "./config/entities.yaml",24 fetchConcurrency: Number(env.WS_FETCH_CONCURRENCY ?? 16),25 perHostConcurrency: Number(env.WS_PER_HOST_CONCURRENCY ?? 2),26 /** hosts that tolerate more parallel requests than the default (large public APIs) */27 perHostOverrides: { "github.com": 6, "api.github.com": 4, "data.sec.gov": 4, "huggingface.co": 4, "www.federalregister.gov": 4, "services.nvd.nist.gov": 1, "export.arxiv.org": 1, "rss.arxiv.org": 2, "api.crossref.org": 2, "api.openalex.org": 2, "clinicaltrials.gov": 3, "www.gov.uk": 4, "api.io.canada.ca": 4, "www.nature.com": 1, ...parseHostOverrides(env.WS_PER_HOST_OVERRIDES) } as Record<string, number>,28 metricsPort: Number(env.ENGINE_METRICS_PORT ?? 8262),29 metricsHost: env.ENGINE_METRICS_HOST ?? "127.0.0.1",30 processingVersion: "event-pipeline-v2",31 /** heuristic signal threshold to promote a raw change to an event */32 meaningfulSignal: Number(env.WS_MEANINGFUL_SIGNAL ?? 0.32),33 llm: {34 apiKey: env.ANTHROPIC_API_KEY ?? "",35 modelFast: env.WS_LLM_MODEL_FAST ?? "claude-haiku-4-5",36 modelDeep: env.WS_LLM_MODEL_DEEP ?? "claude-opus-5",37 dailyCallBudget: Number(env.WS_LLM_DAILY_CALL_BUDGET ?? 600),38 minImportance: Number(env.WS_LLM_MIN_IMPORTANCE ?? 35),39 deepMinImportance: Number(env.WS_LLM_DEEP_MIN_IMPORTANCE ?? 78),40 },41 discovery: {42 enabled: (env.WS_DISCOVERY ?? "1") !== "0",43 /** re-run discovery for a source after this many days */44 intervalDays: Number(env.WS_DISCOVERY_INTERVAL_DAYS ?? 7),45 },46 deletion: { confirmations: Number(env.WS_DELETE_CONFIRMATIONS ?? 3), minSeparationMin: Number(env.WS_DELETE_SEPARATION_MIN ?? 60) },47 /** storage lifecycle (spec §58): raw bodies of unchanged snapshots are dropped after N days; canonical + hashes kept */48 retention: { rawDays: Number(env.WS_RETENTION_RAW_DAYS ?? 21), changedRawDays: Number(env.WS_RETENTION_CHANGED_RAW_DAYS ?? 60), enabled: (env.WS_RETENTION ?? "1") !== "0" },49 /** silent-change bar: only eligible types with importance ≥ this become "silent" */50 silentMinImportance: Number(env.WS_SILENT_MIN_IMPORTANCE ?? 45),51 version: "0.3.0",52};5354export const log = pino({55 level: config.logLevel,56 ...(config.env !== "production" ? { transport: { target: "pino-pretty", options: { colorize: true, translateTime: "HH:MM:ss" } } } : {}),57 base: { service: "websensor-engine" },58});5960/** Source Factory (apps/engine/src/factory) — organization expansion, shadow monitoring, acceptance. */61export const factoryConfig = {62 enabled: (env.WS_FACTORY ?? "1") !== "0",63 /** seeds discovered in parallel */64 concurrency: Number(env.WS_FACTORY_CONCURRENCY ?? 10),65 /** HTTP requests per organization during discovery */66 budget: Number(env.WS_FACTORY_BUDGET ?? 90),67 /** wall-clock deadline per organization (ms) */68 deadlineMs: Number(env.WS_FACTORY_DEADLINE_MS ?? 120_000),69 /** minimum candidate score to enter shadow monitoring */70 minScore: Number(env.WS_FACTORY_MIN_SCORE ?? 0.35),71 /** shadow evaluation: checks needed, minimum and maximum shadow duration */72 shadowMinChecks: Number(env.WS_FACTORY_SHADOW_MIN_CHECKS ?? 5),73 shadowMinHours: Number(env.WS_FACTORY_SHADOW_MIN_HOURS ?? 24),74 shadowMaxHours: Number(env.WS_FACTORY_SHADOW_MAX_HOURS ?? 120),75 /** per-organization promotion caps */76 caps: { feeds: 4, pages: 6, releases: 4, sitemap: 1, status: 1, models: 1, filings: 1, api: 1, posture: 3, legal: 2, total: 18 },77 /** seconds between factory ticks */78 tickSeconds: Number(env.WS_FACTORY_TICK_SECONDS ?? 20),79 coverageDir: env.WS_COVERAGE_DIR ?? "./config/coverage",80 seedsDir: env.WS_FACTORY_SEEDS_DIR ?? "./config/factory/seeds",81};82