TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { newId, slugify, sourceImportanceFromTier } from "@websensor/core";2import { db, discoveryCandidates, entities, entityAliases, eq, sensors, sourceEntities, sources, sql, textArray } from "@websensor/db";3import { discoverDomain } from "@websensor/connectors";4import { config, log } from "./config";56/**7 * Source registry: `config/sources.yaml` is the seed (organizations + curated sensors).8 * Booleans under `discover:` only allow discovery; every candidate is validated by a real9 * fetch+parse before becoming a sensor.10 */11export type { SourceSeed } from "./seeds";12import { loadSeedsDetailed, type SourceSeed } from "./seeds";1314export function loadSeeds(file = config.sourcesFile, dir = config.sourcesDir): SourceSeed[] {15 const { seeds, issues, files } = loadSeedsDetailed(file, dir);16 for (const i of issues) log.error({ file: i.file, source: i.source, issue: i.message }, "invalid source seed");17 log.info({ files: files.length, sources: seeds.length, sensors: seeds.reduce((n, s) => n + s.sensors.length, 0) }, "registry seeds loaded");18 return seeds;19}2021/**22 * Upsert one organization (source row + organization entity + aliases + product entities). Shared by the YAML23 * registry sync (`origin = seed`) and the Source Factory (`origin = factory`). Sensors are handled by the caller.24 */25export async function upsertSourceRecord(s: SourceSeed, opts: { origin?: "seed" | "import" | "factory"; sector?: string | null } = {}): Promise<void> {26 // Provenance: media/aggregators are third-party; everything else is the organization's own channel.27 const firstParty = s.first_party ?? !s.categories.some((c) => c === "news" || c === "media");28 const country = s.country ?? inferCountry(s.domain, s.categories);29 const language = s.language ?? inferLanguage(s.domain);30 const origin = opts.origin ?? "seed";31 const base = { name: s.name, domain: s.domain, homepage: s.homepage ?? `https://${s.domain}`, description: s.description, categories: s.categories, tier: s.tier, importanceWeight: s.weight, discover: s.discover, fallback: s.fallback, notes: s.notes, enabled: s.enabled, llmEnabled: s.llm, firstParty, country, language };32 await db33 .insert(sources)34 .values({ id: s.id, ...base, origin, sector: opts.sector ?? null })35 .onConflictDoUpdate({ target: sources.id, set: origin === "seed" ? { ...base, origin, updatedAt: new Date() } : { ...base, updatedAt: new Date(), ...(opts.sector ? { sector: opts.sector } : {}) } });3637 // Organization entity + aliases38 const entId = `org_${s.id}`;39 await db40 .insert(entities)41 .values({ id: entId, name: s.name, type: s.entity_type, domain: s.domain, homepage: s.homepage ?? `https://${s.domain}`, description: s.description, importance: sourceImportanceFromTier(s.tier, s.weight), categories: s.categories })42 .onConflictDoUpdate({ target: entities.id, set: { name: s.name, type: s.entity_type, domain: s.domain, description: s.description, importance: sourceImportanceFromTier(s.tier, s.weight), categories: s.categories } });43 await db.insert(sourceEntities).values({ sourceId: s.id, entityId: entId }).onConflictDoNothing();44 for (const alias of new Set([s.name, s.domain, s.domain.replace(/^www\./, ""), ...s.aliases])) {45 await db.insert(entityAliases).values({ alias: alias.toLowerCase(), entityId: entId }).onConflictDoNothing();46 }47 for (const p of s.products) {48 const pid = p.id ?? `prd_${s.id}_${slugify(p.name)}`;49 await db50 .insert(entities)51 .values({ id: pid, name: p.name, type: p.type, parentId: entId, importance: Math.max(30, sourceImportanceFromTier(s.tier, s.weight) - 10), categories: s.categories, domain: s.domain })52 .onConflictDoUpdate({ target: entities.id, set: { name: p.name, type: p.type, parentId: entId } });53 await db.insert(sourceEntities).values({ sourceId: s.id, entityId: pid }).onConflictDoNothing();54 for (const alias of new Set([p.name, ...p.aliases])) await db.insert(entityAliases).values({ alias: alias.toLowerCase(), entityId: pid }).onConflictDoNothing();55 await db.execute(sql`insert into entity_relations (from_id, relation, to_id) values (${entId}, 'owns', ${pid}) on conflict do nothing`);56 }57}5859export async function syncRegistry(seeds = loadSeeds()): Promise<{ sources: number; sensors: number }> {60 let nSensors = 0;61 for (const s of seeds) {62 await upsertSourceRecord(s, { origin: "seed" });6364 const seedIds: string[] = [];65 for (const sen of s.sensors) {66 const id = sen.id ?? `${s.id}_${slugify(sen.name)}`;67 seedIds.push(id);68 const cfg = { ...sen.config, seed: true };69 const tier = sen.tier ?? s.tier;70 const priority = priorityFor(tier, s.categories);71 await db72 .insert(sensors)73 .values({ id, sourceId: s.id, name: sen.name, url: sen.url, type: sen.type, connector: sen.connector, tier, importanceWeight: sen.weight ?? 1, config: cfg, baseIntervalSeconds: sen.interval ?? null, priority, status: "VALIDATED", validatedAt: new Date() })74 .onConflictDoUpdate({ target: sensors.id, set: { name: sen.name, url: sen.url, type: sen.type, connector: sen.connector, tier, importanceWeight: sen.weight ?? 1, config: cfg, baseIntervalSeconds: sen.interval ?? null, priority, enabled: true, updatedAt: new Date() } });75 nSensors++;76 }77 // Seed sensors removed from the YAML are disabled (history kept), discovery-created ones are untouched.78 await db.execute(sql`update sensors set enabled = false, status = 'DISABLED', updated_at = now() where source_id = ${s.id} and enabled and (config->>'seed') = 'true' and not (id = any(${textArray(seedIds)}))`);79 }80 log.info({ sources: seeds.length, sensors: nSensors }, "registry synced");81 return { sources: seeds.length, sensors: nSensors };82}8384/**85 * Discovery: for each enabled source whose `discover` flags allow it, probe the domain,86 * store candidates and promote validated feeds / sitemaps / statuspages to sensors that do87 * not already exist for that URL.88 */89export async function runDiscovery(opts: { onlyMissing?: boolean; sourceIds?: string[] } = {}): Promise<number> {90 const rows = await db.select().from(sources).where(eq(sources.enabled, true));91 let promoted = 0;92 const targets = rows.filter((r) => !opts.sourceIds || opts.sourceIds.includes(r.id));93 let i = 0;94 const workers = Array.from({ length: 4 }, async () => {95 while (i < targets.length) {96 const src = targets[i++]!;97 const d = src.discover as { rss?: boolean; sitemap?: boolean; status?: boolean; pages?: boolean };98 if (!d.rss && !d.sitemap && !d.status && !d.pages) continue;99 const existing = await db.select({ url: sensors.url, type: sensors.type }).from(sensors).where(eq(sensors.sourceId, src.id));100 const lastRun = (src.notes ?? "").match(/discovered_at=(\S+)/)?.[1];101 if (opts.onlyMissing && lastRun && Date.now() - new Date(lastRun).getTime() < config.discovery.intervalDays * 86400e3) continue;102 const started = Date.now();103 let found;104 try {105 found = await discoverDomain(src.domain, { probePages: Boolean(d.pages), sensorIdForLogs: `discover_${src.id}` });106 } catch (e) {107 log.warn({ source: src.id, err: (e as Error).message }, "discovery failed");108 continue;109 }110 const existingUrls = new Set(existing.map((e) => e.url.replace(/\/$/, "")));111 const hasFeed = existing.some((e) => e.type === "RSS" || e.type === "ATOM");112 let feedsAdded = 0;113 for (const f of found) {114 await db115 .insert(discoveryCandidates)116 .values({ id: newId("cand"), sourceId: src.id, url: f.url, kind: f.type, evidence: f.evidence, score: { value: f.value, itemCount: f.itemCount ?? null, title: f.title ?? null } })117 .onConflictDoNothing();118 if (existingUrls.has(f.url.replace(/\/$/, ""))) continue;119 const allowed = (f.connector === "rss" && d.rss) || (f.connector === "sitemap" && d.sitemap) || (f.connector === "statuspage" && d.status) || (f.connector === "http" && d.pages);120 if (!allowed) continue;121 // Promotion policy: feeds (max 3 per source, best first), one sitemap, one statuspage, pricing/changelog/security pages.122 if (f.connector === "rss" && feedsAdded + (hasFeed ? 1 : 0) >= 3) continue;123 if (f.connector === "sitemap" && existing.some((e) => e.type === "SITEMAP")) continue;124 if (f.connector === "statuspage" && existing.some((e) => e.type === "STATUSPAGE")) continue;125 if (f.connector === "http" && !/pricing|changelog|security|releases/i.test(f.url)) continue;126 const name = f.connector === "rss" ? feedName(f.url, f.title) : f.connector === "sitemap" ? "sitemap" : f.connector === "statuspage" ? "status" : new URL(f.url).pathname.replace(/\W+/g, " ").trim() || "page";127 const id = `${src.id}_${slugify(name)}`.slice(0, 80);128 const tier = f.connector === "statuspage" ? "S" : f.connector === "sitemap" ? (src.tier === "S" ? "A" : src.tier) : src.tier;129 await db130 .insert(sensors)131 .values({ id, sourceId: src.id, name, url: f.url, type: f.type, connector: f.connector, tier, config: f.connector === "sitemap" ? { maxChildren: 4, maxUrls: 3000 } : {}, status: "VALIDATED", validatedAt: new Date(), priority: priorityFor(tier, src.categories) })132 .onConflictDoNothing();133 await db.update(discoveryCandidates).set({ status: "promoted" }).where(sql`source_id = ${src.id} and url = ${f.url}`);134 existingUrls.add(f.url.replace(/\/$/, ""));135 if (f.connector === "rss") feedsAdded++;136 promoted++;137 }138 const notes = ((src.notes ?? "").replace(/\s*discovered_at=\S+/, "") + ` discovered_at=${new Date().toISOString()}`).trim();139 await db.update(sources).set({ notes, robotsCheckedAt: new Date(), updatedAt: new Date() }).where(eq(sources.id, src.id));140 log.info({ source: src.id, candidates: found.length, ms: Date.now() - started }, "discovery done");141 }142 });143 await Promise.all(workers);144 log.info({ promoted }, "discovery promoted sensors");145 return promoted;146}147148/** Priority tier (spec §76): 0 critical infrastructure/government/major AI/cyber · 1 major companies/markets/science/health · 2 specialized · 3 low. */149export function priorityFor(tier: string, categories: string[]): number {150 if (tier === "S") return 0;151 if (tier === "A" && categories.some((c) => ["cyber", "government", "ai", "cloud", "internet", "infrastructure", "finance", "health"].includes(c))) return 0;152 if (tier === "A" || (tier === "B" && categories.some((c) => ["finance", "science", "health", "pharma", "ai", "cyber", "government"].includes(c)))) return 1;153 if (tier === "B" || tier === "C") return 2;154 return 3;155}156157const TLD_COUNTRY: Record<string, string> = { ca: "CA", "gc.ca": "CA", "gouv.qc.ca": "CA", uk: "GB", "gov.uk": "GB", fr: "FR", "gouv.fr": "FR", de: "DE", it: "IT", es: "ES", jp: "JP", "go.jp": "JP", kr: "KR", "go.kr": "KR", au: "AU", "gov.au": "AU", in: "IN", "gov.in": "IN", br: "BR", "gov.br": "BR", mx: "MX", "gob.mx": "MX", ch: "CH", nl: "NL", se: "SE", no: "NO", fi: "FI", dk: "DK", ie: "IE", be: "BE", at: "AT", pt: "PT", pl: "PL", cn: "CN", hk: "HK", tw: "TW", sg: "SG", il: "IL", ae: "AE", sa: "SA", qa: "QA", za: "ZA", ng: "NG", ke: "KE", ar: "AR", cl: "CL", co: "CO", nz: "NZ", ru: "RU", tr: "TR", id: "ID", th: "TH", vn: "VN", my: "MY", ph: "PH", eg: "EG", ua: "UA", cz: "CZ", gr: "GR", lu: "LU", eu: "EU", "europa.eu": "EU", gov: "US", mil: "US", "us.com": "US" };158159/** Country inference for seeds without an explicit `country:` — only from unambiguous TLDs; `.com/.org/.io` stay null. */160export function inferCountry(domain: string, categories: string[]): string | null {161 const d = domain.toLowerCase();162 for (const [suffix, code] of Object.entries(TLD_COUNTRY).sort((a, b) => b[0].length - a[0].length)) if (d === suffix || d.endsWith("." + suffix)) return code;163 if (/\.int$|^un\.org$|\.who\.int$|\.imf\.org$|\.worldbank\.org$|\.oecd\.org$|\.bis\.org$|\.wto\.org$|\.iso\.org$|\.ietf\.org$|\.w3\.org$|\.icann\.org$/.test(d) || categories.includes("international")) return "INT";164 return null;165}166167export function inferLanguage(domain: string): string | null {168 const d = domain.toLowerCase();169 if (/\.qc\.ca$|quebec\.ca$|gouv\.fr$|\.fr$|lapresse\.ca|ledevoir\.com|journaldemontreal\.com|tvanouvelles\.ca|radio-canada\.ca|lemonde\.fr|lefigaro\.fr|24heures\.ca|noovo\.ca|lesoleil\.com|ledroit\.com|lapresse\.ca/.test(d)) return "fr";170 if (/\.de$|spiegel\.de|faz\.net|sueddeutsche\.de|handelsblatt\.com|\.at$|swissinfo\.ch/.test(d)) return "de";171 if (/\.es$|elpais\.com|elmundo\.es|\.mx$|eluniversal\.com\.mx|\.ar$|\.cl$|\.co$/.test(d)) return "es";172 if (/\.it$|repubblica\.it|corriere\.it/.test(d)) return "it";173 if (/\.br$|\.pt$|globo\.com|folha\.uol\.com\.br/.test(d)) return "pt";174 if (/\.jp$|nhk\.or\.jp|nikkei\.com$/.test(d)) return "ja";175 if (/\.kr$/.test(d)) return "ko";176 if (/\.cn$|xinhuanet\.com/.test(d)) return "zh";177 if (/\.nl$/.test(d)) return "nl";178 if (/\.ru$|tass\.(ru|com)/.test(d)) return "ru";179 return null;180}181182function feedName(url: string, title?: string): string {183 const p = new URL(url).pathname.toLowerCase();184 if (/blog/.test(p)) return "blog feed";185 if (/news|press/.test(p)) return "news feed";186 if (/release|changelog/.test(p)) return "changelog feed";187 if (/security|advisor/.test(p)) return "security feed";188 if (title) return slugify(title).replace(/-/g, " ").slice(0, 40) + " feed";189 return "feed";190}191