TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";2import { join } from "node:path";3import YAML from "yaml";45/**6 * Source registry seeds. `config/sources.yaml` is the founding file; every `config/sources.d/*.yaml`7 * (sorted by name) is merged on top. A fragment entry may carry `extend: true` to add sensors,8 * products and aliases to a source declared earlier (same `id`) instead of redefining it — used to9 * attach connector-class sensors (EDGAR filings, TLS/DNS posture, package registries…) to existing10 * organizations without touching the founding file.11 *12 * This module has no database dependency so that the validator CLI can load seeds anywhere.13 */14export { sensorSchema, productSchema, sourceSchema, extendSchema } from "@websensor/core";15export type { SensorSeed, SourceSeed } from "@websensor/core";16import { extendSchema, sourceSchema, type SourceSeed } from "@websensor/core";1718export interface SeedIssue {19 file: string;20 source: string | undefined;21 message: string;22}2324export interface LoadedSeeds {25 seeds: SourceSeed[];26 issues: SeedIssue[];27 files: string[];28 /** source id → files that declared or extended it */29 origins: Map<string, string[]>;30}3132export function listSeedFiles(file: string, dir: string): string[] {33 const files = existsSync(file) ? [file] : [];34 if (existsSync(dir) && statSync(dir).isDirectory()) {35 for (const f of readdirSync(dir).sort()) if (/\.ya?ml$/.test(f) && !f.startsWith("_") && !f.startsWith(".")) files.push(join(dir, f));36 }37 return files;38}3940export function loadSeedsDetailed(file: string, dir: string): LoadedSeeds {41 const files = listSeedFiles(file, dir);42 const byId = new Map<string, SourceSeed>();43 const origins = new Map<string, string[]>();44 const issues: SeedIssue[] = [];45 const touch = (id: string, f: string): void => {46 const arr = origins.get(id) ?? [];47 if (!arr.includes(f)) arr.push(f);48 origins.set(id, arr);49 };50 for (const f of files) {51 let raw: { sources?: unknown[] };52 try {53 raw = (YAML.parse(readFileSync(f, "utf8")) ?? {}) as { sources?: unknown[] };54 } catch (e) {55 issues.push({ file: f, source: undefined, message: `YAML parse error: ${(e as Error).message}` });56 continue;57 }58 for (const s of raw.sources ?? []) {59 const id = (s as { id?: string })?.id;60 if ((s as { extend?: boolean })?.extend === true) {61 const parsed = extendSchema.safeParse(s);62 if (!parsed.success) {63 issues.push({ file: f, source: id, message: parsed.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ") });64 continue;65 }66 const base = byId.get(parsed.data.id);67 if (!base) {68 issues.push({ file: f, source: id, message: "extend: true but no source with this id was declared before this file" });69 continue;70 }71 touch(base.id, f);72 base.aliases = [...new Set([...base.aliases, ...parsed.data.aliases])];73 base.products = [...base.products, ...parsed.data.products.filter((p) => !base.products.some((q) => q.name === p.name))];74 base.categories = [...new Set([...base.categories, ...parsed.data.categories])];75 base.sensors = [...base.sensors, ...parsed.data.sensors.filter((n) => !base.sensors.some((o) => o.url === n.url))];76 if (parsed.data.fallback) base.fallback = { ...base.fallback, ...parsed.data.fallback };77 if (parsed.data.notes) base.notes = [base.notes, parsed.data.notes].filter(Boolean).join(" ");78 if (parsed.data.country && !base.country) base.country = parsed.data.country;79 if (parsed.data.language && !base.language) base.language = parsed.data.language;80 if (parsed.data.first_party !== undefined && base.first_party === undefined) base.first_party = parsed.data.first_party;81 continue;82 }83 const parsed = sourceSchema.safeParse(s);84 if (!parsed.success) {85 issues.push({ file: f, source: id, message: parsed.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ") });86 continue;87 }88 if (byId.has(parsed.data.id)) {89 issues.push({ file: f, source: parsed.data.id, message: "duplicate source id (use extend: true to add sensors to an existing source)" });90 continue;91 }92 byId.set(parsed.data.id, parsed.data);93 touch(parsed.data.id, f);94 }95 }96 // Sensor-level sanity: duplicate URLs inside a source.97 for (const s of byId.values()) {98 const seen = new Set<string>();99 for (const sen of s.sensors) {100 const k = sen.url.replace(/\/$/, "");101 if (seen.has(k)) issues.push({ file: "-", source: s.id, message: `duplicate sensor url ${sen.url}` });102 seen.add(k);103 }104 }105 return { seeds: [...byId.values()], issues, files, origins };106}107