TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { writeFileSync } from "node:fs";2import { resolve } from "node:path";3import type { SensorEndpoint, Tier } from "@websensor/core";4import { closeDispatcher, discoverDomain, getConnector, NormalizeError } from "@websensor/connectors";5import { loadSeedsDetailed, type SourceSeed } from "./seeds";67/**8 * Registry validator — no database. Loads the seeds (founding file + fragments), then runs every9 * curated sensor through its connector (fetch + normalize) exactly as the engine would, and prints10 * one line per sensor:11 *12 * OK 200 list items=42 openai_news_feed https://…13 * WARN 200 text thin acme_pricing https://… (JS shell / interstitial)14 * FAIL 403 - - foo_blog_feed https://… http_40315 *16 * Usage (from the repo root):17 * node node_modules/tsx/dist/cli.mjs apps/engine/src/validate.ts [fragment.yaml…] [--all]18 * [--source id[,id…]] [--connector key] [--concurrency 8] [--json report.json] [--quiet]19 * node node_modules/tsx/dist/cli.mjs apps/engine/src/validate.ts --probe <domain> (discovery dry-run, no DB)20 *21 * Without a file argument the founding file is validated; with fragment paths only the sources22 * declared or extended by those fragments are checked (the whole registry is still loaded so that23 * `extend: true` resolves). Exit code 1 when any sensor FAILs or any seed is invalid.24 */25interface Result {26 source: string;27 sensor: string;28 connector: string;29 type: string;30 url: string;31 status: "OK" | "WARN" | "FAIL";32 http: number;33 mode?: string;34 items?: number;35 note: string;36 ms: number;37}3839function arg(name: string, def?: string): string | undefined {40 const i = process.argv.indexOf(`--${name}`);41 return i >= 0 ? process.argv[i + 1] : def;42}43const flag = (name: string): boolean => process.argv.includes(`--${name}`);4445async function checkSensor(src: SourceSeed, sen: SourceSeed["sensors"][number]): Promise<Result> {46 const t0 = Date.now();47 const id = sen.id ?? `${src.id}_${sen.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "")}`;48 const base: Omit<Result, "status" | "http" | "note" | "ms"> = { source: src.id, sensor: id, connector: sen.connector, type: sen.type, url: sen.url };49 let connector;50 try {51 connector = getConnector(sen.connector);52 } catch (e) {53 return { ...base, status: "FAIL", http: 0, note: (e as Error).message, ms: 0 };54 }55 const endpoint: SensorEndpoint = { id, sourceId: src.id, name: sen.name, url: sen.url, type: sen.type, tier: (sen.tier ?? src.tier) as Tier, connector: sen.connector, config: sen.config, etag: null, lastModified: null, state: null };56 const obs = await connector.fetch(endpoint);57 if (obs.error) return { ...base, status: "FAIL", http: obs.meta.status, note: `${obs.error.code}: ${obs.error.message.slice(0, 160)}`, ms: Date.now() - t0 };58 if (obs.meta.status >= 400) return { ...base, status: "FAIL", http: obs.meta.status, note: `http_${obs.meta.status}${obs.meta.finalUrl !== sen.url ? " → " + obs.meta.finalUrl : ""}`, ms: Date.now() - t0 };59 try {60 const norm = await connector.normalize(endpoint, obs);61 const items = norm.items?.length;62 const thin = Boolean((norm.extra as { thin?: boolean } | undefined)?.thin);63 let status: Result["status"] = "OK";64 let note = "";65 if (norm.mode === "list" && (items ?? 0) === 0) {66 status = "WARN";67 note = "empty list";68 } else if (thin || norm.extractionConfidence < 0.5) {69 status = "WARN";70 note = `thin (confidence ${norm.extractionConfidence})`;71 } else if (norm.mode === "text" && (norm.text?.length ?? 0) < (sen.type === "FILE" ? 20 : 200)) {72 status = "WARN";73 note = `short text (${norm.text?.length ?? 0} chars)`;74 }75 if (["http", "rss", "sitemap", "statuspage", "github", "jsonlist", "statusjson", "openapi", "csv", "pdf", "headers"].includes(sen.connector) && obs.meta.finalUrl && obs.meta.finalUrl.replace(/\/$/, "") !== sen.url.replace(/\/$/, "")) note = `${note ? note + "; " : ""}redirect → ${obs.meta.finalUrl}`;76 return { ...base, status, http: obs.meta.status, mode: norm.mode, items, note, ms: Date.now() - t0 };77 } catch (e) {78 const msg = e instanceof NormalizeError ? `${e.code}: ${e.message}` : (e as Error).message;79 return { ...base, status: "FAIL", http: obs.meta.status, note: msg.slice(0, 200), ms: Date.now() - t0 };80 }81}8283async function main(): Promise<void> {84 const probe = arg("probe");85 if (probe) {86 const r = await discoverDomain(probe, { probePages: true });87 if (!r.length) console.log(`no validated endpoint found for ${probe}`);88 for (const e of r) console.log(`${e.type.padEnd(10)} ${e.connector.padEnd(10)} ${e.value.toFixed(2)} ${String(e.itemCount ?? "").padStart(5)} ${e.url} (${e.evidence}${e.title ? " · " + e.title : ""})`);89 return;90 }91 const files = process.argv.slice(2).filter((a) => !a.startsWith("--") && /\.ya?ml$/.test(a) && (process.argv[process.argv.indexOf(a) - 1] ?? "").startsWith("--") === false);92 const baseFile = process.env.WS_SOURCES_FILE ?? "./config/sources.yaml";93 const dir = process.env.WS_SOURCES_DIR ?? "./config/sources.d";94 const loaded = loadSeedsDetailed(baseFile, dir);95 for (const i of loaded.issues) console.error(`SEED ${i.file} ${i.source ?? "-"}: ${i.message}`);9697 const wanted = new Set(files.map((f) => resolve(f)));98 const onlySources = arg("source")?.split(",").filter(Boolean);99 const onlyConnector = arg("connector");100 let targets = loaded.seeds;101 if (!flag("all") && wanted.size) targets = targets.filter((s) => (loaded.origins.get(s.id) ?? []).some((f) => wanted.has(resolve(f))));102 else if (!flag("all") && !wanted.size) targets = targets.filter((s) => (loaded.origins.get(s.id) ?? []).some((f) => resolve(f) === resolve(baseFile)));103 if (onlySources) targets = targets.filter((s) => onlySources.includes(s.id));104105 // For fragment validation, only the sensors that the fragment contributed are relevant.106 const jobs: { src: SourceSeed; sen: SourceSeed["sensors"][number] }[] = [];107 for (const src of targets) for (const sen of src.sensors) if (!onlyConnector || sen.connector === onlyConnector) jobs.push({ src, sen });108 if (wanted.size && !flag("all")) {109 const fragmentUrls = new Set<string>();110 const YAML = (await import("yaml")).default;111 const { readFileSync } = await import("node:fs");112 for (const f of wanted) {113 const raw = YAML.parse(readFileSync(f, "utf8")) as { sources?: { sensors?: { url: string }[] }[] };114 for (const s of raw.sources ?? []) for (const sen of s.sensors ?? []) fragmentUrls.add(sen.url);115 }116 for (let i = jobs.length - 1; i >= 0; i--) if (!fragmentUrls.has(jobs[i]!.sen.url)) jobs.splice(i, 1);117 }118119 const concurrency = Number(arg("concurrency", "8"));120 const quiet = flag("quiet");121 const results: Result[] = [];122 // Per-host politeness: never more than 2 in flight for the same host.123 const inflight = new Map<string, number>();124 const host = (u: string): string => {125 try {126 return new URL(u).hostname;127 } catch {128 return u;129 }130 };131 const queue = [...jobs];132 await Promise.all(133 Array.from({ length: concurrency }, async () => {134 while (queue.length) {135 const idx = queue.findIndex((j) => (inflight.get(host(j.sen.url)) ?? 0) < 2);136 if (idx < 0) {137 await new Promise((r) => setTimeout(r, 150));138 continue;139 }140 const job = queue.splice(idx, 1)[0]!;141 const h = host(job.sen.url);142 inflight.set(h, (inflight.get(h) ?? 0) + 1);143 try {144 const r = await checkSensor(job.src, job.sen);145 results.push(r);146 if (!quiet || r.status !== "OK") console.log(`${r.status.padEnd(5)} ${String(r.http).padStart(3)} ${(r.mode ?? "-").padEnd(4)} ${(r.items !== undefined ? `items=${r.items}` : "-").padEnd(10)} ${r.sensor.padEnd(52).slice(0, 52)} ${r.url}${r.note ? ` (${r.note})` : ""}`);147 } finally {148 inflight.set(h, (inflight.get(h) ?? 1) - 1);149 }150 }151 }),152 );153154 const ok = results.filter((r) => r.status === "OK").length;155 const warn = results.filter((r) => r.status === "WARN").length;156 const fail = results.filter((r) => r.status === "FAIL").length;157 const byConnector = new Map<string, { ok: number; warn: number; fail: number }>();158 for (const r of results) {159 const c = byConnector.get(r.connector) ?? { ok: 0, warn: 0, fail: 0 };160 c[r.status.toLowerCase() as "ok" | "warn" | "fail"]++;161 byConnector.set(r.connector, c);162 }163 console.log(`\n${targets.length} sources · ${results.length} sensors → OK ${ok} · WARN ${warn} · FAIL ${fail}`);164 for (const [c, n] of [...byConnector.entries()].sort()) console.log(` ${c.padEnd(12)} ok ${String(n.ok).padStart(4)} warn ${String(n.warn).padStart(3)} fail ${String(n.fail).padStart(3)}`);165 const out = arg("json");166 if (out) writeFileSync(out, JSON.stringify({ generatedAt: new Date().toISOString(), sources: targets.length, results }, null, 2));167 if (fail > 0 || loaded.issues.length > 0) process.exitCode = 1;168}169170main()171 .catch((e) => {172 console.error(e);173 process.exitCode = 1;174 })175 .finally(() => closeDispatcher());176