TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { canonicalizeHtml, jaccard, newId, shingles, type SensorType } from "@websensor/core";2import { httpFetch } from "./fetcher";3import { parseFeed } from "./rss";4import { parseSitemap } from "./sitemap";56/**7 * Discovery engine: for a domain, probe well-known paths, robots.txt sitemaps, HTML8 * `<link rel="alternate">` feeds and known status providers; validate each candidate by9 * actually fetching and parsing it. Nothing is assumed from booleans in the registry.10 */11export interface DiscoveredEndpoint {12 url: string;13 type: SensorType;14 connector: string;15 evidence: string;16 /** rough information value 0–1 used to rank candidates */17 value: number;18 itemCount?: number;19 title?: string;20}2122const FEED_PATHS = ["/feed", "/rss", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml", "/feed/", "/blog/feed", "/blog/rss.xml", "/blog/feed.xml", "/news/feed", "/news/rss", "/news/rss.xml", "/newsroom/rss", "/rss/news", "/feeds/all.atom.xml", "/en/feed", "/changelog/feed", "/changelog.rss", "/changelog/rss.xml", "/releases.atom", "/security/feed", "/blog/index.xml"];23const SITEMAP_PATHS = ["/sitemap.xml", "/sitemap_index.xml", "/sitemap-index.xml", "/sitemaps/sitemap.xml", "/news-sitemap.xml", "/sitemap/news.xml"];24const PAGE_PATHS: [string, string][] = [25 ["/changelog", "changelog"],26 ["/news", "news"],27 ["/newsroom", "news"],28 ["/blog", "blog"],29 ["/releases", "releases"],30 ["/security", "security"],31 ["/pricing", "pricing"],32 ["/status", "status"],33 ["/docs", "docs"],34];3536export async function discoverDomain(domain: string, opts: { probePages?: boolean; sensorIdForLogs?: string } = {}): Promise<DiscoveredEndpoint[]> {37 const base = `https://${domain}`;38 const found = new Map<string, DiscoveredEndpoint>();39 const id = opts.sensorIdForLogs ?? `discover_${domain}`;4041 const add = (e: DiscoveredEndpoint): void => {42 const k = e.url.replace(/\/$/, "");43 if (!found.has(k) || (found.get(k)!.value < e.value)) found.set(k, e);44 };4546 // robots.txt → Sitemap: lines47 const robots = await httpFetch(id, `${base}/robots.txt`, { timeoutMs: 12_000, maxBytes: 512 * 1024 });48 const sitemapsFromRobots: string[] = [];49 if (robots.body && robots.meta.status === 200) {50 for (const line of robots.body.toString("utf8").split(/\r?\n/)) {51 const m = line.match(/^\s*sitemap:\s*(\S+)/i);52 if (m) sitemapsFromRobots.push(m[1]!);53 }54 }5556 // Homepage: <link rel="alternate"> and links to status/changelog57 const home = await httpFetch(id, base + "/", { timeoutMs: 15_000, maxBytes: 3 * 1024 * 1024 });58 const homeHtml = home.body && home.meta.status < 400 ? home.body.toString("utf8") : "";59 const alternates = [...homeHtml.matchAll(/<link[^>]+rel=["']alternate["'][^>]*>/gi)]60 .map((m) => m[0])61 .filter((tag) => /application\/(rss|atom)\+xml|application\/feed\+json/i.test(tag))62 .map((tag) => tag.match(/href=["']([^"']+)["']/i)?.[1])63 .filter((h): h is string => Boolean(h))64 .map((h) => safeAbs(h, home.meta.finalUrl || base))65 .filter((h): h is string => Boolean(h));66 const statusLinks = [...homeHtml.matchAll(/href=["'](https?:\/\/(?:status|statuspage|health)\.[^"'\s]+|https?:\/\/[^"'\s]*\.statuspage\.io[^"'\s]*)["']/gi)].map((m) => m[1]!);6768 const feedCandidates = [...new Set([...alternates, ...FEED_PATHS.map((p) => base + p)])];69 const sitemapCandidates = [...new Set([...sitemapsFromRobots.filter((u) => !/image|video/i.test(u)).sort((a, b) => Number(/news/i.test(b)) - Number(/news/i.test(a))).slice(0, 8), ...SITEMAP_PATHS.map((p) => base + p)])];7071 await parallel(feedCandidates, 6, async (url) => {72 const o = await httpFetch(id, url, { timeoutMs: 12_000, maxBytes: 4 * 1024 * 1024 });73 if (!o.body || o.meta.status !== 200) return;74 const text = o.body.toString("utf8");75 if (/^\s*<!doctype html|<html/i.test(text.slice(0, 400))) return;76 try {77 const f = parseFeed(text, o.meta.finalUrl);78 if (!f.items.length) return;79 const dated = f.items.filter((i) => i.publishedAt).length / f.items.length;80 add({ url: o.meta.finalUrl || url, type: f.kind === "atom" ? "ATOM" : "RSS", connector: "rss", evidence: alternates.includes(url) ? "link[rel=alternate]" : "well-known path", value: 0.8 + 0.2 * dated, itemCount: f.items.length, title: f.title });81 } catch {82 // not a feed83 }84 });8586 await parallel(sitemapCandidates, 4, async (url) => {87 const o = await httpFetch(id, url, { timeoutMs: 15_000, maxBytes: 20 * 1024 * 1024 });88 if (!o.body || o.meta.status !== 200) return;89 const text = o.body.toString("utf8");90 if (/^\s*<!doctype html|<html/i.test(text.slice(0, 400))) return;91 try {92 const s = parseSitemap(text);93 const n = s.kind === "sitemapindex" ? s.children.length : s.entries.length;94 if (!n) return;95 const lastmods = s.entries.filter((e) => e.lastmod).length;96 add({ url: o.meta.finalUrl || url, type: "SITEMAP", connector: "sitemap", evidence: sitemapsFromRobots.includes(url) ? "robots.txt" : "well-known path", value: 0.5 + (s.entries.length ? 0.3 * (lastmods / s.entries.length) : 0.2) + (/news/i.test(url) ? 0.2 : 0), itemCount: n });97 } catch {98 // not a sitemap99 }100 });101102 for (const s of statusLinks) {103 try {104 const u = new URL(s);105 const api = `${u.origin}/api/v2/summary.json`;106 const o = await httpFetch(id, api, { timeoutMs: 12_000, accept: "application/json" });107 if (o.body && o.meta.status === 200 && /"incidents"/.test(o.body.toString("utf8").slice(0, 20_000))) add({ url: api, type: "STATUSPAGE", connector: "statuspage", evidence: `linked from homepage (${u.host})`, value: 0.95 });108 } catch {109 // ignore110 }111 }112113 if (opts.probePages) {114 // Catch-all sites answer 200 for any path: a page candidate must be a real, distinct, non-thin document115 // (GET, no redirect back to the homepage, canonical text different from the homepage's).116 const homeCanon = homeHtml ? canonicalizeHtml(homeHtml, home.meta.finalUrl || base) : null;117 await parallel(PAGE_PATHS, 4, async ([path, kind]) => {118 const url = base + path;119 const o = await httpFetch(id, url, { timeoutMs: 15_000, maxBytes: 3 * 1024 * 1024 });120 if (o.meta.status !== 200 || !o.body) return;121 const finalPath = safePath(o.meta.finalUrl);122 if (finalPath === "/" || finalPath === "") return; // redirected home123 const c = canonicalizeHtml(o.body.toString("utf8"), o.meta.finalUrl);124 if (c.text.length < 200 || (homeCanon && (c.canonicalHash === homeCanon.canonicalHash || (c.title && c.title === homeCanon.title && jaccard(shingles(c.text), shingles(homeCanon.text)) > 0.8)))) return;125 add({ url: o.meta.finalUrl || url, type: "HTML", connector: "http", evidence: `GET 200, ${c.text.length} chars, distinct from homepage`, value: kind === "pricing" || kind === "changelog" || kind === "security" ? 0.6 : 0.4, title: c.title ?? undefined });126 });127 }128129 return [...found.values()].sort((a, b) => b.value - a.value);130}131132function safePath(u: string): string {133 try {134 return new URL(u).pathname.replace(/\/$/, "");135 } catch {136 return "";137 }138}139140function safeAbs(h: string, base: string): string | null {141 try {142 const u = new URL(h, base);143 return u.protocol.startsWith("http") ? u.toString() : null;144 } catch {145 return null;146 }147}148149async function parallel<T>(items: T[], limit: number, fn: (item: T) => Promise<void>): Promise<void> {150 let i = 0;151 const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {152 while (i < items.length) {153 const item = items[i++]!;154 try {155 await fn(item);156 } catch {157 // swallow: discovery is best effort158 }159 }160 });161 await Promise.all(workers);162}163164export function candidateId(): string {165 return newId("cand");166}167