import { canonicalizeHtml, jaccard, newId, shingles, type SensorType } from "@websensor/core"; import { httpFetch } from "./fetcher"; import { parseFeed } from "./rss"; import { parseSitemap } from "./sitemap"; /** * Discovery engine: for a domain, probe well-known paths, robots.txt sitemaps, HTML * `` feeds and known status providers; validate each candidate by * actually fetching and parsing it. Nothing is assumed from booleans in the registry. */ export interface DiscoveredEndpoint { url: string; type: SensorType; connector: string; evidence: string; /** rough information value 0–1 used to rank candidates */ value: number; itemCount?: number; title?: string; } const FEED_PATHS = ["/feed", "/rss", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml", "/feed/", "/blog/feed", "/blog/rss.xml", "/blog/feed.xml", "/news/feed", "/news/rss", "/news/rss.xml", "/newsroom/rss", "/rss/news", "/feeds/all.atom.xml", "/en/feed", "/changelog/feed", "/changelog.rss", "/changelog/rss.xml", "/releases.atom", "/security/feed", "/blog/index.xml"]; const SITEMAP_PATHS = ["/sitemap.xml", "/sitemap_index.xml", "/sitemap-index.xml", "/sitemaps/sitemap.xml", "/news-sitemap.xml", "/sitemap/news.xml"]; const PAGE_PATHS: [string, string][] = [ ["/changelog", "changelog"], ["/news", "news"], ["/newsroom", "news"], ["/blog", "blog"], ["/releases", "releases"], ["/security", "security"], ["/pricing", "pricing"], ["/status", "status"], ["/docs", "docs"], ]; export async function discoverDomain(domain: string, opts: { probePages?: boolean; sensorIdForLogs?: string } = {}): Promise { const base = `https://${domain}`; const found = new Map(); const id = opts.sensorIdForLogs ?? `discover_${domain}`; const add = (e: DiscoveredEndpoint): void => { const k = e.url.replace(/\/$/, ""); if (!found.has(k) || (found.get(k)!.value < e.value)) found.set(k, e); }; // robots.txt → Sitemap: lines const robots = await httpFetch(id, `${base}/robots.txt`, { timeoutMs: 12_000, maxBytes: 512 * 1024 }); const sitemapsFromRobots: string[] = []; if (robots.body && robots.meta.status === 200) { for (const line of robots.body.toString("utf8").split(/\r?\n/)) { const m = line.match(/^\s*sitemap:\s*(\S+)/i); if (m) sitemapsFromRobots.push(m[1]!); } } // Homepage: and links to status/changelog const home = await httpFetch(id, base + "/", { timeoutMs: 15_000, maxBytes: 3 * 1024 * 1024 }); const homeHtml = home.body && home.meta.status < 400 ? home.body.toString("utf8") : ""; const alternates = [...homeHtml.matchAll(/]+rel=["']alternate["'][^>]*>/gi)] .map((m) => m[0]) .filter((tag) => /application\/(rss|atom)\+xml|application\/feed\+json/i.test(tag)) .map((tag) => tag.match(/href=["']([^"']+)["']/i)?.[1]) .filter((h): h is string => Boolean(h)) .map((h) => safeAbs(h, home.meta.finalUrl || base)) .filter((h): h is string => Boolean(h)); const statusLinks = [...homeHtml.matchAll(/href=["'](https?:\/\/(?:status|statuspage|health)\.[^"'\s]+|https?:\/\/[^"'\s]*\.statuspage\.io[^"'\s]*)["']/gi)].map((m) => m[1]!); const feedCandidates = [...new Set([...alternates, ...FEED_PATHS.map((p) => base + p)])]; const sitemapCandidates = [...new Set([...sitemapsFromRobots.filter((u) => !/image|video/i.test(u)).sort((a, b) => Number(/news/i.test(b)) - Number(/news/i.test(a))).slice(0, 8), ...SITEMAP_PATHS.map((p) => base + p)])]; await parallel(feedCandidates, 6, async (url) => { const o = await httpFetch(id, url, { timeoutMs: 12_000, maxBytes: 4 * 1024 * 1024 }); if (!o.body || o.meta.status !== 200) return; const text = o.body.toString("utf8"); if (/^\s* i.publishedAt).length / f.items.length; add({ url: o.meta.finalUrl || url, type: f.kind === "atom" ? "ATOM" : "RSS", connector: "rss", evidence: alternates.includes(url) ? "link[rel=alternate]" : "well-known path", value: 0.8 + 0.2 * dated, itemCount: f.items.length, title: f.title }); } catch { // not a feed } }); await parallel(sitemapCandidates, 4, async (url) => { const o = await httpFetch(id, url, { timeoutMs: 15_000, maxBytes: 20 * 1024 * 1024 }); if (!o.body || o.meta.status !== 200) return; const text = o.body.toString("utf8"); if (/^\s* e.lastmod).length; add({ url: o.meta.finalUrl || url, type: "SITEMAP", connector: "sitemap", evidence: sitemapsFromRobots.includes(url) ? "robots.txt" : "well-known path", value: 0.5 + (s.entries.length ? 0.3 * (lastmods / s.entries.length) : 0.2) + (/news/i.test(url) ? 0.2 : 0), itemCount: n }); } catch { // not a sitemap } }); for (const s of statusLinks) { try { const u = new URL(s); const api = `${u.origin}/api/v2/summary.json`; const o = await httpFetch(id, api, { timeoutMs: 12_000, accept: "application/json" }); if (o.body && o.meta.status === 200 && /"incidents"/.test(o.body.toString("utf8").slice(0, 20_000))) add({ url: api, type: "STATUSPAGE", connector: "statuspage", evidence: `linked from homepage (${u.host})`, value: 0.95 }); } catch { // ignore } } if (opts.probePages) { // Catch-all sites answer 200 for any path: a page candidate must be a real, distinct, non-thin document // (GET, no redirect back to the homepage, canonical text different from the homepage's). const homeCanon = homeHtml ? canonicalizeHtml(homeHtml, home.meta.finalUrl || base) : null; await parallel(PAGE_PATHS, 4, async ([path, kind]) => { const url = base + path; const o = await httpFetch(id, url, { timeoutMs: 15_000, maxBytes: 3 * 1024 * 1024 }); if (o.meta.status !== 200 || !o.body) return; const finalPath = safePath(o.meta.finalUrl); if (finalPath === "/" || finalPath === "") return; // redirected home const c = canonicalizeHtml(o.body.toString("utf8"), o.meta.finalUrl); if (c.text.length < 200 || (homeCanon && (c.canonicalHash === homeCanon.canonicalHash || (c.title && c.title === homeCanon.title && jaccard(shingles(c.text), shingles(homeCanon.text)) > 0.8)))) return; add({ url: o.meta.finalUrl || url, type: "HTML", connector: "http", evidence: `GET 200, ${c.text.length} chars, distinct from homepage`, value: kind === "pricing" || kind === "changelog" || kind === "security" ? 0.6 : 0.4, title: c.title ?? undefined }); }); } return [...found.values()].sort((a, b) => b.value - a.value); } function safePath(u: string): string { try { return new URL(u).pathname.replace(/\/$/, ""); } catch { return ""; } } function safeAbs(h: string, base: string): string | null { try { const u = new URL(h, base); return u.protocol.startsWith("http") ? u.toString() : null; } catch { return null; } } async function parallel(items: T[], limit: number, fn: (item: T) => Promise): Promise { let i = 0; const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { while (i < items.length) { const item = items[i++]!; try { await fn(item); } catch { // swallow: discovery is best effort } } }); await Promise.all(workers); } export function candidateId(): string { return newId("cand"); }