1import { promises as dns } from "node:dns";2import { canonicalizeHtml, jaccard, registrableDomain, sha256, shingles, type SensorType, type Tier } from "@websensor/core";3import { httpFetch } from "./fetcher";4import { parseFeed } from "./rss";5import { parseSitemap } from "./sitemap";6import { detectFlavor } from "./statusjson";78/**9 * Deep discovery — the acquisition stage of the Source Factory. Starting from an organization (domain +10 * optional hints) it walks robots.txt → sitemaps → feeds (`<link rel=alternate>`, well-known paths, official11 * sub-domains) → the homepage's own navigation (newsroom, press, investors, changelog, security, pricing,12 * legal, careers, docs, status) → known status-page providers → GitHub organization → EDGAR → Hugging Face →13 * OpenAPI documents → web posture, and validates EVERY candidate by fetching and parsing it exactly as the14 * connector would. Nothing is assumed from a URL pattern.15 *16 * The result is a ranked list of candidate sensors with the evidence the Factory needs to score them:17 * page class, first-party confidence, expected change frequency, fetch cost and a content fingerprint used18 * to drop duplicates (two feeds exposing the same items, two pages with the same text).19 */20export type CandidateKind = "news" | "press" | "blog" | "ir" | "filings" | "changelog" | "releases" | "security" | "pricing" | "legal" | "careers" | "leadership" | "docs" | "api" | "status" | "sitemap" | "models" | "data" | "posture" | "other";2122export interface DeepCandidate {23 url: string;24 type: SensorType;25 connector: string;26 kind: CandidateKind;27 name: string;28 config: Record<string, unknown>;29 suggestedTier: Tier;30 evidence: string;31 /** rough information value 0–1 (before the Factory's importance-aware scoring) */32 value: number;33 itemCount?: number;34 /** items per day observed in the feed/list window (null when unknown) */35 itemsPerDay?: number | null;36 /** 1 = the organization's own registrable domain; 0.9 = official channel on a platform (GitHub, HF, status provider) */37 firstPartyConfidence: number;38 fetchCost: { bytes: number; ms: number };39 /** content fingerprint for de-duplication (sorted item keys, or canonical text hash) */40 fingerprint?: string;41 itemKeys?: string[];42 textShingles?: Set<string>;43 title?: string;44}4546export interface DeepDiscoveryHints {47 cik?: string | number;48 github_org?: string;49 github_repos?: string[];50 hf_author?: string;51 status_url?: string;52 hosts?: string[];53 urls?: { url: string; kind?: string; connector?: string; type?: string; tier?: Tier; config?: Record<string, unknown>; name?: string }[];54 posture?: boolean;55}5657export interface DeepDiscoveryOptions {58 hints?: DeepDiscoveryHints;59 /** maximum number of HTTP requests for this organization (default 90) */60 budget?: number;61 concurrency?: number;62 sensorIdForLogs?: string;63 probePages?: boolean;64 probeSubdomains?: boolean;65 probePosture?: boolean;66 probeOpenApi?: boolean;67}6869export interface DeepDiscoveryResult {70 domain: string;71 /** origin actually used after following the homepage redirect (may be another registrable domain) */72 origin: string;73 canonicalDomain: string;74 homeStatus: number;75 /** anti-bot / challenge detected on the homepage */76 blocked: boolean;77 requests: number;78 candidates: DeepCandidate[];79 rejected: { url: string; reason: string }[];80 notes: string[];81 durationMs: number;82}8384// ------------------------------------------------------------------------------------------------85// Vocabularies86// ------------------------------------------------------------------------------------------------8788const FEED_PATHS = ["/feed", "/rss", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml", "/feed/", "/feeds/all.atom.xml", "/blog/feed", "/blog/rss", "/blog/rss.xml", "/blog/feed.xml", "/blog/atom.xml", "/blog/index.xml", "/news/feed", "/news/rss", "/news/rss.xml", "/news/feed.xml", "/news.rss", "/news.xml", "/newsroom/rss", "/newsroom/feed", "/newsroom/rss.xml", "/press/rss", "/press/feed", "/press-releases/rss", "/press-releases/feed", "/rss/news", "/rss/press-releases.xml", "/rss/news-releases.xml", "/rss/pressreleases.xml", "/investors/rss", "/investors/feed", "/en/feed", "/en/rss", "/en/rss.xml", "/en/news/rss", "/changelog/feed", "/changelog.rss", "/changelog/rss.xml", "/changelog.xml", "/releases.atom", "/releases/feed", "/security/feed", "/security/rss", "/advisories/rss", "/advisories.rss", "/updates/feed", "/updates/rss", "/whats-new/feed", "/?feed=rss2", "/feed/atom", "/feeds/posts/default", "/feed.json"];89const SITEMAP_PATHS = ["/sitemap.xml", "/sitemap_index.xml", "/sitemap-index.xml", "/sitemaps/sitemap.xml", "/news-sitemap.xml", "/sitemap/news.xml", "/sitemap-news.xml", "/sitemap.txt"];90const OPENAPI_PATHS = ["/openapi.json", "/swagger.json", "/api/openapi.json", "/api/swagger.json", "/v1/openapi.json", "/api/v1/openapi.json", "/.well-known/openapi.json", "/swagger/v1/swagger.json", "/api-docs", "/openapi.yaml", "/api/openapi.yaml"];91const SUBDOMAIN_KINDS: [string, CandidateKind][] = [["blog", "blog"], ["news", "news"], ["newsroom", "news"], ["press", "press"], ["media", "press"], ["investors", "ir"], ["investor", "ir"], ["ir", "ir"], ["status", "status"], ["docs", "docs"], ["developer", "docs"], ["developers", "docs"], ["security", "security"], ["trust", "security"], ["careers", "careers"], ["jobs", "careers"], ["changelog", "changelog"], ["updates", "changelog"], ["engineering", "blog"], ["research", "blog"], ["about", "other"], ["corporate", "news"], ["api", "api"]];9293/** Page classes probed from navigation links and well-known paths. `paths` are fallbacks when the navigation has no link. */94const PAGE_KINDS: { kind: CandidateKind; label: string; hrefRe: RegExp; textRe: RegExp; paths: string[]; tier: Tier; value: number; max: number }[] = [95 { kind: "pricing", label: "pricing", hrefRe: /\/(pricing|plans|plans-and-pricing|tarifs|preise|precios)(\/|$|\?)/i, textRe: /^(pricing|plans|plans & pricing|tarifs|preise|precios)$/i, paths: ["/pricing", "/plans", "/pricing/"], tier: "C", value: 0.75, max: 1 },96 { kind: "changelog", label: "changelog", hrefRe: /\/(changelog|change-log|release-notes|releasenotes|whats-new|what-s-new|updates|product-updates)(\/|$|\?)/i, textRe: /^(changelog|change log|release notes|what'?s new|product updates|updates)$/i, paths: ["/changelog", "/release-notes", "/whats-new", "/updates"], tier: "B", value: 0.75, max: 1 },97 { kind: "security", label: "security advisories", hrefRe: /\/(security|security-advisories|advisories|psirt|trust|security-bulletins|bulletins|vulnerabilit)/i, textRe: /^(security|security advisories|advisories|trust center|trust|psirt|security bulletins)$/i, paths: ["/security", "/security/advisories", "/trust", "/.well-known/security.txt"], tier: "B", value: 0.7, max: 2 },98 { kind: "news", label: "newsroom", hrefRe: /\/(news|newsroom|news-room|media|media-center|media-centre|actualites|actualit%C3%A9s|nouvelles|aktuelles|noticias|notizie)(\/|$|\?)/i, textRe: /^(news|newsroom|news ?room|media|media cent(er|re)|latest news|in the news|actualit[ée]s|nouvelles|aktuelles|noticias)$/i, paths: ["/news", "/newsroom", "/news/", "/en/news", "/media"], tier: "B", value: 0.65, max: 1 },99 { kind: "press", label: "press releases", hrefRe: /\/(press|press-releases|pressreleases|press-room|pressroom|press-center|communiques|communiqu%C3%A9s|presse|prensa)(\/|$|\?)/i, textRe: /^(press|press releases|press ?room|press cent(er|re)|communiqu[ée]s( de presse)?|presse|prensa)$/i, paths: ["/press", "/press-releases", "/press/", "/newsroom/press-releases"], tier: "B", value: 0.65, max: 1 },100 { kind: "ir", label: "investor relations", hrefRe: /\/(investors|investor-relations|investor|ir|relations-investisseurs|investor-center)(\/|$|\?)/i, textRe: /^(investors|investor relations|investor cent(er|re)|ir|relations investisseurs|shareholders)$/i, paths: ["/investors", "/investor-relations", "/investors/", "/investor"], tier: "C", value: 0.6, max: 1 },101 { kind: "legal", label: "terms of service", hrefRe: /\/(terms|terms-of-service|terms-of-use|tos|legal\/terms|conditions|cgu|cgv|legal)(\/|$|\?|\.)/i, textRe: /^(terms|terms of (service|use)|terms & conditions|terms and conditions|legal|conditions g[ée]n[ée]rales|cgu|cgv)$/i, paths: ["/terms", "/legal/terms", "/terms-of-service", "/legal"], tier: "C", value: 0.6, max: 1 },102 { kind: "legal", label: "privacy policy", hrefRe: /\/(privacy|privacy-policy|privacypolicy|legal\/privacy|politique-de-confidentialite|datenschutz|privacidad)(\/|$|\?|\.)/i, textRe: /^(privacy|privacy policy|privacy notice|politique de confidentialit[ée]|datenschutz|privacidad)$/i, paths: ["/privacy", "/legal/privacy", "/privacy-policy"], tier: "C", value: 0.55, max: 1 },103 { kind: "careers", label: "careers", hrefRe: /\/(careers|jobs|join-us|work-with-us|carrieres|carri%C3%A8res|karriere|empleo)(\/|$|\?)/i, textRe: /^(careers|jobs|join us|work with us|carri[èe]res|karriere|empleo|we'?re hiring)$/i, paths: ["/careers", "/jobs", "/careers/"], tier: "D", value: 0.35, max: 1 },104 { kind: "leadership", label: "leadership", hrefRe: /\/(leadership|management|executive-team|our-team|team|board-of-directors|board|about\/leadership|company\/leadership|direction|governance)(\/|$|\?)/i, textRe: /^(leadership|leadership team|management|management team|executive team|our team|board of directors|governance|direction)$/i, paths: ["/leadership", "/about/leadership", "/company/leadership", "/about/team"], tier: "D", value: 0.45, max: 1 },105 { kind: "docs", label: "documentation", hrefRe: /\/(docs|documentation|developers|developer|api-docs|reference)(\/|$|\?)/i, textRe: /^(docs|documentation|developers|developer|api|api reference|api docs)$/i, paths: ["/docs", "/documentation", "/developers"], tier: "D", value: 0.4, max: 1 },106 { kind: "data", label: "open data", hrefRe: /\/(open-data|opendata|data|statistics|statistiques|datasets|publications)(\/|$|\?)/i, textRe: /^(open data|data|statistics|statistiques|datasets|publications)$/i, paths: [], tier: "C", value: 0.4, max: 1 },107];108109const STATUS_HOST_RE = /^(https?:\/\/)?((status|statuspage|uptime|health|trust|system-status|servicestatus)\.[^/\s"']+|[^/\s"']*\.(statuspage\.io|instatus\.com|betteruptime\.com|betterstack\.com|incident\.io|status\.io|hund\.io|statuspal\.io|freshstatus\.io|status\.page|statusgator\.com))/i;110const CHALLENGE_RE = /cf-browser-verification|challenge-platform|__cf_chl|Attention Required!|Just a moment\.\.\.|Access Denied|Request unsuccessful\. Incapsula|akamai.*bot|_Incapsula_Resource|Please enable JavaScript to view|captcha/i;111112// ------------------------------------------------------------------------------------------------113114export async function discoverOrganization(domain: string, opts: DeepDiscoveryOptions = {}): Promise<DeepDiscoveryResult> {115 const started = Date.now();116 const id = opts.sensorIdForLogs ?? `factory_${domain}`;117 const budget = opts.budget ?? 90;118 const hints = opts.hints ?? {};119 const notes: string[] = [];120 const rejected: { url: string; reason: string }[] = [];121 const found = new Map<string, DeepCandidate>();122 let requests = 0;123124 const add = (c: DeepCandidate): void => {125 const k = c.url.replace(/\/$/, "");126 const prev = found.get(k);127 if (!prev || prev.value < c.value) found.set(k, c);128 };129 const budgetLeft = (): boolean => requests < budget;130 const get = async (url: string, o: { timeoutMs?: number; maxBytes?: number; accept?: string; headers?: Record<string, string>; userAgent?: string } = {}) => {131 requests++;132 return httpFetch(id, url, { timeoutMs: o.timeoutMs ?? 15_000, maxBytes: o.maxBytes ?? 3 * 1024 * 1024, accept: o.accept, headers: o.headers, userAgent: o.userAgent });133 };134135 // ---- 1. robots.txt --------------------------------------------------------------------------------------136 let base = `https://${domain}`;137 const sitemapsFromRobots: string[] = [];138 let robotsOk = false;139 let robotsMentionsAi = false;140 const robots = await get(`${base}/robots.txt`, { timeoutMs: 12_000, maxBytes: 512 * 1024 }).catch(() => null);141 if (robots?.body && robots.meta.status === 200 && !/^\s*<!doctype html|<html/i.test(robots.body.toString("utf8").slice(0, 300))) {142 robotsOk = true;143 const text = robots.body.toString("utf8");144 for (const line of text.split(/\r?\n/)) {145 const m = line.match(/^\s*sitemap:\s*(\S+)/i);146 if (m) sitemapsFromRobots.push(m[1]!);147 }148 robotsMentionsAi = /GPTBot|ClaudeBot|Google-Extended|CCBot|anthropic-ai|PerplexityBot|Bytespider|Applebot-Extended|meta-externalagent/i.test(text);149 }150151 // ---- 2. homepage ------------------------------------------------------------------------------------------152 const home = await get(`${base}/`, { timeoutMs: 20_000, maxBytes: 4 * 1024 * 1024 });153 const homeStatus = home.meta.status;154 let blocked = false;155 const homeHtml = home.body && home.meta.status < 400 ? home.body.toString("utf8") : "";156 if ([403, 429, 503].includes(homeStatus) || (homeHtml && CHALLENGE_RE.test(homeHtml.slice(0, 20_000)) && homeHtml.length < 60_000)) {157 blocked = true;158 notes.push(`homepage ${homeStatus}${homeHtml ? " (anti-bot challenge)" : ""}`);159 }160 let origin = base;161 try {162 const fu = new URL(home.meta.finalUrl || base);163 origin = fu.origin;164 if (registrableDomain(fu.hostname) !== registrableDomain(domain)) notes.push(`homepage redirects to ${fu.hostname}`);165 } catch {166 // keep base167 }168 base = origin;169 const canonicalDomain = registrableDomain(new URL(base).hostname);170 const ownHosts = new Set<string>([registrableDomain(domain), canonicalDomain, ...(hints.hosts ?? []).map((h) => registrableDomain(h))]);171 const isOwn = (u: string): boolean => {172 try {173 return ownHosts.has(registrableDomain(new URL(u).hostname));174 } catch {175 return false;176 }177 };178179 const homeCanon = homeHtml ? canonicalizeHtml(homeHtml, home.meta.finalUrl || base) : null;180 const homeShingles = homeCanon ? shingles(homeCanon.text) : null;181 const alternates: string[] = [];182 const statusLinks = new Set<string>();183 const githubOrgs = new Set<string>();184 const hfAuthors = new Set<string>();185 const pageLinks = new Map<CandidateKind, string[]>();186 if (homeHtml) {187 for (const m of homeHtml.matchAll(/<link[^>]+rel=["']alternate["'][^>]*>/gi)) {188 const tag = m[0];189 if (!/application\/(rss|atom)\+xml|application\/feed\+json/i.test(tag)) continue;190 const href = tag.match(/href=["']([^"']+)["']/i)?.[1];191 const abs = href ? safeAbs(href, home.meta.finalUrl || base) : null;192 if (abs) alternates.push(abs);193 }194 // Navigation links: classify by href pattern first, then by anchor text.195 const anchors = [...homeHtml.matchAll(/<a\b[^>]*href=["']([^"'#]+)["'][^>]*>([\s\S]{0,300}?)<\/a>/gi)].map((m) => ({ href: safeAbs(m[1]!, home.meta.finalUrl || base), text: m[2]!.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim() }));196 for (const a of anchors) {197 if (!a.href) continue;198 const gh = a.href.match(/^https?:\/\/(?:www\.)?github\.com\/([A-Za-z0-9_.-]+)\/?(?:$|\?)/);199 if (gh && !/^(features|about|pricing|topics|login|signup|sponsors|marketplace|explore|apps|orgs|enterprise|solutions|security|team|customer-stories|readme|collections|events|trending|search|settings|notifications|new|organizations|site|contact)$/i.test(gh[1]!)) githubOrgs.add(gh[1]!);200 const hf = a.href.match(/^https?:\/\/huggingface\.co\/([A-Za-z0-9_.-]+)\/?$/);201 if (hf && !/^(models|datasets|spaces|docs|pricing|papers|blog|join|login|enterprise|posts|tasks|learn|chat)$/i.test(hf[1]!)) hfAuthors.add(hf[1]!);202 if (STATUS_HOST_RE.test(a.href)) statusLinks.add(a.href);203 if (!isOwn(a.href)) continue;204 let path = "";205 try {206 path = new URL(a.href).pathname;207 } catch {208 continue;209 }210 if (path.split("/").filter(Boolean).length > 3) continue; // deep article links are not sections211 for (const pk of PAGE_KINDS) {212 if (pk.hrefRe.test(path) || pk.textRe.test(a.text)) {213 const arr = pageLinks.get(pk.kind) ?? [];214 if (!arr.includes(a.href) && arr.length < 4) arr.push(a.href);215 pageLinks.set(pk.kind, arr);216 }217 }218 }219 }220 if (hints.github_org) githubOrgs.add(hints.github_org);221 if (hints.hf_author) hfAuthors.add(hints.hf_author);222 if (hints.status_url) statusLinks.add(hints.status_url);223224 // ---- 3. official sub-domains (cheap DNS first) ------------------------------------------------------------225 const subHosts: { host: string; kind: CandidateKind }[] = [];226 if (opts.probeSubdomains !== false) {227 const rd = registrableDomain(domain);228 // Wildcard DNS (everything resolves) makes existence checks meaningless: only probe the core labels then.229 const wildcard = await resolves(`ws-probe-${Date.now().toString(36)}.${rd}`);230 if (wildcard) notes.push("wildcard DNS");231 const labels = wildcard ? ["blog", "news", "status", "investors", "docs", "security"] : [...new Set([...SUBDOMAIN_KINDS.map(([l]) => l)])];232 await parallel(labels, 8, async (label) => {233 const host = `${label}.${rd}`;234 if (host === new URL(base).hostname) return;235 if (wildcard || (await resolves(host))) subHosts.push({ host, kind: SUBDOMAIN_KINDS.find(([l]) => l === label)![1] });236 });237 if (wildcard) {238 // keep only sub-domains that actually answer with a distinct page239 const alive: typeof subHosts = [];240 await parallel(subHosts, 4, async (s) => {241 if (!budgetLeft()) return;242 const o = await get(`https://${s.host}/`, { timeoutMs: 10_000, maxBytes: 512 * 1024 });243 const fh = o.meta.finalUrl ? new URL(o.meta.finalUrl).hostname : "";244 if (o.meta.status === 200 && fh === s.host) alive.push(s);245 });246 subHosts.splice(0, subHosts.length, ...alive);247 }248 for (const h of hints.hosts ?? []) if (!subHosts.some((s) => s.host === h)) subHosts.push({ host: h, kind: "other" });249 if (subHosts.length) notes.push(`sub-domains: ${subHosts.map((s) => s.host).join(", ")}`);250 }251252 // ---- 4. feeds ---------------------------------------------------------------------------------------------253 const feedUrls = new Set<string>(alternates);254 for (const p of FEED_PATHS) feedUrls.add(base + p);255 for (const s of subHosts) {256 if (s.kind === "status" || s.kind === "api") continue;257 for (const p of ["/feed", "/rss", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml", "/feed/", "/?feed=rss2"]) feedUrls.add(`https://${s.host}${p}`);258 }259 // Feeds are cheap and high-value: spend up to 55 % of the budget here.260 const feedList = [...feedUrls].slice(0, Math.max(10, Math.floor(budget * 0.55)));261 const feeds: DeepCandidate[] = [];262 await parallel(feedList, opts.concurrency ?? 4, async (url) => {263 if (!budgetLeft()) return;264 const o = await get(url, { timeoutMs: 12_000, maxBytes: 4 * 1024 * 1024, accept: "application/rss+xml, application/atom+xml, application/feed+json, application/xml;q=0.9, */*;q=0.5" });265 if (!o.body || o.meta.status !== 200) return;266 const text = o.body.toString("utf8");267 if (/^\s*<!doctype html|<html/i.test(text.slice(0, 400))) return;268 try {269 const f = parseFeed(text, o.meta.finalUrl);270 if (!f.items.length) return rejected.push({ url, reason: "empty feed" });271 const dates = f.items.map((i) => (i.publishedAt ? new Date(i.publishedAt).getTime() : NaN)).filter((t) => !Number.isNaN(t));272 const dated = dates.length / f.items.length;273 const itemsPerDay = itemsPerDayOf(dates);274 const finalUrl = o.meta.finalUrl || url;275 const kind = feedKind(finalUrl, f.title);276 const keys = f.items.map((i) => i.key).sort();277 const stale = dates.length && Math.max(...dates) < Date.now() - 3 * 365 * 86400e3;278 if (stale) return rejected.push({ url, reason: "feed abandoned (> 3 years)" });279 feeds.push({280 url: finalUrl,281 type: f.kind === "atom" ? "ATOM" : "RSS",282 connector: "rss",283 kind,284 name: feedName(kind, finalUrl),285 config: {},286 suggestedTier: itemsPerDay !== null && itemsPerDay < 0.05 ? "C" : "B",287 evidence: `${alternates.includes(url) ? "link[rel=alternate]" : "well-known path"} · ${f.items.length} items${itemsPerDay !== null ? ` · ${itemsPerDay}/day` : ""}`,288 value: 0.75 + 0.15 * dated + (kind === "news" || kind === "press" ? 0.1 : kind === "security" || kind === "changelog" ? 0.08 : 0),289 itemCount: f.items.length,290 itemsPerDay,291 firstPartyConfidence: isOwn(finalUrl) ? 1 : 0.7,292 fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs },293 fingerprint: sha256(keys.join("\n")),294 itemKeys: keys,295 title: f.title,296 });297 } catch {298 // not a feed299 }300 });301 // De-duplicate feeds sharing ≥ 70 % of their items (rss + atom of the same channel, /feed vs /rss.xml).302 feeds.sort((a, b) => b.value - a.value || (b.itemCount ?? 0) - (a.itemCount ?? 0));303 const keptFeeds: DeepCandidate[] = [];304 for (const f of feeds) {305 const dup = keptFeeds.find((k) => k.itemKeys && f.itemKeys && jaccard(new Set(k.itemKeys), new Set(f.itemKeys)) >= 0.7);306 if (dup) rejected.push({ url: f.url, reason: `duplicate of ${dup.url}` });307 else keptFeeds.push(f);308 }309 for (const f of keptFeeds) add(f);310311 // ---- 5. sitemaps -------------------------------------------------------------------------------------------312 const sitemapCandidates = [...new Set([...sitemapsFromRobots.filter((u) => !/image|video/i.test(u)).sort((a, b) => Number(/news/i.test(b)) - Number(/news/i.test(a))).slice(0, 6), ...SITEMAP_PATHS.map((p) => base + p)])];313 const sitemaps: DeepCandidate[] = [];314 await parallel(sitemapCandidates, 3, async (url) => {315 if (!budgetLeft() || sitemaps.length >= 3) return;316 const o = await get(url, { timeoutMs: 15_000, maxBytes: 20 * 1024 * 1024, accept: "application/xml, text/xml, text/plain;q=0.8, */*;q=0.5" });317 if (!o.body || o.meta.status !== 200) return;318 const text = o.body.toString("utf8");319 if (/^\s*<!doctype html|<html/i.test(text.slice(0, 400))) return;320 try {321 const s = parseSitemap(text);322 const n = s.kind === "sitemapindex" ? s.children.length : s.entries.length;323 if (!n) return;324 const lastmods = s.entries.filter((e) => e.lastmod).length;325 const isNews = /news/i.test(url) || s.entries.some((e) => e.publishedAt);326 sitemaps.push({ url: o.meta.finalUrl || url, type: "SITEMAP", connector: "sitemap", kind: "sitemap", name: isNews ? "news sitemap" : "sitemap", config: { maxChildren: 4, maxUrls: 3000 }, suggestedTier: "B", evidence: `${sitemapsFromRobots.includes(url) ? "robots.txt" : "well-known path"} · ${n} ${s.kind === "sitemapindex" ? "children" : "urls"}`, value: 0.45 + (s.entries.length ? 0.3 * (lastmods / s.entries.length) : 0.15) + (isNews ? 0.2 : 0), itemCount: n, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });327 } catch {328 // not a sitemap329 }330 });331 sitemaps.sort((a, b) => b.value - a.value);332 if (sitemaps[0]) add(sitemaps[0]);333334 // ---- 6. status pages ----------------------------------------------------------------------------------------335 const statusHosts = new Set<string>();336 if (hints.status_url) {337 try {338 statusHosts.add(new URL(hints.status_url).origin);339 } catch {340 // ignore341 }342 }343 for (const s of statusLinks) {344 try {345 statusHosts.add(new URL(s).origin);346 } catch {347 // ignore348 }349 }350 for (const s of subHosts) if (s.kind === "status") statusHosts.add(`https://${s.host}`);351 let statusFound = false;352 for (const root of statusHosts) {353 if (statusFound || !budgetLeft()) break;354 for (const [path, connector] of [["/api/v2/summary.json", "statuspage"], ["/summary.json", "statusjson"], ["/api/v1/summary", "statusjson"]] as const) {355 if (statusFound || !budgetLeft()) break;356 const o = await get(root + path, { timeoutMs: 12_000, accept: "application/json" });357 if (!o.body || o.meta.status !== 200) continue;358 const text = o.body.toString("utf8");359 try {360 const j = JSON.parse(text) as Record<string, unknown>;361 if (connector === "statuspage" && Array.isArray(j.incidents) && (j.components || j.status)) {362 add({ url: root + path, type: "STATUSPAGE", connector: "statuspage", kind: "status", name: "status", config: {}, suggestedTier: "S", evidence: `Atlassian Statuspage (${new URL(root).host})`, value: 0.95, itemsPerDay: null, firstPartyConfidence: 0.9, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });363 statusFound = true;364 } else if (connector === "statusjson") {365 const flavor = detectFlavor(j);366 if (flavor) {367 add({ url: root + path, type: "STATUSPAGE", connector: "statusjson", kind: "status", name: "status", config: { flavor }, suggestedTier: "S", evidence: `${flavor} status JSON (${new URL(root).host})`, value: 0.93, itemsPerDay: null, firstPartyConfidence: 0.9, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });368 statusFound = true;369 }370 }371 } catch {372 // not JSON373 }374 }375 }376377 // ---- 7. pages (server-rendered sections) --------------------------------------------------------------------378 if (opts.probePages !== false && !blocked) {379 const pageJobs: { kind: CandidateKind; label: string; tier: Tier; value: number; urls: string[]; max: number }[] = [];380 for (const pk of PAGE_KINDS) {381 const linked = pageLinks.get(pk.kind) ?? [];382 // feeds already cover news/press/blog for this org → the HTML index page adds little383 if ((pk.kind === "news" || pk.kind === "press") && keptFeeds.some((f) => f.kind === pk.kind || f.kind === "news")) continue;384 const shallowFirst = (a: string, b: string): number => safePath(a).split("/").length - safePath(b).split("/").length || a.length - b.length;385 const urls = [...new Set([...linked.filter((u) => pk.hrefRe.test(safePath(u)) || linked.length <= 2).sort(shallowFirst), ...pk.paths.map((p) => base + p)])].slice(0, 4);386 if (!urls.length) continue;387 pageJobs.push({ kind: pk.kind, label: pk.label, tier: pk.tier, value: pk.value, urls, max: pk.max });388 }389 const pageCanon: { url: string; sh: Set<string>; hash: string }[] = [];390 const probed = new Set<string>();391 await parallel(pageJobs, 3, async (job) => {392 let kept = 0;393 for (const url of job.urls) {394 if (kept >= job.max || !budgetLeft()) break;395 const pk = url.replace(/\/$/, "");396 if (probed.has(pk) || found.has(pk)) continue;397 probed.add(pk);398 const o = await get(url, { timeoutMs: 15_000, maxBytes: 3 * 1024 * 1024, accept: "text/html, application/xhtml+xml;q=0.9, */*;q=0.5" });399 if (o.meta.status !== 200 || !o.body) continue;400 const finalPath = safePath(o.meta.finalUrl);401 if (finalPath === "/" || finalPath === "") continue; // redirected home402 if (!isOwn(o.meta.finalUrl || url)) continue;403 if (!/text\/html|xhtml/i.test(o.meta.contentType ?? "") && /^\s*[{[]/.test(o.body.toString("utf8").slice(0, 50))) continue;404 const c = canonicalizeHtml(o.body.toString("utf8"), o.meta.finalUrl || url);405 if (c.text.length < 300) {406 rejected.push({ url, reason: `thin (${c.text.length} chars)` });407 continue;408 }409 const sh = shingles(c.text);410 if (homeCanon && (c.canonicalHash === homeCanon.canonicalHash || (homeShingles && jaccard(sh, homeShingles) > 0.8))) {411 rejected.push({ url, reason: "same as homepage" });412 continue;413 }414 const dup = pageCanon.find((p) => p.hash === c.canonicalHash || jaccard(p.sh, sh) > 0.85);415 if (dup) {416 rejected.push({ url, reason: `duplicate of ${dup.url}` });417 continue;418 }419 pageCanon.push({ url: o.meta.finalUrl || url, sh, hash: c.canonicalHash });420 const isSecurityTxt = /security\.txt$/.test(url);421 add({ url: o.meta.finalUrl || url, type: "HTML", connector: "http", kind: job.kind, name: isSecurityTxt ? "security.txt" : job.label, config: {}, suggestedTier: isSecurityTxt ? "D" : job.tier, evidence: `${pageLinks.get(job.kind)?.includes(url) ? "linked from homepage" : "well-known path"} · GET 200 · ${c.text.length} chars · distinct from homepage`, value: isSecurityTxt ? 0.3 : job.value, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs }, fingerprint: c.canonicalHash, textShingles: sh, title: c.title ?? undefined });422 kept++;423 }424 });425 }426427 // ---- 8. GitHub organization → repositories' releases ----------------------------------------------------------428 const repos = new Set<string>(hints.github_repos ?? []);429 for (const org of [...githubOrgs].slice(0, 1)) {430 if (repos.size >= 4 || !budgetLeft()) break;431 const token = process.env.GITHUB_TOKEN;432 if (token) {433 const o = await get(`https://api.github.com/search/repositories?q=org:${encodeURIComponent(org)}+fork:false&sort=stars&order=desc&per_page=5`, { accept: "application/vnd.github+json", headers: { authorization: `Bearer ${token}`, "x-github-api-version": "2022-11-28" } });434 if (o.body && o.meta.status === 200) {435 try {436 const j = JSON.parse(o.body.toString("utf8")) as { items?: { full_name: string; archived?: boolean; stargazers_count?: number }[] };437 for (const r of j.items ?? []) if (!r.archived && (r.stargazers_count ?? 0) >= 50) repos.add(r.full_name);438 } catch {439 // ignore440 }441 }442 } else {443 // Unauthenticated: the organization page lists pinned / popular repositories server-side.444 const o = await get(`https://github.com/${org}`, { timeoutMs: 15_000, maxBytes: 3 * 1024 * 1024 });445 if (o.body && o.meta.status === 200) {446 const html = o.body.toString("utf8");447 const counts = new Map<string, number>();448 for (const m of html.matchAll(new RegExp(`href="/${org.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/([A-Za-z0-9_.-]+)"`, "g"))) {449 const r = m[1]!;450 if (/^(followers|following|repositories|projects|packages|people|teams|sponsoring|discussions|orgs|\.github)$/i.test(r)) continue;451 counts.set(r, (counts.get(r) ?? 0) + 1);452 }453 const pinned = [...html.matchAll(/class="[^"]*pinned-item-list-item[^"]*"[\s\S]{0,1500}?href="\/([^/"]+)\/([^/"]+)"/g)].map((m) => `${m[1]}/${m[2]}`).filter((f) => f.toLowerCase().startsWith(org.toLowerCase() + "/"));454 for (const f of pinned) repos.add(f);455 for (const [r] of [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 6)) if (repos.size < 6) repos.add(`${org}/${r}`);456 }457 }458 }459 const repoList = [...repos].slice(0, 6);460 let releasesKept = 0;461 await parallel(repoList, 3, async (full) => {462 if (releasesKept >= 4 || !budgetLeft()) return;463 for (const kind of ["releases", "tags"] as const) {464 const url = `https://github.com/${full}/${kind}.atom`;465 const o = await get(url, { timeoutMs: 12_000, accept: "application/atom+xml, application/xml;q=0.9" });466 if (!o.body || o.meta.status !== 200) {467 if (o.meta.status === 404) return;468 continue;469 }470 try {471 const f = parseFeed(o.body.toString("utf8"), url);472 if (!f.items.length) continue;473 const dates = f.items.map((i) => (i.publishedAt ? new Date(i.publishedAt).getTime() : NaN)).filter((t) => !Number.isNaN(t));474 if (dates.length && Math.max(...dates) < Date.now() - 2 * 365 * 86400e3) return rejected.push({ url, reason: "repository inactive (> 2 years)" });475 add({ url, type: "GITHUB_RELEASE", connector: "github", kind: "releases", name: `${full.split("/")[1]} ${kind}`, config: { repo: full, kind }, suggestedTier: "B", evidence: `GitHub ${kind}.atom · ${f.items.length} items`, value: kind === "releases" ? 0.8 : 0.65, itemCount: f.items.length, itemsPerDay: null, firstPartyConfidence: 0.9, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs }, fingerprint: sha256(f.items.map((i) => i.key).sort().join("\n")) });476 releasesKept++;477 return;478 } catch {479 // not a feed480 }481 }482 });483484 // ---- 9. Hugging Face models ------------------------------------------------------------------------------------485 for (const author of [...hfAuthors].slice(0, 2)) {486 if (!budgetLeft()) break;487 const url = `https://huggingface.co/api/models?author=${encodeURIComponent(author)}&sort=lastModified&direction=-1&limit=50`;488 const o = await get(url, { accept: "application/json" });489 if (!o.body || o.meta.status !== 200) continue;490 try {491 const j = JSON.parse(o.body.toString("utf8")) as unknown[];492 if (Array.isArray(j) && j.length) add({ url, type: "REST_API", connector: "jsonlist", kind: "models", name: "hugging face models", config: { keyField: "id", titleField: "id", dateField: "lastModified", compareFields: ["pipeline_tag", "library_name"], maxItems: 50 }, suggestedTier: "B", evidence: `Hugging Face author ${author} · ${j.length} models`, value: 0.85, itemCount: j.length, itemsPerDay: null, firstPartyConfidence: 0.9, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });493 } catch {494 // ignore495 }496 }497498 // ---- 10. EDGAR filings -------------------------------------------------------------------------------------------499 if (hints.cik !== undefined && budgetLeft()) {500 const cik = String(hints.cik).replace(/\D/g, "").padStart(10, "0");501 const url = `https://data.sec.gov/submissions/CIK${cik}.json`;502 const ua = process.env.WS_EDGAR_USER_AGENT ?? "WebSensor (contact@websensor.io)";503 const o = await get(url, { accept: "application/json", userAgent: ua, headers: { "user-agent": ua }, timeoutMs: 30_000, maxBytes: 30 * 1024 * 1024 });504 if (o.body && o.meta.status === 200 && /"filings"/.test(o.body.toString("utf8").slice(0, 200_000))) {505 add({ url, type: "REST_API", connector: "edgar", kind: "filings", name: "edgar filings", config: { forms: ["8-K", "10-K", "10-Q", "6-K", "20-F", "40-F", "S-1", "SC 13D", "DEF 14A"] }, suggestedTier: "B", evidence: `SEC EDGAR submissions CIK ${cik}`, value: 0.9, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });506 } else rejected.push({ url, reason: `edgar ${o.meta.status}` });507 }508509 // ---- 11. OpenAPI documents ---------------------------------------------------------------------------------------510 if (opts.probeOpenApi !== false && (subHosts.some((s) => s.kind === "docs" || s.kind === "api") || pageLinks.has("docs")) && budgetLeft()) {511 const roots = [base, ...subHosts.filter((s) => s.kind === "docs" || s.kind === "api").map((s) => `https://${s.host}`)].slice(0, 3);512 let openapiFound = false;513 for (const root of roots) {514 if (openapiFound) break;515 for (const p of OPENAPI_PATHS.slice(0, 7)) {516 if (openapiFound || !budgetLeft()) break;517 const o = await get(root + p, { timeoutMs: 15_000, maxBytes: 8 * 1024 * 1024, accept: "application/json, application/yaml, */*;q=0.5" });518 if (!o.body || o.meta.status !== 200) continue;519 const text = o.body.toString("utf8").slice(0, 4000);520 if (/^\s*[{[]/.test(text) && /"(openapi|swagger)"\s*:/.test(text)) {521 add({ url: root + p, type: "JSON", connector: "openapi", kind: "api", name: "openapi", config: {}, suggestedTier: "C", evidence: `OpenAPI document (${new URL(root).host})`, value: 0.7, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });522 openapiFound = true;523 }524 }525 }526 }527528 // ---- 12. explicit hint URLs (validated like everything else) --------------------------------------------------------529 for (const h of hints.urls ?? []) {530 if (!budgetLeft()) break;531 const known = h.url.replace(/\/$/, "");532 if (found.has(known)) continue;533 const o = await get(h.url, { timeoutMs: 15_000, maxBytes: 6 * 1024 * 1024 });534 if (!o.body || o.meta.status !== 200) {535 rejected.push({ url: h.url, reason: `hint ${o.meta.status || o.error?.code}` });536 continue;537 }538 const text = o.body.toString("utf8");539 const kind = (h.kind ?? "other") as CandidateKind;540 const tier = h.tier ?? (kind === "status" ? "S" : kind === "pricing" || kind === "legal" || kind === "ir" ? "C" : "B");541 const cost = { bytes: o.meta.contentLength, ms: o.meta.durationMs };542 if (h.connector) {543 add({ url: h.url, type: (h.type ?? (h.connector === "rss" ? "RSS" : h.connector === "jsonlist" || h.connector === "edgar" ? "REST_API" : h.connector === "statuspage" || h.connector === "statusjson" ? "STATUSPAGE" : h.connector === "sitemap" ? "SITEMAP" : "HTML")) as SensorType, connector: h.connector, kind, name: h.name ?? kind, config: h.config ?? {}, suggestedTier: tier, evidence: "explicit hint · GET 200", value: 0.8, itemsPerDay: null, firstPartyConfidence: isOwn(h.url) ? 1 : 0.8, fetchCost: cost });544 continue;545 }546 // Sniff: feed → rss; JSON → http/json; HTML → http547 if (!/^\s*<!doctype html|<html/i.test(text.slice(0, 400)) && /<(rss|feed|rdf:RDF)\b/i.test(text.slice(0, 2000))) {548 try {549 const f = parseFeed(text, o.meta.finalUrl);550 if (f.items.length) {551 add({ url: o.meta.finalUrl || h.url, type: f.kind === "atom" ? "ATOM" : "RSS", connector: "rss", kind, name: h.name ?? feedName(kind, h.url), config: {}, suggestedTier: tier, evidence: `explicit hint · feed · ${f.items.length} items`, value: 0.85, itemCount: f.items.length, itemsPerDay: null, firstPartyConfidence: isOwn(h.url) ? 1 : 0.8, fetchCost: cost, fingerprint: sha256(f.items.map((i) => i.key).sort().join("\n")), itemKeys: f.items.map((i) => i.key).sort() });552 continue;553 }554 } catch {555 // fallthrough556 }557 }558 if (/^\s*[{[]/.test(text.slice(0, 50))) {559 add({ url: h.url, type: "JSON", connector: "http", kind, name: h.name ?? kind, config: h.config ?? {}, suggestedTier: tier, evidence: "explicit hint · JSON 200", value: 0.65, itemsPerDay: null, firstPartyConfidence: isOwn(h.url) ? 1 : 0.8, fetchCost: cost });560 continue;561 }562 const c = canonicalizeHtml(text, o.meta.finalUrl || h.url);563 if (c.text.length < 300) {564 rejected.push({ url: h.url, reason: `hint thin (${c.text.length} chars)` });565 continue;566 }567 add({ url: o.meta.finalUrl || h.url, type: "HTML", connector: "http", kind, name: h.name ?? PAGE_KINDS.find((p) => p.kind === kind)?.label ?? kind, config: h.config ?? {}, suggestedTier: tier, evidence: `explicit hint · GET 200 · ${c.text.length} chars`, value: 0.7, itemsPerDay: null, firstPartyConfidence: isOwn(h.url) ? 1 : 0.8, fetchCost: cost, fingerprint: c.canonicalHash, title: c.title ?? undefined });568 }569570 // ---- 13. web posture (no network here: validated in shadow by the dns/tls connectors) ------------------------------571 if (opts.probePosture || hints.posture) {572 const rd = registrableDomain(domain);573 add({ url: `dns://${rd}`, type: "DNS", connector: "dns", kind: "posture", name: "dns", config: {}, suggestedTier: "D", evidence: "posture", value: 0.3, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: 0, ms: 0 } });574 add({ url: `tls://${new URL(base).hostname}`, type: "TLS", connector: "tls", kind: "posture", name: "tls certificate", config: {}, suggestedTier: "D", evidence: "posture", value: 0.3, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: 0, ms: 0 } });575 if (robotsOk) add({ url: `https://${domain}/robots.txt`, type: "HTML", connector: "http", kind: "posture", name: "robots.txt", config: {}, suggestedTier: "D", evidence: robotsMentionsAi ? "robots.txt names AI crawlers" : "robots.txt", value: robotsMentionsAi ? 0.45 : 0.3, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: robots?.meta.contentLength ?? 0, ms: robots?.meta.durationMs ?? 0 } });576 }577578 const candidates = [...found.values()].sort((a, b) => b.value - a.value);579 // Strip the heavy de-dup helpers before returning (they are not serializable and no longer needed).580 for (const c of candidates) {581 delete c.textShingles;582 if (c.itemKeys && c.itemKeys.length > 60) c.itemKeys = c.itemKeys.slice(0, 60);583 }584 return { domain, origin: base, canonicalDomain, homeStatus, blocked, requests, candidates, rejected, notes, durationMs: Date.now() - started };585}586587// ------------------------------------------------------------------------------------------------588589/** Expected publication rate from item dates: the median gap between consecutive items (robust to one bogus date). */590export function itemsPerDayOf(dates: number[]): number | null {591 const d = [...dates].filter((t) => Number.isFinite(t)).sort((a, b) => b - a).slice(0, 30);592 if (d.length < 3) return null;593 const gaps = d.slice(1).map((t, i) => (d[i]! - t) / 86400e3).filter((g) => g >= 0).sort((a, b) => a - b);594 if (!gaps.length) return null;595 const median = gaps[Math.floor(gaps.length / 2)]!;596 if (median <= 0) return 50;597 return Math.round((1 / median) * 1000) / 1000;598}599600function feedKind(url: string, title?: string): CandidateKind {601 const s = `${url} ${title ?? ""}`.toLowerCase();602 if (/press|communiqu|pressemitteilung|prensa|news-release|newsrelease/.test(s)) return "press";603 if (/investor|\/ir\/|ir\.|sec-filing|filings/.test(s)) return "ir";604 if (/security|advisor|psirt|bulletin|vulnerab|cve/.test(s)) return "security";605 if (/changelog|release-notes|releasenotes|releases|whats-new|updates/.test(s)) return "changelog";606 if (/blog|engineering|research|insights|stories/.test(s)) return "blog";607 if (/news|actualit|nouvelles|aktuell|noticias|announcement/.test(s)) return "news";608 if (/jobs|careers/.test(s)) return "careers";609 return "news";610}611612function feedName(kind: CandidateKind, url: string): string {613 switch (kind) {614 case "press":615 return "press releases feed";616 case "ir":617 return "investor news feed";618 case "security":619 return "security feed";620 case "changelog":621 return "changelog feed";622 case "blog":623 return /engineering/i.test(url) ? "engineering blog feed" : /research/i.test(url) ? "research feed" : "blog feed";624 case "careers":625 return "jobs feed";626 default:627 return "news feed";628 }629}630631function safePath(u: string): string {632 try {633 return new URL(u).pathname.replace(/\/$/, "");634 } catch {635 return "";636 }637}638639function safeAbs(h: string, base: string): string | null {640 try {641 const u = new URL(h, base);642 if (!u.protocol.startsWith("http")) return null;643 u.hash = "";644 return u.toString();645 } catch {646 return null;647 }648}649650async function resolves(host: string): Promise<boolean> {651 try {652 const r = await dns.lookup(host, { all: true });653 return r.length > 0;654 } catch {655 return false;656 }657}658659async function parallel<T>(items: T[], limit: number, fn: (item: T) => Promise<unknown>): Promise<void> {660 let i = 0;661 const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {662 while (i < items.length) {663 const item = items[i++]!;664 try {665 await fn(item);666 } catch {667 // best effort668 }669 }670 });671 await Promise.all(workers);672}673