TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { promises as dns } from "node:dns";2import { canonicalizeHtml, jaccard, registrableDomain, sha256, shingles, type SensorType, type Tier } from "@websensor/core";3import { httpFetch } from "./fetcher";4import { parseFeed } from "./rss";5import { parseSitemap } from "./sitemap";6import { detectFlavor } from "./statusjson";78/**9 * Deep discovery — the acquisition stage of the Source Factory. Starting from an organization (domain +10 * optional hints) it walks robots.txt → sitemaps → feeds (`<link rel=alternate>`, well-known paths, official11 * sub-domains) → the homepage's own navigation (newsroom, press, investors, changelog, security, pricing,12 * legal, careers, docs, status) → known status-page providers → GitHub organization → EDGAR → Hugging Face →13 * OpenAPI documents → web posture, and validates EVERY candidate by fetching and parsing it exactly as the14 * connector would. Nothing is assumed from a URL pattern.15 *16 * The result is a ranked list of candidate sensors with the evidence the Factory needs to score them:17 * page class, first-party confidence, expected change frequency, fetch cost and a content fingerprint used18 * to drop duplicates (two feeds exposing the same items, two pages with the same text).19 */20export type CandidateKind = "news" | "press" | "blog" | "ir" | "filings" | "changelog" | "releases" | "security" | "pricing" | "legal" | "careers" | "leadership" | "docs" | "api" | "status" | "sitemap" | "models" | "data" | "posture" | "other";2122export interface DeepCandidate {23 url: string;24 type: SensorType;25 connector: string;26 kind: CandidateKind;27 name: string;28 config: Record<string, unknown>;29 suggestedTier: Tier;30 evidence: string;31 /** rough information value 0–1 (before the Factory's importance-aware scoring) */32 value: number;33 itemCount?: number;34 /** items per day observed in the feed/list window (null when unknown) */35 itemsPerDay?: number | null;36 /** 1 = the organization's own registrable domain; 0.9 = official channel on a platform (GitHub, HF, status provider) */37 firstPartyConfidence: number;38 fetchCost: { bytes: number; ms: number };39 /** content fingerprint for de-duplication (sorted item keys, or canonical text hash) */40 fingerprint?: string;41 itemKeys?: string[];42 textShingles?: Set<string>;43 title?: string;44}4546export interface DeepDiscoveryHints {47 cik?: string | number;48 github_org?: string;49 github_repos?: string[];50 hf_author?: string;51 status_url?: string;52 hosts?: string[];53 urls?: { url: string; kind?: string; connector?: string; type?: string; tier?: Tier; config?: Record<string, unknown>; name?: string }[];54 posture?: boolean;55}5657export interface DeepDiscoveryOptions {58 hints?: DeepDiscoveryHints;59 /** maximum number of HTTP requests for this organization (default 90) */60 budget?: number;61 /** wall-clock deadline for the whole organization (default 120 s); optional stages are skipped once reached */62 deadlineMs?: number;63 concurrency?: number;64 sensorIdForLogs?: string;65 probePages?: boolean;66 probeSubdomains?: boolean;67 probePosture?: boolean;68 probeOpenApi?: boolean;69}7071export interface DeepDiscoveryResult {72 domain: string;73 /** origin actually used after following the homepage redirect (may be another registrable domain) */74 origin: string;75 canonicalDomain: string;76 homeStatus: number;77 /** anti-bot / challenge detected on the homepage */78 blocked: boolean;79 requests: number;80 candidates: DeepCandidate[];81 rejected: { url: string; reason: string }[];82 notes: string[];83 durationMs: number;84}8586// ------------------------------------------------------------------------------------------------87// Vocabularies88// ------------------------------------------------------------------------------------------------8990const FEED_PATHS = ["/feed", "/rss", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml", "/feed/", "/feeds/all.atom.xml", "/blog/feed", "/blog/rss", "/blog/rss.xml", "/blog/feed.xml", "/blog/atom.xml", "/blog/index.xml", "/news/feed", "/news/rss", "/news/rss.xml", "/news/feed.xml", "/news.rss", "/news.xml", "/newsroom/rss", "/newsroom/feed", "/newsroom/rss.xml", "/press/rss", "/press/feed", "/press-releases/rss", "/press-releases/feed", "/rss/news", "/rss/press-releases.xml", "/rss/news-releases.xml", "/rss/pressreleases.xml", "/investors/rss", "/investors/feed", "/en/feed", "/en/rss", "/en/rss.xml", "/en/news/rss", "/changelog/feed", "/changelog.rss", "/changelog/rss.xml", "/changelog.xml", "/releases.atom", "/releases/feed", "/security/feed", "/security/rss", "/advisories/rss", "/advisories.rss", "/updates/feed", "/updates/rss", "/whats-new/feed", "/?feed=rss2", "/feed/atom", "/feeds/posts/default", "/feed.json"];91const SITEMAP_PATHS = ["/sitemap.xml", "/sitemap_index.xml", "/sitemap-index.xml", "/sitemaps/sitemap.xml", "/news-sitemap.xml", "/sitemap/news.xml", "/sitemap-news.xml", "/sitemap.txt"];92const OPENAPI_PATHS = ["/openapi.json", "/swagger.json", "/api/openapi.json", "/api/swagger.json", "/v1/openapi.json", "/api/v1/openapi.json", "/.well-known/openapi.json", "/swagger/v1/swagger.json", "/api-docs", "/openapi.yaml", "/api/openapi.yaml"];93const SUBDOMAIN_KINDS: [string, CandidateKind][] = [["blog", "blog"], ["news", "news"], ["newsroom", "news"], ["press", "press"], ["media", "press"], ["investors", "ir"], ["investor", "ir"], ["ir", "ir"], ["status", "status"], ["docs", "docs"], ["developer", "docs"], ["developers", "docs"], ["security", "security"], ["trust", "security"], ["careers", "careers"], ["jobs", "careers"], ["changelog", "changelog"], ["updates", "changelog"], ["engineering", "blog"], ["research", "blog"], ["about", "other"], ["corporate", "news"], ["api", "api"]];9495/** Page classes probed from navigation links and well-known paths. `paths` are fallbacks when the navigation has no link. */96const PAGE_KINDS: { kind: CandidateKind; label: string; hrefRe: RegExp; textRe: RegExp; paths: string[]; tier: Tier; value: number; max: number }[] = [97 { kind: "pricing", label: "pricing", hrefRe: /\/(pricing|plans|plans-and-pricing|tarifs|preise|precios)(\/|$|\?)/i, textRe: /^(pricing|plans|plans & pricing|tarifs|preise|precios)$/i, paths: ["/pricing", "/plans", "/pricing/"], tier: "C", value: 0.75, max: 1 },98 { kind: "changelog", label: "changelog", hrefRe: /\/(changelog|change-log|release-notes|releasenotes|whats-new|what-s-new|updates|product-updates)(\/|$|\?)/i, textRe: /^(changelog|change log|release notes|what'?s new|product updates|updates)$/i, paths: ["/changelog", "/release-notes", "/whats-new", "/updates"], tier: "B", value: 0.75, max: 1 },99 { kind: "security", label: "security advisories", hrefRe: /\/(security|security-advisories|advisories|psirt|trust|security-bulletins|bulletins|vulnerabilit)/i, textRe: /^(security|security advisories|advisories|trust center|trust|psirt|security bulletins)$/i, paths: ["/security", "/security/advisories", "/trust", "/.well-known/security.txt"], tier: "B", value: 0.7, max: 2 },100 { kind: "news", label: "newsroom", hrefRe: /\/(news|newsroom|news-room|media|media-center|media-centre|actualites|actualit%C3%A9s|nouvelles|aktuelles|noticias|notizie)(\/|$|\?)/i, textRe: /^(news|newsroom|news ?room|media|media cent(er|re)|latest news|in the news|actualit[ée]s|nouvelles|aktuelles|noticias)$/i, paths: ["/news", "/newsroom", "/news/", "/en/news", "/media"], tier: "B", value: 0.65, max: 1 },101 { kind: "press", label: "press releases", hrefRe: /\/(press|press-releases|pressreleases|press-room|pressroom|press-center|communiques|communiqu%C3%A9s|presse|prensa)(\/|$|\?)/i, textRe: /^(press|press releases|press ?room|press cent(er|re)|communiqu[ée]s( de presse)?|presse|prensa)$/i, paths: ["/press", "/press-releases", "/press/", "/newsroom/press-releases"], tier: "B", value: 0.65, max: 1 },102 { kind: "ir", label: "investor relations", hrefRe: /\/(investors|investor-relations|investor|ir|relations-investisseurs|investor-center)(\/|$|\?)/i, textRe: /^(investors|investor relations|investor cent(er|re)|ir|relations investisseurs|shareholders)$/i, paths: ["/investors", "/investor-relations", "/investors/", "/investor"], tier: "C", value: 0.6, max: 1 },103 { kind: "legal", label: "terms of service", hrefRe: /\/(terms|terms-of-service|terms-of-use|tos|legal\/terms|conditions|cgu|cgv|legal)(\/|$|\?|\.)/i, textRe: /^(terms|terms of (service|use)|terms & conditions|terms and conditions|legal|conditions g[ée]n[ée]rales|cgu|cgv)$/i, paths: ["/terms", "/legal/terms", "/terms-of-service", "/legal"], tier: "C", value: 0.6, max: 1 },104 { kind: "legal", label: "privacy policy", hrefRe: /\/(privacy|privacy-policy|privacypolicy|legal\/privacy|politique-de-confidentialite|datenschutz|privacidad)(\/|$|\?|\.)/i, textRe: /^(privacy|privacy policy|privacy notice|politique de confidentialit[ée]|datenschutz|privacidad)$/i, paths: ["/privacy", "/legal/privacy", "/privacy-policy"], tier: "C", value: 0.55, max: 1 },105 { kind: "careers", label: "careers", hrefRe: /\/(careers|jobs|join-us|work-with-us|carrieres|carri%C3%A8res|karriere|empleo)(\/|$|\?)/i, textRe: /^(careers|jobs|join us|work with us|carri[èe]res|karriere|empleo|we'?re hiring)$/i, paths: ["/careers", "/jobs", "/careers/"], tier: "D", value: 0.35, max: 1 },106 { kind: "leadership", label: "leadership", hrefRe: /\/(leadership|management|executive-team|our-team|team|board-of-directors|board|about\/leadership|company\/leadership|direction|governance)(\/|$|\?)/i, textRe: /^(leadership|leadership team|management|management team|executive team|our team|board of directors|governance|direction)$/i, paths: ["/leadership", "/about/leadership", "/company/leadership", "/about/team"], tier: "D", value: 0.45, max: 1 },107 { kind: "docs", label: "documentation", hrefRe: /\/(docs|documentation|developers|developer|api-docs|reference)(\/|$|\?)/i, textRe: /^(docs|documentation|developers|developer|api|api reference|api docs)$/i, paths: ["/docs", "/documentation", "/developers"], tier: "D", value: 0.4, max: 1 },108 { kind: "data", label: "open data", hrefRe: /\/(open-data|opendata|data|statistics|statistiques|datasets|publications)(\/|$|\?)/i, textRe: /^(open data|data|statistics|statistiques|datasets|publications)$/i, paths: [], tier: "C", value: 0.4, max: 1 },109];110111const STATUS_HOST_RE = /^(https?:\/\/)?((status|statuspage|uptime|health|trust|system-status|servicestatus)\.[^/\s"']+|[^/\s"']*\.(statuspage\.io|instatus\.com|betteruptime\.com|betterstack\.com|incident\.io|status\.io|hund\.io|statuspal\.io|freshstatus\.io|status\.page|statusgator\.com))/i;112const CHALLENGE_RE = /cf-browser-verification|challenge-platform|__cf_chl|Attention Required!|Just a moment\.\.\.|Access Denied|Request unsuccessful\. Incapsula|akamai.*bot|_Incapsula_Resource|Please enable JavaScript to view|captcha/i;113114// ------------------------------------------------------------------------------------------------115116export async function discoverOrganization(domain: string, opts: DeepDiscoveryOptions = {}): Promise<DeepDiscoveryResult> {117 const started = Date.now();118 const id = opts.sensorIdForLogs ?? `factory_${domain}`;119 const budget = opts.budget ?? 90;120 const hints = opts.hints ?? {};121 const notes: string[] = [];122 const rejected: { url: string; reason: string }[] = [];123 const found = new Map<string, DeepCandidate>();124 let requests = 0;125126 const add = (c: DeepCandidate): void => {127 const k = c.url.replace(/\/$/, "");128 const prev = found.get(k);129 if (!prev || prev.value < c.value) found.set(k, c);130 };131 const deadline = started + (opts.deadlineMs ?? 120_000);132 const budgetLeft = (): boolean => requests < budget && Date.now() < deadline;133 const get = async (url: string, o: { timeoutMs?: number; maxBytes?: number; accept?: string; headers?: Record<string, string>; userAgent?: string } = {}) => {134 requests++;135 return httpFetch(id, url, { timeoutMs: o.timeoutMs ?? 15_000, maxBytes: o.maxBytes ?? 3 * 1024 * 1024, accept: o.accept, headers: o.headers, userAgent: o.userAgent });136 };137138 // ---- 1. robots.txt --------------------------------------------------------------------------------------139 let base = `https://${domain}`;140 const sitemapsFromRobots: string[] = [];141 let robotsOk = false;142 let robotsMentionsAi = false;143 const robots = await get(`${base}/robots.txt`, { timeoutMs: 12_000, maxBytes: 512 * 1024 }).catch(() => null);144 if (robots?.body && robots.meta.status === 200 && !/^\s*<!doctype html|<html/i.test(robots.body.toString("utf8").slice(0, 300))) {145 robotsOk = true;146 const text = robots.body.toString("utf8");147 for (const line of text.split(/\r?\n/)) {148 const m = line.match(/^\s*sitemap:\s*(\S+)/i);149 if (m) sitemapsFromRobots.push(m[1]!);150 }151 robotsMentionsAi = /GPTBot|ClaudeBot|Google-Extended|CCBot|anthropic-ai|PerplexityBot|Bytespider|Applebot-Extended|meta-externalagent/i.test(text);152 }153154 // ---- 2. homepage ------------------------------------------------------------------------------------------155 const home = await get(`${base}/`, { timeoutMs: 20_000, maxBytes: 4 * 1024 * 1024 });156 const homeStatus = home.meta.status;157 let blocked = false;158 const homeHtml = home.body && home.meta.status < 400 ? home.body.toString("utf8") : "";159 if ([403, 429, 503].includes(homeStatus) || (homeHtml && CHALLENGE_RE.test(homeHtml.slice(0, 20_000)) && homeHtml.length < 60_000)) {160 blocked = true;161 notes.push(`homepage ${homeStatus}${homeHtml ? " (anti-bot challenge)" : ""}`);162 }163 let origin = base;164 try {165 const fu = new URL(home.meta.finalUrl || base);166 origin = fu.origin;167 if (registrableDomain(fu.hostname) !== registrableDomain(domain)) notes.push(`homepage redirects to ${fu.hostname}`);168 } catch {169 // keep base170 }171 base = origin;172 const canonicalDomain = registrableDomain(new URL(base).hostname);173 const ownHosts = new Set<string>([registrableDomain(domain), canonicalDomain, ...(hints.hosts ?? []).map((h) => registrableDomain(h))]);174 const isOwn = (u: string): boolean => {175 try {176 return ownHosts.has(registrableDomain(new URL(u).hostname));177 } catch {178 return false;179 }180 };181182 const homeCanon = homeHtml ? canonicalizeHtml(homeHtml, home.meta.finalUrl || base) : null;183 const homeShingles = homeCanon ? shingles(homeCanon.text) : null;184 const alternates: string[] = [];185 const statusLinks = new Set<string>();186 const githubOrgs = new Set<string>();187 const hfAuthors = new Set<string>();188 const pageLinks = new Map<CandidateKind, string[]>();189 if (homeHtml) {190 for (const m of homeHtml.matchAll(/<link[^>]+rel=["']alternate["'][^>]*>/gi)) {191 const tag = m[0];192 if (!/application\/(rss|atom)\+xml|application\/feed\+json/i.test(tag)) continue;193 const href = tag.match(/href=["']([^"']+)["']/i)?.[1];194 const abs = href ? safeAbs(href, home.meta.finalUrl || base) : null;195 if (abs) alternates.push(abs);196 }197 // Navigation links: classify by href pattern first, then by anchor text.198 const anchors = [...homeHtml.matchAll(/<a\b[^>]*href=["']([^"'#]+)["'][^>]*>([\s\S]{0,300}?)<\/a>/gi)].map((m) => ({ href: safeAbs(m[1]!, home.meta.finalUrl || base), text: m[2]!.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim() }));199 for (const a of anchors) {200 if (!a.href) continue;201 const gh = a.href.match(/^https?:\/\/(?:www\.)?github\.com\/([A-Za-z0-9_.-]+)\/?(?:$|\?)/);202 if (gh && !/^(features|about|pricing|topics|login|signup|sponsors|marketplace|explore|apps|orgs|enterprise|solutions|security|team|customer-stories|readme|collections|events|trending|search|settings|notifications|new|organizations|site|contact)$/i.test(gh[1]!)) githubOrgs.add(gh[1]!);203 const hf = a.href.match(/^https?:\/\/huggingface\.co\/([A-Za-z0-9_.-]+)\/?$/);204 if (hf && !/^(models|datasets|spaces|docs|pricing|papers|blog|join|login|enterprise|posts|tasks|learn|chat)$/i.test(hf[1]!)) hfAuthors.add(hf[1]!);205 if (STATUS_HOST_RE.test(a.href)) statusLinks.add(a.href);206 if (!isOwn(a.href)) continue;207 let path = "";208 try {209 path = new URL(a.href).pathname;210 } catch {211 continue;212 }213 if (path.split("/").filter(Boolean).length > 3) continue; // deep article links are not sections214 for (const pk of PAGE_KINDS) {215 if (pk.hrefRe.test(path) || pk.textRe.test(a.text)) {216 const arr = pageLinks.get(pk.kind) ?? [];217 if (!arr.includes(a.href) && arr.length < 4) arr.push(a.href);218 pageLinks.set(pk.kind, arr);219 }220 }221 }222 }223 if (hints.github_org) githubOrgs.add(hints.github_org);224 if (hints.hf_author) hfAuthors.add(hints.hf_author);225 if (hints.status_url) statusLinks.add(hints.status_url);226227 // ---- 3. official sub-domains (cheap DNS first) ------------------------------------------------------------228 const subHosts: { host: string; kind: CandidateKind }[] = [];229 if (opts.probeSubdomains !== false) {230 const rd = registrableDomain(domain);231 // Wildcard DNS (everything resolves) makes existence checks meaningless: only probe the core labels then.232 const wildcard = await resolves(`ws-probe-${Date.now().toString(36)}.${rd}`);233 if (wildcard) notes.push("wildcard DNS");234 const labels = wildcard ? ["blog", "news", "status", "investors", "docs", "security"] : [...new Set([...SUBDOMAIN_KINDS.map(([l]) => l)])];235 await parallel(labels, 8, async (label) => {236 const host = `${label}.${rd}`;237 if (host === new URL(base).hostname) return;238 if (wildcard || (await resolves(host))) subHosts.push({ host, kind: SUBDOMAIN_KINDS.find(([l]) => l === label)![1] });239 });240 if (wildcard) {241 // keep only sub-domains that actually answer with a distinct page242 const alive: typeof subHosts = [];243 await parallel(subHosts, 4, async (s) => {244 if (!budgetLeft()) return;245 const o = await get(`https://${s.host}/`, { timeoutMs: 10_000, maxBytes: 512 * 1024 });246 const fh = o.meta.finalUrl ? new URL(o.meta.finalUrl).hostname : "";247 if (o.meta.status === 200 && fh === s.host) alive.push(s);248 });249 subHosts.splice(0, subHosts.length, ...alive);250 }251 for (const h of hints.hosts ?? []) if (!subHosts.some((s) => s.host === h)) subHosts.push({ host: h, kind: "other" });252 if (subHosts.length) notes.push(`sub-domains: ${subHosts.map((s) => s.host).join(", ")}`);253 }254255 // ---- 4. feeds ---------------------------------------------------------------------------------------------256 const feedUrls = new Set<string>(alternates);257 for (const p of FEED_PATHS) feedUrls.add(base + p);258 for (const s of subHosts) {259 if (s.kind === "status" || s.kind === "api") continue;260 for (const p of ["/feed", "/rss", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml", "/feed/", "/?feed=rss2"]) feedUrls.add(`https://${s.host}${p}`);261 }262 // Feeds are cheap and high-value: spend up to 55 % of the budget here.263 const feedList = [...feedUrls].slice(0, Math.max(10, Math.floor(budget * 0.55)));264 const feeds: DeepCandidate[] = [];265 await parallel(feedList, opts.concurrency ?? 4, async (url) => {266 if (!budgetLeft()) return;267 const o = await get(url, { timeoutMs: 12_000, maxBytes: 4 * 1024 * 1024, accept: "application/rss+xml, application/atom+xml, application/feed+json, application/xml;q=0.9, */*;q=0.5" });268 if (!o.body || o.meta.status !== 200) return;269 const text = o.body.toString("utf8");270 if (/^\s*<!doctype html|<html/i.test(text.slice(0, 400))) return;271 try {272 const f = parseFeed(text, o.meta.finalUrl);273 if (!f.items.length) return rejected.push({ url, reason: "empty feed" });274 const dates = f.items.map((i) => (i.publishedAt ? new Date(i.publishedAt).getTime() : NaN)).filter((t) => !Number.isNaN(t));275 const dated = dates.length / f.items.length;276 const itemsPerDay = itemsPerDayOf(dates);277 const finalUrl = o.meta.finalUrl || url;278 const kind = feedKind(finalUrl, f.title);279 const keys = f.items.map((i) => i.key).sort();280 const stale = dates.length && Math.max(...dates) < Date.now() - 3 * 365 * 86400e3;281 if (stale) return rejected.push({ url, reason: "feed abandoned (> 3 years)" });282 feeds.push({283 url: finalUrl,284 type: f.kind === "atom" ? "ATOM" : "RSS",285 connector: "rss",286 kind,287 name: feedName(kind, finalUrl),288 config: {},289 suggestedTier: itemsPerDay !== null && itemsPerDay < 0.05 ? "C" : "B",290 evidence: `${alternates.includes(url) ? "link[rel=alternate]" : "well-known path"} · ${f.items.length} items${itemsPerDay !== null ? ` · ${itemsPerDay}/day` : ""}`,291 value: 0.75 + 0.15 * dated + (kind === "news" || kind === "press" ? 0.1 : kind === "security" || kind === "changelog" ? 0.08 : 0),292 itemCount: f.items.length,293 itemsPerDay,294 firstPartyConfidence: isOwn(finalUrl) ? 1 : 0.7,295 fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs },296 fingerprint: sha256(keys.join("\n")),297 itemKeys: keys,298 title: f.title,299 });300 } catch {301 // not a feed302 }303 });304 // De-duplicate feeds sharing ≥ 70 % of their items (rss + atom of the same channel, /feed vs /rss.xml).305 feeds.sort((a, b) => b.value - a.value || (b.itemCount ?? 0) - (a.itemCount ?? 0));306 const keptFeeds: DeepCandidate[] = [];307 for (const f of feeds) {308 const dup = keptFeeds.find((k) => k.itemKeys && f.itemKeys && jaccard(new Set(k.itemKeys), new Set(f.itemKeys)) >= 0.7);309 if (dup) rejected.push({ url: f.url, reason: `duplicate of ${dup.url}` });310 else keptFeeds.push(f);311 }312 for (const f of keptFeeds) add(f);313314 // ---- 5. sitemaps -------------------------------------------------------------------------------------------315 const sitemapCandidates = [...new Set([...sitemapsFromRobots.filter((u) => !/image|video/i.test(u)).sort((a, b) => Number(/news/i.test(b)) - Number(/news/i.test(a))).slice(0, 6), ...SITEMAP_PATHS.map((p) => base + p)])];316 const sitemaps: DeepCandidate[] = [];317 await parallel(sitemapCandidates, 3, async (url) => {318 if (!budgetLeft() || sitemaps.length >= 3) return;319 const o = await get(url, { timeoutMs: 15_000, maxBytes: 20 * 1024 * 1024, accept: "application/xml, text/xml, text/plain;q=0.8, */*;q=0.5" });320 if (!o.body || o.meta.status !== 200) return;321 const text = o.body.toString("utf8");322 if (/^\s*<!doctype html|<html/i.test(text.slice(0, 400))) return;323 try {324 const s = parseSitemap(text);325 const n = s.kind === "sitemapindex" ? s.children.length : s.entries.length;326 if (!n) return;327 const lastmods = s.entries.filter((e) => e.lastmod).length;328 const isNews = /news/i.test(url) || s.entries.some((e) => e.publishedAt);329 sitemaps.push({ url: o.meta.finalUrl || url, type: "SITEMAP", connector: "sitemap", kind: "sitemap", name: isNews ? "news sitemap" : "sitemap", config: { maxChildren: 4, maxUrls: 3000 }, suggestedTier: isNews ? "B" : "C", evidence: `${sitemapsFromRobots.includes(url) ? "robots.txt" : "well-known path"} · ${n} ${s.kind === "sitemapindex" ? "children" : "urls"}`, value: 0.45 + (s.entries.length ? 0.3 * (lastmods / s.entries.length) : 0.15) + (isNews ? 0.2 : 0), itemCount: n, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });330 } catch {331 // not a sitemap332 }333 });334 sitemaps.sort((a, b) => b.value - a.value);335 if (sitemaps[0]) add(sitemaps[0]);336337 // ---- 6. status pages ----------------------------------------------------------------------------------------338 const statusHosts = new Set<string>();339 if (hints.status_url) {340 try {341 statusHosts.add(new URL(hints.status_url).origin);342 } catch {343 // ignore344 }345 }346 for (const s of statusLinks) {347 try {348 statusHosts.add(new URL(s).origin);349 } catch {350 // ignore351 }352 }353 for (const s of subHosts) if (s.kind === "status") statusHosts.add(`https://${s.host}`);354 let statusFound = false;355 for (const root of statusHosts) {356 if (statusFound || !budgetLeft()) break;357 for (const [path, connector] of [["/api/v2/summary.json", "statuspage"], ["/summary.json", "statusjson"], ["/api/v1/summary", "statusjson"]] as const) {358 if (statusFound || !budgetLeft()) break;359 const o = await get(root + path, { timeoutMs: 12_000, accept: "application/json" });360 if (!o.body || o.meta.status !== 200) continue;361 const text = o.body.toString("utf8");362 try {363 const j = JSON.parse(text) as Record<string, unknown>;364 if (connector === "statuspage" && Array.isArray(j.incidents) && (j.components || j.status)) {365 add({ url: root + path, type: "STATUSPAGE", connector: "statuspage", kind: "status", name: "status", config: {}, suggestedTier: "S", evidence: `Atlassian Statuspage (${new URL(root).host})`, value: 0.95, itemsPerDay: null, firstPartyConfidence: 0.9, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });366 statusFound = true;367 } else if (connector === "statusjson") {368 const flavor = detectFlavor(j);369 if (flavor) {370 add({ url: root + path, type: "STATUSPAGE", connector: "statusjson", kind: "status", name: "status", config: { flavor }, suggestedTier: "S", evidence: `${flavor} status JSON (${new URL(root).host})`, value: 0.93, itemsPerDay: null, firstPartyConfidence: 0.9, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });371 statusFound = true;372 }373 }374 } catch {375 // not JSON376 }377 }378 }379380 // ---- 7. pages (server-rendered sections) --------------------------------------------------------------------381 if (opts.probePages !== false && !blocked) {382 const pageJobs: { kind: CandidateKind; label: string; tier: Tier; value: number; urls: string[]; max: number }[] = [];383 for (const pk of PAGE_KINDS) {384 const linked = pageLinks.get(pk.kind) ?? [];385 // feeds already cover news/press/blog for this org → the HTML index page adds little386 if ((pk.kind === "news" || pk.kind === "press") && keptFeeds.some((f) => f.kind === pk.kind || f.kind === "news")) continue;387 const shallowFirst = (a: string, b: string): number => safePath(a).split("/").length - safePath(b).split("/").length || a.length - b.length;388 const urls = [...new Set([...linked.filter((u) => pk.hrefRe.test(safePath(u)) || linked.length <= 2).sort(shallowFirst), ...pk.paths.map((p) => base + p)])].slice(0, 4);389 if (!urls.length) continue;390 pageJobs.push({ kind: pk.kind, label: pk.label, tier: pk.tier, value: pk.value, urls, max: pk.max });391 }392 const pageCanon: { url: string; sh: Set<string>; hash: string }[] = [];393 const probed = new Set<string>();394 await parallel(pageJobs, 3, async (job) => {395 let kept = 0;396 for (const url of job.urls) {397 if (kept >= job.max || !budgetLeft()) break;398 const pk = url.replace(/\/$/, "");399 if (probed.has(pk) || found.has(pk)) continue;400 probed.add(pk);401 const o = await get(url, { timeoutMs: 15_000, maxBytes: 3 * 1024 * 1024, accept: "text/html, application/xhtml+xml;q=0.9, */*;q=0.5" });402 if (o.meta.status !== 200 || !o.body) continue;403 const finalPath = safePath(o.meta.finalUrl);404 if (finalPath === "/" || finalPath === "") continue; // redirected home405 if (!isOwn(o.meta.finalUrl || url)) continue;406 if (!/text\/html|xhtml/i.test(o.meta.contentType ?? "") && /^\s*[{[]/.test(o.body.toString("utf8").slice(0, 50))) continue;407 const c = canonicalizeHtml(o.body.toString("utf8"), o.meta.finalUrl || url);408 if (c.text.length < 300) {409 rejected.push({ url, reason: `thin (${c.text.length} chars)` });410 continue;411 }412 const sh = shingles(c.text);413 if (homeCanon && (c.canonicalHash === homeCanon.canonicalHash || (homeShingles && jaccard(sh, homeShingles) > 0.8))) {414 rejected.push({ url, reason: "same as homepage" });415 continue;416 }417 const dup = pageCanon.find((p) => p.hash === c.canonicalHash || jaccard(p.sh, sh) > 0.85);418 if (dup) {419 rejected.push({ url, reason: `duplicate of ${dup.url}` });420 continue;421 }422 pageCanon.push({ url: o.meta.finalUrl || url, sh, hash: c.canonicalHash });423 const isSecurityTxt = /security\.txt$/.test(url);424 add({ url: o.meta.finalUrl || url, type: "HTML", connector: "http", kind: job.kind, name: isSecurityTxt ? "security.txt" : job.label, config: {}, suggestedTier: isSecurityTxt ? "D" : job.tier, evidence: `${pageLinks.get(job.kind)?.includes(url) ? "linked from homepage" : "well-known path"} · GET 200 · ${c.text.length} chars · distinct from homepage`, value: isSecurityTxt ? 0.3 : job.value, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs }, fingerprint: c.canonicalHash, textShingles: sh, title: c.title ?? undefined });425 kept++;426 }427 });428 }429430 // ---- 8. GitHub organization → repositories' releases ----------------------------------------------------------431 const repos = new Set<string>(hints.github_repos ?? []);432 for (const org of [...githubOrgs].slice(0, 1)) {433 if (repos.size >= 4 || !budgetLeft()) break;434 const token = process.env.GITHUB_TOKEN;435 if (token) {436 const o = await get(`https://api.github.com/search/repositories?q=org:${encodeURIComponent(org)}+fork:false&sort=stars&order=desc&per_page=5`, { accept: "application/vnd.github+json", headers: { authorization: `Bearer ${token}`, "x-github-api-version": "2022-11-28" } });437 if (o.body && o.meta.status === 200) {438 try {439 const j = JSON.parse(o.body.toString("utf8")) as { items?: { full_name: string; archived?: boolean; stargazers_count?: number }[] };440 for (const r of j.items ?? []) if (!r.archived && (r.stargazers_count ?? 0) >= 50) repos.add(r.full_name);441 } catch {442 // ignore443 }444 }445 } else {446 // Unauthenticated: the organization page lists pinned / popular repositories server-side.447 const o = await get(`https://github.com/${org}`, { timeoutMs: 15_000, maxBytes: 3 * 1024 * 1024 });448 if (o.body && o.meta.status === 200) {449 const html = o.body.toString("utf8");450 const counts = new Map<string, number>();451 for (const m of html.matchAll(new RegExp(`href="/${org.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/([A-Za-z0-9_.-]+)"`, "g"))) {452 const r = m[1]!;453 if (/^(followers|following|repositories|projects|packages|people|teams|sponsoring|discussions|orgs|\.github)$/i.test(r)) continue;454 counts.set(r, (counts.get(r) ?? 0) + 1);455 }456 const pinned = [...html.matchAll(/class="[^"]*pinned-item-list-item[^"]*"[\s\S]{0,1500}?href="\/([^/"]+)\/([^/"]+)"/g)].map((m) => `${m[1]}/${m[2]}`).filter((f) => f.toLowerCase().startsWith(org.toLowerCase() + "/"));457 for (const f of pinned) repos.add(f);458 for (const [r] of [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 6)) if (repos.size < 6) repos.add(`${org}/${r}`);459 }460 }461 }462 const repoList = [...repos].slice(0, 6);463 let releasesKept = 0;464 await parallel(repoList, 3, async (full) => {465 if (releasesKept >= 4 || !budgetLeft()) return;466 for (const kind of ["releases", "tags"] as const) {467 const url = `https://github.com/${full}/${kind}.atom`;468 const o = await get(url, { timeoutMs: 12_000, accept: "application/atom+xml, application/xml;q=0.9" });469 if (!o.body || o.meta.status !== 200) {470 if (o.meta.status === 404) return;471 continue;472 }473 try {474 const f = parseFeed(o.body.toString("utf8"), url);475 if (!f.items.length) continue;476 const dates = f.items.map((i) => (i.publishedAt ? new Date(i.publishedAt).getTime() : NaN)).filter((t) => !Number.isNaN(t));477 if (dates.length && Math.max(...dates) < Date.now() - 2 * 365 * 86400e3) return rejected.push({ url, reason: "repository inactive (> 2 years)" });478 add({ url, type: "GITHUB_RELEASE", connector: "github", kind: "releases", name: `${full.split("/")[1]} ${kind}`, config: { repo: full, kind }, suggestedTier: "B", evidence: `GitHub ${kind}.atom · ${f.items.length} items`, value: kind === "releases" ? 0.8 : 0.65, itemCount: f.items.length, itemsPerDay: null, firstPartyConfidence: 0.9, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs }, fingerprint: sha256(f.items.map((i) => i.key).sort().join("\n")) });479 releasesKept++;480 return;481 } catch {482 // not a feed483 }484 }485 });486487 // ---- 9. Hugging Face models ------------------------------------------------------------------------------------488 for (const author of [...hfAuthors].slice(0, 2)) {489 if (!budgetLeft()) break;490 const url = `https://huggingface.co/api/models?author=${encodeURIComponent(author)}&sort=lastModified&direction=-1&limit=50`;491 const o = await get(url, { accept: "application/json" });492 if (!o.body || o.meta.status !== 200) continue;493 try {494 const j = JSON.parse(o.body.toString("utf8")) as unknown[];495 if (Array.isArray(j) && j.length) add({ url, type: "REST_API", connector: "jsonlist", kind: "models", name: "hugging face models", config: { keyField: "id", titleField: "id", dateField: "lastModified", compareFields: ["pipeline_tag", "library_name"], maxItems: 50 }, suggestedTier: "B", evidence: `Hugging Face author ${author} · ${j.length} models`, value: 0.85, itemCount: j.length, itemsPerDay: null, firstPartyConfidence: 0.9, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });496 } catch {497 // ignore498 }499 }500501 // ---- 10. EDGAR filings -------------------------------------------------------------------------------------------502 if (hints.cik !== undefined && budgetLeft()) {503 const cik = String(hints.cik).replace(/\D/g, "").padStart(10, "0");504 const url = `https://data.sec.gov/submissions/CIK${cik}.json`;505 const ua = process.env.WS_EDGAR_USER_AGENT ?? "WebSensor (contact@websensor.io)";506 const o = await get(url, { accept: "application/json", userAgent: ua, headers: { "user-agent": ua }, timeoutMs: 30_000, maxBytes: 30 * 1024 * 1024 });507 if (o.body && o.meta.status === 200 && /"filings"/.test(o.body.toString("utf8").slice(0, 200_000))) {508 add({ url, type: "REST_API", connector: "edgar", kind: "filings", name: "edgar filings", config: { forms: ["8-K", "10-K", "10-Q", "6-K", "20-F", "40-F", "S-1", "SC 13D", "DEF 14A"] }, suggestedTier: "B", evidence: `SEC EDGAR submissions CIK ${cik}`, value: 0.9, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });509 } else rejected.push({ url, reason: `edgar ${o.meta.status}` });510 }511512 // ---- 11. OpenAPI documents ---------------------------------------------------------------------------------------513 if (opts.probeOpenApi !== false && (subHosts.some((s) => s.kind === "docs" || s.kind === "api") || pageLinks.has("docs")) && budgetLeft()) {514 const roots = [base, ...subHosts.filter((s) => s.kind === "docs" || s.kind === "api").map((s) => `https://${s.host}`)].slice(0, 3);515 let openapiFound = false;516 for (const root of roots) {517 if (openapiFound) break;518 for (const p of OPENAPI_PATHS.slice(0, 7)) {519 if (openapiFound || !budgetLeft()) break;520 const o = await get(root + p, { timeoutMs: 15_000, maxBytes: 8 * 1024 * 1024, accept: "application/json, application/yaml, */*;q=0.5" });521 if (!o.body || o.meta.status !== 200) continue;522 const text = o.body.toString("utf8").slice(0, 4000);523 if (/^\s*[{[]/.test(text) && /"(openapi|swagger)"\s*:/.test(text)) {524 add({ url: root + p, type: "JSON", connector: "openapi", kind: "api", name: "openapi", config: {}, suggestedTier: "C", evidence: `OpenAPI document (${new URL(root).host})`, value: 0.7, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: o.meta.contentLength, ms: o.meta.durationMs } });525 openapiFound = true;526 }527 }528 }529 }530531 // ---- 12. explicit hint URLs (validated like everything else) --------------------------------------------------------532 for (const h of hints.urls ?? []) {533 if (!budgetLeft()) break;534 const known = h.url.replace(/\/$/, "");535 if (found.has(known)) continue;536 const o = await get(h.url, { timeoutMs: 15_000, maxBytes: 6 * 1024 * 1024 });537 if (!o.body || o.meta.status !== 200) {538 rejected.push({ url: h.url, reason: `hint ${o.meta.status || o.error?.code}` });539 continue;540 }541 const text = o.body.toString("utf8");542 const kind = (h.kind ?? "other") as CandidateKind;543 const tier = h.tier ?? (kind === "status" ? "S" : kind === "pricing" || kind === "legal" || kind === "ir" ? "C" : "B");544 const cost = { bytes: o.meta.contentLength, ms: o.meta.durationMs };545 if (h.connector) {546 add({ url: h.url, type: (h.type ?? (h.connector === "rss" ? "RSS" : h.connector === "jsonlist" || h.connector === "edgar" ? "REST_API" : h.connector === "statuspage" || h.connector === "statusjson" ? "STATUSPAGE" : h.connector === "sitemap" ? "SITEMAP" : "HTML")) as SensorType, connector: h.connector, kind, name: h.name ?? kind, config: h.config ?? {}, suggestedTier: tier, evidence: "explicit hint · GET 200", value: 0.8, itemsPerDay: null, firstPartyConfidence: isOwn(h.url) ? 1 : 0.8, fetchCost: cost });547 continue;548 }549 // Sniff: feed → rss; JSON → http/json; HTML → http550 if (!/^\s*<!doctype html|<html/i.test(text.slice(0, 400)) && /<(rss|feed|rdf:RDF)\b/i.test(text.slice(0, 2000))) {551 try {552 const f = parseFeed(text, o.meta.finalUrl);553 if (f.items.length) {554 add({ url: o.meta.finalUrl || h.url, type: f.kind === "atom" ? "ATOM" : "RSS", connector: "rss", kind, name: h.name ?? feedName(kind, h.url), config: {}, suggestedTier: tier, evidence: `explicit hint · feed · ${f.items.length} items`, value: 0.85, itemCount: f.items.length, itemsPerDay: null, firstPartyConfidence: isOwn(h.url) ? 1 : 0.8, fetchCost: cost, fingerprint: sha256(f.items.map((i) => i.key).sort().join("\n")), itemKeys: f.items.map((i) => i.key).sort() });555 continue;556 }557 } catch {558 // fallthrough559 }560 }561 if (/^\s*[{[]/.test(text.slice(0, 50))) {562 add({ url: h.url, type: "JSON", connector: "http", kind, name: h.name ?? kind, config: h.config ?? {}, suggestedTier: tier, evidence: "explicit hint · JSON 200", value: 0.65, itemsPerDay: null, firstPartyConfidence: isOwn(h.url) ? 1 : 0.8, fetchCost: cost });563 continue;564 }565 const c = canonicalizeHtml(text, o.meta.finalUrl || h.url);566 if (c.text.length < 300) {567 rejected.push({ url: h.url, reason: `hint thin (${c.text.length} chars)` });568 continue;569 }570 add({ url: o.meta.finalUrl || h.url, type: "HTML", connector: "http", kind, name: h.name ?? PAGE_KINDS.find((p) => p.kind === kind)?.label ?? kind, config: h.config ?? {}, suggestedTier: tier, evidence: `explicit hint · GET 200 · ${c.text.length} chars`, value: 0.7, itemsPerDay: null, firstPartyConfidence: isOwn(h.url) ? 1 : 0.8, fetchCost: cost, fingerprint: c.canonicalHash, title: c.title ?? undefined });571 }572573 // ---- 13. web posture (no network here: validated in shadow by the dns/tls connectors) ------------------------------574 if (opts.probePosture || hints.posture) {575 const rd = registrableDomain(domain);576 add({ url: `dns://${rd}`, type: "DNS", connector: "dns", kind: "posture", name: "dns", config: {}, suggestedTier: "D", evidence: "posture", value: 0.3, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: 0, ms: 0 } });577 add({ url: `tls://${new URL(base).hostname}`, type: "TLS", connector: "tls", kind: "posture", name: "tls certificate", config: {}, suggestedTier: "D", evidence: "posture", value: 0.3, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: 0, ms: 0 } });578 if (robotsOk) add({ url: `https://${domain}/robots.txt`, type: "HTML", connector: "http", kind: "posture", name: "robots.txt", config: {}, suggestedTier: "D", evidence: robotsMentionsAi ? "robots.txt names AI crawlers" : "robots.txt", value: robotsMentionsAi ? 0.45 : 0.3, itemsPerDay: null, firstPartyConfidence: 1, fetchCost: { bytes: robots?.meta.contentLength ?? 0, ms: robots?.meta.durationMs ?? 0 } });579 }580581 const candidates = [...found.values()].sort((a, b) => b.value - a.value);582 // Strip the heavy de-dup helpers before returning (they are not serializable and no longer needed).583 for (const c of candidates) {584 delete c.textShingles;585 if (c.itemKeys && c.itemKeys.length > 60) c.itemKeys = c.itemKeys.slice(0, 60);586 }587 return { domain, origin: base, canonicalDomain, homeStatus, blocked, requests, candidates, rejected, notes, durationMs: Date.now() - started };588}589590// ------------------------------------------------------------------------------------------------591592/** Expected publication rate from item dates: the median gap between consecutive items (robust to one bogus date). */593export function itemsPerDayOf(dates: number[]): number | null {594 const d = [...dates].filter((t) => Number.isFinite(t)).sort((a, b) => b - a).slice(0, 30);595 if (d.length < 3) return null;596 const gaps = d.slice(1).map((t, i) => (d[i]! - t) / 86400e3).filter((g) => g >= 0).sort((a, b) => a - b);597 if (!gaps.length) return null;598 const median = gaps[Math.floor(gaps.length / 2)]!;599 if (median <= 0) return 50;600 return Math.round((1 / median) * 1000) / 1000;601}602603function feedKind(url: string, title?: string): CandidateKind {604 const s = `${url} ${title ?? ""}`.toLowerCase();605 if (/press|communiqu|pressemitteilung|prensa|news-release|newsrelease/.test(s)) return "press";606 if (/investor|\/ir\/|ir\.|sec-filing|filings/.test(s)) return "ir";607 if (/security|advisor|psirt|bulletin|vulnerab|cve/.test(s)) return "security";608 if (/changelog|release-notes|releasenotes|releases|whats-new|updates/.test(s)) return "changelog";609 if (/blog|engineering|research|insights|stories/.test(s)) return "blog";610 if (/news|actualit|nouvelles|aktuell|noticias|announcement/.test(s)) return "news";611 if (/jobs|careers/.test(s)) return "careers";612 return "news";613}614615function feedName(kind: CandidateKind, url: string): string {616 switch (kind) {617 case "press":618 return "press releases feed";619 case "ir":620 return "investor news feed";621 case "security":622 return "security feed";623 case "changelog":624 return "changelog feed";625 case "blog":626 return /engineering/i.test(url) ? "engineering blog feed" : /research/i.test(url) ? "research feed" : "blog feed";627 case "careers":628 return "jobs feed";629 default:630 return "news feed";631 }632}633634function safePath(u: string): string {635 try {636 return new URL(u).pathname.replace(/\/$/, "");637 } catch {638 return "";639 }640}641642function safeAbs(h: string, base: string): string | null {643 try {644 const u = new URL(h, base);645 if (!u.protocol.startsWith("http")) return null;646 u.hash = "";647 return u.toString();648 } catch {649 return null;650 }651}652653async function resolves(host: string): Promise<boolean> {654 try {655 const r = await dns.lookup(host, { all: true });656 return r.length > 0;657 } catch {658 return false;659 }660}661662async function parallel<T>(items: T[], limit: number, fn: (item: T) => Promise<unknown>): Promise<void> {663 let i = 0;664 const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {665 while (i < items.length) {666 const item = items[i++]!;667 try {668 await fn(item);669 } catch {670 // best effort671 }672 }673 });674 await Promise.all(workers);675}676