import { sha256, simhash, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core"; import { httpFetchWithRetry } from "./fetcher"; import { NormalizeError, type WebSensorConnector } from "./types"; import { asArray, parseXml, textOf } from "./xml"; export type SitemapEntry = { key: string; url: string; lastmod: string | null; title?: string; publishedAt?: string | null; [k: string]: unknown; }; export interface ParsedSitemap { kind: "urlset" | "sitemapindex"; entries: SitemapEntry[]; children: string[]; } export function parseSitemap(text: string): ParsedSitemap { const doc = parseXml(text); if (doc.sitemapindex) { const children = asArray((doc.sitemapindex as Record).sitemap as unknown[]).map((s) => textOf((s as Record).loc)); return { kind: "sitemapindex", entries: [], children: children.filter(Boolean) }; } if (doc.urlset) { const urls = asArray((doc.urlset as Record).url as unknown[]).map((u) => { const o = u as Record; const url = textOf(o.loc); const news = o["news:news"] as Record | undefined; const title = news ? textOf(news["news:title"]) : undefined; const pub = news ? textOf(news["news:publication_date"]) : undefined; return { key: url, url, lastmod: textOf(o.lastmod) || null, title: title || undefined, publishedAt: pub || null } satisfies SitemapEntry; }); return { kind: "urlset", entries: urls.filter((e) => e.url), children: [] }; } // Plain-text sitemaps (one URL per line) const lines = text.split(/\r?\n/).map((l) => l.trim()).filter((l) => /^https?:\/\//.test(l)); if (lines.length) return { kind: "urlset", entries: lines.map((u) => ({ key: u, url: u, lastmod: null })), children: [] }; throw new NormalizeError("not_a_sitemap", "Document is not a sitemap"); } /** * Sitemap connector. Handles sitemap indexes (follows up to `maxChildren` children, newest * first when lastmod is available) and compressed sitemaps. Emits list diffs: * new_url / removed_url / modified lastmod. */ export class SitemapConnector implements WebSensorConnector { mode = "list" as const; metadata(): ConnectorMetadata { return { key: "sitemap", name: "Sitemap", sensorTypes: ["SITEMAP"], description: "sitemap.xml / index / news sitemaps → URL created/removed/lastmod", version: "1.0.0" }; } async fetch(endpoint: SensorEndpoint): Promise { return httpFetchWithRetry(endpoint.id, endpoint.url, { etag: endpoint.etag, lastModified: endpoint.lastModified, accept: "application/xml, text/xml, application/gzip, */*;q=0.5", maxBytes: 30 * 1024 * 1024 }); } async normalize(endpoint: SensorEndpoint, obs: Observation): Promise { if (!obs.body) throw new NormalizeError("no_body", "Observation has no body"); const cfg = endpoint.config as { maxChildren?: number; maxUrls?: number; include?: string; exclude?: string }; const text = obs.body.toString("utf8"); if (/^\s*= 400) continue; try { const p = parseSitemap(o.body.toString("utf8")); entries.push(...p.entries); fetchedChildren.push(child); } catch { // skip unparseable child } } parsed = { ...parsed, entries }; } if (cfg.include) { const re = new RegExp(cfg.include, "i"); entries = entries.filter((e) => re.test(e.url)); } if (cfg.exclude) { const re = new RegExp(cfg.exclude, "i"); entries = entries.filter((e) => !re.test(e.url)); } // Newest first when lastmod exists, bounded. entries.sort((a, b) => (b.lastmod ?? "").localeCompare(a.lastmod ?? "")); const max = cfg.maxUrls ?? 5000; const items = entries.slice(0, max); const canonical = items.map((e) => `${e.url}\t${e.lastmod ?? ""}`).join("\n"); const newest = items.map((i) => i.lastmod).filter((x): x is string => Boolean(x)).sort().at(-1); return { mode: "list", items, compareFields: ["lastmod"], rawHash: sha256(text), canonicalHash: sha256(canonical), semanticHash: simhash(items.map((i) => i.url).join("\n")), publishedAt: newest ? new Date(newest) : null, extractionConfidence: 1, extra: { kind: parsed.kind, total: entries.length, children: fetchedChildren }, }; } }