TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { sha256, simhash, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core";2import { httpFetchWithRetry } from "./fetcher";3import { NormalizeError, type WebSensorConnector } from "./types";4import { asArray, parseXml, textOf } from "./xml";56export type SitemapEntry = {7 key: string;8 url: string;9 lastmod: string | null;10 title?: string;11 publishedAt?: string | null;12 [k: string]: unknown;13};1415export interface ParsedSitemap {16 kind: "urlset" | "sitemapindex";17 entries: SitemapEntry[];18 children: string[];19}2021export function parseSitemap(text: string): ParsedSitemap {22 const doc = parseXml(text);23 if (doc.sitemapindex) {24 const children = asArray((doc.sitemapindex as Record<string, unknown>).sitemap as unknown[]).map((s) => textOf((s as Record<string, unknown>).loc));25 return { kind: "sitemapindex", entries: [], children: children.filter(Boolean) };26 }27 if (doc.urlset) {28 const urls = asArray((doc.urlset as Record<string, unknown>).url as unknown[]).map((u) => {29 const o = u as Record<string, unknown>;30 const url = textOf(o.loc);31 const news = o["news:news"] as Record<string, unknown> | undefined;32 const title = news ? textOf(news["news:title"]) : undefined;33 const pub = news ? textOf(news["news:publication_date"]) : undefined;34 return { key: url, url, lastmod: textOf(o.lastmod) || null, title: title || undefined, publishedAt: pub || null } satisfies SitemapEntry;35 });36 return { kind: "urlset", entries: urls.filter((e) => e.url), children: [] };37 }38 // Plain-text sitemaps (one URL per line)39 const lines = text.split(/\r?\n/).map((l) => l.trim()).filter((l) => /^https?:\/\//.test(l));40 if (lines.length) return { kind: "urlset", entries: lines.map((u) => ({ key: u, url: u, lastmod: null })), children: [] };41 throw new NormalizeError("not_a_sitemap", "Document is not a sitemap");42}4344/**45 * Sitemap connector. Handles sitemap indexes (follows up to `maxChildren` children, newest46 * first when lastmod is available) and compressed sitemaps. Emits list diffs:47 * new_url / removed_url / modified lastmod.48 */49export class SitemapConnector implements WebSensorConnector {50 mode = "list" as const;51 metadata(): ConnectorMetadata {52 return { key: "sitemap", name: "Sitemap", sensorTypes: ["SITEMAP"], description: "sitemap.xml / index / news sitemaps → URL created/removed/lastmod", version: "1.0.0" };53 }54 async fetch(endpoint: SensorEndpoint): Promise<Observation> {55 return httpFetchWithRetry(endpoint.id, endpoint.url, { etag: endpoint.etag, lastModified: endpoint.lastModified, accept: "application/xml, text/xml, application/gzip, */*;q=0.5", maxBytes: 30 * 1024 * 1024 });56 }57 async normalize(endpoint: SensorEndpoint, obs: Observation): Promise<NormalizedContent> {58 if (!obs.body) throw new NormalizeError("no_body", "Observation has no body");59 const cfg = endpoint.config as { maxChildren?: number; maxUrls?: number; include?: string; exclude?: string };60 const text = obs.body.toString("utf8");61 if (/^\s*<!doctype html|<html/i.test(text.slice(0, 500))) throw new NormalizeError("html_not_sitemap", "Endpoint returned HTML instead of a sitemap");62 let parsed = parseSitemap(text);63 let entries = parsed.entries;64 const fetchedChildren: string[] = [];65 if (parsed.kind === "sitemapindex") {66 const children = parsed.children.slice(0, cfg.maxChildren ?? 6);67 for (const child of children) {68 const o = await httpFetchWithRetry(endpoint.id, child, { accept: "application/xml, text/xml, application/gzip, */*;q=0.5", maxBytes: 30 * 1024 * 1024 }, 0);69 if (!o.body || o.meta.status >= 400) continue;70 try {71 const p = parseSitemap(o.body.toString("utf8"));72 entries.push(...p.entries);73 fetchedChildren.push(child);74 } catch {75 // skip unparseable child76 }77 }78 parsed = { ...parsed, entries };79 }80 if (cfg.include) {81 const re = new RegExp(cfg.include, "i");82 entries = entries.filter((e) => re.test(e.url));83 }84 if (cfg.exclude) {85 const re = new RegExp(cfg.exclude, "i");86 entries = entries.filter((e) => !re.test(e.url));87 }88 // Newest first when lastmod exists, bounded.89 entries.sort((a, b) => (b.lastmod ?? "").localeCompare(a.lastmod ?? ""));90 const max = cfg.maxUrls ?? 5000;91 const items = entries.slice(0, max);92 const canonical = items.map((e) => `${e.url}\t${e.lastmod ?? ""}`).join("\n");93 const newest = items.map((i) => i.lastmod).filter((x): x is string => Boolean(x)).sort().at(-1);94 return {95 mode: "list",96 items,97 compareFields: ["lastmod"],98 rawHash: sha256(text),99 canonicalHash: sha256(canonical),100 semanticHash: simhash(items.map((i) => i.url).join("\n")),101 publishedAt: newest ? new Date(newest) : null,102 extractionConfidence: 1,103 extra: { kind: parsed.kind, total: entries.length, children: fetchedChildren },104 };105 }106}107