TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { sha256, simhash, stripTrackingParams, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core";2import { httpFetchWithRetry } from "./fetcher";3import { NormalizeError, type WebSensorConnector } from "./types";4import { asArray, parseDate, parseXml, stripHtml, textOf } from "./xml";56export type FeedItem = {7 key: string;8 title: string;9 url: string;10 summary: string;11 publishedAt: string | null;12 updatedAt: string | null;13 author?: string;14 categories?: string[];15 [k: string]: unknown;16};1718export interface ParsedFeed {19 kind: "rss" | "atom" | "rdf" | "jsonfeed";20 title: string;21 link: string;22 items: FeedItem[];23}2425/** RSS 2.0 / Atom / RSS 1.0 (RDF) / JSON Feed parser tolerant to common malformations. */26export function parseFeed(text: string, baseUrl?: string): ParsedFeed {27 const trimmed = text.trimStart();28 if (trimmed.startsWith("{")) return parseJsonFeed(trimmed);29 const doc = parseXml(text);30 if (doc.rss) {31 const ch = ((doc.rss as Record<string, unknown>).channel ?? {}) as Record<string, unknown>;32 const items = asArray(ch.item as unknown[]).map((raw) => rssItem(raw as Record<string, unknown>, baseUrl));33 return { kind: "rss", title: stripHtml(textOf(ch.title)), link: textOf(ch.link), items: dedupe(items) };34 }35 if (doc.feed) {36 const f = doc.feed as Record<string, unknown>;37 const items = asArray(f.entry as unknown[]).map((raw) => atomEntry(raw as Record<string, unknown>, baseUrl));38 const link = asArray(f.link as unknown[])39 .map((l) => l as Record<string, unknown>)40 .find((l) => !l["@_rel"] || l["@_rel"] === "alternate");41 return { kind: "atom", title: stripHtml(textOf(f.title)), link: textOf(link?.["@_href"]), items: dedupe(items) };42 }43 if (doc["rdf:RDF"]) {44 const r = doc["rdf:RDF"] as Record<string, unknown>;45 const ch = (r.channel ?? {}) as Record<string, unknown>;46 const items = asArray(r.item as unknown[]).map((raw) => rssItem(raw as Record<string, unknown>, baseUrl));47 return { kind: "rdf", title: stripHtml(textOf(ch.title)), link: textOf(ch.link), items: dedupe(items) };48 }49 throw new NormalizeError("not_a_feed", "Document is neither RSS, Atom, RDF nor JSON Feed");50}5152function rssItem(raw: Record<string, unknown>, baseUrl?: string): FeedItem {53 const guid = textOf(raw.guid);54 let link = textOf(raw.link);55 if (!link && typeof raw.link === "object") link = textOf((raw.link as Record<string, unknown>)["@_href"]);56 if (!link && guid.startsWith("http")) link = guid;57 if (!link && raw["feedburner:origLink"]) link = textOf(raw["feedburner:origLink"]);58 const url = safeUrl(link, baseUrl);59 const title = stripHtml(textOf(raw.title));60 const summary = stripHtml(textOf(raw.description) || textOf(raw["content:encoded"]) || textOf(raw.summary)).slice(0, 1200);61 const pub = parseDate(raw.pubDate) ?? parseDate(raw["dc:date"]) ?? parseDate(raw.published);62 const key = guid || url || sha256(title + summary).slice(0, 24);63 const categories = asArray(raw.category as unknown[])64 .map((c) => stripHtml(textOf(c)))65 .filter(Boolean);66 return { key, title, url, summary, publishedAt: pub?.toISOString() ?? null, updatedAt: null, author: stripHtml(textOf(raw.author) || textOf(raw["dc:creator"])) || undefined, categories };67}6869function atomEntry(raw: Record<string, unknown>, baseUrl?: string): FeedItem {70 const id = textOf(raw.id);71 const links = asArray(raw.link as unknown[]).map((l) => l as Record<string, unknown>);72 const alt = links.find((l) => !l["@_rel"] || l["@_rel"] === "alternate") ?? links[0];73 const link = textOf(alt?.["@_href"]) || (id.startsWith("http") ? id : "");74 const url = safeUrl(link, baseUrl);75 const title = stripHtml(textOf(raw.title));76 const summary = stripHtml(textOf(raw.summary) || textOf(raw.content)).slice(0, 1200);77 const pub = parseDate(raw.published) ?? parseDate(raw.issued);78 const upd = parseDate(raw.updated);79 const author = raw.author && typeof raw.author === "object" ? stripHtml(textOf((raw.author as Record<string, unknown>).name)) : stripHtml(textOf(raw.author));80 const categories = asArray(raw.category as unknown[])81 .map((c) => textOf((c as Record<string, unknown>)["@_term"]) || textOf((c as Record<string, unknown>)["@_label"]))82 .filter(Boolean);83 return { key: id || url || sha256(title + summary).slice(0, 24), title, url, summary, publishedAt: (pub ?? upd)?.toISOString() ?? null, updatedAt: upd?.toISOString() ?? null, author: author || undefined, categories };84}8586function parseJsonFeed(text: string): ParsedFeed {87 let j: Record<string, unknown>;88 try {89 j = JSON.parse(text) as Record<string, unknown>;90 } catch {91 throw new NormalizeError("bad_json", "Invalid JSON Feed");92 }93 const items = asArray(j.items as Record<string, unknown>[]).map((it) => {94 const url = String(it.url ?? it.external_url ?? "");95 const title = stripHtml(String(it.title ?? ""));96 const summary = stripHtml(String(it.summary ?? it.content_text ?? it.content_html ?? "")).slice(0, 1200);97 return { key: String(it.id ?? url), title, url, summary, publishedAt: it.date_published ? new Date(String(it.date_published)).toISOString() : null, updatedAt: it.date_modified ? new Date(String(it.date_modified)).toISOString() : null } satisfies FeedItem;98 });99 return { kind: "jsonfeed", title: String(j.title ?? ""), link: String(j.home_page_url ?? ""), items: dedupe(items) };100}101102function safeUrl(link: string, baseUrl?: string): string {103 if (!link) return "";104 try {105 return stripTrackingParams(new URL(link, baseUrl).toString());106 } catch {107 return link;108 }109}110111function dedupe(items: FeedItem[]): FeedItem[] {112 const seen = new Set<string>();113 const out: FeedItem[] = [];114 for (const it of items) {115 if (seen.has(it.key)) continue;116 seen.add(it.key);117 out.push(it);118 }119 return out;120}121122/**123 * RSS/Atom connector. Sensor state keeps the set of seen item keys so items that scroll124 * out of the feed window are not reported as "removed".125 */126export class RssConnector implements WebSensorConnector {127 mode = "list" as const;128 metadata(): ConnectorMetadata {129 return { key: "rss", name: "RSS / Atom", sensorTypes: ["RSS", "ATOM"], description: "Feed parser with GUID deduplication and update detection", version: "1.0.0" };130 }131 async fetch(endpoint: SensorEndpoint): Promise<Observation> {132 return httpFetchWithRetry(endpoint.id, endpoint.url, { etag: endpoint.etag, lastModified: endpoint.lastModified, accept: "application/rss+xml, application/atom+xml, application/xml, text/xml, application/feed+json, application/json;q=0.8, */*;q=0.5" });133 }134 async normalize(endpoint: SensorEndpoint, obs: Observation): Promise<NormalizedContent> {135 if (!obs.body) throw new NormalizeError("no_body", "Observation has no body");136 const text = obs.body.toString("utf8");137 if (/^\s*<!doctype html|<html/i.test(text.slice(0, 500))) throw new NormalizeError("html_not_feed", "Endpoint returned HTML instead of a feed");138 const feed = parseFeed(text, obs.meta.finalUrl);139 const cfg = endpoint.config as { maxItems?: number };140 const items = feed.items.slice(0, cfg.maxItems ?? 100).map((it) => ({ ...it, key: it.key }));141 const newest = items.map((i) => i.publishedAt).filter((x): x is string => Boolean(x)).sort().at(-1);142 // Items comparison: key is the GUID; title/summary changes count as modifications.143 const canonical = items.map((i) => `${i.key}\t${i.title}\t${sha256(i.summary).slice(0, 12)}`).join("\n");144 const seen = new Set<string>(Array.isArray(endpoint.state?.seenKeys) ? (endpoint.state!.seenKeys as string[]) : []);145 for (const i of items) seen.add(i.key);146 const seenKeys = [...seen].slice(-2000);147 return {148 mode: "list",149 items,150 compareFields: ["title", "summary"],151 title: feed.title,152 rawHash: sha256(text),153 canonicalHash: sha256(canonical),154 semanticHash: simhash(items.map((i) => i.title).join("\n")),155 publishedAt: newest ? new Date(newest) : null,156 state: { seenKeys, feedKind: feed.kind },157 extractionConfidence: 1,158 extra: { feedKind: feed.kind, itemCount: items.length },159 };160 }161}162