TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import * as cheerio from "cheerio";2import { sha256, simhash } from "./hash";34/**5 * Canonical content extraction. Before diffing we strip rendering noise (scripts, styles,6 * nav/footer chrome, tracking params, timestamps generated at render time, random ids,7 * CSRF tokens…) so `canonical_hash` only moves when the content moves. `semantic_hash`8 * is a simhash of the canonical text and is robust to small reorderings.9 */1011export interface CanonicalResult {12 /** Canonical text (one block per line). */13 text: string;14 title: string | null;15 /** Headings (h1–h3) in order — used for section-level diffs. */16 headings: string[];17 /** Absolute links found in the main content. */18 links: { href: string; text: string }[];19 rawHash: string;20 canonicalHash: string;21 semanticHash: string;22 /** Structural signature: element counts by tag (used for DOM-level diff summaries). */23 structure: Record<string, number>;24 meta: Record<string, string>;25}2627const NOISE_SELECTORS = [28 "script",29 "style",30 "noscript",31 "template",32 "svg",33 "iframe",34 "canvas",35 "video",36 "audio",37 "link",38 "meta",39 "[aria-hidden='true']",40 "[hidden]",41 ".cookie-banner",42 ".cookie-consent",43 "#cookie-banner",44 "#onetrust-consent-sdk",45 ".advertisement",46 ".ad-slot",47 "[class*='cookie']",48 "[id*='cookie']",49 "[class*='banner-consent']",50 "[class*='newsletter']",51 "[class*='social-share']",52 "[class*='skip-link']",53];5455const CHROME_SELECTORS = ["header", "nav", "footer", "aside", "[role='navigation']", "[role='banner']", "[role='contentinfo']", ".sidebar", ".breadcrumb", ".breadcrumbs"];5657const TRACKING_PARAMS = /^(utm_|fbclid|gclid|dclid|msclkid|mc_cid|mc_eid|ref|ref_src|igshid|_hs|hsa_|vero_|yclid|wickedid|oly_|s_kwcid|ncid|cmpid|_ga|_gl|spm)/i;5859export function stripTrackingParams(href: string, base?: string): string {60 try {61 const u = new URL(href, base);62 const keys = [...u.searchParams.keys()];63 for (const k of keys) if (TRACKING_PARAMS.test(k)) u.searchParams.delete(k);64 u.hash = "";65 return u.toString();66 } catch {67 return href;68 }69}7071/** Patterns that change on every render and carry no information. */72const VOLATILE_PATTERNS: RegExp[] = [73 /\b\d{1,2}:\d{2}(:\d{2})?\s?(am|pm|utc|gmt|est|pst|cet)?\b/gi, // clock times74 /\b(generated|rendered|last updated|page last modified)\s*(on|at)?[:\s]+[^\n]{4,40}/gi,75 /\b[0-9a-f]{32,64}\b/gi, // hashes / tokens76 /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, // uuids77 /\b(csrf|nonce|token|session)[=:]\s*[\w-]+/gi,78 /\b\d{1,3}(,\d{3})*\s+(views|visitors|online|reading now|comments?)\b/gi,79 /\b(\d+|a few|several)\s+(seconds?|minutes?|hours?)\s+ago\b/gi,80];8182export function normalizeText(text: string): string {83 return text84 .replace(/\r\n?/g, "\n")85 .replace(/ /g, " ")86 .replace(/[ \t\f\v]+/g, " ")87 .split("\n")88 .map((l) => l.trim())89 .filter((l) => l.length > 0)90 .join("\n");91}9293export function scrubVolatile(text: string): string {94 let out = text;95 for (const re of VOLATILE_PATTERNS) out = out.replace(re, " ");96 return normalizeText(out);97}9899const BLOCK_TAGS = new Set(["p", "div", "section", "article", "li", "ul", "ol", "h1", "h2", "h3", "h4", "h5", "h6", "tr", "td", "th", "table", "blockquote", "pre", "dd", "dt", "dl", "figcaption", "main", "br", "hr", "details", "summary"]);100101export function canonicalizeHtml(html: string, baseUrl?: string, opts: { keepChrome?: boolean } = {}): CanonicalResult {102 const rawHash = sha256(html);103 const $ = cheerio.load(html, { xml: false });104 const meta: Record<string, string> = {};105 $("meta").each((_, el) => {106 const name = $(el).attr("name") ?? $(el).attr("property");107 const content = $(el).attr("content");108 if (name && content && /^(description|og:title|og:description|og:type|article:published_time|article:modified_time|last-modified|generator)$/i.test(name)) meta[name.toLowerCase()] = content.trim();109 });110 const title = normalizeText($("title").first().text() || $("h1").first().text() || "") || null;111112 $(NOISE_SELECTORS.join(",")).remove();113 if (!opts.keepChrome) $(CHROME_SELECTORS.join(",")).remove();114 // Comments115 $("*")116 .contents()117 .each((_, node) => {118 if (node.type === "comment") $(node).remove();119 });120121 const root: cheerio.Cheerio<any> = $("main").length ? $("main").first() : $("article").length && $("article").text().trim().length > 200 ? $("article").first() : $("body").length ? $("body") : $.root();122123 const headings: string[] = [];124 root.find("h1,h2,h3").each((_, el) => {125 const t = normalizeText($(el).text());126 if (t) headings.push(t);127 });128129 const links: { href: string; text: string }[] = [];130 const seen = new Set<string>();131 root.find("a[href]").each((_, el) => {132 const hrefRaw = $(el).attr("href") ?? "";133 if (!hrefRaw || hrefRaw.startsWith("#") || /^(javascript|mailto|tel):/i.test(hrefRaw)) return;134 const href = stripTrackingParams(hrefRaw, baseUrl);135 if (!/^https?:/i.test(href) || seen.has(href)) return;136 seen.add(href);137 links.push({ href, text: normalizeText($(el).text()).slice(0, 200) });138 });139140 const structure: Record<string, number> = {};141 root.find("*").each((_, el) => {142 if (el.type !== "tag") return;143 structure[el.name] = (structure[el.name] ?? 0) + 1;144 });145146 // Text extraction with block boundaries147 const parts: string[] = [];148 const walk = (node: cheerio.Cheerio<any>): void => {149 node.contents().each((_, child) => {150 if (child.type === "text") {151 const t = (child as { data?: string }).data ?? "";152 if (t.trim()) parts.push(t);153 } else if (child.type === "tag") {154 const tag = (child as { name: string }).name;155 const isBlock = BLOCK_TAGS.has(tag);156 if (isBlock) parts.push("\n");157 walk($(child));158 if (isBlock) parts.push("\n");159 }160 });161 };162 walk(root);163 const text = scrubVolatile(normalizeText(parts.join("")));164165 return {166 text,167 title,168 headings,169 links,170 rawHash,171 canonicalHash: sha256(text),172 semanticHash: simhash(text),173 structure,174 meta,175 };176}177178/** Deterministic JSON serialization (sorted keys) for structured feeds. */179export function canonicalJson(value: unknown): string {180 return JSON.stringify(sortKeys(value));181}182183function sortKeys(v: unknown): unknown {184 if (Array.isArray(v)) return v.map(sortKeys);185 if (v && typeof v === "object") {186 const out: Record<string, unknown> = {};187 for (const k of Object.keys(v as Record<string, unknown>).sort()) out[k] = sortKeys((v as Record<string, unknown>)[k]);188 return out;189 }190 return v;191}192193export function canonicalizeText(text: string): { text: string; rawHash: string; canonicalHash: string; semanticHash: string } {194 const t = scrubVolatile(normalizeText(text));195 return { text: t, rawHash: sha256(text), canonicalHash: sha256(t), semanticHash: simhash(t) };196}197