// author: simon-pierre boucher import { parseHTML } from "linkedom"; export interface Dom { readonly document: Document; readonly window: Window; } export function parse(html: string): Dom { const { document, window } = parseHTML(html) as unknown as { document: Document; window: Window }; return { document, window }; } /** * Parse an HTML fragment (e.g. Readability output or an element's innerHTML), * guaranteeing the content lands in `document.body`. linkedom leaves bare * fragments outside ``, which silently empties the pipeline otherwise. */ export function parseFragment(fragment: string): Dom { return parse(`${fragment}`); } const DROP_TAGS = ["script", "style", "svg", "noscript", "template", "iframe", "object", "embed"]; const AD_CLASS_RE = /(^|[-_ ])(ad|ads|advert|sponsor|promo|share|social|related|comment)([-_ ]|$)/i; const CHROME_SELECTORS = [ "header", "footer", "nav", "[role=navigation]", "[role=banner]", "[role=complementary]", "[aria-hidden=true]", "[hidden]", ]; export function sanitize(document: Document): void { for (const tag of DROP_TAGS) { for (const el of Array.from(document.querySelectorAll(tag))) el.remove(); } for (const img of Array.from(document.querySelectorAll("img"))) { const w = img.getAttribute("width"); const h = img.getAttribute("height"); if ((w === "1" && h === "1") || img.getAttribute("aria-hidden") === "true") img.remove(); } } export function stripChrome(document: Document): void { for (const sel of CHROME_SELECTORS) { for (const el of Array.from(document.querySelectorAll(sel))) el.remove(); } for (const el of Array.from(document.querySelectorAll("[class]"))) { const cls = el.getAttribute("class") ?? ""; if (AD_CLASS_RE.test(cls)) el.remove(); } } export function dropSelectors(document: Document, selectors: readonly string[]): void { for (const sel of selectors) { let matches: Element[]; try { matches = Array.from(document.querySelectorAll(sel)); } catch { continue; } for (const el of matches) el.remove(); } } export function keepOnly(document: Document, selectors: readonly string[]): void { const kept: Element[] = []; for (const sel of selectors) { try { kept.push(...Array.from(document.querySelectorAll(sel))); } catch { continue; } } if (kept.length === 0) return; const container = document.createElement("div"); for (const el of kept) container.appendChild(el.cloneNode(true) as Node); const body = document.body; body.textContent = ""; body.appendChild(container); } export function absolutizeUrls(document: Document, base: string): void { const attrs: Array<[string, string]> = [ ["a", "href"], ["img", "src"], ["source", "src"], ["link", "href"], ]; for (const [tag, attr] of attrs) { for (const el of Array.from(document.querySelectorAll(tag))) { const raw = el.getAttribute(attr); if (raw === null || raw === "") continue; if (/^(data:|javascript:|mailto:|tel:|#)/i.test(raw)) continue; try { el.setAttribute(attr, new URL(raw, base).toString()); } catch { continue; } } } for (const img of Array.from(document.querySelectorAll("img"))) { resolveLazyImage(img, base); } } function resolveLazyImage(img: Element, base: string): void { if ((img.getAttribute("src") ?? "") !== "") return; const lazy = img.getAttribute("data-src") ?? bestFromSrcset(img.getAttribute("srcset")); if (lazy === null) return; try { img.setAttribute("src", new URL(lazy, base).toString()); } catch { /* ignore malformed lazy URL */ } } function bestFromSrcset(srcset: string | null): string | null { if (srcset === null || srcset.trim() === "") return null; const candidates = srcset.split(",").map((part) => { const [url, size] = part.trim().split(/\s+/); const width = size?.endsWith("w") ? Number.parseInt(size, 10) : 0; return { url: url ?? "", width }; }); candidates.sort((a, b) => b.width - a.width); return candidates[0]?.url ?? null; } export function textContentLength(document: Document): number { return (document.body.textContent ?? "").replace(/\s+/g, " ").trim().length; }