spb/tendril Public
Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.
JavaScript 82.6%
TypeScript 11.8%
HTML 5.3%
1// author: simon-pierre boucher <contact@spboucher.ai>2import { parseHTML } from "linkedom";34export interface Dom {5 readonly document: Document;6 readonly window: Window;7}89export function parse(html: string): Dom {10 const { document, window } = parseHTML(html) as unknown as { document: Document; window: Window };11 return { document, window };12}1314/**15 * Parse an HTML fragment (e.g. Readability output or an element's innerHTML),16 * guaranteeing the content lands in `document.body`. linkedom leaves bare17 * fragments outside `<body>`, which silently empties the pipeline otherwise.18 */19export function parseFragment(fragment: string): Dom {20 return parse(`<!doctype html><html><body>${fragment}</body></html>`);21}2223const DROP_TAGS = ["script", "style", "svg", "noscript", "template", "iframe", "object", "embed"];2425const AD_CLASS_RE = /(^|[-_ ])(ad|ads|advert|sponsor|promo|share|social|related|comment)([-_ ]|$)/i;26const CHROME_SELECTORS = [27 "header",28 "footer",29 "nav",30 "[role=navigation]",31 "[role=banner]",32 "[role=complementary]",33 "[aria-hidden=true]",34 "[hidden]",35];3637export function sanitize(document: Document): void {38 for (const tag of DROP_TAGS) {39 for (const el of Array.from(document.querySelectorAll(tag))) el.remove();40 }41 for (const img of Array.from(document.querySelectorAll("img"))) {42 const w = img.getAttribute("width");43 const h = img.getAttribute("height");44 if ((w === "1" && h === "1") || img.getAttribute("aria-hidden") === "true") img.remove();45 }46}4748export function stripChrome(document: Document): void {49 for (const sel of CHROME_SELECTORS) {50 for (const el of Array.from(document.querySelectorAll(sel))) el.remove();51 }52 for (const el of Array.from(document.querySelectorAll("[class]"))) {53 const cls = el.getAttribute("class") ?? "";54 if (AD_CLASS_RE.test(cls)) el.remove();55 }56}5758export function dropSelectors(document: Document, selectors: readonly string[]): void {59 for (const sel of selectors) {60 let matches: Element[];61 try {62 matches = Array.from(document.querySelectorAll(sel));63 } catch {64 continue;65 }66 for (const el of matches) el.remove();67 }68}6970export function keepOnly(document: Document, selectors: readonly string[]): void {71 const kept: Element[] = [];72 for (const sel of selectors) {73 try {74 kept.push(...Array.from(document.querySelectorAll(sel)));75 } catch {76 continue;77 }78 }79 if (kept.length === 0) return;80 const container = document.createElement("div");81 for (const el of kept) container.appendChild(el.cloneNode(true) as Node);82 const body = document.body;83 body.textContent = "";84 body.appendChild(container);85}8687export function absolutizeUrls(document: Document, base: string): void {88 const attrs: Array<[string, string]> = [89 ["a", "href"],90 ["img", "src"],91 ["source", "src"],92 ["link", "href"],93 ];94 for (const [tag, attr] of attrs) {95 for (const el of Array.from(document.querySelectorAll(tag))) {96 const raw = el.getAttribute(attr);97 if (raw === null || raw === "") continue;98 if (/^(data:|javascript:|mailto:|tel:|#)/i.test(raw)) continue;99 try {100 el.setAttribute(attr, new URL(raw, base).toString());101 } catch {102 continue;103 }104 }105 }106 for (const img of Array.from(document.querySelectorAll("img"))) {107 resolveLazyImage(img, base);108 }109}110111function resolveLazyImage(img: Element, base: string): void {112 if ((img.getAttribute("src") ?? "") !== "") return;113 const lazy = img.getAttribute("data-src") ?? bestFromSrcset(img.getAttribute("srcset"));114 if (lazy === null) return;115 try {116 img.setAttribute("src", new URL(lazy, base).toString());117 } catch {118 /* ignore malformed lazy URL */119 }120}121122function bestFromSrcset(srcset: string | null): string | null {123 if (srcset === null || srcset.trim() === "") return null;124 const candidates = srcset.split(",").map((part) => {125 const [url, size] = part.trim().split(/\s+/);126 const width = size?.endsWith("w") ? Number.parseInt(size, 10) : 0;127 return { url: url ?? "", width };128 });129 candidates.sort((a, b) => b.width - a.width);130 return candidates[0]?.url ?? null;131}132133export function textContentLength(document: Document): number {134 return (document.body.textContent ?? "").replace(/\s+/g, " ").trim().length;135}136