spb/tendril Public

1// author: simon-pierre boucher <contact@spboucher.ai>2import { Readability } from "@mozilla/readability";3import { parse, stripChrome } from "./dom.js";45const MIN_READABILITY_CHARS = 200;67function textLen(el: Element): number {8  return (el.textContent ?? "").replace(/\s+/g, " ").trim().length;9}1011function linkTextLen(el: Element): number {12  let sum = 0;13  for (const a of Array.from(el.querySelectorAll("a"))) sum += textLen(a);14  return sum;15}1617/**18 * Density-based main-content fallback (§8.2). For each candidate block, score it19 * by `textLength^2 / (1 + linkTextLength)` — favouring large, low-link subtrees —20 * and return the winner's HTML. This is the safety net for listing/product pages21 * where Readability silently returns almost nothing.22 */23export function densityExtract(document: Document): string {24  const candidates = Array.from(document.querySelectorAll("article, main, section, div, td"));25  let best: Element | null = document.body;26  let bestScore = -1;27  for (const el of candidates) {28    const t = textLen(el);29    if (t < 100) continue;30    const score = (t * t) / (1 + linkTextLen(el));31    if (score > bestScore) {32      bestScore = score;33      best = el;34    }35  }36  return (best ?? document.body).innerHTML;37}3839export interface MainContent {40  readonly html: string;41  readonly usedReadability: boolean;42}4344export function extractMainContent(html: string, onlyMainContent: boolean): MainContent {45  if (!onlyMainContent) {46    const { document } = parse(html);47    return { html: document.body.innerHTML, usedReadability: false };48  }4950  try {51    const { document } = parse(html);52    const article = new Readability(document as unknown as Document, { charThreshold: MIN_READABILITY_CHARS }).parse();53    if (article && (article.textContent ?? "").trim().length >= MIN_READABILITY_CHARS && article.content) {54      return { html: article.content, usedReadability: true };55    }56  } catch {57    /* fall through to density */58  }5960  const { document } = parse(html);61  stripChrome(document);62  return { html: densityExtract(document), usedReadability: false };63}64