spb/tendril Public

1// author: simon-pierre boucher <contact@spboucher.ai>2import { parse } from "./dom.js";3import type { PageLink } from "./types.js";45export function extractLinksFromHtml(html: string, base: string): PageLink[] {6  return extractLinks(parse(html).document, base);7}89export function extractLinks(document: Document, base: string): PageLink[] {10  let baseHost: string;11  try {12    baseHost = new URL(base).hostname.toLowerCase();13  } catch {14    baseHost = "";15  }1617  const seen = new Set<string>();18  const links: PageLink[] = [];19  for (const a of Array.from(document.querySelectorAll("a[href]"))) {20    const raw = a.getAttribute("href");21    if (raw === null || raw === "" || /^(javascript:|mailto:|tel:|#)/i.test(raw)) continue;22    let abs: URL;23    try {24      abs = new URL(raw, base);25    } catch {26      continue;27    }28    if (abs.protocol !== "http:" && abs.protocol !== "https:") continue;29    const url = abs.toString();30    if (seen.has(url)) continue;31    seen.add(url);32    links.push({33      url,34      text: (a.textContent ?? "").replace(/\s+/g, " ").trim(),35      rel: a.getAttribute("rel"),36      isInternal: abs.hostname.toLowerCase() === baseHost,37    });38  }39  return links;40}41