spb/tendril Public

1// author: simon-pierre boucher <contact@spboucher.ai>2import TurndownService from "turndown";34interface MinimalEl {5  textContent: string | null;6  getAttribute(name: string): string | null;7  querySelectorAll(sel: string): ArrayLike<MinimalEl>;8  querySelector(sel: string): MinimalEl | null;9}1011function asEl(node: unknown): MinimalEl {12  return node as MinimalEl;13}1415function cellText(el: MinimalEl): string {16  return (el.textContent ?? "")17    .replace(/\r?\n/g, " ")18    .replace(/\s+/g, " ")19    .trim()20    .replace(/\|/g, "\\|");21}2223function buildTable(node: MinimalEl): string {24  const rows = Array.from(node.querySelectorAll("tr"));25  if (rows.length === 0) return "";26  const grid: string[][] = [];27  for (const row of rows) {28    const cells = Array.from(asEl(row).querySelectorAll("th, td"));29    grid.push(cells.map((c) => cellText(c)));30  }31  const firstRow = grid[0];32  if (firstRow === undefined) return "";33  const width = grid.reduce((m, r) => Math.max(m, r.length), 0);34  const pad = (r: string[]): string[] => {35    const copy = r.slice();36    while (copy.length < width) copy.push("");37    return copy;38  };39  const header = pad(firstRow);40  const sep = header.map(() => "---");41  const body = grid.slice(1).map((r) => pad(r));42  const line = (r: string[]): string => `| ${r.join(" | ")} |`;43  return ["", line(header), line(sep), ...body.map(line), ""].join("\n");44}4546function detectLanguage(node: MinimalEl): string {47  const code = node.querySelector("code");48  const cls = (code ?? node).getAttribute("class") ?? "";49  const m = /language-([a-z0-9+#-]+)/i.exec(cls) ?? /lang-([a-z0-9+#-]+)/i.exec(cls);50  return m?.[1] ?? "";51}5253export function createTurndown(): TurndownService {54  const td = new TurndownService({55    headingStyle: "atx",56    codeBlockStyle: "fenced",57    bulletListMarker: "-",58    emDelimiter: "_",59    hr: "---",60    linkStyle: "inlined",61  });6263  td.remove(["script", "style", "noscript"]);6465  td.addRule("removeAnchorLinks", {66    filter: (node) => {67      const el = asEl(node);68      if ((node as { nodeName?: string }).nodeName !== "A") return false;69      const href = el.getAttribute("href") ?? "";70      const cls = el.getAttribute("class") ?? "";71      const text = (el.textContent ?? "").trim();72      return href.startsWith("#") && (/anchor|headerlink|permalink/i.test(cls) || text === "" || text === "¶" || text === "#");73    },74    replacement: () => "",75  });7677  td.addRule("fencedCodeWithLang", {78    filter: (node) => (node as { nodeName?: string }).nodeName === "PRE",79    replacement: (_content, node) => {80      const el = asEl(node);81      const code = el.querySelector("code") ?? el;82      const text = code.textContent ?? "";83      const lang = detectLanguage(el);84      return `\n\n\`\`\`${lang}\n${text.replace(/\n$/, "")}\n\`\`\`\n\n`;85    },86  });8788  td.addRule("gfmTable", {89    filter: (node) => (node as { nodeName?: string }).nodeName === "TABLE",90    replacement: (_content, node) => {91      const el = asEl(node);92      const nested = Array.from(el.querySelectorAll("table")).some((x) => x !== node);93      if (nested) {94        return `\n\n${(node as { outerHTML?: string }).outerHTML ?? ""}\n\n`;95      }96      return `\n${buildTable(el)}\n`;97    },98  });99100  td.addRule("figureCaption", {101    filter: (node) => (node as { nodeName?: string }).nodeName === "FIGURE",102    replacement: (_content, node) => {103      const el = asEl(node);104      const img = el.querySelector("img");105      const cap = el.querySelector("figcaption");106      const src = img?.getAttribute("src") ?? "";107      const alt = img?.getAttribute("alt") ?? "";108      const caption = (cap?.textContent ?? "").trim();109      const image = src !== "" ? `![${alt}](${src})` : "";110      return caption !== "" ? `\n\n${image}\n\n_${caption}_\n\n` : `\n\n${image}\n\n`;111    },112  });113114  td.addRule("definitionList", {115    filter: (node) => (node as { nodeName?: string }).nodeName === "DL",116    replacement: (_content, node) => {117      const el = asEl(node);118      const parts: string[] = [];119      const children = Array.from(el.querySelectorAll("dt, dd"));120      for (const child of children) {121        const name = (child as { nodeName?: string }).nodeName;122        const text = (child.textContent ?? "").replace(/\s+/g, " ").trim();123        if (text === "") continue;124        parts.push(name === "DT" ? `\n**${text}**` : `\n: ${text}`);125      }126      return `\n\n${parts.join("").trim()}\n\n`;127    },128  });129130  td.addRule("strikethrough", {131    filter: ["del", "s"],132    replacement: (content) => `~~${content}~~`,133  });134135  return td;136}137138const sharedTurndown = createTurndown();139140export function postProcess(markdown: string): string {141  const lines = markdown142    .split("\n")143    .map((l) => l.replace(/[ \t]+$/, ""))144    .map((l) => l.replace(/^(\s*)[-*+][ \t]+/, "$1- "));145  const deduped: string[] = [];146  const linkRe = /^\s*\[[^\]]*\]\([^)]*\)\s*$/;147  for (const line of lines) {148    const prev = deduped[deduped.length - 1];149    if (linkRe.test(line) && prev !== undefined && prev === line) continue;150    deduped.push(line);151  }152  return deduped153    .join("\n")154    .replace(/\n{3,}/g, "\n\n")155    .trim();156}157158export function htmlToMarkdown(html: string): string {159  return postProcess(sharedTurndown.turndown(html));160}161