// author: simon-pierre boucher import TurndownService from "turndown"; interface MinimalEl { textContent: string | null; getAttribute(name: string): string | null; querySelectorAll(sel: string): ArrayLike; querySelector(sel: string): MinimalEl | null; } function asEl(node: unknown): MinimalEl { return node as MinimalEl; } function cellText(el: MinimalEl): string { return (el.textContent ?? "") .replace(/\r?\n/g, " ") .replace(/\s+/g, " ") .trim() .replace(/\|/g, "\\|"); } function buildTable(node: MinimalEl): string { const rows = Array.from(node.querySelectorAll("tr")); if (rows.length === 0) return ""; const grid: string[][] = []; for (const row of rows) { const cells = Array.from(asEl(row).querySelectorAll("th, td")); grid.push(cells.map((c) => cellText(c))); } const firstRow = grid[0]; if (firstRow === undefined) return ""; const width = grid.reduce((m, r) => Math.max(m, r.length), 0); const pad = (r: string[]): string[] => { const copy = r.slice(); while (copy.length < width) copy.push(""); return copy; }; const header = pad(firstRow); const sep = header.map(() => "---"); const body = grid.slice(1).map((r) => pad(r)); const line = (r: string[]): string => `| ${r.join(" | ")} |`; return ["", line(header), line(sep), ...body.map(line), ""].join("\n"); } function detectLanguage(node: MinimalEl): string { const code = node.querySelector("code"); const cls = (code ?? node).getAttribute("class") ?? ""; const m = /language-([a-z0-9+#-]+)/i.exec(cls) ?? /lang-([a-z0-9+#-]+)/i.exec(cls); return m?.[1] ?? ""; } export function createTurndown(): TurndownService { const td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-", emDelimiter: "_", hr: "---", linkStyle: "inlined", }); td.remove(["script", "style", "noscript"]); td.addRule("removeAnchorLinks", { filter: (node) => { const el = asEl(node); if ((node as { nodeName?: string }).nodeName !== "A") return false; const href = el.getAttribute("href") ?? ""; const cls = el.getAttribute("class") ?? ""; const text = (el.textContent ?? "").trim(); return href.startsWith("#") && (/anchor|headerlink|permalink/i.test(cls) || text === "" || text === "ΒΆ" || text === "#"); }, replacement: () => "", }); td.addRule("fencedCodeWithLang", { filter: (node) => (node as { nodeName?: string }).nodeName === "PRE", replacement: (_content, node) => { const el = asEl(node); const code = el.querySelector("code") ?? el; const text = code.textContent ?? ""; const lang = detectLanguage(el); return `\n\n\`\`\`${lang}\n${text.replace(/\n$/, "")}\n\`\`\`\n\n`; }, }); td.addRule("gfmTable", { filter: (node) => (node as { nodeName?: string }).nodeName === "TABLE", replacement: (_content, node) => { const el = asEl(node); const nested = Array.from(el.querySelectorAll("table")).some((x) => x !== node); if (nested) { return `\n\n${(node as { outerHTML?: string }).outerHTML ?? ""}\n\n`; } return `\n${buildTable(el)}\n`; }, }); td.addRule("figureCaption", { filter: (node) => (node as { nodeName?: string }).nodeName === "FIGURE", replacement: (_content, node) => { const el = asEl(node); const img = el.querySelector("img"); const cap = el.querySelector("figcaption"); const src = img?.getAttribute("src") ?? ""; const alt = img?.getAttribute("alt") ?? ""; const caption = (cap?.textContent ?? "").trim(); const image = src !== "" ? `![${alt}](${src})` : ""; return caption !== "" ? `\n\n${image}\n\n_${caption}_\n\n` : `\n\n${image}\n\n`; }, }); td.addRule("definitionList", { filter: (node) => (node as { nodeName?: string }).nodeName === "DL", replacement: (_content, node) => { const el = asEl(node); const parts: string[] = []; const children = Array.from(el.querySelectorAll("dt, dd")); for (const child of children) { const name = (child as { nodeName?: string }).nodeName; const text = (child.textContent ?? "").replace(/\s+/g, " ").trim(); if (text === "") continue; parts.push(name === "DT" ? `\n**${text}**` : `\n: ${text}`); } return `\n\n${parts.join("").trim()}\n\n`; }, }); td.addRule("strikethrough", { filter: ["del", "s"], replacement: (content) => `~~${content}~~`, }); return td; } const sharedTurndown = createTurndown(); export function postProcess(markdown: string): string { const lines = markdown .split("\n") .map((l) => l.replace(/[ \t]+$/, "")) .map((l) => l.replace(/^(\s*)[-*+][ \t]+/, "$1- ")); const deduped: string[] = []; const linkRe = /^\s*\[[^\]]*\]\([^)]*\)\s*$/; for (const line of lines) { const prev = deduped[deduped.length - 1]; if (linkRe.test(line) && prev !== undefined && prev === line) continue; deduped.push(line); } return deduped .join("\n") .replace(/\n{3,}/g, "\n\n") .trim(); } export function htmlToMarkdown(html: string): string { return postProcess(sharedTurndown.turndown(html)); }