| 263 | return s.replace(/([\\`*_{}[\]<>])/g, "\\$1"); |
263 | return s.replace(/([\\`*_{}[\]<>])/g, "\\$1"); |
| 264 | } |
264 | } |
| 265 | |
265 | |
| |
266 | +type WriterOpts = Required<Pick<MarkdownOptions, "images" | "links" | "stripNoise">> & { baseUrl?: string; /** Text length of the rendered scope, used to keep "noise" landmarks that actually hold the content (e.g. a <nav> of language links on a portal). */ scopeLen?: number }; |
| |
267 | + |
| 266 | class MdWriter { |
268 | class MdWriter { |
| 267 | out: string[] = []; |
269 | out: string[] = []; |
| 268 | − private listStack: Array<{ ordered: boolean; index: number }> = []; |
270 | + constructor(private readonly opts: WriterOpts) {} |
| 269 | − constructor(private readonly opts: Required<Pick<MarkdownOptions, "images" | "links" | "stripNoise">> & { baseUrl?: string }) {} |
271 | + |
| |
272 | + /** Noise check that spares landmark elements carrying a large share of the scope's text. */ |
| |
273 | + private noise(n: HNode): boolean { |
| |
274 | + if (!this.opts.stripNoise || !isNoise(n)) return false; |
| |
275 | + if (n.type === "element" && (n.tag === "nav" || n.tag === "aside" || n.tag === "header" || n.tag === "footer" || n.tag === "form") && this.opts.scopeLen) { |
| |
276 | + const t = textOf(n).length; |
| |
277 | + if (t >= 120 && t >= this.opts.scopeLen * 0.35) return false; |
| |
278 | + } |
| |
279 | + return true; |
| |
280 | + } |
| 270 | |
281 | |
| 271 | block(s: string) { |
282 | block(s: string) { |
| 272 | const t = s.replace(/\n{3,}/g, "\n\n").trim(); |
283 | const t = s.replace(/\n{3,}/g, "\n\n").trim(); |
| 287 | s += decodeEntities(c.text).replace(/\s+/g, " "); |
298 | s += decodeEntities(c.text).replace(/\s+/g, " "); |
| 288 | continue; |
299 | continue; |
| 289 | } |
300 | } |
| 290 | − if (this.opts.stripNoise && isNoise(c)) continue; |
301 | + if (this.noise(c)) continue; |
| 291 | switch (c.tag) { |
302 | switch (c.tag) { |
| 292 | case "br": |
303 | case "br": |
| 293 | s += " \n"; |
304 | s += " \n"; |
| 375 | inlineBuf += decodeEntities(c.text).replace(/\s+/g, " "); |
386 | inlineBuf += decodeEntities(c.text).replace(/\s+/g, " "); |
| 376 | continue; |
387 | continue; |
| 377 | } |
388 | } |
| 378 | − if (this.opts.stripNoise && isNoise(c)) continue; |
389 | + if (this.noise(c)) continue; |
| 379 | if (!BLOCK.has(c.tag) && !["table", "ul", "ol", "dl"].includes(c.tag)) { |
390 | if (!BLOCK.has(c.tag) && !["table", "ul", "ol", "dl"].includes(c.tag)) { |
| 380 | inlineBuf += this.inline({ ...c, children: [c] } as HNode); |
391 | inlineBuf += this.inline({ ...c, children: [c] } as HNode); |
| 381 | continue; |
392 | continue; |
| 535 | export function htmlToMarkdown(html: string, opts: MarkdownOptions = {}): string { |
546 | export function htmlToMarkdown(html: string, opts: MarkdownOptions = {}): string { |
| 536 | const root = parseHtml(html); |
547 | const root = parseHtml(html); |
| 537 | const scope = opts.mainContent === false ? (findAll(root, (n) => n.tag === "body")[0] ?? root) : mainContent(root); |
548 | const scope = opts.mainContent === false ? (findAll(root, (n) => n.tag === "body")[0] ?? root) : mainContent(root); |
| 538 | − const writer = new MdWriter({ images: opts.images ?? true, links: opts.links ?? true, stripNoise: opts.stripNoise ?? true, baseUrl: opts.baseUrl }); |
549 | + const writer = new MdWriter({ images: opts.images ?? true, links: opts.links ?? true, stripNoise: opts.stripNoise ?? true, baseUrl: opts.baseUrl, scopeLen: textOf(scope).length }); |
| 539 | let md = writer.render(scope); |
550 | let md = writer.render(scope); |
| 540 | // Prepend the document title when the content does not already start with a heading. |
551 | // Prepend the document title when the content does not already start with a heading. |
| 541 | const title = findAll(root, (n) => n.tag === "title")[0]; |
552 | const title = findAll(root, (n) => n.tag === "title")[0]; |
| 549 | export function htmlToMainText(html: string): string { |
560 | export function htmlToMainText(html: string): string { |
| 550 | const root = parseHtml(html); |
561 | const root = parseHtml(html); |
| 551 | const scope = mainContent(root); |
562 | const scope = mainContent(root); |
| 552 | − const w = new MdWriter({ images: false, links: false, stripNoise: true }); |
563 | + const w = new MdWriter({ images: false, links: false, stripNoise: true, scopeLen: textOf(scope).length }); |
| 553 | return w |
564 | return w |
| 554 | .render(scope) |
565 | .render(scope) |
| 555 | .replace(/^#+\s*/gm, "") |
566 | .replace(/^#+\s*/gm, "") |