// author: simon-pierre boucher import { ok, type Result } from "@tendril/shared"; import { absolutizeUrls, dropSelectors, keepOnly, parse, parseFragment, sanitize, textContentLength } from "./dom.js"; import { extractMainContent } from "./boilerplate.js"; import { harvestStructured } from "./structured.js"; import { extractLinks } from "./links.js"; import { htmlToMarkdown } from "./markdown.js"; import type { ExtractOptions, ExtractResult } from "./types.js"; /** * The deterministic extraction pipeline (ยง8.1). Pure: `(html, options) => result`, * no I/O, safe to run offline against saved fixtures. Structured data is harvested * from the full document before boilerplate removal, so JSON-LD in survives. */ export function extract(html: string, options: ExtractOptions): Result { const base = options.url; const full = parse(html); const { metadata, structured } = harvestStructured(full.document); const links = extractLinks(full.document, base); const main = extractMainContent(html, options.onlyMainContent ?? true); const content = parseFragment(main.html); if (options.includeTags && options.includeTags.length > 0) { keepOnly(content.document, options.includeTags); } if (options.excludeTags && options.excludeTags.length > 0) { dropSelectors(content.document, options.excludeTags); } sanitize(content.document); absolutizeUrls(content.document, base); const cleanedHtml = content.document.body.innerHTML; const markdown = htmlToMarkdown(cleanedHtml); const textLength = textContentLength(content.document); const result: ExtractResult = { markdown, html: cleanedHtml, links, metadata, structured, textLength, }; return ok(result); }