spb/tendril Public
Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.
JavaScript 82.6%
TypeScript 11.8%
HTML 5.3%
1// author: simon-pierre boucher <contact@spboucher.ai>2import { ok, type Result } from "@tendril/shared";3import { absolutizeUrls, dropSelectors, keepOnly, parse, parseFragment, sanitize, textContentLength } from "./dom.js";4import { extractMainContent } from "./boilerplate.js";5import { harvestStructured } from "./structured.js";6import { extractLinks } from "./links.js";7import { htmlToMarkdown } from "./markdown.js";8import type { ExtractOptions, ExtractResult } from "./types.js";910/**11 * The deterministic extraction pipeline (§8.1). Pure: `(html, options) => result`,12 * no I/O, safe to run offline against saved fixtures. Structured data is harvested13 * from the full document before boilerplate removal, so JSON-LD in <head> survives.14 */15export function extract(html: string, options: ExtractOptions): Result<ExtractResult> {16 const base = options.url;1718 const full = parse(html);19 const { metadata, structured } = harvestStructured(full.document);20 const links = extractLinks(full.document, base);2122 const main = extractMainContent(html, options.onlyMainContent ?? true);23 const content = parseFragment(main.html);2425 if (options.includeTags && options.includeTags.length > 0) {26 keepOnly(content.document, options.includeTags);27 }28 if (options.excludeTags && options.excludeTags.length > 0) {29 dropSelectors(content.document, options.excludeTags);30 }3132 sanitize(content.document);33 absolutizeUrls(content.document, base);3435 const cleanedHtml = content.document.body.innerHTML;36 const markdown = htmlToMarkdown(cleanedHtml);37 const textLength = textContentLength(content.document);3839 const result: ExtractResult = {40 markdown,41 html: cleanedHtml,42 links,43 metadata,44 structured,45 textLength,46 };47 return ok(result);48}49