spb/tendril Public
Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.
JavaScript 82.6%
TypeScript 11.8%
HTML 5.3%
1// author: simon-pierre boucher <contact@spboucher.ai>2import { describe, expect, it } from "vitest";3import { extract } from "./pipeline.js";45const article = `<!doctype html><html lang="en"><head>6 <title>How Tendrils Climb</title>7 <meta name="description" content="A study of climbing plants.">8 <script type="application/ld+json">{"@type":"Article","headline":"How Tendrils Climb","author":{"name":"Botanist"}}</script>9</head><body>10 <header><nav><a href="/home">Home</a></nav></header>11 <article>12 <h1>How Tendrils Climb</h1>13 <p>${"Tendrils are specialized stems that coil around supports to hoist the plant upward. ".repeat(6)}</p>14 <p>${"They respond to touch through a process called thigmotropism which is fascinating. ".repeat(6)}</p>15 <a href="/deep-dive">Read the deep dive</a>16 </article>17 <footer><a href="/privacy">Privacy</a></footer>18</body></html>`;1920describe("extract pipeline", () => {21 it("returns markdown, metadata, structured and links", () => {22 const r = extract(article, { url: "https://plants.example/guide", onlyMainContent: true });23 expect(r.ok).toBe(true);24 if (!r.ok) return;25 const d = r.value;26 expect(d.metadata.title).toBe("How Tendrils Climb");27 expect(d.metadata.author).toBe("Botanist");28 expect(d.structured.jsonld.length).toBe(1);29 expect(d.markdown).toContain("thigmotropism");30 expect(d.textLength).toBeGreaterThan(200);31 });3233 it("absolutizes links and flags internal vs external", () => {34 const r = extract(article, { url: "https://plants.example/guide" });35 if (!r.ok) throw new Error("expected ok");36 const deep = r.value.links.find((l) => l.url.endsWith("/deep-dive"));37 expect(deep?.url).toBe("https://plants.example/deep-dive");38 expect(deep?.isInternal).toBe(true);39 });4041 it("drops chrome (nav/footer) content from markdown in onlyMainContent mode", () => {42 const r = extract(article, { url: "https://plants.example/guide", onlyMainContent: true });43 if (!r.ok) throw new Error("expected ok");44 expect(r.value.markdown).not.toContain("Privacy");45 });4647 it("honors excludeTags", () => {48 const html = `<html><body><main><p>${"keep this important content here for density ".repeat(8)}</p>` +49 `<div class="promo-box">buy now discount offer</div></main></body></html>`;50 const r = extract(html, { url: "https://x.example/", onlyMainContent: false, excludeTags: [".promo-box"] });51 if (!r.ok) throw new Error("expected ok");52 expect(r.value.markdown).not.toContain("buy now");53 expect(r.value.markdown).toContain("keep this");54 });5556 it("uses density fallback on a listing page with little prose", () => {57 const items = Array.from({ length: 8 }, (_, i) => `<li><a href="/p/${i}">Product ${i}</a> $${i}0.00 in stock</li>`).join("");58 const html = `<html><body><nav><a href="/">nav</a></nav><main><ul class="grid">${items}</ul>` +59 `<p>${"Some descriptive listing text that gives the container real density weight. ".repeat(4)}</p></main></body></html>`;60 const r = extract(html, { url: "https://shop.example/list", onlyMainContent: true });61 if (!r.ok) throw new Error("expected ok");62 expect(r.value.markdown).toContain("Product 0");63 expect(r.value.markdown).toContain("Product 7");64 });65});66