spb/tendril Public
Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.
JavaScript 82.6%
TypeScript 11.8%
HTML 5.3%
1// author: simon-pierre boucher <contact@spboucher.ai>2import { readFileSync } from "node:fs";3import { fileURLToPath } from "node:url";4import { dirname, resolve } from "node:path";5import { describe, expect, it } from "vitest";6import { extract } from "./pipeline.js";78const here = dirname(fileURLToPath(import.meta.url));9const fixture = (name: string): string =>10 readFileSync(resolve(here, "../../../test/fixtures/html", name), "utf8");1112describe("fixture: shopify-product", () => {13 const html = fixture("shopify-product.html");14 const r = extract(html, { url: "https://gardenco.example/products/trellis-frame", onlyMainContent: false });15 if (!r.ok) throw new Error("extraction failed");16 const d = r.value;1718 it("harvests the Product JSON-LD without a model", () => {19 const product = d.structured.jsonld.find(20 (n): n is Record<string, unknown> => typeof n === "object" && n !== null && (n as Record<string, unknown>)["@type"] === "Product",21 );22 expect(product?.["sku"]).toBe("TRL-2026");23 const offers = product?.["offers"] as Record<string, unknown> | undefined;24 expect(offers?.["price"]).toBe("49.00");25 });2627 it("prefers canonical and OpenGraph metadata", () => {28 expect(d.metadata.canonical).toBe("https://gardenco.example/products/trellis-frame");29 expect(d.metadata.title).toBe("Trellis Climbing Frame");30 expect(d.metadata.siteName).toBe("GardenCo");31 });3233 it("renders the spec table as a GFM table", () => {34 expect(d.markdown).toContain("| Spec | Value |");35 expect(d.markdown).toContain("| Material | Powder-coated steel |");36 });3738 it("resolves lazy data-src images to absolute URLs", () => {39 expect(d.html).toContain("https://gardenco.example/img/trellis-800.jpg");40 });4142 it("absolutizes and classifies links", () => {43 const reviews = d.links.find((l) => l.url.includes("/reviews"));44 expect(reviews?.url).toBe("https://gardenco.example/products/trellis-frame/reviews");45 expect(reviews?.isInternal).toBe(true);46 });47});48