SPB Git

spb/tendril Public

Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.

JavaScript 82.6% TypeScript 11.8% HTML 5.3%
2.0 KB · 48 lines typescript
Raw Blame History
1// author: simon-pierre boucher <contact@spboucher.ai>2import { readFileSync } from "node:fs";3import { fileURLToPath } from "node:url";4import { dirname, resolve } from "node:path";5import { describe, expect, it } from "vitest";6import { extract } from "./pipeline.js";78const here = dirname(fileURLToPath(import.meta.url));9const fixture = (name: string): string =>10  readFileSync(resolve(here, "../../../test/fixtures/html", name), "utf8");1112describe("fixture: shopify-product", () => {13  const html = fixture("shopify-product.html");14  const r = extract(html, { url: "https://gardenco.example/products/trellis-frame", onlyMainContent: false });15  if (!r.ok) throw new Error("extraction failed");16  const d = r.value;1718  it("harvests the Product JSON-LD without a model", () => {19    const product = d.structured.jsonld.find(20      (n): n is Record<string, unknown> => typeof n === "object" && n !== null && (n as Record<string, unknown>)["@type"] === "Product",21    );22    expect(product?.["sku"]).toBe("TRL-2026");23    const offers = product?.["offers"] as Record<string, unknown> | undefined;24    expect(offers?.["price"]).toBe("49.00");25  });2627  it("prefers canonical and OpenGraph metadata", () => {28    expect(d.metadata.canonical).toBe("https://gardenco.example/products/trellis-frame");29    expect(d.metadata.title).toBe("Trellis Climbing Frame");30    expect(d.metadata.siteName).toBe("GardenCo");31  });3233  it("renders the spec table as a GFM table", () => {34    expect(d.markdown).toContain("| Spec | Value |");35    expect(d.markdown).toContain("| Material | Powder-coated steel |");36  });3738  it("resolves lazy data-src images to absolute URLs", () => {39    expect(d.html).toContain("https://gardenco.example/img/trellis-800.jpg");40  });4142  it("absolutizes and classifies links", () => {43    const reviews = d.links.find((l) => l.url.includes("/reviews"));44    expect(reviews?.url).toBe("https://gardenco.example/products/trellis-frame/reviews");45    expect(reviews?.isInternal).toBe(true);46  });47});48