spb/tendril Public
Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.
JavaScript 82.6%
TypeScript 11.8%
HTML 5.3%
1// author: simon-pierre boucher <contact@spboucher.ai>2import { describe, expect, it } from "vitest";3import { parse } from "./dom.js";4import { harvestStructured } from "./structured.js";56describe("harvestStructured", () => {7 it("harvests JSON-LD, resolving @graph", () => {8 const html = `<html><head>9 <script type="application/ld+json">10 {"@context":"https://schema.org","@graph":[11 {"@type":"Article","headline":"The Headline","datePublished":"2026-01-01","author":{"@type":"Person","name":"Ada"}},12 {"@type":"Organization","name":"Acme"}13 ]}</script>14 </head><body><h1>Fallback</h1></body></html>`;15 const { metadata, structured } = harvestStructured(parse(html).document);16 expect(structured.jsonld.length).toBe(2);17 expect(metadata.title).toBe("The Headline");18 expect(metadata.publishedTime).toBe("2026-01-01");19 expect(metadata.author).toBe("Ada");20 });2122 it("prefers JSON-LD over OpenGraph over meta over heuristics", () => {23 const html = `<html><head>24 <title>Doc Title</title>25 <meta name="description" content="meta desc">26 <meta property="og:title" content="OG Title">27 <meta property="og:description" content="og desc">28 <script type="application/ld+json">{"@type":"Article","headline":"LD Title"}</script>29 </head><body><h1>H1 Title</h1></body></html>`;30 const { metadata } = harvestStructured(parse(html).document);31 expect(metadata.title).toBe("LD Title");32 expect(metadata.description).toBe("og desc");33 });3435 it("falls back to og:title then <title> then h1", () => {36 const ogOnly = harvestStructured(37 parse('<html><head><meta property="og:title" content="OG"><title>T</title></head><body><h1>H</h1></body></html>')38 .document,39 );40 expect(ogOnly.metadata.title).toBe("OG");41 const titleOnly = harvestStructured(parse("<html><head><title>T</title></head><body><h1>H</h1></body></html>").document);42 expect(titleOnly.metadata.title).toBe("T");43 const h1Only = harvestStructured(parse("<html><body><h1>H</h1></body></html>").document);44 expect(h1Only.metadata.title).toBe("H");45 });4647 it("harvests OpenGraph and Twitter cards", () => {48 const html = `<html><head>49 <meta property="og:site_name" content="Acme">50 <meta property="og:image" content="https://x/og.png">51 <meta name="twitter:image" content="https://x/tw.png">52 </head><body></body></html>`;53 const { metadata, structured } = harvestStructured(parse(html).document);54 expect(structured.openGraph["site_name"]).toBe("Acme");55 expect(metadata.siteName).toBe("Acme");56 expect(metadata.image).toBe("https://x/og.png");57 expect(structured.twitter["image"]).toBe("https://x/tw.png");58 });5960 it("ignores malformed JSON-LD without throwing", () => {61 const { structured } = harvestStructured(62 parse('<html><head><script type="application/ld+json">{bad json,</script></head><body></body></html>').document,63 );64 expect(structured.jsonld).toEqual([]);65 });6667 it("splits keywords and reads canonical + lang", () => {68 const html = `<html lang="fr"><head>69 <meta name="keywords" content="a, b ,c">70 <link rel="canonical" href="https://x/canon">71 </head><body></body></html>`;72 const { metadata } = harvestStructured(parse(html).document);73 expect(metadata.keywords).toEqual(["a", "b", "c"]);74 expect(metadata.canonical).toBe("https://x/canon");75 expect(metadata.lang).toBe("fr");76 });77});78