SPB Git

spb/tendril Public

Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.

JavaScript 82.6% TypeScript 11.8% HTML 5.3%
1.6 KB · 42 lines typescript
Raw Blame History
1// author: simon-pierre boucher <contact@spboucher.ai>2import { describe, expect, it } from "vitest";3import { parseSitemap } from "./sitemap.js";45describe("parseSitemap", () => {6  it("parses a urlset with lastmod", () => {7    const xml = `<?xml version="1.0"?><urlset>8      <url><loc>https://e.com/a</loc><lastmod>2026-01-01</lastmod></url>9      <url><loc>https://e.com/b</loc></url>10    </urlset>`;11    const r = parseSitemap(xml);12    expect(r.urls).toEqual([13      { loc: "https://e.com/a", lastModified: "2026-01-01" },14      { loc: "https://e.com/b" },15    ]);16    expect(r.childSitemaps).toEqual([]);17  });1819  it("parses a sitemap index into child sitemaps", () => {20    const xml = `<sitemapindex>21      <sitemap><loc>https://e.com/sitemap-1.xml</loc></sitemap>22      <sitemap><loc>https://e.com/sitemap-2.xml</loc></sitemap>23    </sitemapindex>`;24    const r = parseSitemap(xml);25    expect(r.childSitemaps).toEqual(["https://e.com/sitemap-1.xml", "https://e.com/sitemap-2.xml"]);26    expect(r.urls).toEqual([]);27  });2829  it("decodes entities and CDATA in loc", () => {30    const xml = `<urlset><url><loc>https://e.com/s?a=1&amp;b=2</loc></url>31      <url><loc><![CDATA[https://e.com/c]]></loc></url></urlset>`;32    const r = parseSitemap(xml);33    expect(r.urls[0]?.loc).toBe("https://e.com/s?a=1&b=2");34    expect(r.urls[1]?.loc).toBe("https://e.com/c");35  });3637  it("falls back to bare <loc> tags", () => {38    const r = parseSitemap("<loc>https://e.com/x</loc><loc>https://e.com/y</loc>");39    expect(r.urls.map((u) => u.loc)).toEqual(["https://e.com/x", "https://e.com/y"]);40  });41});42