SPB Git

spb/tendril Public

Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.

JavaScript 82.6% TypeScript 11.8% HTML 5.3%
1.2 KB · 38 lines typescript
Raw Blame History
1// author: simon-pierre boucher <contact@spboucher.ai>2import { describe, expect, it } from "vitest";3import { buildHeaders, BOT_SUFFIX } from "./headers.js";45describe("buildHeaders", () => {6  it("emits headers in Safari order", () => {7    const keys = Object.keys(buildHeaders("example.com"));8    expect(keys).toEqual([9      "Host",10      "Accept",11      "Accept-Language",12      "Accept-Encoding",13      "User-Agent",14      "Connection",15      "Sec-Fetch-Dest",16      "Sec-Fetch-Mode",17      "Sec-Fetch-Site",18      "Sec-Fetch-User",19      "Upgrade-Insecure-Requests",20    ]);21  });2223  it("always appends the bot suffix to the UA (§26)", () => {24    expect(buildHeaders("example.com")["User-Agent"]).toContain(BOT_SUFFIX);25    expect(buildHeaders("example.com", {}, "Custom/9")["User-Agent"]).toBe("Custom/9" + BOT_SUFFIX);26  });2728  it("sets the Host header from the argument", () => {29    expect(buildHeaders("shop.example")["Host"]).toBe("shop.example");30  });3132  it("merges overrides over defaults without reordering base keys", () => {33    const h = buildHeaders("example.com", { "Accept-Language": "fr-CA,fr;q=0.9", "X-Extra": "1" });34    expect(h["Accept-Language"]).toBe("fr-CA,fr;q=0.9");35    expect(h["X-Extra"]).toBe("1");36  });37});38