spb/tendril Public
Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.
JavaScript 82.6%
TypeScript 11.8%
HTML 5.3%
1// author: simon-pierre boucher <contact@spboucher.ai>2import { describe, expect, it } from "vitest";3import { buildHeaders, BOT_SUFFIX } from "./headers.js";45describe("buildHeaders", () => {6 it("emits headers in Safari order", () => {7 const keys = Object.keys(buildHeaders("example.com"));8 expect(keys).toEqual([9 "Host",10 "Accept",11 "Accept-Language",12 "Accept-Encoding",13 "User-Agent",14 "Connection",15 "Sec-Fetch-Dest",16 "Sec-Fetch-Mode",17 "Sec-Fetch-Site",18 "Sec-Fetch-User",19 "Upgrade-Insecure-Requests",20 ]);21 });2223 it("always appends the bot suffix to the UA (§26)", () => {24 expect(buildHeaders("example.com")["User-Agent"]).toContain(BOT_SUFFIX);25 expect(buildHeaders("example.com", {}, "Custom/9")["User-Agent"]).toBe("Custom/9" + BOT_SUFFIX);26 });2728 it("sets the Host header from the argument", () => {29 expect(buildHeaders("shop.example")["Host"]).toBe("shop.example");30 });3132 it("merges overrides over defaults without reordering base keys", () => {33 const h = buildHeaders("example.com", { "Accept-Language": "fr-CA,fr;q=0.9", "X-Extra": "1" });34 expect(h["Accept-Language"]).toBe("fr-CA,fr;q=0.9");35 expect(h["X-Extra"]).toBe("1");36 });37});38