SPB Git

spb/tendril Public

Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.

JavaScript 82.6% TypeScript 11.8% HTML 5.3%
1.3 KB · 37 lines typescript
Raw Blame History
1// author: simon-pierre boucher <contact@spboucher.ai>2export const SAFARI_UA =3  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15";45export const BOT_SUFFIX = " Tendril/1.0 (+https://www.ten-dril.com/bot)";67export const DEFAULT_USER_AGENT = SAFARI_UA + BOT_SUFFIX;89/**10 * Build request headers in Safari's emission order (§3.2). Header order is a11 * fingerprint, so this returns an insertion-ordered object; do not sort it.12 * The bot suffix in the UA is non-negotiable (§26).13 */14export function buildHeaders(15  host: string,16  overrides?: Readonly<Record<string, string>>,17  userAgent?: string,18): Record<string, string> {19  const headers: Record<string, string> = {20    Host: host,21    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",22    "Accept-Language": "en-US,en;q=0.9",23    "Accept-Encoding": "gzip, deflate, br",24    "User-Agent": (userAgent ?? SAFARI_UA) + BOT_SUFFIX,25    Connection: "keep-alive",26    "Sec-Fetch-Dest": "document",27    "Sec-Fetch-Mode": "navigate",28    "Sec-Fetch-Site": "none",29    "Sec-Fetch-User": "?1",30    "Upgrade-Insecure-Requests": "1",31  };32  if (overrides) {33    for (const [key, value] of Object.entries(overrides)) headers[key] = value;34  }35  return headers;36}37