SPB Git

spb/tendril Public

Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.

JavaScript 82.6% TypeScript 11.8% HTML 5.3%
2.2 KB · 72 lines typescript
Raw Blame History
1// author: simon-pierre boucher <contact@spboucher.ai>2import { err, ok, type Result } from "./result.js";3import { tendrilError } from "./errors.js";45const DEFAULT_PORTS: Readonly<Record<string, string>> = {6  "http:": "80",7  "https:": "443",8};910const TRACKING_PARAMS = new Set(["fbclid", "gclid", "gclsrc", "ref", "mc_cid", "mc_eid", "_ga", "_gl"]);1112function isTrackingParam(key: string): boolean {13  const lower = key.toLowerCase();14  return lower.startsWith("utm_") || TRACKING_PARAMS.has(lower);15}1617/**18 * Canonicalize a URL for dedup and cache keys (§10 rule 1, §10.1). The result is19 * intended for comparison only — keep the original string for output. Returns20 * ERR_INVALID_URL for anything that is not an absolute http(s) URL.21 */22export function normalizeUrl(raw: string, base?: string): Result<string> {23  let u: URL;24  try {25    u = base !== undefined ? new URL(raw, base) : new URL(raw);26  } catch {27    return err(tendrilError("ERR_INVALID_URL", { details: { url: raw } }));28  }2930  if (u.protocol !== "http:" && u.protocol !== "https:") {31    return err(tendrilError("ERR_INVALID_URL", { details: { url: raw, scheme: u.protocol } }));32  }3334  u.hostname = u.hostname.toLowerCase();3536  if (u.port !== "" && DEFAULT_PORTS[u.protocol] === u.port) {37    u.port = "";38  }3940  const kept: Array<[string, string]> = [];41  for (const [key, value] of u.searchParams.entries()) {42    if (!isTrackingParam(key)) kept.push([key, value]);43  }44  kept.sort((a, b) => (a[0] === b[0] ? (a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0) : a[0] < b[0] ? -1 : 1));45  u.search = "";46  for (const [key, value] of kept) u.searchParams.append(key, value);4748  const hashbang = u.hash.startsWith("#!");49  if (!hashbang) u.hash = "";5051  if (u.pathname.length > 1 && u.pathname.endsWith("/")) {52    u.pathname = u.pathname.replace(/\/+$/, "");53    if (u.pathname === "") u.pathname = "/";54  }5556  return ok(u.toString());57}5859export interface ParsedUrl {60  readonly url: URL;61  readonly host: string;62}6364export function parseHost(raw: string): Result<ParsedUrl> {65  try {66    const url = new URL(raw);67    return ok({ url, host: url.hostname.toLowerCase() });68  } catch {69    return err(tendrilError("ERR_INVALID_URL", { details: { url: raw } }));70  }71}72