spb/tendril Public
Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.
JavaScript 82.6%
TypeScript 11.8%
HTML 5.3%
1// author: simon-pierre boucher <contact@spboucher.ai>2import { err, ok, type Result } from "./result.js";3import { tendrilError } from "./errors.js";45const DEFAULT_PORTS: Readonly<Record<string, string>> = {6 "http:": "80",7 "https:": "443",8};910const TRACKING_PARAMS = new Set(["fbclid", "gclid", "gclsrc", "ref", "mc_cid", "mc_eid", "_ga", "_gl"]);1112function isTrackingParam(key: string): boolean {13 const lower = key.toLowerCase();14 return lower.startsWith("utm_") || TRACKING_PARAMS.has(lower);15}1617/**18 * Canonicalize a URL for dedup and cache keys (§10 rule 1, §10.1). The result is19 * intended for comparison only — keep the original string for output. Returns20 * ERR_INVALID_URL for anything that is not an absolute http(s) URL.21 */22export function normalizeUrl(raw: string, base?: string): Result<string> {23 let u: URL;24 try {25 u = base !== undefined ? new URL(raw, base) : new URL(raw);26 } catch {27 return err(tendrilError("ERR_INVALID_URL", { details: { url: raw } }));28 }2930 if (u.protocol !== "http:" && u.protocol !== "https:") {31 return err(tendrilError("ERR_INVALID_URL", { details: { url: raw, scheme: u.protocol } }));32 }3334 u.hostname = u.hostname.toLowerCase();3536 if (u.port !== "" && DEFAULT_PORTS[u.protocol] === u.port) {37 u.port = "";38 }3940 const kept: Array<[string, string]> = [];41 for (const [key, value] of u.searchParams.entries()) {42 if (!isTrackingParam(key)) kept.push([key, value]);43 }44 kept.sort((a, b) => (a[0] === b[0] ? (a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0) : a[0] < b[0] ? -1 : 1));45 u.search = "";46 for (const [key, value] of kept) u.searchParams.append(key, value);4748 const hashbang = u.hash.startsWith("#!");49 if (!hashbang) u.hash = "";5051 if (u.pathname.length > 1 && u.pathname.endsWith("/")) {52 u.pathname = u.pathname.replace(/\/+$/, "");53 if (u.pathname === "") u.pathname = "/";54 }5556 return ok(u.toString());57}5859export interface ParsedUrl {60 readonly url: URL;61 readonly host: string;62}6364export function parseHost(raw: string): Result<ParsedUrl> {65 try {66 const url = new URL(raw);67 return ok({ url, host: url.hostname.toLowerCase() });68 } catch {69 return err(tendrilError("ERR_INVALID_URL", { details: { url: raw } }));70 }71}72