spb/tendril Public
Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.
JavaScript 82.6%
TypeScript 11.8%
HTML 5.3%
1// author: simon-pierre boucher <contact@spboucher.ai>2import { describe, expect, it } from "vitest";3import { normalizeUrl } from "./url.js";45function norm(raw: string, base?: string): string {6 const r = normalizeUrl(raw, base);7 if (!r.ok) throw new Error(`expected ok, got ${r.error.code}`);8 return r.value;9}1011describe("normalizeUrl", () => {12 it("lowercases the host but not the path", () => {13 expect(norm("https://EXAMPLE.com/Path")).toBe("https://example.com/Path");14 });1516 it("strips default ports", () => {17 expect(norm("http://example.com:80/a")).toBe("http://example.com/a");18 expect(norm("https://example.com:443/a")).toBe("https://example.com/a");19 });2021 it("keeps non-default ports", () => {22 expect(norm("https://example.com:8443/a")).toBe("https://example.com:8443/a");23 });2425 it("removes tracking params but keeps real ones", () => {26 expect(norm("https://e.com/?utm_source=x&q=1&fbclid=z&_ga=2")).toBe("https://e.com/?q=1");27 });2829 it("sorts query params deterministically", () => {30 expect(norm("https://e.com/?b=2&a=1&a=0")).toBe("https://e.com/?a=0&a=1&b=2");31 });3233 it("drops the fragment unless hashbang", () => {34 expect(norm("https://e.com/p#section")).toBe("https://e.com/p");35 expect(norm("https://e.com/p#!/route")).toBe("https://e.com/p#!/route");36 });3738 it("removes trailing slash except at root", () => {39 expect(norm("https://e.com/foo/")).toBe("https://e.com/foo");40 expect(norm("https://e.com/")).toBe("https://e.com/");41 });4243 it("resolves dot segments", () => {44 expect(norm("https://e.com/a/b/../c")).toBe("https://e.com/a/c");45 });4647 it("punycodes international hosts", () => {48 expect(norm("https://münchen.de/")).toBe("https://xn--mnchen-3ya.de/");49 });5051 it("resolves relative URLs against a base", () => {52 expect(norm("../x", "https://e.com/a/b")).toBe("https://e.com/x");53 });5455 it("rejects non-http schemes", () => {56 const r = normalizeUrl("ftp://e.com/x");57 expect(r.ok).toBe(false);58 if (!r.ok) expect(r.error.code).toBe("ERR_INVALID_URL");59 });6061 it("rejects malformed input", () => {62 const r = normalizeUrl("not a url");63 expect(r.ok).toBe(false);64 });6566 it("produces identical output for equivalent URLs", () => {67 expect(norm("https://E.com:443/a/../b/?utm_x=1&z=9#frag")).toBe(norm("https://e.com/b?z=9"));68 });69});70