SPB Git

spb/tendril Public

Tendril — web ingestion platform (scrape/crawl/map/search) on macOS Apple Silicon: WebKit fidelity, authenticated pages, deterministic testable extraction. A self-hosted Firecrawl alternative.

JavaScript 82.6% TypeScript 11.8% HTML 5.3%
2.3 KB · 70 lines typescript
Raw Blame History
1// author: simon-pierre boucher <contact@spboucher.ai>2import { describe, expect, it } from "vitest";3import { normalizeUrl } from "./url.js";45function norm(raw: string, base?: string): string {6  const r = normalizeUrl(raw, base);7  if (!r.ok) throw new Error(`expected ok, got ${r.error.code}`);8  return r.value;9}1011describe("normalizeUrl", () => {12  it("lowercases the host but not the path", () => {13    expect(norm("https://EXAMPLE.com/Path")).toBe("https://example.com/Path");14  });1516  it("strips default ports", () => {17    expect(norm("http://example.com:80/a")).toBe("http://example.com/a");18    expect(norm("https://example.com:443/a")).toBe("https://example.com/a");19  });2021  it("keeps non-default ports", () => {22    expect(norm("https://example.com:8443/a")).toBe("https://example.com:8443/a");23  });2425  it("removes tracking params but keeps real ones", () => {26    expect(norm("https://e.com/?utm_source=x&q=1&fbclid=z&_ga=2")).toBe("https://e.com/?q=1");27  });2829  it("sorts query params deterministically", () => {30    expect(norm("https://e.com/?b=2&a=1&a=0")).toBe("https://e.com/?a=0&a=1&b=2");31  });3233  it("drops the fragment unless hashbang", () => {34    expect(norm("https://e.com/p#section")).toBe("https://e.com/p");35    expect(norm("https://e.com/p#!/route")).toBe("https://e.com/p#!/route");36  });3738  it("removes trailing slash except at root", () => {39    expect(norm("https://e.com/foo/")).toBe("https://e.com/foo");40    expect(norm("https://e.com/")).toBe("https://e.com/");41  });4243  it("resolves dot segments", () => {44    expect(norm("https://e.com/a/b/../c")).toBe("https://e.com/a/c");45  });4647  it("punycodes international hosts", () => {48    expect(norm("https://münchen.de/")).toBe("https://xn--mnchen-3ya.de/");49  });5051  it("resolves relative URLs against a base", () => {52    expect(norm("../x", "https://e.com/a/b")).toBe("https://e.com/x");53  });5455  it("rejects non-http schemes", () => {56    const r = normalizeUrl("ftp://e.com/x");57    expect(r.ok).toBe(false);58    if (!r.ok) expect(r.error.code).toBe("ERR_INVALID_URL");59  });6061  it("rejects malformed input", () => {62    const r = normalizeUrl("not a url");63    expect(r.ok).toBe(false);64  });6566  it("produces identical output for equivalent URLs", () => {67    expect(norm("https://E.com:443/a/../b/?utm_x=1&z=9#frag")).toBe(norm("https://e.com/b?z=9"));68  });69});70