spb/worthdoing Public
Autonomous investigation agent that discovers, challenges, and ranks things genuinely worth doing — Claude + Firecrawl, Next.js 16, PostgreSQL
TypeScript 91.5%
SQL 5.8%
CSS 2.2%
1/**2 * WorthDoing.ai3 * Author: Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * File: src/lib/firecrawl/url.ts6 * Description: Pure URL canonicalization and content hashing (no I/O) — the dedup primitives.7 */8import { createHash } from "crypto";910const TRACKING_PARAMS = new Set([11 "utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content",12 "gclid", "fbclid", "ref", "ref_src", "igshid", "mc_cid", "mc_eid",13]);1415/** Normalize a URL to its canonical form for dedup. */16export function canonicalizeUrl(raw: string): string {17 const u = new URL(raw);18 u.hash = "";19 u.hostname = u.hostname.toLowerCase().replace(/^www\./, "");20 for (const key of [...u.searchParams.keys()]) {21 if (TRACKING_PARAMS.has(key.toLowerCase())) u.searchParams.delete(key);22 }23 u.searchParams.sort();24 let path = u.pathname.replace(/\/+$/, "");25 if (path === "") path = "/";26 u.pathname = path;27 return u.toString();28}2930export function sha256(text: string): string {31 return createHash("sha256").update(text).digest("hex");32}33