/**
 * WorthDoing.ai
 * Author: Simon-Pierre Boucher
 * Contact: contact@spboucher.ai
 * File: src/lib/firecrawl/url.ts
 * Description: Pure URL canonicalization and content hashing (no I/O) — the dedup primitives.
 */
import { createHash } from "crypto";

const TRACKING_PARAMS = new Set([
  "utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content",
  "gclid", "fbclid", "ref", "ref_src", "igshid", "mc_cid", "mc_eid",
]);

/** Normalize a URL to its canonical form for dedup. */
export function canonicalizeUrl(raw: string): string {
  const u = new URL(raw);
  u.hash = "";
  u.hostname = u.hostname.toLowerCase().replace(/^www\./, "");
  for (const key of [...u.searchParams.keys()]) {
    if (TRACKING_PARAMS.has(key.toLowerCase())) u.searchParams.delete(key);
  }
  u.searchParams.sort();
  let path = u.pathname.replace(/\/+$/, "");
  if (path === "") path = "/";
  u.pathname = path;
  return u.toString();
}

export function sha256(text: string): string {
  return createHash("sha256").update(text).digest("hex");
}