spb/social-runtime-crawler

1import { createHash, randomUUID } from "node:crypto";23export function nowIso(): string {4  return new Date().toISOString();5}67export function newId(prefix: string): string {8  return `${prefix}_${randomUUID().replace(/-/g, "").slice(0, 20)}`;9}1011export function sha1(input: string): string {12  return createHash("sha1").update(input).digest("hex");13}1415export function shortHash(input: string, len = 12): string {16  return sha1(input).slice(0, len);17}1819export function clamp01(n: number): number {20  if (Number.isNaN(n)) return 0;21  return Math.max(0, Math.min(1, n));22}2324export function truncate(s: string | undefined, max: number): string {25  if (!s) return "";26  const t = s.replace(/\s+/g, " ").trim();27  return t.length > max ? t.slice(0, max - 1) + "…" : t;28}2930export function sleep(ms: number): Promise<void> {31  return new Promise((r) => setTimeout(r, ms));32}3334/** Parse "1.2M", "483", "12 k", "3,4 M de vues" → number. Returns undefined when nothing numeric. */35export function parseCount(text: string | undefined | null): number | undefined {36  if (!text) return undefined;37  const m = text.replace(/ /g, " ").replace(/\s+/g, " ").match(/(\d{1,3}(?:[ ,.]\d{3})+|\d+(?:[.,]\d+)?)\s*([kKmMbBG])?(?![a-z])/i);38  // matches "3 624", "12,345", "1,2 M", "48 k", "483" (all unicode spaces normalised first)39  if (!m) return undefined;40  let raw = m[1]!;41  const suffix = (m[2] ?? "").toLowerCase();42  if (/^\d{1,3}(?:[ ,.]\d{3})+$/.test(raw) && !suffix) raw = raw.replace(/[ ,.]/g, ""); // thousands groups43  else raw = raw.replace(/ /g, "").replace(",", "."); // decimal comma (fr)44  let n = Number(raw);45  if (Number.isNaN(n)) return undefined;46  if (suffix === "k") n *= 1e3;47  else if (suffix === "m") n *= 1e6;48  else if (suffix === "b" || suffix === "g") n *= 1e9;49  return Math.round(n);50}5152/** Parse "12:34" / "1:02:03" → seconds. */53export function parseDuration(text: string | undefined | null): number | undefined {54  if (!text) return undefined;55  const m = text.trim().match(/^(?:(\d+):)?(\d{1,2}):(\d{2})$/);56  if (!m) return undefined;57  const h = m[1] ? Number(m[1]) : 0;58  return h * 3600 + Number(m[2]) * 60 + Number(m[3]);59}6061const STOPWORDS = new Set(["the", "and", "for", "with", "that", "this", "from", "les", "des", "une", "sur", "pour", "dans", "est", "qui", "que", "de", "la", "le", "du", "en", "un", "et", "au", "of", "to", "in", "on", "is", "it", "or", "by", "an", "as", "at", "be", "we", "vs", "ce", "se", "sa", "son", "ses", "ne", "pas", "plus", "how", "what", "why", "who", "public", "discover", "discussing", "about"]);6263/** Tokenize for cheap lexical similarity (novelty fallback when no embedding model is present). */64export function tokenize(text: string): Set<string> {65  return new Set(66    text67      .toLowerCase()68      .normalize("NFD")69      .replace(/[̀-ͯ]/g, "")70      .split(/[^a-z0-9#@]+/)71      // keep 2-letter tokens: "ai", "ia", "qc" carry meaning in this domain72      .filter((t) => t.length >= 2 && !STOPWORDS.has(t)),73  );74}7576export function jaccard(a: Set<string>, b: Set<string>): number {77  if (a.size === 0 || b.size === 0) return 0;78  let inter = 0;79  for (const t of a) if (b.has(t)) inter++;80  return inter / (a.size + b.size - inter);81}8283export function canonicalUrl(url: string): string {84  try {85    const u = new URL(url);86    u.hash = "";87    // strip common tracking params88    for (const k of [...u.searchParams.keys()]) {89      if (/^(utm_|fbclid|gclid|igshid|si|feature|pp|ref_src|ref_url|t|ref|refid|rdid|mibextid|locale|__cft__|__tn__|__xts__|eid|hc_ref|notif_id|notif_t|comment_tracking|source|sfnsn|extid)/i.test(k)) u.searchParams.delete(k);90    }91    u.hostname = u.hostname.toLowerCase().replace(/^(www|m|mobile)\./, "");92    let s = u.toString();93    if (s.endsWith("/")) s = s.slice(0, -1);94    return s;95  } catch {96    return url;97  }98}99100export function safeJsonParse(text: string): unknown | undefined {101  try {102    return JSON.parse(text);103  } catch {104    return undefined;105  }106}107108/** Walk any JSON value depth-first. Visitor receives (value, path). Return false to stop descending. */109export function walkJson(110  value: unknown,111  visitor: (v: unknown, path: string[]) => boolean | void,112  path: string[] = [],113  depth = 0,114): void {115  if (depth > 40) return;116  const cont = visitor(value, path);117  if (cont === false) return;118  if (Array.isArray(value)) {119    value.forEach((v, i) => walkJson(v, visitor, [...path, `[${i}]`], depth + 1));120  } else if (value && typeof value === "object") {121    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {122      walkJson(v, visitor, [...path, k], depth + 1);123    }124  }125}126