TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { createHash } from "node:crypto";23export function sha256(data: string | Uint8Array): string {4 return createHash("sha256").update(data).digest("hex");5}67export function sha1(data: string | Uint8Array): string {8 return createHash("sha1").update(data).digest("hex");9}1011const STOP = new Set(12 "a an the and or of to in on for with by from at as is are was were be been this that these those it its into over under about after before than then there their they we you your our not no yes can will may".split(" "),13);1415export function tokens(text: string): string[] {16 return text17 .toLowerCase()18 .replace(/[^\p{L}\p{N}$%.\-/]+/gu, " ")19 .split(/\s+/)20 .filter((t) => t.length > 1 && !STOP.has(t));21}2223/** Word 3-shingles used for Jaccard similarity / novelty. */24export function shingles(text: string, k = 3): Set<string> {25 const t = tokens(text);26 const out = new Set<string>();27 if (t.length < k) {28 if (t.length) out.add(t.join(" "));29 return out;30 }31 for (let i = 0; i <= t.length - k; i++) out.add(t.slice(i, i + k).join(" "));32 return out;33}3435export function jaccard(a: Set<string>, b: Set<string>): number {36 if (!a.size && !b.size) return 1;37 let inter = 0;38 for (const x of a) if (b.has(x)) inter++;39 return inter / (a.size + b.size - inter);40}4142/** 64-bit simhash over tokens as a hex string. Cheap semantic fingerprint for near-duplicate detection. */43export function simhash(text: string): string {44 const v = new Array<number>(64).fill(0);45 for (const tok of tokens(text)) {46 const h = createHash("md5").update(tok).digest();47 for (let i = 0; i < 64; i++) {48 const bit = (h[i >> 3]! >> (i & 7)) & 1;49 v[i]! += bit ? 1 : -1;50 }51 }52 let hi = 0n;53 for (let i = 0; i < 64; i++) if (v[i]! > 0) hi |= 1n << BigInt(i);54 return hi.toString(16).padStart(16, "0");55}5657export function hammingHex(a: string, b: string): number {58 let x = BigInt("0x" + a) ^ BigInt("0x" + b);59 let c = 0;60 while (x) {61 c += Number(x & 1n);62 x >>= 1n;63 }64 return c;65}66