Repository Public
1/** Explainable lexical embedding: hashed, sub-linear TF with L2 normalisation. No model, no network. */2export const DIM = 512;3const STOP = new Set("a an and are as at be but by for from has have he her his in is it its of on or she that the their they this to was were will with you your un une des les le la de du et en est que qui dans pour par sur au aux ce ces se sa son ses il elle ils elles nous vous ne pas plus ou où a à d l qu s n".split(" "));45export function tokens(text: string): string[] {6 return text.normalize("NFKD").replace(/[\u0300-\u036f]/g, "").toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 2 && !STOP.has(w));7}8function h(s: string): number {9 let x = 2166136261;10 for (let i = 0; i < s.length; i++) { x ^= s.charCodeAt(i); x = Math.imul(x, 16777619); }11 return x >>> 0;12}13export function embed(text: string): number[] {14 const v = new Array<number>(DIM).fill(0);15 const tf = new Map<string, number>();16 const toks = tokens(text);17 for (const w of toks) tf.set(w, (tf.get(w) ?? 0) + 1);18 for (let i = 0; i < toks.length - 1; i++) { const bi = `${toks[i]}_${toks[i + 1]}`; tf.set(bi, (tf.get(bi) ?? 0) + 0.5); }19 for (const [w, n] of tf) { const k = h(w); v[k % DIM] += (k & 0x80000000 ? -1 : 1) * (1 + Math.log(n)); }20 const norm = Math.hypot(...v) || 1;21 return v.map((x) => Math.round((x / norm) * 1e5) / 1e5);22}23export function cosine(a: number[], b: number[]): number {24 if (a.length !== b.length || !a.length) return 0;25 let s = 0;26 for (let i = 0; i < a.length; i++) s += a[i] * b[i];27 return s;28}29/** Shared meaningful words, for "why this was recommended". */30export function sharedTerms(a: string, b: string, max = 4): string[] {31 const B = new Set(tokens(b));32 return [...new Set(tokens(a).filter((w) => B.has(w)))].slice(0, max);33}34