spb/wp-spboucher-ai

1/** Deterministic hashed TF-IDF-style vector (512 dims) — a provider-free similarity signal. Labelled "lexical (inferred)" in the UI. */2const DIM = 512;3const STOP = new Set("the of and in to a is for on with that this by as are from we an be at or it its which these those their not but also than into can has have was were our such more one two across between within using used use based paper results model models data".split(" "));4function fnv(s: string): number { let h = 0x811c9dc5; for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 0x01000193) >>> 0; } return h; }5export function tokenize(text: string): string[] {6  return text.toLowerCase().normalize("NFD").replace(/[̀-ͯ]/g, "").replace(/[^a-z0-9 ]+/g, " ").split(/\s+/).filter((w) => w.length > 2 && !STOP.has(w) && !/^\d+$/.test(w));7}8export function lexvec(text: string): number[] {9  const v = new Float64Array(DIM); const toks = tokenize(text); const tf = new Map<string, number>();10  for (const t of toks) tf.set(t, (tf.get(t) ?? 0) + 1);11  for (let i = 0; i < toks.length - 1; i++) { const bg = toks[i] + "_" + toks[i + 1]; tf.set(bg, (tf.get(bg) ?? 0) + 0.5); }12  for (const [t, c] of tf) { const h = fnv(t); const sign = (h & 1) ? 1 : -1; v[h % DIM] += sign * (1 + Math.log(c)); }13  let n = 0; for (let i = 0; i < DIM; i++) n += v[i] * v[i]; n = Math.sqrt(n) || 1;14  return Array.from(v, (x) => x / n);15}16export function cosine(a: number[], b: number[]): number { let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; return s; }17export const LEXVEC_DIM = DIM;18