/** Deterministic hashed TF-IDF-style vector (512 dims) — a provider-free similarity signal. Labelled "lexical (inferred)" in the UI. */
const DIM = 512;
const STOP = new Set("the of and in to a is for on with that this by as are from we an be at or it its which these those their not but also than into can has have was were our such more one two across between within using used use based paper results model models data".split(" "));
function fnv(s: string): number { let h = 0x811c9dc5; for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 0x01000193) >>> 0; } return h; }
export function tokenize(text: string): string[] {
  return text.toLowerCase().normalize("NFD").replace(/[̀-ͯ]/g, "").replace(/[^a-z0-9 ]+/g, " ").split(/\s+/).filter((w) => w.length > 2 && !STOP.has(w) && !/^\d+$/.test(w));
}
export function lexvec(text: string): number[] {
  const v = new Float64Array(DIM); const toks = tokenize(text); const tf = new Map<string, number>();
  for (const t of toks) tf.set(t, (tf.get(t) ?? 0) + 1);
  for (let i = 0; i < toks.length - 1; i++) { const bg = toks[i] + "_" + toks[i + 1]; tf.set(bg, (tf.get(bg) ?? 0) + 0.5); }
  for (const [t, c] of tf) { const h = fnv(t); const sign = (h & 1) ? 1 : -1; v[h % DIM] += sign * (1 + Math.log(c)); }
  let n = 0; for (let i = 0; i < DIM; i++) n += v[i] * v[i]; n = Math.sqrt(n) || 1;
  return Array.from(v, (x) => x / n);
}
export function cosine(a: number[], b: number[]): number { let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; return s; }
export const LEXVEC_DIM = DIM;