spb/tools-spboucher-ai

1/** Measurable search-quality evaluation: for each (query → expected slug) in tests/fixtures/ranking.json, compute rank of the expected result; report MRR, hit@1, hit@5. Fails if MRR < threshold. */2import "./_env";3import { readFileSync } from "node:fs";4import { join } from "node:path";5import { sqlPublic, closeDb } from "../src/db";6const cases = JSON.parse(readFileSync(join(import.meta.dirname, "..", "tests", "fixtures", "ranking.json"), "utf8")) as { q: string; expect: string }[];7const threshold = Number(process.argv[2] ?? 0.7);8// Minimal re-implementation of the lexical ranker (same SQL as src/lib/public/queries.ts) to avoid importing server-only modules.9function prefixQuery(q: string) { const toks = q.toLowerCase().split(/[^\p{L}\p{N}+#.]+/u).filter(Boolean).slice(0, 8); return toks.map((t, i) => (i === toks.length - 1 ? `${t}:*` : t)).join(" & "); }10let rr = 0, h1 = 0, h5 = 0; const misses: string[] = [];11for (const c of cases) {12  const pq = prefixQuery(c.q);13  const rows = await sqlPublic<{ slug: string }[]>`SELECT t.slug FROM public_tools t WHERE (t.search_vector @@ to_tsquery('english', f_unaccent(${pq})) OR similarity(f_unaccent(lower(t.name)), f_unaccent(lower(${c.q}))) > 0.3 OR f_unaccent(lower(t.name)) LIKE '%' || f_unaccent(lower(${c.q})) || '%' OR EXISTS (SELECT 1 FROM unnest(t.aliases) a WHERE similarity(f_unaccent(lower(a)), f_unaccent(lower(${c.q}))) > 0.3))14    ORDER BY (ts_rank_cd(t.search_vector, to_tsquery('english', f_unaccent(${pq})), 32) + greatest(similarity(f_unaccent(lower(t.name)), f_unaccent(lower(${c.q}))), (SELECT coalesce(max(similarity(f_unaccent(lower(a)), f_unaccent(lower(${c.q})))),0) FROM unnest(t.aliases) a)) * 0.9 + CASE WHEN f_unaccent(lower(t.name)) LIKE f_unaccent(lower(${c.q})) || '%' THEN 0.6 ELSE 0 END + CASE WHEN f_unaccent(lower(t.name)) = f_unaccent(lower(${c.q})) THEN 1.0 ELSE 0 END + t.quality_score / 1000.0 + CASE WHEN t.featured THEN 0.03 ELSE 0 END) DESC, t.name LIMIT 10`;15  const rank = rows.findIndex((r) => r.slug === c.expect) + 1;16  if (rank > 0) { rr += 1 / rank; if (rank === 1) h1++; if (rank <= 5) h5++; } else misses.push(`${c.q} → expected ${c.expect}, got [${rows.slice(0, 3).map((r) => r.slug).join(", ")}]`);17}18const mrr = rr / cases.length;19console.log(JSON.stringify({ cases: cases.length, mrr: +mrr.toFixed(3), hitAt1: +(h1 / cases.length).toFixed(3), hitAt5: +(h5 / cases.length).toFixed(3), threshold, pass: mrr >= threshold, at: new Date().toISOString() }));20for (const m of misses) console.log("MISS", m);21await closeDb(); if (mrr < threshold) process.exit(1);22