spb/spbgit

1/**2 * ─────────────────────────────────────────────3 *  SPB Git — Personal Git Platform4 * ─────────────────────────────────────────────5 *  Author  : Simon-Pierre Boucher6 *  Contact : contact@spboucher.ai7 *  File    : src/ai/index.mjs8 *  Purpose : Repository text index for grounded Q&A — per-blob chunk cache (incremental across commits), BM25 retrieval, no external calls9 *  License : MIT © Simon-Pierre Boucher10 * ─────────────────────────────────────────────11 */1213import { mapLimit } from '../lib/util.mjs';1415const MAX_FILE_BYTES = 96 * 1024;16const MAX_FILES = 1500;17const CHUNK_LINES = 60;18const CHUNK_OVERLAP = 10;19const TEXT_EXT = /\.(m?js|cjs|ts|tsx|jsx|py|r|jl|go|rs|c|h|cpp|hpp|java|kt|swift|rb|php|sh|zsh|sql|json|ya?ml|toml|ini|cfg|md|markdown|rst|txt|tex|bib|csv|html|css|scss|njk|ejs|vue|svelte|dockerfile|mk|cmake|proto|graphql|env\.example)$/i;20const NAME_OK = /^(Dockerfile|Makefile|README|LICENSE|CMakeLists\.txt|\.gitignore|\.env\.example)$/i;21const SKIP_RE = /(^|\/)(node_modules|vendor|dist|build|\.git|coverage|__pycache__|\.venv)\//;2223/** Lower-case word tokens (≥ 2 chars), camelCase split. */24export function tokenize(text) {25  return String(text)26    .replace(/([a-z0-9])([A-Z])/g, '$1 $2')27    .toLowerCase()28    .split(/[^a-z0-9_]+/)29    .filter((t) => t.length >= 2 && t.length <= 40);30}3132/** Split file content into overlapping line chunks. */33export function chunkFile(path, content) {34  const lines = content.split('\n');35  const chunks = [];36  for (let start = 0; start < lines.length; start += CHUNK_LINES - CHUNK_OVERLAP) {37    const end = Math.min(lines.length, start + CHUNK_LINES);38    const text = lines.slice(start, end).join('\n');39    if (text.trim()) chunks.push({ path, start: start + 1, end, text });40    if (end >= lines.length) break;41  }42  return chunks;43}4445/** @returns {boolean} whether the path looks like indexable text */46export function indexable(path, size) {47  if (size > MAX_FILE_BYTES || SKIP_RE.test(path)) return false;48  const base = path.slice(path.lastIndexOf('/') + 1);49  return TEXT_EXT.test(base) || NAME_OK.test(base);50}5152/**53 * Build (or load) the chunk index of a repo at a commit. Chunks are cached per54 * blob sha under `global/ai-blobs/<sha>.json`, so a new commit only re-reads55 * files whose content changed.56 * @param {{repos: object, cache: object}} ctx57 * @returns {Promise<{sha: string, chunks: Array<{path: string, start: number, end: number, text: string, tf: Record<string, number>, len: number}>, df: Record<string, number>, avgLen: number, files: number, builtAt: string}>}58 */59export async function buildRepoIndex(ctx, name, sha) {60  const cachePath = ctx.cache.repoPath(name, sha, 'ai-index-v1.json');61  const cached = ctx.cache.getJSON(cachePath);62  if (cached) return cached;63  const listing = await ctx.repos.tryGit(name, ['ls-tree', '-r', '-l', '-z', sha]);64  const entries = (listing ?? '').split('\0').filter(Boolean).map((rec) => {65    const tab = rec.indexOf('\t');66    const [, type, blobSha, sizeRaw] = rec.slice(0, tab).split(/\s+/);67    return { type, blobSha, size: sizeRaw === '-' ? 0 : Number(sizeRaw), path: rec.slice(tab + 1) };68  }).filter((e) => e.type === 'blob' && indexable(e.path, e.size)).slice(0, MAX_FILES);6970  const perFile = await mapLimit(entries, 4, async (e) => {71    const blobCache = ctx.cache.path('global', 'ai-blobs', `${e.blobSha}.json`);72    const hit = ctx.cache.getJSON(blobCache);73    if (hit) return hit.map((c) => ({ ...c, path: e.path }));74    const blob = await ctx.repos.blob(name, sha, e.path, { maxBytes: MAX_FILE_BYTES });75    if (!blob?.content || blob.binary) return [];76    const chunks = chunkFile(e.path, blob.content.toString('utf8')).map((c) => {77      const tf = {};78      const toks = tokenize(`${c.path} ${c.text}`);79      for (const t of toks) tf[t] = (tf[t] ?? 0) + 1;80      return { ...c, tf, len: toks.length };81    });82    ctx.cache.setJSON(blobCache, chunks.map(({ path: _p, ...rest }) => rest));83    return chunks;84  });85  const chunks = perFile.flat();86  const df = {};87  for (const c of chunks) for (const t of Object.keys(c.tf)) df[t] = (df[t] ?? 0) + 1;88  const avgLen = chunks.length ? chunks.reduce((n, c) => n + c.len, 0) / chunks.length : 1;89  const index = { sha, chunks, df, avgLen, files: entries.length, builtAt: new Date().toISOString() };90  ctx.cache.setJSON(cachePath, index);91  return index;92}9394/**95 * BM25 retrieval over the chunk index.96 * @returns {Array<{path: string, start: number, end: number, text: string, score: number}>}97 */98export function retrieve(index, query, k = 8) {99  const terms = [...new Set(tokenize(query))];100  if (!terms.length || !index.chunks.length) return [];101  const N = index.chunks.length;102  const k1 = 1.4;103  const b = 0.75;104  const scored = [];105  for (const c of index.chunks) {106    let score = 0;107    for (const t of terms) {108      const f = c.tf[t];109      if (!f) continue;110      const n = index.df[t] ?? 0;111      const idf = Math.log(1 + (N - n + 0.5) / (n + 0.5));112      score += idf * ((f * (k1 + 1)) / (f + k1 * (1 - b + (b * c.len) / index.avgLen)));113    }114    if (score > 0) scored.push({ path: c.path, start: c.start, end: c.end, text: c.text, score });115  }116  scored.sort((a, b2) => b2.score - a.score);117  return scored.slice(0, k);118}119