/** * KHAELOR * File: src/repograph/extractor.ts * Description: Symbol/import extraction for TS/JS/Python — dependency-free heuristic parser (tree-sitter is the documented upgrade path) (v2 design §3). * * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai */ export type SymbolKind = "function" | "class" | "type" | "export" | "variable" | "method"; export interface ExtractedSymbol { name: string; kind: SymbolKind; /** 1-based line of the declaration. */ line: number; /** The declaration line, trimmed. */ signature: string; /** First line of the preceding doc comment, when present. */ docComment?: string; exported: boolean; } export interface ExtractedImport { /** Module specifier as written: "./kernel.js", "node:path", "react". */ spec: string; /** Imported names ("default" for default imports, "*" for namespace). */ names: string[]; line: number; } export interface ExtractionResult { symbols: ExtractedSymbol[]; imports: ExtractedImport[]; } const TS_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"]); /** Extensions the extractor understands. */ export function isIndexablePath(path: string): boolean { const dot = path.lastIndexOf("."); if (dot === -1) return false; const ext = path.slice(dot); return TS_EXTENSIONS.has(ext) || ext === ".py"; } const MAX_SIGNATURE = 160; function clip(line: string): string { const trimmed = line.trim(); return trimmed.length > MAX_SIGNATURE ? `${trimmed.slice(0, MAX_SIGNATURE)}…` : trimmed; } /** Extract the first sentence of a `/** … *\/` block ending just above `index`. */ function docCommentAbove(lines: readonly string[], index: number): string | undefined { let i = index - 1; while (i >= 0 && (lines[i] as string).trim().length === 0) i -= 1; if (i < 0) return undefined; const above = (lines[i] as string).trim(); if (above.endsWith("*/")) { // Walk up to the /** opener collecting the first content line. for (let j = i; j >= 0 && j > i - 20; j -= 1) { const candidate = (lines[j] as string).trim(); if (candidate.startsWith("/**")) { const inline = candidate.replace(/^\/\*\*\s*/, "").replace(/\s*\*\/$/, ""); if (inline.length > 0) return clip(inline); const next = (lines[j + 1] as string | undefined)?.trim().replace(/^\*\s?/, ""); return next !== undefined && next.length > 0 ? clip(next) : undefined; } } } if (above.startsWith("//")) return clip(above.replace(/^\/\/\s?/, "")); if (above.startsWith("#")) return clip(above.replace(/^#\s?/, "")); return undefined; } // TS/JS declaration patterns — anchored to line starts, tolerant of export modifiers. const TS_PATTERNS: { re: RegExp; kind: SymbolKind }[] = [ { re: /^(export\s+)?(default\s+)?(async\s+)?function\s*\*?\s+([A-Za-z_$][\w$]*)/, kind: "function" }, { re: /^(export\s+)?(default\s+)?(abstract\s+)?class\s+([A-Za-z_$][\w$]*)/, kind: "class" }, { re: /^(export\s+)?interface\s+([A-Za-z_$][\w$]*)/, kind: "type" }, { re: /^(export\s+)?type\s+([A-Za-z_$][\w$]*)\s*=/, kind: "type" }, { re: /^(export\s+)?enum\s+([A-Za-z_$][\w$]*)/, kind: "type" }, { re: /^(export\s+)?const\s+([A-Za-z_$][\w$]*)\s*(?::[^=]+)?=/, kind: "variable" }, { re: /^(export\s+)?let\s+([A-Za-z_$][\w$]*)\s*(?::[^=]+)?=/, kind: "variable" }, ]; const TS_METHOD = /^(?:public\s+|private\s+|protected\s+|static\s+|readonly\s+)*(?:async\s+)?(?:get\s+|set\s+)?([A-Za-z_$][\w$]*)\s*(?:<[^>]*>)?\([^;]*\)\s*(?::[^{;]+)?\{\s*$/; const TS_KEYWORD_NOT_METHOD = new Set([ "if", "for", "while", "switch", "catch", "return", "function", "constructor", "new", "typeof", "else", "do", "try", ]); const TS_IMPORT = /^import\s+(?:type\s+)?(.+?)\s+from\s+["']([^"']+)["']/; const TS_SIDE_EFFECT_IMPORT = /^import\s+["']([^"']+)["']/; const TS_EXPORT_LIST = /^export\s*\{([^}]*)\}/; function parseImportNames(clause: string): string[] { const names: string[] = []; const braces = /\{([^}]*)\}/.exec(clause); if (braces !== null) { for (const part of (braces[1] as string).split(",")) { const name = part.trim().split(/\s+as\s+/)[0]?.trim(); if (name !== undefined && name.length > 0) names.push(name); } } const withoutBraces = clause.replace(/\{[^}]*\}/, "").trim(); if (withoutBraces.startsWith("* as")) names.push("*"); else { const first = withoutBraces.split(",")[0]?.trim(); if (first !== undefined && first.length > 0 && first !== "*") names.push("default"); } return names; } function extractTs(lines: readonly string[]): ExtractionResult { const symbols: ExtractedSymbol[] = []; const imports: ExtractedImport[] = []; let braceDepth = 0; let inClassAtDepth = -1; for (let i = 0; i < lines.length; i += 1) { const raw = lines[i] as string; const line = raw.trim(); const importMatch = TS_IMPORT.exec(line); if (importMatch !== null) { imports.push({ spec: importMatch[2] as string, names: parseImportNames(importMatch[1] as string), line: i + 1, }); } else { const sideEffect = TS_SIDE_EFFECT_IMPORT.exec(line); if (sideEffect !== null) { imports.push({ spec: sideEffect[1] as string, names: [], line: i + 1 }); } } if (braceDepth === 0) { const exportList = TS_EXPORT_LIST.exec(line); if (exportList !== null && !line.includes(" from ")) { for (const part of (exportList[1] as string).split(",")) { const name = part.trim().split(/\s+as\s+/)[0]?.trim(); if (name !== undefined && name.length > 0) { symbols.push({ name, kind: "export", line: i + 1, signature: clip(line), exported: true }); } } } for (const pattern of TS_PATTERNS) { const match = pattern.re.exec(line); if (match !== null) { const name = match[match.length - 1] as string; const doc = docCommentAbove(lines, i); symbols.push({ name, kind: pattern.kind, line: i + 1, signature: clip(line), ...(doc !== undefined ? { docComment: doc } : {}), exported: /^export\b/.test(line), }); if (pattern.kind === "class") inClassAtDepth = braceDepth; break; } } } else if (braceDepth === 1 && inClassAtDepth === 0) { const method = TS_METHOD.exec(line); if (method !== null) { const name = method[1] as string; if (!TS_KEYWORD_NOT_METHOD.has(name)) { const doc = docCommentAbove(lines, i); symbols.push({ name, kind: "method", line: i + 1, signature: clip(line), ...(doc !== undefined ? { docComment: doc } : {}), exported: false, }); } } } // Cheap brace tracking, ignoring string/comment contents well enough for indexing. for (const ch of raw) { if (ch === "{") braceDepth += 1; else if (ch === "}") braceDepth = Math.max(0, braceDepth - 1); } if (braceDepth === 0) inClassAtDepth = -1; } return { symbols, imports }; } const PY_DEF = /^(\s*)def\s+([A-Za-z_]\w*)/; const PY_CLASS = /^(\s*)class\s+([A-Za-z_]\w*)/; const PY_IMPORT = /^import\s+([\w.]+)/; const PY_FROM_IMPORT = /^from\s+([\w.]+)\s+import\s+(.+)/; function extractPy(lines: readonly string[]): ExtractionResult { const symbols: ExtractedSymbol[] = []; const imports: ExtractedImport[] = []; for (let i = 0; i < lines.length; i += 1) { const raw = lines[i] as string; const def = PY_DEF.exec(raw); if (def !== null) { const indent = (def[1] as string).length; const doc = docCommentAbove(lines, i); symbols.push({ name: def[2] as string, kind: indent > 0 ? "method" : "function", line: i + 1, signature: clip(raw), ...(doc !== undefined ? { docComment: doc } : {}), exported: indent === 0 && !(def[2] as string).startsWith("_"), }); continue; } const cls = PY_CLASS.exec(raw); if (cls !== null) { const doc = docCommentAbove(lines, i); symbols.push({ name: cls[2] as string, kind: "class", line: i + 1, signature: clip(raw), ...(doc !== undefined ? { docComment: doc } : {}), exported: !(cls[2] as string).startsWith("_"), }); continue; } const from = PY_FROM_IMPORT.exec(raw.trim()); if (from !== null) { const names = (from[2] as string).split(",").map((n) => n.trim().split(/\s+as\s+/)[0] ?? ""); imports.push({ spec: from[1] as string, names: names.filter((n) => n.length > 0), line: i + 1 }); continue; } const imp = PY_IMPORT.exec(raw.trim()); if (imp !== null) { imports.push({ spec: imp[1] as string, names: ["*"], line: i + 1 }); } } return { symbols, imports }; } /** Extract symbols and imports from one file's content. */ export function extractFile(path: string, content: string): ExtractionResult { const lines = content.split("\n"); if (path.endsWith(".py")) return extractPy(lines); return extractTs(lines); }