Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatDataTests/KnowledgeIndexerTests.swift3// Description: Indexing pipeline and search engine: dedupe, progress, failure, cancellation, re-indexing with another embedder, privacy guard, keyword fallback, Apple embeddings end to end.4import Foundation5import LocalChatCore6import NaturalLanguage7import Testing8@testable import LocalChatKnowledge910@Suite struct KnowledgeIndexerTests {11 static let handbook = """12 # Handbook1314 ## Vacation1516 Employees receive twenty five days of paid vacation every year. Unused days carry over until March.1718 ## Security1920 Laptops must use full disk encryption. Report a lost badge to the security desk within one hour.2122 ## Kitchen2324 The espresso machine is descaled every Friday. Please label your food in the fridge.25 """2627 func setUp(chunkSize: Int = 160) throws -> (KnowledgeStore, KnowledgeIndexer, KnowledgeCollection) {28 let store = try KnowledgeStore.inMemory()29 let c = KnowledgeCollection(name: "HR", embedderID: "fake.bow", dimension: 64, chunkSize: chunkSize, chunkOverlap: 20)30 try store.saveCollection(c)31 return (store, KnowledgeIndexer(store: store, batchSize: 2), c)32 }3334 @Test func addsDeduplicatesAndReportsProgress() async throws {35 let (store, indexer, c) = try setUp(); let embedder = FakeEmbedder()36 let updates = await indexer.progressUpdates()37 let doc = try await indexer.add(data: Data(Self.handbook.utf8), name: "handbook.md", to: c.id, provider: embedder)38 #expect(doc.status == .ready && doc.kind == "md" && doc.chunkCount >= 3 && doc.byteCount == Self.handbook.utf8.count && doc.contentHash.count == 64)39 #expect(try store.counts().vectors == doc.chunkCount)40 #expect(embedder.calls == (doc.chunkCount + 1) / 2) // batches of two41 let again = try await indexer.add(data: Data(Self.handbook.utf8), name: "copy.md", to: c.id, provider: embedder)42 #expect(again.id == doc.id)43 #expect(try store.documents(in: c.id).count == 1)44 var stages: [IndexingProgress.Stage] = []; var maxDone = 045 for await event in updates { stages.append(event.stage); maxDone = max(maxDone, event.chunksDone); if event.stage == .duplicate { break } }46 #expect(stages.first == .extracting && stages.contains(.chunking) && stages.contains(.embedding) && stages.contains(.writing) && stages.contains(.finished))47 #expect(maxDone == doc.chunkCount)48 let chunks = try store.chunks(of: doc.id)49 #expect(chunks.contains { $0.heading == "Handbook › Security" && $0.text.contains("encryption") })50 // a file on disk goes through the same path51 let url = FileManager.default.temporaryDirectory.appendingPathComponent("lc-\(UUID().uuidString).txt")52 try Data("The parking garage closes at midnight.".utf8).write(to: url); defer { try? FileManager.default.removeItem(at: url) }53 #expect(try await indexer.add(fileURL: url, to: c.id, provider: embedder).status == .ready)54 }5556 @Test func failuresAreRecordedAndCanBeRetried() async throws {57 let (store, indexer, c) = try setUp()58 await #expect(throws: KnowledgeIndexingError.self) { try await indexer.add(data: Data(Self.handbook.utf8), name: "h.md", to: c.id, provider: FakeEmbedder(failing: true)) }59 let failed = try #require(try store.documents(in: c.id).first)60 #expect(failed.status == .failed && failed.errorText?.contains("embedding model failed") == true && failed.chunkCount == 0)61 #expect(try store.counts().chunks == 0)62 await #expect(throws: ExtractionError.self) { try await indexer.add(data: Data([0xFF, 0x00, 0x00, 0x01, 0x00, 0x00]), name: "broken.pdf", to: c.id, provider: FakeEmbedder()) }63 let retried = try await indexer.add(data: Data(Self.handbook.utf8), name: "h.md", to: c.id, provider: FakeEmbedder())64 #expect(retried.id == failed.id && retried.status == .ready && retried.errorText == nil)65 await #expect(throws: KnowledgeStoreError.collectionNotFound) { try await indexer.add(data: Data("x".utf8), name: "x.txt", to: UUID(), provider: FakeEmbedder()) }66 }6768 @Test func cancellationLeavesNothingBehind() async throws {69 let (store, indexer, c) = try setUp(chunkSize: 60)70 let slow = FakeEmbedder(delay: .milliseconds(80))71 let task = Task { try await indexer.add(data: Data(Self.handbook.utf8), name: "h.md", to: c.id, provider: slow) }72 try await Task.sleep(for: .milliseconds(120))73 task.cancel()74 await #expect(throws: CancellationError.self) { try await task.value }75 #expect(try store.counts() == StoreCounts(collections: 1, documents: 0, sections: 0, chunks: 0, vectors: 0, ftsRows: 0))76 #expect(slow.calls >= 1)77 }7879 @Test func embeddersAreNeverMixedAndReindexSwitchesAtomically() async throws {80 let (store, indexer, c) = try setUp()81 let small = FakeEmbedder(identifier: "fake.small", dimension: 32)82 let doc = try await indexer.add(data: Data(Self.handbook.utf8), name: "h.md", to: c.id, provider: small) // empty collection adopts the embedder83 #expect(try store.collection(c.id)?.embedderID == "fake.small" && store.collection(c.id)?.dimension == 32)84 let big = FakeEmbedder(identifier: "fake.big", dimension: 96)85 await #expect(throws: KnowledgeIndexingError.embedderMismatch(collection: "fake.small", provider: "fake.big")) {86 try await indexer.add(data: Data("another file".utf8), name: "b.txt", to: c.id, provider: big)87 }88 await #expect(throws: KnowledgeIndexingError.self) { try await indexer.reindex(documentID: doc.id, provider: big) }89 // a failing re-index keeps the old index intact90 await #expect(throws: KnowledgeIndexingError.self) { try await indexer.reindex(collectionID: c.id, provider: FakeEmbedder(identifier: "fake.bad", dimension: 16, failing: true)) }91 #expect(try store.collection(c.id)?.embedderID == "fake.small")92 #expect(try store.vectorSearch(collection: c.id, query: FakeEmbedder.vector("vacation days", dimension: 32), limit: 1).count == 1)9394 try await indexer.reindex(collectionID: c.id, provider: big)95 let updated = try #require(try store.collection(c.id))96 #expect(updated.embedderID == "fake.big" && updated.dimension == 96)97 #expect(try store.counts().vectors == doc.chunkCount && store.counts().documents == 1)98 #expect(try store.vectorSearch(collection: c.id, query: FakeEmbedder.vector("vacation days", dimension: 32), limit: 1).isEmpty)99 let hit = try #require(try store.hits(for: store.vectorSearch(collection: c.id, query: FakeEmbedder.vector("paid vacation days", dimension: 96), limit: 1)).first)100 #expect(hit.text.contains("vacation"))101102 // new chunk settings are applied when a document is re-indexed from its stored text103 var resized = updated; resized.chunkSize = 60; resized.chunkOverlap = 0; try store.saveCollection(resized)104 let rechunked = try await indexer.reindex(documentID: doc.id, provider: big)105 #expect(rechunked.chunkCount > doc.chunkCount && rechunked.status == .ready)106 #expect(try store.chunks(of: doc.id).allSatisfy { $0.text.count <= 60 })107 }108109 @Test func remoteEmbeddersNeedExplicitConsent() async throws {110 let (store, indexer, c) = try setUp(); let remote = FakeEmbedder(identifier: "remote.embed", isRemote: true)111 await #expect(throws: KnowledgeIndexingError.remoteProcessingNotAllowed("remote.embed")) { try await indexer.add(data: Data(Self.handbook.utf8), name: "h.md", to: c.id, provider: remote) }112 #expect(remote.calls == 0)113 #expect(try store.counts().documents == 0)114 await #expect(throws: KnowledgeIndexingError.self) { try await indexer.reindex(collectionID: c.id, provider: remote) }115 let doc = try await indexer.add(data: Data(Self.handbook.utf8), name: "h.md", to: c.id, provider: remote, allowRemoteProcessing: true)116 #expect(doc.status == .ready)117 // queries are not sent to a remote embedder either unless allowed118 let guarded = KnowledgeEngine(store: store) { _ in remote }119 let before = remote.calls120 let hits = try await guarded.search("disk encryption", collections: [c.id], limit: 3)121 #expect(remote.calls == before && guarded.lastSearchMode == .keywordOnly && hits.first?.text.contains("encryption") == true)122 #expect(guarded.lastSearchNotes.first?.contains("remote") == true)123 let allowed = KnowledgeEngine(store: store, allowRemoteProcessing: true) { _ in remote }124 _ = try await allowed.search("disk encryption", collections: [c.id], limit: 3)125 #expect(remote.calls == before + 1 && allowed.lastSearchMode == .hybrid)126 }127128 @Test func engineSearchesHybridAndFallsBackToKeywords() async throws {129 let (store, indexer, c) = try setUp(); let embedder = FakeEmbedder()130 try await indexer.add(data: Data(Self.handbook.utf8), name: "handbook.md", to: c.id, provider: embedder)131 let engine = KnowledgeEngine(store: store) { id in id == "fake.bow" ? embedder : nil }132 #expect(engine.lastSearchMode == .none)133 let hits = try await engine.search("how many vacation days do employees get", collections: [c.id], limit: 3)134 #expect(engine.lastSearchMode == .hybrid && engine.lastSearchNotes.isEmpty)135 #expect(hits.first?.text.contains("twenty five days") == true && hits.first?.document.name == "handbook.md")136 #expect(hits.first?.reference.documentName == "handbook.md")137 let offline = KnowledgeEngine(store: store) { _ in nil }138 let fallback = try await offline.search("espresso machine", collections: [c.id], limit: 3)139 #expect(offline.lastSearchMode == .keywordOnly && offline.lastSearchNotes.count == 1)140 #expect(fallback.first?.text.contains("espresso") == true)141 #expect(try await engine.search(" ", collections: [c.id], limit: 3).isEmpty)142 #expect(try await engine.search("vacation", collections: [], limit: 3).isEmpty)143 }144145 @Test func appleSentenceEmbeddingsEndToEnd() async throws {146 let languages = AppleNLEmbeddingProvider.availableLanguages()147 print("KNOWLEDGE-NL available sentence embeddings: \(languages.map { "\($0.rawValue)(\(NLEmbedding.sentenceEmbedding(for: $0)?.dimension ?? 0))" }.joined(separator: " "))")148 guard let provider = AppleNLEmbeddingProvider(language: .english) else { print("KNOWLEDGE-NL English sentence embedding unavailable on this host; end-to-end test skipped"); return }149 #expect(provider.dimension > 0 && provider.identifier == "apple.nl.sentence.en" && !provider.isRemote)150 #expect(AppleNLEmbeddingProvider(identifier: provider.identifier)?.dimension == provider.dimension)151 #expect(AppleNLEmbeddingProvider(identifier: "mlx:something") == nil)152 #expect(AppleNLEmbeddingProvider.best(forSample: "The quick brown fox jumps over the lazy dog near the river.")?.language == .english)153 let store = try KnowledgeStore.inMemory()154 let c = KnowledgeCollection(name: "Notes", embedderID: provider.identifier, dimension: provider.dimension, chunkSize: 80, chunkOverlap: 0)155 try store.saveCollection(c)156 let notes = """157 The cat sat quietly on the warm windowsill watching birds.158159 Interest rates were raised by the central bank to fight inflation.160161 The recipe needs two cups of flour, three eggs and a pinch of salt.162163 The rocket reached orbit eight minutes after leaving the launch pad.164 """165 let indexer = KnowledgeIndexer(store: store)166 let doc = try await indexer.add(data: Data(notes.utf8), name: "notes.txt", to: c.id, provider: provider)167 #expect(doc.chunkCount == 4)168 let engine = KnowledgeEngine(store: store, mmrLambda: nil) { AppleNLEmbeddingProvider(identifier: $0) }169 // semantic matches with no keyword in common (vector search only decides)170 for (query, expected) in [("monetary policy and the economy", "central bank"), ("a spacecraft launch", "rocket"), ("baking ingredients", "flour"), ("a kitten resting", "cat")] {171 let vector = try await provider.embed([query])[0]172 let top = try store.hits(for: store.vectorSearch(collection: c.id, query: vector, limit: 1)).first173 #expect(top?.text.contains(expected) == true, "\(query) → \(top?.text ?? "nil")")174 }175 let hits = try await engine.search("How do central banks fight inflation?", collections: [c.id], limit: 2)176 #expect(engine.lastSearchMode == .hybrid && hits.first?.text.contains("central bank") == true)177 await #expect(throws: AppleNLEmbeddingError.noVectors) { try await provider.embed([" "]) }178 }179}180