SPB Git

Repository Public

KnowledgeIndexerTests.swift 12.6 KB 180 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatDataTests/KnowledgeIndexerTests.swift3// Description: Indexing pipeline and search engine: dedupe, progress, failure, cancellation, re-indexing with another embedder, privacy guard, keyword fallback, Apple embeddings end to end.4import Foundation5import LocalChatCore6import NaturalLanguage7import Testing8@testable import LocalChatKnowledge910@Suite struct KnowledgeIndexerTests {11    static let handbook = """12    # Handbook1314    ## Vacation1516    Employees receive twenty five days of paid vacation every year. Unused days carry over until March.1718    ## Security1920    Laptops must use full disk encryption. Report a lost badge to the security desk within one hour.2122    ## Kitchen2324    The espresso machine is descaled every Friday. Please label your food in the fridge.25    """2627    func setUp(chunkSize: Int = 160) throws -> (KnowledgeStore, KnowledgeIndexer, KnowledgeCollection) {28        let store = try KnowledgeStore.inMemory()29        let c = KnowledgeCollection(name: "HR", embedderID: "fake.bow", dimension: 64, chunkSize: chunkSize, chunkOverlap: 20)30        try store.saveCollection(c)31        return (store, KnowledgeIndexer(store: store, batchSize: 2), c)32    }3334    @Test func addsDeduplicatesAndReportsProgress() async throws {35        let (store, indexer, c) = try setUp(); let embedder = FakeEmbedder()36        let updates = await indexer.progressUpdates()37        let doc = try await indexer.add(data: Data(Self.handbook.utf8), name: "handbook.md", to: c.id, provider: embedder)38        #expect(doc.status == .ready && doc.kind == "md" && doc.chunkCount >= 3 && doc.byteCount == Self.handbook.utf8.count && doc.contentHash.count == 64)39        #expect(try store.counts().vectors == doc.chunkCount)40        #expect(embedder.calls == (doc.chunkCount + 1) / 2)       // batches of two41        let again = try await indexer.add(data: Data(Self.handbook.utf8), name: "copy.md", to: c.id, provider: embedder)42        #expect(again.id == doc.id)43        #expect(try store.documents(in: c.id).count == 1)44        var stages: [IndexingProgress.Stage] = []; var maxDone = 045        for await event in updates { stages.append(event.stage); maxDone = max(maxDone, event.chunksDone); if event.stage == .duplicate { break } }46        #expect(stages.first == .extracting && stages.contains(.chunking) && stages.contains(.embedding) && stages.contains(.writing) && stages.contains(.finished))47        #expect(maxDone == doc.chunkCount)48        let chunks = try store.chunks(of: doc.id)49        #expect(chunks.contains { $0.heading == "Handbook › Security" && $0.text.contains("encryption") })50        // a file on disk goes through the same path51        let url = FileManager.default.temporaryDirectory.appendingPathComponent("lc-\(UUID().uuidString).txt")52        try Data("The parking garage closes at midnight.".utf8).write(to: url); defer { try? FileManager.default.removeItem(at: url) }53        #expect(try await indexer.add(fileURL: url, to: c.id, provider: embedder).status == .ready)54    }5556    @Test func failuresAreRecordedAndCanBeRetried() async throws {57        let (store, indexer, c) = try setUp()58        await #expect(throws: KnowledgeIndexingError.self) { try await indexer.add(data: Data(Self.handbook.utf8), name: "h.md", to: c.id, provider: FakeEmbedder(failing: true)) }59        let failed = try #require(try store.documents(in: c.id).first)60        #expect(failed.status == .failed && failed.errorText?.contains("embedding model failed") == true && failed.chunkCount == 0)61        #expect(try store.counts().chunks == 0)62        await #expect(throws: ExtractionError.self) { try await indexer.add(data: Data([0xFF, 0x00, 0x00, 0x01, 0x00, 0x00]), name: "broken.pdf", to: c.id, provider: FakeEmbedder()) }63        let retried = try await indexer.add(data: Data(Self.handbook.utf8), name: "h.md", to: c.id, provider: FakeEmbedder())64        #expect(retried.id == failed.id && retried.status == .ready && retried.errorText == nil)65        await #expect(throws: KnowledgeStoreError.collectionNotFound) { try await indexer.add(data: Data("x".utf8), name: "x.txt", to: UUID(), provider: FakeEmbedder()) }66    }6768    @Test func cancellationLeavesNothingBehind() async throws {69        let (store, indexer, c) = try setUp(chunkSize: 60)70        let slow = FakeEmbedder(delay: .milliseconds(80))71        let task = Task { try await indexer.add(data: Data(Self.handbook.utf8), name: "h.md", to: c.id, provider: slow) }72        try await Task.sleep(for: .milliseconds(120))73        task.cancel()74        await #expect(throws: CancellationError.self) { try await task.value }75        #expect(try store.counts() == StoreCounts(collections: 1, documents: 0, sections: 0, chunks: 0, vectors: 0, ftsRows: 0))76        #expect(slow.calls >= 1)77    }7879    @Test func embeddersAreNeverMixedAndReindexSwitchesAtomically() async throws {80        let (store, indexer, c) = try setUp()81        let small = FakeEmbedder(identifier: "fake.small", dimension: 32)82        let doc = try await indexer.add(data: Data(Self.handbook.utf8), name: "h.md", to: c.id, provider: small)   // empty collection adopts the embedder83        #expect(try store.collection(c.id)?.embedderID == "fake.small" && store.collection(c.id)?.dimension == 32)84        let big = FakeEmbedder(identifier: "fake.big", dimension: 96)85        await #expect(throws: KnowledgeIndexingError.embedderMismatch(collection: "fake.small", provider: "fake.big")) {86            try await indexer.add(data: Data("another file".utf8), name: "b.txt", to: c.id, provider: big)87        }88        await #expect(throws: KnowledgeIndexingError.self) { try await indexer.reindex(documentID: doc.id, provider: big) }89        // a failing re-index keeps the old index intact90        await #expect(throws: KnowledgeIndexingError.self) { try await indexer.reindex(collectionID: c.id, provider: FakeEmbedder(identifier: "fake.bad", dimension: 16, failing: true)) }91        #expect(try store.collection(c.id)?.embedderID == "fake.small")92        #expect(try store.vectorSearch(collection: c.id, query: FakeEmbedder.vector("vacation days", dimension: 32), limit: 1).count == 1)9394        try await indexer.reindex(collectionID: c.id, provider: big)95        let updated = try #require(try store.collection(c.id))96        #expect(updated.embedderID == "fake.big" && updated.dimension == 96)97        #expect(try store.counts().vectors == doc.chunkCount && store.counts().documents == 1)98        #expect(try store.vectorSearch(collection: c.id, query: FakeEmbedder.vector("vacation days", dimension: 32), limit: 1).isEmpty)99        let hit = try #require(try store.hits(for: store.vectorSearch(collection: c.id, query: FakeEmbedder.vector("paid vacation days", dimension: 96), limit: 1)).first)100        #expect(hit.text.contains("vacation"))101102        // new chunk settings are applied when a document is re-indexed from its stored text103        var resized = updated; resized.chunkSize = 60; resized.chunkOverlap = 0; try store.saveCollection(resized)104        let rechunked = try await indexer.reindex(documentID: doc.id, provider: big)105        #expect(rechunked.chunkCount > doc.chunkCount && rechunked.status == .ready)106        #expect(try store.chunks(of: doc.id).allSatisfy { $0.text.count <= 60 })107    }108109    @Test func remoteEmbeddersNeedExplicitConsent() async throws {110        let (store, indexer, c) = try setUp(); let remote = FakeEmbedder(identifier: "remote.embed", isRemote: true)111        await #expect(throws: KnowledgeIndexingError.remoteProcessingNotAllowed("remote.embed")) { try await indexer.add(data: Data(Self.handbook.utf8), name: "h.md", to: c.id, provider: remote) }112        #expect(remote.calls == 0)113        #expect(try store.counts().documents == 0)114        await #expect(throws: KnowledgeIndexingError.self) { try await indexer.reindex(collectionID: c.id, provider: remote) }115        let doc = try await indexer.add(data: Data(Self.handbook.utf8), name: "h.md", to: c.id, provider: remote, allowRemoteProcessing: true)116        #expect(doc.status == .ready)117        // queries are not sent to a remote embedder either unless allowed118        let guarded = KnowledgeEngine(store: store) { _ in remote }119        let before = remote.calls120        let hits = try await guarded.search("disk encryption", collections: [c.id], limit: 3)121        #expect(remote.calls == before && guarded.lastSearchMode == .keywordOnly && hits.first?.text.contains("encryption") == true)122        #expect(guarded.lastSearchNotes.first?.contains("remote") == true)123        let allowed = KnowledgeEngine(store: store, allowRemoteProcessing: true) { _ in remote }124        _ = try await allowed.search("disk encryption", collections: [c.id], limit: 3)125        #expect(remote.calls == before + 1 && allowed.lastSearchMode == .hybrid)126    }127128    @Test func engineSearchesHybridAndFallsBackToKeywords() async throws {129        let (store, indexer, c) = try setUp(); let embedder = FakeEmbedder()130        try await indexer.add(data: Data(Self.handbook.utf8), name: "handbook.md", to: c.id, provider: embedder)131        let engine = KnowledgeEngine(store: store) { id in id == "fake.bow" ? embedder : nil }132        #expect(engine.lastSearchMode == .none)133        let hits = try await engine.search("how many vacation days do employees get", collections: [c.id], limit: 3)134        #expect(engine.lastSearchMode == .hybrid && engine.lastSearchNotes.isEmpty)135        #expect(hits.first?.text.contains("twenty five days") == true && hits.first?.document.name == "handbook.md")136        #expect(hits.first?.reference.documentName == "handbook.md")137        let offline = KnowledgeEngine(store: store) { _ in nil }138        let fallback = try await offline.search("espresso machine", collections: [c.id], limit: 3)139        #expect(offline.lastSearchMode == .keywordOnly && offline.lastSearchNotes.count == 1)140        #expect(fallback.first?.text.contains("espresso") == true)141        #expect(try await engine.search("   ", collections: [c.id], limit: 3).isEmpty)142        #expect(try await engine.search("vacation", collections: [], limit: 3).isEmpty)143    }144145    @Test func appleSentenceEmbeddingsEndToEnd() async throws {146        let languages = AppleNLEmbeddingProvider.availableLanguages()147        print("KNOWLEDGE-NL available sentence embeddings: \(languages.map { "\($0.rawValue)(\(NLEmbedding.sentenceEmbedding(for: $0)?.dimension ?? 0))" }.joined(separator: " "))")148        guard let provider = AppleNLEmbeddingProvider(language: .english) else { print("KNOWLEDGE-NL English sentence embedding unavailable on this host; end-to-end test skipped"); return }149        #expect(provider.dimension > 0 && provider.identifier == "apple.nl.sentence.en" && !provider.isRemote)150        #expect(AppleNLEmbeddingProvider(identifier: provider.identifier)?.dimension == provider.dimension)151        #expect(AppleNLEmbeddingProvider(identifier: "mlx:something") == nil)152        #expect(AppleNLEmbeddingProvider.best(forSample: "The quick brown fox jumps over the lazy dog near the river.")?.language == .english)153        let store = try KnowledgeStore.inMemory()154        let c = KnowledgeCollection(name: "Notes", embedderID: provider.identifier, dimension: provider.dimension, chunkSize: 80, chunkOverlap: 0)155        try store.saveCollection(c)156        let notes = """157        The cat sat quietly on the warm windowsill watching birds.158159        Interest rates were raised by the central bank to fight inflation.160161        The recipe needs two cups of flour, three eggs and a pinch of salt.162163        The rocket reached orbit eight minutes after leaving the launch pad.164        """165        let indexer = KnowledgeIndexer(store: store)166        let doc = try await indexer.add(data: Data(notes.utf8), name: "notes.txt", to: c.id, provider: provider)167        #expect(doc.chunkCount == 4)168        let engine = KnowledgeEngine(store: store, mmrLambda: nil) { AppleNLEmbeddingProvider(identifier: $0) }169        // semantic matches with no keyword in common (vector search only decides)170        for (query, expected) in [("monetary policy and the economy", "central bank"), ("a spacecraft launch", "rocket"), ("baking ingredients", "flour"), ("a kitten resting", "cat")] {171            let vector = try await provider.embed([query])[0]172            let top = try store.hits(for: store.vectorSearch(collection: c.id, query: vector, limit: 1)).first173            #expect(top?.text.contains(expected) == true, "\(query) → \(top?.text ?? "nil")")174        }175        let hits = try await engine.search("How do central banks fight inflation?", collections: [c.id], limit: 2)176        #expect(engine.lastSearchMode == .hybrid && hits.first?.text.contains("central bank") == true)177        await #expect(throws: AppleNLEmbeddingError.noVectors) { try await provider.embed(["   "]) }178    }179}180
Files main
Menu

Theme