SPB Git

Repository Public

KnowledgeStoreTests.swift 10.5 KB 146 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatDataTests/KnowledgeStoreTests.swift3// Description: Knowledge store: CRUD, real deletion (row counts), exact vector search against brute force, BM25, hybrid fusion, MMR, 100k-chunk timing.4import Foundation5import LocalChatCore6import Testing7@testable import LocalChatKnowledge89@Suite struct KnowledgeStoreTests {10    func makeCollection(_ store: KnowledgeStore, dimension: Int = 64, name: String = "Docs") throws -> KnowledgeCollection {11        let c = KnowledgeCollection(name: name, embedderID: "fake.bow", dimension: dimension)12        try store.saveCollection(c); return c13    }14    func index(_ store: KnowledgeStore, _ collection: KnowledgeCollection, name: String, texts: [String]) throws -> KnowledgeDocument {15        let doc = KnowledgeDocument(collectionID: collection.id, name: name, kind: "txt", byteCount: 1, contentHash: name)16        let chunks = texts.enumerated().map { TextChunk(index: $0, text: $1, page: $0 + 1, heading: "H\($0)") }17        try store.replaceIndex([.init(document: doc, extracted: ExtractedDocument(kind: .txt, sections: texts.map { .init(text: $0) }), chunks: chunks,18                                      vectors: texts.map { FakeEmbedder.vector($0, dimension: collection.dimension) })])19        return try #require(try store.document(doc.id))20    }2122    @Test func crudAndRealDeletion() throws {23        let url = FileManager.default.temporaryDirectory.appendingPathComponent("lc-know-\(UUID().uuidString)/k.sqlite")24        defer { try? FileManager.default.removeItem(at: url.deletingLastPathComponent()) }25        let store = try KnowledgeStore(url: url)26        var a = try makeCollection(store, name: "Alpha"); let b = try makeCollection(store, name: "Beta")27        a.name = "Alpha 2"; try store.saveCollection(a)28        #expect(try store.collections().map(\.name) == ["Alpha 2", "Beta"])29        let d1 = try index(store, a, name: "one.txt", texts: ["the secret zebra password is marmalade", "bananas are yellow fruit"])30        let d2 = try index(store, a, name: "two.txt", texts: ["quantum computers use qubits"])31        _ = try index(store, b, name: "three.txt", texts: ["zebra stripes confuse lions", "lions sleep a lot", "giraffes are tall"])32        #expect(d1.status == .ready && d1.chunkCount == 2 && d1.characterCount > 0)33        #expect(try store.counts() == StoreCounts(collections: 2, documents: 3, sections: 6, chunks: 6, vectors: 6, ftsRows: 6))34        let stored = try store.chunks(of: d1.id)35        #expect(stored.map(\.index) == [0, 1] && stored[1].page == 2 && stored[1].heading == "H1" && stored.allSatisfy(\.hasVector))36        #expect(try store.extractedDocument(d1.id)?.sections.count == 2)37        #expect(try store.document(inCollection: a.id, contentHash: "two.txt")?.id == d2.id)38        #expect(try store.keywordSearch("zebra", collections: [a.id, b.id], limit: 10).count == 2)3940        try store.deleteDocument(d1.id)41        #expect(try store.counts() == StoreCounts(collections: 2, documents: 2, sections: 4, chunks: 4, vectors: 4, ftsRows: 4))42        #expect(try store.keywordSearch("marmalade", collections: [a.id], limit: 10).isEmpty)43        #expect(try store.keywordSearch("zebra", collections: [a.id, b.id], limit: 10).count == 1)44        #expect(try store.vectorSearch(collection: a.id, query: FakeEmbedder.vector("marmalade zebra", dimension: 64), limit: 5).count == 1)4546        try store.deleteCollection(b.id)47        #expect(try store.counts() == StoreCounts(collections: 1, documents: 1, sections: 1, chunks: 1, vectors: 1, ftsRows: 1))48        #expect(try store.keywordSearch("zebra lions giraffes", collections: [a.id, b.id], limit: 10).isEmpty)49        try store.vacuum()50        // the deleted text is gone from the database file itself51        let raw = try Data(contentsOf: url)52        #expect(raw.range(of: Data("marmalade".utf8)) == nil && raw.range(of: Data("giraffes".utf8)) == nil)53        #expect(raw.range(of: Data("qubits".utf8)) != nil)54    }5556    @Test func rejectsWrongDimensionsAtomically() throws {57        let store = try KnowledgeStore.inMemory(); let c = try makeCollection(store, dimension: 8)58        let doc = KnowledgeDocument(collectionID: c.id, name: "x", kind: "txt", byteCount: 1, contentHash: "x")59        #expect(throws: KnowledgeStoreError.dimensionMismatch(expected: 8, got: 3)) {60            try store.replaceIndex([.init(document: doc, extracted: nil, chunks: [TextChunk(index: 0, text: "a"), TextChunk(index: 1, text: "b")],61                                          vectors: [[Float](repeating: 1, count: 8), [1, 2, 3]])])62        }63        #expect(try store.counts().chunks == 0 && store.counts().documents == 0)64    }6566    @Test func vectorSearchMatchesBruteForce() throws {67        let store = try KnowledgeStore.inMemory(); let dim = 48; let c = try makeCollection(store, dimension: dim)68        var rng = SystemRandomNumberGenerator()69        let vectors = (0..<3000).map { _ in (0..<dim).map { _ in Float.random(in: -1...1, using: &rng) } }70        let doc = KnowledgeDocument(collectionID: c.id, name: "r", kind: "txt", byteCount: 1, contentHash: "r")71        try store.replaceIndex([.init(document: doc, extracted: nil, chunks: vectors.indices.map { TextChunk(index: $0, text: "chunk \($0)") }, vectors: vectors)])72        let rowIDs = try store.chunks(of: doc.id).map(\.rowID)73        for _ in 0..<5 {74            let q = (0..<dim).map { _ in Float.random(in: -1...1, using: &rng) }75            func cosine(_ v: [Float]) -> Double {76                let d = zip(v, q).reduce(0.0) { $0 + Double($1.0) * Double($1.1) }77                return d / (v.reduce(0.0) { $0 + Double($1) * Double($1) }.squareRoot() * q.reduce(0.0) { $0 + Double($1) * Double($1) }.squareRoot())78            }79            let reference = vectors.enumerated().map { (rowIDs[$0], cosine($1)) }.sorted { $0.1 > $1.1 }.prefix(10)80            let found = try store.vectorSearch(collection: c.id, query: q, limit: 10)81            #expect(found.map(\.rowID) == reference.map(\.0))82            for (f, r) in zip(found, reference) { #expect(abs(f.score - r.1) < 1e-4) }83        }84        #expect(try store.vectorSearch(collection: c.id, query: [1, 2], limit: 5).isEmpty)        // wrong dimension85        #expect(try store.vectorSearch(collection: UUID(), query: [1, 2], limit: 5).isEmpty)      // unknown collection86    }8788    @Test func hybridFusionKeywordsAndMMR() throws {89        let store = try KnowledgeStore.inMemory(); let c = try makeCollection(store)90        let texts = ["The warranty covers battery replacement for two years.",91                     "The warranty covers battery replacement for two years.",       // exact duplicate92                     "Shipping takes five business days to Canada.",93                     "Reference code ZX-9000 identifies the premium model.",94                     "Our office dog is called Biscuit."]95        _ = try index(store, c, name: "faq.txt", texts: texts)96        func run(_ q: String, vector: Bool = true, mmr: Double? = nil, limit: Int = 3) throws -> KnowledgeSearchResult {97            try store.search(text: q, queryVectors: vector ? [c.id: FakeEmbedder.vector(q, dimension: 64)] : [:], collections: [c.id], limit: limit, mmrLambda: mmr)98        }99        let hybrid = try run("battery warranty")100        #expect(hybrid.usedVectors && hybrid.usedKeywords)101        #expect(hybrid.hits.first?.text.contains("warranty") == true)102        #expect(hybrid.hits.first?.document.name == "faq.txt" && hybrid.hits.first?.page != nil)103        // a rare identifier is found by keywords even when no vector is available104        let keyword = try run("ZX-9000", vector: false)105        #expect(!keyword.usedVectors && keyword.usedKeywords && keyword.hits.first?.text.contains("ZX-9000") == true)106        // vectors only: scores are cosine similarities107        let vectorOnly = try store.search(text: "battery warranty", queryVectors: [c.id: FakeEmbedder.vector("battery warranty", dimension: 64)], collections: [c.id], limit: 2, useKeywords: false)108        #expect(vectorOnly.hits.allSatisfy { $0.score > 0.3 && $0.score <= 1.0001 })109        // without MMR both duplicates lead; with MMR the second result is something else110        let plain = try run("battery warranty", limit: 2)111        #expect(plain.hits.allSatisfy { $0.text.contains("warranty") })112        let diverse = try run("battery warranty", mmr: 0.3, limit: 2)113        #expect(diverse.hits.count == 2 && diverse.hits[0].text.contains("warranty") && !diverse.hits[1].text.contains("warranty"))114        #expect(try run("", vector: false).hits.isEmpty)115        #expect(KnowledgeStore.ftsQuery("a \" OR * (") == "\"or\"")116    }117118    @Test func oneHundredThousandChunksSearchTiming() throws {119        let store = try KnowledgeStore.inMemory(); let dim = 384; let n = 100_000120        let c = try makeCollection(store, dimension: dim)121        var seed: UInt64 = 42122        func next() -> Float { seed = seed &* 6364136223846793005 &+ 1442695040888963407; return Float(Int64(bitPattern: seed >> 33) % 2001 - 1000) / 1000 }123        let clock = ContinuousClock()124        var target: [Float] = []125        let build = try clock.measure {126            for batch in 0..<10 {127                let vectors = (0..<(n / 10)).map { _ in (0..<dim).map { _ in next() } }128                if batch == 7 { target = vectors[1234] }129                let doc = KnowledgeDocument(collectionID: c.id, name: "d\(batch)", kind: "txt", byteCount: 1, contentHash: "d\(batch)")130                try store.replaceIndex([.init(document: doc, extracted: nil, chunks: vectors.indices.map { TextChunk(index: $0, text: "lorem ipsum chunk \(batch) \($0)") }, vectors: vectors)])131            }132        }133        var first: [ScoredChunk] = []134        let cold = try clock.measure { first = try store.vectorSearch(collection: c.id, query: target, limit: 10) }135        var warm = Duration.seconds(100)136        for _ in 0..<10 { warm = min(warm, try clock.measure { _ = try store.vectorSearch(collection: c.id, query: target, limit: 10) }) }137        var hybridHits = 0138        let hybrid = try clock.measure { hybridHits = try store.search(text: "lorem chunk 1234", queryVectors: [c.id: target], collections: [c.id], limit: 10, mmrLambda: 0.7).hits.count }139        func ms(_ d: Duration) -> String { String(format: "%.1f ms", Double(d.components.seconds) * 1000 + Double(d.components.attoseconds) / 1e15) }140        print("KNOWLEDGE-BENCH 100k x \(dim): build \(ms(build)), cold search (index load) \(ms(cold)), warm exact top-10 \(ms(warm)), hybrid+MMR \(ms(hybrid))")141        #expect(first.count == 10 && first[0].score > 0.999)142        #expect(hybridHits == 10)143        #expect(warm < .milliseconds(500))144    }145}146
Files main
Menu

Theme