SPB Git

Repository Public

KnowledgeChunkerTests.swift 4.1 KB 68 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatDataTests/KnowledgeChunkerTests.swift3// Description: Chunker invariants: size bound, coverage, bounded overlap, metadata, grapheme safety, determinism.4import Foundation5import Testing6@testable import LocalChatKnowledge78@Suite struct KnowledgeChunkerTests {9    static let prose: String = (1...40).map { p in10        (1...(3 + p % 5)).map { s in "Paragraph \(p) sentence \(s) talks about topic\(p)x\(s) in some detail." }.joined(separator: " ")11    }.joined(separator: "\n\n")1213    func words(_ s: String) -> [String] { s.split(whereSeparator: \.isWhitespace).map(String.init) }1415    @Test(arguments: [(120, 0), (200, 40), (500, 100), (1200, 200)]) func sizeCoverageAndOverlap(size: Int, overlap: Int) {16        let doc = ExtractedDocument(kind: .txt, sections: [.init(text: Self.prose, page: 7, heading: "H")])17        let chunker = Chunker(chunkSize: size, overlap: overlap)18        let chunks = chunker.chunk(doc)19        #expect(chunks.count > 1)20        #expect(chunks.map(\.index) == Array(0..<chunks.count))21        #expect(chunks.allSatisfy { $0.text.count <= size && !$0.text.isEmpty && $0.page == 7 && $0.heading == "H" })22        // coverage: every word of the source appears, in order, once the repeated tails are skipped23        let source = words(Self.prose); var position = 024        for chunk in chunks {25            let w = words(chunk.text)26            // the chunk may start with at most `overlap` characters already seen27            var best = 028            for k in 0...min(w.count, position) where Array(w.prefix(k)) == Array(source[(position - k)..<position]) { best = k }29            #expect(w.prefix(best).joined(separator: " ").count <= overlap || overlap == 0 && best == 0)30            for word in w.dropFirst(best) { #expect(position < source.count && source[position] == word); position += 1 }31        }32        #expect(position == source.count)33        #expect(chunker.chunk(doc) == chunks)   // deterministic34    }3536    @Test func noOverlapReproducesTheText() {37        let chunks = Chunker(chunkSize: 300, overlap: 0).chunk(ExtractedDocument(kind: .txt, sections: [.init(text: Self.prose)]))38        #expect(words(chunks.map(\.text).joined(separator: " ")) == words(Self.prose))39    }4041    @Test func longTokensAndGraphemesAreNeverBroken() {42        let family = "👨‍👩‍👧‍👦"; let flag = "🇨🇦"; let accent = "e\u{301}"43        let text = String(repeating: family + flag + accent, count: 200)       // 600 Characters, no whitespace44        let chunks = Chunker(chunkSize: 50, overlap: 10).chunk(ExtractedDocument(kind: .txt, sections: [.init(text: text)]))45        #expect(chunks.count > 10)46        #expect(chunks.allSatisfy { $0.text.count <= 50 })47        let allowed: Set<Character> = [Character(family), Character(flag), Character(accent)]48        #expect(chunks.allSatisfy { $0.text.allSatisfy(allowed.contains) })49        #expect(chunks.map(\.text).joined() == text)     // no word boundary → no overlap, nothing lost50    }5152    @Test func codeSplitsOnBlankLinesAndKeepsIndentation() {53        let functions = (1...12).map { "func f\($0)() {\n    let value = \($0)\n    print(value)\n}" }.joined(separator: "\n\n")54        let chunks = Chunker(chunkSize: 160, overlap: 0).chunk(ExtractedDocument(kind: .code, language: "swift", sections: [.init(text: functions)]))55        #expect(chunks.count >= 4)56        #expect(chunks.allSatisfy { $0.text.hasPrefix("func f") && $0.text.hasSuffix("}") && $0.text.contains("\n    let value") })57        #expect(chunks.map(\.text).joined(separator: "\n\n") == functions)58    }5960    @Test func sectionsAreNeverMergedAndSettingsAreClamped() {61        let doc = ExtractedDocument(kind: .pdf, sections: [.init(text: "Page one text.", page: 1), .init(text: "Page two text.", page: 2)])62        let chunks = Chunker(chunkSize: 1000, overlap: 5000).chunk(doc)63        #expect(chunks.map(\.page) == [1, 2])64        #expect(Chunker(chunkSize: 1000, overlap: 5000).overlap == 500)65        #expect(TextChunk(index: 0, text: "body", heading: "Guide").embeddingText == "Guide\nbody")66    }67}68
Files main
Menu

Theme