Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatDataTests/KnowledgeChunkerTests.swift3// Description: Chunker invariants: size bound, coverage, bounded overlap, metadata, grapheme safety, determinism.4import Foundation5import Testing6@testable import LocalChatKnowledge78@Suite struct KnowledgeChunkerTests {9 static let prose: String = (1...40).map { p in10 (1...(3 + p % 5)).map { s in "Paragraph \(p) sentence \(s) talks about topic\(p)x\(s) in some detail." }.joined(separator: " ")11 }.joined(separator: "\n\n")1213 func words(_ s: String) -> [String] { s.split(whereSeparator: \.isWhitespace).map(String.init) }1415 @Test(arguments: [(120, 0), (200, 40), (500, 100), (1200, 200)]) func sizeCoverageAndOverlap(size: Int, overlap: Int) {16 let doc = ExtractedDocument(kind: .txt, sections: [.init(text: Self.prose, page: 7, heading: "H")])17 let chunker = Chunker(chunkSize: size, overlap: overlap)18 let chunks = chunker.chunk(doc)19 #expect(chunks.count > 1)20 #expect(chunks.map(\.index) == Array(0..<chunks.count))21 #expect(chunks.allSatisfy { $0.text.count <= size && !$0.text.isEmpty && $0.page == 7 && $0.heading == "H" })22 // coverage: every word of the source appears, in order, once the repeated tails are skipped23 let source = words(Self.prose); var position = 024 for chunk in chunks {25 let w = words(chunk.text)26 // the chunk may start with at most `overlap` characters already seen27 var best = 028 for k in 0...min(w.count, position) where Array(w.prefix(k)) == Array(source[(position - k)..<position]) { best = k }29 #expect(w.prefix(best).joined(separator: " ").count <= overlap || overlap == 0 && best == 0)30 for word in w.dropFirst(best) { #expect(position < source.count && source[position] == word); position += 1 }31 }32 #expect(position == source.count)33 #expect(chunker.chunk(doc) == chunks) // deterministic34 }3536 @Test func noOverlapReproducesTheText() {37 let chunks = Chunker(chunkSize: 300, overlap: 0).chunk(ExtractedDocument(kind: .txt, sections: [.init(text: Self.prose)]))38 #expect(words(chunks.map(\.text).joined(separator: " ")) == words(Self.prose))39 }4041 @Test func longTokensAndGraphemesAreNeverBroken() {42 let family = "👨👩👧👦"; let flag = "🇨🇦"; let accent = "e\u{301}"43 let text = String(repeating: family + flag + accent, count: 200) // 600 Characters, no whitespace44 let chunks = Chunker(chunkSize: 50, overlap: 10).chunk(ExtractedDocument(kind: .txt, sections: [.init(text: text)]))45 #expect(chunks.count > 10)46 #expect(chunks.allSatisfy { $0.text.count <= 50 })47 let allowed: Set<Character> = [Character(family), Character(flag), Character(accent)]48 #expect(chunks.allSatisfy { $0.text.allSatisfy(allowed.contains) })49 #expect(chunks.map(\.text).joined() == text) // no word boundary → no overlap, nothing lost50 }5152 @Test func codeSplitsOnBlankLinesAndKeepsIndentation() {53 let functions = (1...12).map { "func f\($0)() {\n let value = \($0)\n print(value)\n}" }.joined(separator: "\n\n")54 let chunks = Chunker(chunkSize: 160, overlap: 0).chunk(ExtractedDocument(kind: .code, language: "swift", sections: [.init(text: functions)]))55 #expect(chunks.count >= 4)56 #expect(chunks.allSatisfy { $0.text.hasPrefix("func f") && $0.text.hasSuffix("}") && $0.text.contains("\n let value") })57 #expect(chunks.map(\.text).joined(separator: "\n\n") == functions)58 }5960 @Test func sectionsAreNeverMergedAndSettingsAreClamped() {61 let doc = ExtractedDocument(kind: .pdf, sections: [.init(text: "Page one text.", page: 1), .init(text: "Page two text.", page: 2)])62 let chunks = Chunker(chunkSize: 1000, overlap: 5000).chunk(doc)63 #expect(chunks.map(\.page) == [1, 2])64 #expect(Chunker(chunkSize: 1000, overlap: 5000).overlap == 500)65 #expect(TextChunk(index: 0, text: "body", heading: "Guide").embeddingText == "Guide\nbody")66 }67}68