Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatKnowledge/Sources/Chunker.swift3// Description: Structure-aware, deterministic chunking: paragraphs, then sentences, then words; code on blank lines and lines. Sizes are in Characters (never inside a grapheme).4import Foundation5import LocalChatCore6import NaturalLanguage78public struct TextChunk: Sendable, Hashable {9 public var index: Int10 public var text: String11 public var page: Int?12 public var heading: String?13 public init(index: Int, text: String, page: Int? = nil, heading: String? = nil) { self.index = index; self.text = text; self.page = page; self.heading = heading }14 /// What is embedded: the heading path gives short chunks their context.15 public var embeddingText: String {16 guard let heading, !heading.isEmpty, !text.hasPrefix(heading) else { return text }17 return heading + "\n" + text18 }19}2021public struct Chunker: Sendable {22 public let chunkSize: Int // maximum Characters per chunk23 public let overlap: Int // maximum Characters repeated from the previous chunk of the same section2425 public init(chunkSize: Int = 1200, overlap: Int = 200) {26 self.chunkSize = max(16, chunkSize)27 self.overlap = min(max(0, overlap), self.chunkSize / 2)28 }29 public init(collection: KnowledgeCollection) { self.init(chunkSize: collection.chunkSize, overlap: collection.chunkOverlap) }3031 private struct Unit { var text: String; var count: Int; var joiner: String }32 private var unitLimit: Int { max(1, chunkSize - overlap - 2) }3334 public func chunk(_ document: ExtractedDocument) -> [TextChunk] {35 var out: [TextChunk] = []36 let isCode = document.kind == .code37 for section in document.sections {38 let units = isCode ? codeUnits(section.text) : proseUnits(section.text)39 var current = ""; var count = 0; var fresh = false40 func emit() {41 let text = isCode ? current.trimmingCharacters(in: .newlines) : current.trimmingCharacters(in: .whitespacesAndNewlines)42 if !text.isEmpty { out.append(TextChunk(index: out.count, text: text, page: section.page, heading: section.heading)) }43 }44 for unit in units {45 let added = (current.isEmpty ? 0 : unit.joiner.count) + unit.count46 if fresh, count + added > chunkSize {47 emit()48 current = tail(of: current, code: isCode); count = current.count; fresh = false49 }50 if current.isEmpty { current = unit.text; count = unit.count } else { current += unit.joiner + unit.text; count += unit.joiner.count + unit.count }51 fresh = true52 }53 if fresh { emit() }54 }55 return out56 }5758 /// The end of a chunk repeated at the start of the next one, cut at a word (or line) boundary.59 private func tail(of text: String, code: Bool) -> String {60 guard overlap > 0 else { return "" }61 var suffix = text.suffix(overlap)62 if suffix.startIndex > text.startIndex {63 // do not start in the middle of a word / line64 guard let boundary = suffix.firstIndex(where: { code ? $0.isNewline : $0.isWhitespace }) else { return "" }65 suffix = suffix[suffix.index(after: boundary)...]66 }67 return code ? String(suffix).trimmingCharacters(in: .newlines) : String(suffix).trimmingCharacters(in: .whitespacesAndNewlines)68 }6970 private func proseUnits(_ text: String) -> [Unit] {71 var units: [Unit] = []72 for paragraph in Self.paragraphs(text) {73 let p = paragraph.trimmingCharacters(in: .whitespacesAndNewlines)74 if p.isEmpty { continue }75 let n = p.count76 if n <= unitLimit { units.append(Unit(text: p, count: n, joiner: "\n\n")); continue }77 var first = true78 for sentence in Self.sentences(p) {79 let s = sentence.trimmingCharacters(in: .whitespacesAndNewlines)80 if s.isEmpty { continue }81 let c = s.count82 if c <= unitLimit { units.append(Unit(text: s, count: c, joiner: first ? "\n\n" : " ")); first = false; continue }83 for (piece, glued) in hardSplit(s, trim: true) {84 units.append(Unit(text: piece, count: piece.count, joiner: first ? "\n\n" : (glued ? "" : " "))); first = false85 }86 }87 }88 return units89 }9091 private func codeUnits(_ text: String) -> [Unit] {92 var units: [Unit] = []93 for block in Self.paragraphs(text) {94 let b = block.trimmingCharacters(in: .newlines)95 if b.trimmingCharacters(in: .whitespaces).isEmpty { continue }96 let n = b.count97 if n <= unitLimit { units.append(Unit(text: b, count: n, joiner: "\n\n")); continue }98 var first = true99 for line in b.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline) {100 let l = String(line); let c = l.count101 if c <= unitLimit { units.append(Unit(text: l, count: c, joiner: first ? "\n\n" : "\n")); first = false; continue }102 var firstPiece = true103 for (piece, _) in hardSplit(l, trim: false) {104 units.append(Unit(text: piece, count: piece.count, joiner: first ? "\n\n" : (firstPiece ? "\n" : ""))); first = false; firstPiece = false105 }106 }107 }108 return units109 }110111 /// Pieces of at most `unitLimit` Characters, cut at whitespace when one exists in the second half. `glued` is true when the piece continues the previous one without a space.112 private func hardSplit(_ s: String, trim: Bool) -> [(String, Bool)] {113 var out: [(String, Bool)] = []; var start = s.startIndex; var glued = false114 let limit = unitLimit115 while start < s.endIndex {116 let end = s.index(start, offsetBy: limit, limitedBy: s.endIndex) ?? s.endIndex117 var cut = end; var atSpace = false118 if end < s.endIndex, let ws = s[start..<end].lastIndex(where: \.isWhitespace), s.distance(from: start, to: ws) >= limit / 2 {119 cut = s.index(after: ws); atSpace = true120 }121 let piece = trim ? s[start..<cut].trimmingCharacters(in: .whitespacesAndNewlines) : String(s[start..<cut])122 if !piece.isEmpty { out.append((piece, glued)) }123 glued = !atSpace; start = cut124 }125 return out126 }127128 /// Splits on blank lines (lines that contain only whitespace).129 static func paragraphs(_ text: String) -> [String] {130 var out: [String] = []; var current: [Substring] = []131 for line in text.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline) {132 if line.allSatisfy(\.isWhitespace) { if !current.isEmpty { out.append(current.joined(separator: "\n")); current = [] } }133 else { current.append(line) }134 }135 if !current.isEmpty { out.append(current.joined(separator: "\n")) }136 return out137 }138139 static func sentences(_ text: String) -> [String] {140 let tokenizer = NLTokenizer(unit: .sentence); tokenizer.string = text141 var out: [String] = []142 tokenizer.enumerateTokens(in: text.startIndex..<text.endIndex) { range, _ in out.append(String(text[range])); return true }143 return out.isEmpty ? [text] : out144 }145}146