SPB Git

Repository Public

Chunker.swift 7.3 KB 146 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatKnowledge/Sources/Chunker.swift3// Description: Structure-aware, deterministic chunking: paragraphs, then sentences, then words; code on blank lines and lines. Sizes are in Characters (never inside a grapheme).4import Foundation5import LocalChatCore6import NaturalLanguage78public struct TextChunk: Sendable, Hashable {9    public var index: Int10    public var text: String11    public var page: Int?12    public var heading: String?13    public init(index: Int, text: String, page: Int? = nil, heading: String? = nil) { self.index = index; self.text = text; self.page = page; self.heading = heading }14    /// What is embedded: the heading path gives short chunks their context.15    public var embeddingText: String {16        guard let heading, !heading.isEmpty, !text.hasPrefix(heading) else { return text }17        return heading + "\n" + text18    }19}2021public struct Chunker: Sendable {22    public let chunkSize: Int     // maximum Characters per chunk23    public let overlap: Int       // maximum Characters repeated from the previous chunk of the same section2425    public init(chunkSize: Int = 1200, overlap: Int = 200) {26        self.chunkSize = max(16, chunkSize)27        self.overlap = min(max(0, overlap), self.chunkSize / 2)28    }29    public init(collection: KnowledgeCollection) { self.init(chunkSize: collection.chunkSize, overlap: collection.chunkOverlap) }3031    private struct Unit { var text: String; var count: Int; var joiner: String }32    private var unitLimit: Int { max(1, chunkSize - overlap - 2) }3334    public func chunk(_ document: ExtractedDocument) -> [TextChunk] {35        var out: [TextChunk] = []36        let isCode = document.kind == .code37        for section in document.sections {38            let units = isCode ? codeUnits(section.text) : proseUnits(section.text)39            var current = ""; var count = 0; var fresh = false40            func emit() {41                let text = isCode ? current.trimmingCharacters(in: .newlines) : current.trimmingCharacters(in: .whitespacesAndNewlines)42                if !text.isEmpty { out.append(TextChunk(index: out.count, text: text, page: section.page, heading: section.heading)) }43            }44            for unit in units {45                let added = (current.isEmpty ? 0 : unit.joiner.count) + unit.count46                if fresh, count + added > chunkSize {47                    emit()48                    current = tail(of: current, code: isCode); count = current.count; fresh = false49                }50                if current.isEmpty { current = unit.text; count = unit.count } else { current += unit.joiner + unit.text; count += unit.joiner.count + unit.count }51                fresh = true52            }53            if fresh { emit() }54        }55        return out56    }5758    /// The end of a chunk repeated at the start of the next one, cut at a word (or line) boundary.59    private func tail(of text: String, code: Bool) -> String {60        guard overlap > 0 else { return "" }61        var suffix = text.suffix(overlap)62        if suffix.startIndex > text.startIndex {63            // do not start in the middle of a word / line64            guard let boundary = suffix.firstIndex(where: { code ? $0.isNewline : $0.isWhitespace }) else { return "" }65            suffix = suffix[suffix.index(after: boundary)...]66        }67        return code ? String(suffix).trimmingCharacters(in: .newlines) : String(suffix).trimmingCharacters(in: .whitespacesAndNewlines)68    }6970    private func proseUnits(_ text: String) -> [Unit] {71        var units: [Unit] = []72        for paragraph in Self.paragraphs(text) {73            let p = paragraph.trimmingCharacters(in: .whitespacesAndNewlines)74            if p.isEmpty { continue }75            let n = p.count76            if n <= unitLimit { units.append(Unit(text: p, count: n, joiner: "\n\n")); continue }77            var first = true78            for sentence in Self.sentences(p) {79                let s = sentence.trimmingCharacters(in: .whitespacesAndNewlines)80                if s.isEmpty { continue }81                let c = s.count82                if c <= unitLimit { units.append(Unit(text: s, count: c, joiner: first ? "\n\n" : " ")); first = false; continue }83                for (piece, glued) in hardSplit(s, trim: true) {84                    units.append(Unit(text: piece, count: piece.count, joiner: first ? "\n\n" : (glued ? "" : " "))); first = false85                }86            }87        }88        return units89    }9091    private func codeUnits(_ text: String) -> [Unit] {92        var units: [Unit] = []93        for block in Self.paragraphs(text) {94            let b = block.trimmingCharacters(in: .newlines)95            if b.trimmingCharacters(in: .whitespaces).isEmpty { continue }96            let n = b.count97            if n <= unitLimit { units.append(Unit(text: b, count: n, joiner: "\n\n")); continue }98            var first = true99            for line in b.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline) {100                let l = String(line); let c = l.count101                if c <= unitLimit { units.append(Unit(text: l, count: c, joiner: first ? "\n\n" : "\n")); first = false; continue }102                var firstPiece = true103                for (piece, _) in hardSplit(l, trim: false) {104                    units.append(Unit(text: piece, count: piece.count, joiner: first ? "\n\n" : (firstPiece ? "\n" : ""))); first = false; firstPiece = false105                }106            }107        }108        return units109    }110111    /// Pieces of at most `unitLimit` Characters, cut at whitespace when one exists in the second half. `glued` is true when the piece continues the previous one without a space.112    private func hardSplit(_ s: String, trim: Bool) -> [(String, Bool)] {113        var out: [(String, Bool)] = []; var start = s.startIndex; var glued = false114        let limit = unitLimit115        while start < s.endIndex {116            let end = s.index(start, offsetBy: limit, limitedBy: s.endIndex) ?? s.endIndex117            var cut = end; var atSpace = false118            if end < s.endIndex, let ws = s[start..<end].lastIndex(where: \.isWhitespace), s.distance(from: start, to: ws) >= limit / 2 {119                cut = s.index(after: ws); atSpace = true120            }121            let piece = trim ? s[start..<cut].trimmingCharacters(in: .whitespacesAndNewlines) : String(s[start..<cut])122            if !piece.isEmpty { out.append((piece, glued)) }123            glued = !atSpace; start = cut124        }125        return out126    }127128    /// Splits on blank lines (lines that contain only whitespace).129    static func paragraphs(_ text: String) -> [String] {130        var out: [String] = []; var current: [Substring] = []131        for line in text.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline) {132            if line.allSatisfy(\.isWhitespace) { if !current.isEmpty { out.append(current.joined(separator: "\n")); current = [] } }133            else { current.append(line) }134        }135        if !current.isEmpty { out.append(current.joined(separator: "\n")) }136        return out137    }138139    static func sentences(_ text: String) -> [String] {140        let tokenizer = NLTokenizer(unit: .sentence); tokenizer.string = text141        var out: [String] = []142        tokenizer.enumerateTokens(in: text.startIndex..<text.endIndex) { range, _ in out.append(String(text[range])); return true }143        return out.isEmpty ? [text] : out144    }145}146
Files main
Menu

Theme