spb/zyquo-atlas Public License

1//2//  Chunker.swift3//  Zyquo Atlas4//5//  Author: Simon-Pierre Boucher6//  Mail: contact@spboucher.ai7//8//  Splits a PageContext's markdown into heading-aware chunks for long-page9//  summarization (map-reduce) and chat-with-page retrieval (docs/10//  AI-BROWSER-RESEARCH.md §3). Each chunk carries its heading path and11//  character range so answers can cite and scroll-to-highlight the source.12//1314import Foundation1516struct Chunk: Identifiable, Hashable {17    let id: Int18    /// e.g. "Setup > macOS".19    let headingPath: String20    let text: String21    let charRange: Range<Int>22    var estimatedTokens: Int { text.count / 4 }23}2425enum Chunker {26    /// ~600-token Q&A chunks by default; pass a larger target for summarization27    /// (map-reduce tolerates coarse chunks).28    static func chunk(_ context: PageContext, targetTokens: Int = 600, overlapTokens: Int = 50) -> [Chunk] {29        let md = context.markdown30        guard !md.isEmpty else { return [] }3132        let targetChars = targetTokens * 433        let overlapChars = overlapTokens * 434        let scalars = Array(md)3536        // Section boundaries from the heading index (fall back to the whole doc).37        var boundaries = context.headings.map { $0.offset }.sorted()38        if boundaries.first != 0 { boundaries.insert(0, at: 0) }39        boundaries.append(scalars.count)4041        var chunks: [Chunk] = []42        var id = 043        var headingStack: [(level: Int, title: String)] = []4445        for i in 0..<(boundaries.count - 1) {46            let start = boundaries[i]47            let end = boundaries[i + 1]48            guard start < end else { continue }4950            // Track heading path from the heading that opens this section.51            if let h = context.headings.first(where: { $0.offset == start }) {52                while let last = headingStack.last, last.level >= h.level { headingStack.removeLast() }53                headingStack.append((h.level, h.title))54            }55            let path = headingStack.map(\.title).joined(separator: " > ")5657            // Sub-split large sections with overlap.58            var cursor = start59            while cursor < end {60                let sliceEnd = min(cursor + targetChars, end)61                let text = String(scalars[cursor..<sliceEnd]).trimmingCharacters(in: .whitespacesAndNewlines)62                if !text.isEmpty {63                    chunks.append(Chunk(id: id, headingPath: path, text: text,64                                        charRange: cursor..<sliceEnd))65                    id += 166                }67                if sliceEnd >= end { break }68                cursor = max(cursor + targetChars - overlapChars, cursor + 1)69            }70        }71        return chunks72    }73}74