SPB Git

Repository Public MIT

Documents.swift 14.7 KB 305 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraPersistence/Sources/Documents.swift10// Module           : Packages11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import GRDB90import AstraCore9192/// Splits text into overlapping chunks for retrieval. Sizes are in characters. Paragraphs are kept whole when they fit;93/// a longer paragraph is cut at sentence ends, then at spaces, and only as a last resort in the middle of a word.94public struct Chunker: Sendable, Hashable {95    public var targetSize: Int96    public var overlap: Int9798    public init(targetSize: Int = 1_200, overlap: Int = 150) {99        self.targetSize = max(16, targetSize)100        self.overlap = max(0, min(overlap, self.targetSize / 2))101    }102103    public func chunks(_ text: String) -> [String] {104        let normalized = text.replacingOccurrences(of: "\r\n", with: "\n").replacingOccurrences(of: "\r", with: "\n")105        var pieces: [String] = []106        for paragraph in Self.paragraphs(normalized) {107            if paragraph.count <= targetSize { pieces.append(paragraph) } else { pieces += split(paragraph) }108        }109        var out: [String] = []110        var current = ""111        for piece in pieces {112            if current.isEmpty { current = piece; continue }113            if current.count + 2 + piece.count <= targetSize { current += "\n\n" + piece; continue }114            out.append(current)115            let carried = tail(of: current)116            current = !carried.isEmpty && carried.count + 2 + piece.count <= targetSize ? carried + "\n\n" + piece : piece117        }118        if !current.isEmpty { out.append(current) }119        return out120    }121122    static func paragraphs(_ text: String) -> [String] {123        var out: [String] = []; var lines: [Substring] = []124        func flush() {125            let paragraph = lines.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines)126            if !paragraph.isEmpty { out.append(paragraph) }127            lines.removeAll(keepingCapacity: true)128        }129        for line in text.split(separator: "\n", omittingEmptySubsequences: false) {130            if line.allSatisfy(\.isWhitespace) { flush() } else { lines.append(line) }131        }132        flush()133        return out134    }135136    /// Cuts one oversized paragraph into windows of at most `targetSize`, each starting `overlap` characters before the previous end.137    private func split(_ paragraph: String) -> [String] {138        let characters = Array(paragraph)139        var out: [String] = []140        var start = 0141        while start < characters.count {142            let hardEnd = min(start + targetSize, characters.count)143            var end = hardEnd144            if hardEnd < characters.count {145                let floor = start + targetSize / 2146                var sentence: Int?, space: Int?147                var index = hardEnd - 1148                while index >= floor {149                    let character = characters[index]150                    if sentence == nil, ".!?。!?\n".contains(character), index + 1 < characters.count, characters[index + 1].isWhitespace || "。!?".contains(character) {151                        sentence = index + 1; break152                    }153                    if space == nil, character.isWhitespace { space = index + 1 }154                    index -= 1155                }156                end = sentence ?? space ?? hardEnd157            }158            let piece = String(characters[start..<end]).trimmingCharacters(in: .whitespacesAndNewlines)159            if !piece.isEmpty { out.append(piece) }160            if end >= characters.count { break }161            var next = max(end - overlap, start + 1)162            // Start the overlap on a word boundary when there is one nearby.163            if overlap > 0, let boundary = (next..<end).first(where: { characters[$0].isWhitespace }) { next = boundary + 1 }164            start = min(max(next, start + 1), end)165        }166        return out167    }168169    /// The last `overlap` characters of a chunk, starting at a word boundary.170    private func tail(of chunk: String) -> String {171        guard overlap > 0, chunk.count > overlap else { return "" }172        let suffix = chunk.suffix(overlap)173        guard let boundary = suffix.firstIndex(where: \.isWhitespace) else { return "" }174        return suffix[boundary...].trimmingCharacters(in: .whitespacesAndNewlines)175    }176}177178extension ChunkRecord {179    static let selection = "chunk.id, chunk.documentID, chunk.ordinal, chunk.text, chunk.embedding"180    init(row: Row, withEmbedding: Bool = true) {181        id = row[0]; documentID = row[1]; ordinal = row[2]; text = row[3]182        embedding = withEmbedding ? (row[4] as Data?).map { [Float](blob: $0) } : nil183    }184}185186public extension AstraStore {187    /// Adds a document with its chunks in one transaction; the full-text index follows automatically.188    @discardableResult189    func addDocument(name: String, kind: String, byteSize: Int, sha256: String, chunks: [ChunkDraft], at date: Date = Date()) async throws -> DocumentRecord {190        let document = DocumentRecord(id: AstraID.new(), name: name, kind: kind, byteSize: byteSize, sha256: sha256, createdAt: date)191        try await write { db in192            try document.insert(db)193            let statement = try db.cachedStatement(sql: "INSERT INTO chunk(id, documentID, ordinal, text, embedding) VALUES (?, ?, ?, ?, ?)")194            for (ordinal, chunk) in chunks.enumerated() {195                try statement.execute(arguments: [AstraID.new(), document.id, ordinal, chunk.text, chunk.embedding?.blob])196            }197        }198        return document199    }200201    /// Chunks `text` with `chunker` and adds the document.202    @discardableResult203    func addDocument(name: String, kind: String, text: String, chunker: Chunker = Chunker(), at date: Date = Date()) async throws -> DocumentRecord {204        let data = Data(text.utf8)205        return try await addDocument(name: name, kind: kind, byteSize: data.count, sha256: Hashing.sha256(data),206                                     chunks: chunker.chunks(text).map { ChunkDraft(text: $0) }, at: date)207    }208209    /// Most recent first.210    func documents() async throws -> [DocumentRecord] {211        try await read { db in try DocumentRecord.fetchAll(db, sql: "SELECT * FROM document ORDER BY createdAt DESC, id") }212    }213214    func document(id: String) async throws -> DocumentRecord? { try await read { db in try DocumentRecord.find(db, id: id) } }215216    func document(sha256: String) async throws -> DocumentRecord? {217        try await read { db in try DocumentRecord.fetchOne(db, sql: "SELECT * FROM document WHERE sha256 = ? LIMIT 1", arguments: [sha256]) }218    }219220    func chunks(documentID: String, includeEmbeddings: Bool = false) async throws -> [ChunkRecord] {221        try await read { db in222            try Row.fetchAll(db, sql: "SELECT \(ChunkRecord.selection) FROM chunk WHERE documentID = ? ORDER BY ordinal", arguments: [documentID])223                .map { ChunkRecord(row: $0, withEmbedding: includeEmbeddings) }224        }225    }226227    func chunkCount(documentID: String? = nil) async throws -> Int {228        try await read { db in229            if let documentID { return try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM chunk WHERE documentID = ?", arguments: [documentID]) ?? 0 }230            return try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM chunk") ?? 0231        }232    }233234    func setChunkEmbedding(chunkID: String, _ embedding: [Float]?) async throws {235        try await write { db in try db.execute(sql: "UPDATE chunk SET embedding = ? WHERE id = ?", arguments: [embedding?.blob, chunkID]) }236    }237238    /// Chunk ids and vectors for a vector index built in memory by the caller.239    func chunkEmbeddings(documentIDs: [String]? = nil) async throws -> [String: [Float]] {240        try await read { db in241            var out: [String: [Float]] = [:]242            let rows = try Row.fetchCursor(db, sql: "SELECT id, documentID, embedding FROM chunk WHERE embedding IS NOT NULL")243            let wanted = documentIDs.map(Set.init)244            while let row = try rows.next() {245                if let wanted, !wanted.contains(row[1]) { continue }246                out[row[0]] = [Float](blob: row[2])247            }248            return out249        }250    }251252    func chunks(ids: [String]) async throws -> [ChunkHit] {253        try await read { db in254            try ids.compactMap { id in255                try Row.fetchOne(db, sql: """256                    SELECT \(ChunkRecord.selection), document.name FROM chunk JOIN document ON document.id = chunk.documentID WHERE chunk.id = ?257                    """, arguments: [id]).map { ChunkHit(chunk: ChunkRecord(row: $0, withEmbedding: false), documentName: $0[5], score: 0) }258            }259        }260    }261262    /// BM25-ranked keyword search over chunk text, best first. The query is free text from a person or a model: it is reduced to263    /// quoted terms joined by OR, so it can never be read as FTS5 syntax. Text in scripts without word spacing falls back to264    /// substring matching (score 0).265    func searchChunks(_ query: String, limit: Int = 8, documentIDs: [String]? = nil) async throws -> [ChunkHit] {266        let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines)267        guard !trimmed.isEmpty, limit > 0 else { return [] }268        return try await read { db in269            let wanted = documentIDs.map(Set.init)270            let fetchLimit = wanted == nil ? limit : max(limit * 8, 200)271            var hits: [ChunkHit] = []272            if let pattern = FTSQuery.pattern(trimmed, mode: .anyTerm) {273                do {274                    hits = try Row.fetchAll(db, sql: """275                        SELECT \(ChunkRecord.selection), document.name, bm25(chunk_fts) FROM chunk_fts276                        JOIN chunk ON chunk.seq = chunk_fts.rowid JOIN document ON document.id = chunk.documentID277                        WHERE chunk_fts MATCH ? ORDER BY bm25(chunk_fts), chunk.documentID, chunk.ordinal LIMIT ?278                        """, arguments: [pattern, fetchLimit])279                        .map { ChunkHit(chunk: ChunkRecord(row: $0, withEmbedding: false), documentName: $0[5], score: -($0[6] as Double)) }280                } catch is DatabaseError { hits = [] }281            }282            if hits.isEmpty, FTSQuery.containsUnsegmentedScript(trimmed) || FTSQuery.pattern(trimmed) == nil {283                hits = try Row.fetchAll(db, sql: """284                    SELECT \(ChunkRecord.selection), document.name FROM chunk JOIN document ON document.id = chunk.documentID285                    WHERE chunk.text LIKE ? ESCAPE '\\' ORDER BY chunk.documentID, chunk.ordinal LIMIT ?286                    """, arguments: [FTSQuery.likePattern(trimmed), fetchLimit])287                    .map { ChunkHit(chunk: ChunkRecord(row: $0, withEmbedding: false), documentName: $0[5], score: 0) }288            }289            if let wanted { hits = hits.filter { wanted.contains($0.chunk.documentID) } }290            return Array(hits.prefix(limit))291        }292    }293294    /// Deletes a document, its chunks and their index entries.295    @discardableResult296    func deleteDocument(id: String) async throws -> DeletionResult {297        try await write { db in298            let before = try Self.usedBytes(db)299            let chunks = try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM chunk WHERE documentID = ?", arguments: [id]) ?? 0300            let removed = try DocumentRecord.remove(db, id: id)301            return DeletionResult(rowsDeleted: removed ? chunks + 1 : 0, bytesFreed: max(0, before - (try Self.usedBytes(db))))302        }303    }304}305
Files main
Menu

Theme