Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Packages/AstraPersistence/Sources/Documents.swift10// Module : Packages11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import GRDB90import AstraCore9192/// Splits text into overlapping chunks for retrieval. Sizes are in characters. Paragraphs are kept whole when they fit;93/// a longer paragraph is cut at sentence ends, then at spaces, and only as a last resort in the middle of a word.94public struct Chunker: Sendable, Hashable {95 public var targetSize: Int96 public var overlap: Int9798 public init(targetSize: Int = 1_200, overlap: Int = 150) {99 self.targetSize = max(16, targetSize)100 self.overlap = max(0, min(overlap, self.targetSize / 2))101 }102103 public func chunks(_ text: String) -> [String] {104 let normalized = text.replacingOccurrences(of: "\r\n", with: "\n").replacingOccurrences(of: "\r", with: "\n")105 var pieces: [String] = []106 for paragraph in Self.paragraphs(normalized) {107 if paragraph.count <= targetSize { pieces.append(paragraph) } else { pieces += split(paragraph) }108 }109 var out: [String] = []110 var current = ""111 for piece in pieces {112 if current.isEmpty { current = piece; continue }113 if current.count + 2 + piece.count <= targetSize { current += "\n\n" + piece; continue }114 out.append(current)115 let carried = tail(of: current)116 current = !carried.isEmpty && carried.count + 2 + piece.count <= targetSize ? carried + "\n\n" + piece : piece117 }118 if !current.isEmpty { out.append(current) }119 return out120 }121122 static func paragraphs(_ text: String) -> [String] {123 var out: [String] = []; var lines: [Substring] = []124 func flush() {125 let paragraph = lines.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines)126 if !paragraph.isEmpty { out.append(paragraph) }127 lines.removeAll(keepingCapacity: true)128 }129 for line in text.split(separator: "\n", omittingEmptySubsequences: false) {130 if line.allSatisfy(\.isWhitespace) { flush() } else { lines.append(line) }131 }132 flush()133 return out134 }135136 /// Cuts one oversized paragraph into windows of at most `targetSize`, each starting `overlap` characters before the previous end.137 private func split(_ paragraph: String) -> [String] {138 let characters = Array(paragraph)139 var out: [String] = []140 var start = 0141 while start < characters.count {142 let hardEnd = min(start + targetSize, characters.count)143 var end = hardEnd144 if hardEnd < characters.count {145 let floor = start + targetSize / 2146 var sentence: Int?, space: Int?147 var index = hardEnd - 1148 while index >= floor {149 let character = characters[index]150 if sentence == nil, ".!?。!?\n".contains(character), index + 1 < characters.count, characters[index + 1].isWhitespace || "。!?".contains(character) {151 sentence = index + 1; break152 }153 if space == nil, character.isWhitespace { space = index + 1 }154 index -= 1155 }156 end = sentence ?? space ?? hardEnd157 }158 let piece = String(characters[start..<end]).trimmingCharacters(in: .whitespacesAndNewlines)159 if !piece.isEmpty { out.append(piece) }160 if end >= characters.count { break }161 var next = max(end - overlap, start + 1)162 // Start the overlap on a word boundary when there is one nearby.163 if overlap > 0, let boundary = (next..<end).first(where: { characters[$0].isWhitespace }) { next = boundary + 1 }164 start = min(max(next, start + 1), end)165 }166 return out167 }168169 /// The last `overlap` characters of a chunk, starting at a word boundary.170 private func tail(of chunk: String) -> String {171 guard overlap > 0, chunk.count > overlap else { return "" }172 let suffix = chunk.suffix(overlap)173 guard let boundary = suffix.firstIndex(where: \.isWhitespace) else { return "" }174 return suffix[boundary...].trimmingCharacters(in: .whitespacesAndNewlines)175 }176}177178extension ChunkRecord {179 static let selection = "chunk.id, chunk.documentID, chunk.ordinal, chunk.text, chunk.embedding"180 init(row: Row, withEmbedding: Bool = true) {181 id = row[0]; documentID = row[1]; ordinal = row[2]; text = row[3]182 embedding = withEmbedding ? (row[4] as Data?).map { [Float](blob: $0) } : nil183 }184}185186public extension AstraStore {187 /// Adds a document with its chunks in one transaction; the full-text index follows automatically.188 @discardableResult189 func addDocument(name: String, kind: String, byteSize: Int, sha256: String, chunks: [ChunkDraft], at date: Date = Date()) async throws -> DocumentRecord {190 let document = DocumentRecord(id: AstraID.new(), name: name, kind: kind, byteSize: byteSize, sha256: sha256, createdAt: date)191 try await write { db in192 try document.insert(db)193 let statement = try db.cachedStatement(sql: "INSERT INTO chunk(id, documentID, ordinal, text, embedding) VALUES (?, ?, ?, ?, ?)")194 for (ordinal, chunk) in chunks.enumerated() {195 try statement.execute(arguments: [AstraID.new(), document.id, ordinal, chunk.text, chunk.embedding?.blob])196 }197 }198 return document199 }200201 /// Chunks `text` with `chunker` and adds the document.202 @discardableResult203 func addDocument(name: String, kind: String, text: String, chunker: Chunker = Chunker(), at date: Date = Date()) async throws -> DocumentRecord {204 let data = Data(text.utf8)205 return try await addDocument(name: name, kind: kind, byteSize: data.count, sha256: Hashing.sha256(data),206 chunks: chunker.chunks(text).map { ChunkDraft(text: $0) }, at: date)207 }208209 /// Most recent first.210 func documents() async throws -> [DocumentRecord] {211 try await read { db in try DocumentRecord.fetchAll(db, sql: "SELECT * FROM document ORDER BY createdAt DESC, id") }212 }213214 func document(id: String) async throws -> DocumentRecord? { try await read { db in try DocumentRecord.find(db, id: id) } }215216 func document(sha256: String) async throws -> DocumentRecord? {217 try await read { db in try DocumentRecord.fetchOne(db, sql: "SELECT * FROM document WHERE sha256 = ? LIMIT 1", arguments: [sha256]) }218 }219220 func chunks(documentID: String, includeEmbeddings: Bool = false) async throws -> [ChunkRecord] {221 try await read { db in222 try Row.fetchAll(db, sql: "SELECT \(ChunkRecord.selection) FROM chunk WHERE documentID = ? ORDER BY ordinal", arguments: [documentID])223 .map { ChunkRecord(row: $0, withEmbedding: includeEmbeddings) }224 }225 }226227 func chunkCount(documentID: String? = nil) async throws -> Int {228 try await read { db in229 if let documentID { return try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM chunk WHERE documentID = ?", arguments: [documentID]) ?? 0 }230 return try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM chunk") ?? 0231 }232 }233234 func setChunkEmbedding(chunkID: String, _ embedding: [Float]?) async throws {235 try await write { db in try db.execute(sql: "UPDATE chunk SET embedding = ? WHERE id = ?", arguments: [embedding?.blob, chunkID]) }236 }237238 /// Chunk ids and vectors for a vector index built in memory by the caller.239 func chunkEmbeddings(documentIDs: [String]? = nil) async throws -> [String: [Float]] {240 try await read { db in241 var out: [String: [Float]] = [:]242 let rows = try Row.fetchCursor(db, sql: "SELECT id, documentID, embedding FROM chunk WHERE embedding IS NOT NULL")243 let wanted = documentIDs.map(Set.init)244 while let row = try rows.next() {245 if let wanted, !wanted.contains(row[1]) { continue }246 out[row[0]] = [Float](blob: row[2])247 }248 return out249 }250 }251252 func chunks(ids: [String]) async throws -> [ChunkHit] {253 try await read { db in254 try ids.compactMap { id in255 try Row.fetchOne(db, sql: """256 SELECT \(ChunkRecord.selection), document.name FROM chunk JOIN document ON document.id = chunk.documentID WHERE chunk.id = ?257 """, arguments: [id]).map { ChunkHit(chunk: ChunkRecord(row: $0, withEmbedding: false), documentName: $0[5], score: 0) }258 }259 }260 }261262 /// BM25-ranked keyword search over chunk text, best first. The query is free text from a person or a model: it is reduced to263 /// quoted terms joined by OR, so it can never be read as FTS5 syntax. Text in scripts without word spacing falls back to264 /// substring matching (score 0).265 func searchChunks(_ query: String, limit: Int = 8, documentIDs: [String]? = nil) async throws -> [ChunkHit] {266 let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines)267 guard !trimmed.isEmpty, limit > 0 else { return [] }268 return try await read { db in269 let wanted = documentIDs.map(Set.init)270 let fetchLimit = wanted == nil ? limit : max(limit * 8, 200)271 var hits: [ChunkHit] = []272 if let pattern = FTSQuery.pattern(trimmed, mode: .anyTerm) {273 do {274 hits = try Row.fetchAll(db, sql: """275 SELECT \(ChunkRecord.selection), document.name, bm25(chunk_fts) FROM chunk_fts276 JOIN chunk ON chunk.seq = chunk_fts.rowid JOIN document ON document.id = chunk.documentID277 WHERE chunk_fts MATCH ? ORDER BY bm25(chunk_fts), chunk.documentID, chunk.ordinal LIMIT ?278 """, arguments: [pattern, fetchLimit])279 .map { ChunkHit(chunk: ChunkRecord(row: $0, withEmbedding: false), documentName: $0[5], score: -($0[6] as Double)) }280 } catch is DatabaseError { hits = [] }281 }282 if hits.isEmpty, FTSQuery.containsUnsegmentedScript(trimmed) || FTSQuery.pattern(trimmed) == nil {283 hits = try Row.fetchAll(db, sql: """284 SELECT \(ChunkRecord.selection), document.name FROM chunk JOIN document ON document.id = chunk.documentID285 WHERE chunk.text LIKE ? ESCAPE '\\' ORDER BY chunk.documentID, chunk.ordinal LIMIT ?286 """, arguments: [FTSQuery.likePattern(trimmed), fetchLimit])287 .map { ChunkHit(chunk: ChunkRecord(row: $0, withEmbedding: false), documentName: $0[5], score: 0) }288 }289 if let wanted { hits = hits.filter { wanted.contains($0.chunk.documentID) } }290 return Array(hits.prefix(limit))291 }292 }293294 /// Deletes a document, its chunks and their index entries.295 @discardableResult296 func deleteDocument(id: String) async throws -> DeletionResult {297 try await write { db in298 let before = try Self.usedBytes(db)299 let chunks = try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM chunk WHERE documentID = ?", arguments: [id]) ?? 0300 let removed = try DocumentRecord.remove(db, id: id)301 return DeletionResult(rowsDeleted: removed ? chunks + 1 : 0, bytesFreed: max(0, before - (try Self.usedBytes(db))))302 }303 }304}305