Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Packages/AstraUI/Sources/Knowledge/KnowledgeController.swift10// Module : Knowledge11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import Observation90import CryptoKit91import UniformTypeIdentifiers92import AstraCore93import AstraPersistence94#if canImport(PDFKit)95import PDFKit96#endif9798/// Text pulled out of a picked file, ready to be chunked and indexed.99struct KnowledgeExtraction: Sendable {100 let text: String101 let kind: String102 /// SHA-256 of the extracted text, the same value the store keeps for a document.103 let sha256: String104 /// True when the text was cut at `KnowledgeExtractor.maxCharacters`.105 let truncated: Bool106}107108enum KnowledgeImportError: Error, LocalizedError {109 case folder, tooLarge, unreadable, unsupported, image, encryptedPDF, scannedPDF, empty110111 var errorDescription: String? {112 switch self {113 case .folder: "This is a folder. Pick individual files."114 case .tooLarge: "Larger than 20 MB, which is more than Knowledge imports from one file."115 case .unreadable: "The file could not be read."116 case .unsupported: "Not a supported type. PDFs with a text layer, text, Markdown and source files can be imported."117 case .image: "Images have no text to index. ORIX Astra does not run text recognition on imports."118 case .encryptedPDF: "This PDF is password-protected. Remove the password in Files, then import it again."119 case .scannedPDF: "This PDF has no text layer: it is probably a scan. No text recognition is run, so there is nothing to index."120 case .empty: "The file contains no text."121 }122 }123}124125enum KnowledgeExtractor {126 static let maxCharacters = 4_000_000127 private static let sourceExtensions: Set<String> = ["swift", "py", "js", "ts", "tsx", "jsx", "java", "kt", "c", "h", "cpp", "hpp", "m", "mm", "rs", "go", "rb",128 "php", "sh", "sql", "css", "html", "r", "jl", "lua", "cs", "scala", "tex"]129130 /// Reads and extracts off the caller's actor. Security-scoped access to the picked file is held only while its bytes are read.131 static func extract(_ url: URL) async throws -> KnowledgeExtraction {132 try await Task.detached(priority: .userInitiated) {133 let name = url.lastPathComponent134 let (data, type) = try read(url)135 try Task.checkCancellation()136 if type.conforms(to: .image) { throw KnowledgeImportError.image }137 var text: String138 var kind: String139 if type.conforms(to: .pdf) {140 text = try pdfText(data)141 kind = "pdf"142 } else {143 // The chat's attachment importer decides what counts as a text file; the full text is then decoded here without its excerpt cap.144 do { _ = try AttachmentImporter.draft(data: data, name: name, type: type) }145 catch AttachmentImporter.ImportError.unsupported { throw KnowledgeImportError.unsupported }146 catch AttachmentImporter.ImportError.tooLarge { throw KnowledgeImportError.tooLarge }147 catch { throw KnowledgeImportError.unreadable }148 guard !data.prefix(4_096).contains(0), let decoded = String(data: data, encoding: .utf8) ?? String(data: data, encoding: .isoLatin1) else {149 throw KnowledgeImportError.unsupported150 }151 text = decoded152 let ext = url.pathExtension.lowercased()153 kind = ["md", "markdown"].contains(ext) ? "markdown" : (type.conforms(to: .sourceCode) || sourceExtensions.contains(ext) ? "source" : "text")154 }155 text = text.trimmingCharacters(in: .whitespacesAndNewlines)156 guard !text.isEmpty else { throw kind == "pdf" ? KnowledgeImportError.scannedPDF : KnowledgeImportError.empty }157 let truncated = text.count > maxCharacters158 if truncated { text = String(text.prefix(maxCharacters)) }159 let digest = SHA256.hash(data: Data(text.utf8)).map { String(format: "%02x", $0) }.joined()160 return KnowledgeExtraction(text: text, kind: kind, sha256: digest, truncated: truncated)161 }.value162 }163164 private static func read(_ url: URL) throws -> (Data, UTType) {165 let scoped = url.startAccessingSecurityScopedResource()166 defer { if scoped { url.stopAccessingSecurityScopedResource() } }167 let values = try? url.resourceValues(forKeys: [.fileSizeKey, .contentTypeKey, .isRegularFileKey])168 guard values?.isRegularFile ?? true else { throw KnowledgeImportError.folder }169 if let size = values?.fileSize, size > AttachmentImporter.maxFileBytes { throw KnowledgeImportError.tooLarge }170 // A full copy, not a mapping: nothing may touch the file once the access above is released.171 guard let data = try? Data(contentsOf: url) else { throw KnowledgeImportError.unreadable }172 guard data.count <= AttachmentImporter.maxFileBytes else { throw KnowledgeImportError.tooLarge }173 return (data, values?.contentType ?? UTType(filenameExtension: url.pathExtension) ?? .data)174 }175176 private static func pdfText(_ data: Data) throws -> String {177 #if canImport(PDFKit)178 guard let document = PDFDocument(data: data) else { throw KnowledgeImportError.unreadable }179 if document.isLocked { throw KnowledgeImportError.encryptedPDF }180 var text = ""181 for index in 0..<document.pageCount {182 if let page = document.page(at: index)?.string, !page.isEmpty { text += page + "\n\n" }183 if text.count > maxCharacters { break }184 }185 return text186 #else187 throw KnowledgeImportError.unsupported188 #endif189 }190}191192/// The documents indexed for retrieval, their import, and the search the person can run to see what an agent would get.193@MainActor @Observable final class KnowledgeController {194 struct Item: Identifiable, Hashable, Sendable {195 let document: DocumentRecord196 let chunkCount: Int197 var id: String { document.id }198 }199200 struct Issue: Identifiable, Hashable, Sendable {201 enum Kind: Sendable { case failed, duplicate, truncated }202 let id = UUID()203 let fileName: String204 let message: String205 let kind: Kind206 }207208 struct Progress: Equatable, Sendable {209 var fileName: String210 var index: Int211 var total: Int212 var stage: String213 /// Files fully handled over files picked: the only progress that is really known.214 var fraction: Double { total > 0 ? Double(index - 1) / Double(total) : 0 }215 }216217 /// What the agent tool returns at most; the screen shows the same number of passages.218 static let searchLimit = 10219220 private(set) var items: [Item] = []221 private(set) var isLoaded = false222 private(set) var loadError: String?223 private(set) var progress: Progress?224 private(set) var issues: [Issue] = []225 private(set) var hits: [ChunkHit] = []226 private(set) var searchedQuery = ""227 private(set) var isSearching = false228 private(set) var searchError: String?229 var query = ""230231 @ObservationIgnored private let store: AstraStore232 @ObservationIgnored private var importTask: Task<Void, Never>?233234 init(store: AstraStore) { self.store = store }235236 var totalBytes: Int { items.reduce(0) { $0 + $1.document.byteSize } }237 var totalChunks: Int { items.reduce(0) { $0 + $1.chunkCount } }238 var isImporting: Bool { progress != nil }239240 func reload() async {241 do {242 var loaded: [Item] = []243 for document in try await store.documents() {244 loaded.append(Item(document: document, chunkCount: try await store.chunkCount(documentID: document.id)))245 }246 items = loaded247 loadError = nil248 } catch { loadError = AstraError.wrap(error).message }249 isLoaded = true250 }251252 // MARK: Import253254 /// Imports the picked files one after the other. `done` receives the number of documents added.255 func importFiles(_ urls: [URL], done: @escaping @MainActor (Int) -> Void) {256 guard !isImporting, !urls.isEmpty else { return }257 issues = []258 progress = Progress(fileName: urls[0].lastPathComponent, index: 1, total: urls.count, stage: "Reading")259 importTask = Task {260 var added = 0261 for (offset, url) in urls.enumerated() {262 if Task.isCancelled { break }263 let name = url.lastPathComponent264 progress = Progress(fileName: name, index: offset + 1, total: urls.count, stage: "Reading")265 do {266 let extraction = try await KnowledgeExtractor.extract(url)267 if Task.isCancelled { break }268 if let existing = try await store.document(sha256: extraction.sha256) {269 issues.append(Issue(fileName: name, message: "Already in Knowledge as “\(existing.name)” (same text). Nothing was added.", kind: .duplicate))270 continue271 }272 progress?.stage = "Indexing"273 try await store.addDocument(name: name, kind: extraction.kind, text: extraction.text)274 added += 1275 if extraction.truncated {276 issues.append(Issue(fileName: name, message: "Only the first \(KnowledgeExtractor.maxCharacters.formatted()) characters were indexed; the rest of this file was left out.", kind: .truncated))277 }278 } catch is CancellationError {279 break280 } catch {281 issues.append(Issue(fileName: name, message: (error as? LocalizedError)?.errorDescription ?? AstraError.wrap(error).message, kind: .failed))282 }283 }284 progress = nil285 importTask = nil286 await reload()287 if !searchedQuery.isEmpty { await search() }288 done(added)289 }290 }291292 /// Stops after the file being handled; documents already added stay.293 func cancelImport() { importTask?.cancel() }294295 func dismissIssues() { issues = [] }296297 // MARK: Delete298299 /// Deletes a document with its passages and index entries; returns the space released in the database.300 func delete(_ item: Item) async throws -> Int64 {301 let result = try await store.deleteDocument(id: item.id)302 await reload()303 if !searchedQuery.isEmpty { await search() }304 return result.bytesFreed305 }306307 // MARK: Search308309 /// Runs the query exactly as the agent's `document_search` does: same index, same ranking.310 func search() async {311 let text = query.trimmingCharacters(in: .whitespacesAndNewlines)312 guard !text.isEmpty else { hits = []; searchedQuery = ""; searchError = nil; isSearching = false; return }313 isSearching = true314 defer { isSearching = false }315 do {316 let found = try await store.searchChunks(text, limit: Self.searchLimit)317 guard !Task.isCancelled else { return }318 hits = found; searchedQuery = text; searchError = nil319 } catch {320 guard !Task.isCancelled else { return }321 hits = []; searchedQuery = text; searchError = AstraError.wrap(error).message322 }323 }324}325326/// Builds the excerpt shown for a search hit: a window around the first query word found, with the matches emphasised.327enum KnowledgeSnippet {328 static func terms(_ query: String) -> [String] {329 var seen = Set<String>()330 return query.lowercased().split(whereSeparator: { !$0.isLetter && !$0.isNumber }).map(String.init).filter { $0.count >= 2 && seen.insert($0).inserted }331 }332333 static func excerpt(_ text: String, query: String, radius: Int = 150) -> String {334 let flat = text.split(whereSeparator: \.isNewline).joined(separator: " ")335 let options: String.CompareOptions = [.caseInsensitive, .diacriticInsensitive]336 let first = terms(query).compactMap { flat.range(of: $0, options: options)?.lowerBound }.min()337 guard let first else { return flat.count > radius * 2 ? String(flat.prefix(radius * 2)) + "…" : flat }338 let start = flat.index(first, offsetBy: -radius, limitedBy: flat.startIndex) ?? flat.startIndex339 let end = flat.index(first, offsetBy: radius, limitedBy: flat.endIndex) ?? flat.endIndex340 return (start > flat.startIndex ? "…" : "") + flat[start..<end] + (end < flat.endIndex ? "…" : "")341 }342343 static func highlighted(_ excerpt: String, query: String) -> AttributedString {344 var out = AttributedString(excerpt)345 for term in terms(query) {346 var from = out.startIndex347 while from < out.endIndex, let range = out[from...].range(of: term, options: [.caseInsensitive, .diacriticInsensitive]) {348 out[range].inlinePresentationIntent = .stronglyEmphasized349 from = range.upperBound350 }351 }352 return out353 }354}355