SPB Git

Repository Public MIT

KnowledgeController.swift 16.2 KB 355 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraUI/Sources/Knowledge/KnowledgeController.swift10// Module           : Knowledge11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import Observation90import CryptoKit91import UniformTypeIdentifiers92import AstraCore93import AstraPersistence94#if canImport(PDFKit)95import PDFKit96#endif9798/// Text pulled out of a picked file, ready to be chunked and indexed.99struct KnowledgeExtraction: Sendable {100    let text: String101    let kind: String102    /// SHA-256 of the extracted text, the same value the store keeps for a document.103    let sha256: String104    /// True when the text was cut at `KnowledgeExtractor.maxCharacters`.105    let truncated: Bool106}107108enum KnowledgeImportError: Error, LocalizedError {109    case folder, tooLarge, unreadable, unsupported, image, encryptedPDF, scannedPDF, empty110111    var errorDescription: String? {112        switch self {113        case .folder: "This is a folder. Pick individual files."114        case .tooLarge: "Larger than 20 MB, which is more than Knowledge imports from one file."115        case .unreadable: "The file could not be read."116        case .unsupported: "Not a supported type. PDFs with a text layer, text, Markdown and source files can be imported."117        case .image: "Images have no text to index. ORIX Astra does not run text recognition on imports."118        case .encryptedPDF: "This PDF is password-protected. Remove the password in Files, then import it again."119        case .scannedPDF: "This PDF has no text layer: it is probably a scan. No text recognition is run, so there is nothing to index."120        case .empty: "The file contains no text."121        }122    }123}124125enum KnowledgeExtractor {126    static let maxCharacters = 4_000_000127    private static let sourceExtensions: Set<String> = ["swift", "py", "js", "ts", "tsx", "jsx", "java", "kt", "c", "h", "cpp", "hpp", "m", "mm", "rs", "go", "rb",128                                                        "php", "sh", "sql", "css", "html", "r", "jl", "lua", "cs", "scala", "tex"]129130    /// Reads and extracts off the caller's actor. Security-scoped access to the picked file is held only while its bytes are read.131    static func extract(_ url: URL) async throws -> KnowledgeExtraction {132        try await Task.detached(priority: .userInitiated) {133            let name = url.lastPathComponent134            let (data, type) = try read(url)135            try Task.checkCancellation()136            if type.conforms(to: .image) { throw KnowledgeImportError.image }137            var text: String138            var kind: String139            if type.conforms(to: .pdf) {140                text = try pdfText(data)141                kind = "pdf"142            } else {143                // The chat's attachment importer decides what counts as a text file; the full text is then decoded here without its excerpt cap.144                do { _ = try AttachmentImporter.draft(data: data, name: name, type: type) }145                catch AttachmentImporter.ImportError.unsupported { throw KnowledgeImportError.unsupported }146                catch AttachmentImporter.ImportError.tooLarge { throw KnowledgeImportError.tooLarge }147                catch { throw KnowledgeImportError.unreadable }148                guard !data.prefix(4_096).contains(0), let decoded = String(data: data, encoding: .utf8) ?? String(data: data, encoding: .isoLatin1) else {149                    throw KnowledgeImportError.unsupported150                }151                text = decoded152                let ext = url.pathExtension.lowercased()153                kind = ["md", "markdown"].contains(ext) ? "markdown" : (type.conforms(to: .sourceCode) || sourceExtensions.contains(ext) ? "source" : "text")154            }155            text = text.trimmingCharacters(in: .whitespacesAndNewlines)156            guard !text.isEmpty else { throw kind == "pdf" ? KnowledgeImportError.scannedPDF : KnowledgeImportError.empty }157            let truncated = text.count > maxCharacters158            if truncated { text = String(text.prefix(maxCharacters)) }159            let digest = SHA256.hash(data: Data(text.utf8)).map { String(format: "%02x", $0) }.joined()160            return KnowledgeExtraction(text: text, kind: kind, sha256: digest, truncated: truncated)161        }.value162    }163164    private static func read(_ url: URL) throws -> (Data, UTType) {165        let scoped = url.startAccessingSecurityScopedResource()166        defer { if scoped { url.stopAccessingSecurityScopedResource() } }167        let values = try? url.resourceValues(forKeys: [.fileSizeKey, .contentTypeKey, .isRegularFileKey])168        guard values?.isRegularFile ?? true else { throw KnowledgeImportError.folder }169        if let size = values?.fileSize, size > AttachmentImporter.maxFileBytes { throw KnowledgeImportError.tooLarge }170        // A full copy, not a mapping: nothing may touch the file once the access above is released.171        guard let data = try? Data(contentsOf: url) else { throw KnowledgeImportError.unreadable }172        guard data.count <= AttachmentImporter.maxFileBytes else { throw KnowledgeImportError.tooLarge }173        return (data, values?.contentType ?? UTType(filenameExtension: url.pathExtension) ?? .data)174    }175176    private static func pdfText(_ data: Data) throws -> String {177        #if canImport(PDFKit)178        guard let document = PDFDocument(data: data) else { throw KnowledgeImportError.unreadable }179        if document.isLocked { throw KnowledgeImportError.encryptedPDF }180        var text = ""181        for index in 0..<document.pageCount {182            if let page = document.page(at: index)?.string, !page.isEmpty { text += page + "\n\n" }183            if text.count > maxCharacters { break }184        }185        return text186        #else187        throw KnowledgeImportError.unsupported188        #endif189    }190}191192/// The documents indexed for retrieval, their import, and the search the person can run to see what an agent would get.193@MainActor @Observable final class KnowledgeController {194    struct Item: Identifiable, Hashable, Sendable {195        let document: DocumentRecord196        let chunkCount: Int197        var id: String { document.id }198    }199200    struct Issue: Identifiable, Hashable, Sendable {201        enum Kind: Sendable { case failed, duplicate, truncated }202        let id = UUID()203        let fileName: String204        let message: String205        let kind: Kind206    }207208    struct Progress: Equatable, Sendable {209        var fileName: String210        var index: Int211        var total: Int212        var stage: String213        /// Files fully handled over files picked: the only progress that is really known.214        var fraction: Double { total > 0 ? Double(index - 1) / Double(total) : 0 }215    }216217    /// What the agent tool returns at most; the screen shows the same number of passages.218    static let searchLimit = 10219220    private(set) var items: [Item] = []221    private(set) var isLoaded = false222    private(set) var loadError: String?223    private(set) var progress: Progress?224    private(set) var issues: [Issue] = []225    private(set) var hits: [ChunkHit] = []226    private(set) var searchedQuery = ""227    private(set) var isSearching = false228    private(set) var searchError: String?229    var query = ""230231    @ObservationIgnored private let store: AstraStore232    @ObservationIgnored private var importTask: Task<Void, Never>?233234    init(store: AstraStore) { self.store = store }235236    var totalBytes: Int { items.reduce(0) { $0 + $1.document.byteSize } }237    var totalChunks: Int { items.reduce(0) { $0 + $1.chunkCount } }238    var isImporting: Bool { progress != nil }239240    func reload() async {241        do {242            var loaded: [Item] = []243            for document in try await store.documents() {244                loaded.append(Item(document: document, chunkCount: try await store.chunkCount(documentID: document.id)))245            }246            items = loaded247            loadError = nil248        } catch { loadError = AstraError.wrap(error).message }249        isLoaded = true250    }251252    // MARK: Import253254    /// Imports the picked files one after the other. `done` receives the number of documents added.255    func importFiles(_ urls: [URL], done: @escaping @MainActor (Int) -> Void) {256        guard !isImporting, !urls.isEmpty else { return }257        issues = []258        progress = Progress(fileName: urls[0].lastPathComponent, index: 1, total: urls.count, stage: "Reading")259        importTask = Task {260            var added = 0261            for (offset, url) in urls.enumerated() {262                if Task.isCancelled { break }263                let name = url.lastPathComponent264                progress = Progress(fileName: name, index: offset + 1, total: urls.count, stage: "Reading")265                do {266                    let extraction = try await KnowledgeExtractor.extract(url)267                    if Task.isCancelled { break }268                    if let existing = try await store.document(sha256: extraction.sha256) {269                        issues.append(Issue(fileName: name, message: "Already in Knowledge as “\(existing.name)” (same text). Nothing was added.", kind: .duplicate))270                        continue271                    }272                    progress?.stage = "Indexing"273                    try await store.addDocument(name: name, kind: extraction.kind, text: extraction.text)274                    added += 1275                    if extraction.truncated {276                        issues.append(Issue(fileName: name, message: "Only the first \(KnowledgeExtractor.maxCharacters.formatted()) characters were indexed; the rest of this file was left out.", kind: .truncated))277                    }278                } catch is CancellationError {279                    break280                } catch {281                    issues.append(Issue(fileName: name, message: (error as? LocalizedError)?.errorDescription ?? AstraError.wrap(error).message, kind: .failed))282                }283            }284            progress = nil285            importTask = nil286            await reload()287            if !searchedQuery.isEmpty { await search() }288            done(added)289        }290    }291292    /// Stops after the file being handled; documents already added stay.293    func cancelImport() { importTask?.cancel() }294295    func dismissIssues() { issues = [] }296297    // MARK: Delete298299    /// Deletes a document with its passages and index entries; returns the space released in the database.300    func delete(_ item: Item) async throws -> Int64 {301        let result = try await store.deleteDocument(id: item.id)302        await reload()303        if !searchedQuery.isEmpty { await search() }304        return result.bytesFreed305    }306307    // MARK: Search308309    /// Runs the query exactly as the agent's `document_search` does: same index, same ranking.310    func search() async {311        let text = query.trimmingCharacters(in: .whitespacesAndNewlines)312        guard !text.isEmpty else { hits = []; searchedQuery = ""; searchError = nil; isSearching = false; return }313        isSearching = true314        defer { isSearching = false }315        do {316            let found = try await store.searchChunks(text, limit: Self.searchLimit)317            guard !Task.isCancelled else { return }318            hits = found; searchedQuery = text; searchError = nil319        } catch {320            guard !Task.isCancelled else { return }321            hits = []; searchedQuery = text; searchError = AstraError.wrap(error).message322        }323    }324}325326/// Builds the excerpt shown for a search hit: a window around the first query word found, with the matches emphasised.327enum KnowledgeSnippet {328    static func terms(_ query: String) -> [String] {329        var seen = Set<String>()330        return query.lowercased().split(whereSeparator: { !$0.isLetter && !$0.isNumber }).map(String.init).filter { $0.count >= 2 && seen.insert($0).inserted }331    }332333    static func excerpt(_ text: String, query: String, radius: Int = 150) -> String {334        let flat = text.split(whereSeparator: \.isNewline).joined(separator: " ")335        let options: String.CompareOptions = [.caseInsensitive, .diacriticInsensitive]336        let first = terms(query).compactMap { flat.range(of: $0, options: options)?.lowerBound }.min()337        guard let first else { return flat.count > radius * 2 ? String(flat.prefix(radius * 2)) + "…" : flat }338        let start = flat.index(first, offsetBy: -radius, limitedBy: flat.startIndex) ?? flat.startIndex339        let end = flat.index(first, offsetBy: radius, limitedBy: flat.endIndex) ?? flat.endIndex340        return (start > flat.startIndex ? "…" : "") + flat[start..<end] + (end < flat.endIndex ? "…" : "")341    }342343    static func highlighted(_ excerpt: String, query: String) -> AttributedString {344        var out = AttributedString(excerpt)345        for term in terms(query) {346            var from = out.startIndex347            while from < out.endIndex, let range = out[from...].range(of: term, options: [.caseInsensitive, .diacriticInsensitive]) {348                out[range].inlinePresentationIntent = .stronglyEmphasized349                from = range.upperBound350            }351        }352        return out353    }354}355
Files main
Menu

Theme