SPB Git

Repository Public MIT

Export.swift 19.4 KB 372 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraPersistence/Sources/Export.swift10// Module           : Packages11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import GRDB90import AstraCore9192/// Shape of the JSON export of one conversation: the whole message tree, which branch was showing, and where it forks.93public struct ConversationExport: Sendable, Codable, Hashable {94    public struct BranchPoint: Sendable, Codable, Hashable {95        /// Nil for the alternatives of the first message.96        public var parentID: String?97        public var childIDs: [String]98        /// The alternative on the active path, if the active path goes through this fork.99        public var activeChildID: String?100    }101    public var format: String102    public var exportedAt: Date103    public var conversation: ConversationRecord104    /// Message ids of the branch that was showing, root → leaf.105    public var activePath: [String]106    public var messages: [MessageRecord]107    public var branchPoints: [BranchPoint]108    public var attachments: [AttachmentRecord]109}110111/// Shape of `astra-export.json`, the full data export. It holds no credential: keys live in the Keychain and are never exported.112public struct AstraFullExport: Sendable, Codable, Hashable {113    public struct AttachmentFile: Sendable, Codable, Hashable {114        /// File name inside the export's `attachments` directory (the SHA-256 of the content).115        public var name: String116        public var byteSize: Int64117    }118    public struct Chunk: Sendable, Codable, Hashable {119        public var id: String120        public var documentID: String121        public var ordinal: Int122        public var text: String123    }124    public var format: String125    public var schemaVersion: Int126    public var exportedAt: Date127    public var conversations: [ConversationRecord]128    public var messages: [MessageRecord]129    public var attachments: [AttachmentRecord]130    public var attachmentFiles: [AttachmentFile]131    public var arenaSessions: [ArenaSessionRecord]132    public var arenaEntries: [ArenaEntryRecord]133    public var agentProfiles: [AgentProfileRecord]134    public var agentRuns: [AgentRunRecord]135    public var agentSteps: [AgentStepRecord]136    public var promptTemplates: [PromptTemplateRecord]137    public var promptTemplateVersions: [PromptTemplateVersionRecord]138    public var presets: [PresetRecord]139    public var usage: [UsageRecord]140    public var diagnostics: [RequestDiagnostic]141    public var memories: [MemoryRecord]142    public var documents: [DocumentRecord]143    public var chunks: [Chunk]144    public var endpoints: [EndpointProfile]145    public var settings: [String: JSONValue]146    public var benchmarks: [BenchmarkRecord]147}148149public struct FullExportSummary: Sendable, Hashable {150    public var jsonURL: URL151    public var attachmentsDirectory: URL152    public var attachmentFileCount: Int153    public var totalBytes: Int64154}155156public extension AstraStore {157    // MARK: One conversation158159    /// The whole message tree with the active path and fork points. `activePathOnly` leaves the other branches out.160    func conversationExport(id: String, activePathOnly: Bool = false, at date: Date = Date()) async throws -> ConversationExport {161        try await read { db in162            let conversation = try Self.requireConversation(db, id)163            let path = try MessageRecord.path(db, conversationID: id, limit: nil)164            let active = Set(path.map(\.id))165            let messages = activePathOnly ? path : try MessageRecord.fetch(db, "WHERE conversationID = ? ORDER BY seq", [id])166            var children: [String: [MessageRecord]] = [:]167            for message in try MessageRecord.fetch(db, "WHERE conversationID = ? ORDER BY createdAt, id", [id]) {168                children[message.parentID ?? "", default: []].append(message)169            }170            let forks = children.filter { $0.value.count > 1 }.map { parent, list in171                ConversationExport.BranchPoint(parentID: parent.isEmpty ? nil : parent, childIDs: list.map(\.id),172                                               activeChildID: list.first { active.contains($0.id) }?.id)173            }.sorted { ($0.childIDs.first ?? "") < ($1.childIDs.first ?? "") }174            let kept = Set(messages.map(\.id))175            let attachments = try AttachmentRecord.fetchAll(db, sql: """176                SELECT a.* FROM attachment a JOIN message m ON m.id = a.messageID WHERE m.conversationID = ? ORDER BY m.seq, a.createdAt, a.id177                """, arguments: [id]).filter { kept.contains($0.messageID) }178            return ConversationExport(format: "orix-astra-conversation/1", exportedAt: date, conversation: conversation, activePath: path.map(\.id),179                                      messages: messages, branchPoints: forks, attachments: attachments)180        }181    }182183    func exportConversationJSON(id: String, activePathOnly: Bool = false) async throws -> Data {184        try ExportCoding.encoder.encode(try await conversationExport(id: id, activePathOnly: activePathOnly))185    }186187    /// The active branch as a readable document.188    func exportConversationMarkdown(id: String) async throws -> String {189        let export = try await conversationExport(id: id, activePathOnly: true)190        let attachments = Dictionary(grouping: export.attachments, by: \.messageID)191        let forks = export.branchPoints.reduce(into: [String: Int]()) { counts, fork in192            if let active = fork.activeChildID { counts[active] = fork.childIDs.count }193        }194        let conversation = export.conversation195        var out = "# \(conversation.title.isEmpty ? "Conversation" : conversation.title)\n\n"196        out += "- Created: \(ExportCoding.iso(conversation.createdAt))\n- Model: \(conversation.model.modelID) (\(conversation.model.provider.displayName))\n\n"197        if !conversation.systemPrompt.isEmpty { out += "## System\n\n\(conversation.systemPrompt)\n\n" }198        for message in export.messages {199            switch message.role {200            case .user: out += "## You\n\n"201            case .assistant: out += "## Assistant" + ((message.reportedModel ?? message.model?.modelID).map { " (\($0))" } ?? "") + "\n\n"202            case .tool: out += "## Tool result" + (message.toolName.map { " (\($0))" } ?? "") + "\n\n"203            }204            if let count = forks[message.id] { out += "_One of \(count) alternatives at this point._\n\n" }205            for attachment in attachments[message.id] ?? [] {206                out += "- Attachment: \(Markdown.cell(attachment.filename)) (\(attachment.mimeType), \(attachment.byteSize) bytes)\n"207            }208            if attachments[message.id] != nil { out += "\n" }209            if !message.reasoningSummary.isEmpty {210                out += message.reasoningSummary.split(separator: "\n", omittingEmptySubsequences: false).map { "> " + $0 }.joined(separator: "\n") + "\n\n"211            }212            if message.role == .tool { out += Markdown.fenced(message.text) + "\n\n" } else if !message.text.isEmpty { out += message.text + "\n\n" }213            for call in message.toolCalls {214                out += "Tool call `\(call.name)`:\n\n" + Markdown.fenced(call.arguments.serialized(pretty: true), language: "json") + "\n\n"215            }216            switch message.status {217            case .stopped: out += "_Stopped before the end._\n\n"218            case .failed: out += "_Failed: \((message.error?.message ?? "unknown error").replacingOccurrences(of: "\n", with: " "))_\n\n"219            case .streaming: out += "_Still being written when exported._\n\n"220            case .complete: break221            }222        }223        return out224    }225226    // MARK: Everything227228    /// A consistent snapshot of every table (one read transaction).229    func fullExport(at date: Date = Date()) async throws -> AstraFullExport {230        let files = blobs.files().map { AstraFullExport.AttachmentFile(name: $0.name, byteSize: $0.bytes) }231        return try await read { db in232            let referenced = Set(try String.fetchAll(db, sql: "SELECT DISTINCT sha256 FROM attachment"))233            return AstraFullExport(234                format: "orix-astra-export/1", schemaVersion: try AstraSchema.migrator().appliedMigrations(db).count, exportedAt: date,235                conversations: try ConversationRecord.fetchAll(db, sql: "SELECT * FROM conversation ORDER BY seq"),236                messages: try MessageRecord.fetch(db, "ORDER BY seq"),237                attachments: try AttachmentRecord.fetchAll(db, sql: "SELECT * FROM attachment ORDER BY createdAt, id"),238                attachmentFiles: files.filter { referenced.contains($0.name) },239                arenaSessions: try ArenaSessionRecord.fetchAll(db, sql: "SELECT * FROM arenaSession ORDER BY createdAt, id"),240                arenaEntries: try ArenaEntryRecord.fetchAll(db, sql: "SELECT * FROM arenaEntry ORDER BY sessionID, ordinal"),241                agentProfiles: try AgentProfileRecord.fetchAll(db, sql: "SELECT * FROM agentProfile ORDER BY createdAt, id"),242                agentRuns: try AgentRunRecord.fetchAll(db, sql: "SELECT * FROM agentRun ORDER BY startedAt, id"),243                agentSteps: try AgentStepRecord.fetchAll(db, sql: "SELECT * FROM agentStep ORDER BY runID, ordinal"),244                promptTemplates: try PromptTemplateRecord.fetchAll(db, sql: "SELECT * FROM promptTemplate ORDER BY createdAt, id"),245                promptTemplateVersions: try PromptTemplateVersionRecord.fetchAll(db, sql: "SELECT * FROM promptTemplateVersion ORDER BY templateID, version"),246                presets: try PresetRecord.fetchAll(db, sql: "SELECT * FROM preset ORDER BY createdAt, id"),247                usage: try UsageRecord.fetchAll(db, sql: "SELECT * FROM usage ORDER BY date, id"),248                diagnostics: try Self.diagnostics(db, limit: -1).reversed(),249                memories: try MemoryRecord.fetchAll(db, sql: "SELECT * FROM memory ORDER BY createdAt, id"),250                documents: try DocumentRecord.fetchAll(db, sql: "SELECT * FROM document ORDER BY createdAt, id"),251                chunks: try Row.fetchAll(db, sql: "SELECT id, documentID, ordinal, text FROM chunk ORDER BY documentID, ordinal")252                    .map { AstraFullExport.Chunk(id: $0[0], documentID: $0[1], ordinal: $0[2], text: $0[3]) },253                endpoints: try Self.endpoints(db).map { profile in254                    var clean = profile255                    clean.extraHeaders = Redactor.redact(headers: profile.extraHeaders)256                    return clean257                },258                settings: try Self.allSettings(db),259                benchmarks: try BenchmarkRecord.fetchAll(db, sql: "SELECT * FROM benchmark ORDER BY createdAt, id"))260        }261    }262263    /// Writes `astra-export.json` and an `attachments` directory (one file per distinct content, named by SHA-256) into `directory`.264    @discardableResult265    func exportAll(to directory: URL, at date: Date = Date()) async throws -> FullExportSummary {266        let export = try await fullExport(at: date)267        let manager = FileManager.default268        try manager.createDirectory(at: directory, withIntermediateDirectories: true)269        let jsonURL = directory.appendingPathComponent("astra-export.json")270        let data = try ExportCoding.encoder.encode(export)271        try data.write(to: jsonURL, options: [.atomic])272        let target = directory.appendingPathComponent("attachments", isDirectory: true)273        try manager.createDirectory(at: target, withIntermediateDirectories: true)274        var total = Int64(data.count); var count = 0275        for file in export.attachmentFiles {276            let source = try blobs.fileURL(sha256: file.name)277            let destination = target.appendingPathComponent(file.name)278            if manager.fileExists(atPath: destination.path) { try manager.removeItem(at: destination) }279            try manager.copyItem(at: source, to: destination)280            total += file.byteSize; count += 1281        }282        return FullExportSummary(jsonURL: jsonURL, attachmentsDirectory: target, attachmentFileCount: count, totalBytes: total)283    }284285    // MARK: Deletion286287    @discardableResult288    func deleteAllConversations() async throws -> DeletionResult { try await deleteAll(["conversation"], counting: ["conversation", "message"]) }289290    /// Removes every attachment (rows and files); the messages stay.291    @discardableResult292    func deleteAllAttachments() async throws -> DeletionResult { try await deleteAll(["attachment"]) }293294    /// Removes every document, chunk and index entry used for retrieval.295    @discardableResult296    func deleteAllDocuments() async throws -> DeletionResult { try await deleteAll(["document"], counting: ["document", "chunk"]) }297298    @discardableResult299    func deleteAllUsage() async throws -> DeletionResult { try await deleteAll(["usage"]) }300301    @discardableResult302    func deleteAllDiagnostics() async throws -> DeletionResult { try await deleteAll(["diagnostic"]) }303304    @discardableResult305    func deleteAllMemory() async throws -> DeletionResult { try await deleteAll(["memory"]) }306307    @discardableResult308    func deleteAllArenaSessions() async throws -> DeletionResult { try await deleteAll(["arenaSession"], counting: ["arenaSession", "arenaEntry"]) }309310    @discardableResult311    func deleteAllAgentRuns() async throws -> DeletionResult { try await deleteAll(["agentRun"], counting: ["agentRun", "agentStep"]) }312313    /// Erases the database (every table is dropped and the schema recreated) and removes every attachment file.314    /// Credentials are not stored here: clear the Keychain separately.315    @discardableResult316    func deleteEverything() async throws -> DeletionResult {317        try await checkpoint()318        let before = try await databaseFileBytes()319        let rows = try await read { db in320            try AstraTable.allCases.reduce(0) { $0 + (try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM \"\($1.rawValue)\"") ?? 0) }321        }322        try await writer.erase()323        try AstraSchema.migrator().migrate(writer)324        try await checkpoint()325        var files: Int64 = 0326        for file in blobs.files() {327            if (try? FileManager.default.removeItem(at: blobs.url.appendingPathComponent(file.name))) != nil { files += file.bytes }328        }329        let after = try await databaseFileBytes()330        return DeletionResult(rowsDeleted: rows, bytesFreed: max(0, before - after) + files)331    }332333    /// Returns free pages to the system (`VACUUM`) and truncates the write-ahead log. Returns the bytes the files shrank by.334    @discardableResult335    func compact() async throws -> Int64 {336        try await checkpoint()337        let before = try await databaseFileBytes()338        try await writer.writeWithoutTransaction { db in try db.execute(sql: "VACUUM") }339        try await checkpoint()340        return max(0, before - (try await databaseFileBytes()))341    }342}343344extension AstraStore {345    /// Size on disk of the database and its write-ahead log; for an in-memory store, the bytes its pages hold.346    func databaseFileBytes() async throws -> Int64 {347        guard let databaseURL else { return try await read { db in try Self.usedBytes(db) } }348        return ["", "-wal"].reduce(Int64(0)) { total, suffix in349            let attributes = try? FileManager.default.attributesOfItem(atPath: databaseURL.path + suffix)350            return total + ((attributes?[.size] as? NSNumber)?.int64Value ?? 0)351        }352    }353354    /// Moves the write-ahead log into the database file and truncates it, so file sizes mean something.355    func checkpoint() async throws {356        guard databaseURL != nil else { return }357        try await writer.writeWithoutTransaction { db in try db.execute(sql: "PRAGMA wal_checkpoint(TRUNCATE)") }358    }359360    func deleteAll(_ tables: [String], counting: [String]? = nil) async throws -> DeletionResult {361        let blobs = self.blobs362        return try await write { db in363            let before = try Self.usedBytes(db)364            var rows = 0365            for table in counting ?? tables { rows += try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM \"\(table)\"") ?? 0 }366            for table in tables { try db.execute(sql: "DELETE FROM \"\(table)\"") }367            let files = try blobs.removeOrphans(db)368            return DeletionResult(rowsDeleted: rows, bytesFreed: max(0, before - (try Self.usedBytes(db))) + files)369        }370    }371}372
Files main
Menu

Theme