Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Packages/AstraPersistence/Sources/Export.swift10// Module : Packages11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import GRDB90import AstraCore9192/// Shape of the JSON export of one conversation: the whole message tree, which branch was showing, and where it forks.93public struct ConversationExport: Sendable, Codable, Hashable {94 public struct BranchPoint: Sendable, Codable, Hashable {95 /// Nil for the alternatives of the first message.96 public var parentID: String?97 public var childIDs: [String]98 /// The alternative on the active path, if the active path goes through this fork.99 public var activeChildID: String?100 }101 public var format: String102 public var exportedAt: Date103 public var conversation: ConversationRecord104 /// Message ids of the branch that was showing, root → leaf.105 public var activePath: [String]106 public var messages: [MessageRecord]107 public var branchPoints: [BranchPoint]108 public var attachments: [AttachmentRecord]109}110111/// Shape of `astra-export.json`, the full data export. It holds no credential: keys live in the Keychain and are never exported.112public struct AstraFullExport: Sendable, Codable, Hashable {113 public struct AttachmentFile: Sendable, Codable, Hashable {114 /// File name inside the export's `attachments` directory (the SHA-256 of the content).115 public var name: String116 public var byteSize: Int64117 }118 public struct Chunk: Sendable, Codable, Hashable {119 public var id: String120 public var documentID: String121 public var ordinal: Int122 public var text: String123 }124 public var format: String125 public var schemaVersion: Int126 public var exportedAt: Date127 public var conversations: [ConversationRecord]128 public var messages: [MessageRecord]129 public var attachments: [AttachmentRecord]130 public var attachmentFiles: [AttachmentFile]131 public var arenaSessions: [ArenaSessionRecord]132 public var arenaEntries: [ArenaEntryRecord]133 public var agentProfiles: [AgentProfileRecord]134 public var agentRuns: [AgentRunRecord]135 public var agentSteps: [AgentStepRecord]136 public var promptTemplates: [PromptTemplateRecord]137 public var promptTemplateVersions: [PromptTemplateVersionRecord]138 public var presets: [PresetRecord]139 public var usage: [UsageRecord]140 public var diagnostics: [RequestDiagnostic]141 public var memories: [MemoryRecord]142 public var documents: [DocumentRecord]143 public var chunks: [Chunk]144 public var endpoints: [EndpointProfile]145 public var settings: [String: JSONValue]146 public var benchmarks: [BenchmarkRecord]147}148149public struct FullExportSummary: Sendable, Hashable {150 public var jsonURL: URL151 public var attachmentsDirectory: URL152 public var attachmentFileCount: Int153 public var totalBytes: Int64154}155156public extension AstraStore {157 // MARK: One conversation158159 /// The whole message tree with the active path and fork points. `activePathOnly` leaves the other branches out.160 func conversationExport(id: String, activePathOnly: Bool = false, at date: Date = Date()) async throws -> ConversationExport {161 try await read { db in162 let conversation = try Self.requireConversation(db, id)163 let path = try MessageRecord.path(db, conversationID: id, limit: nil)164 let active = Set(path.map(\.id))165 let messages = activePathOnly ? path : try MessageRecord.fetch(db, "WHERE conversationID = ? ORDER BY seq", [id])166 var children: [String: [MessageRecord]] = [:]167 for message in try MessageRecord.fetch(db, "WHERE conversationID = ? ORDER BY createdAt, id", [id]) {168 children[message.parentID ?? "", default: []].append(message)169 }170 let forks = children.filter { $0.value.count > 1 }.map { parent, list in171 ConversationExport.BranchPoint(parentID: parent.isEmpty ? nil : parent, childIDs: list.map(\.id),172 activeChildID: list.first { active.contains($0.id) }?.id)173 }.sorted { ($0.childIDs.first ?? "") < ($1.childIDs.first ?? "") }174 let kept = Set(messages.map(\.id))175 let attachments = try AttachmentRecord.fetchAll(db, sql: """176 SELECT a.* FROM attachment a JOIN message m ON m.id = a.messageID WHERE m.conversationID = ? ORDER BY m.seq, a.createdAt, a.id177 """, arguments: [id]).filter { kept.contains($0.messageID) }178 return ConversationExport(format: "orix-astra-conversation/1", exportedAt: date, conversation: conversation, activePath: path.map(\.id),179 messages: messages, branchPoints: forks, attachments: attachments)180 }181 }182183 func exportConversationJSON(id: String, activePathOnly: Bool = false) async throws -> Data {184 try ExportCoding.encoder.encode(try await conversationExport(id: id, activePathOnly: activePathOnly))185 }186187 /// The active branch as a readable document.188 func exportConversationMarkdown(id: String) async throws -> String {189 let export = try await conversationExport(id: id, activePathOnly: true)190 let attachments = Dictionary(grouping: export.attachments, by: \.messageID)191 let forks = export.branchPoints.reduce(into: [String: Int]()) { counts, fork in192 if let active = fork.activeChildID { counts[active] = fork.childIDs.count }193 }194 let conversation = export.conversation195 var out = "# \(conversation.title.isEmpty ? "Conversation" : conversation.title)\n\n"196 out += "- Created: \(ExportCoding.iso(conversation.createdAt))\n- Model: \(conversation.model.modelID) (\(conversation.model.provider.displayName))\n\n"197 if !conversation.systemPrompt.isEmpty { out += "## System\n\n\(conversation.systemPrompt)\n\n" }198 for message in export.messages {199 switch message.role {200 case .user: out += "## You\n\n"201 case .assistant: out += "## Assistant" + ((message.reportedModel ?? message.model?.modelID).map { " (\($0))" } ?? "") + "\n\n"202 case .tool: out += "## Tool result" + (message.toolName.map { " (\($0))" } ?? "") + "\n\n"203 }204 if let count = forks[message.id] { out += "_One of \(count) alternatives at this point._\n\n" }205 for attachment in attachments[message.id] ?? [] {206 out += "- Attachment: \(Markdown.cell(attachment.filename)) (\(attachment.mimeType), \(attachment.byteSize) bytes)\n"207 }208 if attachments[message.id] != nil { out += "\n" }209 if !message.reasoningSummary.isEmpty {210 out += message.reasoningSummary.split(separator: "\n", omittingEmptySubsequences: false).map { "> " + $0 }.joined(separator: "\n") + "\n\n"211 }212 if message.role == .tool { out += Markdown.fenced(message.text) + "\n\n" } else if !message.text.isEmpty { out += message.text + "\n\n" }213 for call in message.toolCalls {214 out += "Tool call `\(call.name)`:\n\n" + Markdown.fenced(call.arguments.serialized(pretty: true), language: "json") + "\n\n"215 }216 switch message.status {217 case .stopped: out += "_Stopped before the end._\n\n"218 case .failed: out += "_Failed: \((message.error?.message ?? "unknown error").replacingOccurrences(of: "\n", with: " "))_\n\n"219 case .streaming: out += "_Still being written when exported._\n\n"220 case .complete: break221 }222 }223 return out224 }225226 // MARK: Everything227228 /// A consistent snapshot of every table (one read transaction).229 func fullExport(at date: Date = Date()) async throws -> AstraFullExport {230 let files = blobs.files().map { AstraFullExport.AttachmentFile(name: $0.name, byteSize: $0.bytes) }231 return try await read { db in232 let referenced = Set(try String.fetchAll(db, sql: "SELECT DISTINCT sha256 FROM attachment"))233 return AstraFullExport(234 format: "orix-astra-export/1", schemaVersion: try AstraSchema.migrator().appliedMigrations(db).count, exportedAt: date,235 conversations: try ConversationRecord.fetchAll(db, sql: "SELECT * FROM conversation ORDER BY seq"),236 messages: try MessageRecord.fetch(db, "ORDER BY seq"),237 attachments: try AttachmentRecord.fetchAll(db, sql: "SELECT * FROM attachment ORDER BY createdAt, id"),238 attachmentFiles: files.filter { referenced.contains($0.name) },239 arenaSessions: try ArenaSessionRecord.fetchAll(db, sql: "SELECT * FROM arenaSession ORDER BY createdAt, id"),240 arenaEntries: try ArenaEntryRecord.fetchAll(db, sql: "SELECT * FROM arenaEntry ORDER BY sessionID, ordinal"),241 agentProfiles: try AgentProfileRecord.fetchAll(db, sql: "SELECT * FROM agentProfile ORDER BY createdAt, id"),242 agentRuns: try AgentRunRecord.fetchAll(db, sql: "SELECT * FROM agentRun ORDER BY startedAt, id"),243 agentSteps: try AgentStepRecord.fetchAll(db, sql: "SELECT * FROM agentStep ORDER BY runID, ordinal"),244 promptTemplates: try PromptTemplateRecord.fetchAll(db, sql: "SELECT * FROM promptTemplate ORDER BY createdAt, id"),245 promptTemplateVersions: try PromptTemplateVersionRecord.fetchAll(db, sql: "SELECT * FROM promptTemplateVersion ORDER BY templateID, version"),246 presets: try PresetRecord.fetchAll(db, sql: "SELECT * FROM preset ORDER BY createdAt, id"),247 usage: try UsageRecord.fetchAll(db, sql: "SELECT * FROM usage ORDER BY date, id"),248 diagnostics: try Self.diagnostics(db, limit: -1).reversed(),249 memories: try MemoryRecord.fetchAll(db, sql: "SELECT * FROM memory ORDER BY createdAt, id"),250 documents: try DocumentRecord.fetchAll(db, sql: "SELECT * FROM document ORDER BY createdAt, id"),251 chunks: try Row.fetchAll(db, sql: "SELECT id, documentID, ordinal, text FROM chunk ORDER BY documentID, ordinal")252 .map { AstraFullExport.Chunk(id: $0[0], documentID: $0[1], ordinal: $0[2], text: $0[3]) },253 endpoints: try Self.endpoints(db).map { profile in254 var clean = profile255 clean.extraHeaders = Redactor.redact(headers: profile.extraHeaders)256 return clean257 },258 settings: try Self.allSettings(db),259 benchmarks: try BenchmarkRecord.fetchAll(db, sql: "SELECT * FROM benchmark ORDER BY createdAt, id"))260 }261 }262263 /// Writes `astra-export.json` and an `attachments` directory (one file per distinct content, named by SHA-256) into `directory`.264 @discardableResult265 func exportAll(to directory: URL, at date: Date = Date()) async throws -> FullExportSummary {266 let export = try await fullExport(at: date)267 let manager = FileManager.default268 try manager.createDirectory(at: directory, withIntermediateDirectories: true)269 let jsonURL = directory.appendingPathComponent("astra-export.json")270 let data = try ExportCoding.encoder.encode(export)271 try data.write(to: jsonURL, options: [.atomic])272 let target = directory.appendingPathComponent("attachments", isDirectory: true)273 try manager.createDirectory(at: target, withIntermediateDirectories: true)274 var total = Int64(data.count); var count = 0275 for file in export.attachmentFiles {276 let source = try blobs.fileURL(sha256: file.name)277 let destination = target.appendingPathComponent(file.name)278 if manager.fileExists(atPath: destination.path) { try manager.removeItem(at: destination) }279 try manager.copyItem(at: source, to: destination)280 total += file.byteSize; count += 1281 }282 return FullExportSummary(jsonURL: jsonURL, attachmentsDirectory: target, attachmentFileCount: count, totalBytes: total)283 }284285 // MARK: Deletion286287 @discardableResult288 func deleteAllConversations() async throws -> DeletionResult { try await deleteAll(["conversation"], counting: ["conversation", "message"]) }289290 /// Removes every attachment (rows and files); the messages stay.291 @discardableResult292 func deleteAllAttachments() async throws -> DeletionResult { try await deleteAll(["attachment"]) }293294 /// Removes every document, chunk and index entry used for retrieval.295 @discardableResult296 func deleteAllDocuments() async throws -> DeletionResult { try await deleteAll(["document"], counting: ["document", "chunk"]) }297298 @discardableResult299 func deleteAllUsage() async throws -> DeletionResult { try await deleteAll(["usage"]) }300301 @discardableResult302 func deleteAllDiagnostics() async throws -> DeletionResult { try await deleteAll(["diagnostic"]) }303304 @discardableResult305 func deleteAllMemory() async throws -> DeletionResult { try await deleteAll(["memory"]) }306307 @discardableResult308 func deleteAllArenaSessions() async throws -> DeletionResult { try await deleteAll(["arenaSession"], counting: ["arenaSession", "arenaEntry"]) }309310 @discardableResult311 func deleteAllAgentRuns() async throws -> DeletionResult { try await deleteAll(["agentRun"], counting: ["agentRun", "agentStep"]) }312313 /// Erases the database (every table is dropped and the schema recreated) and removes every attachment file.314 /// Credentials are not stored here: clear the Keychain separately.315 @discardableResult316 func deleteEverything() async throws -> DeletionResult {317 try await checkpoint()318 let before = try await databaseFileBytes()319 let rows = try await read { db in320 try AstraTable.allCases.reduce(0) { $0 + (try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM \"\($1.rawValue)\"") ?? 0) }321 }322 try await writer.erase()323 try AstraSchema.migrator().migrate(writer)324 try await checkpoint()325 var files: Int64 = 0326 for file in blobs.files() {327 if (try? FileManager.default.removeItem(at: blobs.url.appendingPathComponent(file.name))) != nil { files += file.bytes }328 }329 let after = try await databaseFileBytes()330 return DeletionResult(rowsDeleted: rows, bytesFreed: max(0, before - after) + files)331 }332333 /// Returns free pages to the system (`VACUUM`) and truncates the write-ahead log. Returns the bytes the files shrank by.334 @discardableResult335 func compact() async throws -> Int64 {336 try await checkpoint()337 let before = try await databaseFileBytes()338 try await writer.writeWithoutTransaction { db in try db.execute(sql: "VACUUM") }339 try await checkpoint()340 return max(0, before - (try await databaseFileBytes()))341 }342}343344extension AstraStore {345 /// Size on disk of the database and its write-ahead log; for an in-memory store, the bytes its pages hold.346 func databaseFileBytes() async throws -> Int64 {347 guard let databaseURL else { return try await read { db in try Self.usedBytes(db) } }348 return ["", "-wal"].reduce(Int64(0)) { total, suffix in349 let attributes = try? FileManager.default.attributesOfItem(atPath: databaseURL.path + suffix)350 return total + ((attributes?[.size] as? NSNumber)?.int64Value ?? 0)351 }352 }353354 /// Moves the write-ahead log into the database file and truncates it, so file sizes mean something.355 func checkpoint() async throws {356 guard databaseURL != nil else { return }357 try await writer.writeWithoutTransaction { db in try db.execute(sql: "PRAGMA wal_checkpoint(TRUNCATE)") }358 }359360 func deleteAll(_ tables: [String], counting: [String]? = nil) async throws -> DeletionResult {361 let blobs = self.blobs362 return try await write { db in363 let before = try Self.usedBytes(db)364 var rows = 0365 for table in counting ?? tables { rows += try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM \"\(table)\"") ?? 0 }366 for table in tables { try db.execute(sql: "DELETE FROM \"\(table)\"") }367 let files = try blobs.removeOrphans(db)368 return DeletionResult(rowsDeleted: rows, bytesFreed: max(0, before - (try Self.usedBytes(db))) + files)369 }370 }371}372