Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Tests/AstraPersistenceTests/StudioTests.swift10// Module : AstraPersistenceTests11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import Testing90import AstraCore91@testable import AstraPersistence9293@Suite struct ArenaTests {94 static func session(_ store: AstraStore, blind: Bool = false) async throws -> (ArenaSessionRecord, [ArenaEntryRecord]) {95 let created = try await store.createArenaSession(title: "Tides, \"round\" 1", prompt: "Explain tides,\nin two lines. Use \"quotes\".", system: "Be brief.",96 parameters: GenerationParameters(temperature: 0), blind: blind, templateID: "tpl",97 competitors: [gpt, claude, local], at: t(0))98 var entries = created.entries99 entries[0].text = "The Moon pulls, \"mostly\".\nTwice a day."; entries[0].status = .complete; entries[0].reportedModel = "gpt-test-2026"100 entries[0].usage = TokenUsage(inputTokens: 20, outputTokens: 12); entries[0].timeToFirstToken = 0.31; entries[0].totalTime = 1.5101 entries[0].tokensPerSecond = 10.1; entries[0].cost = CostEstimate(amountUSD: 0.00042, basis: .estimatedFromUsage, pricingAsOf: "2026-11-01")102 entries[0].stopReason = .endOfTurn103 entries[1].text = "=SUM(A1:A9) is not a formula here"; entries[1].status = .complete; entries[1].usage = TokenUsage(inputTokens: 21, outputTokens: 9)104 entries[1].cost = CostEstimate(amountUSD: 0.0003, basis: .reportedByProvider); entries[1].stopReason = .maxTokens105 entries[2].status = .failed; entries[2].error = .local("Out of memory, with key sk-ant-abcdefghijklmnop"); entries[2].stopReason = .error106 entries[2].cost = .onDevice107 for entry in entries { try await store.finishArenaEntry(entry, at: t(5)) }108 return (created.session, entries)109 }110111 @Test func sessionsEntriesAndWinner() async throws {112 let store = try makeStore()113 let (session, entries) = try await Self.session(store)114 #expect(try await store.arenaSessions().map(\.id) == [session.id])115 #expect(try await store.arenaSession(id: session.id) == session)116 let read = try await store.arenaEntries(sessionID: session.id)117 #expect(read == entries)118 #expect(read.map(\.revealOrder) == [0, 1, 2])119 #expect(read.map(\.model) == [gpt, claude, local])120 #expect(read[0].blindLabel == "Model A")121 try await store.setArenaWinner(sessionID: session.id, entryID: entries[1].id)122 #expect(try await store.arenaEntries(sessionID: session.id).map(\.isWinner) == [false, true, false])123 try await store.setArenaWinner(sessionID: session.id, entryID: entries[0].id)124 #expect(try await store.arenaEntries(sessionID: session.id).map(\.isWinner) == [true, false, false])125 try await store.setArenaWinner(sessionID: session.id, entryID: nil)126 #expect(try await store.arenaEntries(sessionID: session.id).allSatisfy { !$0.isWinner })127 // One ledger row per entry that reported usage or a cost.128 let usage = try await store.usageRecords()129 #expect(usage.count == 3)130 #expect(usage.allSatisfy { $0.origin == .arena && $0.originID == session.id })131 try await store.renameArenaSession(id: session.id, title: "Renamed")132 #expect(try await store.arenaSession(id: session.id)?.title == "Renamed")133 try await store.deleteArenaSession(id: session.id)134 #expect(try await store.arenaEntries(sessionID: session.id).isEmpty)135 #expect(try await store.usageRecords().count == 3) // the ledger outlives the session136 await #expect(throws: AstraStoreError.self) { try await store.exportArenaSession(id: session.id, format: .json) }137 }138139 @Test func blindSessionsShuffleTheDisplayOrder() async throws {140 let store = try makeStore()141 let models = (0..<8).map { ModelRef(provider: .openRouter, modelID: "vendor/model-\($0)") }142 var orders = Set<[Int]>()143 for _ in 0..<12 {144 let created = try await store.createArenaSession(prompt: "p", blind: true, competitors: models)145 let shown = try await store.arenaEntries(sessionID: created.session.id)146 #expect(shown.map(\.revealOrder) == Array(0..<8))147 #expect(Set(shown.map(\.ordinal)) == Set(0..<8))148 #expect(Set(shown.map(\.blindLabel)).count == 8)149 orders.insert(shown.map(\.ordinal))150 }151 #expect(orders.count > 1)152 }153154 @Test func csvExportEscapesEverything() async throws {155 let store = try makeStore()156 let (session, entries) = try await Self.session(store)157 try await store.setArenaWinner(sessionID: session.id, entryID: entries[0].id)158 let csv = try await store.exportArenaSession(id: session.id, format: .csv)159 #expect(csv.hasSuffix("\r\n"))160 let rows = parseCSV(csv)161 #expect(rows.count == 4)162 #expect(rows.allSatisfy { $0.count == rows[0].count })163 let header = rows[0]164 func cell(_ row: Int, _ name: String) -> String { rows[row][header.firstIndex(of: name)!] }165 #expect(cell(1, "model") == "gpt-test")166 #expect(cell(1, "reported_model") == "gpt-test-2026")167 #expect(cell(1, "winner") == "yes")168 #expect(cell(2, "winner") == "no")169 // Commas, quotes and line breaks survive a round trip.170 #expect(cell(1, "response") == "The Moon pulls, \"mostly\".\nTwice a day.")171 #expect(cell(1, "prompt") == "Explain tides,\nin two lines. Use \"quotes\".")172 #expect(cell(1, "output_tokens") == "12")173 #expect(cell(1, "ttft_s") == "0.31")174 #expect(cell(1, "cost_usd") == "0.00042")175 #expect(cell(1, "cost_basis") == "estimatedFromUsage")176 #expect(cell(2, "stop_reason") == "maxTokens")177 // A cell that a spreadsheet would run as a formula is neutralized.178 #expect(cell(2, "response") == "'=SUM(A1:A9) is not a formula here")179 #expect(cell(3, "status") == "failed")180 #expect(cell(3, "cost_usd") == "0.0")181 // Errors are redacted on the way out.182 #expect(cell(3, "error").contains("Out of memory"))183 #expect(!csv.contains("abcdefghijklmnop"))184 }185186 @Test func csvWriterRules() {187 #expect(CSVWriter.field("plain") == "plain")188 #expect(CSVWriter.field("a,b") == "\"a,b\"")189 #expect(CSVWriter.field("say \"hi\"") == "\"say \"\"hi\"\"\"")190 #expect(CSVWriter.field("line\nbreak") == "\"line\nbreak\"")191 #expect(CSVWriter.field("cr\rlf") == "\"cr\rlf\"")192 #expect(CSVWriter.field(" padded ") == "\" padded \"")193 #expect(CSVWriter.field("") == "")194 #expect(CSVWriter.field("-12.5") == "-12.5")195 #expect(CSVWriter.field("+1") == "+1")196 #expect(CSVWriter.field("-cmd|' /C calc'!A0") == "'-cmd|' /C calc'!A0")197 #expect(CSVWriter.field("@SUM(1,2)") == "\"'@SUM(1,2)\"")198 #expect(CSVWriter.field("\tTabbed") == "'\tTabbed")199 #expect(CSVWriter.line(["a", "b,c", ""]) == "a,\"b,c\",")200 #expect(parseCSV(CSVWriter.document(header: ["x", "y"], rows: [["1", "a\"b,c\nd"], ["", "é🙂"]])) == [["x", "y"], ["1", "a\"b,c\nd"], ["", "é🙂"]])201 }202203 @Test func jsonAndMarkdownExports() async throws {204 let store = try makeStore()205 let (session, entries) = try await Self.session(store, blind: true)206 try await store.setArenaWinner(sessionID: session.id, entryID: entries[1].id)207 let json = try await store.exportArenaSession(id: session.id, format: .json)208 let decoded = try ExportCoding.decoder.decode(ArenaExport.self, from: Data(json.utf8))209 #expect(decoded.format == "orix-astra-arena/1")210 #expect(decoded.prompt == session.prompt)211 #expect(decoded.blind)212 #expect(decoded.entries.count == 3)213 #expect(Set(decoded.entries.compactMap(\.model)) == ["gpt-test", "claude-test", "mlx-community/test-4bit"])214 #expect(decoded.entries.first { $0.isWinner }?.model == "claude-test")215 #expect(decoded.entries.first { $0.model == "gpt-test" }?.usage == TokenUsage(inputTokens: 20, outputTokens: 12))216 #expect(!json.contains("abcdefghijklmnop"))217218 // Anonymized: no model identity anywhere.219 let anonymous = try await store.exportArenaSession(id: session.id, format: .json, anonymized: true)220 let hidden = try ExportCoding.decoder.decode(ArenaExport.self, from: Data(anonymous.utf8))221 #expect(hidden.entries.map(\.label) == ["Model A", "Model B", "Model C"])222 #expect(hidden.entries.allSatisfy { $0.model == nil && $0.provider == nil && $0.reportedModel == nil })223 for name in ["gpt-test", "claude-test", "mlx-community", "openAI", "anthropic"] { #expect(!anonymous.contains(name)) }224 let anonymousCSV = try await store.exportArenaSession(id: session.id, format: .csv, anonymized: true)225 #expect(!anonymousCSV.contains("gpt-test"))226227 let markdown = try await store.exportArenaSession(id: session.id, format: .markdown)228 #expect(markdown.hasPrefix("# Tides, \"round\" 1\n"))229 #expect(markdown.contains("## Prompt"))230 #expect(markdown.contains("| claude-test | yes |"))231 #expect(markdown.contains("## claude-test (winner)"))232 #expect(markdown.contains("The Moon pulls, \"mostly\".\nTwice a day."))233 #expect(markdown.contains("> Error: Out of memory"))234 #expect(Markdown.cell("a|b\nc") == "a\\|b c")235 #expect(Markdown.fenced("has ``` inside").hasPrefix("````\n"))236 #expect(ExportFormat.markdown.fileExtension == "md")237 }238}239240@Suite struct AgentTests {241 @Test func profilesRoundTrip() async throws {242 let store = try makeStore()243 let profile = AgentProfileRecord(name: "Researcher", instructions: "Cite sources.", model: claude, allowedTools: ["web.search", "calc"],244 maxSteps: 8, maxToolCalls: 5, maxOutputTokens: 2000, maxCostUSD: 0.25, memoryPolicy: .readOnly, createdAt: t(0))245 try await store.saveAgentProfile(profile, at: t(1))246 var saved = try #require(try await store.agentProfile(id: profile.id))247 #expect(saved.allowedTools == ["web.search", "calc"])248 #expect(saved.maxCostUSD == 0.25)249 #expect(saved.memoryPolicy == .readOnly)250 #expect(saved.createdAt == t(0))251 #expect(saved.updatedAt == t(1))252 saved.name = "Analyst"; saved.allowedTools = []253 try await store.saveAgentProfile(saved, at: t(2))254 try await store.saveAgentProfile(AgentProfileRecord(name: "beta", model: gpt))255 #expect(try await store.agentProfiles().map(\.name) == ["Analyst", "beta"])256 }257258 @Test func auditTrailIsOrderedRedactedAndTotalled() async throws {259 let store = try makeStore()260 let profile = AgentProfileRecord(name: "Ops", model: gpt, allowedTools: ["http.get"])261 try await store.saveAgentProfile(profile)262 let run = try await store.startAgentRun(profile: profile, goal: "Fetch the report with Authorization: Bearer abcdef1234567890XYZ", at: t(0))263 #expect(run.status == .running)264 #expect(!run.goal.contains("abcdef1234567890XYZ"))265 try await store.appendAgentStep(runID: run.id, AgentStepDraft(kind: .plan, text: "Use key sk-proj-ABCDEFGHIJKLMNOP1234 to call the API", usage: TokenUsage(inputTokens: 100, outputTokens: 40), costUSD: 0.002, createdAt: t(1)))266 let call = try await store.appendAgentStep(runID: run.id, AgentStepDraft(267 kind: .toolCall, toolName: "http.get",268 input: ["url": "https://example.com/r?access_token=tok_live_123456789", "headers": ["Authorization": "Bearer zzzzYYYY99998888", "X-Api-Key": "k-1234567890"],269 "password": "abc", "max_tokens": 256, "nested": [["api_key": "short"], "sk-ant-api03-SECRETSECRETSECRET"]],270 createdAt: t(2)))271 try await store.appendAgentStep(runID: run.id, AgentStepDraft(kind: .approvalRequest, toolName: "http.get", text: "Allow http.get?", createdAt: t(3)))272 try await store.completeAgentStep(id: call.id, output: ["status": 200, "body": "token=hf_abcdefghijklmnopqrstuv done", "secret": "s3cr3t-value"], approval: .approved, at: t(4))273 try await store.appendAgentStep(runID: run.id, AgentStepDraft(kind: .final, text: "Done.", usage: TokenUsage(inputTokens: 300, outputTokens: 20, cachedInputTokens: 250), costUSD: 0.001, createdAt: t(5)))274275 let steps = try await store.agentSteps(runID: run.id)276 #expect(steps.map(\.ordinal) == [0, 1, 2, 3])277 #expect(steps.map(\.kind) == [.plan, .toolCall, .approvalRequest, .final])278 let everything = steps.map { [$0.text, $0.inputJSON ?? "", $0.outputJSON ?? ""].joined(separator: "\n") }.joined(separator: "\n")279 for secret in ["sk-proj-ABCDEFGHIJKLMNOP1234", "tok_live_123456789", "zzzzYYYY99998888", "k-1234567890", "\"abc\"", "\"short\"", "SECRETSECRETSECRET",280 "hf_abcdefghijklmnopqrstuv", "s3cr3t-value"] {281 #expect(!everything.contains(secret), "Leaked \(secret)")282 }283 // What is not a secret stays readable and the JSON stays valid.284 let input = try #require(steps[1].input)285 #expect(input["url"]?.stringValue?.hasPrefix("https://example.com/r?access_token=") == true)286 #expect(input["max_tokens"] == 256)287 #expect(input["password"] == "•••")288 let output = try #require(steps[1].output)289 #expect(output["status"] == 200)290 #expect(steps[1].approval == .approved)291 #expect(steps[1].finishedAt == t(4))292 #expect(steps[0].text.hasPrefix("Use key sk-proj-•••"))293294 var saved = try #require(try await store.agentRun(id: run.id))295 #expect(saved.stepCount == 4)296 #expect(saved.toolCallCount == 1)297 #expect(saved.usage == TokenUsage(inputTokens: 400, outputTokens: 60, cachedInputTokens: 250))298 #expect(abs((saved.costUSD ?? 0) - 0.003) < 1e-12)299 #expect(saved.profileName == "Ops")300 try await store.setAgentRunStatus(id: run.id, .awaitingApproval)301 #expect(try await store.agentRun(id: run.id)?.finishedAt == nil)302 try await store.setAgentRunStatus(id: run.id, .completed, result: "Report fetched, password=hunter2hunter2", at: t(9))303 saved = try #require(try await store.agentRun(id: run.id))304 #expect(saved.status == .completed)305 #expect(saved.finishedAt == t(9))306 #expect(saved.result?.contains("hunter2") == false)307 try await store.recordAgentRunUsage(runID: run.id)308 let ledger = try #require(try await store.usageRecords().first)309 #expect(ledger.origin == .agent)310 #expect(ledger.inputTokens == 400)311312 // Deleting the profile keeps the audit trail; deleting the run removes its steps.313 try await store.deleteAgentProfile(id: profile.id)314 #expect(try await store.agentRun(id: run.id)?.profileID == nil)315 #expect(try await store.agentRuns().count == 1)316 #expect(try await store.agentRuns(profileID: profile.id).isEmpty)317 try await store.deleteAgentRun(id: run.id)318 #expect(try await store.agentSteps(runID: run.id).isEmpty)319 await #expect(throws: AstraStoreError.self) { try await store.appendAgentStep(runID: run.id, AgentStepDraft(kind: .note)) }320 }321322 @Test func concurrentStepsGetDistinctOrdinalsAndInterruptedRunsAreClosed() async throws {323 let store = try makeStore()324 let unsaved = AgentProfileRecord(name: "Ad hoc", model: gpt)325 let run = try await store.startAgentRun(profile: unsaved, goal: "g")326 #expect(run.profileID == nil)327 try await withThrowingTaskGroup(of: Void.self) { group in328 for index in 0..<40 { group.addTask { try await store.appendAgentStep(runID: run.id, AgentStepDraft(kind: .note, text: "n\(index)")) } }329 try await group.waitForAll()330 }331 #expect(try await store.agentSteps(runID: run.id).map(\.ordinal) == Array(0..<40))332 #expect(try await store.agentRun(id: run.id)?.stepCount == 40)333 #expect(try await store.recoverInterruptedAgentRuns(at: t(3)) == 1)334 #expect(try await store.agentRun(id: run.id)?.status == .cancelled)335 #expect(try await store.recoverInterruptedAgentRuns() == 0)336 #expect(AgentRunStatus.budgetExceeded.isFinished)337 #expect(!AgentRunStatus.awaitingApproval.isFinished)338 }339}340341@Suite struct PromptStudioTests {342 @Test func everySaveAppendsAVersionAndRestoreNeverRewritesHistory() async throws {343 let store = try makeStore()344 let v1 = try await store.saveTemplate(name: "Summary", system: "You summarize.", userTemplate: "Summarize {{text}} in {{count}} points.",345 parameters: GenerationParameters(temperature: 0.3), at: t(1))346 #expect(v1.currentVersion == 1)347 #expect(v1.variables == ["text", "count"])348 let v2 = try await store.saveTemplate(id: v1.id, name: "Summary", system: "You summarize for {{audience}}.", userTemplate: "Summarize {{text}}.",349 note: "audience", at: t(2))350 let v3 = try await store.saveTemplate(id: v1.id, name: "Summary (short)", system: "", userTemplate: "TL;DR {{text}}", at: t(3))351 #expect(v2.currentVersion == 2)352 #expect(v3.currentVersion == 3)353 #expect(v3.createdAt == t(1))354 #expect(v3.updatedAt == t(3))355 var versions = try await store.templateVersions(templateID: v1.id)356 #expect(versions.map(\.version) == [3, 2, 1])357 #expect(versions[1].note == "audience")358 #expect(versions[2].userTemplate == "Summarize {{text}} in {{count}} points.")359 #expect(versions[2].parameters == GenerationParameters(temperature: 0.3))360361 let restored = try await store.restoreTemplateVersion(templateID: v1.id, version: 1, at: t(4))362 #expect(restored.currentVersion == 4)363 #expect(restored.name == "Summary")364 #expect(restored.userTemplate == "Summarize {{text}} in {{count}} points.")365 #expect(restored.parameters == GenerationParameters(temperature: 0.3))366 versions = try await store.templateVersions(templateID: v1.id)367 #expect(versions.map(\.version) == [4, 3, 2, 1])368 #expect(versions[0].note == "Restored from version 1")369 #expect(versions[1].userTemplate == "TL;DR {{text}}")370 #expect(try await store.template(id: v1.id) == restored)371 await #expect(throws: AstraStoreError.self) { try await store.restoreTemplateVersion(templateID: v1.id, version: 99) }372373 try await store.saveTemplate(name: "another", system: "", userTemplate: "x")374 #expect(try await store.templates().map(\.name) == ["another", "Summary"])375 try await store.deleteTemplate(id: v1.id)376 #expect(try await store.templateVersions(templateID: v1.id).isEmpty)377 #expect(try await store.templates().count == 1)378 }379380 @Test func variablesAreExtractedInOrder() {381 #expect(PromptRenderer.variables(in: "Hello {{name}}, meet {{ other_name }} and {{name}} again. {{a.b-c}} {{été}} {{x1}}") == ["name", "other_name", "a.b-c", "été", "x1"])382 #expect(PromptRenderer.variables(in: "No {placeholders} here, {{}} {{ }} {{bad name}} {{-x}} {{{triple}}}") == ["triple"])383 #expect(PromptRenderer.variables(in: "") == [])384 }385386 @Test func renderingReportsMissingAndUnusedVariables() {387 let template = "Dear {{name}},\nYour order {{ order }} ships to {{city}}. Bye {{name}}."388 let complete = PromptRenderer.render(template, values: ["name": "Ada", "order": "#42", "city": "Québec"])389 #expect(complete.text == "Dear Ada,\nYour order #42 ships to Québec. Bye Ada.")390 #expect(complete.isComplete)391 #expect(complete.unused.isEmpty)392 let partial = PromptRenderer.render(template, values: ["name": "Ada", "extra": "x", "zeta": "z"])393 #expect(partial.text == "Dear Ada,\nYour order {{ order }} ships to {{city}}. Bye Ada.")394 #expect(partial.missing == ["order", "city"])395 #expect(partial.unused == ["extra", "zeta"])396 #expect(!partial.isComplete)397 // Values are inserted literally and never expanded again; an empty value is a value.398 let literal = PromptRenderer.render("{{a}} {{b}} {{c}}", values: ["a": "{{b}}", "b": "$1 \\0 🙂", "c": ""])399 #expect(literal.text == "{{b}} $1 \\0 🙂 ")400 #expect(literal.missing.isEmpty)401 #expect(PromptRenderer.render("plain", values: [:]).text == "plain")402 }403404 @Test func presets() async throws {405 let store = try makeStore()406 var preset = PresetRecord(name: "Tutor", system: "Explain step by step.", parameters: GenerationParameters(temperature: 0.4, reasoning: .medium), createdAt: t(0))407 try await store.savePreset(preset, at: t(0))408 try await store.savePreset(PresetRecord(name: "analyst", system: "Numbers first."))409 preset.system = "Explain patiently."410 try await store.savePreset(preset, at: t(5))411 #expect(try await store.presets().map(\.name) == ["analyst", "Tutor"])412 let saved = try #require(try await store.preset(id: preset.id))413 #expect(saved.system == "Explain patiently.")414 #expect(saved.parameters.reasoning == .medium)415 #expect(saved.updatedAt == t(5))416 try await store.deletePreset(id: preset.id)417 #expect(try await store.presets().count == 1)418 }419}420