SPB Git

Repository Public MIT

StudioTests.swift 24.4 KB 420 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraPersistenceTests/StudioTests.swift10// Module           : AstraPersistenceTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import Testing90import AstraCore91@testable import AstraPersistence9293@Suite struct ArenaTests {94    static func session(_ store: AstraStore, blind: Bool = false) async throws -> (ArenaSessionRecord, [ArenaEntryRecord]) {95        let created = try await store.createArenaSession(title: "Tides, \"round\" 1", prompt: "Explain tides,\nin two lines. Use \"quotes\".", system: "Be brief.",96                                                         parameters: GenerationParameters(temperature: 0), blind: blind, templateID: "tpl",97                                                         competitors: [gpt, claude, local], at: t(0))98        var entries = created.entries99        entries[0].text = "The Moon pulls, \"mostly\".\nTwice a day."; entries[0].status = .complete; entries[0].reportedModel = "gpt-test-2026"100        entries[0].usage = TokenUsage(inputTokens: 20, outputTokens: 12); entries[0].timeToFirstToken = 0.31; entries[0].totalTime = 1.5101        entries[0].tokensPerSecond = 10.1; entries[0].cost = CostEstimate(amountUSD: 0.00042, basis: .estimatedFromUsage, pricingAsOf: "2026-11-01")102        entries[0].stopReason = .endOfTurn103        entries[1].text = "=SUM(A1:A9) is not a formula here"; entries[1].status = .complete; entries[1].usage = TokenUsage(inputTokens: 21, outputTokens: 9)104        entries[1].cost = CostEstimate(amountUSD: 0.0003, basis: .reportedByProvider); entries[1].stopReason = .maxTokens105        entries[2].status = .failed; entries[2].error = .local("Out of memory, with key sk-ant-abcdefghijklmnop"); entries[2].stopReason = .error106        entries[2].cost = .onDevice107        for entry in entries { try await store.finishArenaEntry(entry, at: t(5)) }108        return (created.session, entries)109    }110111    @Test func sessionsEntriesAndWinner() async throws {112        let store = try makeStore()113        let (session, entries) = try await Self.session(store)114        #expect(try await store.arenaSessions().map(\.id) == [session.id])115        #expect(try await store.arenaSession(id: session.id) == session)116        let read = try await store.arenaEntries(sessionID: session.id)117        #expect(read == entries)118        #expect(read.map(\.revealOrder) == [0, 1, 2])119        #expect(read.map(\.model) == [gpt, claude, local])120        #expect(read[0].blindLabel == "Model A")121        try await store.setArenaWinner(sessionID: session.id, entryID: entries[1].id)122        #expect(try await store.arenaEntries(sessionID: session.id).map(\.isWinner) == [false, true, false])123        try await store.setArenaWinner(sessionID: session.id, entryID: entries[0].id)124        #expect(try await store.arenaEntries(sessionID: session.id).map(\.isWinner) == [true, false, false])125        try await store.setArenaWinner(sessionID: session.id, entryID: nil)126        #expect(try await store.arenaEntries(sessionID: session.id).allSatisfy { !$0.isWinner })127        // One ledger row per entry that reported usage or a cost.128        let usage = try await store.usageRecords()129        #expect(usage.count == 3)130        #expect(usage.allSatisfy { $0.origin == .arena && $0.originID == session.id })131        try await store.renameArenaSession(id: session.id, title: "Renamed")132        #expect(try await store.arenaSession(id: session.id)?.title == "Renamed")133        try await store.deleteArenaSession(id: session.id)134        #expect(try await store.arenaEntries(sessionID: session.id).isEmpty)135        #expect(try await store.usageRecords().count == 3)   // the ledger outlives the session136        await #expect(throws: AstraStoreError.self) { try await store.exportArenaSession(id: session.id, format: .json) }137    }138139    @Test func blindSessionsShuffleTheDisplayOrder() async throws {140        let store = try makeStore()141        let models = (0..<8).map { ModelRef(provider: .openRouter, modelID: "vendor/model-\($0)") }142        var orders = Set<[Int]>()143        for _ in 0..<12 {144            let created = try await store.createArenaSession(prompt: "p", blind: true, competitors: models)145            let shown = try await store.arenaEntries(sessionID: created.session.id)146            #expect(shown.map(\.revealOrder) == Array(0..<8))147            #expect(Set(shown.map(\.ordinal)) == Set(0..<8))148            #expect(Set(shown.map(\.blindLabel)).count == 8)149            orders.insert(shown.map(\.ordinal))150        }151        #expect(orders.count > 1)152    }153154    @Test func csvExportEscapesEverything() async throws {155        let store = try makeStore()156        let (session, entries) = try await Self.session(store)157        try await store.setArenaWinner(sessionID: session.id, entryID: entries[0].id)158        let csv = try await store.exportArenaSession(id: session.id, format: .csv)159        #expect(csv.hasSuffix("\r\n"))160        let rows = parseCSV(csv)161        #expect(rows.count == 4)162        #expect(rows.allSatisfy { $0.count == rows[0].count })163        let header = rows[0]164        func cell(_ row: Int, _ name: String) -> String { rows[row][header.firstIndex(of: name)!] }165        #expect(cell(1, "model") == "gpt-test")166        #expect(cell(1, "reported_model") == "gpt-test-2026")167        #expect(cell(1, "winner") == "yes")168        #expect(cell(2, "winner") == "no")169        // Commas, quotes and line breaks survive a round trip.170        #expect(cell(1, "response") == "The Moon pulls, \"mostly\".\nTwice a day.")171        #expect(cell(1, "prompt") == "Explain tides,\nin two lines. Use \"quotes\".")172        #expect(cell(1, "output_tokens") == "12")173        #expect(cell(1, "ttft_s") == "0.31")174        #expect(cell(1, "cost_usd") == "0.00042")175        #expect(cell(1, "cost_basis") == "estimatedFromUsage")176        #expect(cell(2, "stop_reason") == "maxTokens")177        // A cell that a spreadsheet would run as a formula is neutralized.178        #expect(cell(2, "response") == "'=SUM(A1:A9) is not a formula here")179        #expect(cell(3, "status") == "failed")180        #expect(cell(3, "cost_usd") == "0.0")181        // Errors are redacted on the way out.182        #expect(cell(3, "error").contains("Out of memory"))183        #expect(!csv.contains("abcdefghijklmnop"))184    }185186    @Test func csvWriterRules() {187        #expect(CSVWriter.field("plain") == "plain")188        #expect(CSVWriter.field("a,b") == "\"a,b\"")189        #expect(CSVWriter.field("say \"hi\"") == "\"say \"\"hi\"\"\"")190        #expect(CSVWriter.field("line\nbreak") == "\"line\nbreak\"")191        #expect(CSVWriter.field("cr\rlf") == "\"cr\rlf\"")192        #expect(CSVWriter.field(" padded ") == "\" padded \"")193        #expect(CSVWriter.field("") == "")194        #expect(CSVWriter.field("-12.5") == "-12.5")195        #expect(CSVWriter.field("+1") == "+1")196        #expect(CSVWriter.field("-cmd|' /C calc'!A0") == "'-cmd|' /C calc'!A0")197        #expect(CSVWriter.field("@SUM(1,2)") == "\"'@SUM(1,2)\"")198        #expect(CSVWriter.field("\tTabbed") == "'\tTabbed")199        #expect(CSVWriter.line(["a", "b,c", ""]) == "a,\"b,c\",")200        #expect(parseCSV(CSVWriter.document(header: ["x", "y"], rows: [["1", "a\"b,c\nd"], ["", "é🙂"]])) == [["x", "y"], ["1", "a\"b,c\nd"], ["", "é🙂"]])201    }202203    @Test func jsonAndMarkdownExports() async throws {204        let store = try makeStore()205        let (session, entries) = try await Self.session(store, blind: true)206        try await store.setArenaWinner(sessionID: session.id, entryID: entries[1].id)207        let json = try await store.exportArenaSession(id: session.id, format: .json)208        let decoded = try ExportCoding.decoder.decode(ArenaExport.self, from: Data(json.utf8))209        #expect(decoded.format == "orix-astra-arena/1")210        #expect(decoded.prompt == session.prompt)211        #expect(decoded.blind)212        #expect(decoded.entries.count == 3)213        #expect(Set(decoded.entries.compactMap(\.model)) == ["gpt-test", "claude-test", "mlx-community/test-4bit"])214        #expect(decoded.entries.first { $0.isWinner }?.model == "claude-test")215        #expect(decoded.entries.first { $0.model == "gpt-test" }?.usage == TokenUsage(inputTokens: 20, outputTokens: 12))216        #expect(!json.contains("abcdefghijklmnop"))217218        // Anonymized: no model identity anywhere.219        let anonymous = try await store.exportArenaSession(id: session.id, format: .json, anonymized: true)220        let hidden = try ExportCoding.decoder.decode(ArenaExport.self, from: Data(anonymous.utf8))221        #expect(hidden.entries.map(\.label) == ["Model A", "Model B", "Model C"])222        #expect(hidden.entries.allSatisfy { $0.model == nil && $0.provider == nil && $0.reportedModel == nil })223        for name in ["gpt-test", "claude-test", "mlx-community", "openAI", "anthropic"] { #expect(!anonymous.contains(name)) }224        let anonymousCSV = try await store.exportArenaSession(id: session.id, format: .csv, anonymized: true)225        #expect(!anonymousCSV.contains("gpt-test"))226227        let markdown = try await store.exportArenaSession(id: session.id, format: .markdown)228        #expect(markdown.hasPrefix("# Tides, \"round\" 1\n"))229        #expect(markdown.contains("## Prompt"))230        #expect(markdown.contains("| claude-test | yes |"))231        #expect(markdown.contains("## claude-test (winner)"))232        #expect(markdown.contains("The Moon pulls, \"mostly\".\nTwice a day."))233        #expect(markdown.contains("> Error: Out of memory"))234        #expect(Markdown.cell("a|b\nc") == "a\\|b c")235        #expect(Markdown.fenced("has ``` inside").hasPrefix("````\n"))236        #expect(ExportFormat.markdown.fileExtension == "md")237    }238}239240@Suite struct AgentTests {241    @Test func profilesRoundTrip() async throws {242        let store = try makeStore()243        let profile = AgentProfileRecord(name: "Researcher", instructions: "Cite sources.", model: claude, allowedTools: ["web.search", "calc"],244                                         maxSteps: 8, maxToolCalls: 5, maxOutputTokens: 2000, maxCostUSD: 0.25, memoryPolicy: .readOnly, createdAt: t(0))245        try await store.saveAgentProfile(profile, at: t(1))246        var saved = try #require(try await store.agentProfile(id: profile.id))247        #expect(saved.allowedTools == ["web.search", "calc"])248        #expect(saved.maxCostUSD == 0.25)249        #expect(saved.memoryPolicy == .readOnly)250        #expect(saved.createdAt == t(0))251        #expect(saved.updatedAt == t(1))252        saved.name = "Analyst"; saved.allowedTools = []253        try await store.saveAgentProfile(saved, at: t(2))254        try await store.saveAgentProfile(AgentProfileRecord(name: "beta", model: gpt))255        #expect(try await store.agentProfiles().map(\.name) == ["Analyst", "beta"])256    }257258    @Test func auditTrailIsOrderedRedactedAndTotalled() async throws {259        let store = try makeStore()260        let profile = AgentProfileRecord(name: "Ops", model: gpt, allowedTools: ["http.get"])261        try await store.saveAgentProfile(profile)262        let run = try await store.startAgentRun(profile: profile, goal: "Fetch the report with Authorization: Bearer abcdef1234567890XYZ", at: t(0))263        #expect(run.status == .running)264        #expect(!run.goal.contains("abcdef1234567890XYZ"))265        try await store.appendAgentStep(runID: run.id, AgentStepDraft(kind: .plan, text: "Use key sk-proj-ABCDEFGHIJKLMNOP1234 to call the API", usage: TokenUsage(inputTokens: 100, outputTokens: 40), costUSD: 0.002, createdAt: t(1)))266        let call = try await store.appendAgentStep(runID: run.id, AgentStepDraft(267            kind: .toolCall, toolName: "http.get",268            input: ["url": "https://example.com/r?access_token=tok_live_123456789", "headers": ["Authorization": "Bearer zzzzYYYY99998888", "X-Api-Key": "k-1234567890"],269                    "password": "abc", "max_tokens": 256, "nested": [["api_key": "short"], "sk-ant-api03-SECRETSECRETSECRET"]],270            createdAt: t(2)))271        try await store.appendAgentStep(runID: run.id, AgentStepDraft(kind: .approvalRequest, toolName: "http.get", text: "Allow http.get?", createdAt: t(3)))272        try await store.completeAgentStep(id: call.id, output: ["status": 200, "body": "token=hf_abcdefghijklmnopqrstuv done", "secret": "s3cr3t-value"], approval: .approved, at: t(4))273        try await store.appendAgentStep(runID: run.id, AgentStepDraft(kind: .final, text: "Done.", usage: TokenUsage(inputTokens: 300, outputTokens: 20, cachedInputTokens: 250), costUSD: 0.001, createdAt: t(5)))274275        let steps = try await store.agentSteps(runID: run.id)276        #expect(steps.map(\.ordinal) == [0, 1, 2, 3])277        #expect(steps.map(\.kind) == [.plan, .toolCall, .approvalRequest, .final])278        let everything = steps.map { [$0.text, $0.inputJSON ?? "", $0.outputJSON ?? ""].joined(separator: "\n") }.joined(separator: "\n")279        for secret in ["sk-proj-ABCDEFGHIJKLMNOP1234", "tok_live_123456789", "zzzzYYYY99998888", "k-1234567890", "\"abc\"", "\"short\"", "SECRETSECRETSECRET",280                       "hf_abcdefghijklmnopqrstuv", "s3cr3t-value"] {281            #expect(!everything.contains(secret), "Leaked \(secret)")282        }283        // What is not a secret stays readable and the JSON stays valid.284        let input = try #require(steps[1].input)285        #expect(input["url"]?.stringValue?.hasPrefix("https://example.com/r?access_token=") == true)286        #expect(input["max_tokens"] == 256)287        #expect(input["password"] == "•••")288        let output = try #require(steps[1].output)289        #expect(output["status"] == 200)290        #expect(steps[1].approval == .approved)291        #expect(steps[1].finishedAt == t(4))292        #expect(steps[0].text.hasPrefix("Use key sk-proj-•••"))293294        var saved = try #require(try await store.agentRun(id: run.id))295        #expect(saved.stepCount == 4)296        #expect(saved.toolCallCount == 1)297        #expect(saved.usage == TokenUsage(inputTokens: 400, outputTokens: 60, cachedInputTokens: 250))298        #expect(abs((saved.costUSD ?? 0) - 0.003) < 1e-12)299        #expect(saved.profileName == "Ops")300        try await store.setAgentRunStatus(id: run.id, .awaitingApproval)301        #expect(try await store.agentRun(id: run.id)?.finishedAt == nil)302        try await store.setAgentRunStatus(id: run.id, .completed, result: "Report fetched, password=hunter2hunter2", at: t(9))303        saved = try #require(try await store.agentRun(id: run.id))304        #expect(saved.status == .completed)305        #expect(saved.finishedAt == t(9))306        #expect(saved.result?.contains("hunter2") == false)307        try await store.recordAgentRunUsage(runID: run.id)308        let ledger = try #require(try await store.usageRecords().first)309        #expect(ledger.origin == .agent)310        #expect(ledger.inputTokens == 400)311312        // Deleting the profile keeps the audit trail; deleting the run removes its steps.313        try await store.deleteAgentProfile(id: profile.id)314        #expect(try await store.agentRun(id: run.id)?.profileID == nil)315        #expect(try await store.agentRuns().count == 1)316        #expect(try await store.agentRuns(profileID: profile.id).isEmpty)317        try await store.deleteAgentRun(id: run.id)318        #expect(try await store.agentSteps(runID: run.id).isEmpty)319        await #expect(throws: AstraStoreError.self) { try await store.appendAgentStep(runID: run.id, AgentStepDraft(kind: .note)) }320    }321322    @Test func concurrentStepsGetDistinctOrdinalsAndInterruptedRunsAreClosed() async throws {323        let store = try makeStore()324        let unsaved = AgentProfileRecord(name: "Ad hoc", model: gpt)325        let run = try await store.startAgentRun(profile: unsaved, goal: "g")326        #expect(run.profileID == nil)327        try await withThrowingTaskGroup(of: Void.self) { group in328            for index in 0..<40 { group.addTask { try await store.appendAgentStep(runID: run.id, AgentStepDraft(kind: .note, text: "n\(index)")) } }329            try await group.waitForAll()330        }331        #expect(try await store.agentSteps(runID: run.id).map(\.ordinal) == Array(0..<40))332        #expect(try await store.agentRun(id: run.id)?.stepCount == 40)333        #expect(try await store.recoverInterruptedAgentRuns(at: t(3)) == 1)334        #expect(try await store.agentRun(id: run.id)?.status == .cancelled)335        #expect(try await store.recoverInterruptedAgentRuns() == 0)336        #expect(AgentRunStatus.budgetExceeded.isFinished)337        #expect(!AgentRunStatus.awaitingApproval.isFinished)338    }339}340341@Suite struct PromptStudioTests {342    @Test func everySaveAppendsAVersionAndRestoreNeverRewritesHistory() async throws {343        let store = try makeStore()344        let v1 = try await store.saveTemplate(name: "Summary", system: "You summarize.", userTemplate: "Summarize {{text}} in {{count}} points.",345                                              parameters: GenerationParameters(temperature: 0.3), at: t(1))346        #expect(v1.currentVersion == 1)347        #expect(v1.variables == ["text", "count"])348        let v2 = try await store.saveTemplate(id: v1.id, name: "Summary", system: "You summarize for {{audience}}.", userTemplate: "Summarize {{text}}.",349                                              note: "audience", at: t(2))350        let v3 = try await store.saveTemplate(id: v1.id, name: "Summary (short)", system: "", userTemplate: "TL;DR {{text}}", at: t(3))351        #expect(v2.currentVersion == 2)352        #expect(v3.currentVersion == 3)353        #expect(v3.createdAt == t(1))354        #expect(v3.updatedAt == t(3))355        var versions = try await store.templateVersions(templateID: v1.id)356        #expect(versions.map(\.version) == [3, 2, 1])357        #expect(versions[1].note == "audience")358        #expect(versions[2].userTemplate == "Summarize {{text}} in {{count}} points.")359        #expect(versions[2].parameters == GenerationParameters(temperature: 0.3))360361        let restored = try await store.restoreTemplateVersion(templateID: v1.id, version: 1, at: t(4))362        #expect(restored.currentVersion == 4)363        #expect(restored.name == "Summary")364        #expect(restored.userTemplate == "Summarize {{text}} in {{count}} points.")365        #expect(restored.parameters == GenerationParameters(temperature: 0.3))366        versions = try await store.templateVersions(templateID: v1.id)367        #expect(versions.map(\.version) == [4, 3, 2, 1])368        #expect(versions[0].note == "Restored from version 1")369        #expect(versions[1].userTemplate == "TL;DR {{text}}")370        #expect(try await store.template(id: v1.id) == restored)371        await #expect(throws: AstraStoreError.self) { try await store.restoreTemplateVersion(templateID: v1.id, version: 99) }372373        try await store.saveTemplate(name: "another", system: "", userTemplate: "x")374        #expect(try await store.templates().map(\.name) == ["another", "Summary"])375        try await store.deleteTemplate(id: v1.id)376        #expect(try await store.templateVersions(templateID: v1.id).isEmpty)377        #expect(try await store.templates().count == 1)378    }379380    @Test func variablesAreExtractedInOrder() {381        #expect(PromptRenderer.variables(in: "Hello {{name}}, meet {{ other_name }} and {{name}} again. {{a.b-c}} {{été}} {{x1}}") == ["name", "other_name", "a.b-c", "été", "x1"])382        #expect(PromptRenderer.variables(in: "No {placeholders} here, {{}} {{ }} {{bad name}} {{-x}} {{{triple}}}") == ["triple"])383        #expect(PromptRenderer.variables(in: "") == [])384    }385386    @Test func renderingReportsMissingAndUnusedVariables() {387        let template = "Dear {{name}},\nYour order {{ order }} ships to {{city}}. Bye {{name}}."388        let complete = PromptRenderer.render(template, values: ["name": "Ada", "order": "#42", "city": "Québec"])389        #expect(complete.text == "Dear Ada,\nYour order #42 ships to Québec. Bye Ada.")390        #expect(complete.isComplete)391        #expect(complete.unused.isEmpty)392        let partial = PromptRenderer.render(template, values: ["name": "Ada", "extra": "x", "zeta": "z"])393        #expect(partial.text == "Dear Ada,\nYour order {{ order }} ships to {{city}}. Bye Ada.")394        #expect(partial.missing == ["order", "city"])395        #expect(partial.unused == ["extra", "zeta"])396        #expect(!partial.isComplete)397        // Values are inserted literally and never expanded again; an empty value is a value.398        let literal = PromptRenderer.render("{{a}} {{b}} {{c}}", values: ["a": "{{b}}", "b": "$1 \\0 🙂", "c": ""])399        #expect(literal.text == "{{b}} $1 \\0 🙂 ")400        #expect(literal.missing.isEmpty)401        #expect(PromptRenderer.render("plain", values: [:]).text == "plain")402    }403404    @Test func presets() async throws {405        let store = try makeStore()406        var preset = PresetRecord(name: "Tutor", system: "Explain step by step.", parameters: GenerationParameters(temperature: 0.4, reasoning: .medium), createdAt: t(0))407        try await store.savePreset(preset, at: t(0))408        try await store.savePreset(PresetRecord(name: "analyst", system: "Numbers first."))409        preset.system = "Explain patiently."410        try await store.savePreset(preset, at: t(5))411        #expect(try await store.presets().map(\.name) == ["analyst", "Tutor"])412        let saved = try #require(try await store.preset(id: preset.id))413        #expect(saved.system == "Explain patiently.")414        #expect(saved.parameters.reasoning == .medium)415        #expect(saved.updatedAt == t(5))416        try await store.deletePreset(id: preset.id)417        #expect(try await store.presets().count == 1)418    }419}420
Files main
Menu

Theme