Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Tests/AstraCoreTests/StreamTests.swift10// Module : AstraCoreTests11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2214//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import Testing90@testable import AstraCore9192/// A provider that plays scripted attempts: each attempt is a list of events, optionally ending in a thrown error.93final class ScriptedProvider: ChatProvider, @unchecked Sendable {94 struct Attempt { var events: [StreamEvent]; var error: Error?; var hang = false }95 let kind: ProviderKind = .openAI96 private let lock = NSLock()97 private var attempts: [Attempt]98 private(set) var calls = 099 init(_ attempts: [Attempt]) { self.attempts = attempts }100 func listModels() async throws -> [ModelInfo] { [] }101 func verify() async throws -> ConnectionStatus { ConnectionStatus(ok: true, detail: "ok") }102 func stream(_ request: GenerationRequest) -> AsyncThrowingStream<StreamEvent, Error> {103 let attempt: Attempt = lock.withLock { calls += 1; return attempts.isEmpty ? Attempt(events: []) : attempts.removeFirst() }104 return AsyncThrowingStream { continuation in105 let task = Task {106 for e in attempt.events { continuation.yield(e) }107 if attempt.hang { try? await Task.sleep(nanoseconds: 30_000_000_000) }108 continuation.finish(throwing: attempt.error)109 }110 continuation.onTermination = { _ in task.cancel() }111 }112 }113 var callCount: Int { lock.withLock { calls } }114}115116private let request = GenerationRequest(model: ModelRef(provider: .openAI, modelID: "m"), messages: [.user("hi")])117118private func collect(_ provider: ScriptedProvider, maxRetries: Int = 1) async -> [StreamEvent] {119 let o = InferenceOrchestrator(maxRetries: maxRetries, sleep: { _ in }) { _ in provider }120 var out: [StreamEvent] = []121 for await e in o.stream(request) { out.append(e) }122 return out123}124125@Suite struct ValidatorTests {126 @Test func synthesizesAStartAndTracksToolCalls() throws {127 var v = StreamValidator()128 #expect(try v.accept(.textDelta("a")) == [.messageStarted(MessageStart()), .textDelta("a")])129 #expect(try v.accept(.messageStarted(MessageStart(reportedModel: "late"))) == [])130 _ = try v.accept(.toolCallStarted(id: "1", name: "t"))131 _ = try v.accept(.toolCallDelta(id: "1", argumentsFragment: "{"))132 _ = try v.accept(.toolCallCompleted(ToolCall(id: "1", name: "t", arguments: [:])))133 _ = try v.accept(.messageCompleted(.toolCalls))134 #expect(v.finished)135 #expect(throws: AstraError.self) { try v.accept(.textDelta("late")) }136 }137138 @Test func rejectsOutOfOrderToolEvents() {139 var v = StreamValidator()140 #expect(throws: AstraError.self) { try v.accept(.toolCallDelta(id: "x", argumentsFragment: "{")) }141 var w = StreamValidator()142 _ = try? w.accept(.toolCallStarted(id: "1", name: "t"))143 #expect(throws: AstraError.self) { try w.accept(.toolCallStarted(id: "1", name: "t")) }144 var z = StreamValidator()145 _ = try? z.accept(.toolCallCompleted(ToolCall(id: "1", name: "t", arguments: [:])))146 #expect(throws: AstraError.self) { try z.accept(.toolCallCompleted(ToolCall(id: "1", name: "t", arguments: [:]))) }147 }148}149150@Suite struct OrchestratorTests {151 @Test func forwardsACleanStream() async {152 let p = ScriptedProvider([.init(events: [.messageStarted(MessageStart(reportedModel: "m-1")), .textDelta("Hel"), .textDelta("lo"),153 .usage(TokenUsage(inputTokens: 3, outputTokens: 2)), .messageCompleted(.endOfTurn)])])154 let events = await collect(p)155 #expect(events.count == 5 && events.last == .messageCompleted(.endOfTurn))156 var outcome = GenerationOutcome()157 for e in events { outcome.apply(e) }158 #expect(outcome.text == "Hello" && outcome.start.reportedModel == "m-1" && outcome.usage?.outputTokens == 2 && outcome.error == nil)159 }160161 @Test func alwaysEndsWithExactlyOneTerminalEvent() async {162 let cases: [[ScriptedProvider.Attempt]] = [163 [.init(events: [.textDelta("a")])], // ends without completing164 [.init(events: [.textDelta("a")], error: AstraError.server(status: 500, message: nil))], // fails after content165 [.init(events: [.messageCompleted(.endOfTurn), .textDelta("late")])], // content after the end166 [.init(events: [.toolCallDelta(id: "x", argumentsFragment: "{")])], // protocol violation167 [.init(events: [], error: AstraError.unauthorized(nil))],168 ]169 for attempts in cases {170 let events = await collect(ScriptedProvider(attempts))171 #expect(events.filter(\.isTerminal).count == 1, "\(events)")172 #expect(events.last?.isTerminal == true)173 }174 }175176 @Test func retriesOnlyWhenNothingWasReceived() async {177 let ok: [StreamEvent] = [.textDelta("ok"), .messageCompleted(.endOfTurn)]178 let before = ScriptedProvider([.init(events: [], error: AstraError.overloaded(nil)), .init(events: ok)])179 let a = await collect(before)180 #expect(before.callCount == 2 && a.last == .messageCompleted(.endOfTurn))181182 let after = ScriptedProvider([.init(events: [.textDelta("par")], error: AstraError.overloaded(nil)), .init(events: ok)])183 let b = await collect(after)184 #expect(after.callCount == 1, "a partial answer must never be retried")185 #expect(b.last == .error(.overloaded(nil)))186 #expect(b.contains(.textDelta("par")))187 }188189 @Test func doesNotRetryPermanentErrorsOrBeyondTheLimit() async {190 let auth = ScriptedProvider([.init(events: [], error: AstraError.unauthorized(nil)), .init(events: [.messageCompleted(.endOfTurn)])])191 _ = await collect(auth)192 #expect(auth.callCount == 1)193 let flaky = ScriptedProvider(Array(repeating: .init(events: [], error: AstraError.timeout), count: 5))194 let e = await collect(flaky, maxRetries: 2)195 #expect(flaky.callCount == 3 && e.last == .error(.timeout))196 }197198 @Test func longRateLimitDelaysAreNotWaitedOut() async {199 let p = ScriptedProvider([.init(events: [], error: AstraError.rateLimited(retryAfter: 120, message: nil)), .init(events: [.messageCompleted(.endOfTurn)])])200 let e = await collect(p)201 #expect(p.callCount == 1 && e.last == .error(.rateLimited(retryAfter: 120, message: nil)))202 }203204 @Test func cancellationStopsTheProviderAndTheStream() async {205 let p = ScriptedProvider([.init(events: [.textDelta("a")], hang: true)])206 let o = InferenceOrchestrator(sleep: { _ in }) { _ in p }207 let task = Task { () -> [StreamEvent] in208 var out: [StreamEvent] = []209 for await e in o.stream(request) { out.append(e); if case .textDelta = e { break } }210 return out211 }212 let got = await task.value213 #expect(got.contains(.textDelta("a")))214 }215216 @Test func resolverFailureBecomesATerminalError() async {217 let o = InferenceOrchestrator(sleep: { _ in }) { _ in throw AstraError.missingCredential(.anthropic) }218 var out: [StreamEvent] = []219 for await e in o.stream(request) { out.append(e) }220 #expect(out == [.error(.missingCredential(.anthropic))])221 }222223 @Test func outcomeRecordsTimingAndErrors() {224 var o = GenerationOutcome()225 let t0 = o.timing.startedAt226 o.apply(.textDelta("a"), now: t0.addingTimeInterval(0.5))227 o.apply(.textDelta("b"), now: t0.addingTimeInterval(0.9))228 o.apply(.messageCompleted(.maxTokens), now: t0.addingTimeInterval(2.5))229 #expect(o.timing.timeToFirstToken == 0.5 && o.timing.totalTime == 2.5 && o.stopReason == .maxTokens)230 #expect(o.timing.tokensPerSecond(outputTokens: 40) == 20)231 #expect(o.timing.tokensPerSecond(outputTokens: nil) == nil)232 #expect(o.timing.tokensPerSecond(outputTokens: 60, reasoningTokens: 20) == 20, "hidden reasoning tokens are not visible output")233 // an answer delivered in one burst right after the first token has no meaningful speed234 let burst = GenerationTiming(timeToFirstToken: 2.41, totalTime: 2.47)235 #expect(burst.tokensPerSecond(outputTokens: 119, reasoningTokens: 55) == nil)236 #expect(GenerationTiming(timeToFirstToken: 1, totalTime: 1.4).tokensPerSecond(outputTokens: 500) == nil)237 var f = GenerationOutcome(); f.apply(.error(.cancelled))238 #expect(f.stopReason == .cancelled)239 }240}241242@Suite struct CredentialTests {243 @Test func inMemoryStore() throws {244 let s = InMemoryCredentialStore()245 #expect(try s.secret(for: CredentialAccount.provider(.openAI)) == nil)246 try s.setSecret(" sk-test-value ", for: CredentialAccount.provider(.openAI))247 #expect(try s.secret(for: "provider.openAI") == "sk-test-value")248 #expect(throws: CredentialError.self) { try s.setSecret(" ", for: "x") }249 try s.removeSecret(for: "provider.openAI")250 #expect(try s.accounts().isEmpty)251 }252}253