SPB Git

Repository Public MIT

StreamTests.swift 12.3 KB 253 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraCoreTests/StreamTests.swift10// Module           : AstraCoreTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2214//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import Testing90@testable import AstraCore9192/// A provider that plays scripted attempts: each attempt is a list of events, optionally ending in a thrown error.93final class ScriptedProvider: ChatProvider, @unchecked Sendable {94    struct Attempt { var events: [StreamEvent]; var error: Error?; var hang = false }95    let kind: ProviderKind = .openAI96    private let lock = NSLock()97    private var attempts: [Attempt]98    private(set) var calls = 099    init(_ attempts: [Attempt]) { self.attempts = attempts }100    func listModels() async throws -> [ModelInfo] { [] }101    func verify() async throws -> ConnectionStatus { ConnectionStatus(ok: true, detail: "ok") }102    func stream(_ request: GenerationRequest) -> AsyncThrowingStream<StreamEvent, Error> {103        let attempt: Attempt = lock.withLock { calls += 1; return attempts.isEmpty ? Attempt(events: []) : attempts.removeFirst() }104        return AsyncThrowingStream { continuation in105            let task = Task {106                for e in attempt.events { continuation.yield(e) }107                if attempt.hang { try? await Task.sleep(nanoseconds: 30_000_000_000) }108                continuation.finish(throwing: attempt.error)109            }110            continuation.onTermination = { _ in task.cancel() }111        }112    }113    var callCount: Int { lock.withLock { calls } }114}115116private let request = GenerationRequest(model: ModelRef(provider: .openAI, modelID: "m"), messages: [.user("hi")])117118private func collect(_ provider: ScriptedProvider, maxRetries: Int = 1) async -> [StreamEvent] {119    let o = InferenceOrchestrator(maxRetries: maxRetries, sleep: { _ in }) { _ in provider }120    var out: [StreamEvent] = []121    for await e in o.stream(request) { out.append(e) }122    return out123}124125@Suite struct ValidatorTests {126    @Test func synthesizesAStartAndTracksToolCalls() throws {127        var v = StreamValidator()128        #expect(try v.accept(.textDelta("a")) == [.messageStarted(MessageStart()), .textDelta("a")])129        #expect(try v.accept(.messageStarted(MessageStart(reportedModel: "late"))) == [])130        _ = try v.accept(.toolCallStarted(id: "1", name: "t"))131        _ = try v.accept(.toolCallDelta(id: "1", argumentsFragment: "{"))132        _ = try v.accept(.toolCallCompleted(ToolCall(id: "1", name: "t", arguments: [:])))133        _ = try v.accept(.messageCompleted(.toolCalls))134        #expect(v.finished)135        #expect(throws: AstraError.self) { try v.accept(.textDelta("late")) }136    }137138    @Test func rejectsOutOfOrderToolEvents() {139        var v = StreamValidator()140        #expect(throws: AstraError.self) { try v.accept(.toolCallDelta(id: "x", argumentsFragment: "{")) }141        var w = StreamValidator()142        _ = try? w.accept(.toolCallStarted(id: "1", name: "t"))143        #expect(throws: AstraError.self) { try w.accept(.toolCallStarted(id: "1", name: "t")) }144        var z = StreamValidator()145        _ = try? z.accept(.toolCallCompleted(ToolCall(id: "1", name: "t", arguments: [:])))146        #expect(throws: AstraError.self) { try z.accept(.toolCallCompleted(ToolCall(id: "1", name: "t", arguments: [:]))) }147    }148}149150@Suite struct OrchestratorTests {151    @Test func forwardsACleanStream() async {152        let p = ScriptedProvider([.init(events: [.messageStarted(MessageStart(reportedModel: "m-1")), .textDelta("Hel"), .textDelta("lo"),153                                                 .usage(TokenUsage(inputTokens: 3, outputTokens: 2)), .messageCompleted(.endOfTurn)])])154        let events = await collect(p)155        #expect(events.count == 5 && events.last == .messageCompleted(.endOfTurn))156        var outcome = GenerationOutcome()157        for e in events { outcome.apply(e) }158        #expect(outcome.text == "Hello" && outcome.start.reportedModel == "m-1" && outcome.usage?.outputTokens == 2 && outcome.error == nil)159    }160161    @Test func alwaysEndsWithExactlyOneTerminalEvent() async {162        let cases: [[ScriptedProvider.Attempt]] = [163            [.init(events: [.textDelta("a")])],                                                            // ends without completing164            [.init(events: [.textDelta("a")], error: AstraError.server(status: 500, message: nil))],        // fails after content165            [.init(events: [.messageCompleted(.endOfTurn), .textDelta("late")])],                           // content after the end166            [.init(events: [.toolCallDelta(id: "x", argumentsFragment: "{")])],                             // protocol violation167            [.init(events: [], error: AstraError.unauthorized(nil))],168        ]169        for attempts in cases {170            let events = await collect(ScriptedProvider(attempts))171            #expect(events.filter(\.isTerminal).count == 1, "\(events)")172            #expect(events.last?.isTerminal == true)173        }174    }175176    @Test func retriesOnlyWhenNothingWasReceived() async {177        let ok: [StreamEvent] = [.textDelta("ok"), .messageCompleted(.endOfTurn)]178        let before = ScriptedProvider([.init(events: [], error: AstraError.overloaded(nil)), .init(events: ok)])179        let a = await collect(before)180        #expect(before.callCount == 2 && a.last == .messageCompleted(.endOfTurn))181182        let after = ScriptedProvider([.init(events: [.textDelta("par")], error: AstraError.overloaded(nil)), .init(events: ok)])183        let b = await collect(after)184        #expect(after.callCount == 1, "a partial answer must never be retried")185        #expect(b.last == .error(.overloaded(nil)))186        #expect(b.contains(.textDelta("par")))187    }188189    @Test func doesNotRetryPermanentErrorsOrBeyondTheLimit() async {190        let auth = ScriptedProvider([.init(events: [], error: AstraError.unauthorized(nil)), .init(events: [.messageCompleted(.endOfTurn)])])191        _ = await collect(auth)192        #expect(auth.callCount == 1)193        let flaky = ScriptedProvider(Array(repeating: .init(events: [], error: AstraError.timeout), count: 5))194        let e = await collect(flaky, maxRetries: 2)195        #expect(flaky.callCount == 3 && e.last == .error(.timeout))196    }197198    @Test func longRateLimitDelaysAreNotWaitedOut() async {199        let p = ScriptedProvider([.init(events: [], error: AstraError.rateLimited(retryAfter: 120, message: nil)), .init(events: [.messageCompleted(.endOfTurn)])])200        let e = await collect(p)201        #expect(p.callCount == 1 && e.last == .error(.rateLimited(retryAfter: 120, message: nil)))202    }203204    @Test func cancellationStopsTheProviderAndTheStream() async {205        let p = ScriptedProvider([.init(events: [.textDelta("a")], hang: true)])206        let o = InferenceOrchestrator(sleep: { _ in }) { _ in p }207        let task = Task { () -> [StreamEvent] in208            var out: [StreamEvent] = []209            for await e in o.stream(request) { out.append(e); if case .textDelta = e { break } }210            return out211        }212        let got = await task.value213        #expect(got.contains(.textDelta("a")))214    }215216    @Test func resolverFailureBecomesATerminalError() async {217        let o = InferenceOrchestrator(sleep: { _ in }) { _ in throw AstraError.missingCredential(.anthropic) }218        var out: [StreamEvent] = []219        for await e in o.stream(request) { out.append(e) }220        #expect(out == [.error(.missingCredential(.anthropic))])221    }222223    @Test func outcomeRecordsTimingAndErrors() {224        var o = GenerationOutcome()225        let t0 = o.timing.startedAt226        o.apply(.textDelta("a"), now: t0.addingTimeInterval(0.5))227        o.apply(.textDelta("b"), now: t0.addingTimeInterval(0.9))228        o.apply(.messageCompleted(.maxTokens), now: t0.addingTimeInterval(2.5))229        #expect(o.timing.timeToFirstToken == 0.5 && o.timing.totalTime == 2.5 && o.stopReason == .maxTokens)230        #expect(o.timing.tokensPerSecond(outputTokens: 40) == 20)231        #expect(o.timing.tokensPerSecond(outputTokens: nil) == nil)232        #expect(o.timing.tokensPerSecond(outputTokens: 60, reasoningTokens: 20) == 20, "hidden reasoning tokens are not visible output")233        // an answer delivered in one burst right after the first token has no meaningful speed234        let burst = GenerationTiming(timeToFirstToken: 2.41, totalTime: 2.47)235        #expect(burst.tokensPerSecond(outputTokens: 119, reasoningTokens: 55) == nil)236        #expect(GenerationTiming(timeToFirstToken: 1, totalTime: 1.4).tokensPerSecond(outputTokens: 500) == nil)237        var f = GenerationOutcome(); f.apply(.error(.cancelled))238        #expect(f.stopReason == .cancelled)239    }240}241242@Suite struct CredentialTests {243    @Test func inMemoryStore() throws {244        let s = InMemoryCredentialStore()245        #expect(try s.secret(for: CredentialAccount.provider(.openAI)) == nil)246        try s.setSecret("  sk-test-value  ", for: CredentialAccount.provider(.openAI))247        #expect(try s.secret(for: "provider.openAI") == "sk-test-value")248        #expect(throws: CredentialError.self) { try s.setSecret("   ", for: "x") }249        try s.removeSecret(for: "provider.openAI")250        #expect(try s.accounts().isEmpty)251    }252}253
Files main
Menu

Theme