SPB Git

Repository Public License

FixtureRetryAndFactoryTests.swift 18.7 KB 314 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX OS Automation9// File             : Packages/ORIXProviderAdapters/Tests/ORIXProviderAdaptersTests/FixtureRetryAndFactoryTests.swift10// Description      : Mocked tests of RetryPolicy math, RetryingProvider (retry11//                    only before the first event), ProviderFactory,12//                    ProviderHealth, ModelRegistry and the DEBUG-only13//                    credential source.14// Module           : ORIXProviderAdaptersTests15// Repository       : https://git.spboucher.ai/orix-automation16// Created          : 2026-11-2117// Last Modified    : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author           : Simon-Pierre Boucher24// Position         : Professor of Finance25// Institution      : Université du Québec en Outaouais (UQO)26// Department       : Département des sciences administratives27// Organization     : SP Boucher28//29// Website          : https://spboucher.ai30// Git              : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General          : info@spboucher.ai37// Professional     : contact@spboucher.ai38// Direct           : simon@spboucher.ai39// Research         : research@spboucher.ai40// Development      : dev@spboucher.ai41// Support          : support@spboucher.ai42// Security/Admin   : admin@spboucher.ai43// Notifications    : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification   : Public70// Distribution     : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License          : Proprietary, all rights reserved (see license file)84// License File     : LICENSE85// Maintainer       : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Testing93import Foundation94import Synchronization95import ORIXCore96import ORIXTestingSupport97@testable import ORIXProviderAdapters9899/// Yields scripted events, then optionally fails: models a stream that breaks after output was delivered.100final class BreakingProvider: LLMProvider {101    let id = ProviderID(rawValue: "breaking")102    let attempts = Mutex(0)103    let script: @Sendable (Int) -> (events: [LLMStreamEvent], error: LLMError?)104    init(script: @escaping @Sendable (Int) -> (events: [LLMStreamEvent], error: LLMError?)) { self.script = script }105    func listModels() async throws -> [ModelInfo] {106        let n = attempts.withLock { $0 += 1; return $0 }107        if let e = script(n).error { throw e }108        return [ModelInfo(id: "m", provider: id)]109    }110    func stream(_ request: LLMRequest) -> AsyncThrowingStream<LLMStreamEvent, Error> {111        let n = attempts.withLock { $0 += 1; return $0 }112        let (events, error) = script(n)113        return AsyncThrowingStream { c in114            for e in events { c.yield(e) }115            c.finish(throwing: error)116        }117    }118}119120final class SleepRecorder: Sendable {121    let waits = Mutex<[Double]>([])122    var sleep: RetryingProvider.Sleep { { seconds in self.waits.withLock { $0.append(seconds) } } }123    var recorded: [Double] { waits.withLock { $0 } }124}125126@Suite("Fixture retry policy and retrying provider (mocked)")127struct FixtureRetryTests {128    @Test func backoffIsExponentialJitteredAndBounded() {129        let p = RetryPolicy(maxAttempts: 5, baseDelaySeconds: 1, multiplier: 2, maxDelaySeconds: 5, jitter: 0.25, totalDelayCapSeconds: 100)130        #expect(p.delay(afterFailedAttempts: 1, error: .overloaded, randomUnit: 0.5) == 1)131        #expect(p.delay(afterFailedAttempts: 2, error: .overloaded, randomUnit: 0.5) == 2)132        #expect(p.delay(afterFailedAttempts: 3, error: .overloaded, randomUnit: 0.5) == 4)133        #expect(p.delay(afterFailedAttempts: 4, error: .overloaded, randomUnit: 0.5) == 5, "capped by maxDelaySeconds")134        #expect(p.delay(afterFailedAttempts: 1, error: .overloaded, randomUnit: 0) == 0.75)135        #expect(p.delay(afterFailedAttempts: 1, error: .overloaded, randomUnit: 1) == 1.25)136        #expect(p.delay(afterFailedAttempts: 5, error: .overloaded, randomUnit: 0.5) == nil, "no wait after the last allowed attempt")137        for _ in 0..<200 {138            let d = p.delay(afterFailedAttempts: 2, error: .network("x"))!139            #expect(d >= 1.5 && d <= 2.5)140        }141    }142143    @Test func onlyRetryableErrorsAreRetried() {144        let p = RetryPolicy()145        for e in [LLMError.rateLimited(retryAfterSeconds: nil), .overloaded, .network("x"), .server(status: 500, message: ""), .server(status: 503, message: "")] {146            #expect(p.delay(afterFailedAttempts: 1, error: e, randomUnit: 0.5) != nil, "\(e)")147        }148        for e in [LLMError.authentication("x"), .invalidRequest("x"), .modelNotFound("x"), .contextTooLong("x"), .unsupported("x"), .decoding("x"), .canceled,149                  .missingCredential(.openai), .server(status: 404, message: "")] {150            #expect(p.delay(afterFailedAttempts: 1, error: e, randomUnit: 0.5) == nil, "\(e)")151        }152        #expect(RetryPolicy.none.delay(afterFailedAttempts: 1, error: .overloaded) == nil)153    }154155    @Test func retryAfterIsHonouredWithinTheTotalCap() {156        let p = RetryPolicy(maxAttempts: 4, baseDelaySeconds: 0.5, maxDelaySeconds: 20, jitter: 0, totalDelayCapSeconds: 30)157        #expect(p.delay(afterFailedAttempts: 1, error: .rateLimited(retryAfterSeconds: 12)) == 12, "never earlier than the server asked")158        #expect(p.delay(afterFailedAttempts: 3, error: .rateLimited(retryAfterSeconds: 0.1)) == 2, "a short retry-after does not shorten the backoff")159        #expect(p.delay(afterFailedAttempts: 1, error: .rateLimited(retryAfterSeconds: 120)) == nil, "a wait beyond the total cap is not started")160        #expect(p.delay(afterFailedAttempts: 2, error: .overloaded, elapsedDelaySeconds: 29.5) == nil)161        #expect(p.delay(afterFailedAttempts: 2, error: .overloaded, elapsedDelaySeconds: 29) == 1)162    }163164    @Test func retriesBeforeTheFirstEventThenSucceeds() async throws {165        let scripted = ScriptedProvider(turns: [.failure(.overloaded), .failure(.rateLimited(retryAfterSeconds: 3)), ScriptedProvider.text("done")])166        let recorder = SleepRecorder()167        let observed = Mutex<[String]>([])168        let retrying = RetryingProvider(scripted, policy: RetryPolicy(maxAttempts: 4, baseDelaySeconds: 1, jitter: 0),169                                        onRetry: { n, e, d in observed.withLock { $0.append("\(n) \(e) \(d)") } }, sleep: recorder.sleep)170        let (events, error) = await collect(retrying.stream(LLMRequest(model: "scripted-1", messages: [.user("x")])))171        #expect(error == nil)172        #expect(events == [.textDelta("done"), .usage(.init(inputTokens: 10, outputTokens: 5)), .stop(.endTurn)], "only the successful attempt's events, nothing synthetic")173        #expect(scripted.requests.count == 3)174        #expect(recorder.recorded == [1, 3])175        #expect(observed.withLock { $0.count } == 2)176        #expect(retrying.id == scripted.id)177    }178179    @Test func neverRetriesOnceAnEventWasDelivered() async {180        let base = BreakingProvider { _ in ([.textDelta("partial")], .network("connection lost")) }181        let recorder = SleepRecorder()182        let (events, error) = await collect(RetryingProvider(base, policy: RetryPolicy(maxAttempts: 5), sleep: recorder.sleep).stream(LLMRequest(model: "m", messages: [.user("x")])))183        #expect(events == [.textDelta("partial")])184        #expect(error == .network("connection lost"))185        #expect(base.attempts.withLock { $0 } == 1, "replaying would duplicate delivered output")186        #expect(recorder.recorded.isEmpty)187    }188189    @Test func doesNotRetryPermanentErrorsAndStopsAtMaxAttempts() async {190        let recorder = SleepRecorder()191        let auth = BreakingProvider { _ in ([], .authentication("bad key")) }192        let a = await collect(RetryingProvider(auth, sleep: recorder.sleep).stream(LLMRequest(model: "m", messages: [.user("x")])))193        #expect(a.error == .authentication("bad key") && auth.attempts.withLock { $0 } == 1)194195        let down = BreakingProvider { _ in ([], .server(status: 503, message: "down")) }196        let b = await collect(RetryingProvider(down, policy: RetryPolicy(maxAttempts: 3, baseDelaySeconds: 0.01), sleep: recorder.sleep).stream(LLMRequest(model: "m", messages: [.user("x")])))197        #expect(b.events.isEmpty && b.error == .server(status: 503, message: "down"))198        #expect(down.attempts.withLock { $0 } == 3)199        #expect(recorder.recorded.count == 2)200    }201202    @Test func listModelsIsRetriedToo() async throws {203        let flaky = BreakingProvider { n in ([], n < 3 ? .network("blip") : nil) }204        let recorder = SleepRecorder()205        let models = try await RetryingProvider(flaky, policy: RetryPolicy(maxAttempts: 3, baseDelaySeconds: 0.01), sleep: recorder.sleep).listModels()206        #expect(models.map(\.id) == ["m"] && recorder.recorded.count == 2)207    }208209    @Test func retriesARealAdapterOn529ThenStreams() async throws {210        let server = StubServer()211            .json(529, #"{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}"#)212            .json(429, #"{"type":"error","error":{"type":"rate_limit_error","message":"slow down"}}"#, headers: ["retry-after": "2"])213            .sse(FixtureAnthropicTests.textTranscript)214        let recorder = SleepRecorder()215        let provider = RetryingProvider(AnthropicProvider(apiKey: fixtureKey, session: server.session, baseURL: server.baseURL),216                                        policy: RetryPolicy(maxAttempts: 3, baseDelaySeconds: 0.5, jitter: 0), sleep: recorder.sleep)217        let response = try await provider.complete(LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")]))218        #expect(response.text == "Bonjour, monde — ✓" && response.stopReason == .endTurn)219        #expect(server.requests.count == 3)220        #expect(recorder.recorded == [0.5, 2], "second wait follows the server's retry-after")221    }222223    @Test func cancellingTheConsumerSurfacesCanceled() async {224        let scripted = ScriptedProvider(turns: [.failure(.overloaded), ScriptedProvider.text("late")])225        let retrying = RetryingProvider(scripted, policy: RetryPolicy(maxAttempts: 3, baseDelaySeconds: 30, jitter: 0, totalDelayCapSeconds: 100))226        let task = Task { await collect(retrying.stream(LLMRequest(model: "scripted-1", messages: [.user("x")]))) }227        try? await Task.sleep(nanoseconds: 100_000_000)228        task.cancel()229        let (events, error) = await task.value230        #expect(events.isEmpty)231        #expect(error == nil || error == .canceled, "a cancelled consumer gets no event; the producer ends as canceled")232        #expect(scripted.requests.count == 1)233    }234}235236@Suite("Fixture factory, health check, registry and dev credentials (mocked)")237struct FixtureFactoryTests {238    @Test func factoryBuildsTheRightAdapterAndRefusesUnknownProviders() async {239        #expect(ProviderFactory.make(.anthropic, apiKey: fixtureKey) is AnthropicProvider)240        #expect(ProviderFactory.make(.openai, apiKey: fixtureKey) is OpenAIProvider)241        #expect(ProviderFactory.make(.openrouter, apiKey: fixtureKey) is OpenRouterProvider)242        #expect(ProviderFactory.makeRetrying(.openai, apiKey: fixtureKey) is RetryingProvider)243        for id in ProviderID.all { #expect(ProviderFactory.make(id, apiKey: fixtureKey).id == id) }244        let unknown = ProviderFactory.make(ProviderID(rawValue: "acme"), apiKey: fixtureKey)245        let (events, error) = await collect(unknown.stream(LLMRequest(model: "m", messages: [.user("x")])))246        #expect(events.isEmpty && error == .unsupported("No adapter exists for provider \"acme\"."))247        await #expect(throws: LLMError.self) { _ = try await unknown.listModels() }248    }249250    @Test func healthCheckSendsAMinimalRequestAndReportsTypedFailures() async {251        let server = StubServer().sse(FixtureAnthropicTests.textTranscript).json(401, #"{"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}"#)252        let provider = AnthropicProvider(apiKey: fixtureKey, session: server.session, baseURL: server.baseURL)253        let ok = await ProviderHealth.check(provider: provider, model: "claude-haiku-5-5")254        #expect(ok.ok && ok.error == nil && ok.message == nil && ok.provider == .anthropic && ok.model == "claude-haiku-5-5")255        #expect(ok.latencyMs >= 0 && ok.firstEventMs != nil && ok.usage?.outputTokens == 9)256        #expect(server.requests[0].body?["max_tokens"] == 8, "the probe is capped at 8 output tokens")257        #expect(server.requests[0].body?["tools"] == nil)258259        let bad = await ProviderHealth.check(provider: provider, model: "claude-haiku-5-5")260        #expect(!bad.ok && bad.error == .authentication("invalid x-api-key"))261        #expect(bad.message?.contains("rejected the API key") == true && bad.message?.contains(fixtureKey) == false)262263        let offline = await ProviderHealth.check(provider: AnthropicProvider(apiKey: fixtureKey, session: StubServer().enqueue(.init(failure: .notConnectedToInternet)).session,264                                                                              baseURL: URL(string: "https://unregistered-host.test")!), model: "m")265        if case .network = offline.error {} else { Issue.record("expected a network error, got \(String(describing: offline.error))") }266        #expect(!offline.ok)267    }268269    @Test func registryIsDatedPricedAndNeverGuesses() {270        #expect(ModelRegistry.lastVerified.range(of: #"^\d{4}-\d{2}-\d{2}$"#, options: .regularExpression) != nil)271        let anthropic = ModelRegistry.models(for: .anthropic), openai = ModelRegistry.models(for: .openai)272        #expect(!anthropic.isEmpty && !openai.isEmpty && ModelRegistry.models(for: .openrouter).isEmpty)273        #expect((anthropic + openai).allSatisfy { $0.source == "registry" })274        #expect(Set(ModelRegistry.entries.map { "\($0.provider)/\($0.id)" }).count == ModelRegistry.entries.count, "no duplicate ids")275        for e in ModelRegistry.entries {276            if let i = e.inputPricePerMTok, let o = e.outputPricePerMTok { #expect(i > 0 && o >= i, "\(e.id)") }277            if let r = e.cacheReadPricePerMTok, let i = e.inputPricePerMTok { #expect(r < i, "\(e.id)") }278        }279        let haiku = ModelRegistry.info(provider: .anthropic, id: "claude-haiku-5-5")280        #expect(haiku?.inputPricePerMTok == 0.10 && haiku?.outputPricePerMTok == 0.50 && haiku?.contextWindow == 1_000_000)281        #expect(ModelRegistry.info(provider: .anthropic, id: "claude-haiku-4-5-20251001")?.id == "claude-haiku-4-5-20251001")282        #expect(ModelRegistry.info(provider: .openai, id: "gpt-4o-mini-2024-07-18")?.outputPricePerMTok == 0.60)283        #expect(ModelRegistry.info(provider: .openai, id: "gpt-unknown") == nil)284        #expect(ModelRegistry.info(provider: .openai, id: "claude-haiku-5-5") == nil, "entries are per provider")285        #expect(ModelRegistry.entry(provider: .anthropic, id: "claude-opus-5-5")?.acceptsForcedToolChoice == false)286        // Cost from usage and registry prices: 1M in + 1M out on Haiku 5.5 = $0.60.287        #expect(TokenUsage(inputTokens: 1_000_000, outputTokens: 1_000_000).costUSD(model: haiku) == 0.60)288        let merged = ModelRegistry.merge(api: [ModelInfo(id: "gpt-4o-mini", provider: .openai), ModelInfo(id: "x", provider: .openai, inputPricePerMTok: 9)])289        #expect(merged[0].inputPricePerMTok == 0.15 && merged[0].source == "api" && merged[1].inputPricePerMTok == 9)290    }291292    @Test func toolResultWithoutErrorOmitsTheErrorFlag() {293        let p = AnthropicProvider(apiKey: fixtureKey, session: StubServer().session)294        let body = p.requestBody(for: LLMRequest(model: "claude-haiku-5-5", messages: [295            .user("x"), LLMMessage(role: .assistant, content: [.toolUse(LLMToolUse(id: "toolu_fixture_01", name: "t", input: [:]))]),296            LLMMessage(role: .user, content: [.toolResult(toolUseID: "toolu_fixture_01", content: "ok", isError: false)]),297        ])).body298        #expect(body["messages"]?[2]?["content"] == [["type": "tool_result", "tool_use_id": "toolu_fixture_01", "content": "ok"]])299        #expect(ChatCompletionsEncoder.encode(.assistant("hi")) == [["role": "assistant", "content": "hi"]])300    }301302    #if DEBUG303    @Test func devCredentialsReadOnlyTheGivenEnvironment() {304        let creds = DevEnvironmentCredentials(environment: ["ORIX_DEV_ANTHROPIC_API_KEY": "  \(fixtureKey)\n", "ORIX_DEV_OPENAI_API_KEY": "   ", "UNRELATED": "x"])305        #expect(creds.apiKey(for: .anthropic) == fixtureKey)306        #expect(creds.apiKey(for: .openai) == nil, "a blank variable is no key")307        #expect(creds.apiKey(for: .openrouter) == nil && creds.apiKey(for: ProviderID(rawValue: "acme")) == nil)308        #expect(creds.availableProviders == [.anthropic])309        #expect(DevEnvironmentCredentials.variableName(for: .openrouter) == "ORIX_DEV_OPENROUTER_API_KEY")310        #expect(DevEnvironmentCredentials(environment: [:]).availableProviders.isEmpty)311    }312    #endif313}314
Files main
Menu

Theme