Repository Public License
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX OS Automation9// File : Packages/ORIXProviderAdapters/Tests/ORIXProviderAdaptersTests/FixtureRetryAndFactoryTests.swift10// Description : Mocked tests of RetryPolicy math, RetryingProvider (retry11// only before the first event), ProviderFactory,12// ProviderHealth, ModelRegistry and the DEBUG-only13// credential source.14// Module : ORIXProviderAdaptersTests15// Repository : https://git.spboucher.ai/orix-automation16// Created : 2026-11-2117// Last Modified : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author : Simon-Pierre Boucher24// Position : Professor of Finance25// Institution : Université du Québec en Outaouais (UQO)26// Department : Département des sciences administratives27// Organization : SP Boucher28//29// Website : https://spboucher.ai30// Git : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General : info@spboucher.ai37// Professional : contact@spboucher.ai38// Direct : simon@spboucher.ai39// Research : research@spboucher.ai40// Development : dev@spboucher.ai41// Support : support@spboucher.ai42// Security/Admin : admin@spboucher.ai43// Notifications : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification : Public70// Distribution : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License : Proprietary, all rights reserved (see license file)84// License File : LICENSE85// Maintainer : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Testing93import Foundation94import Synchronization95import ORIXCore96import ORIXTestingSupport97@testable import ORIXProviderAdapters9899/// Yields scripted events, then optionally fails: models a stream that breaks after output was delivered.100final class BreakingProvider: LLMProvider {101 let id = ProviderID(rawValue: "breaking")102 let attempts = Mutex(0)103 let script: @Sendable (Int) -> (events: [LLMStreamEvent], error: LLMError?)104 init(script: @escaping @Sendable (Int) -> (events: [LLMStreamEvent], error: LLMError?)) { self.script = script }105 func listModels() async throws -> [ModelInfo] {106 let n = attempts.withLock { $0 += 1; return $0 }107 if let e = script(n).error { throw e }108 return [ModelInfo(id: "m", provider: id)]109 }110 func stream(_ request: LLMRequest) -> AsyncThrowingStream<LLMStreamEvent, Error> {111 let n = attempts.withLock { $0 += 1; return $0 }112 let (events, error) = script(n)113 return AsyncThrowingStream { c in114 for e in events { c.yield(e) }115 c.finish(throwing: error)116 }117 }118}119120final class SleepRecorder: Sendable {121 let waits = Mutex<[Double]>([])122 var sleep: RetryingProvider.Sleep { { seconds in self.waits.withLock { $0.append(seconds) } } }123 var recorded: [Double] { waits.withLock { $0 } }124}125126@Suite("Fixture retry policy and retrying provider (mocked)")127struct FixtureRetryTests {128 @Test func backoffIsExponentialJitteredAndBounded() {129 let p = RetryPolicy(maxAttempts: 5, baseDelaySeconds: 1, multiplier: 2, maxDelaySeconds: 5, jitter: 0.25, totalDelayCapSeconds: 100)130 #expect(p.delay(afterFailedAttempts: 1, error: .overloaded, randomUnit: 0.5) == 1)131 #expect(p.delay(afterFailedAttempts: 2, error: .overloaded, randomUnit: 0.5) == 2)132 #expect(p.delay(afterFailedAttempts: 3, error: .overloaded, randomUnit: 0.5) == 4)133 #expect(p.delay(afterFailedAttempts: 4, error: .overloaded, randomUnit: 0.5) == 5, "capped by maxDelaySeconds")134 #expect(p.delay(afterFailedAttempts: 1, error: .overloaded, randomUnit: 0) == 0.75)135 #expect(p.delay(afterFailedAttempts: 1, error: .overloaded, randomUnit: 1) == 1.25)136 #expect(p.delay(afterFailedAttempts: 5, error: .overloaded, randomUnit: 0.5) == nil, "no wait after the last allowed attempt")137 for _ in 0..<200 {138 let d = p.delay(afterFailedAttempts: 2, error: .network("x"))!139 #expect(d >= 1.5 && d <= 2.5)140 }141 }142143 @Test func onlyRetryableErrorsAreRetried() {144 let p = RetryPolicy()145 for e in [LLMError.rateLimited(retryAfterSeconds: nil), .overloaded, .network("x"), .server(status: 500, message: ""), .server(status: 503, message: "")] {146 #expect(p.delay(afterFailedAttempts: 1, error: e, randomUnit: 0.5) != nil, "\(e)")147 }148 for e in [LLMError.authentication("x"), .invalidRequest("x"), .modelNotFound("x"), .contextTooLong("x"), .unsupported("x"), .decoding("x"), .canceled,149 .missingCredential(.openai), .server(status: 404, message: "")] {150 #expect(p.delay(afterFailedAttempts: 1, error: e, randomUnit: 0.5) == nil, "\(e)")151 }152 #expect(RetryPolicy.none.delay(afterFailedAttempts: 1, error: .overloaded) == nil)153 }154155 @Test func retryAfterIsHonouredWithinTheTotalCap() {156 let p = RetryPolicy(maxAttempts: 4, baseDelaySeconds: 0.5, maxDelaySeconds: 20, jitter: 0, totalDelayCapSeconds: 30)157 #expect(p.delay(afterFailedAttempts: 1, error: .rateLimited(retryAfterSeconds: 12)) == 12, "never earlier than the server asked")158 #expect(p.delay(afterFailedAttempts: 3, error: .rateLimited(retryAfterSeconds: 0.1)) == 2, "a short retry-after does not shorten the backoff")159 #expect(p.delay(afterFailedAttempts: 1, error: .rateLimited(retryAfterSeconds: 120)) == nil, "a wait beyond the total cap is not started")160 #expect(p.delay(afterFailedAttempts: 2, error: .overloaded, elapsedDelaySeconds: 29.5) == nil)161 #expect(p.delay(afterFailedAttempts: 2, error: .overloaded, elapsedDelaySeconds: 29) == 1)162 }163164 @Test func retriesBeforeTheFirstEventThenSucceeds() async throws {165 let scripted = ScriptedProvider(turns: [.failure(.overloaded), .failure(.rateLimited(retryAfterSeconds: 3)), ScriptedProvider.text("done")])166 let recorder = SleepRecorder()167 let observed = Mutex<[String]>([])168 let retrying = RetryingProvider(scripted, policy: RetryPolicy(maxAttempts: 4, baseDelaySeconds: 1, jitter: 0),169 onRetry: { n, e, d in observed.withLock { $0.append("\(n) \(e) \(d)") } }, sleep: recorder.sleep)170 let (events, error) = await collect(retrying.stream(LLMRequest(model: "scripted-1", messages: [.user("x")])))171 #expect(error == nil)172 #expect(events == [.textDelta("done"), .usage(.init(inputTokens: 10, outputTokens: 5)), .stop(.endTurn)], "only the successful attempt's events, nothing synthetic")173 #expect(scripted.requests.count == 3)174 #expect(recorder.recorded == [1, 3])175 #expect(observed.withLock { $0.count } == 2)176 #expect(retrying.id == scripted.id)177 }178179 @Test func neverRetriesOnceAnEventWasDelivered() async {180 let base = BreakingProvider { _ in ([.textDelta("partial")], .network("connection lost")) }181 let recorder = SleepRecorder()182 let (events, error) = await collect(RetryingProvider(base, policy: RetryPolicy(maxAttempts: 5), sleep: recorder.sleep).stream(LLMRequest(model: "m", messages: [.user("x")])))183 #expect(events == [.textDelta("partial")])184 #expect(error == .network("connection lost"))185 #expect(base.attempts.withLock { $0 } == 1, "replaying would duplicate delivered output")186 #expect(recorder.recorded.isEmpty)187 }188189 @Test func doesNotRetryPermanentErrorsAndStopsAtMaxAttempts() async {190 let recorder = SleepRecorder()191 let auth = BreakingProvider { _ in ([], .authentication("bad key")) }192 let a = await collect(RetryingProvider(auth, sleep: recorder.sleep).stream(LLMRequest(model: "m", messages: [.user("x")])))193 #expect(a.error == .authentication("bad key") && auth.attempts.withLock { $0 } == 1)194195 let down = BreakingProvider { _ in ([], .server(status: 503, message: "down")) }196 let b = await collect(RetryingProvider(down, policy: RetryPolicy(maxAttempts: 3, baseDelaySeconds: 0.01), sleep: recorder.sleep).stream(LLMRequest(model: "m", messages: [.user("x")])))197 #expect(b.events.isEmpty && b.error == .server(status: 503, message: "down"))198 #expect(down.attempts.withLock { $0 } == 3)199 #expect(recorder.recorded.count == 2)200 }201202 @Test func listModelsIsRetriedToo() async throws {203 let flaky = BreakingProvider { n in ([], n < 3 ? .network("blip") : nil) }204 let recorder = SleepRecorder()205 let models = try await RetryingProvider(flaky, policy: RetryPolicy(maxAttempts: 3, baseDelaySeconds: 0.01), sleep: recorder.sleep).listModels()206 #expect(models.map(\.id) == ["m"] && recorder.recorded.count == 2)207 }208209 @Test func retriesARealAdapterOn529ThenStreams() async throws {210 let server = StubServer()211 .json(529, #"{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}"#)212 .json(429, #"{"type":"error","error":{"type":"rate_limit_error","message":"slow down"}}"#, headers: ["retry-after": "2"])213 .sse(FixtureAnthropicTests.textTranscript)214 let recorder = SleepRecorder()215 let provider = RetryingProvider(AnthropicProvider(apiKey: fixtureKey, session: server.session, baseURL: server.baseURL),216 policy: RetryPolicy(maxAttempts: 3, baseDelaySeconds: 0.5, jitter: 0), sleep: recorder.sleep)217 let response = try await provider.complete(LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")]))218 #expect(response.text == "Bonjour, monde — ✓" && response.stopReason == .endTurn)219 #expect(server.requests.count == 3)220 #expect(recorder.recorded == [0.5, 2], "second wait follows the server's retry-after")221 }222223 @Test func cancellingTheConsumerSurfacesCanceled() async {224 let scripted = ScriptedProvider(turns: [.failure(.overloaded), ScriptedProvider.text("late")])225 let retrying = RetryingProvider(scripted, policy: RetryPolicy(maxAttempts: 3, baseDelaySeconds: 30, jitter: 0, totalDelayCapSeconds: 100))226 let task = Task { await collect(retrying.stream(LLMRequest(model: "scripted-1", messages: [.user("x")]))) }227 try? await Task.sleep(nanoseconds: 100_000_000)228 task.cancel()229 let (events, error) = await task.value230 #expect(events.isEmpty)231 #expect(error == nil || error == .canceled, "a cancelled consumer gets no event; the producer ends as canceled")232 #expect(scripted.requests.count == 1)233 }234}235236@Suite("Fixture factory, health check, registry and dev credentials (mocked)")237struct FixtureFactoryTests {238 @Test func factoryBuildsTheRightAdapterAndRefusesUnknownProviders() async {239 #expect(ProviderFactory.make(.anthropic, apiKey: fixtureKey) is AnthropicProvider)240 #expect(ProviderFactory.make(.openai, apiKey: fixtureKey) is OpenAIProvider)241 #expect(ProviderFactory.make(.openrouter, apiKey: fixtureKey) is OpenRouterProvider)242 #expect(ProviderFactory.makeRetrying(.openai, apiKey: fixtureKey) is RetryingProvider)243 for id in ProviderID.all { #expect(ProviderFactory.make(id, apiKey: fixtureKey).id == id) }244 let unknown = ProviderFactory.make(ProviderID(rawValue: "acme"), apiKey: fixtureKey)245 let (events, error) = await collect(unknown.stream(LLMRequest(model: "m", messages: [.user("x")])))246 #expect(events.isEmpty && error == .unsupported("No adapter exists for provider \"acme\"."))247 await #expect(throws: LLMError.self) { _ = try await unknown.listModels() }248 }249250 @Test func healthCheckSendsAMinimalRequestAndReportsTypedFailures() async {251 let server = StubServer().sse(FixtureAnthropicTests.textTranscript).json(401, #"{"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}"#)252 let provider = AnthropicProvider(apiKey: fixtureKey, session: server.session, baseURL: server.baseURL)253 let ok = await ProviderHealth.check(provider: provider, model: "claude-haiku-5-5")254 #expect(ok.ok && ok.error == nil && ok.message == nil && ok.provider == .anthropic && ok.model == "claude-haiku-5-5")255 #expect(ok.latencyMs >= 0 && ok.firstEventMs != nil && ok.usage?.outputTokens == 9)256 #expect(server.requests[0].body?["max_tokens"] == 8, "the probe is capped at 8 output tokens")257 #expect(server.requests[0].body?["tools"] == nil)258259 let bad = await ProviderHealth.check(provider: provider, model: "claude-haiku-5-5")260 #expect(!bad.ok && bad.error == .authentication("invalid x-api-key"))261 #expect(bad.message?.contains("rejected the API key") == true && bad.message?.contains(fixtureKey) == false)262263 let offline = await ProviderHealth.check(provider: AnthropicProvider(apiKey: fixtureKey, session: StubServer().enqueue(.init(failure: .notConnectedToInternet)).session,264 baseURL: URL(string: "https://unregistered-host.test")!), model: "m")265 if case .network = offline.error {} else { Issue.record("expected a network error, got \(String(describing: offline.error))") }266 #expect(!offline.ok)267 }268269 @Test func registryIsDatedPricedAndNeverGuesses() {270 #expect(ModelRegistry.lastVerified.range(of: #"^\d{4}-\d{2}-\d{2}$"#, options: .regularExpression) != nil)271 let anthropic = ModelRegistry.models(for: .anthropic), openai = ModelRegistry.models(for: .openai)272 #expect(!anthropic.isEmpty && !openai.isEmpty && ModelRegistry.models(for: .openrouter).isEmpty)273 #expect((anthropic + openai).allSatisfy { $0.source == "registry" })274 #expect(Set(ModelRegistry.entries.map { "\($0.provider)/\($0.id)" }).count == ModelRegistry.entries.count, "no duplicate ids")275 for e in ModelRegistry.entries {276 if let i = e.inputPricePerMTok, let o = e.outputPricePerMTok { #expect(i > 0 && o >= i, "\(e.id)") }277 if let r = e.cacheReadPricePerMTok, let i = e.inputPricePerMTok { #expect(r < i, "\(e.id)") }278 }279 let haiku = ModelRegistry.info(provider: .anthropic, id: "claude-haiku-5-5")280 #expect(haiku?.inputPricePerMTok == 0.10 && haiku?.outputPricePerMTok == 0.50 && haiku?.contextWindow == 1_000_000)281 #expect(ModelRegistry.info(provider: .anthropic, id: "claude-haiku-4-5-20251001")?.id == "claude-haiku-4-5-20251001")282 #expect(ModelRegistry.info(provider: .openai, id: "gpt-4o-mini-2024-07-18")?.outputPricePerMTok == 0.60)283 #expect(ModelRegistry.info(provider: .openai, id: "gpt-unknown") == nil)284 #expect(ModelRegistry.info(provider: .openai, id: "claude-haiku-5-5") == nil, "entries are per provider")285 #expect(ModelRegistry.entry(provider: .anthropic, id: "claude-opus-5-5")?.acceptsForcedToolChoice == false)286 // Cost from usage and registry prices: 1M in + 1M out on Haiku 5.5 = $0.60.287 #expect(TokenUsage(inputTokens: 1_000_000, outputTokens: 1_000_000).costUSD(model: haiku) == 0.60)288 let merged = ModelRegistry.merge(api: [ModelInfo(id: "gpt-4o-mini", provider: .openai), ModelInfo(id: "x", provider: .openai, inputPricePerMTok: 9)])289 #expect(merged[0].inputPricePerMTok == 0.15 && merged[0].source == "api" && merged[1].inputPricePerMTok == 9)290 }291292 @Test func toolResultWithoutErrorOmitsTheErrorFlag() {293 let p = AnthropicProvider(apiKey: fixtureKey, session: StubServer().session)294 let body = p.requestBody(for: LLMRequest(model: "claude-haiku-5-5", messages: [295 .user("x"), LLMMessage(role: .assistant, content: [.toolUse(LLMToolUse(id: "toolu_fixture_01", name: "t", input: [:]))]),296 LLMMessage(role: .user, content: [.toolResult(toolUseID: "toolu_fixture_01", content: "ok", isError: false)]),297 ])).body298 #expect(body["messages"]?[2]?["content"] == [["type": "tool_result", "tool_use_id": "toolu_fixture_01", "content": "ok"]])299 #expect(ChatCompletionsEncoder.encode(.assistant("hi")) == [["role": "assistant", "content": "hi"]])300 }301302 #if DEBUG303 @Test func devCredentialsReadOnlyTheGivenEnvironment() {304 let creds = DevEnvironmentCredentials(environment: ["ORIX_DEV_ANTHROPIC_API_KEY": " \(fixtureKey)\n", "ORIX_DEV_OPENAI_API_KEY": " ", "UNRELATED": "x"])305 #expect(creds.apiKey(for: .anthropic) == fixtureKey)306 #expect(creds.apiKey(for: .openai) == nil, "a blank variable is no key")307 #expect(creds.apiKey(for: .openrouter) == nil && creds.apiKey(for: ProviderID(rawValue: "acme")) == nil)308 #expect(creds.availableProviders == [.anthropic])309 #expect(DevEnvironmentCredentials.variableName(for: .openrouter) == "ORIX_DEV_OPENROUTER_API_KEY")310 #expect(DevEnvironmentCredentials(environment: [:]).availableProviders.isEmpty)311 }312 #endif313}314