Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Tests/AstraProvidersTests/CommonContractTests.swift10// Module : AstraProvidersTests11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraProviders9293/// The six adapters, built over one transport, so the same contract can be checked against each.94enum Adapter: String, CaseIterable, Sendable, CustomTestStringConvertible {95 case openAI, anthropic, openRouter, xAI, google, custom96 var testDescription: String { rawValue }9798 var kind: ProviderKind {99 switch self {100 case .openAI: .openAI101 case .anthropic: .anthropic102 case .openRouter: .openRouter103 case .xAI: .xAI104 case .google: .google105 case .custom: .custom106 }107 }108 /// A key in the provider's own format, plus one (custom) that no redaction pattern recognises.109 var key: String {110 switch self {111 case .openAI: "sk-proj-LEAKCHECK0123456789abcdefLEAKCHECK"112 case .anthropic: "sk-ant-api03-LEAKCHECK0123456789abcdefLEAKCHECK"113 case .openRouter: "sk-or-v1-LEAKCHECK0123456789abcdef0123456789"114 case .xAI: "xai-LEAKCHECK0123456789abcdefLEAKCHECK"115 case .google: "AIzaLEAKCHECK0123456789abcdefLEAKCHECK0"116 case .custom: "leakcheck-opaque-token-7f3a9c"117 }118 }119 var model: String {120 switch self {121 case .openAI: "gpt-6-luna"122 case .anthropic: "claude-haiku-5-5"123 case .openRouter: "openai/gpt-6-luna"124 case .xAI: "grok-4.3"125 case .google: "gemini-3.1-flash-lite"126 case .custom: "maclustr-qwen3"127 }128 }129 var textFixture: String {130 switch self {131 case .openAI: "openai_text.sse"132 case .anthropic: "anthropic_text.sse"133 case .openRouter: "openrouter_text.sse"134 case .xAI: "xai_text.sse"135 case .google: "gemini_text.sse"136 case .custom: "custom_text.sse"137 }138 }139 static let profile = EndpointProfile(id: "ep1", name: "Mine", baseURL: "https://llm.example.com/v1")140141 func make(_ transport: FixtureTransport, key: String?? = .none, diagnostics: RecordingDiagnostics? = nil) -> any ChatProvider {142 let resolved: String? = key ?? self.key143 let keyProvider: KeyProvider = { resolved }144 switch self {145 case .openAI: return OpenAIProvider(transport: transport, apiKey: keyProvider, diagnostics: diagnostics)146 case .anthropic: return AnthropicProvider(transport: transport, apiKey: keyProvider, diagnostics: diagnostics)147 case .openRouter: return OpenRouterProvider(transport: transport, apiKey: keyProvider, diagnostics: diagnostics)148 case .xAI: return XAIProvider(transport: transport, apiKey: keyProvider, diagnostics: diagnostics)149 case .google: return GeminiProvider(transport: transport, apiKey: keyProvider, diagnostics: diagnostics)150 case .custom: return OpenAICompatibleProvider(profile: Self.profile, transport: transport, apiKey: keyProvider, diagnostics: diagnostics)151 }152 }153 var request: GenerationRequest { AstraProvidersTests.request(kind, model, endpointID: self == .custom ? "ep1" : nil) }154155 /// An error body in this provider's dialect.156 func errorBody(type: String, message: String) -> String {157 let m = JSONValue.string(message).serialized()158 switch self {159 case .anthropic: return #"{"type":"error","error":{"type":"\#(type)","message":\#(m)},"request_id":"req_1"}"#160 case .openRouter: return #"{"error":{"code":400,"message":\#(m),"metadata":{"error_type":"\#(type)"}}}"#161 case .xAI: return #"{"code":"\#(type)","error":\#(m)}"#162 case .google: return #"{"error":{"code":400,"message":\#(m),"status":"\#(type.uppercased())"}}"#163 case .openAI, .custom: return #"{"error":{"message":\#(m),"type":"\#(type)","param":null,"code":"\#(type)"}}"#164 }165 }166}167168struct HTTPCase: Sendable, CustomTestStringConvertible {169 var status: Int170 var type: String171 var message: String172 var headers: [String: String] = [:]173 var check: @Sendable (AstraError?, String) -> Bool174 var testDescription: String { "HTTP \(status) \(type)" }175176 static let all: [HTTPCase] = [177 HTTPCase(status: 401, type: "authentication_error", message: "The key is not valid.") { e, _ in if case .unauthorized = e { true } else { false } },178 HTTPCase(status: 402, type: "billing_error", message: "Insufficient credits.") { e, _ in if case .quotaExceeded = e { true } else { false } },179 HTTPCase(status: 403, type: "permission_error", message: "This key may not use that resource.") { e, _ in if case .forbidden = e { true } else { false } },180 HTTPCase(status: 404, type: "not_found_error", message: "The model does not exist.") { e, model in e == .modelNotFound(model) },181 HTTPCase(status: 429, type: "rate_limit_error", message: "Too many requests.", headers: ["Retry-After": "7"]) { e, _ in e == .rateLimited(retryAfter: 7, message: "Too many requests.") },182 HTTPCase(status: 429, type: "insufficient_quota", message: "You exceeded your current quota.") { e, _ in if case .quotaExceeded = e { true } else { false } },183 HTTPCase(status: 400, type: "invalid_request_error", message: "This model's maximum context length is 1000 tokens; the prompt is too long.") { e, _ in if case .contextTooLong = e { true } else { false } },184 HTTPCase(status: 400, type: "invalid_request_error", message: "messages: field required") { e, _ in e == .invalidRequest("messages: field required") },185 HTTPCase(status: 500, type: "api_error", message: "Internal error.") { e, _ in e == .server(status: 500, message: "Internal error.") },186 HTTPCase(status: 502, type: "bad_gateway", message: "Upstream failed.") { e, _ in e == .server(status: 502, message: "Upstream failed.") },187 HTTPCase(status: 503, type: "service_unavailable", message: "Overloaded.", headers: ["Retry-After": "3"]) { e, _ in e == .overloaded("Overloaded.") },188 HTTPCase(status: 529, type: "overloaded_error", message: "Overloaded.") { e, _ in e == .overloaded("Overloaded.") },189 ]190}191192@Suite("Contract shared by every adapter")193struct CommonContractTests {194 @Test(arguments: Adapter.allCases, HTTPCase.all)195 func httpErrorClasses(adapter: Adapter, c: HTTPCase) async {196 let transport = FixtureTransport(.json(c.status, adapter.errorBody(type: c.type, message: c.message), headers: c.headers))197 let diagnostics = RecordingDiagnostics()198 let result = await collect(adapter.make(transport, diagnostics: diagnostics).stream(adapter.request))199 #expect(result.events.isEmpty) // nothing is emitted before a failed status, so a retry is always safe200 #expect(c.check(result.error, adapter.model), "got \(String(describing: result.error))")201 #expect(transport.requests.count == 1)202 #expect(diagnostics.records.map(\.status) == [c.status])203 // The typed error says whether retrying can help.204 #expect(result.error?.isTransient == ([429, 500, 502, 503, 529].contains(c.status) && c.type != "insufficient_quota"))205 }206207 @Test(arguments: Adapter.allCases)208 func modelListAndVerifyMapErrorsToo(adapter: Adapter) async {209 let transport = FixtureTransport(.json(401, adapter.errorBody(type: "authentication_error", message: "The key is not valid.")))210 await #expect(throws: AstraError.unauthorized("The key is not valid.")) { _ = try await adapter.make(transport).verify() }211 }212213 @Test(arguments: Adapter.allCases)214 func missingKey(adapter: Adapter) async {215 let transport = FixtureTransport(.sse(adapter.textFixture))216 for key in [String?.none, "", " "] {217 let result = await collect(adapter.make(transport, key: .some(key)).stream(adapter.request))218 if adapter == .custom {219 #expect(result.error == nil) // a key is optional for a custom endpoint220 } else {221 #expect(result.error == .missingCredential(adapter.kind))222 #expect(result.events.isEmpty)223 }224 }225 if adapter != .custom { #expect(transport.requests.isEmpty) }226 }227228 @Test(arguments: Adapter.allCases)229 func exactlyOneStartAndOneCompletion(adapter: Adapter) async {230 let diagnostics = RecordingDiagnostics()231 let result = await collect(adapter.make(FixtureTransport(.sse(adapter.textFixture)), diagnostics: diagnostics).stream(adapter.request))232 expectWellFormed(result)233 #expect(result.text == "ASTRA-OK")234 #expect(result.starts.first?.reportedModel?.isEmpty == false)235 if case .messageStarted = result.events.first {} else { Issue.record("the first event must be messageStarted") }236 #expect(result.events.last == .messageCompleted(.endOfTurn))237 #expect(result.outcome.usage?.inputTokens != nil)238 #expect(diagnostics.records.count == 1)239 #expect((diagnostics.records.first?.timeToHeaders ?? -1) >= 0)240 }241242 @Test(arguments: Adapter.allCases)243 func keyNeverAppearsInURLErrorsOrDiagnostics(adapter: Adapter) async {244 let key = adapter.key245 let echo = "Rejected credential \(key) (Authorization: Bearer \(key); x-api-key: \(key); ?key=\(key))"246 var seen: [String] = []247 let diagnostics = RecordingDiagnostics()248 let replies: [FixtureTransport.Reply] = [249 .json(401, adapter.errorBody(type: "authentication_error", message: echo)),250 .json(400, adapter.errorBody(type: "invalid_request_error", message: echo)),251 .json(404, adapter.errorBody(type: "not_found_error", message: echo)),252 .json(429, adapter.errorBody(type: "rate_limit_error", message: echo)),253 .json(500, adapter.errorBody(type: "api_error", message: echo)),254 .init(status: 502, body: "<html>\(echo)</html>"),255 .init(failure: .network("could not reach host with \(key)")),256 .init(headers: ["Content-Type": "text/event-stream"], body: "data: {not json \(key)\n\n"),257 .init(headers: ["Content-Type": "text/event-stream"], body: "event: error\ndata: {\"type\":\"error\",\"error\":{\"type\":\"api_error\",\"code\":500,\"message\":\"\(echo)\"},\"code\":\"server_error\",\"message\":\"\(echo)\"}\n\n"),258 ]259 for reply in replies {260 let transport = FixtureTransport(reply)261 let provider = adapter.make(transport, diagnostics: diagnostics)262 let result = await collect(provider.stream(adapter.request))263 if let error = result.error { seen += [error.message, error.title, error.localizedDescription, String(describing: error), String(reflecting: error)] }264 seen += result.events.map { String(describing: $0) }265 for call in [{ _ = try await provider.listModels() }, { _ = try await provider.verify() }] as [@Sendable () async throws -> Void] {266 do { try await call() } catch { seen += [String(describing: error), (error as? AstraError)?.message ?? ""] }267 }268 for request in transport.requests {269 #expect(request.url?.absoluteString.contains(key) == false)270 if let body = request.httpBody { #expect(String(decoding: body, as: UTF8.self).contains(key) == false) }271 }272 }273 seen += diagnostics.records.map { String(describing: $0) }274 #expect(!seen.isEmpty)275 for text in seen { #expect(!text.contains(key), "leaked in: \(Redactor.redact(text).prefix(160))") }276 }277278 @Test(arguments: Adapter.allCases)279 func promptNeverAppearsInErrors(adapter: Adapter) async {280 let secretPrompt = "PRIVATE-PROMPT-7731 my diagnosis is"281 var req = adapter.request282 req.messages = [.user(secretPrompt)]283 for status in [400, 401, 500] {284 let result = await collect(adapter.make(FixtureTransport(.json(status, adapter.errorBody(type: "error", message: "Something failed.")))).stream(req))285 #expect(result.error != nil)286 #expect(result.error?.message.contains("PRIVATE-PROMPT") == false)287 }288 }289290 @Test(arguments: Adapter.allCases)291 func cancellingTheConsumerCancelsTheHTTPStream(adapter: Adapter) async throws {292 var reply = FixtureTransport.Reply.sse(adapter.textFixture)293 // Keep only the opening of the transcript, then leave the connection open.294 reply.body = reply.body.components(separatedBy: "\n\n").prefix(adapter == .anthropic || adapter == .openAI || adapter == .xAI ? 5 : 2).joined(separator: "\n\n") + "\n\n"295 reply.hangs = true296 let transport = FixtureTransport(reply)297 let provider = adapter.make(transport)298 let request = adapter.request299 let task = Task { () -> Int in300 var count = 0301 do { for try await _ in provider.stream(request) { count += 1 } } catch {}302 return count303 }304 // Wait until the request is open and the first events were delivered, then cancel the consumer.305 for _ in 0..<200 where transport.requests.isEmpty { try await Task.sleep(nanoseconds: 5_000_000) }306 try await Task.sleep(nanoseconds: 50_000_000)307 task.cancel()308 let delivered = await task.value309 #expect(delivered >= 1)310 for _ in 0..<200 where transport.cancelledStreams == 0 { try await Task.sleep(nanoseconds: 5_000_000) }311 #expect(transport.cancelledStreams == 1)312 }313314 @Test func toolArgumentsParsing() {315 #expect(ToolArguments.parse("") == [:])316 #expect(ToolArguments.parse(" ") == [:])317 #expect(ToolArguments.parse(#"{"a":1,"b":{"c":[true,null]}}"#) == ["a": 1, "b": ["c": [true, nil]]])318 #expect(ToolArguments.parse(#"{"a":"#) == ["_raw": #"{"a":"#])319 #expect(ToolArguments.parse(#""just a string""#) == ["_raw": #""just a string""#])320 #expect(ToolArguments.parse("[1,2]") == ["_raw": "[1,2]"])321 // A call that never parsed is replayed with its original text, not with a wrapper object.322 #expect(ToolArguments.wireText(["_raw": #"{"a":"#]) == #"{"a":"#)323 #expect(ToolArguments.wireText(["a": 1]) == #"{"a":1}"#)324 #expect(ToolArguments.wireObject(["_raw": "x"]) == ["_raw": "x"])325 }326327 @Test func errorBodyDialects() {328 #expect(ProviderErrorBody.parse(Data(#"{"error":{"message":"m","type":"t","param":"temperature","code":"c"}}"#.utf8)) == ProviderErrorBody(code: "c t", message: "m", param: "temperature"))329 #expect(ProviderErrorBody.parse(Data(#"{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}"#.utf8)) == ProviderErrorBody(code: "overloaded_error", message: "Overloaded"))330 #expect(ProviderErrorBody.parse(Data(#"{"error":{"code":502,"message":"down","metadata":{"error_type":"provider_unavailable"}}}"#.utf8)) == ProviderErrorBody(code: "502 provider_unavailable", message: "down"))331 #expect(ProviderErrorBody.parse(Data(#"{"code":"invalid-argument","error":"bad"}"#.utf8)) == ProviderErrorBody(code: "invalid-argument", message: "bad"))332 #expect(ProviderErrorBody.parse(Data(#"{"detail":"nope"}"#.utf8)).message == "nope")333 #expect(ProviderErrorBody.parse(Data("<html>".utf8)) == ProviderErrorBody())334 let google = ProviderErrorBody.parse(Data(Fixtures.text("gemini_error_429.json").utf8))335 #expect(google.retryDelay == 17)336 #expect(google.code == "429 RESOURCE_EXHAUSTED")337 }338339 @Test func retryAfterForms() async {340 for (header, expected) in [("12", 12.0), ("0", 0.0)] {341 let transport = FixtureTransport(.json(429, Adapter.openAI.errorBody(type: "rate_limit_error", message: "Slow down."), headers: ["Retry-After": header]))342 let result = await collect(Adapter.openAI.make(transport).stream(Adapter.openAI.request))343 #expect(result.error == .rateLimited(retryAfter: expected, message: "Slow down."))344 }345 let ms = FixtureTransport(.json(429, Adapter.openAI.errorBody(type: "rate_limit_error", message: "Slow down."), headers: ["retry-after-ms": "1500"]))346 let result = await collect(Adapter.openAI.make(ms).stream(Adapter.openAI.request))347 #expect(result.error == .rateLimited(retryAfter: 1.5, message: "Slow down."))348 }349350 @Test func registryBuildsTheRightAdapter() throws {351 let transport = FixtureTransport()352 let store = InMemoryCredentialStore([CredentialAccount.provider(.openAI): "sk-a", CredentialAccount.endpoint("ep1"): "custom"])353 let endpoints = [Adapter.profile]354 func make(_ kind: ProviderKind, endpoint: String? = nil) throws -> any ChatProvider {355 try makeProvider(for: ModelRef(provider: kind, modelID: "m", endpointID: endpoint), credentials: store, endpoints: endpoints, transport: transport, diagnostics: nil)356 }357 #expect(try make(.openAI) is OpenAIProvider)358 #expect(try make(.anthropic) is AnthropicProvider)359 #expect(try make(.openRouter) is OpenRouterProvider)360 #expect(try make(.xAI) is XAIProvider)361 #expect(try make(.google) is GeminiProvider)362 #expect(try make(.custom, endpoint: "ep1") is OpenAICompatibleProvider)363 #expect(throws: AstraError.self) { _ = try make(.custom, endpoint: "gone") }364 #expect(throws: AstraError.self) { _ = try make(.custom) }365 #expect(throws: AstraError.self) { _ = try make(.mlx) }366 #expect(throws: AstraError.self) { _ = try make(.appleFoundation) }367 for kind in ProviderKind.cloudKeyed { #expect(try make(kind).kind == kind) }368 }369370 @Test func registryReadsKeysAtRequestTime() async throws {371 let transport = FixtureTransport(.sse("openai_text.sse"))372 let store = InMemoryCredentialStore()373 let provider = try ProviderRegistry.makeProvider(kind: .openAI, credentials: store, transport: transport)374 var result = await collect(provider.stream(Adapter.openAI.request))375 #expect(result.error == .missingCredential(.openAI))376 try store.setSecret("sk-added-later-0123456789", for: CredentialAccount.provider(.openAI))377 result = await collect(provider.stream(Adapter.openAI.request))378 #expect(result.error == nil)379 #expect(transport.requests.first?.value(forHTTPHeaderField: "Authorization") == "Bearer sk-added-later-0123456789")380 // The custom endpoint's key lives under its own account.381 let customTransport = FixtureTransport(.sse("custom_text.sse"))382 try store.setSecret("endpoint-secret", for: CredentialAccount.endpoint("ep1"))383 let custom = try ProviderRegistry.makeProvider(for: ModelRef(provider: .custom, modelID: "m", endpointID: "ep1"), credentials: store, endpoints: [Adapter.profile], transport: customTransport)384 _ = await collect(custom.stream(Adapter.custom.request))385 #expect(customTransport.requests.first?.value(forHTTPHeaderField: "Authorization") == "Bearer endpoint-secret")386 }387}388