SPB Git

Repository Public MIT

CommonContractTests.swift 21.9 KB 388 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraProvidersTests/CommonContractTests.swift10// Module           : AstraProvidersTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraProviders9293/// The six adapters, built over one transport, so the same contract can be checked against each.94enum Adapter: String, CaseIterable, Sendable, CustomTestStringConvertible {95    case openAI, anthropic, openRouter, xAI, google, custom96    var testDescription: String { rawValue }9798    var kind: ProviderKind {99        switch self {100        case .openAI: .openAI101        case .anthropic: .anthropic102        case .openRouter: .openRouter103        case .xAI: .xAI104        case .google: .google105        case .custom: .custom106        }107    }108    /// A key in the provider's own format, plus one (custom) that no redaction pattern recognises.109    var key: String {110        switch self {111        case .openAI: "sk-proj-LEAKCHECK0123456789abcdefLEAKCHECK"112        case .anthropic: "sk-ant-api03-LEAKCHECK0123456789abcdefLEAKCHECK"113        case .openRouter: "sk-or-v1-LEAKCHECK0123456789abcdef0123456789"114        case .xAI: "xai-LEAKCHECK0123456789abcdefLEAKCHECK"115        case .google: "AIzaLEAKCHECK0123456789abcdefLEAKCHECK0"116        case .custom: "leakcheck-opaque-token-7f3a9c"117        }118    }119    var model: String {120        switch self {121        case .openAI: "gpt-6-luna"122        case .anthropic: "claude-haiku-5-5"123        case .openRouter: "openai/gpt-6-luna"124        case .xAI: "grok-4.3"125        case .google: "gemini-3.1-flash-lite"126        case .custom: "maclustr-qwen3"127        }128    }129    var textFixture: String {130        switch self {131        case .openAI: "openai_text.sse"132        case .anthropic: "anthropic_text.sse"133        case .openRouter: "openrouter_text.sse"134        case .xAI: "xai_text.sse"135        case .google: "gemini_text.sse"136        case .custom: "custom_text.sse"137        }138    }139    static let profile = EndpointProfile(id: "ep1", name: "Mine", baseURL: "https://llm.example.com/v1")140141    func make(_ transport: FixtureTransport, key: String?? = .none, diagnostics: RecordingDiagnostics? = nil) -> any ChatProvider {142        let resolved: String? = key ?? self.key143        let keyProvider: KeyProvider = { resolved }144        switch self {145        case .openAI: return OpenAIProvider(transport: transport, apiKey: keyProvider, diagnostics: diagnostics)146        case .anthropic: return AnthropicProvider(transport: transport, apiKey: keyProvider, diagnostics: diagnostics)147        case .openRouter: return OpenRouterProvider(transport: transport, apiKey: keyProvider, diagnostics: diagnostics)148        case .xAI: return XAIProvider(transport: transport, apiKey: keyProvider, diagnostics: diagnostics)149        case .google: return GeminiProvider(transport: transport, apiKey: keyProvider, diagnostics: diagnostics)150        case .custom: return OpenAICompatibleProvider(profile: Self.profile, transport: transport, apiKey: keyProvider, diagnostics: diagnostics)151        }152    }153    var request: GenerationRequest { AstraProvidersTests.request(kind, model, endpointID: self == .custom ? "ep1" : nil) }154155    /// An error body in this provider's dialect.156    func errorBody(type: String, message: String) -> String {157        let m = JSONValue.string(message).serialized()158        switch self {159        case .anthropic: return #"{"type":"error","error":{"type":"\#(type)","message":\#(m)},"request_id":"req_1"}"#160        case .openRouter: return #"{"error":{"code":400,"message":\#(m),"metadata":{"error_type":"\#(type)"}}}"#161        case .xAI: return #"{"code":"\#(type)","error":\#(m)}"#162        case .google: return #"{"error":{"code":400,"message":\#(m),"status":"\#(type.uppercased())"}}"#163        case .openAI, .custom: return #"{"error":{"message":\#(m),"type":"\#(type)","param":null,"code":"\#(type)"}}"#164        }165    }166}167168struct HTTPCase: Sendable, CustomTestStringConvertible {169    var status: Int170    var type: String171    var message: String172    var headers: [String: String] = [:]173    var check: @Sendable (AstraError?, String) -> Bool174    var testDescription: String { "HTTP \(status) \(type)" }175176    static let all: [HTTPCase] = [177        HTTPCase(status: 401, type: "authentication_error", message: "The key is not valid.") { e, _ in if case .unauthorized = e { true } else { false } },178        HTTPCase(status: 402, type: "billing_error", message: "Insufficient credits.") { e, _ in if case .quotaExceeded = e { true } else { false } },179        HTTPCase(status: 403, type: "permission_error", message: "This key may not use that resource.") { e, _ in if case .forbidden = e { true } else { false } },180        HTTPCase(status: 404, type: "not_found_error", message: "The model does not exist.") { e, model in e == .modelNotFound(model) },181        HTTPCase(status: 429, type: "rate_limit_error", message: "Too many requests.", headers: ["Retry-After": "7"]) { e, _ in e == .rateLimited(retryAfter: 7, message: "Too many requests.") },182        HTTPCase(status: 429, type: "insufficient_quota", message: "You exceeded your current quota.") { e, _ in if case .quotaExceeded = e { true } else { false } },183        HTTPCase(status: 400, type: "invalid_request_error", message: "This model's maximum context length is 1000 tokens; the prompt is too long.") { e, _ in if case .contextTooLong = e { true } else { false } },184        HTTPCase(status: 400, type: "invalid_request_error", message: "messages: field required") { e, _ in e == .invalidRequest("messages: field required") },185        HTTPCase(status: 500, type: "api_error", message: "Internal error.") { e, _ in e == .server(status: 500, message: "Internal error.") },186        HTTPCase(status: 502, type: "bad_gateway", message: "Upstream failed.") { e, _ in e == .server(status: 502, message: "Upstream failed.") },187        HTTPCase(status: 503, type: "service_unavailable", message: "Overloaded.", headers: ["Retry-After": "3"]) { e, _ in e == .overloaded("Overloaded.") },188        HTTPCase(status: 529, type: "overloaded_error", message: "Overloaded.") { e, _ in e == .overloaded("Overloaded.") },189    ]190}191192@Suite("Contract shared by every adapter")193struct CommonContractTests {194    @Test(arguments: Adapter.allCases, HTTPCase.all)195    func httpErrorClasses(adapter: Adapter, c: HTTPCase) async {196        let transport = FixtureTransport(.json(c.status, adapter.errorBody(type: c.type, message: c.message), headers: c.headers))197        let diagnostics = RecordingDiagnostics()198        let result = await collect(adapter.make(transport, diagnostics: diagnostics).stream(adapter.request))199        #expect(result.events.isEmpty)      // nothing is emitted before a failed status, so a retry is always safe200        #expect(c.check(result.error, adapter.model), "got \(String(describing: result.error))")201        #expect(transport.requests.count == 1)202        #expect(diagnostics.records.map(\.status) == [c.status])203        // The typed error says whether retrying can help.204        #expect(result.error?.isTransient == ([429, 500, 502, 503, 529].contains(c.status) && c.type != "insufficient_quota"))205    }206207    @Test(arguments: Adapter.allCases)208    func modelListAndVerifyMapErrorsToo(adapter: Adapter) async {209        let transport = FixtureTransport(.json(401, adapter.errorBody(type: "authentication_error", message: "The key is not valid.")))210        await #expect(throws: AstraError.unauthorized("The key is not valid.")) { _ = try await adapter.make(transport).verify() }211    }212213    @Test(arguments: Adapter.allCases)214    func missingKey(adapter: Adapter) async {215        let transport = FixtureTransport(.sse(adapter.textFixture))216        for key in [String?.none, "", "   "] {217            let result = await collect(adapter.make(transport, key: .some(key)).stream(adapter.request))218            if adapter == .custom {219                #expect(result.error == nil)        // a key is optional for a custom endpoint220            } else {221                #expect(result.error == .missingCredential(adapter.kind))222                #expect(result.events.isEmpty)223            }224        }225        if adapter != .custom { #expect(transport.requests.isEmpty) }226    }227228    @Test(arguments: Adapter.allCases)229    func exactlyOneStartAndOneCompletion(adapter: Adapter) async {230        let diagnostics = RecordingDiagnostics()231        let result = await collect(adapter.make(FixtureTransport(.sse(adapter.textFixture)), diagnostics: diagnostics).stream(adapter.request))232        expectWellFormed(result)233        #expect(result.text == "ASTRA-OK")234        #expect(result.starts.first?.reportedModel?.isEmpty == false)235        if case .messageStarted = result.events.first {} else { Issue.record("the first event must be messageStarted") }236        #expect(result.events.last == .messageCompleted(.endOfTurn))237        #expect(result.outcome.usage?.inputTokens != nil)238        #expect(diagnostics.records.count == 1)239        #expect((diagnostics.records.first?.timeToHeaders ?? -1) >= 0)240    }241242    @Test(arguments: Adapter.allCases)243    func keyNeverAppearsInURLErrorsOrDiagnostics(adapter: Adapter) async {244        let key = adapter.key245        let echo = "Rejected credential \(key) (Authorization: Bearer \(key); x-api-key: \(key); ?key=\(key))"246        var seen: [String] = []247        let diagnostics = RecordingDiagnostics()248        let replies: [FixtureTransport.Reply] = [249            .json(401, adapter.errorBody(type: "authentication_error", message: echo)),250            .json(400, adapter.errorBody(type: "invalid_request_error", message: echo)),251            .json(404, adapter.errorBody(type: "not_found_error", message: echo)),252            .json(429, adapter.errorBody(type: "rate_limit_error", message: echo)),253            .json(500, adapter.errorBody(type: "api_error", message: echo)),254            .init(status: 502, body: "<html>\(echo)</html>"),255            .init(failure: .network("could not reach host with \(key)")),256            .init(headers: ["Content-Type": "text/event-stream"], body: "data: {not json \(key)\n\n"),257            .init(headers: ["Content-Type": "text/event-stream"], body: "event: error\ndata: {\"type\":\"error\",\"error\":{\"type\":\"api_error\",\"code\":500,\"message\":\"\(echo)\"},\"code\":\"server_error\",\"message\":\"\(echo)\"}\n\n"),258        ]259        for reply in replies {260            let transport = FixtureTransport(reply)261            let provider = adapter.make(transport, diagnostics: diagnostics)262            let result = await collect(provider.stream(adapter.request))263            if let error = result.error { seen += [error.message, error.title, error.localizedDescription, String(describing: error), String(reflecting: error)] }264            seen += result.events.map { String(describing: $0) }265            for call in [{ _ = try await provider.listModels() }, { _ = try await provider.verify() }] as [@Sendable () async throws -> Void] {266                do { try await call() } catch { seen += [String(describing: error), (error as? AstraError)?.message ?? ""] }267            }268            for request in transport.requests {269                #expect(request.url?.absoluteString.contains(key) == false)270                if let body = request.httpBody { #expect(String(decoding: body, as: UTF8.self).contains(key) == false) }271            }272        }273        seen += diagnostics.records.map { String(describing: $0) }274        #expect(!seen.isEmpty)275        for text in seen { #expect(!text.contains(key), "leaked in: \(Redactor.redact(text).prefix(160))") }276    }277278    @Test(arguments: Adapter.allCases)279    func promptNeverAppearsInErrors(adapter: Adapter) async {280        let secretPrompt = "PRIVATE-PROMPT-7731 my diagnosis is"281        var req = adapter.request282        req.messages = [.user(secretPrompt)]283        for status in [400, 401, 500] {284            let result = await collect(adapter.make(FixtureTransport(.json(status, adapter.errorBody(type: "error", message: "Something failed.")))).stream(req))285            #expect(result.error != nil)286            #expect(result.error?.message.contains("PRIVATE-PROMPT") == false)287        }288    }289290    @Test(arguments: Adapter.allCases)291    func cancellingTheConsumerCancelsTheHTTPStream(adapter: Adapter) async throws {292        var reply = FixtureTransport.Reply.sse(adapter.textFixture)293        // Keep only the opening of the transcript, then leave the connection open.294        reply.body = reply.body.components(separatedBy: "\n\n").prefix(adapter == .anthropic || adapter == .openAI || adapter == .xAI ? 5 : 2).joined(separator: "\n\n") + "\n\n"295        reply.hangs = true296        let transport = FixtureTransport(reply)297        let provider = adapter.make(transport)298        let request = adapter.request299        let task = Task { () -> Int in300            var count = 0301            do { for try await _ in provider.stream(request) { count += 1 } } catch {}302            return count303        }304        // Wait until the request is open and the first events were delivered, then cancel the consumer.305        for _ in 0..<200 where transport.requests.isEmpty { try await Task.sleep(nanoseconds: 5_000_000) }306        try await Task.sleep(nanoseconds: 50_000_000)307        task.cancel()308        let delivered = await task.value309        #expect(delivered >= 1)310        for _ in 0..<200 where transport.cancelledStreams == 0 { try await Task.sleep(nanoseconds: 5_000_000) }311        #expect(transport.cancelledStreams == 1)312    }313314    @Test func toolArgumentsParsing() {315        #expect(ToolArguments.parse("") == [:])316        #expect(ToolArguments.parse("  ") == [:])317        #expect(ToolArguments.parse(#"{"a":1,"b":{"c":[true,null]}}"#) == ["a": 1, "b": ["c": [true, nil]]])318        #expect(ToolArguments.parse(#"{"a":"#) == ["_raw": #"{"a":"#])319        #expect(ToolArguments.parse(#""just a string""#) == ["_raw": #""just a string""#])320        #expect(ToolArguments.parse("[1,2]") == ["_raw": "[1,2]"])321        // A call that never parsed is replayed with its original text, not with a wrapper object.322        #expect(ToolArguments.wireText(["_raw": #"{"a":"#]) == #"{"a":"#)323        #expect(ToolArguments.wireText(["a": 1]) == #"{"a":1}"#)324        #expect(ToolArguments.wireObject(["_raw": "x"]) == ["_raw": "x"])325    }326327    @Test func errorBodyDialects() {328        #expect(ProviderErrorBody.parse(Data(#"{"error":{"message":"m","type":"t","param":"temperature","code":"c"}}"#.utf8)) == ProviderErrorBody(code: "c t", message: "m", param: "temperature"))329        #expect(ProviderErrorBody.parse(Data(#"{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}"#.utf8)) == ProviderErrorBody(code: "overloaded_error", message: "Overloaded"))330        #expect(ProviderErrorBody.parse(Data(#"{"error":{"code":502,"message":"down","metadata":{"error_type":"provider_unavailable"}}}"#.utf8)) == ProviderErrorBody(code: "502 provider_unavailable", message: "down"))331        #expect(ProviderErrorBody.parse(Data(#"{"code":"invalid-argument","error":"bad"}"#.utf8)) == ProviderErrorBody(code: "invalid-argument", message: "bad"))332        #expect(ProviderErrorBody.parse(Data(#"{"detail":"nope"}"#.utf8)).message == "nope")333        #expect(ProviderErrorBody.parse(Data("<html>".utf8)) == ProviderErrorBody())334        let google = ProviderErrorBody.parse(Data(Fixtures.text("gemini_error_429.json").utf8))335        #expect(google.retryDelay == 17)336        #expect(google.code == "429 RESOURCE_EXHAUSTED")337    }338339    @Test func retryAfterForms() async {340        for (header, expected) in [("12", 12.0), ("0", 0.0)] {341            let transport = FixtureTransport(.json(429, Adapter.openAI.errorBody(type: "rate_limit_error", message: "Slow down."), headers: ["Retry-After": header]))342            let result = await collect(Adapter.openAI.make(transport).stream(Adapter.openAI.request))343            #expect(result.error == .rateLimited(retryAfter: expected, message: "Slow down."))344        }345        let ms = FixtureTransport(.json(429, Adapter.openAI.errorBody(type: "rate_limit_error", message: "Slow down."), headers: ["retry-after-ms": "1500"]))346        let result = await collect(Adapter.openAI.make(ms).stream(Adapter.openAI.request))347        #expect(result.error == .rateLimited(retryAfter: 1.5, message: "Slow down."))348    }349350    @Test func registryBuildsTheRightAdapter() throws {351        let transport = FixtureTransport()352        let store = InMemoryCredentialStore([CredentialAccount.provider(.openAI): "sk-a", CredentialAccount.endpoint("ep1"): "custom"])353        let endpoints = [Adapter.profile]354        func make(_ kind: ProviderKind, endpoint: String? = nil) throws -> any ChatProvider {355            try makeProvider(for: ModelRef(provider: kind, modelID: "m", endpointID: endpoint), credentials: store, endpoints: endpoints, transport: transport, diagnostics: nil)356        }357        #expect(try make(.openAI) is OpenAIProvider)358        #expect(try make(.anthropic) is AnthropicProvider)359        #expect(try make(.openRouter) is OpenRouterProvider)360        #expect(try make(.xAI) is XAIProvider)361        #expect(try make(.google) is GeminiProvider)362        #expect(try make(.custom, endpoint: "ep1") is OpenAICompatibleProvider)363        #expect(throws: AstraError.self) { _ = try make(.custom, endpoint: "gone") }364        #expect(throws: AstraError.self) { _ = try make(.custom) }365        #expect(throws: AstraError.self) { _ = try make(.mlx) }366        #expect(throws: AstraError.self) { _ = try make(.appleFoundation) }367        for kind in ProviderKind.cloudKeyed { #expect(try make(kind).kind == kind) }368    }369370    @Test func registryReadsKeysAtRequestTime() async throws {371        let transport = FixtureTransport(.sse("openai_text.sse"))372        let store = InMemoryCredentialStore()373        let provider = try ProviderRegistry.makeProvider(kind: .openAI, credentials: store, transport: transport)374        var result = await collect(provider.stream(Adapter.openAI.request))375        #expect(result.error == .missingCredential(.openAI))376        try store.setSecret("sk-added-later-0123456789", for: CredentialAccount.provider(.openAI))377        result = await collect(provider.stream(Adapter.openAI.request))378        #expect(result.error == nil)379        #expect(transport.requests.first?.value(forHTTPHeaderField: "Authorization") == "Bearer sk-added-later-0123456789")380        // The custom endpoint's key lives under its own account.381        let customTransport = FixtureTransport(.sse("custom_text.sse"))382        try store.setSecret("endpoint-secret", for: CredentialAccount.endpoint("ep1"))383        let custom = try ProviderRegistry.makeProvider(for: ModelRef(provider: .custom, modelID: "m", endpointID: "ep1"), credentials: store, endpoints: [Adapter.profile], transport: customTransport)384        _ = await collect(custom.stream(Adapter.custom.request))385        #expect(customTransport.requests.first?.value(forHTTPHeaderField: "Authorization") == "Bearer endpoint-secret")386    }387}388
Files main
Menu

Theme