SPB Git

Repository Public MIT

ChatCompletionsProviderTests.swift 23.6 KB 387 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraProvidersTests/ChatCompletionsProviderTests.swift10// Module           : AstraProvidersTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraProviders9293@Suite("OpenRouter (Chat Completions)")94struct OpenRouterProviderTests {95    func provider(_ transport: FixtureTransport, diagnostics: RecordingDiagnostics? = nil) -> OpenRouterProvider {96        OpenRouterProvider(transport: transport, apiKey: { "sk-or-v1-FIXTUREKEY0123456789abcdefFIXTURE" }, diagnostics: diagnostics)97    }9899    @Test func plainTextFromRecordedTranscript() async {100        let transport = FixtureTransport(.sse("openrouter_text.sse", headers: ["x-generation-id": "gen-fixture"]))101        let diagnostics = RecordingDiagnostics()102        let c = await collect(provider(transport, diagnostics: diagnostics).stream(request(.openRouter, "openai/gpt-6-luna")))103        expectWellFormed(c)104        #expect(c.text == "ASTRA-OK")105        #expect(c.starts.first == MessageStart(responseID: "gen-1795318144-FIXTURE", reportedModel: "openai/gpt-6-luna", servedBy: "OpenAI"))106        #expect(c.completions == [.endOfTurn])       // the finish reason arrives twice on OpenRouter; one terminal event comes out107        #expect(c.outcome.usage?.inputTokens == 22)108        #expect(c.outcome.usage?.outputTokens == 8)109        #expect(c.outcome.usage?.reportedCostUSD == 0.0000062)110        #expect(diagnostics.records.first?.requestID == "gen-fixture")111    }112113    @Test func headersBodyAndRouting() async {114        let transport = FixtureTransport(.sse("openrouter_text.sse"))115        var req = GenerationRequest(model: ModelRef(provider: .openRouter, modelID: "anthropic/claude-haiku-5.5"), system: "Be brief.",116                                    messages: [.user("Look", images: [tinyPNG])] + toolCycle(),117                                    parameters: GenerationParameters(temperature: 0.4, topP: 0.8, maxOutputTokens: 50, reasoning: .low, seed: 7, stop: ["END"]), tools: [weatherTool])118        req.routing = RoutingPreferences(providerOrder: ["anthropic", "amazon-bedrock"], allowFallbacks: false, denyDataCollection: true, fallbackModels: ["openai/gpt-6-luna"])119        _ = await collect(provider(transport).stream(req))120        let r = transport.requests[0]121        #expect(r.url?.absoluteString == "https://openrouter.ai/api/v1/chat/completions")122        #expect(r.value(forHTTPHeaderField: "Authorization")?.hasPrefix("Bearer sk-or-v1-") == true)123        #expect(r.value(forHTTPHeaderField: "HTTP-Referer") == "https://www.orixlab.dev")124        #expect(r.value(forHTTPHeaderField: "X-Title") == "ORIX Astra")125        #expect(r.value(forHTTPHeaderField: "X-OpenRouter-Title") == "ORIX Astra")126        let body = transport.body()127        #expect(body["model"] == "anthropic/claude-haiku-5.5")128        #expect(body["models"] == ["openai/gpt-6-luna"])129        #expect(body["provider"] == ["order": ["anthropic", "amazon-bedrock"], "allow_fallbacks": false, "data_collection": "deny"])130        #expect(body["stream"] == true)131        #expect(body["max_tokens"] == 50)132        #expect(body["temperature"] == 0.4)133        #expect(body["top_p"] == 0.8)134        #expect(body["seed"] == 7)135        #expect(body["stop"] == ["END"])136        #expect(body["reasoning"] == ["effort": "low"])137        #expect(body["stream_options"] == nil)      // OpenRouter always reports usage138        let messages = body["messages"]?.arrayValue ?? []139        #expect(messages[0] == ["role": "system", "content": "Be brief."])140        #expect(messages[1]["content"]?[1] == ["type": "image_url", "image_url": ["url": .string("data:image/png;base64,\(tinyPNG.data.base64EncodedString())")]])141        #expect(messages[3]["tool_calls"]?[0] == ["id": "call_abc123", "type": "function", "function": ["name": "get_weather", "arguments": "{\"city\":\"Paris\"}"]])142        #expect(messages[4] == ["role": "tool", "tool_call_id": "call_abc123", "content": "18 C, cloudy"])143        #expect(body["tools"]?[0]?["function"]?["parameters"] == weatherTool.parameters)144    }145146    @Test func defaultRoutingAddsNothing() async {147        let transport = FixtureTransport(.sse("openrouter_text.sse"))148        var req = request(.openRouter, "openai/gpt-6-luna")149        req.routing = RoutingPreferences()150        _ = await collect(provider(transport).stream(req))151        #expect(transport.body()["provider"] == nil)152        #expect(transport.body()["models"] == nil)153    }154155    @Test func toolCallAccumulatedByIndex() async {156        let c = await collect(provider(FixtureTransport(.sse("openrouter_tool.sse"))).stream(request(.openRouter, "openai/gpt-6-luna", tools: [weatherTool])))157        expectWellFormed(c)158        #expect(c.completions == [.toolCalls])159        #expect(c.outcome.toolCalls.count == 1)160        #expect(c.outcome.toolCalls[0].arguments == ["city": "Paris"])161        #expect(c.deltas(for: c.outcome.toolCalls[0].id) == ["{\"", "city", "\":\"", "Paris", "\"}"])162        #expect(c.outcome.usage?.reportedCostUSD == 0.0000145)163    }164165    @Test func reasoningAndProcessingComments() async {166        let fixture = Fixtures.text("openrouter_reasoning.sse")167        #expect(fixture.hasPrefix(": OPENROUTER PROCESSING"))168        let c = await collect(provider(FixtureTransport(.sse("openrouter_reasoning.sse"))).stream(request(.openRouter, "anthropic/claude-haiku-5.5", parameters: GenerationParameters(reasoning: .medium))))169        expectWellFormed(c)170        #expect(c.reasoning.hasPrefix("I'm multiplying 17 by 23"))171        #expect(c.text == "391")172        #expect(c.starts.first?.reportedModel == "anthropic/claude-haiku-5.5")173        #expect(c.starts.first?.servedBy == "Claude Platform on AWS")174        #expect(c.outcome.usage?.reasoningTokens == 30)175    }176177    @Test func midStreamErrorChunk() async {178        let c = await collect(provider(FixtureTransport(.sse("openrouter_midstream_error.sse"))).stream(request(.openRouter, "openai/gpt-6-luna")))179        #expect(c.text == "ASTRA")180        #expect(c.completions.isEmpty)181        #expect(c.error == .server(status: 502, message: "Provider disconnected unexpectedly"))182    }183184    @Test func errorAsTheOnlyChunkOfA200() async {185        let c = await collect(provider(FixtureTransport(.sse("openrouter_error_only.sse"))).stream(request(.openRouter, "openai/gpt-6-luna")))186        #expect(c.text.isEmpty)187        guard case .rateLimited = c.error else { Issue.record("expected rateLimited, got \(String(describing: c.error))"); return }188    }189190    @Test func recordedErrorBodies() async {191        var c = await collect(provider(FixtureTransport(.file(401, "openrouter_error_401.json"))).stream(request(.openRouter, "openai/gpt-6-luna")))192        #expect(c.error == .unauthorized("User not found."))193        c = await collect(provider(FixtureTransport(.file(400, "openrouter_error_400_model.json"))).stream(request(.openRouter, "nonexistent/model-zzz")))194        #expect(c.error == .modelNotFound("nonexistent/model-zzz"))195        c = await collect(provider(FixtureTransport(.json(403, #"{"error":{"code":403,"message":"Your input was flagged","metadata":{"reasons":["violence"],"flagged_input":"…","provider_name":"OpenAI","model_slug":"openai/gpt-6-luna"}}}"#))).stream(request(.openRouter, "openai/gpt-6-luna")))196        guard case .contentFiltered = c.error else { Issue.record("expected contentFiltered, got \(String(describing: c.error))"); return }197    }198199    @Test func modelList() async throws {200        let transport = FixtureTransport(.file(200, "openrouter_models.json"))201        let models = try await provider(transport).listModels()202        #expect(transport.requests[0].url?.absoluteString == "https://openrouter.ai/api/v1/models")203        #expect(Set(models.map(\.ref.modelID)) == ["anthropic/claude-sonnet-5.5", "openai/gpt-6-luna", "openrouter/auto", "thinkingmachines/inkling-2-mini"])204        let sonnet = try #require(models.first { $0.ref.modelID == "anthropic/claude-sonnet-5.5" })205        #expect(sonnet.displayName == "Anthropic: Claude Sonnet 5.5")206        #expect(sonnet.contextLength == 1_000_000)207        #expect(sonnet.maxOutputTokens == 128_000)208        #expect(sonnet.vendor == "anthropic")209        #expect(sonnet.capabilities.isSuperset(of: [.text, .vision, .documents, .tools, .reasoning, .structuredOutput, .promptCaching]))210        #expect(sonnet.supportedParameters?.contains("tools") == true)211        #expect(sonnet.supportedParameters?.contains("temperature") == false)212        #expect(sonnet.pricing?.inputPerMTok == 2)213        #expect(sonnet.pricing?.outputPerMTok == 10)214        #expect(sonnet.pricing?.cachedInputPerMTok == 0.1)215        #expect(sonnet.pricing?.cacheWritePerMTok == 2.5)216        guard case .providerAPI = sonnet.source else { Issue.record("OpenRouter metadata comes from the provider API"); return }217        // A router priced at -1 has no fixed price.218        #expect(models.first { $0.ref.modelID == "openrouter/auto" }?.pricing == nil)219    }220221    @Test func verifyReadsTheKeyLimitOrFallsBackToCredits() async throws {222        var transport = FixtureTransport(.file(200, "openrouter_key_limited.json"))223        var status = try await provider(transport).verify()224        #expect(transport.requests.map { $0.url!.path } == ["/api/v1/key"])225        #expect(status.ok)226        #expect(status.remainingCreditUSD == 7.5)227        #expect(!status.detail.contains("sk-or"))228        transport = FixtureTransport(.file(200, "openrouter_key.json"), .file(200, "openrouter_credits.json"))229        status = try await provider(transport).verify()230        #expect(transport.requests.map { $0.url!.path } == ["/api/v1/key", "/api/v1/credits"])231        #expect(status.remainingCreditUSD == 21.5)232        // The credits endpoint refusing the key does not fail the check.233        transport = FixtureTransport(.file(200, "openrouter_key.json"), .json(403, #"{"error":{"code":403,"message":"Only management keys can perform this operation"}}"#))234        status = try await provider(transport).verify()235        #expect(status.ok)236        #expect(status.remainingCreditUSD == nil)237    }238}239240@Suite("OpenAI-compatible endpoint")241struct OpenAICompatibleProviderTests {242    static let customKey = "weird-custom-secret-zzz999"243    func profile(_ base: String = "https://llm.example.com/v1", insecure: Bool = false, headers: [String: String] = [:], defaultModel: String? = nil) -> EndpointProfile {244        EndpointProfile(id: "ep1", name: "Mine", baseURL: base, defaultModelID: defaultModel, allowInsecureHTTP: insecure, extraHeaders: headers)245    }246    func provider(_ transport: FixtureTransport, profile: EndpointProfile? = nil, key: String? = customKey) -> OpenAICompatibleProvider {247        OpenAICompatibleProvider(profile: profile ?? self.profile(), transport: transport, apiKey: { key })248    }249250    @Test func plainTextFromRecordedTranscript() async {251        let transport = FixtureTransport(.sse("custom_text.sse"))252        let c = await collect(provider(transport, profile: profile(headers: ["X-Workspace": "astra"]))253            .stream(request(.custom, "maclustr-qwen3", system: "You are terse.", parameters: GenerationParameters(temperature: 0, maxOutputTokens: 32, reasoning: .low, seed: 3), endpointID: "ep1")))254        expectWellFormed(c)255        #expect(c.text == "ASTRA-OK")256        #expect(c.starts.first?.reportedModel == "maclustr-qwen3")257        #expect(c.completions == [.endOfTurn])258        #expect(c.outcome.usage == TokenUsage(inputTokens: 25, outputTokens: 5, cachedInputTokens: 5))259        let r = transport.requests[0]260        #expect(r.url?.absoluteString == "https://llm.example.com/v1/chat/completions")261        #expect(r.value(forHTTPHeaderField: "Authorization") == "Bearer \(Self.customKey)")262        #expect(r.value(forHTTPHeaderField: "X-Workspace") == "astra")263        let body = transport.body()264        #expect(body["stream_options"] == ["include_usage": true])265        #expect(body["reasoning_effort"] == "low")266        #expect(body["reasoning"] == nil)267        #expect(body["provider"] == nil)268        #expect(body["max_tokens"] == 32)269        #expect(body["messages"]?[0] == ["role": "system", "content": "You are terse."])270    }271272    @Test func keyIsOptional() async {273        let transport = FixtureTransport(.sse("custom_text.sse"))274        let c = await collect(provider(transport, profile: profile("http://192.168.1.20:11434/v1", insecure: true), key: nil).stream(request(.custom, "m", endpointID: "ep1")))275        expectWellFormed(c)276        #expect(transport.requests[0].value(forHTTPHeaderField: "Authorization") == nil)277        #expect(transport.requests[0].url?.absoluteString == "http://192.168.1.20:11434/v1/chat/completions")278    }279280    @Test func endpointPolicyIsEnforcedOnEveryRequest() async {281        let transport = FixtureTransport(.sse("custom_text.sse"))282        // Plain HTTP is refused by default, even for a local host.283        var c = await collect(provider(transport, profile: profile("http://192.168.1.20:11434/v1")).stream(request(.custom, "m", endpointID: "ep1")))284        guard case .insecureEndpoint = c.error else { Issue.record("expected insecureEndpoint, got \(String(describing: c.error))"); return }285        // With the flag, plain HTTP is still refused for a host that is not on the local network.286        c = await collect(provider(transport, profile: profile("http://llm.example.com/v1", insecure: true)).stream(request(.custom, "m", endpointID: "ep1")))287        guard case .insecureEndpoint = c.error else { Issue.record("expected insecureEndpoint, got \(String(describing: c.error))"); return }288        // Credentials in the URL and credential-bearing extra headers are refused.289        c = await collect(provider(transport, profile: profile("https://user:pw@llm.example.com/v1")).stream(request(.custom, "m", endpointID: "ep1")))290        guard case .invalidRequest = c.error else { Issue.record("expected invalidRequest, got \(String(describing: c.error))"); return }291        c = await collect(provider(transport, profile: profile(headers: ["Authorization": "Bearer nope"])).stream(request(.custom, "m", endpointID: "ep1")))292        guard case .invalidRequest = c.error else { Issue.record("expected invalidRequest, got \(String(describing: c.error))"); return }293        await #expect(throws: AstraError.self) { try await provider(transport, profile: profile("http://10.0.0.5/v1")).listModels() }294        #expect(transport.requests.isEmpty)      // nothing left the device295        // Allowed: local host with the flag.296        for base in ["http://localhost:1234/v1", "http://10.0.0.5:8080/v1", "http://mac-studio.local/v1", "http://100.100.1.2/v1"] {297            c = await collect(provider(FixtureTransport(.sse("custom_text.sse")), profile: profile(base, insecure: true)).stream(request(.custom, "m", endpointID: "ep1")))298            #expect(c.error == nil)299        }300    }301302    @Test func toolCall() async {303        let c = await collect(provider(FixtureTransport(.sse("custom_tool.sse"))).stream(request(.custom, "maclustr-qwen3", tools: [weatherTool], endpointID: "ep1")))304        expectWellFormed(c)305        #expect(c.completions == [.toolCalls])306        #expect(c.outcome.toolCalls == [ToolCall(id: "call_bfa70d226dea45cd9ebd6a32", name: "get_weather", arguments: ["city": "Paris"])])307    }308309    @Test func fragmentedParallelCallsReasoningContentAndInvalidJSON() async {310        let c = await collect(provider(FixtureTransport(.sse("custom_reasoning_content.sse"))).stream(request(.custom, "deep-local", endpointID: "ep1")))311        expectWellFormed(c)312        #expect(c.reasoning == "Adding the two numbers.")313        #expect(c.text == "4")314        #expect(c.completions == [.toolCalls])   // "stop" with tool calls still means the turn ended in calls315        #expect(c.outcome.toolCalls.count == 2)316        // No id was sent for the first call: one is synthesised; fragments are joined by index.317        #expect(c.outcome.toolCalls[0] == ToolCall(id: "call_chatcmpl-r_0", name: "sum", arguments: ["a": 2, "b": 2]))318        #expect(c.deltas(for: "call_chatcmpl-r_0") == ["{\"a\":", "2,", "\"b\":2}"])319        // Arguments that are not JSON are preserved verbatim.320        #expect(c.outcome.toolCalls[1] == ToolCall(id: "call_b", name: "noop", arguments: ["_raw": "not json"]))321    }322323    @Test func serverThatClosesWithoutDone() async {324        let c = await collect(provider(FixtureTransport(.sse("custom_no_done.sse"))).stream(request(.custom, "maclustr-qwen3", endpointID: "ep1")))325        expectWellFormed(c)326        #expect(c.text == "ASTRA-OK")327        #expect(c.completions == [.endOfTurn])328    }329330    @Test func serverThatIgnoresStreamAndAnswersWithJSON() async {331        let c = await collect(provider(FixtureTransport(.file(200, "custom_plain_json.json"))).stream(request(.custom, "local-model", endpointID: "ep1")))332        expectWellFormed(c)333        #expect(c.text == "ASTRA-OK")334        #expect(c.outcome.usage?.inputTokens == 9)335    }336337    @Test func rejectedOptionalFieldsAreRemovedOneByOne() async {338        let transport = FixtureTransport(.json(400, #"{"error":{"message":"Unrecognized request argument supplied: stream_options","type":"invalid_request_error"}}"#),339                                         .json(400, #"{"error":{"message":"Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.","type":"invalid_request_error","param":"max_tokens","code":"unsupported_parameter"}}"#),340                                         .json(422, #"{"detail":"reasoning_effort: extra fields not permitted"}"#),341                                         .sse("custom_text.sse"))342        let c = await collect(provider(transport).stream(request(.custom, "m", parameters: GenerationParameters(maxOutputTokens: 40, reasoning: .high), endpointID: "ep1")))343        expectWellFormed(c)344        #expect(transport.requests.count == 4)345        #expect(transport.body(1)["stream_options"] == nil)346        #expect(transport.body(2)["max_tokens"] == nil)347        #expect(transport.body(2)["max_completion_tokens"] == 40)348        #expect(transport.body(3)["reasoning_effort"] == nil)349    }350351    @Test func recordedErrorBodies() async {352        var c = await collect(provider(FixtureTransport(.file(401, "custom_error_401.json"))).stream(request(.custom, "maclustr-qwen3", endpointID: "ep1")))353        guard case .unauthorized = c.error else { Issue.record("expected unauthorized"); return }354        c = await collect(provider(FixtureTransport(.file(404, "custom_error_404.json"))).stream(request(.custom, "nope", endpointID: "ep1")))355        #expect(c.error == .modelNotFound("nope"))356        // An HTML gateway page is never shown raw.357        c = await collect(provider(FixtureTransport(.init(status: 502, body: "<html><body>Bad gateway \(Self.customKey)</body></html>"))).stream(request(.custom, "m", endpointID: "ep1")))358        #expect(c.error == .server(status: 502, message: nil))359    }360361    @Test func modelListReportsOnlyWhatTheEndpointSays() async throws {362        let transport = FixtureTransport(.file(200, "custom_models.json"))363        let models = try await provider(transport, profile: profile(defaultModel: "typed-by-user")).listModels()364        #expect(transport.requests[0].url?.absoluteString == "https://llm.example.com/v1/models")365        #expect(models.map(\.ref.modelID) == ["typed-by-user", "maclustr-qwen3"])366        #expect(models[0].source == .user)367        #expect(models[0].capabilities == [.text])368        let reported = models[1]369        #expect(reported.ref == ModelRef(provider: .custom, modelID: "maclustr-qwen3", endpointID: "ep1"))370        #expect(reported.contextLength == 32768)371        #expect(reported.maxOutputTokens == 8192)372        #expect(reported.capabilities == [.text, .tools])     // the endpoint reports tools: true, thinking: false373        #expect(reported.pricing == nil)374        guard case .providerAPI = reported.source else { Issue.record("expected providerAPI"); return }375        // A bare OpenAI-style list claims text only.376        let bare = try await provider(FixtureTransport(.json(200, #"{"object":"list","data":[{"id":"llama3.2:3b","object":"model","owned_by":"library"}]}"#))).listModels()377        #expect(bare.first?.capabilities == [.text])378        #expect(bare.first?.contextLength == nil)379    }380381    @Test func endpointWithoutAModelListFallsBackToTheTypedModel() async throws {382        let models = try await provider(FixtureTransport(.json(404, #"{"error":"not found"}"#)), profile: profile(defaultModel: "my-model")).listModels()383        #expect(models.map(\.ref.modelID) == ["my-model"])384        #expect(models[0].source == .user)385    }386}387
Files main
Menu

Theme