Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Tests/AstraProvidersTests/ChatCompletionsProviderTests.swift10// Module : AstraProvidersTests11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraProviders9293@Suite("OpenRouter (Chat Completions)")94struct OpenRouterProviderTests {95 func provider(_ transport: FixtureTransport, diagnostics: RecordingDiagnostics? = nil) -> OpenRouterProvider {96 OpenRouterProvider(transport: transport, apiKey: { "sk-or-v1-FIXTUREKEY0123456789abcdefFIXTURE" }, diagnostics: diagnostics)97 }9899 @Test func plainTextFromRecordedTranscript() async {100 let transport = FixtureTransport(.sse("openrouter_text.sse", headers: ["x-generation-id": "gen-fixture"]))101 let diagnostics = RecordingDiagnostics()102 let c = await collect(provider(transport, diagnostics: diagnostics).stream(request(.openRouter, "openai/gpt-6-luna")))103 expectWellFormed(c)104 #expect(c.text == "ASTRA-OK")105 #expect(c.starts.first == MessageStart(responseID: "gen-1795318144-FIXTURE", reportedModel: "openai/gpt-6-luna", servedBy: "OpenAI"))106 #expect(c.completions == [.endOfTurn]) // the finish reason arrives twice on OpenRouter; one terminal event comes out107 #expect(c.outcome.usage?.inputTokens == 22)108 #expect(c.outcome.usage?.outputTokens == 8)109 #expect(c.outcome.usage?.reportedCostUSD == 0.0000062)110 #expect(diagnostics.records.first?.requestID == "gen-fixture")111 }112113 @Test func headersBodyAndRouting() async {114 let transport = FixtureTransport(.sse("openrouter_text.sse"))115 var req = GenerationRequest(model: ModelRef(provider: .openRouter, modelID: "anthropic/claude-haiku-5.5"), system: "Be brief.",116 messages: [.user("Look", images: [tinyPNG])] + toolCycle(),117 parameters: GenerationParameters(temperature: 0.4, topP: 0.8, maxOutputTokens: 50, reasoning: .low, seed: 7, stop: ["END"]), tools: [weatherTool])118 req.routing = RoutingPreferences(providerOrder: ["anthropic", "amazon-bedrock"], allowFallbacks: false, denyDataCollection: true, fallbackModels: ["openai/gpt-6-luna"])119 _ = await collect(provider(transport).stream(req))120 let r = transport.requests[0]121 #expect(r.url?.absoluteString == "https://openrouter.ai/api/v1/chat/completions")122 #expect(r.value(forHTTPHeaderField: "Authorization")?.hasPrefix("Bearer sk-or-v1-") == true)123 #expect(r.value(forHTTPHeaderField: "HTTP-Referer") == "https://www.orixlab.dev")124 #expect(r.value(forHTTPHeaderField: "X-Title") == "ORIX Astra")125 #expect(r.value(forHTTPHeaderField: "X-OpenRouter-Title") == "ORIX Astra")126 let body = transport.body()127 #expect(body["model"] == "anthropic/claude-haiku-5.5")128 #expect(body["models"] == ["openai/gpt-6-luna"])129 #expect(body["provider"] == ["order": ["anthropic", "amazon-bedrock"], "allow_fallbacks": false, "data_collection": "deny"])130 #expect(body["stream"] == true)131 #expect(body["max_tokens"] == 50)132 #expect(body["temperature"] == 0.4)133 #expect(body["top_p"] == 0.8)134 #expect(body["seed"] == 7)135 #expect(body["stop"] == ["END"])136 #expect(body["reasoning"] == ["effort": "low"])137 #expect(body["stream_options"] == nil) // OpenRouter always reports usage138 let messages = body["messages"]?.arrayValue ?? []139 #expect(messages[0] == ["role": "system", "content": "Be brief."])140 #expect(messages[1]["content"]?[1] == ["type": "image_url", "image_url": ["url": .string("data:image/png;base64,\(tinyPNG.data.base64EncodedString())")]])141 #expect(messages[3]["tool_calls"]?[0] == ["id": "call_abc123", "type": "function", "function": ["name": "get_weather", "arguments": "{\"city\":\"Paris\"}"]])142 #expect(messages[4] == ["role": "tool", "tool_call_id": "call_abc123", "content": "18 C, cloudy"])143 #expect(body["tools"]?[0]?["function"]?["parameters"] == weatherTool.parameters)144 }145146 @Test func defaultRoutingAddsNothing() async {147 let transport = FixtureTransport(.sse("openrouter_text.sse"))148 var req = request(.openRouter, "openai/gpt-6-luna")149 req.routing = RoutingPreferences()150 _ = await collect(provider(transport).stream(req))151 #expect(transport.body()["provider"] == nil)152 #expect(transport.body()["models"] == nil)153 }154155 @Test func toolCallAccumulatedByIndex() async {156 let c = await collect(provider(FixtureTransport(.sse("openrouter_tool.sse"))).stream(request(.openRouter, "openai/gpt-6-luna", tools: [weatherTool])))157 expectWellFormed(c)158 #expect(c.completions == [.toolCalls])159 #expect(c.outcome.toolCalls.count == 1)160 #expect(c.outcome.toolCalls[0].arguments == ["city": "Paris"])161 #expect(c.deltas(for: c.outcome.toolCalls[0].id) == ["{\"", "city", "\":\"", "Paris", "\"}"])162 #expect(c.outcome.usage?.reportedCostUSD == 0.0000145)163 }164165 @Test func reasoningAndProcessingComments() async {166 let fixture = Fixtures.text("openrouter_reasoning.sse")167 #expect(fixture.hasPrefix(": OPENROUTER PROCESSING"))168 let c = await collect(provider(FixtureTransport(.sse("openrouter_reasoning.sse"))).stream(request(.openRouter, "anthropic/claude-haiku-5.5", parameters: GenerationParameters(reasoning: .medium))))169 expectWellFormed(c)170 #expect(c.reasoning.hasPrefix("I'm multiplying 17 by 23"))171 #expect(c.text == "391")172 #expect(c.starts.first?.reportedModel == "anthropic/claude-haiku-5.5")173 #expect(c.starts.first?.servedBy == "Claude Platform on AWS")174 #expect(c.outcome.usage?.reasoningTokens == 30)175 }176177 @Test func midStreamErrorChunk() async {178 let c = await collect(provider(FixtureTransport(.sse("openrouter_midstream_error.sse"))).stream(request(.openRouter, "openai/gpt-6-luna")))179 #expect(c.text == "ASTRA")180 #expect(c.completions.isEmpty)181 #expect(c.error == .server(status: 502, message: "Provider disconnected unexpectedly"))182 }183184 @Test func errorAsTheOnlyChunkOfA200() async {185 let c = await collect(provider(FixtureTransport(.sse("openrouter_error_only.sse"))).stream(request(.openRouter, "openai/gpt-6-luna")))186 #expect(c.text.isEmpty)187 guard case .rateLimited = c.error else { Issue.record("expected rateLimited, got \(String(describing: c.error))"); return }188 }189190 @Test func recordedErrorBodies() async {191 var c = await collect(provider(FixtureTransport(.file(401, "openrouter_error_401.json"))).stream(request(.openRouter, "openai/gpt-6-luna")))192 #expect(c.error == .unauthorized("User not found."))193 c = await collect(provider(FixtureTransport(.file(400, "openrouter_error_400_model.json"))).stream(request(.openRouter, "nonexistent/model-zzz")))194 #expect(c.error == .modelNotFound("nonexistent/model-zzz"))195 c = await collect(provider(FixtureTransport(.json(403, #"{"error":{"code":403,"message":"Your input was flagged","metadata":{"reasons":["violence"],"flagged_input":"…","provider_name":"OpenAI","model_slug":"openai/gpt-6-luna"}}}"#))).stream(request(.openRouter, "openai/gpt-6-luna")))196 guard case .contentFiltered = c.error else { Issue.record("expected contentFiltered, got \(String(describing: c.error))"); return }197 }198199 @Test func modelList() async throws {200 let transport = FixtureTransport(.file(200, "openrouter_models.json"))201 let models = try await provider(transport).listModels()202 #expect(transport.requests[0].url?.absoluteString == "https://openrouter.ai/api/v1/models")203 #expect(Set(models.map(\.ref.modelID)) == ["anthropic/claude-sonnet-5.5", "openai/gpt-6-luna", "openrouter/auto", "thinkingmachines/inkling-2-mini"])204 let sonnet = try #require(models.first { $0.ref.modelID == "anthropic/claude-sonnet-5.5" })205 #expect(sonnet.displayName == "Anthropic: Claude Sonnet 5.5")206 #expect(sonnet.contextLength == 1_000_000)207 #expect(sonnet.maxOutputTokens == 128_000)208 #expect(sonnet.vendor == "anthropic")209 #expect(sonnet.capabilities.isSuperset(of: [.text, .vision, .documents, .tools, .reasoning, .structuredOutput, .promptCaching]))210 #expect(sonnet.supportedParameters?.contains("tools") == true)211 #expect(sonnet.supportedParameters?.contains("temperature") == false)212 #expect(sonnet.pricing?.inputPerMTok == 2)213 #expect(sonnet.pricing?.outputPerMTok == 10)214 #expect(sonnet.pricing?.cachedInputPerMTok == 0.1)215 #expect(sonnet.pricing?.cacheWritePerMTok == 2.5)216 guard case .providerAPI = sonnet.source else { Issue.record("OpenRouter metadata comes from the provider API"); return }217 // A router priced at -1 has no fixed price.218 #expect(models.first { $0.ref.modelID == "openrouter/auto" }?.pricing == nil)219 }220221 @Test func verifyReadsTheKeyLimitOrFallsBackToCredits() async throws {222 var transport = FixtureTransport(.file(200, "openrouter_key_limited.json"))223 var status = try await provider(transport).verify()224 #expect(transport.requests.map { $0.url!.path } == ["/api/v1/key"])225 #expect(status.ok)226 #expect(status.remainingCreditUSD == 7.5)227 #expect(!status.detail.contains("sk-or"))228 transport = FixtureTransport(.file(200, "openrouter_key.json"), .file(200, "openrouter_credits.json"))229 status = try await provider(transport).verify()230 #expect(transport.requests.map { $0.url!.path } == ["/api/v1/key", "/api/v1/credits"])231 #expect(status.remainingCreditUSD == 21.5)232 // The credits endpoint refusing the key does not fail the check.233 transport = FixtureTransport(.file(200, "openrouter_key.json"), .json(403, #"{"error":{"code":403,"message":"Only management keys can perform this operation"}}"#))234 status = try await provider(transport).verify()235 #expect(status.ok)236 #expect(status.remainingCreditUSD == nil)237 }238}239240@Suite("OpenAI-compatible endpoint")241struct OpenAICompatibleProviderTests {242 static let customKey = "weird-custom-secret-zzz999"243 func profile(_ base: String = "https://llm.example.com/v1", insecure: Bool = false, headers: [String: String] = [:], defaultModel: String? = nil) -> EndpointProfile {244 EndpointProfile(id: "ep1", name: "Mine", baseURL: base, defaultModelID: defaultModel, allowInsecureHTTP: insecure, extraHeaders: headers)245 }246 func provider(_ transport: FixtureTransport, profile: EndpointProfile? = nil, key: String? = customKey) -> OpenAICompatibleProvider {247 OpenAICompatibleProvider(profile: profile ?? self.profile(), transport: transport, apiKey: { key })248 }249250 @Test func plainTextFromRecordedTranscript() async {251 let transport = FixtureTransport(.sse("custom_text.sse"))252 let c = await collect(provider(transport, profile: profile(headers: ["X-Workspace": "astra"]))253 .stream(request(.custom, "maclustr-qwen3", system: "You are terse.", parameters: GenerationParameters(temperature: 0, maxOutputTokens: 32, reasoning: .low, seed: 3), endpointID: "ep1")))254 expectWellFormed(c)255 #expect(c.text == "ASTRA-OK")256 #expect(c.starts.first?.reportedModel == "maclustr-qwen3")257 #expect(c.completions == [.endOfTurn])258 #expect(c.outcome.usage == TokenUsage(inputTokens: 25, outputTokens: 5, cachedInputTokens: 5))259 let r = transport.requests[0]260 #expect(r.url?.absoluteString == "https://llm.example.com/v1/chat/completions")261 #expect(r.value(forHTTPHeaderField: "Authorization") == "Bearer \(Self.customKey)")262 #expect(r.value(forHTTPHeaderField: "X-Workspace") == "astra")263 let body = transport.body()264 #expect(body["stream_options"] == ["include_usage": true])265 #expect(body["reasoning_effort"] == "low")266 #expect(body["reasoning"] == nil)267 #expect(body["provider"] == nil)268 #expect(body["max_tokens"] == 32)269 #expect(body["messages"]?[0] == ["role": "system", "content": "You are terse."])270 }271272 @Test func keyIsOptional() async {273 let transport = FixtureTransport(.sse("custom_text.sse"))274 let c = await collect(provider(transport, profile: profile("http://192.168.1.20:11434/v1", insecure: true), key: nil).stream(request(.custom, "m", endpointID: "ep1")))275 expectWellFormed(c)276 #expect(transport.requests[0].value(forHTTPHeaderField: "Authorization") == nil)277 #expect(transport.requests[0].url?.absoluteString == "http://192.168.1.20:11434/v1/chat/completions")278 }279280 @Test func endpointPolicyIsEnforcedOnEveryRequest() async {281 let transport = FixtureTransport(.sse("custom_text.sse"))282 // Plain HTTP is refused by default, even for a local host.283 var c = await collect(provider(transport, profile: profile("http://192.168.1.20:11434/v1")).stream(request(.custom, "m", endpointID: "ep1")))284 guard case .insecureEndpoint = c.error else { Issue.record("expected insecureEndpoint, got \(String(describing: c.error))"); return }285 // With the flag, plain HTTP is still refused for a host that is not on the local network.286 c = await collect(provider(transport, profile: profile("http://llm.example.com/v1", insecure: true)).stream(request(.custom, "m", endpointID: "ep1")))287 guard case .insecureEndpoint = c.error else { Issue.record("expected insecureEndpoint, got \(String(describing: c.error))"); return }288 // Credentials in the URL and credential-bearing extra headers are refused.289 c = await collect(provider(transport, profile: profile("https://user:pw@llm.example.com/v1")).stream(request(.custom, "m", endpointID: "ep1")))290 guard case .invalidRequest = c.error else { Issue.record("expected invalidRequest, got \(String(describing: c.error))"); return }291 c = await collect(provider(transport, profile: profile(headers: ["Authorization": "Bearer nope"])).stream(request(.custom, "m", endpointID: "ep1")))292 guard case .invalidRequest = c.error else { Issue.record("expected invalidRequest, got \(String(describing: c.error))"); return }293 await #expect(throws: AstraError.self) { try await provider(transport, profile: profile("http://10.0.0.5/v1")).listModels() }294 #expect(transport.requests.isEmpty) // nothing left the device295 // Allowed: local host with the flag.296 for base in ["http://localhost:1234/v1", "http://10.0.0.5:8080/v1", "http://mac-studio.local/v1", "http://100.100.1.2/v1"] {297 c = await collect(provider(FixtureTransport(.sse("custom_text.sse")), profile: profile(base, insecure: true)).stream(request(.custom, "m", endpointID: "ep1")))298 #expect(c.error == nil)299 }300 }301302 @Test func toolCall() async {303 let c = await collect(provider(FixtureTransport(.sse("custom_tool.sse"))).stream(request(.custom, "maclustr-qwen3", tools: [weatherTool], endpointID: "ep1")))304 expectWellFormed(c)305 #expect(c.completions == [.toolCalls])306 #expect(c.outcome.toolCalls == [ToolCall(id: "call_bfa70d226dea45cd9ebd6a32", name: "get_weather", arguments: ["city": "Paris"])])307 }308309 @Test func fragmentedParallelCallsReasoningContentAndInvalidJSON() async {310 let c = await collect(provider(FixtureTransport(.sse("custom_reasoning_content.sse"))).stream(request(.custom, "deep-local", endpointID: "ep1")))311 expectWellFormed(c)312 #expect(c.reasoning == "Adding the two numbers.")313 #expect(c.text == "4")314 #expect(c.completions == [.toolCalls]) // "stop" with tool calls still means the turn ended in calls315 #expect(c.outcome.toolCalls.count == 2)316 // No id was sent for the first call: one is synthesised; fragments are joined by index.317 #expect(c.outcome.toolCalls[0] == ToolCall(id: "call_chatcmpl-r_0", name: "sum", arguments: ["a": 2, "b": 2]))318 #expect(c.deltas(for: "call_chatcmpl-r_0") == ["{\"a\":", "2,", "\"b\":2}"])319 // Arguments that are not JSON are preserved verbatim.320 #expect(c.outcome.toolCalls[1] == ToolCall(id: "call_b", name: "noop", arguments: ["_raw": "not json"]))321 }322323 @Test func serverThatClosesWithoutDone() async {324 let c = await collect(provider(FixtureTransport(.sse("custom_no_done.sse"))).stream(request(.custom, "maclustr-qwen3", endpointID: "ep1")))325 expectWellFormed(c)326 #expect(c.text == "ASTRA-OK")327 #expect(c.completions == [.endOfTurn])328 }329330 @Test func serverThatIgnoresStreamAndAnswersWithJSON() async {331 let c = await collect(provider(FixtureTransport(.file(200, "custom_plain_json.json"))).stream(request(.custom, "local-model", endpointID: "ep1")))332 expectWellFormed(c)333 #expect(c.text == "ASTRA-OK")334 #expect(c.outcome.usage?.inputTokens == 9)335 }336337 @Test func rejectedOptionalFieldsAreRemovedOneByOne() async {338 let transport = FixtureTransport(.json(400, #"{"error":{"message":"Unrecognized request argument supplied: stream_options","type":"invalid_request_error"}}"#),339 .json(400, #"{"error":{"message":"Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.","type":"invalid_request_error","param":"max_tokens","code":"unsupported_parameter"}}"#),340 .json(422, #"{"detail":"reasoning_effort: extra fields not permitted"}"#),341 .sse("custom_text.sse"))342 let c = await collect(provider(transport).stream(request(.custom, "m", parameters: GenerationParameters(maxOutputTokens: 40, reasoning: .high), endpointID: "ep1")))343 expectWellFormed(c)344 #expect(transport.requests.count == 4)345 #expect(transport.body(1)["stream_options"] == nil)346 #expect(transport.body(2)["max_tokens"] == nil)347 #expect(transport.body(2)["max_completion_tokens"] == 40)348 #expect(transport.body(3)["reasoning_effort"] == nil)349 }350351 @Test func recordedErrorBodies() async {352 var c = await collect(provider(FixtureTransport(.file(401, "custom_error_401.json"))).stream(request(.custom, "maclustr-qwen3", endpointID: "ep1")))353 guard case .unauthorized = c.error else { Issue.record("expected unauthorized"); return }354 c = await collect(provider(FixtureTransport(.file(404, "custom_error_404.json"))).stream(request(.custom, "nope", endpointID: "ep1")))355 #expect(c.error == .modelNotFound("nope"))356 // An HTML gateway page is never shown raw.357 c = await collect(provider(FixtureTransport(.init(status: 502, body: "<html><body>Bad gateway \(Self.customKey)</body></html>"))).stream(request(.custom, "m", endpointID: "ep1")))358 #expect(c.error == .server(status: 502, message: nil))359 }360361 @Test func modelListReportsOnlyWhatTheEndpointSays() async throws {362 let transport = FixtureTransport(.file(200, "custom_models.json"))363 let models = try await provider(transport, profile: profile(defaultModel: "typed-by-user")).listModels()364 #expect(transport.requests[0].url?.absoluteString == "https://llm.example.com/v1/models")365 #expect(models.map(\.ref.modelID) == ["typed-by-user", "maclustr-qwen3"])366 #expect(models[0].source == .user)367 #expect(models[0].capabilities == [.text])368 let reported = models[1]369 #expect(reported.ref == ModelRef(provider: .custom, modelID: "maclustr-qwen3", endpointID: "ep1"))370 #expect(reported.contextLength == 32768)371 #expect(reported.maxOutputTokens == 8192)372 #expect(reported.capabilities == [.text, .tools]) // the endpoint reports tools: true, thinking: false373 #expect(reported.pricing == nil)374 guard case .providerAPI = reported.source else { Issue.record("expected providerAPI"); return }375 // A bare OpenAI-style list claims text only.376 let bare = try await provider(FixtureTransport(.json(200, #"{"object":"list","data":[{"id":"llama3.2:3b","object":"model","owned_by":"library"}]}"#))).listModels()377 #expect(bare.first?.capabilities == [.text])378 #expect(bare.first?.contextLength == nil)379 }380381 @Test func endpointWithoutAModelListFallsBackToTheTypedModel() async throws {382 let models = try await provider(FixtureTransport(.json(404, #"{"error":"not found"}"#)), profile: profile(defaultModel: "my-model")).listModels()383 #expect(models.map(\.ref.modelID) == ["my-model"])384 #expect(models[0].source == .user)385 }386}387