Repository Public License
1// ORIX OS Automation2// File: Packages/ORIXProviderAdapters/Tests/ORIXProviderAdaptersTests/FixtureOpenRouterTests.swift3// Description: Mocked tests of the OpenRouter adapter: catalog parsing, reported cost, provider routing, headers, error bodies and mid-stream error chunks, with sanitized recorded-shape fixtures.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Testing8import Foundation9import ORIXCore10@testable import ORIXProviderAdapters1112@Suite("Fixture OpenRouter adapter (mocked network)")13struct FixtureOpenRouterTests {14 func provider(_ server: StubServer, routing: ProviderRouting? = nil) -> OpenRouterProvider {15 OpenRouterProvider(apiKey: fixtureKey, session: server.session, baseURL: server.baseURL.appendingPathComponent("api/v1"), routing: routing)16 }1718 static func chunk(_ delta: String, finish: String? = nil, usage: String = "") -> String {19 #"{"id":"gen-fixture","provider":"FixtureAI","model":"vendor/model-a","object":"chat.completion.chunk","created":1,"choices":[{"index":0,"delta":\#(delta),"finish_reason":\#(finish.map { "\"\($0)\"" } ?? "null"),"native_finish_reason":null}]\#(usage)}"#20 }21 static let usage = #","usage":{"prompt_tokens":100,"completion_tokens":20,"total_tokens":120,"cost":0.0000325,"is_byok":false,"prompt_tokens_details":{"cached_tokens":40,"cache_write_tokens":10},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}"#2223 static let catalog = #"""24 {"data":[25 {"id":"vendor/model-a","canonical_slug":"vendor/model-a","name":"Vendor: Model A","created":1,"context_length":128000,26 "architecture":{"modality":"text+image->text","input_modalities":["text","image"],"output_modalities":["text"],"tokenizer":"GPT"},27 "pricing":{"prompt":"0.00000015","completion":"0.0000006","input_cache_read":"0.000000075"},28 "top_provider":{"context_length":128000,"max_completion_tokens":16384,"is_moderated":true},29 "supported_parameters":["max_tokens","response_format","structured_outputs","temperature","tool_choice","tools"]},30 {"id":"vendor/text-only","name":"Vendor: Text Only","context_length":32768,31 "architecture":{"input_modalities":["text"],"output_modalities":["text"]},32 "pricing":{"prompt":"0.000003","completion":"0.000015"},33 "top_provider":{"context_length":32768,"max_completion_tokens":null,"is_moderated":false},34 "supported_parameters":["max_tokens","temperature"]},35 {"id":"vendor/free:free","name":"Free","context_length":8192,"pricing":{"prompt":"0","completion":"0"},"supported_parameters":["tools"]},36 {"id":"openrouter/auto","name":"Auto Router","context_length":2000000,"pricing":{"prompt":"-1","completion":"-1"},"supported_parameters":["tools","response_format"]}37 ]}38 """#3940 @Test func parsesTheCatalogIntoPricesLimitsAndCapabilities() async throws {41 let server = StubServer().json(200, Self.catalog)42 let models = try await provider(server).listModels()43 #expect(server.requests[0].path == "/api/v1/models")44 #expect(models.count == 4 && models.allSatisfy { $0.source == "api" && $0.provider == .openrouter })45 let a = models[0]46 #expect(a.displayName == "Vendor: Model A" && a.contextWindow == 128_000 && a.maxOutputTokens == 16_384)47 #expect(a.inputPricePerMTok == 0.15 && a.outputPricePerMTok == 0.6, "per-token strings become USD per million tokens")48 #expect(a.supportsTools && a.supportsVision && a.supportsStructuredOutput)49 let b = models[1]50 #expect(b.inputPricePerMTok == 3 && b.outputPricePerMTok == 15 && b.maxOutputTokens == nil)51 #expect(!b.supportsTools && !b.supportsVision && !b.supportsStructuredOutput)52 #expect(models[2].inputPricePerMTok == 0 && models[2].outputPricePerMTok == 0)53 #expect(models[3].inputPricePerMTok == nil && models[3].outputPricePerMTok == nil, "dynamic pricing (-1) is unknown, not negative")54 }5556 @Test func streamsTextThroughKeepAliveCommentsAndReportsCost() async {57 let transcript = ": OPENROUTER PROCESSING\n\n" + chatSSE([58 Self.chunk(#"{"role":"assistant","content":""}"#),59 Self.chunk(#"{"content":"Salut","reasoning":"Greeting in French."}"#),60 ], done: false) + ": OPENROUTER PROCESSING\n\n" + chatSSE([61 Self.chunk(#"{"content":" !"}"#, finish: "stop"),62 Self.chunk(#"{"role":"assistant","content":""}"#, usage: Self.usage),63 ])64 let server = StubServer().sse(transcript, chunkSize: 31)65 let routing = ProviderRouting(order: ["fixture-a", "fixture-b"], allowFallbacks: false, dataCollection: .deny, requireParameters: true)66 let (events, error) = await collect(provider(server, routing: routing).stream(LLMRequest(model: "vendor/model-a", system: "S", messages: [.user("Hi")], maxOutputTokens: 16, temperature: 0)))67 #expect(error == nil)68 #expect(events.text == "Salut !" && events.reasoning == "Greeting in French." && events.stop == .endTurn)69 #expect(events.lastUsage == TokenUsage(inputTokens: 50, outputTokens: 20, cacheReadTokens: 40, cacheWriteTokens: 10, reportedCostUSD: 0.0000325))70 #expect(events.lastUsage?.costUSD(model: nil) == 0.0000325)7172 let req = server.requests[0]73 #expect(req.path == "/api/v1/chat/completions")74 #expect(req.header("Authorization") == "Bearer \(fixtureKey)")75 #expect(req.header("X-Title") == "ORIX OS Automation" && req.header("HTTP-Referer") == "http://localhost")76 #expect(req.body?["provider"] == ["order": ["fixture-a", "fixture-b"], "allow_fallbacks": false, "data_collection": "deny", "require_parameters": true])77 #expect(req.body?["usage"] == ["include": true])78 #expect(req.body?["max_tokens"] == 16 && req.body?["max_completion_tokens"] == nil)79 #expect(req.body?["temperature"]?.doubleValue == 0)80 #expect(req.body?["messages"]?[0] == ["role": "system", "content": "S"])81 }8283 @Test func routingIsOmittedWhenEmptyAndCacheControlOnlyGoesToAnthropicModels() {84 let p = provider(StubServer(), routing: ProviderRouting())85 let plain = p.requestBody(for: LLMRequest(model: "vendor/model-a", system: "S", messages: [.user("x")], cacheSystemAndTools: true))86 #expect(plain["provider"] == nil)87 #expect(plain["messages"]?[0] == ["role": "system", "content": "S"])88 let claude = p.requestBody(for: LLMRequest(model: "anthropic/claude-haiku-5.5", system: "S", messages: [.user("x")], cacheSystemAndTools: true))89 #expect(claude["messages"]?[0] == ["role": "system", "content": [["type": "text", "text": "S", "cache_control": ["type": "ephemeral"]]]])90 }9192 @Test func toolCallWithoutAnIdGetsAStablePositionalId() async {93 let transcript = chatSSE([94 Self.chunk(#"{"role":"assistant","content":null,"tool_calls":[{"index":0,"type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Rome\"}"}}]}"#),95 Self.chunk("{}", finish: "stop", usage: Self.usage),96 ])97 let (events, error) = await collect(provider(StubServer().sse(transcript)).stream(LLMRequest(model: "vendor/model-a", messages: [.user("x")], tools: [weatherTool])))98 #expect(error == nil)99 #expect(events.toolUses == [LLMToolUse(id: "call_0", name: "get_weather", input: ["city": "Rome"])])100 #expect(events.stop == .toolUse, "an upstream that says `stop` while calling a tool still ends in toolUse")101 }102103 @Test func structuredOutputIsRefusedUpFrontForModelsThatLackIt() async throws {104 let server = StubServer().json(200, Self.catalog).sse(chatSSE([Self.chunk(#"{"content":"{}"}"#, finish: "stop")]))105 let p = provider(server)106 _ = try await p.listModels()107 let schema = JSONSchema.object(["a": .string()], required: ["a"])108 let refused = await collect(p.stream(LLMRequest(model: "vendor/text-only", messages: [.user("x")], responseSchema: schema)))109 #expect(refused.error == .unsupported("vendor/text-only does not offer structured outputs on OpenRouter."))110 #expect(server.requests.count == 1, "no completion request was sent")111 let ok = await collect(p.stream(LLMRequest(model: "vendor/model-a", messages: [.user("x")], responseSchema: schema)))112 #expect(ok.error == nil)113 #expect(server.requests[1].body?["response_format"]?["json_schema"]?["strict"] == true)114 }115116 @Test(arguments: [117 (401, #"{"error":{"code":401,"message":"No auth credentials found"}}"#, LLMError.authentication("No auth credentials found")),118 (402, #"{"error":{"code":402,"message":"Insufficient credits"}}"#, LLMError.invalidRequest("The OpenRouter account or key has insufficient credits. Insufficient credits")),119 (403, #"{"error":{"code":403,"message":"Input flagged","metadata":{"reasons":["violence"],"flagged_input":"…","provider_name":"FixtureAI","model_slug":"vendor/model-a"}}}"#, LLMError.invalidRequest("Blocked by moderation. Input flagged [provider: FixtureAI] [flagged: violence]")),120 (403, #"{"error":{"code":403,"message":"Key lacks permission for this model"}}"#, LLMError.authentication("Key lacks permission for this model")),121 (404, #"{"error":{"code":404,"message":"No endpoints found for vendor/nope."}}"#, LLMError.modelNotFound("vendor/nope — No endpoints found for vendor/nope.")),122 (400, #"{"error":{"code":400,"message":"vendor/nope is not a valid model ID"}}"#, LLMError.modelNotFound("vendor/nope — vendor/nope is not a valid model ID")),123 (400, #"{"error":{"code":400,"message":"This endpoint's maximum context length is 32768 tokens. However, you requested about 40000 tokens.","metadata":{"provider_name":"FixtureAI"}}}"#, LLMError.contextTooLong("This endpoint's maximum context length is 32768 tokens. However, you requested about 40000 tokens. [provider: FixtureAI]")),124 (408, #"{"error":{"code":408,"message":"Request timed out"}}"#, LLMError.network("the provider timed out (HTTP 408)")),125 (502, #"{"error":{"code":502,"message":"Provider returned error","metadata":{"provider_name":"FixtureAI","raw":"upstream exploded"}}}"#, LLMError.server(status: 502, message: "Provider returned error [provider: FixtureAI] upstream exploded")),126 (503, #"{"error":{"code":503,"message":"No allowed providers are available for the selected model."}}"#, LLMError.server(status: 503, message: "No allowed providers are available for the selected model.")),127 (529, #"{"error":{"code":529,"message":"Provider overloaded"}}"#, LLMError.overloaded),128 (400, #"{"error":{"code":400,"message":"tools is not supported by this provider"}}"#, LLMError.invalidRequest("tools is not supported by this provider")),129 ] as [(Int, String, LLMError)])130 func mapsErrorBodies(status: Int, body: String, expected: LLMError) async {131 let (events, error) = await collect(provider(StubServer().json(status, body)).stream(LLMRequest(model: "vendor/nope", messages: [.user("x")])))132 #expect(events.isEmpty && error == expected)133 }134135 @Test func rateLimitUsesRetryAfterAndUpstreamRateLimitsAreRecognised() async {136 let direct = await collect(provider(StubServer().json(429, #"{"error":{"code":429,"message":"Rate limit exceeded"}}"#, headers: ["Retry-After": "12"])).stream(LLMRequest(model: "vendor/model-a", messages: [.user("x")])))137 #expect(direct.error == .rateLimited(retryAfterSeconds: 12))138 // The HTTP status and the body code can differ; the body code decides.139 let body = await collect(provider(StubServer().json(200, #"{"error":{"code":429,"message":"Provider returned error","metadata":{"provider_name":"FixtureAI","raw":"{\"error\":\"slow down\"}"}}}"#)).stream(LLMRequest(model: "vendor/model-a", messages: [.user("x")])))140 #expect(body.error == .rateLimited(retryAfterSeconds: nil))141 }142143 @Test func midStreamErrorChunksBecomeTypedErrorsAfterThePartialOutput() async {144 let request = LLMRequest(model: "vendor/model-a", messages: [.user("x")])145 let head = chatSSE([Self.chunk(#"{"role":"assistant","content":"par"}"#), Self.chunk(#"{"content":"tial"}"#)], done: false)146 let errorChunk = #"{"id":"gen-fixture","object":"chat.completion.chunk","created":1,"model":"vendor/model-a","provider":"FixtureAI","error":{"code":502,"message":"Provider disconnected","metadata":{"error_type":"provider_unavailable","provider_name":"FixtureAI"}},"choices":[{"index":0,"delta":{"content":""},"finish_reason":"error"}]}"#147 let a = await collect(provider(StubServer().sse(head + chatSSE([errorChunk], done: false))).stream(request))148 #expect(a.events.text == "partial" && a.events.stop == nil)149 #expect(a.error == .server(status: 502, message: "Provider disconnected [provider: FixtureAI]"))150 #expect(a.error?.isRetryable == true)151152 let limited = #"{"error":{"code":"429","message":"Upstream rate limited","metadata":{"error_type":"rate_limit_exceeded"}},"choices":[{"index":0,"delta":{"content":""},"finish_reason":"error"}]}"#153 let b = await collect(provider(StubServer().sse(head + chatSSE([limited]))).stream(request))154 #expect(b.error == .rateLimited(retryAfterSeconds: nil))155156 // finish_reason "error" without an error object is still an error, never a normal stop.157 let bare = await collect(provider(StubServer().sse(head + chatSSE([Self.chunk(#"{"content":""}"#, finish: "error")]))).stream(request))158 if case .server(let status, _) = bare.error { #expect(status == 502) } else { Issue.record("expected a server error, got \(String(describing: bare.error))") }159 #expect(bare.events.stop == nil)160161 let dropped = await collect(provider(StubServer().sse(head, failure: .timedOut)).stream(request))162 #expect(dropped.error == .network("the request timed out (URLError -1001)"))163 }164165 @Test func errorTextIsScrubbedOfTheKey() async {166 let body = #"{"error":{"code":401,"message":"Invalid key \#(fixtureKey)","metadata":{"raw":"Authorization: Bearer \#(fixtureKey)"}}}"#167 let (_, error) = await collect(provider(StubServer().json(401, body)).stream(LLMRequest(model: "vendor/model-a", messages: [.user("x")])))168 let rendered = "\(String(describing: error)) \(error?.errorDescription ?? "")"169 #expect(!rendered.contains(fixtureKey) && rendered.contains(Redactor.placeholder))170 }171}172