Repository Public License
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX OS Automation9// File : Packages/ORIXProviderAdapters/Tests/ORIXProviderAdaptersTests/FixtureOpenAITests.swift10// Description : Mocked tests of the OpenAI adapter (Chat Completions)11// against sanitized, recorded-shape SSE transcripts:12// streaming, tool calls by index, usage, structured outputs,13// temperature handling, errors.14// Module : ORIXProviderAdaptersTests15// Repository : https://git.spboucher.ai/orix-automation16// Created : 2026-11-2117// Last Modified : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author : Simon-Pierre Boucher24// Position : Professor of Finance25// Institution : Université du Québec en Outaouais (UQO)26// Department : Département des sciences administratives27// Organization : SP Boucher28//29// Website : https://spboucher.ai30// Git : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General : info@spboucher.ai37// Professional : contact@spboucher.ai38// Direct : simon@spboucher.ai39// Research : research@spboucher.ai40// Development : dev@spboucher.ai41// Support : support@spboucher.ai42// Security/Admin : admin@spboucher.ai43// Notifications : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification : Public70// Distribution : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License : Proprietary, all rights reserved (see license file)84// License File : LICENSE85// Maintainer : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Testing93import Foundation94import ORIXCore95@testable import ORIXProviderAdapters9697@Suite("Fixture OpenAI adapter (mocked network)")98struct FixtureOpenAITests {99 func provider(_ server: StubServer, options: OpenAIOptions = .init()) -> OpenAIProvider {100 OpenAIProvider(apiKey: fixtureKey, session: server.session, baseURL: server.baseURL.appendingPathComponent("v1"), options: options)101 }102103 static func chunk(_ delta: String, finish: String? = nil) -> String {104 #"{"id":"chatcmpl-fixture","object":"chat.completion.chunk","created":1,"model":"gpt-4o-mini","choices":[{"index":0,"delta":\#(delta),"finish_reason":\#(finish.map { "\"\($0)\"" } ?? "null")}],"usage":null}"#105 }106 static let usageChunk = #"{"id":"chatcmpl-fixture","object":"chat.completion.chunk","created":1,"model":"gpt-4o-mini","choices":[],"usage":{"prompt_tokens":1200,"completion_tokens":7,"total_tokens":1207,"prompt_tokens_details":{"cached_tokens":1024,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}}"#107108 static let textTranscript = chatSSE([109 chunk(#"{"role":"assistant","content":"","refusal":null}"#),110 chunk(#"{"content":"Hello"}"#),111 chunk(#"{"content":" wörld"}"#),112 chunk("{}", finish: "stop"),113 usageChunk,114 ])115116 @Test func streamsTextAndReadsUsageFromTheFinalChunk() async {117 let server = StubServer().sse(Self.textTranscript, chunkSize: 13)118 let request = LLMRequest(model: "gpt-4o-mini", system: "Be brief.", messages: [.user("Hi")], maxOutputTokens: 16, temperature: 0.3)119 let (events, error) = await collect(provider(server, options: .init(organization: "org-fixture")).stream(request))120 #expect(error == nil)121 #expect(events.text == "Hello wörld")122 #expect(events.lastUsage == TokenUsage(inputTokens: 176, outputTokens: 7, cacheReadTokens: 1024), "cached prompt tokens are counted apart from uncached input")123 #expect(events.suffix(2) == [.usage(events.lastUsage!), .stop(.endTurn)])124125 let req = server.requests[0]126 #expect(req.method == "POST" && req.path == "/v1/chat/completions")127 #expect(req.header("Authorization") == "Bearer \(fixtureKey)")128 #expect(req.header("OpenAI-Organization") == "org-fixture")129 #expect(req.body?["max_completion_tokens"] == 16 && req.body?["max_tokens"] == nil)130 #expect(req.body?["stream"] == true && req.body?["stream_options"] == ["include_usage": true])131 #expect(req.body?["temperature"] == .number(0.3))132 #expect(req.body?["messages"] == [["role": "system", "content": "Be brief."], ["role": "user", "content": "Hi"]])133 }134135 @Test func assemblesParallelToolCallsByIndexFromInterleavedDeltas() async {136 let transcript = chatSSE([137 Self.chunk(#"{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_fixture_a","type":"function","function":{"name":"get_weather","arguments":""}}]}"#),138 Self.chunk(#"{"tool_calls":[{"index":0,"function":{"arguments":"{\"ci"}}]}"#),139 Self.chunk(#"{"tool_calls":[{"index":1,"id":"call_fixture_b","type":"function","function":{"name":"get_weather","arguments":"{\"city\":"}}]}"#),140 Self.chunk(#"{"tool_calls":[{"index":0,"function":{"arguments":"ty\":\"Paris\"}"}},{"index":1,"function":{"arguments":"\"Lyon\"}"}}]}"#),141 Self.chunk("{}", finish: "tool_calls"),142 Self.usageChunk,143 ])144 let (events, error) = await collect(provider(StubServer().sse(transcript, chunkSize: 9)).stream(LLMRequest(model: "gpt-4o-mini", messages: [.user("x")], tools: [weatherTool])))145 #expect(error == nil)146 #expect(events.started == ["0:call_fixture_a:get_weather", "1:call_fixture_b:get_weather"])147 #expect(events.inputDeltas == ["{\"ci", "{\"city\":", "ty\":\"Paris\"}", "\"Lyon\"}"])148 #expect(events.toolUses == [LLMToolUse(id: "call_fixture_a", name: "get_weather", input: ["city": "Paris"]),149 LLMToolUse(id: "call_fixture_b", name: "get_weather", input: ["city": "Lyon"])])150 #expect(events.stop == .toolUse)151 }152153 @Test func encodesToolsToolResultsImagesAndStrictStructuredOutput() {154 let schema = JSONSchema.object(["answer": .string()], required: ["answer"])155 let request = LLMRequest(156 model: "gpt-4o-mini", system: "S",157 messages: [158 .user("Weather?"),159 LLMMessage(role: .assistant, content: [.text("Looking."), .toolUse(LLMToolUse(id: "call_fixture_a", name: "get_weather", input: ["city": "Paris"])),160 .toolUse(LLMToolUse(id: "call_fixture_b", name: "get_weather", input: ["city": "Lyon"]))]),161 LLMMessage(role: .user, content: [.text("thanks"), .toolResult(toolUseID: "call_fixture_a", content: "12°C", isError: false),162 .toolResult(toolUseID: "call_fixture_b", content: "timeout", isError: true)]),163 LLMMessage(role: .user, content: [.text("look"), .image(base64: "AAAA", mediaType: "image/jpeg")]),164 ],165 tools: [weatherTool], toolChoice: .tool(name: "get_weather"), maxOutputTokens: 64, responseSchema: schema, cacheSystemAndTools: true)166 let body = provider(StubServer(), options: .init(reasoningEffort: "low")).requestBody(for: request)167168 #expect(body["tools"] == [["type": "function", "function": ["name": "get_weather", "description": "Returns the weather for a city.", "parameters": weatherTool.inputSchema.raw]]])169 #expect(body["tool_choice"] == ["type": "function", "function": ["name": "get_weather"]])170 #expect(body["response_format"] == ["type": "json_schema", "json_schema": ["name": "orix_response", "strict": true, "schema": schema.raw]])171 #expect(body["reasoning_effort"] == "low")172 let m = body["messages"]?.arrayValue ?? []173 #expect(m.count == 7)174 #expect(m[0] == ["role": "system", "content": "S"])175 #expect(m[2]["content"] == "Looking." && m[2]["tool_calls"]?.arrayValue?.count == 2)176 #expect(m[2]["tool_calls"]?[0] == ["id": "call_fixture_a", "type": "function", "function": ["name": "get_weather", "arguments": "{\"city\":\"Paris\"}"]])177 #expect(m[3] == ["role": "tool", "tool_call_id": "call_fixture_a", "content": "12°C"], "tool results directly follow the assistant turn")178 #expect(m[4] == ["role": "tool", "tool_call_id": "call_fixture_b", "content": "[tool error] timeout"])179 #expect(m[5] == ["role": "user", "content": "thanks"])180 #expect(m[6] == ["role": "user", "content": [["type": "text", "text": "look"], ["type": "image_url", "image_url": ["url": "data:image/jpeg;base64,AAAA"]]]])181182 func choice(_ c: LLMToolChoice) -> JSONValue? { provider(StubServer()).requestBody(for: LLMRequest(model: "gpt-4o-mini", messages: [.user("x")], tools: [weatherTool], toolChoice: c))["tool_choice"] }183 #expect(choice(.auto) == "auto" && choice(.none) == "none" && choice(.required) == "required")184 // An assistant turn that only calls tools has null content.185 #expect(ChatCompletionsEncoder.encode(LLMMessage(role: .assistant, content: [.toolUse(LLMToolUse(id: "c", name: "n", input: [:]))]))[0]["content"] == .null)186 }187188 @Test func omitsTemperatureForModelsKnownToRejectIt() {189 let p = provider(StubServer())190 func temp(_ model: String) -> JSONValue? { p.requestBody(for: LLMRequest(model: model, messages: [.user("x")], temperature: 0.5))["temperature"] }191 #expect(temp("gpt-4o-mini") == .number(0.5))192 #expect(temp("gpt-4o-mini-2024-07-18") == .number(0.5))193 #expect(temp("gpt-5-nano") == nil)194 #expect(temp("gpt-5-mini-2025-08-07") == nil)195 #expect(temp("o3") == nil && temp("o4-mini") == nil && temp("gpt-5.9-unknown") == nil)196 #expect(temp("gpt-6-luna") == .number(0.5), "not known to reject it: sent, with the one-shot retry as safety net")197 }198199 @Test func retriesOnceWithoutTemperatureOnTheSpecific400() async throws {200 let rejection = #"{"error":{"message":"Unsupported value: 'temperature' does not support 0.5 with this model. Only the default (1) value is supported.","type":"invalid_request_error","param":"temperature","code":"unsupported_value"}}"#201 let server = StubServer().json(400, rejection).sse(Self.textTranscript).sse(Self.textTranscript)202 let p = provider(server)203 let request = LLMRequest(model: "gpt-6-luna", messages: [.user("x")], temperature: 0.5)204 #expect(try await p.complete(request).text == "Hello wörld")205 #expect(server.requests.count == 2)206 #expect(server.requests[0].body?["temperature"] == .number(0.5) && server.requests[1].body?["temperature"] == nil)207 // The model is remembered: the next call does not send temperature at all.208 _ = try await p.complete(request)209 #expect(server.requests.count == 3 && server.requests[2].body?["temperature"] == nil)210211 // Any other 400 is not retried, and a second temperature rejection cannot loop.212 let other = StubServer().json(400, #"{"error":{"message":"Invalid schema for response_format","type":"invalid_request_error","param":"response_format","code":null}}"#)213 let (_, error) = await collect(provider(other).stream(LLMRequest(model: "gpt-4o-mini", messages: [.user("x")], temperature: 0.5)))214 #expect(error == .invalidRequest("Invalid schema for response_format"))215 #expect(other.requests.count == 1)216 }217218 @Test(arguments: [219 (401, #"{"error":{"message":"Incorrect API key provided.","type":"invalid_request_error","param":null,"code":"invalid_api_key"}}"#, LLMError.authentication("Incorrect API key provided.")),220 (403, #"{"error":{"message":"Country not supported","type":"request_forbidden","param":null,"code":"unsupported_country_region_territory"}}"#, LLMError.authentication("Country not supported")),221 (404, #"{"error":{"message":"The model `gpt-nope` does not exist or you do not have access to it.","type":"invalid_request_error","param":null,"code":"model_not_found"}}"#, LLMError.modelNotFound("gpt-nope — The model `gpt-nope` does not exist or you do not have access to it.")),222 (400, #"{"error":{"message":"This model's maximum context length is 128000 tokens. However, your messages resulted in 130000 tokens.","type":"invalid_request_error","param":"messages","code":"context_length_exceeded"}}"#, LLMError.contextTooLong("This model's maximum context length is 128000 tokens. However, your messages resulted in 130000 tokens.")),223 (500, #"{"error":{"message":"The server had an error","type":"server_error","param":null,"code":null}}"#, LLMError.server(status: 500, message: "The server had an error")),224 (503, #"{"error":{"message":"The engine is currently overloaded, please try again later","type":"server_error"}}"#, LLMError.overloaded),225 (529, "", LLMError.overloaded),226 (429, #"{"error":{"message":"You exceeded your current quota","type":"insufficient_quota","param":null,"code":"insufficient_quota"}}"#, LLMError.invalidRequest("The provider account has no remaining quota or credit. You exceeded your current quota")),227 ] as [(Int, String, LLMError)])228 func mapsHTTPErrors(status: Int, body: String, expected: LLMError) async {229 let (events, error) = await collect(provider(StubServer().json(status, body)).stream(LLMRequest(model: "gpt-nope", messages: [.user("x")])))230 #expect(events.isEmpty && error == expected)231 }232233 @Test func rateLimitHonoursRetryAfterHeaders() async {234 let body = #"{"error":{"message":"Rate limit reached","type":"requests","param":null,"code":"rate_limit_exceeded"}}"#235 let seconds = await collect(provider(StubServer().json(429, body, headers: ["Retry-After": "3"])).stream(LLMRequest(model: "gpt-4o-mini", messages: [.user("x")])))236 #expect(seconds.error == .rateLimited(retryAfterSeconds: 3))237 let millis = await collect(provider(StubServer().json(429, body, headers: ["retry-after-ms": "250", "retry-after": "1"])).stream(LLMRequest(model: "gpt-4o-mini", messages: [.user("x")])))238 #expect(millis.error == .rateLimited(retryAfterSeconds: 0.25))239 let none = await collect(provider(StubServer().json(429, body)).stream(LLMRequest(model: "gpt-4o-mini", messages: [.user("x")])))240 #expect(none.error == .rateLimited(retryAfterSeconds: nil))241 }242243 @Test func midStreamDisconnectAndStreamsWithoutDoneAreHandled() async {244 let request = LLMRequest(model: "gpt-4o-mini", messages: [.user("x")])245 let head = chatSSE([Self.chunk(#"{"role":"assistant","content":"par"}"#), Self.chunk(#"{"content":"tial"}"#)], done: false)246 let dropped = await collect(provider(StubServer().sse(head, failure: .networkConnectionLost)).stream(request))247 #expect(dropped.events.text == "partial" && dropped.events.stop == nil)248 if case .network = dropped.error {} else { Issue.record("expected a network error, got \(String(describing: dropped.error))") }249250 let early = await collect(provider(StubServer().sse(head)).stream(request))251 #expect(early.error == .network("The response stream ended before the completion was finished."))252253 // A stream that carried its finish reason but no [DONE] sentinel is complete.254 let noDone = await collect(provider(StubServer().sse(head + chatSSE([Self.chunk("{}", finish: "stop")], done: false))).stream(request))255 #expect(noDone.error == nil && noDone.events.stop == .endTurn)256257 let garbage = await collect(provider(StubServer().sse("data: {not json}\n\n")).stream(request))258 #expect(garbage.error == .decoding("A stream chunk was not valid JSON."))259 }260261 @Test func lengthCutToolCallIsDroppedAndRefusalAndFilterAreReported() async {262 let request = LLMRequest(model: "gpt-4o-mini", messages: [.user("x")], tools: [weatherTool])263 func run(_ chunks: [String]) async -> (events: [LLMStreamEvent], error: LLMError?) {264 await collect(provider(StubServer().sse(chatSSE(chunks))).stream(request))265 }266 let call = Self.chunk(#"{"tool_calls":[{"index":0,"id":"call_fixture_a","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Mon"}}]}"#)267 let cut = await run([call, Self.chunk("{}", finish: "length")])268 #expect(cut.error == nil && cut.events.toolUses.isEmpty && cut.events.stop == .maxTokens)269 let broken = await run([call, Self.chunk("{}", finish: "tool_calls")])270 if case .decoding(let m) = broken.error { #expect(m.contains("get_weather")) } else { Issue.record("expected a decoding error") }271 #expect(broken.events.toolUses.isEmpty && broken.events.stop == nil)272273 let refusal = await run([Self.chunk(#"{"role":"assistant","content":null,"refusal":"I can't help with that."}"#), Self.chunk("{}", finish: "stop")])274 #expect(refusal.events.stop == .refusal && refusal.events.text == "I can't help with that.")275 let filtered = await run([Self.chunk(#"{"content":"x"}"#), Self.chunk("{}", finish: "content_filter")])276 #expect(filtered.events.stop == .contentFilter)277 }278279 @Test func listsOnlyChatModelsAndEnrichesFromTheRegistry() async throws {280 let ids = ["gpt-4o-mini", "gpt-4o-mini-2024-07-18", "gpt-5-nano", "o3", "gpt-9-future", "text-embedding-3-small", "whisper-1", "tts-1",281 "gpt-4o-audio-preview", "gpt-4o-realtime-preview", "gpt-image-1", "dall-e-3", "omni-moderation-latest", "gpt-3.5-turbo-instruct",282 "gpt-5-pro", "gpt-5.2-codex", "gpt-4o-mini-transcribe", "gpt-4o-search-preview", "davinci-002", "o3-deep-research"]283 let body = "{\"object\":\"list\",\"data\":[" + ids.map { "{\"id\":\"\($0)\",\"object\":\"model\",\"created\":1,\"owned_by\":\"system\"}" }.joined(separator: ",") + "]}"284 let server = StubServer().json(200, body)285 let models = try await provider(server).listModels()286 #expect(server.requests[0].path == "/v1/models" && server.requests[0].method == "GET")287 #expect(models.map(\.id) == ["gpt-4o-mini", "gpt-4o-mini-2024-07-18", "gpt-5-nano", "gpt-9-future", "o3"])288 #expect(models.allSatisfy { $0.source == "api" && $0.provider == .openai })289 #expect(models[0].inputPricePerMTok == 0.15 && models[0].outputPricePerMTok == 0.60)290 #expect(models[1].inputPricePerMTok == 0.15 && models[1].id == "gpt-4o-mini-2024-07-18")291 #expect(models[2].inputPricePerMTok == 0.05 && models[2].outputPricePerMTok == 0.40)292 #expect(models[3].inputPricePerMTok == nil && models[3].contextWindow == nil, "unknown models carry no invented metadata")293 }294}295