SPB Git

Repository Public License

FixtureOpenAITests.swift 20.2 KB 295 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX OS Automation9// File             : Packages/ORIXProviderAdapters/Tests/ORIXProviderAdaptersTests/FixtureOpenAITests.swift10// Description      : Mocked tests of the OpenAI adapter (Chat Completions)11//                    against sanitized, recorded-shape SSE transcripts:12//                    streaming, tool calls by index, usage, structured outputs,13//                    temperature handling, errors.14// Module           : ORIXProviderAdaptersTests15// Repository       : https://git.spboucher.ai/orix-automation16// Created          : 2026-11-2117// Last Modified    : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author           : Simon-Pierre Boucher24// Position         : Professor of Finance25// Institution      : Université du Québec en Outaouais (UQO)26// Department       : Département des sciences administratives27// Organization     : SP Boucher28//29// Website          : https://spboucher.ai30// Git              : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General          : info@spboucher.ai37// Professional     : contact@spboucher.ai38// Direct           : simon@spboucher.ai39// Research         : research@spboucher.ai40// Development      : dev@spboucher.ai41// Support          : support@spboucher.ai42// Security/Admin   : admin@spboucher.ai43// Notifications    : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification   : Public70// Distribution     : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License          : Proprietary, all rights reserved (see license file)84// License File     : LICENSE85// Maintainer       : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Testing93import Foundation94import ORIXCore95@testable import ORIXProviderAdapters9697@Suite("Fixture OpenAI adapter (mocked network)")98struct FixtureOpenAITests {99    func provider(_ server: StubServer, options: OpenAIOptions = .init()) -> OpenAIProvider {100        OpenAIProvider(apiKey: fixtureKey, session: server.session, baseURL: server.baseURL.appendingPathComponent("v1"), options: options)101    }102103    static func chunk(_ delta: String, finish: String? = nil) -> String {104        #"{"id":"chatcmpl-fixture","object":"chat.completion.chunk","created":1,"model":"gpt-4o-mini","choices":[{"index":0,"delta":\#(delta),"finish_reason":\#(finish.map { "\"\($0)\"" } ?? "null")}],"usage":null}"#105    }106    static let usageChunk = #"{"id":"chatcmpl-fixture","object":"chat.completion.chunk","created":1,"model":"gpt-4o-mini","choices":[],"usage":{"prompt_tokens":1200,"completion_tokens":7,"total_tokens":1207,"prompt_tokens_details":{"cached_tokens":1024,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}}"#107108    static let textTranscript = chatSSE([109        chunk(#"{"role":"assistant","content":"","refusal":null}"#),110        chunk(#"{"content":"Hello"}"#),111        chunk(#"{"content":" wörld"}"#),112        chunk("{}", finish: "stop"),113        usageChunk,114    ])115116    @Test func streamsTextAndReadsUsageFromTheFinalChunk() async {117        let server = StubServer().sse(Self.textTranscript, chunkSize: 13)118        let request = LLMRequest(model: "gpt-4o-mini", system: "Be brief.", messages: [.user("Hi")], maxOutputTokens: 16, temperature: 0.3)119        let (events, error) = await collect(provider(server, options: .init(organization: "org-fixture")).stream(request))120        #expect(error == nil)121        #expect(events.text == "Hello wörld")122        #expect(events.lastUsage == TokenUsage(inputTokens: 176, outputTokens: 7, cacheReadTokens: 1024), "cached prompt tokens are counted apart from uncached input")123        #expect(events.suffix(2) == [.usage(events.lastUsage!), .stop(.endTurn)])124125        let req = server.requests[0]126        #expect(req.method == "POST" && req.path == "/v1/chat/completions")127        #expect(req.header("Authorization") == "Bearer \(fixtureKey)")128        #expect(req.header("OpenAI-Organization") == "org-fixture")129        #expect(req.body?["max_completion_tokens"] == 16 && req.body?["max_tokens"] == nil)130        #expect(req.body?["stream"] == true && req.body?["stream_options"] == ["include_usage": true])131        #expect(req.body?["temperature"] == .number(0.3))132        #expect(req.body?["messages"] == [["role": "system", "content": "Be brief."], ["role": "user", "content": "Hi"]])133    }134135    @Test func assemblesParallelToolCallsByIndexFromInterleavedDeltas() async {136        let transcript = chatSSE([137            Self.chunk(#"{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_fixture_a","type":"function","function":{"name":"get_weather","arguments":""}}]}"#),138            Self.chunk(#"{"tool_calls":[{"index":0,"function":{"arguments":"{\"ci"}}]}"#),139            Self.chunk(#"{"tool_calls":[{"index":1,"id":"call_fixture_b","type":"function","function":{"name":"get_weather","arguments":"{\"city\":"}}]}"#),140            Self.chunk(#"{"tool_calls":[{"index":0,"function":{"arguments":"ty\":\"Paris\"}"}},{"index":1,"function":{"arguments":"\"Lyon\"}"}}]}"#),141            Self.chunk("{}", finish: "tool_calls"),142            Self.usageChunk,143        ])144        let (events, error) = await collect(provider(StubServer().sse(transcript, chunkSize: 9)).stream(LLMRequest(model: "gpt-4o-mini", messages: [.user("x")], tools: [weatherTool])))145        #expect(error == nil)146        #expect(events.started == ["0:call_fixture_a:get_weather", "1:call_fixture_b:get_weather"])147        #expect(events.inputDeltas == ["{\"ci", "{\"city\":", "ty\":\"Paris\"}", "\"Lyon\"}"])148        #expect(events.toolUses == [LLMToolUse(id: "call_fixture_a", name: "get_weather", input: ["city": "Paris"]),149                                    LLMToolUse(id: "call_fixture_b", name: "get_weather", input: ["city": "Lyon"])])150        #expect(events.stop == .toolUse)151    }152153    @Test func encodesToolsToolResultsImagesAndStrictStructuredOutput() {154        let schema = JSONSchema.object(["answer": .string()], required: ["answer"])155        let request = LLMRequest(156            model: "gpt-4o-mini", system: "S",157            messages: [158                .user("Weather?"),159                LLMMessage(role: .assistant, content: [.text("Looking."), .toolUse(LLMToolUse(id: "call_fixture_a", name: "get_weather", input: ["city": "Paris"])),160                                                       .toolUse(LLMToolUse(id: "call_fixture_b", name: "get_weather", input: ["city": "Lyon"]))]),161                LLMMessage(role: .user, content: [.text("thanks"), .toolResult(toolUseID: "call_fixture_a", content: "12°C", isError: false),162                                                  .toolResult(toolUseID: "call_fixture_b", content: "timeout", isError: true)]),163                LLMMessage(role: .user, content: [.text("look"), .image(base64: "AAAA", mediaType: "image/jpeg")]),164            ],165            tools: [weatherTool], toolChoice: .tool(name: "get_weather"), maxOutputTokens: 64, responseSchema: schema, cacheSystemAndTools: true)166        let body = provider(StubServer(), options: .init(reasoningEffort: "low")).requestBody(for: request)167168        #expect(body["tools"] == [["type": "function", "function": ["name": "get_weather", "description": "Returns the weather for a city.", "parameters": weatherTool.inputSchema.raw]]])169        #expect(body["tool_choice"] == ["type": "function", "function": ["name": "get_weather"]])170        #expect(body["response_format"] == ["type": "json_schema", "json_schema": ["name": "orix_response", "strict": true, "schema": schema.raw]])171        #expect(body["reasoning_effort"] == "low")172        let m = body["messages"]?.arrayValue ?? []173        #expect(m.count == 7)174        #expect(m[0] == ["role": "system", "content": "S"])175        #expect(m[2]["content"] == "Looking." && m[2]["tool_calls"]?.arrayValue?.count == 2)176        #expect(m[2]["tool_calls"]?[0] == ["id": "call_fixture_a", "type": "function", "function": ["name": "get_weather", "arguments": "{\"city\":\"Paris\"}"]])177        #expect(m[3] == ["role": "tool", "tool_call_id": "call_fixture_a", "content": "12°C"], "tool results directly follow the assistant turn")178        #expect(m[4] == ["role": "tool", "tool_call_id": "call_fixture_b", "content": "[tool error] timeout"])179        #expect(m[5] == ["role": "user", "content": "thanks"])180        #expect(m[6] == ["role": "user", "content": [["type": "text", "text": "look"], ["type": "image_url", "image_url": ["url": "data:image/jpeg;base64,AAAA"]]]])181182        func choice(_ c: LLMToolChoice) -> JSONValue? { provider(StubServer()).requestBody(for: LLMRequest(model: "gpt-4o-mini", messages: [.user("x")], tools: [weatherTool], toolChoice: c))["tool_choice"] }183        #expect(choice(.auto) == "auto" && choice(.none) == "none" && choice(.required) == "required")184        // An assistant turn that only calls tools has null content.185        #expect(ChatCompletionsEncoder.encode(LLMMessage(role: .assistant, content: [.toolUse(LLMToolUse(id: "c", name: "n", input: [:]))]))[0]["content"] == .null)186    }187188    @Test func omitsTemperatureForModelsKnownToRejectIt() {189        let p = provider(StubServer())190        func temp(_ model: String) -> JSONValue? { p.requestBody(for: LLMRequest(model: model, messages: [.user("x")], temperature: 0.5))["temperature"] }191        #expect(temp("gpt-4o-mini") == .number(0.5))192        #expect(temp("gpt-4o-mini-2024-07-18") == .number(0.5))193        #expect(temp("gpt-5-nano") == nil)194        #expect(temp("gpt-5-mini-2025-08-07") == nil)195        #expect(temp("o3") == nil && temp("o4-mini") == nil && temp("gpt-5.9-unknown") == nil)196        #expect(temp("gpt-6-luna") == .number(0.5), "not known to reject it: sent, with the one-shot retry as safety net")197    }198199    @Test func retriesOnceWithoutTemperatureOnTheSpecific400() async throws {200        let rejection = #"{"error":{"message":"Unsupported value: 'temperature' does not support 0.5 with this model. Only the default (1) value is supported.","type":"invalid_request_error","param":"temperature","code":"unsupported_value"}}"#201        let server = StubServer().json(400, rejection).sse(Self.textTranscript).sse(Self.textTranscript)202        let p = provider(server)203        let request = LLMRequest(model: "gpt-6-luna", messages: [.user("x")], temperature: 0.5)204        #expect(try await p.complete(request).text == "Hello wörld")205        #expect(server.requests.count == 2)206        #expect(server.requests[0].body?["temperature"] == .number(0.5) && server.requests[1].body?["temperature"] == nil)207        // The model is remembered: the next call does not send temperature at all.208        _ = try await p.complete(request)209        #expect(server.requests.count == 3 && server.requests[2].body?["temperature"] == nil)210211        // Any other 400 is not retried, and a second temperature rejection cannot loop.212        let other = StubServer().json(400, #"{"error":{"message":"Invalid schema for response_format","type":"invalid_request_error","param":"response_format","code":null}}"#)213        let (_, error) = await collect(provider(other).stream(LLMRequest(model: "gpt-4o-mini", messages: [.user("x")], temperature: 0.5)))214        #expect(error == .invalidRequest("Invalid schema for response_format"))215        #expect(other.requests.count == 1)216    }217218    @Test(arguments: [219        (401, #"{"error":{"message":"Incorrect API key provided.","type":"invalid_request_error","param":null,"code":"invalid_api_key"}}"#, LLMError.authentication("Incorrect API key provided.")),220        (403, #"{"error":{"message":"Country not supported","type":"request_forbidden","param":null,"code":"unsupported_country_region_territory"}}"#, LLMError.authentication("Country not supported")),221        (404, #"{"error":{"message":"The model `gpt-nope` does not exist or you do not have access to it.","type":"invalid_request_error","param":null,"code":"model_not_found"}}"#, LLMError.modelNotFound("gpt-nope — The model `gpt-nope` does not exist or you do not have access to it.")),222        (400, #"{"error":{"message":"This model's maximum context length is 128000 tokens. However, your messages resulted in 130000 tokens.","type":"invalid_request_error","param":"messages","code":"context_length_exceeded"}}"#, LLMError.contextTooLong("This model's maximum context length is 128000 tokens. However, your messages resulted in 130000 tokens.")),223        (500, #"{"error":{"message":"The server had an error","type":"server_error","param":null,"code":null}}"#, LLMError.server(status: 500, message: "The server had an error")),224        (503, #"{"error":{"message":"The engine is currently overloaded, please try again later","type":"server_error"}}"#, LLMError.overloaded),225        (529, "", LLMError.overloaded),226        (429, #"{"error":{"message":"You exceeded your current quota","type":"insufficient_quota","param":null,"code":"insufficient_quota"}}"#, LLMError.invalidRequest("The provider account has no remaining quota or credit. You exceeded your current quota")),227    ] as [(Int, String, LLMError)])228    func mapsHTTPErrors(status: Int, body: String, expected: LLMError) async {229        let (events, error) = await collect(provider(StubServer().json(status, body)).stream(LLMRequest(model: "gpt-nope", messages: [.user("x")])))230        #expect(events.isEmpty && error == expected)231    }232233    @Test func rateLimitHonoursRetryAfterHeaders() async {234        let body = #"{"error":{"message":"Rate limit reached","type":"requests","param":null,"code":"rate_limit_exceeded"}}"#235        let seconds = await collect(provider(StubServer().json(429, body, headers: ["Retry-After": "3"])).stream(LLMRequest(model: "gpt-4o-mini", messages: [.user("x")])))236        #expect(seconds.error == .rateLimited(retryAfterSeconds: 3))237        let millis = await collect(provider(StubServer().json(429, body, headers: ["retry-after-ms": "250", "retry-after": "1"])).stream(LLMRequest(model: "gpt-4o-mini", messages: [.user("x")])))238        #expect(millis.error == .rateLimited(retryAfterSeconds: 0.25))239        let none = await collect(provider(StubServer().json(429, body)).stream(LLMRequest(model: "gpt-4o-mini", messages: [.user("x")])))240        #expect(none.error == .rateLimited(retryAfterSeconds: nil))241    }242243    @Test func midStreamDisconnectAndStreamsWithoutDoneAreHandled() async {244        let request = LLMRequest(model: "gpt-4o-mini", messages: [.user("x")])245        let head = chatSSE([Self.chunk(#"{"role":"assistant","content":"par"}"#), Self.chunk(#"{"content":"tial"}"#)], done: false)246        let dropped = await collect(provider(StubServer().sse(head, failure: .networkConnectionLost)).stream(request))247        #expect(dropped.events.text == "partial" && dropped.events.stop == nil)248        if case .network = dropped.error {} else { Issue.record("expected a network error, got \(String(describing: dropped.error))") }249250        let early = await collect(provider(StubServer().sse(head)).stream(request))251        #expect(early.error == .network("The response stream ended before the completion was finished."))252253        // A stream that carried its finish reason but no [DONE] sentinel is complete.254        let noDone = await collect(provider(StubServer().sse(head + chatSSE([Self.chunk("{}", finish: "stop")], done: false))).stream(request))255        #expect(noDone.error == nil && noDone.events.stop == .endTurn)256257        let garbage = await collect(provider(StubServer().sse("data: {not json}\n\n")).stream(request))258        #expect(garbage.error == .decoding("A stream chunk was not valid JSON."))259    }260261    @Test func lengthCutToolCallIsDroppedAndRefusalAndFilterAreReported() async {262        let request = LLMRequest(model: "gpt-4o-mini", messages: [.user("x")], tools: [weatherTool])263        func run(_ chunks: [String]) async -> (events: [LLMStreamEvent], error: LLMError?) {264            await collect(provider(StubServer().sse(chatSSE(chunks))).stream(request))265        }266        let call = Self.chunk(#"{"tool_calls":[{"index":0,"id":"call_fixture_a","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Mon"}}]}"#)267        let cut = await run([call, Self.chunk("{}", finish: "length")])268        #expect(cut.error == nil && cut.events.toolUses.isEmpty && cut.events.stop == .maxTokens)269        let broken = await run([call, Self.chunk("{}", finish: "tool_calls")])270        if case .decoding(let m) = broken.error { #expect(m.contains("get_weather")) } else { Issue.record("expected a decoding error") }271        #expect(broken.events.toolUses.isEmpty && broken.events.stop == nil)272273        let refusal = await run([Self.chunk(#"{"role":"assistant","content":null,"refusal":"I can't help with that."}"#), Self.chunk("{}", finish: "stop")])274        #expect(refusal.events.stop == .refusal && refusal.events.text == "I can't help with that.")275        let filtered = await run([Self.chunk(#"{"content":"x"}"#), Self.chunk("{}", finish: "content_filter")])276        #expect(filtered.events.stop == .contentFilter)277    }278279    @Test func listsOnlyChatModelsAndEnrichesFromTheRegistry() async throws {280        let ids = ["gpt-4o-mini", "gpt-4o-mini-2024-07-18", "gpt-5-nano", "o3", "gpt-9-future", "text-embedding-3-small", "whisper-1", "tts-1",281                   "gpt-4o-audio-preview", "gpt-4o-realtime-preview", "gpt-image-1", "dall-e-3", "omni-moderation-latest", "gpt-3.5-turbo-instruct",282                   "gpt-5-pro", "gpt-5.2-codex", "gpt-4o-mini-transcribe", "gpt-4o-search-preview", "davinci-002", "o3-deep-research"]283        let body = "{\"object\":\"list\",\"data\":[" + ids.map { "{\"id\":\"\($0)\",\"object\":\"model\",\"created\":1,\"owned_by\":\"system\"}" }.joined(separator: ",") + "]}"284        let server = StubServer().json(200, body)285        let models = try await provider(server).listModels()286        #expect(server.requests[0].path == "/v1/models" && server.requests[0].method == "GET")287        #expect(models.map(\.id) == ["gpt-4o-mini", "gpt-4o-mini-2024-07-18", "gpt-5-nano", "gpt-9-future", "o3"])288        #expect(models.allSatisfy { $0.source == "api" && $0.provider == .openai })289        #expect(models[0].inputPricePerMTok == 0.15 && models[0].outputPricePerMTok == 0.60)290        #expect(models[1].inputPricePerMTok == 0.15 && models[1].id == "gpt-4o-mini-2024-07-18")291        #expect(models[2].inputPricePerMTok == 0.05 && models[2].outputPricePerMTok == 0.40)292        #expect(models[3].inputPricePerMTok == nil && models[3].contextWindow == nil, "unknown models carry no invented metadata")293    }294}295
Files main
Menu

Theme