SPB Git

Repository Public MIT

OpenAIProviderTests.swift 21.9 KB 404 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraProvidersTests/OpenAIProviderTests.swift10// Module           : AstraProvidersTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraProviders9293@Suite("OpenAI (Responses API)")94struct OpenAIProviderTests {95    func provider(_ transport: FixtureTransport, diagnostics: RecordingDiagnostics? = nil) -> OpenAIProvider {96        OpenAIProvider(transport: transport, apiKey: { testKey }, diagnostics: diagnostics)97    }9899    @Test func plainTextFromRecordedTranscript() async {100        let transport = FixtureTransport(.sse("openai_text.sse", headers: ["x-request-id": "req_fixture_openai"]))101        let diagnostics = RecordingDiagnostics()102        let c = await collect(provider(transport, diagnostics: diagnostics).stream(request(.openAI, "gpt-6-luna", system: "You are terse.")))103        expectWellFormed(c)104        #expect(c.text == "ASTRA-OK")105        #expect(c.starts.first?.reportedModel == "gpt-6-luna")106        #expect(c.starts.first?.responseID?.hasPrefix("resp_") == true)107        #expect(c.completions == [.endOfTurn])108        #expect(c.outcome.usage == TokenUsage(inputTokens: 22, outputTokens: 8, cachedInputTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0))109        #expect(diagnostics.records.count == 1)110        #expect(diagnostics.records.first?.status == 200)111        #expect(diagnostics.records.first?.requestID == "req_fixture_openai")112        #expect(diagnostics.records.first?.modelID == "gpt-6-luna")113    }114115    @Test func requestEncoding() async {116        let transport = FixtureTransport(.sse("openai_text.sse"))117        var messages = toolCycle()118        messages.insert(.user("Look", images: [tinyPNG]), at: 0)119        messages.insert(.assistant("I see a picture."), at: 1)120        let req = GenerationRequest(model: ModelRef(provider: .openAI, modelID: "gpt-6-luna"), system: "Be brief.", messages: messages,121                                    parameters: GenerationParameters(temperature: 0.3, topP: 0.9, maxOutputTokens: 64, reasoning: .low, seed: 1, stop: ["X"]), tools: [weatherTool])122        _ = await collect(provider(transport).stream(req))123        let r = transport.requests[0]124        #expect(r.url?.absoluteString == "https://api.openai.com/v1/responses")125        #expect(r.httpMethod == "POST")126        #expect(r.value(forHTTPHeaderField: "Authorization") == "Bearer \(testKey)")127        #expect(r.value(forHTTPHeaderField: "Content-Type") == "application/json")128        #expect(r.url?.absoluteString.contains(testKey) == false)129        let body = transport.body()130        #expect(body["model"] == "gpt-6-luna")131        #expect(body["stream"] == true)132        #expect(body["store"] == false)133        #expect(body["instructions"] == "Be brief.")134        #expect(body["max_output_tokens"] == 64)135        // gpt-6-luna is a reasoning model: sampling parameters are dropped up front, effort and summary are sent.136        #expect(body["temperature"] == nil)137        #expect(body["top_p"] == nil)138        #expect(body["reasoning"] == ["effort": "low", "summary": "auto"])139        #expect(body["stop"] == nil)140        #expect(body["tools"]?[0] == ["type": "function", "name": "get_weather", "description": "Get the current weather for a city.", "parameters": weatherTool.parameters, "strict": false])141        let input = body["input"]?.arrayValue ?? []142        #expect(input.count == 6)143        #expect(input[0]["content"]?[0] == ["type": "input_image", "image_url": .string("data:image/png;base64,\(tinyPNG.data.base64EncodedString())"), "detail": "auto"])144        #expect(input[0]["content"]?[1] == ["type": "input_text", "text": "Look"])145        #expect(input[1] == ["role": "assistant", "content": "I see a picture."])146        #expect(input[2]["role"] == "user")147        #expect(input[3] == ["role": "assistant", "content": "Let me check."])148        #expect(input[4] == ["type": "function_call", "call_id": "call_abc123", "name": "get_weather", "arguments": "{\"city\":\"Paris\"}"])149    }150151    @Test func toolResultFollowsItsCall() async {152        let transport = FixtureTransport(.sse("openai_text.sse"))153        _ = await collect(provider(transport).stream(GenerationRequest(model: ModelRef(provider: .openAI, modelID: "gpt-6-luna"), messages: toolCycle())))154        let input = transport.body()["input"]?.arrayValue ?? []155        #expect(input.last == ["type": "function_call_output", "call_id": "call_abc123", "output": "18 C, cloudy"])156    }157158    @Test func gpt62SendsSystemAsDeveloperMessageAndNeverEffortNone() async {159        let transport = FixtureTransport(.sse("openai_text.sse"))160        _ = await collect(provider(transport).stream(request(.openAI, "gpt-6.2-sol", system: "Be brief.", parameters: GenerationParameters(reasoning: .off))))161        let body = transport.body()162        #expect(body["instructions"] == nil)163        #expect(body["input"]?[0] == ["role": "developer", "content": "Be brief."])164        #expect(body["reasoning"]?["effort"] == "low")165    }166167    @Test func samplingIsSentToModelsThatAcceptIt() async {168        let transport = FixtureTransport(.sse("openai_text.sse"))169        _ = await collect(provider(transport).stream(request(.openAI, "gpt-4.1-mini", parameters: GenerationParameters(temperature: 0.3, topP: 0.9, reasoning: .high))))170        let body = transport.body()171        #expect(body["temperature"] == 0.3)172        #expect(body["top_p"] == 0.9)173        #expect(body["reasoning"] == nil)   // not a reasoning model174    }175176    @Test func toolCallWithFragmentedArguments() async {177        let c = await collect(provider(FixtureTransport(.sse("openai_tool.sse"))).stream(request(.openAI, "gpt-6-luna", tools: [weatherTool])))178        expectWellFormed(c)179        #expect(c.completions == [.toolCalls])180        #expect(c.outcome.toolCalls.count == 1)181        let call = c.outcome.toolCalls[0]182        #expect(call.id.hasPrefix("call_"))183        #expect(call.name == "get_weather")184        #expect(call.arguments == ["city": "Paris"])185        #expect(c.toolStarts == [call.id])186        #expect(c.deltas(for: call.id) == ["{\"", "city", "\":\"", "Paris", "\"}"])187    }188189    @Test func reasoningSummaryOnly() async {190        let c = await collect(provider(FixtureTransport(.sse("openai_reasoning.sse"))).stream(request(.openAI, "gpt-6-luna", parameters: GenerationParameters(reasoning: .high))))191        expectWellFormed(c)192        #expect(c.reasoning.hasPrefix("**Calculating arrangements of animals**"))193        #expect(!c.text.isEmpty)194        #expect(!c.reasoning.contains("ENCRYPTED"))195        #expect((c.outcome.usage?.reasoningTokens ?? 0) > 0)196    }197198    @Test func rawReasoningTextIsNeverSurfaced() throws {199        var decoder = ResponsesStreamDecoder(modelID: "m")200        let events = try decoder.consume(SSEEvent(event: "response.reasoning_text.delta", data: #"{"type":"response.reasoning_text.delta","item_id":"rs_1","output_index":0,"content_index":0,"delta":"secret chain of thought","sequence_number":1}"#))201        #expect(events.isEmpty)202    }203204    @Test func incompleteMapsToMaxTokens() async {205        let c = await collect(provider(FixtureTransport(.sse("openai_incomplete.sse"))).stream(request(.openAI, "gpt-6-luna")))206        expectWellFormed(c)207        #expect(c.completions == [.maxTokens])208        #expect(c.outcome.usage?.outputTokens == 16)209    }210211    @Test func refusalMapsToContentFilter() async {212        let c = await collect(provider(FixtureTransport(.sse("openai_refusal.sse"))).stream(request(.openAI, "gpt-6-luna")))213        expectWellFormed(c)214        #expect(c.completions == [.contentFilter])215    }216217    @Test func midStreamErrorEvent() async {218        let c = await collect(provider(FixtureTransport(.sse("openai_midstream_error.sse"))).stream(request(.openAI, "gpt-6-luna")))219        #expect(c.text == "ASTRA-OK")220        #expect(c.completions.isEmpty)221        guard case .rateLimited = c.error else { Issue.record("expected rateLimited, got \(String(describing: c.error))"); return }222    }223224    @Test func failedResponse() async {225        let c = await collect(provider(FixtureTransport(.sse("openai_failed.sse"))).stream(request(.openAI, "gpt-6-luna")))226        #expect(c.error == .server(status: 500, message: "The model failed to generate a response."))227    }228229    @Test func unsupportedParameterIsDroppedAndRetriedOnce() async {230        // A model this build has no entry for: temperature is sent, rejected, removed, and the request succeeds.231        let transport = FixtureTransport(.file(400, "openai_error_unsupported_parameter.json"), .sse("openai_text.sse"))232        let diagnostics = RecordingDiagnostics()233        let c = await collect(provider(transport, diagnostics: diagnostics).stream(request(.openAI, "gpt-9-unknown", parameters: GenerationParameters(temperature: 0.2, topP: 0.9))))234        expectWellFormed(c)235        #expect(c.text == "ASTRA-OK")236        #expect(transport.requests.count == 2)237        #expect(transport.body(0)["temperature"] == 0.2)238        #expect(transport.body(1)["temperature"] == nil)239        #expect(transport.body(1)["top_p"] == 0.9)240        #expect(diagnostics.records.map(\.status) == [400, 200])241        #expect(diagnostics.records.map(\.retryCount) == [0, 1])242    }243244    @Test func aRejectionThatIsNotAboutAParameterIsNotRetried() async {245        let transport = FixtureTransport(.json(400, #"{"error":{"message":"Invalid schema for function 'get_weather'.","type":"invalid_request_error","param":"tools[0].parameters","code":"invalid_function_parameters"}}"#))246        let c = await collect(provider(transport).stream(request(.openAI, "gpt-9-unknown", parameters: GenerationParameters(temperature: 0.2))))247        #expect(transport.requests.count == 1)248        #expect(c.error == .invalidRequest("Invalid schema for function 'get_weather'."))249    }250251    @Test func contextLengthError() async {252        let transport = FixtureTransport(.json(400, #"{"error":{"message":"Your input exceeds the context window of this model. Please adjust your input and try again.","type":"invalid_request_error","param":"input","code":"context_length_exceeded"}}"#))253        let c = await collect(provider(transport).stream(request(.openAI, "gpt-6-luna")))254        guard case .contextTooLong = c.error else { Issue.record("expected contextTooLong, got \(String(describing: c.error))"); return }255    }256257    @Test func recordedErrorBodies() async {258        var c = await collect(provider(FixtureTransport(.file(401, "openai_error_401.json"))).stream(request(.openAI, "gpt-6-luna")))259        guard case .unauthorized(let message) = c.error else { Issue.record("expected unauthorized"); return }260        #expect(message?.contains("Incorrect API key") == true)261        c = await collect(provider(FixtureTransport(.file(404, "openai_error_404.json"))).stream(request(.openAI, "gpt-0-nonexistent")))262        #expect(c.error == .modelNotFound("gpt-0-nonexistent"))263    }264265    @Test func modelListIsFilteredAndEnriched() async throws {266        let transport = FixtureTransport(.file(200, "openai_models.json"))267        let models = try await provider(transport).listModels()268        #expect(transport.requests[0].url?.absoluteString == "https://api.openai.com/v1/models")269        let ids = Set(models.map(\.ref.modelID))270        #expect(ids == ["gpt-6-luna", "gpt-6.2-sol", "gpt-6.2-astra", "gpt-4.1-mini", "gpt-5.3-codex", "chat-latest"])271        let luna = try #require(models.first { $0.ref.modelID == "gpt-6-luna" })272        #expect(luna.contextLength == 1_050_000)273        #expect(luna.maxOutputTokens == 128_000)274        #expect(luna.capabilities.isSuperset(of: [.text, .vision, .tools, .reasoning]))275        #expect(luna.pricing?.inputPerMTok == 0.10)276        #expect(luna.pricing?.outputPerMTok == 0.50)277        #expect(luna.pricing?.asOf == CuratedCatalog.verifiedOn)278        guard case .documentation(let asOf, let url) = luna.source else { Issue.record("curated entries must cite documentation"); return }279        #expect(asOf == "2026-11-21")280        #expect(url.hasPrefix("https://developers.openai.com/"))281        // A model outside the curated table: listed, with no claim beyond text.282        let codex = try #require(models.first { $0.ref.modelID == "gpt-5.3-codex" })283        #expect(codex.capabilities == [.text])284        #expect(codex.pricing == nil)285        #expect(codex.contextLength == nil)286        guard case .providerAPI = codex.source else { Issue.record("uncurated entries must cite the provider API"); return }287    }288289    @Test func verifyCountsModels() async throws {290        let status = try await provider(FixtureTransport(.file(200, "openai_models.json"))).verify()291        #expect(status.ok)292        #expect(status.modelCount == 6)293    }294295    @Test func everyCuratedEntryCitesItsSourceAndDate() {296        for entry in CuratedCatalog.openAI {297            let info = CuratedCatalog.info(for: ModelRef(provider: .openAI, modelID: entry.id))298            guard case .documentation(let asOf, let url)? = info?.source else { Issue.record("\(entry.id) has no documentation source"); continue }299            #expect(asOf == CuratedCatalog.verifiedOn)300            #expect(url.hasPrefix("https://"))301            #expect(info?.pricing?.asOf == CuratedCatalog.verifiedOn)302        }303        #expect(CuratedCatalog.openAI("gpt-6-luna-2026-09-08")?.id == "gpt-6-luna")304        #expect(CuratedCatalog.openAI("gpt-6-lunatic") == nil)305    }306}307308@Suite("xAI (Responses API)")309struct XAIProviderTests {310    func provider(_ transport: FixtureTransport) -> XAIProvider { XAIProvider(transport: transport, apiKey: { "xai-FIXTUREKEY0123456789abcdefFIXTURE" }) }311312    @Test func plainTextFromRecordedTranscript() async {313        let transport = FixtureTransport(.sse("xai_text.sse"))314        let c = await collect(provider(transport).stream(request(.xAI, "grok-4.3", system: "You are terse.", parameters: GenerationParameters(temperature: 0.2, maxOutputTokens: 32, reasoning: .off))))315        expectWellFormed(c)316        #expect(c.text == "ASTRA-OK")317        #expect(c.starts.first?.reportedModel == "grok-4.3")318        #expect(c.completions == [.endOfTurn])319        #expect((c.outcome.usage?.inputTokens ?? 0) > 0)320        #expect((c.outcome.usage?.reportedCostUSD ?? 0) > 0)321        #expect((c.outcome.usage?.reportedCostUSD ?? 1) < 0.01)322        let r = transport.requests[0]323        #expect(r.url?.absoluteString == "https://api.x.ai/v1/responses")324        #expect(r.value(forHTTPHeaderField: "Authorization")?.hasPrefix("Bearer xai-") == true)325        let body = transport.body()326        #expect(body["instructions"] == "You are terse.")327        #expect(body["temperature"] == 0.2)          // xAI accepts sampling parameters328        #expect(body["reasoning"] == ["effort": "none"])329        #expect(body["store"] == false)330    }331332    @Test func costTicksAreConvertedToDollars() {333        let decoder = ResponsesStreamDecoder(modelID: "grok-4.3", costTicksPerUSD: XAIProvider.costTicksPerUSD)334        let usage = decoder.usage(["input_tokens": 272, "input_tokens_details": ["cached_tokens": 128], "output_tokens": 10, "output_tokens_details": ["reasoning_tokens": 0], "cost_in_usd_ticks": 2306000])335        #expect(usage?.inputTokens == 272)336        #expect(usage?.cachedInputTokens == 128)337        #expect(abs((usage?.reportedCostUSD ?? 0) - 0.0002306) < 1e-12)338    }339340    @Test func toolCall() async {341        let c = await collect(provider(FixtureTransport(.sse("xai_tool.sse"))).stream(request(.xAI, "grok-4.3", tools: [weatherTool])))342        expectWellFormed(c)343        #expect(c.completions == [.toolCalls])344        #expect(c.outcome.toolCalls.first?.name == "get_weather")345        #expect(c.outcome.toolCalls.first?.arguments == ["city": "Paris"])346    }347348    @Test func reasoningSummary() async {349        let c = await collect(provider(FixtureTransport(.sse("xai_reasoning.sse"))).stream(request(.xAI, "grok-4.3", parameters: GenerationParameters(reasoning: .low))))350        expectWellFormed(c)351        #expect(c.reasoning.hasPrefix("This is a classic algebra"))352        #expect(c.text.contains("23 chickens"))353        #expect(c.outcome.usage?.reasoningTokens == 199)354    }355356    @Test func incomplete() async {357        let c = await collect(provider(FixtureTransport(.sse("xai_incomplete.sse"))).stream(request(.xAI, "grok-4.3")))358        expectWellFormed(c)359        #expect(c.completions == [.maxTokens])360    }361362    @Test func effortNoneRejectedFallsBackToLow() async {363        let transport = FixtureTransport(.file(400, "xai_error_effort_none.json"), .sse("xai_text.sse"))364        let c = await collect(provider(transport).stream(request(.xAI, "grok-4.7", parameters: GenerationParameters(reasoning: .off))))365        expectWellFormed(c)366        #expect(transport.requests.count == 2)367        #expect(transport.body(0)["reasoning"]?["effort"] == "none")368        #expect(transport.body(1)["reasoning"]?["effort"] == "low")369    }370371    @Test func nonReasoningModelGetsNoReasoningField() async {372        let transport = FixtureTransport(.sse("xai_text.sse"))373        _ = await collect(provider(transport).stream(request(.xAI, "grok-4.20-0309-non-reasoning", parameters: GenerationParameters(reasoning: .high))))374        #expect(transport.body()["reasoning"] == nil)375    }376377    @Test func badKeyArrivesAs400AndIsReportedAsUnauthorized() async {378        let c = await collect(provider(FixtureTransport(.file(400, "xai_error_bad_key.json"))).stream(request(.xAI, "grok-4.3")))379        guard case .unauthorized = c.error else { Issue.record("expected unauthorized, got \(String(describing: c.error))"); return }380    }381382    @Test func modelNotFoundDoesNotExposeTheTeamID() async {383        let c = await collect(provider(FixtureTransport(.file(404, "xai_error_404.json"))).stream(request(.xAI, "grok-0-nonexistent")))384        #expect(c.error == .modelNotFound("grok-0-nonexistent"))385        #expect(c.error?.message.contains("00000000-0000") == false)386    }387388    @Test func modelListFromTheProviderAPI() async throws {389        let transport = FixtureTransport(.file(200, "xai_language_models.json"), .file(200, "xai_models.json"))390        let models = try await provider(transport).listModels()391        #expect(transport.requests.map { $0.url!.path } == ["/v1/language-models", "/v1/models"])392        let grok = try #require(models.first { $0.ref.modelID == "grok-4.3" })393        #expect(grok.capabilities.isSuperset(of: [.text, .vision, .reasoning, .promptCaching]))394        #expect(grok.pricing?.inputPerMTok == 1.25)395        #expect(grok.pricing?.outputPerMTok == 2.5)396        #expect(grok.pricing?.cachedInputPerMTok == 0.2)397        #expect(grok.contextLength == 1_000_000)398        guard case .providerAPI = grok.source else { Issue.record("xAI metadata comes from the provider API"); return }399        let plain = try #require(models.first { $0.ref.modelID == "grok-4.20-0309-non-reasoning" })400        #expect(!plain.capabilities.contains(.reasoning))401        #expect(!plain.capabilities.contains(.tools))   // not reported by the API, so not claimed402    }403}404
Files main
Menu

Theme