SPB Git

Repository Public MIT

GeminiProviderTests.swift 16.9 KB 284 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraProvidersTests/GeminiProviderTests.swift10// Module           : AstraProvidersTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraProviders9293@Suite("Google Gemini (generateContent)")94struct GeminiProviderTests {95    static let key = "AIzaFIXTUREKEY0123456789abcdefFIXTURE00"96    func provider(_ transport: FixtureTransport, diagnostics: RecordingDiagnostics? = nil) -> GeminiProvider {97        GeminiProvider(transport: transport, apiKey: { Self.key }, diagnostics: diagnostics)98    }99100    @Test func plainTextFromRecordedTranscript() async {101        let transport = FixtureTransport(.sse("gemini_text.sse"))102        let c = await collect(provider(transport).stream(request(.google, "gemini-3.1-flash-lite", system: "You are terse.",103                                                                  parameters: GenerationParameters(temperature: 0.2, topP: 0.9, maxOutputTokens: 32, seed: 5, stop: ["END"]))))104        expectWellFormed(c)105        #expect(c.text == "ASTRA-OK")106        #expect(c.starts.first == MessageStart(responseID: "FIXTURE-response", reportedModel: "gemini-3.1-flash-lite"))107        #expect(c.completions == [.endOfTurn])108        #expect(c.outcome.usage == TokenUsage(inputTokens: 13, outputTokens: 4))109        let r = transport.requests[0]110        // The key is a header and never part of the URL.111        #expect(r.url?.absoluteString == "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-lite:streamGenerateContent?alt=sse")112        #expect(r.value(forHTTPHeaderField: "x-goog-api-key") == Self.key)113        #expect(r.url?.absoluteString.contains("key=") == false)114        #expect(r.value(forHTTPHeaderField: "Authorization") == nil)115        let body = transport.body()116        #expect(body["systemInstruction"] == ["parts": [["text": "You are terse."]]])117        #expect(body["contents"] == [["role": "user", "parts": [["text": "Reply with exactly: ASTRA-OK"]]]])118        #expect(body["generationConfig"] == ["maxOutputTokens": 32, "temperature": 0.2, "topP": 0.9, "seed": 5, "stopSequences": ["END"], "thinkingConfig": ["includeThoughts": true]])119    }120121    @Test func thinkingConfigPerFamily() {122        #expect(GeminiProvider.thinkingConfig(nil, modelID: "gemini-4-hydrogen") == ["includeThoughts": true])123        #expect(GeminiProvider.thinkingConfig(.off, modelID: "gemini-3.1-flash-lite") == ["thinkingLevel": "minimal"])124        #expect(GeminiProvider.thinkingConfig(.high, modelID: "gemini-4-neon") == ["thinkingLevel": "high", "includeThoughts": true])125        #expect(GeminiProvider.thinkingConfig(.off, modelID: "gemini-2.5-flash-lite") == ["thinkingBudget": 0])126        #expect(GeminiProvider.thinkingConfig(.off, modelID: "gemini-2.5-pro") == ["thinkingBudget": 128])127        #expect(GeminiProvider.thinkingConfig(.low, modelID: "gemini-2.5-flash") == ["thinkingBudget": 1024, "includeThoughts": true])128        #expect(GeminiProvider.thinkingConfig(.high, modelID: "some-other-model") == nil)129    }130131    @Test func thoughtSummariesThenAnswer() async {132        let c = await collect(provider(FixtureTransport(.sse("gemini_thinking.sse"))).stream(request(.google, "gemini-3.1-flash-lite", parameters: GenerationParameters(reasoning: .high))))133        expectWellFormed(c)134        #expect(c.reasoning.hasPrefix("**Defining the Problem**"))135        #expect(c.text.contains("23"))136        #expect(!c.text.contains("Defining the Problem"))137        #expect(!c.reasoning.contains("SIGNATURE"))138        let usage = c.outcome.usage139        #expect((usage?.reasoningTokens ?? 0) > 0)140        // Thinking is billed as output: it is part of outputTokens.141        #expect((usage?.outputTokens ?? 0) > (usage?.reasoningTokens ?? 0))142    }143144    @Test func usageNormalisation() async {145        let c = await collect(provider(FixtureTransport(.sse("gemini_cached_usage.sse"))).stream(request(.google, "gemini-3.1-flash-lite")))146        expectWellFormed(c)147        #expect(c.outcome.usage == TokenUsage(inputTokens: 4013, outputTokens: 24, cachedInputTokens: 4000, reasoningTokens: 20))148    }149150    @Test func parallelToolCallsKeepProviderIDsAndSignatures() async {151        let store = ThoughtSignatureStore()152        var decoder = GeminiStreamDecoder(modelID: "gemini-3.1-flash-lite", signatures: store)153        var events: [StreamEvent] = []154        var parser = SSEParser()155        for line in Fixtures.text("gemini_tool_parallel.sse").components(separatedBy: "\n") {156            if let event = parser.feed(line) { events += (try? decoder.consume(event.data)) ?? [] }157        }158        let calls = events.compactMap { if case .toolCallCompleted(let c) = $0 { return c }; return nil }159        #expect(calls == [ToolCall(id: "call_15701850020745005639", name: "get_weather", arguments: ["city": "Paris"]),160                          ToolCall(id: "call_15701850020745003648", name: "get_weather", arguments: ["city": "Rome"])])161        #expect(events.last == .messageCompleted(.toolCalls))    // Gemini says STOP for a turn that ends in calls162        // Only the first call of the turn carries a signature; it is kept for the replay.163        #expect(store.signature(for: "call_15701850020745005639")?.hasPrefix("SIGNATURE_PLACEHOLDER") == true)164        #expect(store.signature(for: "call_15701850020745003648") == nil)165166        let history: [PromptMessage] = [167            .user("Weather in Paris and Rome?"),168            .assistant("", toolCalls: calls),169            .tool("18 C", callID: calls[0].id, name: "get_weather"),170            .tool("no such city", callID: calls[1].id, name: "get_weather", isError: true),171        ]172        let wire = GeminiProvider.contents(history, signatures: store)173        #expect(wire.map { $0["role"]?.stringValue } == ["user", "model", "user"])174        let modelParts = wire[1]["parts"]?.arrayValue ?? []175        #expect(modelParts[0]["thoughtSignature"]?.stringValue == store.signature(for: calls[0].id))     // echoed verbatim176        #expect(modelParts[0]["functionCall"] == ["name": "get_weather", "args": ["city": "Paris"], "id": "call_15701850020745005639"])177        #expect(modelParts[1]["thoughtSignature"] == nil)178        #expect(wire[2]["parts"] == [179            ["functionResponse": ["name": "get_weather", "response": ["result": "18 C"], "id": "call_15701850020745005639"]],180            ["functionResponse": ["name": "get_weather", "response": ["error": "no such city"], "id": "call_15701850020745003648"]],181        ])182    }183184    @Test func unknownSignatureFallsBackToThePlaceholder() {185        let wire = GeminiProvider.contents(toolCycle(callID: "call_from_another_session"), signatures: ThoughtSignatureStore())186        #expect(wire[1]["parts"]?[1]?["thoughtSignature"] == "skip_thought_signature_validator")187    }188189    @Test func modelsWithoutCallIDsGetStableSynthesisedOnes() async {190        let c = await collect(provider(FixtureTransport(.sse("gemini_tool_no_ids.sse"))).stream(request(.google, "gemini-2.5-flash-lite", tools: [weatherTool])))191        expectWellFormed(c)192        #expect(c.completions == [.toolCalls])193        #expect(c.outcome.toolCalls.map(\.id) == ["gemini-call-FIXTURE-response-0", "gemini-call-FIXTURE-response-1"])194        #expect(c.outcome.toolCalls.map(\.arguments) == [["city": "Paris"], ["city": "Rome"]])195        #expect(c.deltas(for: "gemini-call-FIXTURE-response-0") == ["{\"city\":\"Paris\"}"])196        // Synthesised ids never go back on the wire; results are matched by name and order.197        let history: [PromptMessage] = [.user("q"), .assistant("", toolCalls: c.outcome.toolCalls),198                                        .tool("18 C", callID: c.outcome.toolCalls[0].id, name: "get_weather"), .tool("22 C", callID: c.outcome.toolCalls[1].id, name: "get_weather")]199        let wire = GeminiProvider.contents(history, signatures: ThoughtSignatureStore())200        #expect(wire[1]["parts"]?[0]?["functionCall"] == ["name": "get_weather", "args": ["city": "Paris"]])201        #expect(wire[2]["parts"]?[1] == ["functionResponse": ["name": "get_weather", "response": ["result": "22 C"]]])202    }203204    @Test func toolsUseTheFullJSONSchemaField() async {205        let transport = FixtureTransport(.sse("gemini_text.sse"))206        _ = await collect(provider(transport).stream(request(.google, "gemini-3.1-flash-lite", tools: [weatherTool])))207        #expect(transport.body()["tools"] == [["functionDeclarations": [["name": "get_weather", "description": "Get the current weather for a city.", "parametersJsonSchema": weatherTool.parameters]]]])208    }209210    @Test func imagesAreInlineData() {211        let wire = GeminiProvider.contents([.user("Look", images: [tinyPNG])], signatures: ThoughtSignatureStore())212        #expect(wire[0]["parts"] == [["inlineData": ["mimeType": "image/png", "data": .string(tinyPNG.data.base64EncodedString())]], ["text": "Look"]])213    }214215    @Test func finishReasons() async {216        var c = await collect(provider(FixtureTransport(.sse("gemini_max_tokens.sse"))).stream(request(.google, "gemini-3.1-flash-lite")))217        expectWellFormed(c)218        #expect(c.completions == [.maxTokens])219        c = await collect(provider(FixtureTransport(.sse("gemini_safety.sse"))).stream(request(.google, "gemini-3.1-flash-lite")))220        expectWellFormed(c)221        #expect(c.completions == [.contentFilter])222        c = await collect(provider(FixtureTransport(.sse("gemini_prompt_blocked.sse"))).stream(request(.google, "gemini-3.1-flash-lite")))223        expectWellFormed(c)224        #expect(c.completions == [.contentFilter])225        #expect(c.text.isEmpty)226        #expect(GeminiStreamDecoder.stopReason("RECITATION", calledTools: false) == .contentFilter)227        #expect(GeminiStreamDecoder.stopReason("PROHIBITED_CONTENT", calledTools: false) == .contentFilter)228        #expect(GeminiStreamDecoder.stopReason("OTHER", calledTools: false) == .unknown)229    }230231    @Test func midStreamError() async {232        let c = await collect(provider(FixtureTransport(.sse("gemini_midstream_error.sse"))).stream(request(.google, "gemini-3.1-flash-lite")))233        #expect(c.text == "ASTRA-OK")234        #expect(c.completions.isEmpty)235        #expect(c.error == .overloaded("The model is overloaded. Please try again later."))236    }237238    @Test func recordedErrorBodies() async {239        // Google answers a bad key with HTTP 400 / API_KEY_INVALID.240        var c = await collect(provider(FixtureTransport(.file(400, "gemini_error_bad_key.json"))).stream(request(.google, "gemini-3.1-flash-lite")))241        #expect(c.error == .unauthorized("API key not valid. Please pass a valid API key."))242        c = await collect(provider(FixtureTransport(.file(404, "gemini_error_404.json"))).stream(request(.google, "gemini-0-nonexistent")))243        #expect(c.error == .modelNotFound("gemini-0-nonexistent"))244        // RetryInfo in the body is honoured when there is no Retry-After header.245        c = await collect(provider(FixtureTransport(.file(429, "gemini_error_429.json"))).stream(request(.google, "gemini-3.1-flash-lite")))246        #expect(c.error == .rateLimited(retryAfter: 17, message: nil))247        c = await collect(provider(FixtureTransport(.file(400, "gemini_error_missing_signature.json"))).stream(request(.google, "gemini-3.1-flash-lite")))248        guard case .invalidRequest = c.error else { Issue.record("expected invalidRequest, got \(String(describing: c.error))"); return }249    }250251    @Test func unsupportedThinkingLevelFallsBack() async {252        let transport = FixtureTransport(.file(400, "gemini_error_minimal.json"), .sse("gemini_text.sse"))253        var c = await collect(provider(transport).stream(request(.google, "gemini-4-hydrogen", parameters: GenerationParameters(reasoning: .off))))254        expectWellFormed(c)255        #expect(transport.body(0)["generationConfig"]?["thinkingConfig"] == ["thinkingLevel": "minimal"])256        #expect(transport.body(1)["generationConfig"]?["thinkingConfig"] == ["thinkingLevel": "low"])257        let second = FixtureTransport(.json(400, #"{"error":{"code":400,"message":"Thinking level is not supported for this model.","status":"INVALID_ARGUMENT"}}"#), .sse("gemini_text.sse"))258        c = await collect(provider(second).stream(request(.google, "gemini-9-new", parameters: GenerationParameters(reasoning: .medium))))259        expectWellFormed(c)260        #expect(second.body(1)["generationConfig"] == nil)261    }262263    @Test func modelList() async throws {264        let transport = FixtureTransport(.file(200, "gemini_models.json"))265        let models = try await provider(transport).listModels()266        let r = transport.requests[0]267        #expect(r.url?.absoluteString == "https://generativelanguage.googleapis.com/v1beta/models?pageSize=1000")268        #expect(r.value(forHTTPHeaderField: "x-goog-api-key") == Self.key)269        // Speech, image, music and embedding models are not chat models.270        #expect(Set(models.map(\.ref.modelID)) == ["gemini-3.1-flash-lite", "gemini-4-hydrogen", "gemini-2.5-flash-lite", "gemma-4-31b-it"])271        let lite = try #require(models.first { $0.ref.modelID == "gemini-3.1-flash-lite" })272        #expect(lite.contextLength == 1_048_576)273        #expect(lite.maxOutputTokens == 65_536)274        #expect(lite.capabilities.isSuperset(of: [.text, .reasoning, .vision, .tools]))275        #expect(lite.supportedParameters?.contains("generateContent") == true)276        #expect(lite.pricing?.inputPerMTok == 0.25)277        #expect(lite.pricing?.asOf == CuratedCatalog.verifiedOn)278        guard case .providerAPI = lite.source else { Issue.record("expected providerAPI"); return }279        let gemma = try #require(models.first { $0.ref.modelID == "gemma-4-31b-it" })280        #expect(gemma.capabilities == [.text, .reasoning])     // only what the API reports281        #expect(gemma.pricing == nil)282    }283}284
Files main
Menu

Theme