Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Tests/AstraProvidersTests/LiveSmokeTests.swift10// Module : AstraProvidersTests11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import CoreGraphics90import Foundation91import ImageIO92import Testing93@testable import AstraProviders9495/// Live checks against the real providers. Disabled unless `ASTRA_LIVE_TESTS=1` and the provider's key variable is set96/// (`source Scripts/smoke-env.sh`). Requests are tiny and bounded; no key is ever printed, logged or written.97enum Live {98 static let env = ProcessInfo.processInfo.environment99 static let on = env["ASTRA_LIVE_TESTS"] == "1"100 static func has(_ name: String) -> Bool { on && !(env[name] ?? "").isEmpty }101 /// `ASTRA_LIVE_MATRIX=1` additionally sends one tiny request to every model each provider lists.102 static let matrix = on && env["ASTRA_LIVE_MATRIX"] == "1"103104 static let keyVariable: [Adapter: String] = [.openAI: "OPENAI_API_KEY", .anthropic: "ANTHROPIC_API_KEY", .openRouter: "OPENROUTER_API_KEY",105 .xAI: "XAI_API_KEY", .google: "GEMINI_API_KEY", .custom: "ASTRA_CUSTOM_KEY"]106 /// The cheapest current chat model of each provider on 2026-11-21.107 static let model: [Adapter: String] = [.openAI: "gpt-6-luna", .anthropic: "claude-haiku-5-5", .openRouter: "openai/gpt-6-luna",108 .xAI: "grok-4.3", .google: "gemini-3.1-flash-lite"]109110 static func available(_ adapter: Adapter) -> Bool {111 guard let variable = keyVariable[adapter], has(variable) else { return false }112 return adapter != .custom || !(env["ASTRA_CUSTOM_BASE_URL"] ?? "").isEmpty113 }114115 static func provider(_ adapter: Adapter, label: String = "") -> any ChatProvider {116 let key = env[keyVariable[adapter] ?? ""] ?? ""117 let transport = TeeTransport(label: label.isEmpty ? nil : "\(adapter.rawValue)_\(label)")118 switch adapter {119 case .openAI: return OpenAIProvider(transport: transport, apiKey: { key })120 case .anthropic: return AnthropicProvider(transport: transport, apiKey: { key })121 case .openRouter: return OpenRouterProvider(transport: transport, apiKey: { key })122 case .xAI: return XAIProvider(transport: transport, apiKey: { key })123 case .google: return GeminiProvider(transport: transport, apiKey: { key })124 case .custom:125 let profile = EndpointProfile(id: "live", name: "Live", baseURL: env["ASTRA_CUSTOM_BASE_URL"] ?? "")126 return OpenAICompatibleProvider(profile: profile, transport: transport, apiKey: { key })127 }128 }129130 static func modelRef(_ adapter: Adapter, provider: any ChatProvider) async throws -> ModelRef {131 if let id = model[adapter] { return ModelRef(provider: adapter.kind, modelID: id) }132 // Custom endpoint: use whatever it reports.133 let models = try await provider.listModels()134 return try #require(models.first?.ref)135 }136137 /// Collects a live stream, retrying a transient failure (capacity, rate limit, overload) that produced nothing, as the app's orchestrator does.138 static func collect(_ provider: any ChatProvider, _ request: GenerationRequest) async -> Collected {139 var result = await AstraProvidersTests.collect(provider.stream(request))140 for delay in [10, 30, 60] {141 guard let error = result.error, error.isTransient, result.events.isEmpty else { break }142 print("LIVE retry \(request.model.modelID) after \(error.title)")143 try? await Task.sleep(nanoseconds: UInt64(delay) * 1_000_000_000)144 result = await AstraProvidersTests.collect(provider.stream(request))145 }146 return result147 }148149 static func describe(_ usage: TokenUsage?) -> String {150 guard let u = usage else { return "usage: none" }151 return "in \(u.inputTokens.map(String.init) ?? "-") out \(u.outputTokens.map(String.init) ?? "-") cached \(u.cachedInputTokens.map(String.init) ?? "-") reasoning \(u.reasoningTokens.map(String.init) ?? "-")"152 + (u.reportedCostUSD.map { " cost $\($0)" } ?? "")153 }154}155156/// The real network transport; when `ASTRA_RECORD_DIR` is set it also writes the raw stream lines it received, for fixtures.157struct TeeTransport: HTTPTransport {158 let inner = URLSessionTransport()159 var label: String?160161 func send(_ request: URLRequest) async throws -> (Data, HTTPURLResponse) { try await inner.send(request) }162163 func stream(_ request: URLRequest) async throws -> (HTTPURLResponse, AsyncThrowingStream<String, Error>) {164 let (response, lines) = try await inner.stream(request)165 guard let label, let directory = Live.env["ASTRA_RECORD_DIR"], !directory.isEmpty else { return (response, lines) }166 let url = URL(fileURLWithPath: directory).appendingPathComponent("\(label).sse")167 let (out, continuation) = AsyncThrowingStream<String, Error>.makeStream()168 let task = Task {169 var text = ""170 do {171 for try await line in lines { text += line + "\n"; continuation.yield(line) }172 try? text.write(to: url, atomically: true, encoding: .utf8) // response body only: no request, no headers, no key173 continuation.finish()174 } catch { continuation.finish(throwing: error) }175 }176 continuation.onTermination = { _ in task.cancel() }177 return (response, out)178 }179}180181@Suite("Live smoke tests", .enabled(if: Live.on), .serialized)182struct LiveSmokeTests {183 @Test(arguments: Adapter.allCases)184 func verifyAndListModels(adapter: Adapter) async throws {185 guard Live.available(adapter) else { return }186 let provider = Live.provider(adapter)187 let status = try await provider.verify()188 #expect(status.ok)189 let models = try await provider.listModels()190 #expect(!models.isEmpty)191 if let expected = Live.model[adapter] { #expect(models.contains { $0.ref.modelID == expected }, "\(expected) is no longer listed") }192 for model in models {193 #expect(model.ref.provider == adapter.kind)194 #expect(model.capabilities.contains(.text))195 if let pricing = model.pricing { #expect(!pricing.asOf.isEmpty); #expect(pricing.inputPerMTok >= 0) }196 }197 print("LIVE \(adapter.rawValue) verify ok, \(models.count) models" + (status.remainingCreditUSD.map { _ in ", credit reported" } ?? ""))198 }199200 @Test(arguments: Adapter.allCases)201 func streamedGeneration(adapter: Adapter) async throws {202 guard Live.available(adapter) else { return }203 let provider = Live.provider(adapter, label: "text")204 let ref = try await Live.modelRef(adapter, provider: provider)205 let request = GenerationRequest(model: ref, system: "You follow instructions exactly.", messages: [.user("Reply with exactly: ASTRA-OK")],206 parameters: GenerationParameters(maxOutputTokens: 32, reasoning: .off))207 let c = await Live.collect(provider, request)208 expectWellFormed(c)209 #expect(c.text.contains("ASTRA-OK"))210 let reported = try #require(c.starts.first?.reportedModel)211 #expect(!reported.isEmpty)212 // The provider may answer with a dated snapshot or a vendor-prefixed id of the model that was asked for.213 let asked = ref.modelID.split(separator: "/").last.map(String.init) ?? ref.modelID214 #expect(reported.contains(asked) || asked.contains(reported), "asked for \(ref.modelID), provider reported \(reported)")215 #expect((c.outcome.usage?.inputTokens ?? 0) > 0)216 #expect((c.outcome.usage?.outputTokens ?? 0) > 0)217 #expect(c.completions == [.endOfTurn])218 print("LIVE \(adapter.rawValue) text: asked \(ref.modelID), reported \(reported), served by \(c.starts.first?.servedBy ?? "-"), \(Live.describe(c.outcome.usage))")219 }220221 @Test(arguments: Adapter.allCases)222 func toolCallRoundTrip(adapter: Adapter) async throws {223 guard Live.available(adapter) else { return }224 let provider = Live.provider(adapter, label: "tool")225 try await roundTrip(adapter, provider: provider, ref: try await Live.modelRef(adapter, provider: provider), reasoning: .off)226 }227228 /// The same round trip with reasoning on, on the models whose tool cycles have special rules: OpenAI reasoning items,229 /// Anthropic thinking blocks (never replayed), Sonnet 5.5 `between_tools`, Gemini thought signatures, models without call ids.230 static let toolCycleModels: [(Adapter, String, ReasoningEffort?)] = [231 (.openAI, "gpt-6.2-sol", .low), (.openAI, "gpt-4.1-mini", nil),232 (.anthropic, "claude-sonnet-5-5", .low), (.anthropic, "claude-sonnet-5-5", .off), (.anthropic, "claude-opus-5-5", .off), (.anthropic, "claude-haiku-4-5", .low), (.anthropic, "claude-haiku-5-5", .medium),233 (.google, "gemini-4-hydrogen", .low), (.google, "gemini-2.5-flash-lite", .low), (.google, "gemini-3.1-flash-lite", .medium),234 (.xAI, "grok-4.7", .low), (.xAI, "grok-4.20-0309-non-reasoning", nil),235 (.openRouter, "anthropic/claude-haiku-5.5", .low), (.openRouter, "google/gemini-3.1-flash-lite", .low),236 ]237238 @Test(.enabled(if: Live.matrix), arguments: 0..<LiveSmokeTests.toolCycleModels.count)239 func toolCallRoundTripWithReasoning(index: Int) async throws {240 let (adapter, model, effort) = Self.toolCycleModels[index]241 guard Live.available(adapter) else { return }242 try await roundTrip(adapter, provider: Live.provider(adapter), ref: ModelRef(provider: adapter.kind, modelID: model), reasoning: effort)243 }244245 func roundTrip(_ adapter: Adapter, provider: any ChatProvider, ref: ModelRef, reasoning: ReasoningEffort?) async throws {246 let question = PromptMessage.user("What is the weather in Paris right now? Call the get_weather tool, then answer in one short sentence.")247 let budget = reasoning == .off || reasoning == nil ? 256 : 1500248 let first = await Live.collect(provider, GenerationRequest(model: ref, messages: [question], parameters: GenerationParameters(maxOutputTokens: budget, reasoning: reasoning), tools: [weatherTool]))249 expectWellFormed(first)250 #expect(first.completions == [.toolCalls], "\(ref.modelID): \(String(describing: first.error))")251 let call = try #require(first.outcome.toolCalls.first)252 #expect(call.name == "get_weather")253 #expect(call.arguments["city"]?.stringValue?.contains("Paris") == true)254 #expect(first.toolStarts.contains(call.id))255256 // Second leg: replay the call and its result exactly as the app would.257 let history = [question, .assistant(first.text, toolCalls: first.outcome.toolCalls)]258 + first.outcome.toolCalls.map { PromptMessage.tool("18 C, cloudy", callID: $0.id, name: $0.name) }259 let second = await Live.collect(provider, GenerationRequest(model: ref, messages: history, parameters: GenerationParameters(maxOutputTokens: budget, reasoning: reasoning), tools: [weatherTool]))260 expectWellFormed(second)261 #expect(second.text.contains("18"), "\(ref.modelID): \(String(describing: second.error))")262 #expect(second.completions == [.endOfTurn])263 print("LIVE \(adapter.rawValue) tool: \(ref.modelID) reasoning \(reasoning?.rawValue ?? "default") called \(call.name)(\(call.arguments.serialized())), then answered; \(Live.describe(second.outcome.usage))")264 }265266 @Test(arguments: [Adapter.openAI, .anthropic, .xAI, .google, .openRouter])267 func reasoningSummary(adapter: Adapter) async throws {268 guard Live.available(adapter) else { return }269 let provider = Live.provider(adapter, label: "reasoning")270 let ref = try await Live.modelRef(adapter, provider: provider)271 let request = GenerationRequest(model: ref, messages: [.user("A farmer has chickens and rabbits: 35 heads and 94 legs. How many of each? Answer in one line.")],272 parameters: GenerationParameters(maxOutputTokens: 1500, reasoning: .medium))273 let c = await Live.collect(provider, request)274 expectWellFormed(c)275 #expect(c.text.contains("23"))276 #expect(c.text.contains("12"))277 // A summary is shown when the provider returns one; whether it does for a small problem is the provider's choice.278 print("LIVE \(adapter.rawValue) reasoning: \(ref.modelID) summary \(c.reasoning.count) chars, \(Live.describe(c.outcome.usage))")279 }280281 /// A solid red 64 x 64 PNG, drawn here so no image file ships with the tests.282 static func redPNG() throws -> ImageInput {283 let side = 64284 let context = try #require(CGContext(data: nil, width: side, height: side, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(),285 bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue))286 context.setFillColor(red: 1, green: 0, blue: 0, alpha: 1)287 context.fill(CGRect(x: 0, y: 0, width: side, height: side))288 let image = try #require(context.makeImage())289 let data = NSMutableData()290 let destination = try #require(CGImageDestinationCreateWithData(data, "public.png" as CFString, 1, nil))291 CGImageDestinationAddImage(destination, image, nil)292 #expect(CGImageDestinationFinalize(destination))293 return ImageInput(data: data as Data, mimeType: "image/png")294 }295296 @Test(arguments: [Adapter.openAI, .anthropic, .xAI, .google, .openRouter])297 func imageInput(adapter: Adapter) async throws {298 guard Live.available(adapter) else { return }299 let provider = Live.provider(adapter)300 let ref = try await Live.modelRef(adapter, provider: provider)301 let request = GenerationRequest(model: ref, messages: [.user("What single colour fills this image? Answer with one word.", images: [try Self.redPNG()])],302 parameters: GenerationParameters(maxOutputTokens: 64, reasoning: .off))303 let c = await Live.collect(provider, request)304 expectWellFormed(c)305 #expect(c.text.lowercased().contains("red"), "\(adapter.rawValue) answered: \(c.text.prefix(60))")306 print("LIVE \(adapter.rawValue) image: \(ref.modelID) answered \(c.text.trimmingCharacters(in: .whitespacesAndNewlines).prefix(20)), \(Live.describe(c.outcome.usage))")307 }308309 @Test(arguments: [Adapter.openAI, .anthropic, .xAI, .google, .openRouter])310 func badKeyIsReportedAsUnauthorized(adapter: Adapter) async {311 guard Live.available(adapter) else { return }312 let bad = adapter.key // a syntactically plausible key that no provider issued313 let transport = URLSessionTransport()314 let provider: any ChatProvider = switch adapter {315 case .openAI: OpenAIProvider(transport: transport, apiKey: { bad })316 case .anthropic: AnthropicProvider(transport: transport, apiKey: { bad })317 case .openRouter: OpenRouterProvider(transport: transport, apiKey: { bad })318 case .xAI: XAIProvider(transport: transport, apiKey: { bad })319 default: GeminiProvider(transport: transport, apiKey: { bad })320 }321 let c = await collect(provider.stream(request(adapter.kind, Live.model[adapter] ?? "")))322 guard case .unauthorized = c.error else { Issue.record("\(adapter.rawValue): expected unauthorized, got \(String(describing: c.error))"); return }323 #expect(c.error?.message.contains(bad) == false)324 }325326 /// One tiny request to every chat model each provider lists, with a temperature and a reasoning effort set, so the327 /// parameter fallbacks are exercised against the real APIs. Opt-in: `ASTRA_LIVE_MATRIX=1`.328 @Test(.enabled(if: Live.matrix), arguments: Adapter.allCases)329 func everyListedModelAnswers(adapter: Adapter) async throws {330 guard Live.available(adapter) else { return }331 let provider = Live.provider(adapter)332 var refs = try await provider.listModels().map(\.ref)333 if adapter == .openRouter {334 // The catalogue has hundreds of models: sample one per major upstream.335 let sample = ["anthropic/claude-haiku-5.5", "google/gemini-3.1-flash-lite", "x-ai/grok-4.3", "openai/gpt-6-luna"]336 refs = refs.filter { sample.contains($0.modelID) }337 }338 if adapter == .openAI { refs = refs.filter { !$0.modelID.contains("-pro") } } // the pro tier is priced out of a smoke test339 var failures: [String] = []340 await withTaskGroup(of: (String, Collected).self) { group in341 var pending = refs[...]342 func add(_ group: inout TaskGroup<(String, Collected)>) {343 guard let ref = pending.popFirst() else { return }344 group.addTask {345 let request = GenerationRequest(model: ref, messages: [.user("Reply with exactly: ASTRA-OK")],346 parameters: GenerationParameters(temperature: 0.5, maxOutputTokens: 200, reasoning: .low))347 return (ref.modelID, await Live.collect(provider, request))348 }349 }350 for _ in 0..<4 { add(&group) }351 for await (id, c) in group {352 add(&group)353 let ok = c.error == nil && c.contractViolation == nil && c.completions.count == 1354 let line = "LIVE-MATRIX \(adapter.rawValue) \(id): " + (ok ? "ok" : "FAILED") + " reported \(c.starts.first?.reportedModel ?? "-") stop \(c.completions.first?.rawValue ?? "-") text \(c.text.contains("ASTRA-OK") ? "match" : "\(c.text.count) chars") \(Live.describe(c.outcome.usage))"355 + (c.error.map { " error: \($0.title): \($0.message.prefix(160))" } ?? "")356 print(line)357 if !ok { failures.append(line) }358 }359 }360 #expect(failures.isEmpty, "\(failures.count) of \(refs.count) models failed")361 }362}363