SPB Git

Repository Public MIT

LiveSmokeTests.swift 20.9 KB 363 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraProvidersTests/LiveSmokeTests.swift10// Module           : AstraProvidersTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import CoreGraphics90import Foundation91import ImageIO92import Testing93@testable import AstraProviders9495/// Live checks against the real providers. Disabled unless `ASTRA_LIVE_TESTS=1` and the provider's key variable is set96/// (`source Scripts/smoke-env.sh`). Requests are tiny and bounded; no key is ever printed, logged or written.97enum Live {98    static let env = ProcessInfo.processInfo.environment99    static let on = env["ASTRA_LIVE_TESTS"] == "1"100    static func has(_ name: String) -> Bool { on && !(env[name] ?? "").isEmpty }101    /// `ASTRA_LIVE_MATRIX=1` additionally sends one tiny request to every model each provider lists.102    static let matrix = on && env["ASTRA_LIVE_MATRIX"] == "1"103104    static let keyVariable: [Adapter: String] = [.openAI: "OPENAI_API_KEY", .anthropic: "ANTHROPIC_API_KEY", .openRouter: "OPENROUTER_API_KEY",105                                                 .xAI: "XAI_API_KEY", .google: "GEMINI_API_KEY", .custom: "ASTRA_CUSTOM_KEY"]106    /// The cheapest current chat model of each provider on 2026-11-21.107    static let model: [Adapter: String] = [.openAI: "gpt-6-luna", .anthropic: "claude-haiku-5-5", .openRouter: "openai/gpt-6-luna",108                                           .xAI: "grok-4.3", .google: "gemini-3.1-flash-lite"]109110    static func available(_ adapter: Adapter) -> Bool {111        guard let variable = keyVariable[adapter], has(variable) else { return false }112        return adapter != .custom || !(env["ASTRA_CUSTOM_BASE_URL"] ?? "").isEmpty113    }114115    static func provider(_ adapter: Adapter, label: String = "") -> any ChatProvider {116        let key = env[keyVariable[adapter] ?? ""] ?? ""117        let transport = TeeTransport(label: label.isEmpty ? nil : "\(adapter.rawValue)_\(label)")118        switch adapter {119        case .openAI: return OpenAIProvider(transport: transport, apiKey: { key })120        case .anthropic: return AnthropicProvider(transport: transport, apiKey: { key })121        case .openRouter: return OpenRouterProvider(transport: transport, apiKey: { key })122        case .xAI: return XAIProvider(transport: transport, apiKey: { key })123        case .google: return GeminiProvider(transport: transport, apiKey: { key })124        case .custom:125            let profile = EndpointProfile(id: "live", name: "Live", baseURL: env["ASTRA_CUSTOM_BASE_URL"] ?? "")126            return OpenAICompatibleProvider(profile: profile, transport: transport, apiKey: { key })127        }128    }129130    static func modelRef(_ adapter: Adapter, provider: any ChatProvider) async throws -> ModelRef {131        if let id = model[adapter] { return ModelRef(provider: adapter.kind, modelID: id) }132        // Custom endpoint: use whatever it reports.133        let models = try await provider.listModels()134        return try #require(models.first?.ref)135    }136137    /// Collects a live stream, retrying a transient failure (capacity, rate limit, overload) that produced nothing, as the app's orchestrator does.138    static func collect(_ provider: any ChatProvider, _ request: GenerationRequest) async -> Collected {139        var result = await AstraProvidersTests.collect(provider.stream(request))140        for delay in [10, 30, 60] {141            guard let error = result.error, error.isTransient, result.events.isEmpty else { break }142            print("LIVE retry \(request.model.modelID) after \(error.title)")143            try? await Task.sleep(nanoseconds: UInt64(delay) * 1_000_000_000)144            result = await AstraProvidersTests.collect(provider.stream(request))145        }146        return result147    }148149    static func describe(_ usage: TokenUsage?) -> String {150        guard let u = usage else { return "usage: none" }151        return "in \(u.inputTokens.map(String.init) ?? "-") out \(u.outputTokens.map(String.init) ?? "-") cached \(u.cachedInputTokens.map(String.init) ?? "-") reasoning \(u.reasoningTokens.map(String.init) ?? "-")"152            + (u.reportedCostUSD.map { " cost $\($0)" } ?? "")153    }154}155156/// The real network transport; when `ASTRA_RECORD_DIR` is set it also writes the raw stream lines it received, for fixtures.157struct TeeTransport: HTTPTransport {158    let inner = URLSessionTransport()159    var label: String?160161    func send(_ request: URLRequest) async throws -> (Data, HTTPURLResponse) { try await inner.send(request) }162163    func stream(_ request: URLRequest) async throws -> (HTTPURLResponse, AsyncThrowingStream<String, Error>) {164        let (response, lines) = try await inner.stream(request)165        guard let label, let directory = Live.env["ASTRA_RECORD_DIR"], !directory.isEmpty else { return (response, lines) }166        let url = URL(fileURLWithPath: directory).appendingPathComponent("\(label).sse")167        let (out, continuation) = AsyncThrowingStream<String, Error>.makeStream()168        let task = Task {169            var text = ""170            do {171                for try await line in lines { text += line + "\n"; continuation.yield(line) }172                try? text.write(to: url, atomically: true, encoding: .utf8)     // response body only: no request, no headers, no key173                continuation.finish()174            } catch { continuation.finish(throwing: error) }175        }176        continuation.onTermination = { _ in task.cancel() }177        return (response, out)178    }179}180181@Suite("Live smoke tests", .enabled(if: Live.on), .serialized)182struct LiveSmokeTests {183    @Test(arguments: Adapter.allCases)184    func verifyAndListModels(adapter: Adapter) async throws {185        guard Live.available(adapter) else { return }186        let provider = Live.provider(adapter)187        let status = try await provider.verify()188        #expect(status.ok)189        let models = try await provider.listModels()190        #expect(!models.isEmpty)191        if let expected = Live.model[adapter] { #expect(models.contains { $0.ref.modelID == expected }, "\(expected) is no longer listed") }192        for model in models {193            #expect(model.ref.provider == adapter.kind)194            #expect(model.capabilities.contains(.text))195            if let pricing = model.pricing { #expect(!pricing.asOf.isEmpty); #expect(pricing.inputPerMTok >= 0) }196        }197        print("LIVE \(adapter.rawValue) verify ok, \(models.count) models" + (status.remainingCreditUSD.map { _ in ", credit reported" } ?? ""))198    }199200    @Test(arguments: Adapter.allCases)201    func streamedGeneration(adapter: Adapter) async throws {202        guard Live.available(adapter) else { return }203        let provider = Live.provider(adapter, label: "text")204        let ref = try await Live.modelRef(adapter, provider: provider)205        let request = GenerationRequest(model: ref, system: "You follow instructions exactly.", messages: [.user("Reply with exactly: ASTRA-OK")],206                                        parameters: GenerationParameters(maxOutputTokens: 32, reasoning: .off))207        let c = await Live.collect(provider, request)208        expectWellFormed(c)209        #expect(c.text.contains("ASTRA-OK"))210        let reported = try #require(c.starts.first?.reportedModel)211        #expect(!reported.isEmpty)212        // The provider may answer with a dated snapshot or a vendor-prefixed id of the model that was asked for.213        let asked = ref.modelID.split(separator: "/").last.map(String.init) ?? ref.modelID214        #expect(reported.contains(asked) || asked.contains(reported), "asked for \(ref.modelID), provider reported \(reported)")215        #expect((c.outcome.usage?.inputTokens ?? 0) > 0)216        #expect((c.outcome.usage?.outputTokens ?? 0) > 0)217        #expect(c.completions == [.endOfTurn])218        print("LIVE \(adapter.rawValue) text: asked \(ref.modelID), reported \(reported), served by \(c.starts.first?.servedBy ?? "-"), \(Live.describe(c.outcome.usage))")219    }220221    @Test(arguments: Adapter.allCases)222    func toolCallRoundTrip(adapter: Adapter) async throws {223        guard Live.available(adapter) else { return }224        let provider = Live.provider(adapter, label: "tool")225        try await roundTrip(adapter, provider: provider, ref: try await Live.modelRef(adapter, provider: provider), reasoning: .off)226    }227228    /// The same round trip with reasoning on, on the models whose tool cycles have special rules: OpenAI reasoning items,229    /// Anthropic thinking blocks (never replayed), Sonnet 5.5 `between_tools`, Gemini thought signatures, models without call ids.230    static let toolCycleModels: [(Adapter, String, ReasoningEffort?)] = [231        (.openAI, "gpt-6.2-sol", .low), (.openAI, "gpt-4.1-mini", nil),232        (.anthropic, "claude-sonnet-5-5", .low), (.anthropic, "claude-sonnet-5-5", .off), (.anthropic, "claude-opus-5-5", .off), (.anthropic, "claude-haiku-4-5", .low), (.anthropic, "claude-haiku-5-5", .medium),233        (.google, "gemini-4-hydrogen", .low), (.google, "gemini-2.5-flash-lite", .low), (.google, "gemini-3.1-flash-lite", .medium),234        (.xAI, "grok-4.7", .low), (.xAI, "grok-4.20-0309-non-reasoning", nil),235        (.openRouter, "anthropic/claude-haiku-5.5", .low), (.openRouter, "google/gemini-3.1-flash-lite", .low),236    ]237238    @Test(.enabled(if: Live.matrix), arguments: 0..<LiveSmokeTests.toolCycleModels.count)239    func toolCallRoundTripWithReasoning(index: Int) async throws {240        let (adapter, model, effort) = Self.toolCycleModels[index]241        guard Live.available(adapter) else { return }242        try await roundTrip(adapter, provider: Live.provider(adapter), ref: ModelRef(provider: adapter.kind, modelID: model), reasoning: effort)243    }244245    func roundTrip(_ adapter: Adapter, provider: any ChatProvider, ref: ModelRef, reasoning: ReasoningEffort?) async throws {246        let question = PromptMessage.user("What is the weather in Paris right now? Call the get_weather tool, then answer in one short sentence.")247        let budget = reasoning == .off || reasoning == nil ? 256 : 1500248        let first = await Live.collect(provider, GenerationRequest(model: ref, messages: [question], parameters: GenerationParameters(maxOutputTokens: budget, reasoning: reasoning), tools: [weatherTool]))249        expectWellFormed(first)250        #expect(first.completions == [.toolCalls], "\(ref.modelID): \(String(describing: first.error))")251        let call = try #require(first.outcome.toolCalls.first)252        #expect(call.name == "get_weather")253        #expect(call.arguments["city"]?.stringValue?.contains("Paris") == true)254        #expect(first.toolStarts.contains(call.id))255256        // Second leg: replay the call and its result exactly as the app would.257        let history = [question, .assistant(first.text, toolCalls: first.outcome.toolCalls)]258            + first.outcome.toolCalls.map { PromptMessage.tool("18 C, cloudy", callID: $0.id, name: $0.name) }259        let second = await Live.collect(provider, GenerationRequest(model: ref, messages: history, parameters: GenerationParameters(maxOutputTokens: budget, reasoning: reasoning), tools: [weatherTool]))260        expectWellFormed(second)261        #expect(second.text.contains("18"), "\(ref.modelID): \(String(describing: second.error))")262        #expect(second.completions == [.endOfTurn])263        print("LIVE \(adapter.rawValue) tool: \(ref.modelID) reasoning \(reasoning?.rawValue ?? "default") called \(call.name)(\(call.arguments.serialized())), then answered; \(Live.describe(second.outcome.usage))")264    }265266    @Test(arguments: [Adapter.openAI, .anthropic, .xAI, .google, .openRouter])267    func reasoningSummary(adapter: Adapter) async throws {268        guard Live.available(adapter) else { return }269        let provider = Live.provider(adapter, label: "reasoning")270        let ref = try await Live.modelRef(adapter, provider: provider)271        let request = GenerationRequest(model: ref, messages: [.user("A farmer has chickens and rabbits: 35 heads and 94 legs. How many of each? Answer in one line.")],272                                        parameters: GenerationParameters(maxOutputTokens: 1500, reasoning: .medium))273        let c = await Live.collect(provider, request)274        expectWellFormed(c)275        #expect(c.text.contains("23"))276        #expect(c.text.contains("12"))277        // A summary is shown when the provider returns one; whether it does for a small problem is the provider's choice.278        print("LIVE \(adapter.rawValue) reasoning: \(ref.modelID) summary \(c.reasoning.count) chars, \(Live.describe(c.outcome.usage))")279    }280281    /// A solid red 64 x 64 PNG, drawn here so no image file ships with the tests.282    static func redPNG() throws -> ImageInput {283        let side = 64284        let context = try #require(CGContext(data: nil, width: side, height: side, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(),285                                             bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue))286        context.setFillColor(red: 1, green: 0, blue: 0, alpha: 1)287        context.fill(CGRect(x: 0, y: 0, width: side, height: side))288        let image = try #require(context.makeImage())289        let data = NSMutableData()290        let destination = try #require(CGImageDestinationCreateWithData(data, "public.png" as CFString, 1, nil))291        CGImageDestinationAddImage(destination, image, nil)292        #expect(CGImageDestinationFinalize(destination))293        return ImageInput(data: data as Data, mimeType: "image/png")294    }295296    @Test(arguments: [Adapter.openAI, .anthropic, .xAI, .google, .openRouter])297    func imageInput(adapter: Adapter) async throws {298        guard Live.available(adapter) else { return }299        let provider = Live.provider(adapter)300        let ref = try await Live.modelRef(adapter, provider: provider)301        let request = GenerationRequest(model: ref, messages: [.user("What single colour fills this image? Answer with one word.", images: [try Self.redPNG()])],302                                        parameters: GenerationParameters(maxOutputTokens: 64, reasoning: .off))303        let c = await Live.collect(provider, request)304        expectWellFormed(c)305        #expect(c.text.lowercased().contains("red"), "\(adapter.rawValue) answered: \(c.text.prefix(60))")306        print("LIVE \(adapter.rawValue) image: \(ref.modelID) answered \(c.text.trimmingCharacters(in: .whitespacesAndNewlines).prefix(20)), \(Live.describe(c.outcome.usage))")307    }308309    @Test(arguments: [Adapter.openAI, .anthropic, .xAI, .google, .openRouter])310    func badKeyIsReportedAsUnauthorized(adapter: Adapter) async {311        guard Live.available(adapter) else { return }312        let bad = adapter.key      // a syntactically plausible key that no provider issued313        let transport = URLSessionTransport()314        let provider: any ChatProvider = switch adapter {315        case .openAI: OpenAIProvider(transport: transport, apiKey: { bad })316        case .anthropic: AnthropicProvider(transport: transport, apiKey: { bad })317        case .openRouter: OpenRouterProvider(transport: transport, apiKey: { bad })318        case .xAI: XAIProvider(transport: transport, apiKey: { bad })319        default: GeminiProvider(transport: transport, apiKey: { bad })320        }321        let c = await collect(provider.stream(request(adapter.kind, Live.model[adapter] ?? "")))322        guard case .unauthorized = c.error else { Issue.record("\(adapter.rawValue): expected unauthorized, got \(String(describing: c.error))"); return }323        #expect(c.error?.message.contains(bad) == false)324    }325326    /// One tiny request to every chat model each provider lists, with a temperature and a reasoning effort set, so the327    /// parameter fallbacks are exercised against the real APIs. Opt-in: `ASTRA_LIVE_MATRIX=1`.328    @Test(.enabled(if: Live.matrix), arguments: Adapter.allCases)329    func everyListedModelAnswers(adapter: Adapter) async throws {330        guard Live.available(adapter) else { return }331        let provider = Live.provider(adapter)332        var refs = try await provider.listModels().map(\.ref)333        if adapter == .openRouter {334            // The catalogue has hundreds of models: sample one per major upstream.335            let sample = ["anthropic/claude-haiku-5.5", "google/gemini-3.1-flash-lite", "x-ai/grok-4.3", "openai/gpt-6-luna"]336            refs = refs.filter { sample.contains($0.modelID) }337        }338        if adapter == .openAI { refs = refs.filter { !$0.modelID.contains("-pro") } }   // the pro tier is priced out of a smoke test339        var failures: [String] = []340        await withTaskGroup(of: (String, Collected).self) { group in341            var pending = refs[...]342            func add(_ group: inout TaskGroup<(String, Collected)>) {343                guard let ref = pending.popFirst() else { return }344                group.addTask {345                    let request = GenerationRequest(model: ref, messages: [.user("Reply with exactly: ASTRA-OK")],346                                                    parameters: GenerationParameters(temperature: 0.5, maxOutputTokens: 200, reasoning: .low))347                    return (ref.modelID, await Live.collect(provider, request))348                }349            }350            for _ in 0..<4 { add(&group) }351            for await (id, c) in group {352                add(&group)353                let ok = c.error == nil && c.contractViolation == nil && c.completions.count == 1354                let line = "LIVE-MATRIX \(adapter.rawValue) \(id): " + (ok ? "ok" : "FAILED") + " reported \(c.starts.first?.reportedModel ?? "-") stop \(c.completions.first?.rawValue ?? "-") text \(c.text.contains("ASTRA-OK") ? "match" : "\(c.text.count) chars") \(Live.describe(c.outcome.usage))"355                    + (c.error.map { " error: \($0.title): \($0.message.prefix(160))" } ?? "")356                print(line)357                if !ok { failures.append(line) }358            }359        }360        #expect(failures.isEmpty, "\(failures.count) of \(refs.count) models failed")361    }362}363
Files main
Menu

Theme