SPB Git

Repository Public License

LiveProviderTests.swift 13.9 KB 257 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX OS Automation9// File             : Packages/ORIXProviderAdapters/Tests/ORIXProviderAdaptersTests/LiveProviderTests.swift10// Description      : Live smoke tests against the real provider APIs (model11//                    list, one capped streamed completion, one forced tool-call12//                    round trip). Run only when the provider's13//                    ORIX_DEV_*_API_KEY is set; otherwise skipped out loud.14// Module           : ORIXProviderAdaptersTests15// Repository       : https://git.spboucher.ai/orix-automation16// Created          : 2026-11-2117// Last Modified    : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author           : Simon-Pierre Boucher24// Position         : Professor of Finance25// Institution      : Université du Québec en Outaouais (UQO)26// Department       : Département des sciences administratives27// Organization     : SP Boucher28//29// Website          : https://spboucher.ai30// Git              : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General          : info@spboucher.ai37// Professional     : contact@spboucher.ai38// Direct           : simon@spboucher.ai39// Research         : research@spboucher.ai40// Development      : dev@spboucher.ai41// Support          : support@spboucher.ai42// Security/Admin   : admin@spboucher.ai43// Notifications    : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification   : Public70// Distribution     : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License          : Proprietary, all rights reserved (see license file)84// License File     : LICENSE85// Maintainer       : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Testing93import Foundation94import ORIXCore95@testable import ORIXProviderAdapters9697// Budget: each provider makes 2 model-list calls (free) and at most 4 completions of ≤ 64 output tokens (one Anthropic98// extra of two requests at ≤ 400) on the cheapest model class. Worst case for the whole suite is far below US$0.01.99// Nothing here prints, logs or stores a key: results are printed as `LIVE_RESULT` lines with model ids, timings and token counts only.100101#if DEBUG102struct LiveHarness {103    let provider: ProviderID104    /// Preferred cheap models, in order; the first one the key can use is taken.105    let candidates: [String]106    let make: @Sendable (String) -> any LLMProvider107108    var key: String? { DevEnvironmentCredentials().apiKey(for: provider) }109110    /// The provider client, or nil after printing the skip line.111    func client() -> (any LLMProvider)? {112        guard let key else {113            print("SKIPPED(no \(DevEnvironmentCredentials.variableName(for: provider) ?? "key"))")114            return nil115        }116        return make(key)117    }118119    func pickModel(_ models: [ModelInfo]) -> String? {120        let ids = Set(models.map(\.id))121        return candidates.first(where: ids.contains)122    }123124    func report(_ test: String, model: String, started: Date, usage: TokenUsage?, note: String = "") {125        let ms = Int((Date().timeIntervalSince(started) * 1000).rounded())126        let cost = usage?.reportedCostUSD.map { String(format: " reportedCostUSD=%.6f", $0) } ?? ""127        print("LIVE_RESULT provider=\(provider.rawValue) test=\(test) model=\(model) latencyMs=\(ms) inputTokens=\(usage?.inputTokens ?? 0) outputTokens=\(usage?.outputTokens ?? 0) cacheRead=\(usage?.cacheReadTokens ?? 0)\(cost) result=pass\(note.isEmpty ? "" : " note=\(note)")")128    }129130    static let lookupTool = LLMToolSpec(name: "orix_lookup", description: "Looks up the value stored under a key. Call this whenever the user asks for a stored value.",131                                        inputSchema: .object(["key": .string("The key to look up")], required: ["key"]))132133    func listModels() async throws {134        guard let client = client() else { return }135        let started = Date()136        let models = try await client.listModels()137        #expect(!models.isEmpty)138        #expect(models.allSatisfy { $0.provider == provider && !$0.id.isEmpty })139        let model = try #require(pickModel(models), "none of the candidate models \(candidates) is available to this key")140        let info = try #require(models.first { $0.id == model })141        #expect(info.inputPricePerMTok != nil && info.outputPricePerMTok != nil, "the smoke model must have a known price")142        print("LIVE_RESULT provider=\(provider.rawValue) test=listModels models=\(models.count) priced=\(models.filter { $0.inputPricePerMTok != nil }.count) smokeModel=\(model) latencyMs=\(Int((Date().timeIntervalSince(started) * 1000).rounded())) result=pass")143    }144145    func streamedText() async throws {146        guard let client = client() else { return }147        let model = try #require(pickModel(try await client.listModels()))148        let started = Date()149        var deltas = 0150        var response = LLMResponse(model: model, content: [], stopReason: .other, usage: TokenUsage())151        var text = ""152        for try await event in client.stream(LLMRequest(model: model, messages: [.user("Reply with the single word: pong")], maxOutputTokens: 16)) {153            switch event {154            case .textDelta(let t): deltas += 1; text += t155            case .usage(let u): response.usage = u156            case .stop(let s): response.stopReason = s157            default: break158            }159        }160        response.content = [.text(text)]161        #expect(deltas >= 1, "text must arrive as stream deltas")162        #expect(!response.text.isEmpty)163        #expect(response.stopReason == .endTurn || response.stopReason == .maxTokens)164        #expect(response.usage.inputTokens > 0 && response.usage.outputTokens > 0 && response.usage.outputTokens <= 16)165        report("streamedText", model: model, started: started, usage: response.usage)166    }167168    func toolRoundTrip() async throws {169        guard let client = client() else { return }170        let model = try #require(pickModel(try await client.listModels()))171        let started = Date()172        let question = LLMMessage.user("What value is stored under the key alpha? Use the orix_lookup tool.")173        let first = try await client.complete(LLMRequest(model: model, system: "You answer using the provided tool.", messages: [question],174                                                         tools: [Self.lookupTool], toolChoice: .tool(name: "orix_lookup"), maxOutputTokens: 64))175        #expect(first.stopReason == .toolUse)176        let call = try #require(first.toolUses.first, "the forced tool call was not returned")177        #expect(call.name == "orix_lookup" && !call.id.isEmpty)178        #expect(Self.lookupTool.inputSchema.validate(call.input).isEmpty, "tool input must satisfy the declared schema")179180        let second = try await client.complete(LLMRequest(model: model, system: "You answer using the provided tool.", messages: [181            question,182            LLMMessage(role: .assistant, content: first.content),183            LLMMessage(role: .user, content: [.toolResult(toolUseID: call.id, content: "alpha = 42", isError: false)]),184        ], tools: [Self.lookupTool], toolChoice: .auto, maxOutputTokens: 64))185        #expect(second.toolUses.isEmpty)186        #expect(second.text.contains("42"), "the final answer should use the tool result")187        #expect(second.stopReason == .endTurn || second.stopReason == .maxTokens)188        report("toolRoundTrip", model: model, started: started, usage: first.usage + second.usage)189    }190}191192@Suite("Live Anthropic (real API)", .serialized)193struct LiveAnthropicTests {194    // Thinking is switched off for the capped requests so the 16/64-token caps buy visible output, not reasoning.195    static let harness = LiveHarness(provider: .anthropic, candidates: ["claude-haiku-5-5", "claude-haiku-4-5", "claude-haiku-4-5-20251001"]) {196        AnthropicProvider(apiKey: $0, options: .init(thinking: .disabled))197    }198    @Test func listModels() async throws { try await Self.harness.listModels() }199    @Test func streamedText() async throws { try await Self.harness.streamedText() }200    @Test func toolRoundTrip() async throws { try await Self.harness.toolRoundTrip() }201202    /// The API requires thinking blocks to accompany a replayed tool call. This checks the adapter's in-memory replay against203    /// the real API with thinking left on (the default of current models).204    @Test func toolRoundTripWithThinkingReplay() async throws {205        guard let key = Self.harness.key else { print("SKIPPED(no ORIX_DEV_ANTHROPIC_API_KEY)"); return }206        let client = AnthropicProvider(apiKey: key, options: .init(thinking: .adaptive(summarized: true), effort: "high"))207        let model = "claude-haiku-5-5"208        guard try await client.listModels().contains(where: { $0.id == model }) else { print("SKIPPED(\(model) is not available to this key)"); return }209        let started = Date()210        let question = LLMMessage.user("Work out 17 × 23 − 91 carefully. The result is the name of a key: look that key up with the orix_lookup tool and tell me its value.")211        let first = try await client.complete(LLMRequest(model: model, messages: [question], tools: [LiveHarness.lookupTool], maxOutputTokens: 400))212        guard let call = first.toolUses.first else { print("SKIPPED(the model answered without calling the tool; nothing to replay)"); return }213        let followUp = LLMRequest(model: model, messages: [214            question, LLMMessage(role: .assistant, content: first.content),215            LLMMessage(role: .user, content: [.toolResult(toolUseID: call.id, content: "the stored value is 42", isError: false)]),216        ], tools: [LiveHarness.lookupTool], maxOutputTokens: 400)217        let replayed = client.requestBody(for: followUp).injected218        let second = try await client.complete(followUp)219        #expect(second.text.contains("42"))220        Self.harness.report("toolRoundTripWithThinkingReplay", model: model, started: started, usage: first.usage + second.usage,221                            note: replayed ? "thinking-blocks-replayed-and-accepted" : "model-did-not-think-before-the-call")222    }223}224225@Suite("Live OpenAI (real API)", .serialized)226struct LiveOpenAITests {227    // Non-reasoning small models first: a reasoning model can spend a 16-token cap entirely on reasoning.228    static let harness = LiveHarness(provider: .openai, candidates: ["gpt-4o-mini", "gpt-4.1-mini"]) { OpenAIProvider(apiKey: $0) }229    @Test func listModels() async throws { try await Self.harness.listModels() }230    @Test func streamedText() async throws { try await Self.harness.streamedText() }231    @Test func toolRoundTrip() async throws { try await Self.harness.toolRoundTrip() }232}233234@Suite("Live OpenRouter (real API)", .serialized)235struct LiveOpenRouterTests {236    static let harness = LiveHarness(provider: .openrouter, candidates: ["openai/gpt-4o-mini", "google/gemini-3.1-flash-lite", "mistralai/mistral-nemo"]) {237        OpenRouterProvider(apiKey: $0, routing: ProviderRouting(allowFallbacks: true, requireParameters: true))238    }239    @Test func listModels() async throws { try await Self.harness.listModels() }240    @Test func streamedText() async throws {241        try await Self.harness.streamedText()242    }243    @Test func toolRoundTrip() async throws { try await Self.harness.toolRoundTrip() }244245    /// OpenRouter reports what the request cost; the adapter must surface it.246    @Test func reportsCost() async throws {247        guard let client = Self.harness.client() else { return }248        let model = try #require(Self.harness.pickModel(try await client.listModels()))249        let started = Date()250        let response = try await client.complete(LLMRequest(model: model, messages: [.user("Reply with the single word: pong")], maxOutputTokens: 16))251        let cost = try #require(response.usage.reportedCostUSD, "usage.cost was not reported")252        #expect(cost > 0 && cost < 0.001)253        Self.harness.report("reportsCost", model: model, started: started, usage: response.usage)254    }255}256#endif257
Files main
Menu

Theme