Repository Public License
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX OS Automation9// File : Packages/ORIXProviderAdapters/Tests/ORIXProviderAdaptersTests/LiveProviderTests.swift10// Description : Live smoke tests against the real provider APIs (model11// list, one capped streamed completion, one forced tool-call12// round trip). Run only when the provider's13// ORIX_DEV_*_API_KEY is set; otherwise skipped out loud.14// Module : ORIXProviderAdaptersTests15// Repository : https://git.spboucher.ai/orix-automation16// Created : 2026-11-2117// Last Modified : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author : Simon-Pierre Boucher24// Position : Professor of Finance25// Institution : Université du Québec en Outaouais (UQO)26// Department : Département des sciences administratives27// Organization : SP Boucher28//29// Website : https://spboucher.ai30// Git : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General : info@spboucher.ai37// Professional : contact@spboucher.ai38// Direct : simon@spboucher.ai39// Research : research@spboucher.ai40// Development : dev@spboucher.ai41// Support : support@spboucher.ai42// Security/Admin : admin@spboucher.ai43// Notifications : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification : Public70// Distribution : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License : Proprietary, all rights reserved (see license file)84// License File : LICENSE85// Maintainer : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Testing93import Foundation94import ORIXCore95@testable import ORIXProviderAdapters9697// Budget: each provider makes 2 model-list calls (free) and at most 4 completions of ≤ 64 output tokens (one Anthropic98// extra of two requests at ≤ 400) on the cheapest model class. Worst case for the whole suite is far below US$0.01.99// Nothing here prints, logs or stores a key: results are printed as `LIVE_RESULT` lines with model ids, timings and token counts only.100101#if DEBUG102struct LiveHarness {103 let provider: ProviderID104 /// Preferred cheap models, in order; the first one the key can use is taken.105 let candidates: [String]106 let make: @Sendable (String) -> any LLMProvider107108 var key: String? { DevEnvironmentCredentials().apiKey(for: provider) }109110 /// The provider client, or nil after printing the skip line.111 func client() -> (any LLMProvider)? {112 guard let key else {113 print("SKIPPED(no \(DevEnvironmentCredentials.variableName(for: provider) ?? "key"))")114 return nil115 }116 return make(key)117 }118119 func pickModel(_ models: [ModelInfo]) -> String? {120 let ids = Set(models.map(\.id))121 return candidates.first(where: ids.contains)122 }123124 func report(_ test: String, model: String, started: Date, usage: TokenUsage?, note: String = "") {125 let ms = Int((Date().timeIntervalSince(started) * 1000).rounded())126 let cost = usage?.reportedCostUSD.map { String(format: " reportedCostUSD=%.6f", $0) } ?? ""127 print("LIVE_RESULT provider=\(provider.rawValue) test=\(test) model=\(model) latencyMs=\(ms) inputTokens=\(usage?.inputTokens ?? 0) outputTokens=\(usage?.outputTokens ?? 0) cacheRead=\(usage?.cacheReadTokens ?? 0)\(cost) result=pass\(note.isEmpty ? "" : " note=\(note)")")128 }129130 static let lookupTool = LLMToolSpec(name: "orix_lookup", description: "Looks up the value stored under a key. Call this whenever the user asks for a stored value.",131 inputSchema: .object(["key": .string("The key to look up")], required: ["key"]))132133 func listModels() async throws {134 guard let client = client() else { return }135 let started = Date()136 let models = try await client.listModels()137 #expect(!models.isEmpty)138 #expect(models.allSatisfy { $0.provider == provider && !$0.id.isEmpty })139 let model = try #require(pickModel(models), "none of the candidate models \(candidates) is available to this key")140 let info = try #require(models.first { $0.id == model })141 #expect(info.inputPricePerMTok != nil && info.outputPricePerMTok != nil, "the smoke model must have a known price")142 print("LIVE_RESULT provider=\(provider.rawValue) test=listModels models=\(models.count) priced=\(models.filter { $0.inputPricePerMTok != nil }.count) smokeModel=\(model) latencyMs=\(Int((Date().timeIntervalSince(started) * 1000).rounded())) result=pass")143 }144145 func streamedText() async throws {146 guard let client = client() else { return }147 let model = try #require(pickModel(try await client.listModels()))148 let started = Date()149 var deltas = 0150 var response = LLMResponse(model: model, content: [], stopReason: .other, usage: TokenUsage())151 var text = ""152 for try await event in client.stream(LLMRequest(model: model, messages: [.user("Reply with the single word: pong")], maxOutputTokens: 16)) {153 switch event {154 case .textDelta(let t): deltas += 1; text += t155 case .usage(let u): response.usage = u156 case .stop(let s): response.stopReason = s157 default: break158 }159 }160 response.content = [.text(text)]161 #expect(deltas >= 1, "text must arrive as stream deltas")162 #expect(!response.text.isEmpty)163 #expect(response.stopReason == .endTurn || response.stopReason == .maxTokens)164 #expect(response.usage.inputTokens > 0 && response.usage.outputTokens > 0 && response.usage.outputTokens <= 16)165 report("streamedText", model: model, started: started, usage: response.usage)166 }167168 func toolRoundTrip() async throws {169 guard let client = client() else { return }170 let model = try #require(pickModel(try await client.listModels()))171 let started = Date()172 let question = LLMMessage.user("What value is stored under the key alpha? Use the orix_lookup tool.")173 let first = try await client.complete(LLMRequest(model: model, system: "You answer using the provided tool.", messages: [question],174 tools: [Self.lookupTool], toolChoice: .tool(name: "orix_lookup"), maxOutputTokens: 64))175 #expect(first.stopReason == .toolUse)176 let call = try #require(first.toolUses.first, "the forced tool call was not returned")177 #expect(call.name == "orix_lookup" && !call.id.isEmpty)178 #expect(Self.lookupTool.inputSchema.validate(call.input).isEmpty, "tool input must satisfy the declared schema")179180 let second = try await client.complete(LLMRequest(model: model, system: "You answer using the provided tool.", messages: [181 question,182 LLMMessage(role: .assistant, content: first.content),183 LLMMessage(role: .user, content: [.toolResult(toolUseID: call.id, content: "alpha = 42", isError: false)]),184 ], tools: [Self.lookupTool], toolChoice: .auto, maxOutputTokens: 64))185 #expect(second.toolUses.isEmpty)186 #expect(second.text.contains("42"), "the final answer should use the tool result")187 #expect(second.stopReason == .endTurn || second.stopReason == .maxTokens)188 report("toolRoundTrip", model: model, started: started, usage: first.usage + second.usage)189 }190}191192@Suite("Live Anthropic (real API)", .serialized)193struct LiveAnthropicTests {194 // Thinking is switched off for the capped requests so the 16/64-token caps buy visible output, not reasoning.195 static let harness = LiveHarness(provider: .anthropic, candidates: ["claude-haiku-5-5", "claude-haiku-4-5", "claude-haiku-4-5-20251001"]) {196 AnthropicProvider(apiKey: $0, options: .init(thinking: .disabled))197 }198 @Test func listModels() async throws { try await Self.harness.listModels() }199 @Test func streamedText() async throws { try await Self.harness.streamedText() }200 @Test func toolRoundTrip() async throws { try await Self.harness.toolRoundTrip() }201202 /// The API requires thinking blocks to accompany a replayed tool call. This checks the adapter's in-memory replay against203 /// the real API with thinking left on (the default of current models).204 @Test func toolRoundTripWithThinkingReplay() async throws {205 guard let key = Self.harness.key else { print("SKIPPED(no ORIX_DEV_ANTHROPIC_API_KEY)"); return }206 let client = AnthropicProvider(apiKey: key, options: .init(thinking: .adaptive(summarized: true), effort: "high"))207 let model = "claude-haiku-5-5"208 guard try await client.listModels().contains(where: { $0.id == model }) else { print("SKIPPED(\(model) is not available to this key)"); return }209 let started = Date()210 let question = LLMMessage.user("Work out 17 × 23 − 91 carefully. The result is the name of a key: look that key up with the orix_lookup tool and tell me its value.")211 let first = try await client.complete(LLMRequest(model: model, messages: [question], tools: [LiveHarness.lookupTool], maxOutputTokens: 400))212 guard let call = first.toolUses.first else { print("SKIPPED(the model answered without calling the tool; nothing to replay)"); return }213 let followUp = LLMRequest(model: model, messages: [214 question, LLMMessage(role: .assistant, content: first.content),215 LLMMessage(role: .user, content: [.toolResult(toolUseID: call.id, content: "the stored value is 42", isError: false)]),216 ], tools: [LiveHarness.lookupTool], maxOutputTokens: 400)217 let replayed = client.requestBody(for: followUp).injected218 let second = try await client.complete(followUp)219 #expect(second.text.contains("42"))220 Self.harness.report("toolRoundTripWithThinkingReplay", model: model, started: started, usage: first.usage + second.usage,221 note: replayed ? "thinking-blocks-replayed-and-accepted" : "model-did-not-think-before-the-call")222 }223}224225@Suite("Live OpenAI (real API)", .serialized)226struct LiveOpenAITests {227 // Non-reasoning small models first: a reasoning model can spend a 16-token cap entirely on reasoning.228 static let harness = LiveHarness(provider: .openai, candidates: ["gpt-4o-mini", "gpt-4.1-mini"]) { OpenAIProvider(apiKey: $0) }229 @Test func listModels() async throws { try await Self.harness.listModels() }230 @Test func streamedText() async throws { try await Self.harness.streamedText() }231 @Test func toolRoundTrip() async throws { try await Self.harness.toolRoundTrip() }232}233234@Suite("Live OpenRouter (real API)", .serialized)235struct LiveOpenRouterTests {236 static let harness = LiveHarness(provider: .openrouter, candidates: ["openai/gpt-4o-mini", "google/gemini-3.1-flash-lite", "mistralai/mistral-nemo"]) {237 OpenRouterProvider(apiKey: $0, routing: ProviderRouting(allowFallbacks: true, requireParameters: true))238 }239 @Test func listModels() async throws { try await Self.harness.listModels() }240 @Test func streamedText() async throws {241 try await Self.harness.streamedText()242 }243 @Test func toolRoundTrip() async throws { try await Self.harness.toolRoundTrip() }244245 /// OpenRouter reports what the request cost; the adapter must surface it.246 @Test func reportsCost() async throws {247 guard let client = Self.harness.client() else { return }248 let model = try #require(Self.harness.pickModel(try await client.listModels()))249 let started = Date()250 let response = try await client.complete(LLMRequest(model: model, messages: [.user("Reply with the single word: pong")], maxOutputTokens: 16))251 let cost = try #require(response.usage.reportedCostUSD, "usage.cost was not reported")252 #expect(cost > 0 && cost < 0.001)253 Self.harness.report("reportsCost", model: model, started: started, usage: response.usage)254 }255}256#endif257