Repository Public License
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX OS Automation9// File : Packages/ORIXProviderAdapters/Tests/ORIXProviderAdaptersTests/FixtureAnthropicTests.swift10// Description : Mocked tests of the Anthropic adapter against sanitized,11// recorded-shape SSE transcripts: streaming, tools, usage,12// caching, structured output, thinking continuity, errors.13// Module : ORIXProviderAdaptersTests14// Repository : https://git.spboucher.ai/orix-automation15// Created : 2026-11-2116// Last Modified : 2026-11-2117//18// =============================================================================19// AUTHOR & PROFESSIONAL AFFILIATION20// =============================================================================21//22// Author : Simon-Pierre Boucher23// Position : Professor of Finance24// Institution : Université du Québec en Outaouais (UQO)25// Department : Département des sciences administratives26// Organization : SP Boucher27//28// Website : https://spboucher.ai29// Git : https://git.spboucher.ai30//31// =============================================================================32// OFFICIAL CONTACT DIRECTORY33// =============================================================================34//35// General : info@spboucher.ai36// Professional : contact@spboucher.ai37// Direct : simon@spboucher.ai38// Research : research@spboucher.ai39// Development : dev@spboucher.ai40// Support : support@spboucher.ai41// Security/Admin : admin@spboucher.ai42// Notifications : notifications@spboucher.ai43//44// Preferred Contact: support@spboucher.ai45//46// =============================================================================47// COPYRIGHT & INTELLECTUAL PROPERTY48// =============================================================================49//50// Copyright (c) 202651// Simon-Pierre Boucher52//53// All rights reserved where applicable.54//55// This file is governed by the applicable project license,56// copyright notices, institutional agreements and relevant law.57//58// Unless expressly authorized by the applicable license or59// rights holder, copying, redistribution, sublicensing,60// publication and commercial exploitation are not permitted.61//62// Existing third-party rights and notices remain unaffected.63//64// =============================================================================65// CONFIDENTIALITY & SECURITY66// =============================================================================67//68// Classification : Public69// Distribution : Public repository; use is governed by the project license70// Security Contact : admin@spboucher.ai71//72// Access to this source does not independently grant permission73// to redistribute or disclose confidential materials.74//75// Do not embed credentials, private keys, access tokens,76// certificates or confidential personal information.77//78// =============================================================================79// LICENSE & MAINTENANCE80// =============================================================================81//82// License : Proprietary, all rights reserved (see license file)83// License File : LICENSE84// Maintainer : Simon-Pierre Boucher85// Technical Contact: dev@spboucher.ai86//87// =============================================================================88// END OF SP BOUCHER SOURCE HEADER89// =============================================================================9091import Testing92import Foundation93import ORIXCore94@testable import ORIXProviderAdapters9596@Suite("Fixture Anthropic adapter (mocked network)")97struct FixtureAnthropicTests {98 func provider(_ server: StubServer, options: AnthropicOptions = .init()) -> AnthropicProvider {99 AnthropicProvider(apiKey: fixtureKey, session: server.session, baseURL: server.baseURL, options: options)100 }101102 static let textTranscript = anthropicSSE([103 ("message_start", #"{"type":"message_start","message":{"id":"msg_fixture_01","type":"message","role":"assistant","content":[],"model":"claude-haiku-5-5","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":12,"cache_creation_input_tokens":300,"cache_read_input_tokens":2000,"output_tokens":1}}}"#),104 ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}"#),105 ("ping", #"{"type":"ping"}"#),106 ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Bonjour"}}"#),107 ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":", monde — ✓"}}"#),108 ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),109 ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":9}}"#),110 ("message_stop", #"{"type":"message_stop"}"#),111 ])112113 static let toolTranscript = anthropicSSE([114 ("message_start", #"{"type":"message_start","message":{"id":"msg_fixture_02","type":"message","role":"assistant","content":[],"model":"claude-haiku-5-5","stop_reason":null,"usage":{"input_tokens":400,"output_tokens":2}}}"#),115 ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}"#),116 ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Checking."}}"#),117 ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),118 ("content_block_start", #"{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_fixture_01","name":"get_weather","input":{}}}"#),119 ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""}}"#),120 ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"ci"}}"#),121 ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ty\": \"Qué"}}"#),122 ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"bec\"}"}}"#),123 ("content_block_stop", #"{"type":"content_block_stop","index":1}"#),124 ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":41}}"#),125 ("message_stop", #"{"type":"message_stop"}"#),126 ])127128 @Test func streamsTextWithUsageAndSendsTheRightRequest() async {129 let server = StubServer().sse(Self.textTranscript, chunkSize: 17)130 let (events, error) = await collect(provider(server).stream(LLMRequest(model: "claude-haiku-5-5", system: "Be brief.",131 messages: [.user("Salut")], maxOutputTokens: 16, temperature: 0.2)))132 #expect(error == nil)133 #expect(events.text == "Bonjour, monde — ✓")134 #expect(events.stop == .endTurn)135 #expect(events.lastUsage == TokenUsage(inputTokens: 12, outputTokens: 9, cacheReadTokens: 2000, cacheWriteTokens: 300))136 #expect(events.last == .stop(.endTurn))137138 let req = server.requests[0]139 #expect(req.method == "POST" && req.path == "/v1/messages")140 #expect(req.header("x-api-key") == fixtureKey)141 #expect(req.header("anthropic-version") == "2026-10-21")142 #expect(req.body?["model"] == "claude-haiku-5-5")143 #expect(req.body?["max_tokens"] == 16)144 #expect(req.body?["stream"] == true)145 #expect(req.body?["system"] == "Be brief.")146 #expect(req.body?["temperature"] == nil, "sampling parameters are rejected on API version 2026-10-21")147 #expect(req.body?["messages"] == [["role": "user", "content": [["type": "text", "text": "Salut"]]]])148 #expect(req.body?["tools"] == nil && req.body?["tool_choice"] == nil)149 }150151 @Test func assemblesAToolCallFromSplitJSONDeltas() async {152 let server = StubServer().sse(Self.toolTranscript, chunkSize: 5)153 let (events, error) = await collect(provider(server).stream(LLMRequest(model: "claude-haiku-5-5", messages: [.user("Météo ?")], tools: [weatherTool])))154 #expect(error == nil)155 #expect(events.text == "Checking.")156 #expect(events.started == ["0:toolu_fixture_01:get_weather"])157 #expect(events.inputDeltas == ["{\"ci", "ty\": \"Qué", "bec\"}"])158 #expect(events.toolUses == [LLMToolUse(id: "toolu_fixture_01", name: "get_weather", input: ["city": "Québec"])])159 #expect(events.stop == .toolUse)160 #expect(events.lastUsage?.outputTokens == 41)161 }162163 @Test func parallelToolCallsKeepTheirOrderAndOrdinals() async {164 let transcript = anthropicSSE([165 ("message_start", #"{"type":"message_start","message":{"usage":{"input_tokens":10,"output_tokens":1}}}"#),166 ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_fixture_a","name":"get_weather","input":{}}}"#),167 ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"city\":\"Paris\"}"}}"#),168 ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),169 ("content_block_start", #"{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_fixture_b","name":"get_weather","input":{}}}"#),170 ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"city\":"}}"#),171 ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\"Lyon\"}"}}"#),172 ("content_block_stop", #"{"type":"content_block_stop","index":1}"#),173 ("content_block_start", #"{"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_fixture_c","name":"no_args","input":{}}}"#),174 ("content_block_stop", #"{"type":"content_block_stop","index":2}"#),175 ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":60}}"#),176 ("message_stop", #"{"type":"message_stop"}"#),177 ])178 let server = StubServer().sse(transcript)179 let response = try? await provider(server).complete(LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")], tools: [weatherTool]))180 #expect(response?.toolUses.map(\.id) == ["toolu_fixture_a", "toolu_fixture_b", "toolu_fixture_c"])181 #expect(response?.toolUses.map(\.input) == [["city": "Paris"], ["city": "Lyon"], [:]])182 #expect(response?.stopReason == .toolUse)183 }184185 @Test func encodesToolsToolResultsSystemCachingAndImages() {186 let p = provider(StubServer())187 let request = LLMRequest(188 model: "claude-haiku-5-5", system: "You are ORIX.",189 messages: [190 .user("Weather?"),191 LLMMessage(role: .assistant, content: [.text("Looking."), .toolUse(LLMToolUse(id: "toolu_fixture_01", name: "get_weather", input: ["city": "Paris"]))]),192 LLMMessage(role: .user, content: [.text("and this picture"), .image(base64: "AAAA", mediaType: "image/png"),193 .toolResult(toolUseID: "toolu_fixture_01", content: "boom", isError: true)]),194 .user("merged into the same user turn"),195 ],196 tools: [weatherTool, LLMToolSpec(name: "second", description: "Second tool.", inputSchema: .object([:]))],197 toolChoice: .tool(name: "get_weather"), maxOutputTokens: 64, cacheSystemAndTools: true)198 let body = p.requestBody(for: request).body199200 #expect(body["system"] == [["type": "text", "text": "You are ORIX.", "cache_control": ["type": "ephemeral"]]])201 let tools = body["tools"]?.arrayValue ?? []202 #expect(tools.count == 2)203 #expect(tools[0]["name"] == "get_weather" && tools[0]["input_schema"] == weatherTool.inputSchema.raw)204 #expect(tools[0]["eager_input_streaming"] == true)205 #expect(tools[0]["cache_control"] == nil && tools[1]["cache_control"] == ["type": "ephemeral"], "one breakpoint, on the last tool")206 #expect(body["tool_choice"] == ["type": "tool", "name": "get_weather"])207208 let messages = body["messages"]?.arrayValue ?? []209 #expect(messages.count == 3, "consecutive user messages are merged")210 #expect(messages[1] == ["role": "assistant", "content": [["type": "text", "text": "Looking."],211 ["type": "tool_use", "id": "toolu_fixture_01", "name": "get_weather", "input": ["city": "Paris"]]]])212 let user = messages[2]["content"]?.arrayValue ?? []213 #expect(user.first == ["type": "tool_result", "tool_use_id": "toolu_fixture_01", "content": "boom", "is_error": true], "tool results come first")214 #expect(user[1] == ["type": "text", "text": "and this picture"])215 #expect(user[2] == ["type": "image", "source": ["type": "base64", "media_type": "image/png", "data": "AAAA"]])216 #expect(user[3] == ["type": "text", "text": "merged into the same user turn"])217 }218219 @Test func mapsToolChoiceIncludingModelsThatRejectForcedCalls() {220 let p = provider(StubServer())221 func body(_ model: String, _ choice: LLMToolChoice, system: String? = nil) -> JSONValue {222 p.requestBody(for: LLMRequest(model: model, system: system, messages: [.user("x")], tools: [weatherTool], toolChoice: choice)).body223 }224 #expect(body("claude-haiku-5-5", .auto)["tool_choice"] == ["type": "auto"])225 #expect(body("claude-haiku-5-5", .none)["tool_choice"] == ["type": "none"])226 #expect(body("claude-haiku-5-5", .required)["tool_choice"] == ["type": "any"])227 #expect(body("claude-haiku-4-5-20251001", .tool(name: "get_weather"))["tool_choice"] == ["type": "tool", "name": "get_weather"])228229 // Claude Opus 5.5 answers 400 to forced tool use: auto + an explicit instruction, placed after the caller's system prompt.230 let forced = body("claude-opus-5-5", .tool(name: "get_weather"), system: "Base.")231 #expect(forced["tool_choice"] == ["type": "auto"])232 let system = forced["system"]?.arrayValue ?? []233 #expect(system.count == 2 && system[0]["text"] == "Base.")234 #expect(system[1]["text"]?.stringValue?.contains("`get_weather`") == true)235 #expect(body("claude-sonnet-5-5", .required)["system"]?.stringValue?.contains("calling one of the provided tools") == true)236 }237238 @Test func structuredOutputUsesOutputConfigWithASanitizedSchema() {239 let schema = JSONSchema.object(["name": .string(), "age": .integer(minimum: 0, maximum: 150),240 "tags": .array(of: .object(["k": .string()], required: ["k"]))], required: ["name", "age"])241 let body = provider(StubServer(), options: .init(thinking: .disabled, effort: "low"))242 .requestBody(for: LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")], responseSchema: schema)).body243 let format = body["output_config"]?["format"]244 #expect(format?["type"] == "json_schema")245 #expect(format?["schema"]?["additionalProperties"] == false)246 #expect(format?["schema"]?["required"] == ["age", "name"])247 #expect(format?["schema"]?["properties"]?["age"] == ["type": "integer"], "numeric bounds are not accepted by the API and are removed")248 #expect(format?["schema"]?["properties"]?["tags"]?["items"]?["additionalProperties"] == false)249 #expect(body["output_config"]?["effort"] == "low")250 #expect(body["thinking"] == ["type": "disabled"])251 // A model that rejects disabled thinking gets no thinking field at all.252 let opus = provider(StubServer(), options: .init(thinking: .disabled)).requestBody(for: LLMRequest(model: "claude-opus-5-5", messages: [.user("x")])).body253 #expect(opus["thinking"] == nil)254 let adaptive = provider(StubServer(), options: .init(thinking: .adaptive(summarized: true))).requestBody(for: LLMRequest(model: "claude-opus-5-5", messages: [.user("x")])).body255 #expect(adaptive["thinking"] == ["type": "adaptive", "display": "summarized"])256 }257258 static let thinkingToolTranscript = anthropicSSE([259 ("message_start", #"{"type":"message_start","message":{"usage":{"input_tokens":50,"output_tokens":1}}}"#),260 ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}"#),261 ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"The user wants "}}"#),262 ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"the weather."}}"#),263 ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"FIXTURESIGNATURE=="}}"#),264 ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),265 ("content_block_start", #"{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_fixture_t1","name":"get_weather","input":{}}}"#),266 ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"city\":\"Oslo\"}"}}"#),267 ("content_block_stop", #"{"type":"content_block_stop","index":1}"#),268 ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":70}}"#),269 ("message_stop", #"{"type":"message_stop"}"#),270 ])271272 @Test func thinkingBlocksAreSummarizedThenReplayedWithTheToolResult() async throws {273 let server = StubServer().sse(Self.thinkingToolTranscript).sse(Self.textTranscript)274 let p = provider(server)275 let first = LLMRequest(model: "claude-haiku-5-5", messages: [.user("Weather in Oslo?")], tools: [weatherTool])276 var seen: [LLMStreamEvent] = []277 for try await e in p.stream(first) { seen.append(e) }278 #expect(seen.reasoning == "The user wants the weather.")279 let call = try #require(seen.toolUses.first)280281 // The runtime replays the turn using Core content only (no thinking block exists in LLMContent).282 let second = LLMRequest(model: "claude-haiku-5-5", messages: first.messages + [283 LLMMessage(role: .assistant, content: [.toolUse(call)]),284 LLMMessage(role: .user, content: [.toolResult(toolUseID: call.id, content: "4°C", isError: false)]),285 ], tools: [weatherTool])286 _ = try await p.complete(second)287 let assistant = server.requests[1].body?["messages"]?[1]?["content"]?.arrayValue ?? []288 #expect(assistant.count == 2)289 #expect(assistant[0] == ["type": "thinking", "thinking": "The user wants the weather.", "signature": "FIXTURESIGNATURE=="],290 "the signed thinking block is passed back unchanged, before its tool call")291 #expect(assistant[1]["type"] == "tool_use")292 }293294 @Test func rejectedThinkingBlockIsDroppedAndTheRequestSentOnceMore() async throws {295 let rejection = #"{"type":"error","error":{"type":"invalid_request_error","message":"messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation."}}"#296 let server = StubServer().sse(Self.thinkingToolTranscript).json(400, rejection).sse(Self.textTranscript)297 let p = provider(server)298 let first = LLMRequest(model: "claude-haiku-5-5", messages: [.user("Weather in Oslo?")], tools: [weatherTool])299 let call = try #require(try await p.complete(first).toolUses.first)300 let second = LLMRequest(model: "claude-haiku-5-5", messages: first.messages + [301 LLMMessage(role: .assistant, content: [.toolUse(call)]),302 LLMMessage(role: .user, content: [.toolResult(toolUseID: call.id, content: "4°C", isError: false)]),303 ], tools: [weatherTool])304 let response = try await p.complete(second)305 #expect(response.text == "Bonjour, monde — ✓")306 #expect(server.requests.count == 3)307 #expect(server.requests[1].body?["messages"]?[1]?["content"]?.arrayValue?.count == 2)308 #expect(server.requests[2].body?["messages"]?[1]?["content"]?.arrayValue?.map { $0["type"] } == ["tool_use"], "retry carries no thinking block")309 // A 400 that is not about thinking is never retried.310 let other = StubServer().json(400, #"{"type":"error","error":{"type":"invalid_request_error","message":"max_tokens: must be positive"}}"#)311 let (_, error) = await collect(provider(other).stream(second))312 #expect(error == .invalidRequest("max_tokens: must be positive"))313 #expect(other.requests.count == 1)314 }315316 @Test func refusalStopsWithoutReleasingToolCalls() async {317 let transcript = anthropicSSE([318 ("message_start", #"{"type":"message_start","message":{"usage":{"input_tokens":10,"output_tokens":1}}}"#),319 ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_fixture_r","name":"get_weather","input":{}}}"#),320 ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"city\":\"X\"}"}}"#),321 ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),322 ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"refusal","stop_details":{"type":"refusal","category":"cyber","explanation":null}},"usage":{"output_tokens":12}}"#),323 ("message_stop", #"{"type":"message_stop"}"#),324 ])325 let (events, error) = await collect(provider(StubServer().sse(transcript)).stream(LLMRequest(model: "claude-opus-5", messages: [.user("x")], tools: [weatherTool])))326 #expect(error == nil)327 #expect(events.stop == .refusal)328 #expect(events.toolUses.isEmpty, "a refused turn's tool calls must never be executable")329 }330331 @Test func truncatedToolInputIsDroppedUnderMaxTokensAndAnErrorOtherwise() async {332 func transcript(stop: String) -> String {333 anthropicSSE([334 ("message_start", #"{"type":"message_start","message":{"usage":{"input_tokens":10,"output_tokens":1}}}"#),335 ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_fixture_x","name":"get_weather","input":{}}}"#),336 ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"city\":\"Mon"}}"#),337 ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),338 ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"\#(stop)"},"usage":{"output_tokens":64}}"#),339 ("message_stop", #"{"type":"message_stop"}"#),340 ])341 }342 let request = LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")], tools: [weatherTool])343 let cut = await collect(provider(StubServer().sse(transcript(stop: "max_tokens"))).stream(request))344 #expect(cut.error == nil && cut.events.toolUses.isEmpty && cut.events.stop == .maxTokens)345 let broken = await collect(provider(StubServer().sse(transcript(stop: "tool_use"))).stream(request))346 #expect(broken.events.toolUses.isEmpty && broken.events.stop == nil)347 if case .decoding(let m) = broken.error { #expect(m.contains("get_weather")) } else { Issue.record("expected a decoding error, got \(String(describing: broken.error))") }348 }349350 @Test(arguments: [351 (401, #"{"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}"#, LLMError.authentication("invalid x-api-key")),352 (403, #"{"type":"error","error":{"type":"permission_error","message":"Your API key does not have permission"}}"#, LLMError.authentication("Your API key does not have permission")),353 (404, #"{"type":"error","error":{"type":"not_found_error","message":"model: claude-nope"}}"#, LLMError.modelNotFound("claude-nope — model: claude-nope")),354 (529, #"{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}"#, LLMError.overloaded),355 (400, #"{"type":"error","error":{"type":"invalid_request_error","message":"prompt is too long: 1200000 tokens > 1000000 maximum"}}"#, LLMError.contextTooLong("prompt is too long: 1200000 tokens > 1000000 maximum")),356 (400, #"{"type":"error","error":{"type":"invalid_request_error","message":"messages: roles must alternate"}}"#, LLMError.invalidRequest("messages: roles must alternate")),357 (500, #"{"type":"error","error":{"type":"api_error","message":"Internal server error"}}"#, LLMError.server(status: 500, message: "Internal server error")),358 (503, "upstream connect error", LLMError.server(status: 503, message: "upstream connect error")),359 (402, #"{"type":"error","error":{"type":"billing_error","message":"Credit balance too low"}}"#, LLMError.invalidRequest("Billing problem on the provider account. Credit balance too low")),360 ] as [(Int, String, LLMError)])361 func mapsHTTPErrors(status: Int, body: String, expected: LLMError) async {362 let server = StubServer().json(status, body)363 let (events, error) = await collect(provider(server).stream(LLMRequest(model: "claude-nope", messages: [.user("x")])))364 #expect(events.isEmpty)365 #expect(error == expected)366 }367368 @Test func rateLimitCarriesRetryAfterAndASpendCapIsNotRetryable() async {369 let limited = StubServer().json(429, #"{"type":"error","error":{"type":"rate_limit_error","message":"Too many requests"}}"#, headers: ["retry-after": "7"])370 let a = await collect(provider(limited).stream(LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")])))371 #expect(a.error == .rateLimited(retryAfterSeconds: 7))372 #expect(a.error?.isRetryable == true)373 let capped = StubServer().json(429, #"{"type":"error","error":{"type":"rate_limit_error","message":"Limit reached","details":{"error_code":"enforced_spend_limit_reached"}}}"#)374 let b = await collect(provider(capped).stream(LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")])))375 #expect(b.error?.isRetryable == false)376 }377378 @Test func midStreamErrorEventDisconnectAndEarlyEndAreTypedErrors() async {379 let head = anthropicSSE([380 ("message_start", #"{"type":"message_start","message":{"usage":{"input_tokens":10,"output_tokens":1}}}"#),381 ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}"#),382 ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"partial"}}"#),383 ])384 let request = LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")])385 let overloaded = await collect(provider(StubServer().sse(head + anthropicSSE([("error", #"{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}"#)]))).stream(request))386 #expect(overloaded.events.text == "partial" && overloaded.error == .overloaded)387388 let dropped = await collect(provider(StubServer().sse(head, failure: .networkConnectionLost)).stream(request))389 #expect(dropped.events.text == "partial")390 if case .network = dropped.error {} else { Issue.record("expected a network error, got \(String(describing: dropped.error))") }391 #expect(dropped.events.stop == nil, "no stop event is invented for an interrupted stream")392393 let early = await collect(provider(StubServer().sse(head)).stream(request))394 #expect(early.error == .network("The response stream ended before the message was complete."))395396 let canceled = await collect(provider(StubServer().enqueue(.init(failure: .cancelled))).stream(request))397 #expect(canceled.error == .canceled)398 }399400 @Test func unknownEventsAndBlockTypesAreIgnored() async {401 let transcript = anthropicSSE([402 ("message_start", #"{"type":"message_start","message":{"usage":{"input_tokens":3,"output_tokens":1}}}"#),403 ("future_event", #"{"type":"future_event","payload":{"x":1}}"#),404 ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"server_tool_use","id":"srvtoolu_fixture","name":"web_search","input":{}}}"#),405 ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"q\":1}"}}"#),406 ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),407 ("content_block_start", #"{"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}"#),408 ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"ok"}}"#),409 ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"future_delta","x":1}}"#),410 ("content_block_stop", #"{"type":"content_block_stop","index":1}"#),411 ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"pause_turn"},"usage":{"output_tokens":5}}"#),412 ("message_stop", #"{"type":"message_stop"}"#),413 ])414 let (events, error) = await collect(provider(StubServer().sse(transcript)).stream(LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")])))415 #expect(error == nil && events.text == "ok" && events.toolUses.isEmpty && events.stop == .other)416 }417418 @Test func listsModelsAcrossPagesAndMergesRegistryPrices() async throws {419 let server = StubServer()420 .json(200, #"{"data":[{"type":"model","id":"claude-haiku-5-5","display_name":"Claude Haiku 5.5","created_at":"2026-01-01T00:00:00Z","max_input_tokens":1000000,"max_tokens":128000,"capabilities":{"image_input":{"supported":true},"structured_outputs":{"supported":true}}}],"has_more":true,"first_id":"claude-haiku-5-5","last_id":"claude-haiku-5-5"}"#)421 .json(200, #"{"data":[{"type":"model","id":"claude-haiku-4-5-20251001","display_name":"Claude Haiku 4.5"},{"type":"model","id":"claude-future-9","display_name":"Claude Future 9","max_input_tokens":5,"capabilities":{"image_input":{"supported":false},"structured_outputs":{"supported":false}}}],"has_more":false,"last_id":"claude-future-9"}"#)422 let models = try await provider(server).listModels()423 #expect(models.map(\.id) == ["claude-haiku-5-5", "claude-haiku-4-5-20251001", "claude-future-9"])424 #expect(server.requests[0].path == "/v1/models" && server.requests[1].query?.contains("after_id=claude-haiku-5-5") == true)425 #expect(models[0].inputPricePerMTok == 0.10 && models[0].outputPricePerMTok == 0.50 && models[0].contextWindow == 1_000_000)426 #expect(models[0].supportsVision && models[0].supportsStructuredOutput && models[0].source == "api")427 #expect(models[1].inputPricePerMTok == 1 && models[1].contextWindow == 200_000 && models[1].maxOutputTokens == 64_000, "dated snapshot ids resolve to the registry entry")428 #expect(models[2].inputPricePerMTok == nil && models[2].outputPricePerMTok == nil, "an unknown model has no invented price")429 #expect(models[2].contextWindow == 5 && !models[2].supportsVision)430 let (failed, _) = (try? await provider(StubServer().json(401, #"{"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}"#)).listModels(), 0)431 #expect(failed == nil)432 }433434 @Test func theAPIKeyNeverAppearsInErrorsOrDescriptions() async {435 let echo = #"{"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key: \#(fixtureKey) (also Authorization: Bearer abcdefghijklmnopqrstuvwxyz012345)"}}"#436 let p = provider(StubServer().json(401, echo))437 let (_, error) = await collect(p.stream(LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")])))438 let rendered = "\(String(describing: error)) \(error?.errorDescription ?? "") \(error?.localizedDescription ?? "")"439 #expect(!rendered.contains(fixtureKey) && !rendered.contains("abcdefghijklmnopqrstuvwxyz012345"))440 #expect(rendered.contains(Redactor.placeholder))441 var dumped = ""442 dump(p, to: &dumped)443 #expect(!"\(p) \(String(reflecting: p)) \(dumped)".contains(fixtureKey))444 #expect(Redactor.shared.redact("key=\(fixtureKey)").contains(fixtureKey) == false, "the key is registered with the shared redactor at init")445 }446}447