SPB Git

Repository Public License

FixtureAnthropicTests.swift 32.4 KB 447 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX OS Automation9// File             : Packages/ORIXProviderAdapters/Tests/ORIXProviderAdaptersTests/FixtureAnthropicTests.swift10// Description      : Mocked tests of the Anthropic adapter against sanitized,11//                    recorded-shape SSE transcripts: streaming, tools, usage,12//                    caching, structured output, thinking continuity, errors.13// Module           : ORIXProviderAdaptersTests14// Repository       : https://git.spboucher.ai/orix-automation15// Created          : 2026-11-2116// Last Modified    : 2026-11-2117//18// =============================================================================19// AUTHOR & PROFESSIONAL AFFILIATION20// =============================================================================21//22// Author           : Simon-Pierre Boucher23// Position         : Professor of Finance24// Institution      : Université du Québec en Outaouais (UQO)25// Department       : Département des sciences administratives26// Organization     : SP Boucher27//28// Website          : https://spboucher.ai29// Git              : https://git.spboucher.ai30//31// =============================================================================32// OFFICIAL CONTACT DIRECTORY33// =============================================================================34//35// General          : info@spboucher.ai36// Professional     : contact@spboucher.ai37// Direct           : simon@spboucher.ai38// Research         : research@spboucher.ai39// Development      : dev@spboucher.ai40// Support          : support@spboucher.ai41// Security/Admin   : admin@spboucher.ai42// Notifications    : notifications@spboucher.ai43//44// Preferred Contact: support@spboucher.ai45//46// =============================================================================47// COPYRIGHT & INTELLECTUAL PROPERTY48// =============================================================================49//50// Copyright (c) 202651// Simon-Pierre Boucher52//53// All rights reserved where applicable.54//55// This file is governed by the applicable project license,56// copyright notices, institutional agreements and relevant law.57//58// Unless expressly authorized by the applicable license or59// rights holder, copying, redistribution, sublicensing,60// publication and commercial exploitation are not permitted.61//62// Existing third-party rights and notices remain unaffected.63//64// =============================================================================65// CONFIDENTIALITY & SECURITY66// =============================================================================67//68// Classification   : Public69// Distribution     : Public repository; use is governed by the project license70// Security Contact : admin@spboucher.ai71//72// Access to this source does not independently grant permission73// to redistribute or disclose confidential materials.74//75// Do not embed credentials, private keys, access tokens,76// certificates or confidential personal information.77//78// =============================================================================79// LICENSE & MAINTENANCE80// =============================================================================81//82// License          : Proprietary, all rights reserved (see license file)83// License File     : LICENSE84// Maintainer       : Simon-Pierre Boucher85// Technical Contact: dev@spboucher.ai86//87// =============================================================================88// END OF SP BOUCHER SOURCE HEADER89// =============================================================================9091import Testing92import Foundation93import ORIXCore94@testable import ORIXProviderAdapters9596@Suite("Fixture Anthropic adapter (mocked network)")97struct FixtureAnthropicTests {98    func provider(_ server: StubServer, options: AnthropicOptions = .init()) -> AnthropicProvider {99        AnthropicProvider(apiKey: fixtureKey, session: server.session, baseURL: server.baseURL, options: options)100    }101102    static let textTranscript = anthropicSSE([103        ("message_start", #"{"type":"message_start","message":{"id":"msg_fixture_01","type":"message","role":"assistant","content":[],"model":"claude-haiku-5-5","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":12,"cache_creation_input_tokens":300,"cache_read_input_tokens":2000,"output_tokens":1}}}"#),104        ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}"#),105        ("ping", #"{"type":"ping"}"#),106        ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Bonjour"}}"#),107        ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":", monde — ✓"}}"#),108        ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),109        ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":9}}"#),110        ("message_stop", #"{"type":"message_stop"}"#),111    ])112113    static let toolTranscript = anthropicSSE([114        ("message_start", #"{"type":"message_start","message":{"id":"msg_fixture_02","type":"message","role":"assistant","content":[],"model":"claude-haiku-5-5","stop_reason":null,"usage":{"input_tokens":400,"output_tokens":2}}}"#),115        ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}"#),116        ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Checking."}}"#),117        ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),118        ("content_block_start", #"{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_fixture_01","name":"get_weather","input":{}}}"#),119        ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""}}"#),120        ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"ci"}}"#),121        ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"ty\": \"Qué"}}"#),122        ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"bec\"}"}}"#),123        ("content_block_stop", #"{"type":"content_block_stop","index":1}"#),124        ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":41}}"#),125        ("message_stop", #"{"type":"message_stop"}"#),126    ])127128    @Test func streamsTextWithUsageAndSendsTheRightRequest() async {129        let server = StubServer().sse(Self.textTranscript, chunkSize: 17)130        let (events, error) = await collect(provider(server).stream(LLMRequest(model: "claude-haiku-5-5", system: "Be brief.",131                                                                               messages: [.user("Salut")], maxOutputTokens: 16, temperature: 0.2)))132        #expect(error == nil)133        #expect(events.text == "Bonjour, monde — ✓")134        #expect(events.stop == .endTurn)135        #expect(events.lastUsage == TokenUsage(inputTokens: 12, outputTokens: 9, cacheReadTokens: 2000, cacheWriteTokens: 300))136        #expect(events.last == .stop(.endTurn))137138        let req = server.requests[0]139        #expect(req.method == "POST" && req.path == "/v1/messages")140        #expect(req.header("x-api-key") == fixtureKey)141        #expect(req.header("anthropic-version") == "2026-10-21")142        #expect(req.body?["model"] == "claude-haiku-5-5")143        #expect(req.body?["max_tokens"] == 16)144        #expect(req.body?["stream"] == true)145        #expect(req.body?["system"] == "Be brief.")146        #expect(req.body?["temperature"] == nil, "sampling parameters are rejected on API version 2026-10-21")147        #expect(req.body?["messages"] == [["role": "user", "content": [["type": "text", "text": "Salut"]]]])148        #expect(req.body?["tools"] == nil && req.body?["tool_choice"] == nil)149    }150151    @Test func assemblesAToolCallFromSplitJSONDeltas() async {152        let server = StubServer().sse(Self.toolTranscript, chunkSize: 5)153        let (events, error) = await collect(provider(server).stream(LLMRequest(model: "claude-haiku-5-5", messages: [.user("Météo ?")], tools: [weatherTool])))154        #expect(error == nil)155        #expect(events.text == "Checking.")156        #expect(events.started == ["0:toolu_fixture_01:get_weather"])157        #expect(events.inputDeltas == ["{\"ci", "ty\": \"Qué", "bec\"}"])158        #expect(events.toolUses == [LLMToolUse(id: "toolu_fixture_01", name: "get_weather", input: ["city": "Québec"])])159        #expect(events.stop == .toolUse)160        #expect(events.lastUsage?.outputTokens == 41)161    }162163    @Test func parallelToolCallsKeepTheirOrderAndOrdinals() async {164        let transcript = anthropicSSE([165            ("message_start", #"{"type":"message_start","message":{"usage":{"input_tokens":10,"output_tokens":1}}}"#),166            ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_fixture_a","name":"get_weather","input":{}}}"#),167            ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"city\":\"Paris\"}"}}"#),168            ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),169            ("content_block_start", #"{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_fixture_b","name":"get_weather","input":{}}}"#),170            ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"city\":"}}"#),171            ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\"Lyon\"}"}}"#),172            ("content_block_stop", #"{"type":"content_block_stop","index":1}"#),173            ("content_block_start", #"{"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_fixture_c","name":"no_args","input":{}}}"#),174            ("content_block_stop", #"{"type":"content_block_stop","index":2}"#),175            ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":60}}"#),176            ("message_stop", #"{"type":"message_stop"}"#),177        ])178        let server = StubServer().sse(transcript)179        let response = try? await provider(server).complete(LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")], tools: [weatherTool]))180        #expect(response?.toolUses.map(\.id) == ["toolu_fixture_a", "toolu_fixture_b", "toolu_fixture_c"])181        #expect(response?.toolUses.map(\.input) == [["city": "Paris"], ["city": "Lyon"], [:]])182        #expect(response?.stopReason == .toolUse)183    }184185    @Test func encodesToolsToolResultsSystemCachingAndImages() {186        let p = provider(StubServer())187        let request = LLMRequest(188            model: "claude-haiku-5-5", system: "You are ORIX.",189            messages: [190                .user("Weather?"),191                LLMMessage(role: .assistant, content: [.text("Looking."), .toolUse(LLMToolUse(id: "toolu_fixture_01", name: "get_weather", input: ["city": "Paris"]))]),192                LLMMessage(role: .user, content: [.text("and this picture"), .image(base64: "AAAA", mediaType: "image/png"),193                                                  .toolResult(toolUseID: "toolu_fixture_01", content: "boom", isError: true)]),194                .user("merged into the same user turn"),195            ],196            tools: [weatherTool, LLMToolSpec(name: "second", description: "Second tool.", inputSchema: .object([:]))],197            toolChoice: .tool(name: "get_weather"), maxOutputTokens: 64, cacheSystemAndTools: true)198        let body = p.requestBody(for: request).body199200        #expect(body["system"] == [["type": "text", "text": "You are ORIX.", "cache_control": ["type": "ephemeral"]]])201        let tools = body["tools"]?.arrayValue ?? []202        #expect(tools.count == 2)203        #expect(tools[0]["name"] == "get_weather" && tools[0]["input_schema"] == weatherTool.inputSchema.raw)204        #expect(tools[0]["eager_input_streaming"] == true)205        #expect(tools[0]["cache_control"] == nil && tools[1]["cache_control"] == ["type": "ephemeral"], "one breakpoint, on the last tool")206        #expect(body["tool_choice"] == ["type": "tool", "name": "get_weather"])207208        let messages = body["messages"]?.arrayValue ?? []209        #expect(messages.count == 3, "consecutive user messages are merged")210        #expect(messages[1] == ["role": "assistant", "content": [["type": "text", "text": "Looking."],211                                                                 ["type": "tool_use", "id": "toolu_fixture_01", "name": "get_weather", "input": ["city": "Paris"]]]])212        let user = messages[2]["content"]?.arrayValue ?? []213        #expect(user.first == ["type": "tool_result", "tool_use_id": "toolu_fixture_01", "content": "boom", "is_error": true], "tool results come first")214        #expect(user[1] == ["type": "text", "text": "and this picture"])215        #expect(user[2] == ["type": "image", "source": ["type": "base64", "media_type": "image/png", "data": "AAAA"]])216        #expect(user[3] == ["type": "text", "text": "merged into the same user turn"])217    }218219    @Test func mapsToolChoiceIncludingModelsThatRejectForcedCalls() {220        let p = provider(StubServer())221        func body(_ model: String, _ choice: LLMToolChoice, system: String? = nil) -> JSONValue {222            p.requestBody(for: LLMRequest(model: model, system: system, messages: [.user("x")], tools: [weatherTool], toolChoice: choice)).body223        }224        #expect(body("claude-haiku-5-5", .auto)["tool_choice"] == ["type": "auto"])225        #expect(body("claude-haiku-5-5", .none)["tool_choice"] == ["type": "none"])226        #expect(body("claude-haiku-5-5", .required)["tool_choice"] == ["type": "any"])227        #expect(body("claude-haiku-4-5-20251001", .tool(name: "get_weather"))["tool_choice"] == ["type": "tool", "name": "get_weather"])228229        // Claude Opus 5.5 answers 400 to forced tool use: auto + an explicit instruction, placed after the caller's system prompt.230        let forced = body("claude-opus-5-5", .tool(name: "get_weather"), system: "Base.")231        #expect(forced["tool_choice"] == ["type": "auto"])232        let system = forced["system"]?.arrayValue ?? []233        #expect(system.count == 2 && system[0]["text"] == "Base.")234        #expect(system[1]["text"]?.stringValue?.contains("`get_weather`") == true)235        #expect(body("claude-sonnet-5-5", .required)["system"]?.stringValue?.contains("calling one of the provided tools") == true)236    }237238    @Test func structuredOutputUsesOutputConfigWithASanitizedSchema() {239        let schema = JSONSchema.object(["name": .string(), "age": .integer(minimum: 0, maximum: 150),240                                        "tags": .array(of: .object(["k": .string()], required: ["k"]))], required: ["name", "age"])241        let body = provider(StubServer(), options: .init(thinking: .disabled, effort: "low"))242            .requestBody(for: LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")], responseSchema: schema)).body243        let format = body["output_config"]?["format"]244        #expect(format?["type"] == "json_schema")245        #expect(format?["schema"]?["additionalProperties"] == false)246        #expect(format?["schema"]?["required"] == ["age", "name"])247        #expect(format?["schema"]?["properties"]?["age"] == ["type": "integer"], "numeric bounds are not accepted by the API and are removed")248        #expect(format?["schema"]?["properties"]?["tags"]?["items"]?["additionalProperties"] == false)249        #expect(body["output_config"]?["effort"] == "low")250        #expect(body["thinking"] == ["type": "disabled"])251        // A model that rejects disabled thinking gets no thinking field at all.252        let opus = provider(StubServer(), options: .init(thinking: .disabled)).requestBody(for: LLMRequest(model: "claude-opus-5-5", messages: [.user("x")])).body253        #expect(opus["thinking"] == nil)254        let adaptive = provider(StubServer(), options: .init(thinking: .adaptive(summarized: true))).requestBody(for: LLMRequest(model: "claude-opus-5-5", messages: [.user("x")])).body255        #expect(adaptive["thinking"] == ["type": "adaptive", "display": "summarized"])256    }257258    static let thinkingToolTranscript = anthropicSSE([259        ("message_start", #"{"type":"message_start","message":{"usage":{"input_tokens":50,"output_tokens":1}}}"#),260        ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}"#),261        ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"The user wants "}}"#),262        ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"the weather."}}"#),263        ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"FIXTURESIGNATURE=="}}"#),264        ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),265        ("content_block_start", #"{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_fixture_t1","name":"get_weather","input":{}}}"#),266        ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"city\":\"Oslo\"}"}}"#),267        ("content_block_stop", #"{"type":"content_block_stop","index":1}"#),268        ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":70}}"#),269        ("message_stop", #"{"type":"message_stop"}"#),270    ])271272    @Test func thinkingBlocksAreSummarizedThenReplayedWithTheToolResult() async throws {273        let server = StubServer().sse(Self.thinkingToolTranscript).sse(Self.textTranscript)274        let p = provider(server)275        let first = LLMRequest(model: "claude-haiku-5-5", messages: [.user("Weather in Oslo?")], tools: [weatherTool])276        var seen: [LLMStreamEvent] = []277        for try await e in p.stream(first) { seen.append(e) }278        #expect(seen.reasoning == "The user wants the weather.")279        let call = try #require(seen.toolUses.first)280281        // The runtime replays the turn using Core content only (no thinking block exists in LLMContent).282        let second = LLMRequest(model: "claude-haiku-5-5", messages: first.messages + [283            LLMMessage(role: .assistant, content: [.toolUse(call)]),284            LLMMessage(role: .user, content: [.toolResult(toolUseID: call.id, content: "4°C", isError: false)]),285        ], tools: [weatherTool])286        _ = try await p.complete(second)287        let assistant = server.requests[1].body?["messages"]?[1]?["content"]?.arrayValue ?? []288        #expect(assistant.count == 2)289        #expect(assistant[0] == ["type": "thinking", "thinking": "The user wants the weather.", "signature": "FIXTURESIGNATURE=="],290                "the signed thinking block is passed back unchanged, before its tool call")291        #expect(assistant[1]["type"] == "tool_use")292    }293294    @Test func rejectedThinkingBlockIsDroppedAndTheRequestSentOnceMore() async throws {295        let rejection = #"{"type":"error","error":{"type":"invalid_request_error","message":"messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation."}}"#296        let server = StubServer().sse(Self.thinkingToolTranscript).json(400, rejection).sse(Self.textTranscript)297        let p = provider(server)298        let first = LLMRequest(model: "claude-haiku-5-5", messages: [.user("Weather in Oslo?")], tools: [weatherTool])299        let call = try #require(try await p.complete(first).toolUses.first)300        let second = LLMRequest(model: "claude-haiku-5-5", messages: first.messages + [301            LLMMessage(role: .assistant, content: [.toolUse(call)]),302            LLMMessage(role: .user, content: [.toolResult(toolUseID: call.id, content: "4°C", isError: false)]),303        ], tools: [weatherTool])304        let response = try await p.complete(second)305        #expect(response.text == "Bonjour, monde — ✓")306        #expect(server.requests.count == 3)307        #expect(server.requests[1].body?["messages"]?[1]?["content"]?.arrayValue?.count == 2)308        #expect(server.requests[2].body?["messages"]?[1]?["content"]?.arrayValue?.map { $0["type"] } == ["tool_use"], "retry carries no thinking block")309        // A 400 that is not about thinking is never retried.310        let other = StubServer().json(400, #"{"type":"error","error":{"type":"invalid_request_error","message":"max_tokens: must be positive"}}"#)311        let (_, error) = await collect(provider(other).stream(second))312        #expect(error == .invalidRequest("max_tokens: must be positive"))313        #expect(other.requests.count == 1)314    }315316    @Test func refusalStopsWithoutReleasingToolCalls() async {317        let transcript = anthropicSSE([318            ("message_start", #"{"type":"message_start","message":{"usage":{"input_tokens":10,"output_tokens":1}}}"#),319            ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_fixture_r","name":"get_weather","input":{}}}"#),320            ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"city\":\"X\"}"}}"#),321            ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),322            ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"refusal","stop_details":{"type":"refusal","category":"cyber","explanation":null}},"usage":{"output_tokens":12}}"#),323            ("message_stop", #"{"type":"message_stop"}"#),324        ])325        let (events, error) = await collect(provider(StubServer().sse(transcript)).stream(LLMRequest(model: "claude-opus-5", messages: [.user("x")], tools: [weatherTool])))326        #expect(error == nil)327        #expect(events.stop == .refusal)328        #expect(events.toolUses.isEmpty, "a refused turn's tool calls must never be executable")329    }330331    @Test func truncatedToolInputIsDroppedUnderMaxTokensAndAnErrorOtherwise() async {332        func transcript(stop: String) -> String {333            anthropicSSE([334                ("message_start", #"{"type":"message_start","message":{"usage":{"input_tokens":10,"output_tokens":1}}}"#),335                ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_fixture_x","name":"get_weather","input":{}}}"#),336                ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"city\":\"Mon"}}"#),337                ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),338                ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"\#(stop)"},"usage":{"output_tokens":64}}"#),339                ("message_stop", #"{"type":"message_stop"}"#),340            ])341        }342        let request = LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")], tools: [weatherTool])343        let cut = await collect(provider(StubServer().sse(transcript(stop: "max_tokens"))).stream(request))344        #expect(cut.error == nil && cut.events.toolUses.isEmpty && cut.events.stop == .maxTokens)345        let broken = await collect(provider(StubServer().sse(transcript(stop: "tool_use"))).stream(request))346        #expect(broken.events.toolUses.isEmpty && broken.events.stop == nil)347        if case .decoding(let m) = broken.error { #expect(m.contains("get_weather")) } else { Issue.record("expected a decoding error, got \(String(describing: broken.error))") }348    }349350    @Test(arguments: [351        (401, #"{"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}"#, LLMError.authentication("invalid x-api-key")),352        (403, #"{"type":"error","error":{"type":"permission_error","message":"Your API key does not have permission"}}"#, LLMError.authentication("Your API key does not have permission")),353        (404, #"{"type":"error","error":{"type":"not_found_error","message":"model: claude-nope"}}"#, LLMError.modelNotFound("claude-nope — model: claude-nope")),354        (529, #"{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}"#, LLMError.overloaded),355        (400, #"{"type":"error","error":{"type":"invalid_request_error","message":"prompt is too long: 1200000 tokens > 1000000 maximum"}}"#, LLMError.contextTooLong("prompt is too long: 1200000 tokens > 1000000 maximum")),356        (400, #"{"type":"error","error":{"type":"invalid_request_error","message":"messages: roles must alternate"}}"#, LLMError.invalidRequest("messages: roles must alternate")),357        (500, #"{"type":"error","error":{"type":"api_error","message":"Internal server error"}}"#, LLMError.server(status: 500, message: "Internal server error")),358        (503, "upstream connect error", LLMError.server(status: 503, message: "upstream connect error")),359        (402, #"{"type":"error","error":{"type":"billing_error","message":"Credit balance too low"}}"#, LLMError.invalidRequest("Billing problem on the provider account. Credit balance too low")),360    ] as [(Int, String, LLMError)])361    func mapsHTTPErrors(status: Int, body: String, expected: LLMError) async {362        let server = StubServer().json(status, body)363        let (events, error) = await collect(provider(server).stream(LLMRequest(model: "claude-nope", messages: [.user("x")])))364        #expect(events.isEmpty)365        #expect(error == expected)366    }367368    @Test func rateLimitCarriesRetryAfterAndASpendCapIsNotRetryable() async {369        let limited = StubServer().json(429, #"{"type":"error","error":{"type":"rate_limit_error","message":"Too many requests"}}"#, headers: ["retry-after": "7"])370        let a = await collect(provider(limited).stream(LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")])))371        #expect(a.error == .rateLimited(retryAfterSeconds: 7))372        #expect(a.error?.isRetryable == true)373        let capped = StubServer().json(429, #"{"type":"error","error":{"type":"rate_limit_error","message":"Limit reached","details":{"error_code":"enforced_spend_limit_reached"}}}"#)374        let b = await collect(provider(capped).stream(LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")])))375        #expect(b.error?.isRetryable == false)376    }377378    @Test func midStreamErrorEventDisconnectAndEarlyEndAreTypedErrors() async {379        let head = anthropicSSE([380            ("message_start", #"{"type":"message_start","message":{"usage":{"input_tokens":10,"output_tokens":1}}}"#),381            ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}"#),382            ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"partial"}}"#),383        ])384        let request = LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")])385        let overloaded = await collect(provider(StubServer().sse(head + anthropicSSE([("error", #"{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}"#)]))).stream(request))386        #expect(overloaded.events.text == "partial" && overloaded.error == .overloaded)387388        let dropped = await collect(provider(StubServer().sse(head, failure: .networkConnectionLost)).stream(request))389        #expect(dropped.events.text == "partial")390        if case .network = dropped.error {} else { Issue.record("expected a network error, got \(String(describing: dropped.error))") }391        #expect(dropped.events.stop == nil, "no stop event is invented for an interrupted stream")392393        let early = await collect(provider(StubServer().sse(head)).stream(request))394        #expect(early.error == .network("The response stream ended before the message was complete."))395396        let canceled = await collect(provider(StubServer().enqueue(.init(failure: .cancelled))).stream(request))397        #expect(canceled.error == .canceled)398    }399400    @Test func unknownEventsAndBlockTypesAreIgnored() async {401        let transcript = anthropicSSE([402            ("message_start", #"{"type":"message_start","message":{"usage":{"input_tokens":3,"output_tokens":1}}}"#),403            ("future_event", #"{"type":"future_event","payload":{"x":1}}"#),404            ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"server_tool_use","id":"srvtoolu_fixture","name":"web_search","input":{}}}"#),405            ("content_block_delta", #"{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"q\":1}"}}"#),406            ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),407            ("content_block_start", #"{"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}"#),408            ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"ok"}}"#),409            ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"future_delta","x":1}}"#),410            ("content_block_stop", #"{"type":"content_block_stop","index":1}"#),411            ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"pause_turn"},"usage":{"output_tokens":5}}"#),412            ("message_stop", #"{"type":"message_stop"}"#),413        ])414        let (events, error) = await collect(provider(StubServer().sse(transcript)).stream(LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")])))415        #expect(error == nil && events.text == "ok" && events.toolUses.isEmpty && events.stop == .other)416    }417418    @Test func listsModelsAcrossPagesAndMergesRegistryPrices() async throws {419        let server = StubServer()420            .json(200, #"{"data":[{"type":"model","id":"claude-haiku-5-5","display_name":"Claude Haiku 5.5","created_at":"2026-01-01T00:00:00Z","max_input_tokens":1000000,"max_tokens":128000,"capabilities":{"image_input":{"supported":true},"structured_outputs":{"supported":true}}}],"has_more":true,"first_id":"claude-haiku-5-5","last_id":"claude-haiku-5-5"}"#)421            .json(200, #"{"data":[{"type":"model","id":"claude-haiku-4-5-20251001","display_name":"Claude Haiku 4.5"},{"type":"model","id":"claude-future-9","display_name":"Claude Future 9","max_input_tokens":5,"capabilities":{"image_input":{"supported":false},"structured_outputs":{"supported":false}}}],"has_more":false,"last_id":"claude-future-9"}"#)422        let models = try await provider(server).listModels()423        #expect(models.map(\.id) == ["claude-haiku-5-5", "claude-haiku-4-5-20251001", "claude-future-9"])424        #expect(server.requests[0].path == "/v1/models" && server.requests[1].query?.contains("after_id=claude-haiku-5-5") == true)425        #expect(models[0].inputPricePerMTok == 0.10 && models[0].outputPricePerMTok == 0.50 && models[0].contextWindow == 1_000_000)426        #expect(models[0].supportsVision && models[0].supportsStructuredOutput && models[0].source == "api")427        #expect(models[1].inputPricePerMTok == 1 && models[1].contextWindow == 200_000 && models[1].maxOutputTokens == 64_000, "dated snapshot ids resolve to the registry entry")428        #expect(models[2].inputPricePerMTok == nil && models[2].outputPricePerMTok == nil, "an unknown model has no invented price")429        #expect(models[2].contextWindow == 5 && !models[2].supportsVision)430        let (failed, _) = (try? await provider(StubServer().json(401, #"{"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}"#)).listModels(), 0)431        #expect(failed == nil)432    }433434    @Test func theAPIKeyNeverAppearsInErrorsOrDescriptions() async {435        let echo = #"{"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key: \#(fixtureKey) (also Authorization: Bearer abcdefghijklmnopqrstuvwxyz012345)"}}"#436        let p = provider(StubServer().json(401, echo))437        let (_, error) = await collect(p.stream(LLMRequest(model: "claude-haiku-5-5", messages: [.user("x")])))438        let rendered = "\(String(describing: error)) \(error?.errorDescription ?? "") \(error?.localizedDescription ?? "")"439        #expect(!rendered.contains(fixtureKey) && !rendered.contains("abcdefghijklmnopqrstuvwxyz012345"))440        #expect(rendered.contains(Redactor.placeholder))441        var dumped = ""442        dump(p, to: &dumped)443        #expect(!"\(p) \(String(reflecting: p)) \(dumped)".contains(fixtureKey))444        #expect(Redactor.shared.redact("key=\(fixtureKey)").contains(fixtureKey) == false, "the key is registered with the shared redactor at init")445    }446}447
Files main
Menu

Theme