SPB Git

Repository Public MIT

AnthropicProviderTests.swift 20.8 KB 345 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraProvidersTests/AnthropicProviderTests.swift10// Module           : AstraProvidersTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraProviders9293@Suite("Anthropic (Messages API)")94struct AnthropicProviderTests {95    func provider(_ transport: FixtureTransport, diagnostics: RecordingDiagnostics? = nil) -> AnthropicProvider {96        AnthropicProvider(transport: transport, apiKey: { "sk-ant-FIXTUREKEY0123456789abcdefFIXTURE" }, diagnostics: diagnostics)97    }9899    @Test func plainTextFromRecordedTranscript() async {100        let transport = FixtureTransport(.sse("anthropic_text.sse"))101        let diagnostics = RecordingDiagnostics()102        let c = await collect(provider(transport, diagnostics: diagnostics).stream(request(.anthropic, "claude-haiku-5-5", system: "You are terse.")))103        expectWellFormed(c)104        #expect(c.text == "ASTRA-OK")105        #expect(c.starts.first == MessageStart(responseID: "msg_01FIXTURE", reportedModel: "claude-haiku-5-5"))106        #expect(c.completions == [.endOfTurn])107        #expect(c.outcome.usage == TokenUsage(inputTokens: 30, outputTokens: 10, cachedInputTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0))108        #expect(diagnostics.records.first?.requestID == "req_fixture")109    }110111    @Test func headersAndBody() async {112        let transport = FixtureTransport(.sse("anthropic_text.sse"))113        let req = GenerationRequest(model: ModelRef(provider: .anthropic, modelID: "claude-haiku-5-5"), system: "Be brief.", messages: [.user("Look", images: [tinyPNG])],114                                    parameters: GenerationParameters(temperature: 0.2, topP: 0.9, maxOutputTokens: 64, reasoning: .medium, stop: ["END"]), tools: [weatherTool])115        _ = await collect(provider(transport).stream(req))116        let r = transport.requests[0]117        #expect(r.url?.absoluteString == "https://api.anthropic.com/v1/messages")118        #expect(r.value(forHTTPHeaderField: "x-api-key")?.hasPrefix("sk-ant-") == true)119        #expect(r.value(forHTTPHeaderField: "anthropic-version") == "2026-10-21")120        #expect(r.value(forHTTPHeaderField: "Authorization") == nil)121        #expect(r.url?.query == nil)122        let body = transport.body()123        #expect(body["model"] == "claude-haiku-5-5")124        #expect(body["stream"] == true)125        #expect(body["max_tokens"] == 64)126        // API version 2026-10-21 rejects sampling parameters on every model: they are never sent.127        #expect(body["temperature"] == nil)128        #expect(body["top_p"] == nil)129        #expect(body["stop_sequences"] == ["END"])130        #expect(body["thinking"] == ["type": "adaptive", "display": "summarized"])131        #expect(body["output_config"] == ["effort": "medium"])132        // A short system prompt is below the cache minimum: no cache_control.133        #expect(body["system"] == [["type": "text", "text": "Be brief."]])134        #expect(body["tools"]?[0] == ["name": "get_weather", "description": "Get the current weather for a city.", "input_schema": weatherTool.parameters, "eager_input_streaming": true])135        #expect(body["messages"]?[0]?["content"]?[0] == ["type": "image", "source": ["type": "base64", "media_type": "image/png", "data": .string(tinyPNG.data.base64EncodedString())]])136        #expect(body["messages"]?[0]?["content"]?[1] == ["type": "text", "text": "Look"])137    }138139    @Test func longSystemPromptIsCached() {140        let long = String(repeating: "You answer questions about the ORIX product family with care. ", count: 60)   // about 1,000 tokens141        var body = JSONValue.object(AnthropicProvider.body(request(.anthropic, "claude-haiku-5-5", system: long)))142        #expect(body["system"]?[0]?["cache_control"] == ["type": "ephemeral"])143        // Haiku 4.5 needs 4,096 tokens before it stores anything: marking a shorter prefix would only cost the write premium.144        body = JSONValue.object(AnthropicProvider.body(request(.anthropic, "claude-haiku-4-5", system: long)))145        #expect(body["system"]?[0]?["cache_control"] == nil)146    }147148    @Test func thinkingFollowsWhatEachModelAccepts() {149        func body(_ model: String, _ effort: ReasoningEffort?, max: Int? = 100) -> [String: JSONValue] {150            AnthropicProvider.body(request(.anthropic, model, parameters: GenerationParameters(maxOutputTokens: max, reasoning: effort)))151        }152        // Thinking on by default: ask for the summary so the pause is visible.153        #expect(body("claude-haiku-5-5", nil)["thinking"] == ["type": "adaptive", "display": "summarized"])154        #expect(body("claude-haiku-5-5", nil)["output_config"] == nil)155        #expect(body("claude-haiku-5-5", .off)["thinking"] == ["type": "disabled"])156        // Sonnet 5.5 cannot disable thinking; between_tools is its off switch.157        #expect(body("claude-sonnet-5-5", .off)["thinking"] == ["type": "between_tools"])158        // Opus 5.5 and Fable are always on: the lowest effort is the closest setting.159        #expect(body("claude-opus-5-5", .off)["thinking"] == ["type": "adaptive", "display": "summarized"])160        #expect(body("claude-opus-5-5", .off)["output_config"] == ["effort": "low"])161        #expect(body("claude-fable-5-2", .high)["output_config"] == ["effort": "high"])162        // Opus 4.8: off unless asked.163        #expect(body("claude-opus-4-8", nil)["thinking"] == nil)164        #expect(body("claude-opus-4-8", .low)["thinking"] == ["type": "adaptive", "display": "summarized"])165        // Sonnet 4.6 summarises by default: no display field.166        #expect(body("claude-sonnet-4-6", .low)["thinking"] == ["type": "adaptive"])167        // Haiku 4.5: manual budget, below max_tokens, and no effort parameter.168        let h45 = body("claude-haiku-4-5-20251001", .low)169        #expect(h45["thinking"] == ["type": "enabled", "budget_tokens": 1024])170        #expect(h45["max_tokens"] == 1124)171        #expect(h45["output_config"] == nil)172        #expect(body("claude-haiku-4-5", .off)["thinking"] == nil)173        // No max given: a bounded default, never the model's full 128K.174        #expect(body("claude-haiku-5-5", nil, max: nil)["max_tokens"] == 16_000)175        #expect(body("claude-unknown-1", nil, max: nil)["max_tokens"] == 4096)176    }177178    @Test func toolCycleMergesResultsIntoOneUserTurnAndRolesAlternate() {179        let messages: [PromptMessage] = [180            .user("Weather in Paris and Rome?"),181            .assistant("Checking both.", toolCalls: [ToolCall(id: "toolu_1", name: "get_weather", arguments: ["city": "Paris"]), ToolCall(id: "toolu_2", name: "get_weather", arguments: ["city": "Rome"])]),182            .tool("18 C", callID: "toolu_1", name: "get_weather"),183            .tool("no such city", callID: "toolu_2", name: "get_weather", isError: true),184            .user("And tomorrow?"),185        ]186        let wire = AnthropicProvider.messages(messages)187        #expect(wire.map { $0["role"]?.stringValue } == ["user", "assistant", "user"])188        #expect(wire[1]["content"]?.arrayValue?.map { $0["type"]?.stringValue } == ["text", "tool_use", "tool_use"])189        #expect(wire[1]["content"]?[1] == ["type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": ["city": "Paris"]])190        #expect(wire[2]["content"] == [191            ["type": "tool_result", "tool_use_id": "toolu_1", "content": "18 C"],192            ["type": "tool_result", "tool_use_id": "toolu_2", "content": "no such city", "is_error": true],193            ["type": "text", "text": "And tomorrow?"],194        ])195        // Nothing resembling a thinking block is ever replayed.196        #expect(!JSONValue.array(wire).serialized().contains("thinking"))197    }198199    @Test func toolCallWithFragmentedArguments() async {200        let c = await collect(provider(FixtureTransport(.sse("anthropic_tool.sse"))).stream(request(.anthropic, "claude-haiku-5-5", tools: [weatherTool])))201        expectWellFormed(c)202        #expect(c.completions == [.toolCalls])203        #expect(c.outcome.toolCalls == [ToolCall(id: "toolu_01FIXTURE", name: "get_weather", arguments: ["city": "Paris"])])204        #expect(c.deltas(for: "toolu_01FIXTURE") == ["{\"city\"", ": \"Pari", "s\"}"])205    }206207    @Test func thinkingSummaryThenText() async {208        let c = await collect(provider(FixtureTransport(.sse("anthropic_thinking.sse"))).stream(request(.anthropic, "claude-haiku-5-5", parameters: GenerationParameters(reasoning: .medium))))209        expectWellFormed(c)210        #expect(c.reasoning.hasPrefix("I'm multiplying 17 by 23"))211        #expect(c.text == "391")212        #expect(!c.reasoning.contains("SIGNATURE"))     // signatures are integrity data, never content213        #expect(c.outcome.usage?.reasoningTokens == 30)214        #expect(c.outcome.usage?.outputTokens == 33)215    }216217    @Test func thinkingThenTextThenToolCall() async {218        let c = await collect(provider(FixtureTransport(.sse("anthropic_thinking_tool.sse"))).stream(request(.anthropic, "claude-haiku-5-5", tools: [weatherTool])))219        expectWellFormed(c)220        #expect(!c.reasoning.isEmpty)221        #expect(c.outcome.toolCalls.first?.arguments == ["city": "Paris"])222        #expect(c.completions == [.toolCalls])223    }224225    @Test func cacheUsageIsNormalisedToTheWholePrompt() async {226        let c = await collect(provider(FixtureTransport(.sse("anthropic_cache_usage.sse"))).stream(request(.anthropic, "claude-haiku-5-5")))227        expectWellFormed(c)228        // input_tokens 30 (uncached) + 5,400 read + 1,200 written = 6,630 prompt tokens.229        #expect(c.outcome.usage?.inputTokens == 6630)230        #expect(c.outcome.usage?.cachedInputTokens == 5400)231        #expect(c.outcome.usage?.cacheWriteTokens == 1200)232        #expect(c.outcome.usage?.outputTokens == 10)233    }234235    @Test func outputOnlyUsageDeltaKeepsTheInputTotal() throws {236        var decoder = AnthropicStreamDecoder(modelID: "m")237        _ = try decoder.consume(SSEEvent(event: "message_start", data: #"{"type":"message_start","message":{"id":"msg_1","model":"m","usage":{"input_tokens":25,"cache_read_input_tokens":100,"output_tokens":1}}}"#))238        let events = try decoder.consume(SSEEvent(event: "message_delta", data: #"{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":15}}"#))239        #expect(events == [.usage(TokenUsage(inputTokens: 125, outputTokens: 15, cachedInputTokens: 100))])240    }241242    @Test func maxTokens() async {243        let c = await collect(provider(FixtureTransport(.sse("anthropic_max_tokens.sse"))).stream(request(.anthropic, "claude-haiku-5-5")))244        expectWellFormed(c)245        #expect(c.completions == [.maxTokens])246    }247248    @Test func stopReasons() {249        #expect(AnthropicStreamDecoder.stopReason("end_turn") == .endOfTurn)250        #expect(AnthropicStreamDecoder.stopReason("tool_use") == .toolCalls)251        #expect(AnthropicStreamDecoder.stopReason("stop_sequence") == .stopSequence)252        #expect(AnthropicStreamDecoder.stopReason("refusal") == .contentFilter)253        #expect(AnthropicStreamDecoder.stopReason("model_context_window_exceeded") == .maxTokens)254        #expect(AnthropicStreamDecoder.stopReason("something_new") == .unknown)255    }256257    @Test func midStreamOverloadedError() async {258        let c = await collect(provider(FixtureTransport(.sse("anthropic_midstream_error.sse"))).stream(request(.anthropic, "claude-haiku-5-5")))259        #expect(c.text == "ASTRA-OK")260        #expect(c.completions.isEmpty)261        #expect(c.error == .overloaded("Overloaded"))262    }263264    @Test func connectionDroppedMidAnswer() async {265        let c = await collect(provider(FixtureTransport(.sse("anthropic_truncated.sse"))).stream(request(.anthropic, "claude-haiku-5-5")))266        guard case .network = c.error else { Issue.record("expected a network error, got \(String(describing: c.error))"); return }267        #expect(c.completions.isEmpty)268    }269270    @Test func unknownEventsAndInvalidToolJSON() throws {271        var decoder = AnthropicStreamDecoder(modelID: "m")272        var events: [StreamEvent] = []273        for (name, data) in [274            ("message_start", #"{"type":"message_start","message":{"id":"msg_1","model":"m","usage":{"input_tokens":5,"output_tokens":1}}}"#),275            ("brand_new_event", #"{"type":"brand_new_event","value":1}"#),276            ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"redacted_thinking","data":"opaque"}}"#),277            ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),278            ("content_block_start", #"{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_x","name":"get_weather","input":{}}}"#),279            ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"city\": \"Par"}}"#),280            ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"max_tokens"},"usage":{"output_tokens":9}}"#),281            ("message_stop", #"{"type":"message_stop"}"#),282        ] { events += try decoder.consume(SSEEvent(event: name, data: data)) }283        // The call was cut off by max_tokens: its text is kept verbatim under _raw instead of being lost.284        #expect(events.contains(.toolCallCompleted(ToolCall(id: "toolu_x", name: "get_weather", arguments: ["_raw": "{\"city\": \"Par"]))))285        #expect(events.last == .messageCompleted(.maxTokens))286    }287288    @Test func rejectedThinkingIsRemovedAndRetried() async {289        let transport = FixtureTransport(.json(400, #"{"type":"error","error":{"type":"invalid_request_error","message":"This model does not support the effort parameter."},"request_id":"req_1"}"#),290                                         .json(400, #"{"type":"error","error":{"type":"invalid_request_error","message":"thinking.type.adaptive is not supported for this model"},"request_id":"req_2"}"#),291                                         .sse("anthropic_text.sse"))292        let c = await collect(provider(transport).stream(request(.anthropic, "claude-next-1", parameters: GenerationParameters(reasoning: .high))))293        expectWellFormed(c)294        #expect(transport.requests.count == 3)295        #expect(transport.body(0)["output_config"] != nil)296        #expect(transport.body(1)["output_config"] == nil)297        #expect(transport.body(1)["thinking"] != nil)298        #expect(transport.body(2)["thinking"] == nil)299    }300301    @Test func recordedErrorBodies() async {302        var c = await collect(provider(FixtureTransport(.file(401, "anthropic_error_401.json"))).stream(request(.anthropic, "claude-haiku-5-5")))303        #expect(c.error == .unauthorized("invalid x-api-key"))304        c = await collect(provider(FixtureTransport(.file(404, "anthropic_error_404.json"))).stream(request(.anthropic, "claude-nonexistent-9")))305        #expect(c.error == .modelNotFound("claude-nonexistent-9"))306        c = await collect(provider(FixtureTransport(.json(400, #"{"type":"error","error":{"type":"invalid_request_error","message":"prompt is too long: 1203984 tokens > 1000000 maximum"},"request_id":"req_1"}"#))).stream(request(.anthropic, "claude-haiku-5-5")))307        guard case .contextTooLong = c.error else { Issue.record("expected contextTooLong, got \(String(describing: c.error))"); return }308    }309310    @Test func modelList() async throws {311        let transport = FixtureTransport(.file(200, "anthropic_models.json"))312        let models = try await provider(transport).listModels()313        #expect(transport.requests[0].url?.absoluteString == "https://api.anthropic.com/v1/models?limit=1000")314        #expect(transport.requests[0].value(forHTTPHeaderField: "anthropic-version") == "2026-10-21")315        #expect(models.map(\.ref.modelID) == ["claude-fable-5-2", "claude-haiku-5-5", "claude-sonnet-5-5", "claude-haiku-4-5-20251001", "claude-future-9"])316        let haiku = try #require(models.first { $0.ref.modelID == "claude-haiku-5-5" })317        #expect(haiku.displayName == "Claude Haiku 5.5")318        #expect(haiku.contextLength == 1_000_000)319        #expect(haiku.maxOutputTokens == 128_000)320        #expect(haiku.pricing == Pricing(inputPerMTok: 0.10, outputPerMTok: 0.50, cachedInputPerMTok: 0.01, cacheWritePerMTok: 0.125, asOf: "2026-11-21", sourceURL: CuratedCatalog.anthropicPricingURL))321        guard case .documentation = haiku.source else { Issue.record("curated entries must cite documentation"); return }322        // The dated snapshot resolves to the curated Haiku 4.5 entry, with the limits the API reports.323        let h45 = try #require(models.first { $0.ref.modelID == "claude-haiku-4-5-20251001" })324        #expect(h45.pricing?.inputPerMTok == 1)325        #expect(h45.contextLength == 200_000)326        // A model the table does not know: only what the API reports, and no price.327        let future = try #require(models.first { $0.ref.modelID == "claude-future-9" })328        #expect(future.capabilities == [.text, .vision, .reasoning])329        #expect(future.pricing == nil)330        #expect(future.contextLength == 500_000)331        guard case .providerAPI = future.source else { Issue.record("uncurated entries must cite the provider API"); return }332    }333334    @Test func curatedLookupDoesNotConfuseNeighbours() {335        #expect(CuratedCatalog.anthropic("claude-opus-5")?.id == "claude-opus-5")336        #expect(CuratedCatalog.anthropic("claude-opus-5-5")?.id == "claude-opus-5-5")337        #expect(CuratedCatalog.anthropic("claude-opus-4-5-20251101")?.id == "claude-opus-4-5")338        #expect(CuratedCatalog.anthropic("claude-opus-5-9") == nil)339        for entry in CuratedCatalog.anthropic {340            guard case .documentation(let asOf, _)? = CuratedCatalog.info(for: ModelRef(provider: .anthropic, modelID: entry.id))?.source else { Issue.record("\(entry.id) has no source"); continue }341            #expect(asOf == CuratedCatalog.verifiedOn)342        }343    }344}345
Files main
Menu

Theme