Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Tests/AstraProvidersTests/AnthropicProviderTests.swift10// Module : AstraProvidersTests11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraProviders9293@Suite("Anthropic (Messages API)")94struct AnthropicProviderTests {95 func provider(_ transport: FixtureTransport, diagnostics: RecordingDiagnostics? = nil) -> AnthropicProvider {96 AnthropicProvider(transport: transport, apiKey: { "sk-ant-FIXTUREKEY0123456789abcdefFIXTURE" }, diagnostics: diagnostics)97 }9899 @Test func plainTextFromRecordedTranscript() async {100 let transport = FixtureTransport(.sse("anthropic_text.sse"))101 let diagnostics = RecordingDiagnostics()102 let c = await collect(provider(transport, diagnostics: diagnostics).stream(request(.anthropic, "claude-haiku-5-5", system: "You are terse.")))103 expectWellFormed(c)104 #expect(c.text == "ASTRA-OK")105 #expect(c.starts.first == MessageStart(responseID: "msg_01FIXTURE", reportedModel: "claude-haiku-5-5"))106 #expect(c.completions == [.endOfTurn])107 #expect(c.outcome.usage == TokenUsage(inputTokens: 30, outputTokens: 10, cachedInputTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0))108 #expect(diagnostics.records.first?.requestID == "req_fixture")109 }110111 @Test func headersAndBody() async {112 let transport = FixtureTransport(.sse("anthropic_text.sse"))113 let req = GenerationRequest(model: ModelRef(provider: .anthropic, modelID: "claude-haiku-5-5"), system: "Be brief.", messages: [.user("Look", images: [tinyPNG])],114 parameters: GenerationParameters(temperature: 0.2, topP: 0.9, maxOutputTokens: 64, reasoning: .medium, stop: ["END"]), tools: [weatherTool])115 _ = await collect(provider(transport).stream(req))116 let r = transport.requests[0]117 #expect(r.url?.absoluteString == "https://api.anthropic.com/v1/messages")118 #expect(r.value(forHTTPHeaderField: "x-api-key")?.hasPrefix("sk-ant-") == true)119 #expect(r.value(forHTTPHeaderField: "anthropic-version") == "2026-10-21")120 #expect(r.value(forHTTPHeaderField: "Authorization") == nil)121 #expect(r.url?.query == nil)122 let body = transport.body()123 #expect(body["model"] == "claude-haiku-5-5")124 #expect(body["stream"] == true)125 #expect(body["max_tokens"] == 64)126 // API version 2026-10-21 rejects sampling parameters on every model: they are never sent.127 #expect(body["temperature"] == nil)128 #expect(body["top_p"] == nil)129 #expect(body["stop_sequences"] == ["END"])130 #expect(body["thinking"] == ["type": "adaptive", "display": "summarized"])131 #expect(body["output_config"] == ["effort": "medium"])132 // A short system prompt is below the cache minimum: no cache_control.133 #expect(body["system"] == [["type": "text", "text": "Be brief."]])134 #expect(body["tools"]?[0] == ["name": "get_weather", "description": "Get the current weather for a city.", "input_schema": weatherTool.parameters, "eager_input_streaming": true])135 #expect(body["messages"]?[0]?["content"]?[0] == ["type": "image", "source": ["type": "base64", "media_type": "image/png", "data": .string(tinyPNG.data.base64EncodedString())]])136 #expect(body["messages"]?[0]?["content"]?[1] == ["type": "text", "text": "Look"])137 }138139 @Test func longSystemPromptIsCached() {140 let long = String(repeating: "You answer questions about the ORIX product family with care. ", count: 60) // about 1,000 tokens141 var body = JSONValue.object(AnthropicProvider.body(request(.anthropic, "claude-haiku-5-5", system: long)))142 #expect(body["system"]?[0]?["cache_control"] == ["type": "ephemeral"])143 // Haiku 4.5 needs 4,096 tokens before it stores anything: marking a shorter prefix would only cost the write premium.144 body = JSONValue.object(AnthropicProvider.body(request(.anthropic, "claude-haiku-4-5", system: long)))145 #expect(body["system"]?[0]?["cache_control"] == nil)146 }147148 @Test func thinkingFollowsWhatEachModelAccepts() {149 func body(_ model: String, _ effort: ReasoningEffort?, max: Int? = 100) -> [String: JSONValue] {150 AnthropicProvider.body(request(.anthropic, model, parameters: GenerationParameters(maxOutputTokens: max, reasoning: effort)))151 }152 // Thinking on by default: ask for the summary so the pause is visible.153 #expect(body("claude-haiku-5-5", nil)["thinking"] == ["type": "adaptive", "display": "summarized"])154 #expect(body("claude-haiku-5-5", nil)["output_config"] == nil)155 #expect(body("claude-haiku-5-5", .off)["thinking"] == ["type": "disabled"])156 // Sonnet 5.5 cannot disable thinking; between_tools is its off switch.157 #expect(body("claude-sonnet-5-5", .off)["thinking"] == ["type": "between_tools"])158 // Opus 5.5 and Fable are always on: the lowest effort is the closest setting.159 #expect(body("claude-opus-5-5", .off)["thinking"] == ["type": "adaptive", "display": "summarized"])160 #expect(body("claude-opus-5-5", .off)["output_config"] == ["effort": "low"])161 #expect(body("claude-fable-5-2", .high)["output_config"] == ["effort": "high"])162 // Opus 4.8: off unless asked.163 #expect(body("claude-opus-4-8", nil)["thinking"] == nil)164 #expect(body("claude-opus-4-8", .low)["thinking"] == ["type": "adaptive", "display": "summarized"])165 // Sonnet 4.6 summarises by default: no display field.166 #expect(body("claude-sonnet-4-6", .low)["thinking"] == ["type": "adaptive"])167 // Haiku 4.5: manual budget, below max_tokens, and no effort parameter.168 let h45 = body("claude-haiku-4-5-20251001", .low)169 #expect(h45["thinking"] == ["type": "enabled", "budget_tokens": 1024])170 #expect(h45["max_tokens"] == 1124)171 #expect(h45["output_config"] == nil)172 #expect(body("claude-haiku-4-5", .off)["thinking"] == nil)173 // No max given: a bounded default, never the model's full 128K.174 #expect(body("claude-haiku-5-5", nil, max: nil)["max_tokens"] == 16_000)175 #expect(body("claude-unknown-1", nil, max: nil)["max_tokens"] == 4096)176 }177178 @Test func toolCycleMergesResultsIntoOneUserTurnAndRolesAlternate() {179 let messages: [PromptMessage] = [180 .user("Weather in Paris and Rome?"),181 .assistant("Checking both.", toolCalls: [ToolCall(id: "toolu_1", name: "get_weather", arguments: ["city": "Paris"]), ToolCall(id: "toolu_2", name: "get_weather", arguments: ["city": "Rome"])]),182 .tool("18 C", callID: "toolu_1", name: "get_weather"),183 .tool("no such city", callID: "toolu_2", name: "get_weather", isError: true),184 .user("And tomorrow?"),185 ]186 let wire = AnthropicProvider.messages(messages)187 #expect(wire.map { $0["role"]?.stringValue } == ["user", "assistant", "user"])188 #expect(wire[1]["content"]?.arrayValue?.map { $0["type"]?.stringValue } == ["text", "tool_use", "tool_use"])189 #expect(wire[1]["content"]?[1] == ["type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": ["city": "Paris"]])190 #expect(wire[2]["content"] == [191 ["type": "tool_result", "tool_use_id": "toolu_1", "content": "18 C"],192 ["type": "tool_result", "tool_use_id": "toolu_2", "content": "no such city", "is_error": true],193 ["type": "text", "text": "And tomorrow?"],194 ])195 // Nothing resembling a thinking block is ever replayed.196 #expect(!JSONValue.array(wire).serialized().contains("thinking"))197 }198199 @Test func toolCallWithFragmentedArguments() async {200 let c = await collect(provider(FixtureTransport(.sse("anthropic_tool.sse"))).stream(request(.anthropic, "claude-haiku-5-5", tools: [weatherTool])))201 expectWellFormed(c)202 #expect(c.completions == [.toolCalls])203 #expect(c.outcome.toolCalls == [ToolCall(id: "toolu_01FIXTURE", name: "get_weather", arguments: ["city": "Paris"])])204 #expect(c.deltas(for: "toolu_01FIXTURE") == ["{\"city\"", ": \"Pari", "s\"}"])205 }206207 @Test func thinkingSummaryThenText() async {208 let c = await collect(provider(FixtureTransport(.sse("anthropic_thinking.sse"))).stream(request(.anthropic, "claude-haiku-5-5", parameters: GenerationParameters(reasoning: .medium))))209 expectWellFormed(c)210 #expect(c.reasoning.hasPrefix("I'm multiplying 17 by 23"))211 #expect(c.text == "391")212 #expect(!c.reasoning.contains("SIGNATURE")) // signatures are integrity data, never content213 #expect(c.outcome.usage?.reasoningTokens == 30)214 #expect(c.outcome.usage?.outputTokens == 33)215 }216217 @Test func thinkingThenTextThenToolCall() async {218 let c = await collect(provider(FixtureTransport(.sse("anthropic_thinking_tool.sse"))).stream(request(.anthropic, "claude-haiku-5-5", tools: [weatherTool])))219 expectWellFormed(c)220 #expect(!c.reasoning.isEmpty)221 #expect(c.outcome.toolCalls.first?.arguments == ["city": "Paris"])222 #expect(c.completions == [.toolCalls])223 }224225 @Test func cacheUsageIsNormalisedToTheWholePrompt() async {226 let c = await collect(provider(FixtureTransport(.sse("anthropic_cache_usage.sse"))).stream(request(.anthropic, "claude-haiku-5-5")))227 expectWellFormed(c)228 // input_tokens 30 (uncached) + 5,400 read + 1,200 written = 6,630 prompt tokens.229 #expect(c.outcome.usage?.inputTokens == 6630)230 #expect(c.outcome.usage?.cachedInputTokens == 5400)231 #expect(c.outcome.usage?.cacheWriteTokens == 1200)232 #expect(c.outcome.usage?.outputTokens == 10)233 }234235 @Test func outputOnlyUsageDeltaKeepsTheInputTotal() throws {236 var decoder = AnthropicStreamDecoder(modelID: "m")237 _ = try decoder.consume(SSEEvent(event: "message_start", data: #"{"type":"message_start","message":{"id":"msg_1","model":"m","usage":{"input_tokens":25,"cache_read_input_tokens":100,"output_tokens":1}}}"#))238 let events = try decoder.consume(SSEEvent(event: "message_delta", data: #"{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":15}}"#))239 #expect(events == [.usage(TokenUsage(inputTokens: 125, outputTokens: 15, cachedInputTokens: 100))])240 }241242 @Test func maxTokens() async {243 let c = await collect(provider(FixtureTransport(.sse("anthropic_max_tokens.sse"))).stream(request(.anthropic, "claude-haiku-5-5")))244 expectWellFormed(c)245 #expect(c.completions == [.maxTokens])246 }247248 @Test func stopReasons() {249 #expect(AnthropicStreamDecoder.stopReason("end_turn") == .endOfTurn)250 #expect(AnthropicStreamDecoder.stopReason("tool_use") == .toolCalls)251 #expect(AnthropicStreamDecoder.stopReason("stop_sequence") == .stopSequence)252 #expect(AnthropicStreamDecoder.stopReason("refusal") == .contentFilter)253 #expect(AnthropicStreamDecoder.stopReason("model_context_window_exceeded") == .maxTokens)254 #expect(AnthropicStreamDecoder.stopReason("something_new") == .unknown)255 }256257 @Test func midStreamOverloadedError() async {258 let c = await collect(provider(FixtureTransport(.sse("anthropic_midstream_error.sse"))).stream(request(.anthropic, "claude-haiku-5-5")))259 #expect(c.text == "ASTRA-OK")260 #expect(c.completions.isEmpty)261 #expect(c.error == .overloaded("Overloaded"))262 }263264 @Test func connectionDroppedMidAnswer() async {265 let c = await collect(provider(FixtureTransport(.sse("anthropic_truncated.sse"))).stream(request(.anthropic, "claude-haiku-5-5")))266 guard case .network = c.error else { Issue.record("expected a network error, got \(String(describing: c.error))"); return }267 #expect(c.completions.isEmpty)268 }269270 @Test func unknownEventsAndInvalidToolJSON() throws {271 var decoder = AnthropicStreamDecoder(modelID: "m")272 var events: [StreamEvent] = []273 for (name, data) in [274 ("message_start", #"{"type":"message_start","message":{"id":"msg_1","model":"m","usage":{"input_tokens":5,"output_tokens":1}}}"#),275 ("brand_new_event", #"{"type":"brand_new_event","value":1}"#),276 ("content_block_start", #"{"type":"content_block_start","index":0,"content_block":{"type":"redacted_thinking","data":"opaque"}}"#),277 ("content_block_stop", #"{"type":"content_block_stop","index":0}"#),278 ("content_block_start", #"{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_x","name":"get_weather","input":{}}}"#),279 ("content_block_delta", #"{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"city\": \"Par"}}"#),280 ("message_delta", #"{"type":"message_delta","delta":{"stop_reason":"max_tokens"},"usage":{"output_tokens":9}}"#),281 ("message_stop", #"{"type":"message_stop"}"#),282 ] { events += try decoder.consume(SSEEvent(event: name, data: data)) }283 // The call was cut off by max_tokens: its text is kept verbatim under _raw instead of being lost.284 #expect(events.contains(.toolCallCompleted(ToolCall(id: "toolu_x", name: "get_weather", arguments: ["_raw": "{\"city\": \"Par"]))))285 #expect(events.last == .messageCompleted(.maxTokens))286 }287288 @Test func rejectedThinkingIsRemovedAndRetried() async {289 let transport = FixtureTransport(.json(400, #"{"type":"error","error":{"type":"invalid_request_error","message":"This model does not support the effort parameter."},"request_id":"req_1"}"#),290 .json(400, #"{"type":"error","error":{"type":"invalid_request_error","message":"thinking.type.adaptive is not supported for this model"},"request_id":"req_2"}"#),291 .sse("anthropic_text.sse"))292 let c = await collect(provider(transport).stream(request(.anthropic, "claude-next-1", parameters: GenerationParameters(reasoning: .high))))293 expectWellFormed(c)294 #expect(transport.requests.count == 3)295 #expect(transport.body(0)["output_config"] != nil)296 #expect(transport.body(1)["output_config"] == nil)297 #expect(transport.body(1)["thinking"] != nil)298 #expect(transport.body(2)["thinking"] == nil)299 }300301 @Test func recordedErrorBodies() async {302 var c = await collect(provider(FixtureTransport(.file(401, "anthropic_error_401.json"))).stream(request(.anthropic, "claude-haiku-5-5")))303 #expect(c.error == .unauthorized("invalid x-api-key"))304 c = await collect(provider(FixtureTransport(.file(404, "anthropic_error_404.json"))).stream(request(.anthropic, "claude-nonexistent-9")))305 #expect(c.error == .modelNotFound("claude-nonexistent-9"))306 c = await collect(provider(FixtureTransport(.json(400, #"{"type":"error","error":{"type":"invalid_request_error","message":"prompt is too long: 1203984 tokens > 1000000 maximum"},"request_id":"req_1"}"#))).stream(request(.anthropic, "claude-haiku-5-5")))307 guard case .contextTooLong = c.error else { Issue.record("expected contextTooLong, got \(String(describing: c.error))"); return }308 }309310 @Test func modelList() async throws {311 let transport = FixtureTransport(.file(200, "anthropic_models.json"))312 let models = try await provider(transport).listModels()313 #expect(transport.requests[0].url?.absoluteString == "https://api.anthropic.com/v1/models?limit=1000")314 #expect(transport.requests[0].value(forHTTPHeaderField: "anthropic-version") == "2026-10-21")315 #expect(models.map(\.ref.modelID) == ["claude-fable-5-2", "claude-haiku-5-5", "claude-sonnet-5-5", "claude-haiku-4-5-20251001", "claude-future-9"])316 let haiku = try #require(models.first { $0.ref.modelID == "claude-haiku-5-5" })317 #expect(haiku.displayName == "Claude Haiku 5.5")318 #expect(haiku.contextLength == 1_000_000)319 #expect(haiku.maxOutputTokens == 128_000)320 #expect(haiku.pricing == Pricing(inputPerMTok: 0.10, outputPerMTok: 0.50, cachedInputPerMTok: 0.01, cacheWritePerMTok: 0.125, asOf: "2026-11-21", sourceURL: CuratedCatalog.anthropicPricingURL))321 guard case .documentation = haiku.source else { Issue.record("curated entries must cite documentation"); return }322 // The dated snapshot resolves to the curated Haiku 4.5 entry, with the limits the API reports.323 let h45 = try #require(models.first { $0.ref.modelID == "claude-haiku-4-5-20251001" })324 #expect(h45.pricing?.inputPerMTok == 1)325 #expect(h45.contextLength == 200_000)326 // A model the table does not know: only what the API reports, and no price.327 let future = try #require(models.first { $0.ref.modelID == "claude-future-9" })328 #expect(future.capabilities == [.text, .vision, .reasoning])329 #expect(future.pricing == nil)330 #expect(future.contextLength == 500_000)331 guard case .providerAPI = future.source else { Issue.record("uncurated entries must cite the provider API"); return }332 }333334 @Test func curatedLookupDoesNotConfuseNeighbours() {335 #expect(CuratedCatalog.anthropic("claude-opus-5")?.id == "claude-opus-5")336 #expect(CuratedCatalog.anthropic("claude-opus-5-5")?.id == "claude-opus-5-5")337 #expect(CuratedCatalog.anthropic("claude-opus-4-5-20251101")?.id == "claude-opus-4-5")338 #expect(CuratedCatalog.anthropic("claude-opus-5-9") == nil)339 for entry in CuratedCatalog.anthropic {340 guard case .documentation(let asOf, _)? = CuratedCatalog.info(for: ModelRef(provider: .anthropic, modelID: entry.id))?.source else { Issue.record("\(entry.id) has no source"); continue }341 #expect(asOf == CuratedCatalog.verifiedOn)342 }343 }344}345