Repository Public
1import Foundation2import Testing3import AIAssistant45/// Records every request; never touches the network.6final class MockTransport: HTTPTransport, @unchecked Sendable {7 struct Failure: Error, CustomStringConvertible { let description: String }89 private let lock = NSLock()10 private var recorded: [HTTPRequest] = []11 private let response: HTTPResponse12 private let failure: Failure?13 private let streamLinesOverride: [String]?1415 init(status: Int = 200, headers: [String: String] = [:], json: String = "{}", failure: String? = nil, streamLines: [String]? = nil) {16 self.response = HTTPResponse(statusCode: status, headers: headers, body: Data(json.utf8))17 self.failure = failure.map(Failure.init(description:))18 self.streamLinesOverride = streamLines19 }2021 var requests: [HTTPRequest] { lock.withLock { recorded } }2223 func send(_ request: HTTPRequest) async throws -> HTTPResponse {24 lock.withLock { recorded.append(request) }25 if let failure { throw failure }26 return response27 }2829 func streamLines(_ request: HTTPRequest) async throws -> HTTPStreamResponse {30 lock.withLock { recorded.append(request) }31 if let failure { throw failure }32 let lines = streamLinesOverride ?? String(decoding: response.body, as: UTF8.self).components(separatedBy: "\n")33 let stream = AsyncThrowingStream<String, any Error> { continuation in34 for line in lines { continuation.yield(line) }35 continuation.finish()36 }37 return HTTPStreamResponse(statusCode: response.statusCode, headers: response.headers, lines: stream)38 }39}4041private let fakeKey = "sk-ant-" + "api03-KEYfakeKEYfake1234567890abcdWXYZ"42private let anthropicOK = """43 {"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-5-5",44 "content":[{"type":"thinking","thinking":"","signature":"s"},{"type":"text","text":"Use:\\n```sh\\nls -la\\n```"},{"type":"text","text":"\\nDone."}],45 "stop_reason":"end_turn","usage":{"input_tokens":12,"output_tokens":7}}46 """47private let openAIOK = """48 {"id":"c1","model":"gpt-x","choices":[{"index":0,"message":{"role":"assistant","content":"hello"},"finish_reason":"stop"}],49 "usage":{"prompt_tokens":5,"completion_tokens":2}}50 """5152private func makeApproved(_ provider: AIProviderConfig, question: String = "what is this?", selected: String? = nil,53 mode: AIMode = .explain, policy: NetworkPolicy = NetworkPolicy(), maxOutputTokens: Int = 16_000) throws -> ApprovedPreview {54 let preview = try ContextPreview.make(mode: mode, context: AIContext(question: question, selectedText: selected, workingDirectory: "/tmp", host: "local", shell: "zsh"),55 provider: provider, policy: policy, maxOutputTokens: maxOutputTokens)56 return ConsentGate().approve(preview)57}5859private func store(_ id: String, _ key: String?) throws -> InMemorySecretStore {60 let store = InMemorySecretStore()61 if let key { try store.setSecret(SecretValue(key), for: id) }62 return store63}6465private func jsonObject(_ data: Data) throws -> [String: Any] {66 try #require(try JSONSerialization.jsonObject(with: data) as? [String: Any])67}6869@Suite struct AIClientRequestTests {70 @Test func anthropicMessagesEncoding() async throws {71 let provider = AIProviderConfig(id: "a", kind: .anthropic, model: "claude-fable-5-2")72 let transport = MockTransport(headers: ["Request-Id": "req_123"], json: anthropicOK)73 let approved = try makeApproved(provider, selected: "token=abcFAKE123456", maxOutputTokens: 2048)74 let client = AIClient(transport: transport, secrets: try store("a", fakeKey))7576 let response = try await client.send(approved)7778 #expect(transport.requests.count == 1)79 let request = try #require(transport.requests.first)80 #expect(request.url.absoluteString == "https://api.anthropic.com/v1/messages")81 #expect(request.method == "POST")82 #expect(request.headers == ["x-api-key": fakeKey, "anthropic-version": "2023-06-01", "content-type": "application/json"])83 let body = try jsonObject(request.body)84 #expect(Set(body.keys) == ["model", "max_tokens", "system", "messages"])85 #expect(body["model"] as? String == "claude-fable-5-2")86 #expect(body["max_tokens"] as? Int == 2048)87 #expect(body["system"] as? String == AIMode.explain.systemPrompt)88 let messages = try #require(body["messages"] as? [[String: Any]])89 #expect(messages.count == 1)90 #expect(messages[0]["role"] as? String == "user")91 #expect(messages[0]["content"] as? String == approved.preview.payloadText)92 // What was previewed is what was sent.93 #expect(request.body.count == approved.preview.byteCount)94 let raw = String(decoding: request.body, as: UTF8.self)95 #expect(!raw.contains("abcFAKE123456"))96 #expect(!raw.contains(fakeKey))97 #expect(!String(describing: request).contains(fakeKey))98 #expect(!String(reflecting: request).contains(fakeKey))99100 #expect(response.text == "Use:\n```sh\nls -la\n```\nDone.")101 #expect(response.model == "claude-sonnet-5-5")102 #expect(response.stopReason == "end_turn")103 #expect(response.usage == AIUsage(inputTokens: 12, outputTokens: 7))104 #expect(response.requestID == "req_123")105 #expect(!response.isTruncated)106 #expect(response.proposedCommands().map(\.text) == ["ls -la"])107 }108109 @Test func openAIChatCompletionsEncoding() async throws {110 let provider = AIProviderConfig(id: "o", kind: .openAI, model: "some-model")111 let transport = MockTransport(json: openAIOK)112 let approved = try makeApproved(provider, mode: .draftCommand)113 let response = try await AIClient(transport: transport, secrets: try store("o", "sk-FAKEopenaiKEY1234567890")).send(approved)114115 let request = try #require(transport.requests.first)116 #expect(request.url.absoluteString == "https://api.openai.com/v1/chat/completions")117 #expect(request.headers == ["authorization": "Bearer sk-FAKEopenaiKEY1234567890", "content-type": "application/json"])118 let body = try jsonObject(request.body)119 #expect(Set(body.keys) == ["model", "messages", "max_completion_tokens"])120 #expect(body["model"] as? String == "some-model")121 let messages = try #require(body["messages"] as? [[String: String]])122 #expect(messages == [123 ["role": "system", "content": AIMode.draftCommand.systemPrompt],124 ["role": "user", "content": approved.preview.payloadText],125 ])126 #expect(request.body.count == approved.preview.byteCount)127 #expect(response.text == "hello")128 #expect(response.stopReason == "stop")129 #expect(response.usage == AIUsage(inputTokens: 5, outputTokens: 2))130 }131132 @Test func compatibleServerWorksWithoutKey() async throws {133 let provider = AIProviderConfig(id: "l", kind: .openAICompatible, model: "qwen3:latest")134 let transport = MockTransport(json: openAIOK)135 let approved = try makeApproved(provider, policy: NetworkPolicy(localOnly: true))136 let client = AIClient(transport: transport, secrets: InMemorySecretStore(), policy: NetworkPolicy(localOnly: true))137 _ = try await client.send(approved)138139 let request = try #require(transport.requests.first)140 #expect(request.url.absoluteString == "http://localhost:11434/v1/chat/completions")141 #expect(request.headers == ["content-type": "application/json"])142 let body = try jsonObject(request.body)143 #expect(Set(body.keys) == ["model", "messages", "max_tokens"])144 #expect(body["model"] as? String == "qwen3:latest")145 }146147 @Test func compatibleServerSendsKeyWhenPresent() async throws {148 let provider = AIProviderConfig(id: "l", kind: .openAICompatible, model: "m")149 let transport = MockTransport(json: openAIOK)150 _ = try await AIClient(transport: transport, secrets: try store("l", "local-FAKE-key")).send(try makeApproved(provider))151 #expect(transport.requests.first?.headers["authorization"] == "Bearer local-FAKE-key")152 }153}154155@Suite struct AIClientRefusalTests {156 @Test func missingKeyMakesNoNetworkCall() async throws {157 for kind in [AIProviderKind.anthropic, .openAI] {158 let provider = AIProviderConfig(id: "p", kind: kind, model: "m")159 let transport = MockTransport(json: anthropicOK)160 let client = AIClient(transport: transport, secrets: InMemorySecretStore())161 await #expect(throws: AIClientError.missingAPIKey(providerID: "p")) { try await client.send(try makeApproved(provider)) }162 await #expect(throws: AIClientError.missingAPIKey(providerID: "p")) {163 for try await _ in client.stream(try makeApproved(provider)) {}164 }165 #expect(transport.requests.isEmpty)166 }167 }168169 @Test func emptyKeyCountsAsMissing() async throws {170 let provider = AIProviderConfig(id: "p", kind: .anthropic)171 let transport = MockTransport(json: anthropicOK)172 await #expect(throws: AIClientError.missingAPIKey(providerID: "p")) {173 try await AIClient(transport: transport, secrets: try store("p", "")).send(try makeApproved(provider))174 }175 #expect(transport.requests.isEmpty)176 }177178 @Test func localOnlyClientRefusesRemotePreviewWithoutIO() async throws {179 // The preview was built (and approved) under a permissive policy; the client policy still wins.180 let provider = AIProviderConfig(id: "a", kind: .anthropic)181 let approved = try makeApproved(provider)182 let transport = MockTransport(json: anthropicOK)183 let client = AIClient(transport: transport, secrets: try store("a", fakeKey), policy: NetworkPolicy(localOnly: true))184 let expected = AIClientError.policyViolation(.remoteDestinationRefused(host: "api.anthropic.com"))185 await #expect(throws: expected) { try await client.send(approved) }186 await #expect(throws: expected) { for try await _ in client.stream(approved) {} }187 #expect(transport.requests.isEmpty)188 }189190 @Test func cleartextRemoteRefusedWithoutIO() async throws {191 let provider = AIProviderConfig(id: "r", kind: .openAICompatible, baseURL: URL(string: "http://192.168.2.50:11434")!, model: "m")192 let approved = try makeApproved(provider, policy: NetworkPolicy(allowsInsecureRemoteHTTP: true))193 let transport = MockTransport(json: openAIOK)194 await #expect(throws: AIClientError.policyViolation(.insecureTransport(host: "192.168.2.50"))) {195 try await AIClient(transport: transport, secrets: InMemorySecretStore()).send(approved)196 }197 #expect(transport.requests.isEmpty)198 }199}200201@Suite struct AIClientErrorMappingTests {202 private func failure(status: Int, headers: [String: String] = [:], json: String, kind: AIProviderKind = .anthropic) async throws -> AIAPIError {203 let provider = AIProviderConfig(id: "a", kind: kind, model: "m")204 let transport = MockTransport(status: status, headers: headers, json: json)205 let client = AIClient(transport: transport, secrets: try store("a", fakeKey))206 do {207 _ = try await client.send(try makeApproved(provider))208 } catch let AIClientError.api(error) {209 #expect(!String(describing: error).contains(fakeKey))210 return error211 }212 Issue.record("expected an API error")213 throw CancellationError()214 }215216 @Test func authenticationErrorDoesNotLeakTheKey() async throws {217 let error = try await failure(status: 401, headers: ["request-id": "req_9"], json: """218 {"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key: \(fakeKey)"}}219 """)220 #expect(error.status == 401)221 #expect(error.kind == .authentication)222 #expect(error.type == "authentication_error")223 #expect(error.requestID == "req_9")224 #expect(!error.message.contains(fakeKey))225 #expect(error.message.contains("[REDACTED:"))226 #expect(!error.isRetryable)227 }228229 @Test func rateLimitCarriesRetryAfter() async throws {230 let error = try await failure(status: 429, headers: ["Retry-After": "12"], json: """231 {"type":"error","error":{"type":"rate_limit_error","message":"slow down"}}232 """)233 #expect(error.kind == .rateLimited)234 #expect(error.retryAfter == 12)235 #expect(error.message == "slow down")236 #expect(error.isRetryable)237 }238239 @Test(arguments: [(400, AIAPIErrorKind.invalidRequest), (403, .permission), (404, .notFound), (500, .server), (502, .server), (529, .overloaded), (418, .other)])240 func statusMapping(status: Int, kind: AIAPIErrorKind) async throws {241 let error = try await failure(status: status, json: "<html>gateway</html>")242 #expect(error.status == status)243 #expect(error.kind == kind)244 #expect(error.type == nil)245 #expect(!error.message.isEmpty)246 }247248 @Test func openAIAndOllamaErrorShapes() async throws {249 let openAI = try await failure(status: 401, json: """250 {"error":{"message":"Incorrect API key provided: \(fakeKey)","type":"invalid_request_error","code":"invalid_api_key"}}251 """, kind: .openAI)252 #expect(openAI.type == "invalid_request_error")253 #expect(!openAI.message.contains(fakeKey))254 let ollama = try await failure(status: 404, json: "{\"error\":\"model 'x' not found\"}", kind: .openAICompatible)255 #expect(ollama.kind == .notFound)256 #expect(ollama.message == "model 'x' not found")257 }258259 @Test func transportFailureIsSanitised() async throws {260 let provider = AIProviderConfig(id: "a", kind: .anthropic)261 let transport = MockTransport(failure: "connection reset while sending x-api-key \(fakeKey)")262 let client = AIClient(transport: transport, secrets: try store("a", fakeKey))263 do {264 _ = try await client.send(try makeApproved(provider))265 Issue.record("expected a transport error")266 } catch let AIClientError.transport(message) {267 #expect(message.contains("connection reset"))268 #expect(!message.contains(fakeKey))269 }270 }271272 @Test func refusalAndMalformedBodies() async throws {273 let provider = AIProviderConfig(id: "a", kind: .anthropic)274 let secrets = try store("a", fakeKey)275 let refusal = MockTransport(json: """276 {"content":[],"model":"m","stop_reason":"refusal","stop_details":{"type":"refusal","category":"cyber"}}277 """)278 await #expect(throws: AIClientError.refused(category: "cyber")) {279 try await AIClient(transport: refusal, secrets: secrets).send(try makeApproved(provider))280 }281 await #expect(throws: AIClientError.decoding("the provider response is not in the expected format")) {282 try await AIClient(transport: MockTransport(json: "not json"), secrets: secrets).send(try makeApproved(provider))283 }284 let truncated = MockTransport(json: "{\"content\":[{\"type\":\"text\",\"text\":\"par\"}],\"stop_reason\":\"max_tokens\"}")285 #expect(try await AIClient(transport: truncated, secrets: secrets).send(try makeApproved(provider)).isTruncated)286 }287}288289@Suite struct AIClientStreamingTests {290 private func collect(_ stream: AsyncThrowingStream<String, any Error>) async throws -> [String] {291 var out: [String] = []292 for try await piece in stream { out.append(piece) }293 return out294 }295296 @Test func anthropicSSE() async throws {297 let provider = AIProviderConfig(id: "a", kind: .anthropic)298 let transport = MockTransport(streamLines: [299 "event: message_start",300 "data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\"}}",301 "",302 "event: content_block_delta",303 "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"hmm\"}}",304 "data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"Hel\"}}",305 "event: ping",306 "data: {\"type\":\"ping\"}",307 "data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"lo\"}}",308 "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}",309 "data: {\"type\":\"message_stop\"}",310 "data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"ignored\"}}",311 ])312 let approved = try makeApproved(provider)313 let pieces = try await collect(AIClient(transport: transport, secrets: try store("a", fakeKey)).stream(approved))314 #expect(pieces == ["Hel", "lo"])315316 let request = try #require(transport.requests.first)317 #expect(transport.requests.count == 1)318 #expect(request.headers["x-api-key"] == fakeKey)319 #expect(request.headers["accept"] == "text/event-stream")320 let body = try jsonObject(request.body)321 #expect(body["stream"] as? Bool == true)322 #expect(Set(body.keys) == ["model", "max_tokens", "system", "messages", "stream"])323 }324325 @Test func openAISSE() async throws {326 let provider = AIProviderConfig(id: "l", kind: .openAICompatible, model: "m")327 let transport = MockTransport(streamLines: [328 "data: {\"choices\":[{\"delta\":{\"role\":\"assistant\"}}]}",329 "data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}",330 "data: {\"choices\":[{\"delta\":{\"content\":\"b\"}}]}",331 "data: {\"choices\":[]}",332 "data: [DONE]",333 ])334 let pieces = try await collect(AIClient(transport: transport, secrets: InMemorySecretStore()).stream(try makeApproved(provider)))335 #expect(pieces == ["a", "b"])336 #expect(try jsonObject(try #require(transport.requests.first).body)["stream"] as? Bool == true)337 }338339 @Test func streamErrors() async throws {340 let provider = AIProviderConfig(id: "a", kind: .anthropic)341 let secrets = try store("a", fakeKey)342 let http = MockTransport(status: 429, headers: ["retry-after": "3"], json: "{\"type\":\"error\",\"error\":{\"type\":\"rate_limit_error\",\"message\":\"slow\"}}")343 do {344 _ = try await collect(AIClient(transport: http, secrets: secrets).stream(try makeApproved(provider)))345 Issue.record("expected an error")346 } catch let AIClientError.api(error) {347 #expect(error.status == 429 && error.kind == .rateLimited && error.retryAfter == 3 && error.message == "slow")348 }349350 let midStream = MockTransport(streamLines: [351 "data: {\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"x\"}}",352 "event: error",353 "data: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"Overloaded \(fakeKey)\"}}",354 ])355 var seen: [String] = []356 do {357 for try await piece in AIClient(transport: midStream, secrets: secrets).stream(try makeApproved(provider)) { seen.append(piece) }358 Issue.record("expected an error")359 } catch let AIClientError.api(error) {360 #expect(error.kind == .overloaded)361 #expect(!error.message.contains(fakeKey))362 }363 #expect(seen == ["x"])364365 let broken = MockTransport(failure: "offline")366 await #expect(throws: AIClientError.transport("offline")) {367 _ = try await collect(AIClient(transport: broken, secrets: secrets).stream(try makeApproved(provider)))368 }369 }370371 @Test func defaultStreamLinesFallsBackToSend() async throws {372 struct SendOnly: HTTPTransport {373 func send(_ request: HTTPRequest) async throws -> HTTPResponse {374 HTTPResponse(statusCode: 200, body: Data("data: {\"choices\":[{\"delta\":{\"content\":\"z\"}}]}\n\ndata: [DONE]\n".utf8))375 }376 }377 let provider = AIProviderConfig(id: "l", kind: .openAICompatible, model: "m")378 let pieces = try await collect(AIClient(transport: SendOnly(), secrets: InMemorySecretStore()).stream(try makeApproved(provider)))379 #expect(pieces == ["z"])380 }381}382