SPB Git

Repository Public

ClientTests.swift 20.0 KB 382 lines swift

1import Foundation2import Testing3import AIAssistant45/// Records every request; never touches the network.6final class MockTransport: HTTPTransport, @unchecked Sendable {7    struct Failure: Error, CustomStringConvertible { let description: String }89    private let lock = NSLock()10    private var recorded: [HTTPRequest] = []11    private let response: HTTPResponse12    private let failure: Failure?13    private let streamLinesOverride: [String]?1415    init(status: Int = 200, headers: [String: String] = [:], json: String = "{}", failure: String? = nil, streamLines: [String]? = nil) {16        self.response = HTTPResponse(statusCode: status, headers: headers, body: Data(json.utf8))17        self.failure = failure.map(Failure.init(description:))18        self.streamLinesOverride = streamLines19    }2021    var requests: [HTTPRequest] { lock.withLock { recorded } }2223    func send(_ request: HTTPRequest) async throws -> HTTPResponse {24        lock.withLock { recorded.append(request) }25        if let failure { throw failure }26        return response27    }2829    func streamLines(_ request: HTTPRequest) async throws -> HTTPStreamResponse {30        lock.withLock { recorded.append(request) }31        if let failure { throw failure }32        let lines = streamLinesOverride ?? String(decoding: response.body, as: UTF8.self).components(separatedBy: "\n")33        let stream = AsyncThrowingStream<String, any Error> { continuation in34            for line in lines { continuation.yield(line) }35            continuation.finish()36        }37        return HTTPStreamResponse(statusCode: response.statusCode, headers: response.headers, lines: stream)38    }39}4041private let fakeKey = "sk-ant-" + "api03-KEYfakeKEYfake1234567890abcdWXYZ"42private let anthropicOK = """43    {"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-5-5",44     "content":[{"type":"thinking","thinking":"","signature":"s"},{"type":"text","text":"Use:\\n```sh\\nls -la\\n```"},{"type":"text","text":"\\nDone."}],45     "stop_reason":"end_turn","usage":{"input_tokens":12,"output_tokens":7}}46    """47private let openAIOK = """48    {"id":"c1","model":"gpt-x","choices":[{"index":0,"message":{"role":"assistant","content":"hello"},"finish_reason":"stop"}],49     "usage":{"prompt_tokens":5,"completion_tokens":2}}50    """5152private func makeApproved(_ provider: AIProviderConfig, question: String = "what is this?", selected: String? = nil,53                          mode: AIMode = .explain, policy: NetworkPolicy = NetworkPolicy(), maxOutputTokens: Int = 16_000) throws -> ApprovedPreview {54    let preview = try ContextPreview.make(mode: mode, context: AIContext(question: question, selectedText: selected, workingDirectory: "/tmp", host: "local", shell: "zsh"),55                                          provider: provider, policy: policy, maxOutputTokens: maxOutputTokens)56    return ConsentGate().approve(preview)57}5859private func store(_ id: String, _ key: String?) throws -> InMemorySecretStore {60    let store = InMemorySecretStore()61    if let key { try store.setSecret(SecretValue(key), for: id) }62    return store63}6465private func jsonObject(_ data: Data) throws -> [String: Any] {66    try #require(try JSONSerialization.jsonObject(with: data) as? [String: Any])67}6869@Suite struct AIClientRequestTests {70    @Test func anthropicMessagesEncoding() async throws {71        let provider = AIProviderConfig(id: "a", kind: .anthropic, model: "claude-fable-5-2")72        let transport = MockTransport(headers: ["Request-Id": "req_123"], json: anthropicOK)73        let approved = try makeApproved(provider, selected: "token=abcFAKE123456", maxOutputTokens: 2048)74        let client = AIClient(transport: transport, secrets: try store("a", fakeKey))7576        let response = try await client.send(approved)7778        #expect(transport.requests.count == 1)79        let request = try #require(transport.requests.first)80        #expect(request.url.absoluteString == "https://api.anthropic.com/v1/messages")81        #expect(request.method == "POST")82        #expect(request.headers == ["x-api-key": fakeKey, "anthropic-version": "2023-06-01", "content-type": "application/json"])83        let body = try jsonObject(request.body)84        #expect(Set(body.keys) == ["model", "max_tokens", "system", "messages"])85        #expect(body["model"] as? String == "claude-fable-5-2")86        #expect(body["max_tokens"] as? Int == 2048)87        #expect(body["system"] as? String == AIMode.explain.systemPrompt)88        let messages = try #require(body["messages"] as? [[String: Any]])89        #expect(messages.count == 1)90        #expect(messages[0]["role"] as? String == "user")91        #expect(messages[0]["content"] as? String == approved.preview.payloadText)92        // What was previewed is what was sent.93        #expect(request.body.count == approved.preview.byteCount)94        let raw = String(decoding: request.body, as: UTF8.self)95        #expect(!raw.contains("abcFAKE123456"))96        #expect(!raw.contains(fakeKey))97        #expect(!String(describing: request).contains(fakeKey))98        #expect(!String(reflecting: request).contains(fakeKey))99100        #expect(response.text == "Use:\n```sh\nls -la\n```\nDone.")101        #expect(response.model == "claude-sonnet-5-5")102        #expect(response.stopReason == "end_turn")103        #expect(response.usage == AIUsage(inputTokens: 12, outputTokens: 7))104        #expect(response.requestID == "req_123")105        #expect(!response.isTruncated)106        #expect(response.proposedCommands().map(\.text) == ["ls -la"])107    }108109    @Test func openAIChatCompletionsEncoding() async throws {110        let provider = AIProviderConfig(id: "o", kind: .openAI, model: "some-model")111        let transport = MockTransport(json: openAIOK)112        let approved = try makeApproved(provider, mode: .draftCommand)113        let response = try await AIClient(transport: transport, secrets: try store("o", "sk-FAKEopenaiKEY1234567890")).send(approved)114115        let request = try #require(transport.requests.first)116        #expect(request.url.absoluteString == "https://api.openai.com/v1/chat/completions")117        #expect(request.headers == ["authorization": "Bearer sk-FAKEopenaiKEY1234567890", "content-type": "application/json"])118        let body = try jsonObject(request.body)119        #expect(Set(body.keys) == ["model", "messages", "max_completion_tokens"])120        #expect(body["model"] as? String == "some-model")121        let messages = try #require(body["messages"] as? [[String: String]])122        #expect(messages == [123            ["role": "system", "content": AIMode.draftCommand.systemPrompt],124            ["role": "user", "content": approved.preview.payloadText],125        ])126        #expect(request.body.count == approved.preview.byteCount)127        #expect(response.text == "hello")128        #expect(response.stopReason == "stop")129        #expect(response.usage == AIUsage(inputTokens: 5, outputTokens: 2))130    }131132    @Test func compatibleServerWorksWithoutKey() async throws {133        let provider = AIProviderConfig(id: "l", kind: .openAICompatible, model: "qwen3:latest")134        let transport = MockTransport(json: openAIOK)135        let approved = try makeApproved(provider, policy: NetworkPolicy(localOnly: true))136        let client = AIClient(transport: transport, secrets: InMemorySecretStore(), policy: NetworkPolicy(localOnly: true))137        _ = try await client.send(approved)138139        let request = try #require(transport.requests.first)140        #expect(request.url.absoluteString == "http://localhost:11434/v1/chat/completions")141        #expect(request.headers == ["content-type": "application/json"])142        let body = try jsonObject(request.body)143        #expect(Set(body.keys) == ["model", "messages", "max_tokens"])144        #expect(body["model"] as? String == "qwen3:latest")145    }146147    @Test func compatibleServerSendsKeyWhenPresent() async throws {148        let provider = AIProviderConfig(id: "l", kind: .openAICompatible, model: "m")149        let transport = MockTransport(json: openAIOK)150        _ = try await AIClient(transport: transport, secrets: try store("l", "local-FAKE-key")).send(try makeApproved(provider))151        #expect(transport.requests.first?.headers["authorization"] == "Bearer local-FAKE-key")152    }153}154155@Suite struct AIClientRefusalTests {156    @Test func missingKeyMakesNoNetworkCall() async throws {157        for kind in [AIProviderKind.anthropic, .openAI] {158            let provider = AIProviderConfig(id: "p", kind: kind, model: "m")159            let transport = MockTransport(json: anthropicOK)160            let client = AIClient(transport: transport, secrets: InMemorySecretStore())161            await #expect(throws: AIClientError.missingAPIKey(providerID: "p")) { try await client.send(try makeApproved(provider)) }162            await #expect(throws: AIClientError.missingAPIKey(providerID: "p")) {163                for try await _ in client.stream(try makeApproved(provider)) {}164            }165            #expect(transport.requests.isEmpty)166        }167    }168169    @Test func emptyKeyCountsAsMissing() async throws {170        let provider = AIProviderConfig(id: "p", kind: .anthropic)171        let transport = MockTransport(json: anthropicOK)172        await #expect(throws: AIClientError.missingAPIKey(providerID: "p")) {173            try await AIClient(transport: transport, secrets: try store("p", "")).send(try makeApproved(provider))174        }175        #expect(transport.requests.isEmpty)176    }177178    @Test func localOnlyClientRefusesRemotePreviewWithoutIO() async throws {179        // The preview was built (and approved) under a permissive policy; the client policy still wins.180        let provider = AIProviderConfig(id: "a", kind: .anthropic)181        let approved = try makeApproved(provider)182        let transport = MockTransport(json: anthropicOK)183        let client = AIClient(transport: transport, secrets: try store("a", fakeKey), policy: NetworkPolicy(localOnly: true))184        let expected = AIClientError.policyViolation(.remoteDestinationRefused(host: "api.anthropic.com"))185        await #expect(throws: expected) { try await client.send(approved) }186        await #expect(throws: expected) { for try await _ in client.stream(approved) {} }187        #expect(transport.requests.isEmpty)188    }189190    @Test func cleartextRemoteRefusedWithoutIO() async throws {191        let provider = AIProviderConfig(id: "r", kind: .openAICompatible, baseURL: URL(string: "http://192.168.2.50:11434")!, model: "m")192        let approved = try makeApproved(provider, policy: NetworkPolicy(allowsInsecureRemoteHTTP: true))193        let transport = MockTransport(json: openAIOK)194        await #expect(throws: AIClientError.policyViolation(.insecureTransport(host: "192.168.2.50"))) {195            try await AIClient(transport: transport, secrets: InMemorySecretStore()).send(approved)196        }197        #expect(transport.requests.isEmpty)198    }199}200201@Suite struct AIClientErrorMappingTests {202    private func failure(status: Int, headers: [String: String] = [:], json: String, kind: AIProviderKind = .anthropic) async throws -> AIAPIError {203        let provider = AIProviderConfig(id: "a", kind: kind, model: "m")204        let transport = MockTransport(status: status, headers: headers, json: json)205        let client = AIClient(transport: transport, secrets: try store("a", fakeKey))206        do {207            _ = try await client.send(try makeApproved(provider))208        } catch let AIClientError.api(error) {209            #expect(!String(describing: error).contains(fakeKey))210            return error211        }212        Issue.record("expected an API error")213        throw CancellationError()214    }215216    @Test func authenticationErrorDoesNotLeakTheKey() async throws {217        let error = try await failure(status: 401, headers: ["request-id": "req_9"], json: """218            {"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key: \(fakeKey)"}}219            """)220        #expect(error.status == 401)221        #expect(error.kind == .authentication)222        #expect(error.type == "authentication_error")223        #expect(error.requestID == "req_9")224        #expect(!error.message.contains(fakeKey))225        #expect(error.message.contains("[REDACTED:"))226        #expect(!error.isRetryable)227    }228229    @Test func rateLimitCarriesRetryAfter() async throws {230        let error = try await failure(status: 429, headers: ["Retry-After": "12"], json: """231            {"type":"error","error":{"type":"rate_limit_error","message":"slow down"}}232            """)233        #expect(error.kind == .rateLimited)234        #expect(error.retryAfter == 12)235        #expect(error.message == "slow down")236        #expect(error.isRetryable)237    }238239    @Test(arguments: [(400, AIAPIErrorKind.invalidRequest), (403, .permission), (404, .notFound), (500, .server), (502, .server), (529, .overloaded), (418, .other)])240    func statusMapping(status: Int, kind: AIAPIErrorKind) async throws {241        let error = try await failure(status: status, json: "<html>gateway</html>")242        #expect(error.status == status)243        #expect(error.kind == kind)244        #expect(error.type == nil)245        #expect(!error.message.isEmpty)246    }247248    @Test func openAIAndOllamaErrorShapes() async throws {249        let openAI = try await failure(status: 401, json: """250            {"error":{"message":"Incorrect API key provided: \(fakeKey)","type":"invalid_request_error","code":"invalid_api_key"}}251            """, kind: .openAI)252        #expect(openAI.type == "invalid_request_error")253        #expect(!openAI.message.contains(fakeKey))254        let ollama = try await failure(status: 404, json: "{\"error\":\"model 'x' not found\"}", kind: .openAICompatible)255        #expect(ollama.kind == .notFound)256        #expect(ollama.message == "model 'x' not found")257    }258259    @Test func transportFailureIsSanitised() async throws {260        let provider = AIProviderConfig(id: "a", kind: .anthropic)261        let transport = MockTransport(failure: "connection reset while sending x-api-key \(fakeKey)")262        let client = AIClient(transport: transport, secrets: try store("a", fakeKey))263        do {264            _ = try await client.send(try makeApproved(provider))265            Issue.record("expected a transport error")266        } catch let AIClientError.transport(message) {267            #expect(message.contains("connection reset"))268            #expect(!message.contains(fakeKey))269        }270    }271272    @Test func refusalAndMalformedBodies() async throws {273        let provider = AIProviderConfig(id: "a", kind: .anthropic)274        let secrets = try store("a", fakeKey)275        let refusal = MockTransport(json: """276            {"content":[],"model":"m","stop_reason":"refusal","stop_details":{"type":"refusal","category":"cyber"}}277            """)278        await #expect(throws: AIClientError.refused(category: "cyber")) {279            try await AIClient(transport: refusal, secrets: secrets).send(try makeApproved(provider))280        }281        await #expect(throws: AIClientError.decoding("the provider response is not in the expected format")) {282            try await AIClient(transport: MockTransport(json: "not json"), secrets: secrets).send(try makeApproved(provider))283        }284        let truncated = MockTransport(json: "{\"content\":[{\"type\":\"text\",\"text\":\"par\"}],\"stop_reason\":\"max_tokens\"}")285        #expect(try await AIClient(transport: truncated, secrets: secrets).send(try makeApproved(provider)).isTruncated)286    }287}288289@Suite struct AIClientStreamingTests {290    private func collect(_ stream: AsyncThrowingStream<String, any Error>) async throws -> [String] {291        var out: [String] = []292        for try await piece in stream { out.append(piece) }293        return out294    }295296    @Test func anthropicSSE() async throws {297        let provider = AIProviderConfig(id: "a", kind: .anthropic)298        let transport = MockTransport(streamLines: [299            "event: message_start",300            "data: {\"type\":\"message_start\",\"message\":{\"id\":\"m\"}}",301            "",302            "event: content_block_delta",303            "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"hmm\"}}",304            "data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"Hel\"}}",305            "event: ping",306            "data: {\"type\":\"ping\"}",307            "data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"lo\"}}",308            "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}",309            "data: {\"type\":\"message_stop\"}",310            "data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"ignored\"}}",311        ])312        let approved = try makeApproved(provider)313        let pieces = try await collect(AIClient(transport: transport, secrets: try store("a", fakeKey)).stream(approved))314        #expect(pieces == ["Hel", "lo"])315316        let request = try #require(transport.requests.first)317        #expect(transport.requests.count == 1)318        #expect(request.headers["x-api-key"] == fakeKey)319        #expect(request.headers["accept"] == "text/event-stream")320        let body = try jsonObject(request.body)321        #expect(body["stream"] as? Bool == true)322        #expect(Set(body.keys) == ["model", "max_tokens", "system", "messages", "stream"])323    }324325    @Test func openAISSE() async throws {326        let provider = AIProviderConfig(id: "l", kind: .openAICompatible, model: "m")327        let transport = MockTransport(streamLines: [328            "data: {\"choices\":[{\"delta\":{\"role\":\"assistant\"}}]}",329            "data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}",330            "data: {\"choices\":[{\"delta\":{\"content\":\"b\"}}]}",331            "data: {\"choices\":[]}",332            "data: [DONE]",333        ])334        let pieces = try await collect(AIClient(transport: transport, secrets: InMemorySecretStore()).stream(try makeApproved(provider)))335        #expect(pieces == ["a", "b"])336        #expect(try jsonObject(try #require(transport.requests.first).body)["stream"] as? Bool == true)337    }338339    @Test func streamErrors() async throws {340        let provider = AIProviderConfig(id: "a", kind: .anthropic)341        let secrets = try store("a", fakeKey)342        let http = MockTransport(status: 429, headers: ["retry-after": "3"], json: "{\"type\":\"error\",\"error\":{\"type\":\"rate_limit_error\",\"message\":\"slow\"}}")343        do {344            _ = try await collect(AIClient(transport: http, secrets: secrets).stream(try makeApproved(provider)))345            Issue.record("expected an error")346        } catch let AIClientError.api(error) {347            #expect(error.status == 429 && error.kind == .rateLimited && error.retryAfter == 3 && error.message == "slow")348        }349350        let midStream = MockTransport(streamLines: [351            "data: {\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"x\"}}",352            "event: error",353            "data: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"Overloaded \(fakeKey)\"}}",354        ])355        var seen: [String] = []356        do {357            for try await piece in AIClient(transport: midStream, secrets: secrets).stream(try makeApproved(provider)) { seen.append(piece) }358            Issue.record("expected an error")359        } catch let AIClientError.api(error) {360            #expect(error.kind == .overloaded)361            #expect(!error.message.contains(fakeKey))362        }363        #expect(seen == ["x"])364365        let broken = MockTransport(failure: "offline")366        await #expect(throws: AIClientError.transport("offline")) {367            _ = try await collect(AIClient(transport: broken, secrets: secrets).stream(try makeApproved(provider)))368        }369    }370371    @Test func defaultStreamLinesFallsBackToSend() async throws {372        struct SendOnly: HTTPTransport {373            func send(_ request: HTTPRequest) async throws -> HTTPResponse {374                HTTPResponse(statusCode: 200, body: Data("data: {\"choices\":[{\"delta\":{\"content\":\"z\"}}]}\n\ndata: [DONE]\n".utf8))375            }376        }377        let provider = AIProviderConfig(id: "l", kind: .openAICompatible, model: "m")378        let pieces = try await collect(AIClient(transport: SendOnly(), secrets: InMemorySecretStore()).stream(try makeApproved(provider)))379        #expect(pieces == ["z"])380    }381}382
Files main
Menu

Theme