SPB Git

Repository Public

RemoteBackend.swift 10.4 KB 171 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Services/RemoteInference/Sources/RemoteBackend.swift3// Description: Remote inference backend: streams /chat/completions from a trusted OpenAI-compatible node into Core generation events, and /embeddings.4import Foundation5import LocalChatCore6import LocalChatNetworking78public struct RemoteBackend: InferenceBackend {9    public let kind: BackendKind = .remote10    public var isAvailable: Bool { true }11    public var unavailableReason: String? { nil }1213    private let resolveNode: @Sendable (UUID) async -> RemoteNode?14    private let apiKey: @Sendable (RemoteNode) -> String?15    private let http: HTTPClient1617    /// - Parameters:18    ///   - node: looks a node up by id (typically `RemoteNodeRegistry.node(id:)`).19    ///   - apiKey: reads the node's key (the app reads it from the Keychain under `node.keychainAccount`).20    public init(node: @escaping @Sendable (UUID) async -> RemoteNode?, apiKey: @escaping @Sendable (RemoteNode) -> String?, http: HTTPClient = HTTPClient()) {21        resolveNode = node; self.apiKey = apiKey; self.http = http22    }2324    public init(registry: RemoteNodeRegistry, apiKey: @escaping @Sendable (RemoteNode) -> String?, http: HTTPClient = HTTPClient()) {25        self.init(node: { await registry.node(id: $0) }, apiKey: apiKey, http: http)26    }2728    public func load(_ model: ModelDescriptor, modelsRoot: URL, progress: @escaping @Sendable (LoadProgress) -> Void) async throws -> any ModelRuntime {29        let started = Date()30        progress(LoadProgress(stage: .preparing))31        guard let nodeID = model.remoteNodeID, let remoteName = RemoteNodeClient.remoteModelName(fromID: model.id) else { throw InferenceError.modelNotFound(model.name) }32        guard let node = await resolveNode(nodeID) else { throw InferenceError.backendUnavailable("The remote node of “\(model.name)” is no longer configured.") }33        guard node.isEnabled else { throw InferenceError.backendUnavailable("The remote node “\(node.name)” is disabled.") }34        let client = RemoteNodeClient(node: node, apiKey: apiKey(node), http: http)35        do { _ = try client.baseURL() } catch { throw InferenceError.backendUnavailable(RemoteNodeClient.describe(error)) }36        progress(LoadProgress(stage: .ready, fraction: 1))37        // Nothing is loaded on this device; the "load time" is the time this call took.38        return RemoteRuntime(client: client, remoteModel: remoteName, info: RuntimeInfo(39            modelID: model.id, backend: .remote, loadTime: Date().timeIntervalSince(started), weightsMemoryBytes: nil,40            contextLength: model.contextLength, capabilities: model.capabilities, chatTemplateAvailable: true))41    }42}4344public final class RemoteRuntime: ModelRuntime {45    public let info: RuntimeInfo46    public let client: RemoteNodeClient47    public let remoteModel: String4849    public init(client: RemoteNodeClient, remoteModel: String, info: RuntimeInfo) { self.client = client; self.remoteModel = remoteModel; self.info = info }5051    public func generate(_ request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error> {52        AsyncThrowingStream { continuation in53            let task = Task {54                do { try await self.run(request, continuation); continuation.finish() }55                catch is CancellationError { continuation.finish(throwing: InferenceError.cancelled) }56                catch let error as InferenceError { continuation.finish(throwing: error) }57                catch { continuation.finish(throwing: Task.isCancelled || (error as? URLError)?.code == .cancelled ? InferenceError.cancelled : InferenceError.generationFailed(RemoteNodeClient.describe(error))) }58            }59            continuation.onTermination = { _ in task.cancel() }   // cancels the URLSession transfer60        }61    }6263    private func run(_ request: GenerationRequest, _ continuation: AsyncThrowingStream<GenerationEvent, Error>.Continuation) async throws {64        let url: URL65        do { url = try client.endpoint("chat/completions") } catch { throw InferenceError.backendUnavailable(RemoteNodeClient.describe(error)) }66        let body = try JSONEncoder().encode(OpenAI.ChatRequest(model: remoteModel, request: request, stream: true))67        let started = Date()68        let (bytes, response) = try await client.http.stream("POST", url, body: body, bearer: client.apiKey)6970        guard (200..<300).contains(response.statusCode) else {71            var data = Data()72            for try await byte in bytes { data.append(byte); if data.count > 65_536 { break } }73            let message = HTTPClient.errorMessage(data) ?? "HTTP \(response.statusCode)"74            switch response.statusCode {75            case 401, 403: throw InferenceError.backendUnavailable("“\(client.node.name)” refused the API key: \(message)")76            case 404: throw InferenceError.modelNotFound("\(remoteModel) on “\(client.node.name)” (\(message))")77            case 429: throw InferenceError.backendUnavailable("“\(client.node.name)” is rate limiting this device: \(message)")78            default: throw InferenceError.generationFailed("“\(client.node.name)” answered HTTP \(response.statusCode): \(message)")79            }80        }8182        var parser = SSEParser(); var calls = ToolCallAccumulator()83        var usage: OpenAI.Usage?; var server: OpenAI.ServerMetrics?; var finish: String?84        var firstToken: Date?; var lastToken: Date?; var done = false85        let decoder = JSONDecoder()8687        func handle(_ event: SSEEvent) throws {88            if event.isDone { done = true; return }89            guard let data = event.data.data(using: .utf8), let chunk = try? decoder.decode(OpenAI.ChatChunk.self, from: data) else { return }90            if let error = chunk.error { throw InferenceError.generationFailed("“\(client.node.name)”: \(error.message)") }91            if let u = chunk.usage { usage = u }92            if let m = chunk.localChat { server = m }93            for choice in chunk.choices where choice.index == 0 {94                if let piece = choice.delta.reasoningContent, !piece.isEmpty {95                    if firstToken == nil { firstToken = Date() }; lastToken = Date()96                    continuation.yield(.reasoning(piece))97                }98                if let piece = choice.delta.content, !piece.isEmpty {99                    if firstToken == nil { firstToken = Date() }; lastToken = Date()100                    continuation.yield(.text(piece))101                }102                if let fragments = choice.delta.toolCalls, !fragments.isEmpty {103                    if firstToken == nil { firstToken = Date() }; lastToken = Date()104                    calls.add(fragments)105                }106                if let reason = choice.finishReason { finish = reason }107            }108        }109110        // a non-streaming JSON answer (a server that ignores "stream") is handled too111        let isJSON = (response.value(forHTTPHeaderField: "Content-Type") ?? "").lowercased().contains("application/json")112        var raw = Data(); var line: [UInt8] = []; line.reserveCapacity(4096)113        for try await byte in bytes {114            if isJSON { raw.append(byte); continue }115            line.append(byte)116            if byte == 0x0A {117                for event in parser.feed(bytes: line) { try handle(event) }118                line.removeAll(keepingCapacity: true)119                if done { break }120            }121        }122        try Task.checkCancellation()123        if isJSON {124            guard let full = try? decoder.decode(OpenAI.ChatResponse.self, from: raw), let choice = full.choices.first else {125                throw InferenceError.generationFailed("“\(client.node.name)” sent an answer that is not a chat completion.")126            }127            firstToken = Date(); lastToken = firstToken128            if let r = choice.message.reasoningContent, !r.isEmpty { continuation.yield(.reasoning(r)) }129            if let text = choice.message.content?.plainText, !text.isEmpty { continuation.yield(.text(text)) }130            calls.add(choice.message.toolCalls ?? []); usage = full.usage; server = full.localChat; finish = choice.finishReason131        } else if !done {132            if !line.isEmpty { for event in parser.feed(bytes: line) { try handle(event) } }133            for event in parser.finish() { try handle(event) }134        }135136        let invocations = calls.finish()137        for call in invocations { continuation.yield(.toolCall(call)) }138139        var metrics = GenerationMetrics(usage: usage, server: server, finishReason: invocations.isEmpty ? finish : "tool_calls")140        let now = Date()141        // Client-side measurements (they include the network): time to first streamed piece and total wall time.142        metrics.timeToFirstToken = firstToken.map { $0.timeIntervalSince(started) }143        metrics.totalTime = now.timeIntervalSince(started)144        // Generation time: the node's own figure when it reports one, else the measured span between first and last streamed piece.145        if metrics.generationTime == nil, !isJSON, let first = firstToken, let last = lastToken, last > first, metrics.completionTokens != nil {146            metrics.generationTime = last.timeIntervalSince(first)147        }148        // memory figures of a remote node describe that node, not this device149        metrics.peakMemoryBytes = nil; metrics.activeMemoryBytes = nil150        continuation.yield(.finished(metrics))151    }152153    public func countTokens(_ text: String) async -> Int? { nil }154155    public func embed(_ texts: [String]) async throws -> [[Float]] {156        guard !texts.isEmpty else { return [] }157        do {158            let response = try await client.http.json(OpenAI.EmbeddingResponse.self, "POST", try client.endpoint("embeddings"),159                                                      encoding: OpenAI.EmbeddingRequest(model: remoteModel, input: texts), bearer: client.apiKey, timeout: 120)160            let vectors = response.data.sorted { $0.index < $1.index }.map(\.embedding)161            guard vectors.count == texts.count else { throw InferenceError.generationFailed("The node returned \(vectors.count) embeddings for \(texts.count) inputs.") }162            return vectors163        } catch let error as InferenceError { throw error }164        catch let error as HTTPClientError where error.statusCode == 401 || error.statusCode == 403 {165            throw InferenceError.backendUnavailable("“\(client.node.name)” refused the API key.")166        } catch { throw InferenceError.generationFailed(RemoteNodeClient.describe(error)) }167    }168169    public func unload() async {}170}171
Files main
Menu

Theme