Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Services/RemoteInference/Sources/RemoteBackend.swift3// Description: Remote inference backend: streams /chat/completions from a trusted OpenAI-compatible node into Core generation events, and /embeddings.4import Foundation5import LocalChatCore6import LocalChatNetworking78public struct RemoteBackend: InferenceBackend {9 public let kind: BackendKind = .remote10 public var isAvailable: Bool { true }11 public var unavailableReason: String? { nil }1213 private let resolveNode: @Sendable (UUID) async -> RemoteNode?14 private let apiKey: @Sendable (RemoteNode) -> String?15 private let http: HTTPClient1617 /// - Parameters:18 /// - node: looks a node up by id (typically `RemoteNodeRegistry.node(id:)`).19 /// - apiKey: reads the node's key (the app reads it from the Keychain under `node.keychainAccount`).20 public init(node: @escaping @Sendable (UUID) async -> RemoteNode?, apiKey: @escaping @Sendable (RemoteNode) -> String?, http: HTTPClient = HTTPClient()) {21 resolveNode = node; self.apiKey = apiKey; self.http = http22 }2324 public init(registry: RemoteNodeRegistry, apiKey: @escaping @Sendable (RemoteNode) -> String?, http: HTTPClient = HTTPClient()) {25 self.init(node: { await registry.node(id: $0) }, apiKey: apiKey, http: http)26 }2728 public func load(_ model: ModelDescriptor, modelsRoot: URL, progress: @escaping @Sendable (LoadProgress) -> Void) async throws -> any ModelRuntime {29 let started = Date()30 progress(LoadProgress(stage: .preparing))31 guard let nodeID = model.remoteNodeID, let remoteName = RemoteNodeClient.remoteModelName(fromID: model.id) else { throw InferenceError.modelNotFound(model.name) }32 guard let node = await resolveNode(nodeID) else { throw InferenceError.backendUnavailable("The remote node of “\(model.name)” is no longer configured.") }33 guard node.isEnabled else { throw InferenceError.backendUnavailable("The remote node “\(node.name)” is disabled.") }34 let client = RemoteNodeClient(node: node, apiKey: apiKey(node), http: http)35 do { _ = try client.baseURL() } catch { throw InferenceError.backendUnavailable(RemoteNodeClient.describe(error)) }36 progress(LoadProgress(stage: .ready, fraction: 1))37 // Nothing is loaded on this device; the "load time" is the time this call took.38 return RemoteRuntime(client: client, remoteModel: remoteName, info: RuntimeInfo(39 modelID: model.id, backend: .remote, loadTime: Date().timeIntervalSince(started), weightsMemoryBytes: nil,40 contextLength: model.contextLength, capabilities: model.capabilities, chatTemplateAvailable: true))41 }42}4344public final class RemoteRuntime: ModelRuntime {45 public let info: RuntimeInfo46 public let client: RemoteNodeClient47 public let remoteModel: String4849 public init(client: RemoteNodeClient, remoteModel: String, info: RuntimeInfo) { self.client = client; self.remoteModel = remoteModel; self.info = info }5051 public func generate(_ request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error> {52 AsyncThrowingStream { continuation in53 let task = Task {54 do { try await self.run(request, continuation); continuation.finish() }55 catch is CancellationError { continuation.finish(throwing: InferenceError.cancelled) }56 catch let error as InferenceError { continuation.finish(throwing: error) }57 catch { continuation.finish(throwing: Task.isCancelled || (error as? URLError)?.code == .cancelled ? InferenceError.cancelled : InferenceError.generationFailed(RemoteNodeClient.describe(error))) }58 }59 continuation.onTermination = { _ in task.cancel() } // cancels the URLSession transfer60 }61 }6263 private func run(_ request: GenerationRequest, _ continuation: AsyncThrowingStream<GenerationEvent, Error>.Continuation) async throws {64 let url: URL65 do { url = try client.endpoint("chat/completions") } catch { throw InferenceError.backendUnavailable(RemoteNodeClient.describe(error)) }66 let body = try JSONEncoder().encode(OpenAI.ChatRequest(model: remoteModel, request: request, stream: true))67 let started = Date()68 let (bytes, response) = try await client.http.stream("POST", url, body: body, bearer: client.apiKey)6970 guard (200..<300).contains(response.statusCode) else {71 var data = Data()72 for try await byte in bytes { data.append(byte); if data.count > 65_536 { break } }73 let message = HTTPClient.errorMessage(data) ?? "HTTP \(response.statusCode)"74 switch response.statusCode {75 case 401, 403: throw InferenceError.backendUnavailable("“\(client.node.name)” refused the API key: \(message)")76 case 404: throw InferenceError.modelNotFound("\(remoteModel) on “\(client.node.name)” (\(message))")77 case 429: throw InferenceError.backendUnavailable("“\(client.node.name)” is rate limiting this device: \(message)")78 default: throw InferenceError.generationFailed("“\(client.node.name)” answered HTTP \(response.statusCode): \(message)")79 }80 }8182 var parser = SSEParser(); var calls = ToolCallAccumulator()83 var usage: OpenAI.Usage?; var server: OpenAI.ServerMetrics?; var finish: String?84 var firstToken: Date?; var lastToken: Date?; var done = false85 let decoder = JSONDecoder()8687 func handle(_ event: SSEEvent) throws {88 if event.isDone { done = true; return }89 guard let data = event.data.data(using: .utf8), let chunk = try? decoder.decode(OpenAI.ChatChunk.self, from: data) else { return }90 if let error = chunk.error { throw InferenceError.generationFailed("“\(client.node.name)”: \(error.message)") }91 if let u = chunk.usage { usage = u }92 if let m = chunk.localChat { server = m }93 for choice in chunk.choices where choice.index == 0 {94 if let piece = choice.delta.reasoningContent, !piece.isEmpty {95 if firstToken == nil { firstToken = Date() }; lastToken = Date()96 continuation.yield(.reasoning(piece))97 }98 if let piece = choice.delta.content, !piece.isEmpty {99 if firstToken == nil { firstToken = Date() }; lastToken = Date()100 continuation.yield(.text(piece))101 }102 if let fragments = choice.delta.toolCalls, !fragments.isEmpty {103 if firstToken == nil { firstToken = Date() }; lastToken = Date()104 calls.add(fragments)105 }106 if let reason = choice.finishReason { finish = reason }107 }108 }109110 // a non-streaming JSON answer (a server that ignores "stream") is handled too111 let isJSON = (response.value(forHTTPHeaderField: "Content-Type") ?? "").lowercased().contains("application/json")112 var raw = Data(); var line: [UInt8] = []; line.reserveCapacity(4096)113 for try await byte in bytes {114 if isJSON { raw.append(byte); continue }115 line.append(byte)116 if byte == 0x0A {117 for event in parser.feed(bytes: line) { try handle(event) }118 line.removeAll(keepingCapacity: true)119 if done { break }120 }121 }122 try Task.checkCancellation()123 if isJSON {124 guard let full = try? decoder.decode(OpenAI.ChatResponse.self, from: raw), let choice = full.choices.first else {125 throw InferenceError.generationFailed("“\(client.node.name)” sent an answer that is not a chat completion.")126 }127 firstToken = Date(); lastToken = firstToken128 if let r = choice.message.reasoningContent, !r.isEmpty { continuation.yield(.reasoning(r)) }129 if let text = choice.message.content?.plainText, !text.isEmpty { continuation.yield(.text(text)) }130 calls.add(choice.message.toolCalls ?? []); usage = full.usage; server = full.localChat; finish = choice.finishReason131 } else if !done {132 if !line.isEmpty { for event in parser.feed(bytes: line) { try handle(event) } }133 for event in parser.finish() { try handle(event) }134 }135136 let invocations = calls.finish()137 for call in invocations { continuation.yield(.toolCall(call)) }138139 var metrics = GenerationMetrics(usage: usage, server: server, finishReason: invocations.isEmpty ? finish : "tool_calls")140 let now = Date()141 // Client-side measurements (they include the network): time to first streamed piece and total wall time.142 metrics.timeToFirstToken = firstToken.map { $0.timeIntervalSince(started) }143 metrics.totalTime = now.timeIntervalSince(started)144 // Generation time: the node's own figure when it reports one, else the measured span between first and last streamed piece.145 if metrics.generationTime == nil, !isJSON, let first = firstToken, let last = lastToken, last > first, metrics.completionTokens != nil {146 metrics.generationTime = last.timeIntervalSince(first)147 }148 // memory figures of a remote node describe that node, not this device149 metrics.peakMemoryBytes = nil; metrics.activeMemoryBytes = nil150 continuation.yield(.finished(metrics))151 }152153 public func countTokens(_ text: String) async -> Int? { nil }154155 public func embed(_ texts: [String]) async throws -> [[Float]] {156 guard !texts.isEmpty else { return [] }157 do {158 let response = try await client.http.json(OpenAI.EmbeddingResponse.self, "POST", try client.endpoint("embeddings"),159 encoding: OpenAI.EmbeddingRequest(model: remoteModel, input: texts), bearer: client.apiKey, timeout: 120)160 let vectors = response.data.sorted { $0.index < $1.index }.map(\.embedding)161 guard vectors.count == texts.count else { throw InferenceError.generationFailed("The node returned \(vectors.count) embeddings for \(texts.count) inputs.") }162 return vectors163 } catch let error as InferenceError { throw error }164 catch let error as HTTPClientError where error.statusCode == 401 || error.statusCode == 403 {165 throw InferenceError.backendUnavailable("“\(client.node.name)” refused the API key.")166 } catch { throw InferenceError.generationFailed(RemoteNodeClient.describe(error)) }167 }168169 public func unload() async {}170}171