Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatInference/Sources/MLXBackend.swift3// Description: MLX Swift backend: loads LLM / VLM / embedding models from a local directory and streams real generations with KV-cache reuse across turns.4import CoreImage5import Foundation6import LocalChatCore7import MLX8import MLXEmbedders9import MLXLLM10import MLXLMCommon11import MLXVLM12import Tokenizers1314/// Adapts swift-transformers tokenizers to the mlx-swift-lm protocol.15struct TransformersTokenizerLoader: MLXLMCommon.TokenizerLoader {16 struct Bridge: MLXLMCommon.Tokenizer {17 let upstream: any Tokenizers.Tokenizer18 func encode(text: String, addSpecialTokens: Bool) -> [Int] { upstream.encode(text: text, addSpecialTokens: addSpecialTokens) }19 func decode(tokenIds: [Int], skipSpecialTokens: Bool) -> String { upstream.decode(tokens: tokenIds, skipSpecialTokens: skipSpecialTokens) }20 func convertTokenToId(_ token: String) -> Int? { upstream.convertTokenToId(token) }21 func convertIdToToken(_ id: Int) -> String? { upstream.convertIdToToken(id) }22 var bosToken: String? { upstream.bosToken }23 var eosToken: String? { upstream.eosToken }24 var unknownToken: String? { upstream.unknownToken }25 func applyChatTemplate(messages: [[String: any Sendable]], tools: [[String: any Sendable]]?, additionalContext: [String: any Sendable]?) throws -> [Int] {26 do { return try upstream.applyChatTemplate(messages: messages, tools: tools, additionalContext: additionalContext) }27 catch Tokenizers.TokenizerError.missingChatTemplate { throw MLXLMCommon.TokenizerError.missingChatTemplate }28 }29 }30 func load(from directory: URL) async throws -> any MLXLMCommon.Tokenizer {31 Bridge(upstream: try await AutoTokenizer.from(modelFolder: directory))32 }33}3435public struct MLXBackend: InferenceBackend {36 public let kind: BackendKind = .mlx37 public init() {}3839 public var isAvailable: Bool {40 #if targetEnvironment(simulator)41 false42 #else43 true44 #endif45 }46 public var unavailableReason: String? {47 isAvailable ? nil : "MLX needs a real Apple GPU: it does not run in the iOS Simulator. Use a device, or route to a Mac."48 }4950 public func load(_ model: ModelDescriptor, modelsRoot: URL, progress: @escaping @Sendable (LocalChatCore.LoadProgress) -> Void) async throws -> any ModelRuntime {51 guard isAvailable else { throw InferenceError.backendUnavailable(unavailableReason ?? "MLX is unavailable.") }52 guard let relative = model.localPath else { throw InferenceError.modelNotFound(model.name) }53 let directory = relative.hasPrefix("/") ? URL(fileURLWithPath: relative) : modelsRoot.appendingPathComponent(relative)54 guard FileManager.default.fileExists(atPath: directory.appendingPathComponent("config.json").path) else {55 throw InferenceError.loadFailed("config.json is missing in \(directory.lastPathComponent).")56 }57 progress(LocalChatCore.LoadProgress(stage: .preparing))58 let start = Date()59 let handlers = LoadProgressHandlers.weights { p in progress(LocalChatCore.LoadProgress(stage: .loadingWeights, fraction: p.fractionCompleted)) }60 let loader = TransformersTokenizerLoader()61 let config = Self.readConfig(directory)6263 if model.isEmbeddingOnly {64 do {65 let container = try await EmbedderModelFactory.shared.loadContainer(from: directory, using: loader)66 let bytes = (try? await container.memory().total).map(Int64.init)67 progress(LocalChatCore.LoadProgress(stage: .ready, fraction: 1))68 return MLXEmbeddingRuntime(container: container, info: RuntimeInfo(69 modelID: model.id, backend: .mlx, loadTime: Date().timeIntervalSince(start), weightsMemoryBytes: bytes,70 contextLength: config.contextLength, capabilities: [.embeddings], chatTemplateAvailable: false))71 } catch { throw InferenceError.loadFailed(Self.describe(error)) }72 }7374 let container: ModelContainer75 do {76 if config.hasVision {77 do { container = try await VLMModelFactory.shared.loadContainer(from: directory, using: loader, progress: handlers) }78 catch { container = try await LLMModelFactory.shared.loadContainer(from: directory, using: loader, progress: handlers) }79 } else {80 container = try await LLMModelFactory.shared.loadContainer(from: directory, using: loader, progress: handlers)81 }82 } catch { throw InferenceError.loadFailed(Self.describe(error)) }8384 let bytes = (try? await container.memory().total).map(Int64.init)85 var capabilities = model.capabilities86 if config.hasVision { capabilities.insert(.vision) }87 let hasTemplate = (try? await container.applyChatTemplate(messages: [["role": "user", "content": "hi"]])) != nil88 progress(LocalChatCore.LoadProgress(stage: .ready, fraction: 1))89 return MLXRuntime(container: container, info: RuntimeInfo(90 modelID: model.id, backend: .mlx, loadTime: Date().timeIntervalSince(start), weightsMemoryBytes: bytes,91 contextLength: config.contextLength ?? model.contextLength, capabilities: capabilities, chatTemplateAvailable: hasTemplate))92 }9394 struct ConfigSummary { var hasVision = false; var contextLength: Int?; var modelType: String? }9596 static func readConfig(_ directory: URL) -> ConfigSummary {97 var out = ConfigSummary()98 if let data = try? Data(contentsOf: directory.appendingPathComponent("config.json")),99 let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] {100 out.hasVision = json["vision_config"] != nil || json["vision_tower"] != nil101 out.modelType = json["model_type"] as? String102 let text = json["text_config"] as? [String: Any]103 out.contextLength = (json["max_position_embeddings"] as? Int) ?? (text?["max_position_embeddings"] as? Int)104 }105 if out.contextLength == nil, let data = try? Data(contentsOf: directory.appendingPathComponent("tokenizer_config.json")),106 let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], let n = json["model_max_length"] as? Int, n < 10_000_000 {107 out.contextLength = n108 }109 return out110 }111112 static func describe(_ error: Error) -> String {113 let text = (error as? LocalizedError)?.errorDescription ?? String(describing: error)114 if text.contains("unsupportedModelType") || text.lowercased().contains("unsupported model type") {115 return "This architecture is not implemented in MLX Swift yet (\(text))."116 }117 return text118 }119}120121/// A loaded MLX chat model. One generation at a time per runtime (the engine serializes requests).122actor MLXRuntime: ModelRuntime {123 nonisolated let info: RuntimeInfo124 private var container: ModelContainer?125126 /// A live ChatSession and the transcript it has already consumed, so the next turn only evaluates new tokens.127 private struct Session { var chat: ChatSession; var transcript: [Fingerprint]; var system: String?; var toolsKey: String }128 private struct Fingerprint: Equatable {129 var role: PromptMessage.Role; var hash: Int130 init(_ m: PromptMessage) { role = m.role; hash = m.content.trimmingCharacters(in: .whitespacesAndNewlines).hashValue ^ m.images.count }131 init(role: PromptMessage.Role, text: String) { self.role = role; hash = text.trimmingCharacters(in: .whitespacesAndNewlines).hashValue }132 }133 private var sessions: [String: Session] = [:]134 private var sessionOrder: [String] = []135 private var lastCachedTokens = 0136 private let maxSessions = 2137138 init(container: ModelContainer, info: RuntimeInfo) { self.container = container; self.info = info }139140 nonisolated func generate(_ request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error> {141 AsyncThrowingStream { continuation in142 let task = Task { await self.run(request, continuation) }143 continuation.onTermination = { _ in task.cancel() }144 }145 }146147 private func run(_ request: GenerationRequest, _ continuation: AsyncThrowingStream<GenerationEvent, Error>.Continuation) async {148 guard let container else { continuation.finish(throwing: InferenceError.modelNotLoaded); return }149 guard info.chatTemplateAvailable else {150 continuation.finish(throwing: InferenceError.unsupported("This model has no chat template; it cannot be used for chat.")); return151 }152 let started = Date()153 var messages = request.messages154 let system = messages.first?.role == .system ? messages.removeFirst().content : nil155 guard !messages.isEmpty else { continuation.finish(throwing: InferenceError.generationFailed("Empty prompt.")); return }156157 let s = request.settings158 let parameters = GenerateParameters(159 maxTokens: s.maxTokens, maxKVSize: s.contextLength > 0 ? s.contextLength : nil, kvBits: s.kvBits,160 temperature: Float(s.temperature), topP: Float(s.topP), topK: s.topK, minP: Float(s.minP),161 repetitionPenalty: s.repetitionPenalty != 1 ? Float(s.repetitionPenalty) : nil, repetitionContextSize: s.repetitionContext,162 seed: s.seed)163 let toolSpecs: [ToolSpec]? = request.tools.isEmpty ? nil : request.tools.map { ($0.openAISpec.anyValue as? [String: any Sendable]) ?? [:] }164 let toolsKey = request.tools.map(\.name).joined(separator: ",")165 let context: [String: any Sendable] = ["enable_thinking": s.reasoningEnabled]166167 // Reuse the live session when this request only appends turns to what it already evaluated.168 var chat: ChatSession; var pending: [PromptMessage]; var reused = false169 let prints = messages.map(Fingerprint.init)170 if let key = request.sessionKey, let live = sessions[key], live.system == system, live.toolsKey == toolsKey,171 live.transcript.count < prints.count, Array(prints.prefix(live.transcript.count)) == live.transcript {172 chat = live.chat; pending = Array(messages.suffix(prints.count - live.transcript.count)); reused = true173 } else {174 // split at the last turn the model must answer: everything before it is history175 let cut = Self.pendingStart(messages)176 let history = messages.prefix(cut).map(Self.chatMessage)177 pending = Array(messages.suffix(from: cut))178 chat = ChatSession(container, instructions: system, history: history, generateParameters: parameters,179 processing: .init(resize: CGSize(width: 768, height: 768)), additionalContext: context, tools: toolSpecs)180 }181 chat.generateParameters = parameters182 chat.additionalContext = context183 chat.tools = toolSpecs184 if let key = request.sessionKey { sessions[key] = nil; sessionOrder.removeAll { $0 == key } }185186 MLX.Memory.peakMemory = 0187 var text = ""; var firstToken: Date?; var info: GenerateCompletionInfo?; var sawToolCall = false; var stopped = false188 let stream = chat.streamDetails(to: pending.map(Self.chatMessage))189 do {190 for try await event in stream {191 if Task.isCancelled { break }192 switch event {193 case .chunk(let piece):194 if firstToken == nil { firstToken = Date() }195 text += piece196 if let hit = request.stop.first(where: { !$0.isEmpty && text.contains($0) }), let range = text.range(of: hit) {197 let keep = String(text[..<range.lowerBound]); let already = text.count - piece.count198 if keep.count > already { continuation.yield(.text(String(keep.dropFirst(already)))) }199 text = keep; stopped = true200 } else { continuation.yield(.text(piece)) }201 case .reasoning(let piece):202 if firstToken == nil { firstToken = Date() }203 continuation.yield(.reasoning(piece))204 case .toolCall(let call):205 sawToolCall = true206 continuation.yield(.toolCall(ToolInvocation(id: call.id ?? UUID().uuidString, name: call.function.name,207 arguments: Self.jsonValue(call.function.arguments))))208 case .info(let i): info = i209 case .rejectedToolCall, .steering: break210 }211 if stopped { break }212 }213 } catch {214 continuation.finish(throwing: Task.isCancelled ? InferenceError.cancelled : InferenceError.generationFailed(MLXBackend.describe(error)))215 return216 }217 let cancelled = Task.isCancelled218 // Keep the session only when the turn completed: a cancelled consumer rolls the turn back inside ChatSession.219 if let key = request.sessionKey, !cancelled, !stopped, info != nil {220 var transcript = prints221 if !sawToolCall || !text.isEmpty { transcript.append(Fingerprint(role: .assistant, text: text)) }222 if sawToolCall && text.isEmpty { transcript.append(Fingerprint(role: .assistant, text: "")) }223 sessions[key] = Session(chat: chat, transcript: transcript, system: system, toolsKey: toolsKey)224 sessionOrder.append(key)225 while sessionOrder.count > maxSessions { sessions[sessionOrder.removeFirst()] = nil }226 }227 let total = Date().timeIntervalSince(started)228 let snapshot = MLX.Memory.snapshot()229 lastCachedTokens = (info?.promptTokenCount ?? 0) + (info?.generationTokenCount ?? 0)230 var metrics = GenerationMetrics(231 promptTokens: info?.promptTokenCount, cachedPromptTokens: info.map { reused || $0.cachedPromptTokenCount > 0 ? $0.cachedPromptTokenCount : 0 },232 completionTokens: info?.generationTokenCount, reasoningTokens: info?.reasoningTokenCount,233 timeToFirstToken: firstToken.map { $0.timeIntervalSince(started) }, promptTime: info?.promptTime, generationTime: info?.generateTime,234 totalTime: total, peakMemoryBytes: Int64(snapshot.peakMemory), activeMemoryBytes: Int64(snapshot.activeMemory))235 if cancelled { metrics.stopReason = .cancelled }236 else if stopped { metrics.stopReason = .stopSequence }237 else if sawToolCall { metrics.stopReason = .toolCall }238 else if let info, case .length = info.stopReason { metrics.stopReason = .maxTokens }239 continuation.yield(.finished(metrics))240 continuation.finish()241 }242243 /// Index of the first message of the trailing run the model has not answered yet (the last user turn, or trailing tool results).244 static func pendingStart(_ messages: [PromptMessage]) -> Int {245 var i = messages.count - 1246 if messages[i].role == .tool { while i > 0, messages[i - 1].role == .tool { i -= 1 } }247 return i248 }249250 static func chatMessage(_ m: PromptMessage) -> Chat.Message {251 let images: [UserInput.Image] = m.images.compactMap { CIImage(data: $0.data).map { .ciImage($0) } }252 switch m.role {253 case .system: return .system(m.content)254 case .user: return .user(m.content, images: images)255 case .assistant:256 let calls = m.toolCalls.map { ToolCall(function: .init(name: $0.name, arguments: ($0.arguments.anyValue as? [String: any Sendable]) ?? [:]), id: $0.id) }257 return .assistant(m.content, toolCalls: calls.isEmpty ? nil : calls)258 case .tool: return .tool(m.content, id: m.toolCallID, name: m.toolName)259 }260 }261262 static func jsonValue(_ arguments: [String: MLXLMCommon.JSONValue]) -> LocalChatCore.JSONValue {263 guard let data = try? JSONEncoder().encode(arguments), let value = try? JSONDecoder().decode(LocalChatCore.JSONValue.self, from: data) else { return .object([:]) }264 return value265 }266267 func countTokens(_ text: String) async -> Int? {268 guard let container else { return nil }269 return await container.encode(text).count270 }271272 func cacheStatistics() async -> CacheStatistics? {273 sessions.isEmpty ? CacheStatistics(cachedTokens: 0) : CacheStatistics(cachedTokens: lastCachedTokens)274 }275276 func resetSession(_ key: String?) async {277 if let key { sessions[key] = nil; sessionOrder.removeAll { $0 == key } } else { sessions.removeAll(); sessionOrder.removeAll() }278 }279280 func unload() async {281 sessions.removeAll(); sessionOrder.removeAll(); container = nil282 MLX.Memory.clearCache()283 }284}285286/// A loaded MLX embedding model.287actor MLXEmbeddingRuntime: ModelRuntime {288 nonisolated let info: RuntimeInfo289 private var container: EmbedderModelContainer?290 init(container: EmbedderModelContainer, info: RuntimeInfo) { self.container = container; self.info = info }291292 nonisolated func generate(_ request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error> {293 AsyncThrowingStream { $0.finish(throwing: InferenceError.unsupported("This is an embedding model; it cannot chat.")) }294 }295296 func countTokens(_ text: String) async -> Int? {297 guard let container else { return nil }298 return await container.perform { $0.tokenizer.encode(text: text, addSpecialTokens: true).count }299 }300301 func embed(_ texts: [String]) async throws -> [[Float]] {302 guard let container else { throw InferenceError.modelNotLoaded }303 guard !texts.isEmpty else { return [] }304 let limit = min(info.contextLength ?? 512, 512)305 var out: [[Float]] = []306 // small batches keep peak memory flat on iPhone307 for start in stride(from: 0, to: texts.count, by: 16) {308 let batch = Array(texts[start..<min(start + 16, texts.count)])309 let vectors = await container.perform { context -> [[Float]] in310 let inputs = batch.map { Array(context.tokenizer.encode(text: $0, addSpecialTokens: true).prefix(limit)) }311 let maxLength = inputs.reduce(into: 16) { $0 = max($0, $1.count) }312 let pad = context.tokenizer.eosTokenId ?? 0313 let padded = stacked(inputs.map { MLXArray($0 + Array(repeating: pad, count: maxLength - $0.count)) })314 let mask = stacked(inputs.map { MLXArray(Array(repeating: true, count: $0.count) + Array(repeating: false, count: maxLength - $0.count)) })315 let result = context.pooling(context.model(padded, positionIds: nil, tokenTypeIds: MLXArray.zeros(like: padded), attentionMask: mask),316 normalize: true, applyLayerNorm: true)317 result.eval()318 return result.map { $0.asArray(Float.self) }319 }320 out.append(contentsOf: vectors)321 }322 return out323 }324325 func unload() async { container = nil; MLX.Memory.clearCache() }326}327