SPB Git

Repository Public

MLXBackend.swift 18.7 KB 327 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatInference/Sources/MLXBackend.swift3// Description: MLX Swift backend: loads LLM / VLM / embedding models from a local directory and streams real generations with KV-cache reuse across turns.4import CoreImage5import Foundation6import LocalChatCore7import MLX8import MLXEmbedders9import MLXLLM10import MLXLMCommon11import MLXVLM12import Tokenizers1314/// Adapts swift-transformers tokenizers to the mlx-swift-lm protocol.15struct TransformersTokenizerLoader: MLXLMCommon.TokenizerLoader {16    struct Bridge: MLXLMCommon.Tokenizer {17        let upstream: any Tokenizers.Tokenizer18        func encode(text: String, addSpecialTokens: Bool) -> [Int] { upstream.encode(text: text, addSpecialTokens: addSpecialTokens) }19        func decode(tokenIds: [Int], skipSpecialTokens: Bool) -> String { upstream.decode(tokens: tokenIds, skipSpecialTokens: skipSpecialTokens) }20        func convertTokenToId(_ token: String) -> Int? { upstream.convertTokenToId(token) }21        func convertIdToToken(_ id: Int) -> String? { upstream.convertIdToToken(id) }22        var bosToken: String? { upstream.bosToken }23        var eosToken: String? { upstream.eosToken }24        var unknownToken: String? { upstream.unknownToken }25        func applyChatTemplate(messages: [[String: any Sendable]], tools: [[String: any Sendable]]?, additionalContext: [String: any Sendable]?) throws -> [Int] {26            do { return try upstream.applyChatTemplate(messages: messages, tools: tools, additionalContext: additionalContext) }27            catch Tokenizers.TokenizerError.missingChatTemplate { throw MLXLMCommon.TokenizerError.missingChatTemplate }28        }29    }30    func load(from directory: URL) async throws -> any MLXLMCommon.Tokenizer {31        Bridge(upstream: try await AutoTokenizer.from(modelFolder: directory))32    }33}3435public struct MLXBackend: InferenceBackend {36    public let kind: BackendKind = .mlx37    public init() {}3839    public var isAvailable: Bool {40        #if targetEnvironment(simulator)41        false42        #else43        true44        #endif45    }46    public var unavailableReason: String? {47        isAvailable ? nil : "MLX needs a real Apple GPU: it does not run in the iOS Simulator. Use a device, or route to a Mac."48    }4950    public func load(_ model: ModelDescriptor, modelsRoot: URL, progress: @escaping @Sendable (LocalChatCore.LoadProgress) -> Void) async throws -> any ModelRuntime {51        guard isAvailable else { throw InferenceError.backendUnavailable(unavailableReason ?? "MLX is unavailable.") }52        guard let relative = model.localPath else { throw InferenceError.modelNotFound(model.name) }53        let directory = relative.hasPrefix("/") ? URL(fileURLWithPath: relative) : modelsRoot.appendingPathComponent(relative)54        guard FileManager.default.fileExists(atPath: directory.appendingPathComponent("config.json").path) else {55            throw InferenceError.loadFailed("config.json is missing in \(directory.lastPathComponent).")56        }57        progress(LocalChatCore.LoadProgress(stage: .preparing))58        let start = Date()59        let handlers = LoadProgressHandlers.weights { p in progress(LocalChatCore.LoadProgress(stage: .loadingWeights, fraction: p.fractionCompleted)) }60        let loader = TransformersTokenizerLoader()61        let config = Self.readConfig(directory)6263        if model.isEmbeddingOnly {64            do {65                let container = try await EmbedderModelFactory.shared.loadContainer(from: directory, using: loader)66                let bytes = (try? await container.memory().total).map(Int64.init)67                progress(LocalChatCore.LoadProgress(stage: .ready, fraction: 1))68                return MLXEmbeddingRuntime(container: container, info: RuntimeInfo(69                    modelID: model.id, backend: .mlx, loadTime: Date().timeIntervalSince(start), weightsMemoryBytes: bytes,70                    contextLength: config.contextLength, capabilities: [.embeddings], chatTemplateAvailable: false))71            } catch { throw InferenceError.loadFailed(Self.describe(error)) }72        }7374        let container: ModelContainer75        do {76            if config.hasVision {77                do { container = try await VLMModelFactory.shared.loadContainer(from: directory, using: loader, progress: handlers) }78                catch { container = try await LLMModelFactory.shared.loadContainer(from: directory, using: loader, progress: handlers) }79            } else {80                container = try await LLMModelFactory.shared.loadContainer(from: directory, using: loader, progress: handlers)81            }82        } catch { throw InferenceError.loadFailed(Self.describe(error)) }8384        let bytes = (try? await container.memory().total).map(Int64.init)85        var capabilities = model.capabilities86        if config.hasVision { capabilities.insert(.vision) }87        let hasTemplate = (try? await container.applyChatTemplate(messages: [["role": "user", "content": "hi"]])) != nil88        progress(LocalChatCore.LoadProgress(stage: .ready, fraction: 1))89        return MLXRuntime(container: container, info: RuntimeInfo(90            modelID: model.id, backend: .mlx, loadTime: Date().timeIntervalSince(start), weightsMemoryBytes: bytes,91            contextLength: config.contextLength ?? model.contextLength, capabilities: capabilities, chatTemplateAvailable: hasTemplate))92    }9394    struct ConfigSummary { var hasVision = false; var contextLength: Int?; var modelType: String? }9596    static func readConfig(_ directory: URL) -> ConfigSummary {97        var out = ConfigSummary()98        if let data = try? Data(contentsOf: directory.appendingPathComponent("config.json")),99           let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] {100            out.hasVision = json["vision_config"] != nil || json["vision_tower"] != nil101            out.modelType = json["model_type"] as? String102            let text = json["text_config"] as? [String: Any]103            out.contextLength = (json["max_position_embeddings"] as? Int) ?? (text?["max_position_embeddings"] as? Int)104        }105        if out.contextLength == nil, let data = try? Data(contentsOf: directory.appendingPathComponent("tokenizer_config.json")),106           let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], let n = json["model_max_length"] as? Int, n < 10_000_000 {107            out.contextLength = n108        }109        return out110    }111112    static func describe(_ error: Error) -> String {113        let text = (error as? LocalizedError)?.errorDescription ?? String(describing: error)114        if text.contains("unsupportedModelType") || text.lowercased().contains("unsupported model type") {115            return "This architecture is not implemented in MLX Swift yet (\(text))."116        }117        return text118    }119}120121/// A loaded MLX chat model. One generation at a time per runtime (the engine serializes requests).122actor MLXRuntime: ModelRuntime {123    nonisolated let info: RuntimeInfo124    private var container: ModelContainer?125126    /// A live ChatSession and the transcript it has already consumed, so the next turn only evaluates new tokens.127    private struct Session { var chat: ChatSession; var transcript: [Fingerprint]; var system: String?; var toolsKey: String }128    private struct Fingerprint: Equatable {129        var role: PromptMessage.Role; var hash: Int130        init(_ m: PromptMessage) { role = m.role; hash = m.content.trimmingCharacters(in: .whitespacesAndNewlines).hashValue ^ m.images.count }131        init(role: PromptMessage.Role, text: String) { self.role = role; hash = text.trimmingCharacters(in: .whitespacesAndNewlines).hashValue }132    }133    private var sessions: [String: Session] = [:]134    private var sessionOrder: [String] = []135    private var lastCachedTokens = 0136    private let maxSessions = 2137138    init(container: ModelContainer, info: RuntimeInfo) { self.container = container; self.info = info }139140    nonisolated func generate(_ request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error> {141        AsyncThrowingStream { continuation in142            let task = Task { await self.run(request, continuation) }143            continuation.onTermination = { _ in task.cancel() }144        }145    }146147    private func run(_ request: GenerationRequest, _ continuation: AsyncThrowingStream<GenerationEvent, Error>.Continuation) async {148        guard let container else { continuation.finish(throwing: InferenceError.modelNotLoaded); return }149        guard info.chatTemplateAvailable else {150            continuation.finish(throwing: InferenceError.unsupported("This model has no chat template; it cannot be used for chat.")); return151        }152        let started = Date()153        var messages = request.messages154        let system = messages.first?.role == .system ? messages.removeFirst().content : nil155        guard !messages.isEmpty else { continuation.finish(throwing: InferenceError.generationFailed("Empty prompt.")); return }156157        let s = request.settings158        let parameters = GenerateParameters(159            maxTokens: s.maxTokens, maxKVSize: s.contextLength > 0 ? s.contextLength : nil, kvBits: s.kvBits,160            temperature: Float(s.temperature), topP: Float(s.topP), topK: s.topK, minP: Float(s.minP),161            repetitionPenalty: s.repetitionPenalty != 1 ? Float(s.repetitionPenalty) : nil, repetitionContextSize: s.repetitionContext,162            seed: s.seed)163        let toolSpecs: [ToolSpec]? = request.tools.isEmpty ? nil : request.tools.map { ($0.openAISpec.anyValue as? [String: any Sendable]) ?? [:] }164        let toolsKey = request.tools.map(\.name).joined(separator: ",")165        let context: [String: any Sendable] = ["enable_thinking": s.reasoningEnabled]166167        // Reuse the live session when this request only appends turns to what it already evaluated.168        var chat: ChatSession; var pending: [PromptMessage]; var reused = false169        let prints = messages.map(Fingerprint.init)170        if let key = request.sessionKey, let live = sessions[key], live.system == system, live.toolsKey == toolsKey,171           live.transcript.count < prints.count, Array(prints.prefix(live.transcript.count)) == live.transcript {172            chat = live.chat; pending = Array(messages.suffix(prints.count - live.transcript.count)); reused = true173        } else {174            // split at the last turn the model must answer: everything before it is history175            let cut = Self.pendingStart(messages)176            let history = messages.prefix(cut).map(Self.chatMessage)177            pending = Array(messages.suffix(from: cut))178            chat = ChatSession(container, instructions: system, history: history, generateParameters: parameters,179                               processing: .init(resize: CGSize(width: 768, height: 768)), additionalContext: context, tools: toolSpecs)180        }181        chat.generateParameters = parameters182        chat.additionalContext = context183        chat.tools = toolSpecs184        if let key = request.sessionKey { sessions[key] = nil; sessionOrder.removeAll { $0 == key } }185186        MLX.Memory.peakMemory = 0187        var text = ""; var firstToken: Date?; var info: GenerateCompletionInfo?; var sawToolCall = false; var stopped = false188        let stream = chat.streamDetails(to: pending.map(Self.chatMessage))189        do {190            for try await event in stream {191                if Task.isCancelled { break }192                switch event {193                case .chunk(let piece):194                    if firstToken == nil { firstToken = Date() }195                    text += piece196                    if let hit = request.stop.first(where: { !$0.isEmpty && text.contains($0) }), let range = text.range(of: hit) {197                        let keep = String(text[..<range.lowerBound]); let already = text.count - piece.count198                        if keep.count > already { continuation.yield(.text(String(keep.dropFirst(already)))) }199                        text = keep; stopped = true200                    } else { continuation.yield(.text(piece)) }201                case .reasoning(let piece):202                    if firstToken == nil { firstToken = Date() }203                    continuation.yield(.reasoning(piece))204                case .toolCall(let call):205                    sawToolCall = true206                    continuation.yield(.toolCall(ToolInvocation(id: call.id ?? UUID().uuidString, name: call.function.name,207                                                                arguments: Self.jsonValue(call.function.arguments))))208                case .info(let i): info = i209                case .rejectedToolCall, .steering: break210                }211                if stopped { break }212            }213        } catch {214            continuation.finish(throwing: Task.isCancelled ? InferenceError.cancelled : InferenceError.generationFailed(MLXBackend.describe(error)))215            return216        }217        let cancelled = Task.isCancelled218        // Keep the session only when the turn completed: a cancelled consumer rolls the turn back inside ChatSession.219        if let key = request.sessionKey, !cancelled, !stopped, info != nil {220            var transcript = prints221            if !sawToolCall || !text.isEmpty { transcript.append(Fingerprint(role: .assistant, text: text)) }222            if sawToolCall && text.isEmpty { transcript.append(Fingerprint(role: .assistant, text: "")) }223            sessions[key] = Session(chat: chat, transcript: transcript, system: system, toolsKey: toolsKey)224            sessionOrder.append(key)225            while sessionOrder.count > maxSessions { sessions[sessionOrder.removeFirst()] = nil }226        }227        let total = Date().timeIntervalSince(started)228        let snapshot = MLX.Memory.snapshot()229        lastCachedTokens = (info?.promptTokenCount ?? 0) + (info?.generationTokenCount ?? 0)230        var metrics = GenerationMetrics(231            promptTokens: info?.promptTokenCount, cachedPromptTokens: info.map { reused || $0.cachedPromptTokenCount > 0 ? $0.cachedPromptTokenCount : 0 },232            completionTokens: info?.generationTokenCount, reasoningTokens: info?.reasoningTokenCount,233            timeToFirstToken: firstToken.map { $0.timeIntervalSince(started) }, promptTime: info?.promptTime, generationTime: info?.generateTime,234            totalTime: total, peakMemoryBytes: Int64(snapshot.peakMemory), activeMemoryBytes: Int64(snapshot.activeMemory))235        if cancelled { metrics.stopReason = .cancelled }236        else if stopped { metrics.stopReason = .stopSequence }237        else if sawToolCall { metrics.stopReason = .toolCall }238        else if let info, case .length = info.stopReason { metrics.stopReason = .maxTokens }239        continuation.yield(.finished(metrics))240        continuation.finish()241    }242243    /// Index of the first message of the trailing run the model has not answered yet (the last user turn, or trailing tool results).244    static func pendingStart(_ messages: [PromptMessage]) -> Int {245        var i = messages.count - 1246        if messages[i].role == .tool { while i > 0, messages[i - 1].role == .tool { i -= 1 } }247        return i248    }249250    static func chatMessage(_ m: PromptMessage) -> Chat.Message {251        let images: [UserInput.Image] = m.images.compactMap { CIImage(data: $0.data).map { .ciImage($0) } }252        switch m.role {253        case .system: return .system(m.content)254        case .user: return .user(m.content, images: images)255        case .assistant:256            let calls = m.toolCalls.map { ToolCall(function: .init(name: $0.name, arguments: ($0.arguments.anyValue as? [String: any Sendable]) ?? [:]), id: $0.id) }257            return .assistant(m.content, toolCalls: calls.isEmpty ? nil : calls)258        case .tool: return .tool(m.content, id: m.toolCallID, name: m.toolName)259        }260    }261262    static func jsonValue(_ arguments: [String: MLXLMCommon.JSONValue]) -> LocalChatCore.JSONValue {263        guard let data = try? JSONEncoder().encode(arguments), let value = try? JSONDecoder().decode(LocalChatCore.JSONValue.self, from: data) else { return .object([:]) }264        return value265    }266267    func countTokens(_ text: String) async -> Int? {268        guard let container else { return nil }269        return await container.encode(text).count270    }271272    func cacheStatistics() async -> CacheStatistics? {273        sessions.isEmpty ? CacheStatistics(cachedTokens: 0) : CacheStatistics(cachedTokens: lastCachedTokens)274    }275276    func resetSession(_ key: String?) async {277        if let key { sessions[key] = nil; sessionOrder.removeAll { $0 == key } } else { sessions.removeAll(); sessionOrder.removeAll() }278    }279280    func unload() async {281        sessions.removeAll(); sessionOrder.removeAll(); container = nil282        MLX.Memory.clearCache()283    }284}285286/// A loaded MLX embedding model.287actor MLXEmbeddingRuntime: ModelRuntime {288    nonisolated let info: RuntimeInfo289    private var container: EmbedderModelContainer?290    init(container: EmbedderModelContainer, info: RuntimeInfo) { self.container = container; self.info = info }291292    nonisolated func generate(_ request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error> {293        AsyncThrowingStream { $0.finish(throwing: InferenceError.unsupported("This is an embedding model; it cannot chat.")) }294    }295296    func countTokens(_ text: String) async -> Int? {297        guard let container else { return nil }298        return await container.perform { $0.tokenizer.encode(text: text, addSpecialTokens: true).count }299    }300301    func embed(_ texts: [String]) async throws -> [[Float]] {302        guard let container else { throw InferenceError.modelNotLoaded }303        guard !texts.isEmpty else { return [] }304        let limit = min(info.contextLength ?? 512, 512)305        var out: [[Float]] = []306        // small batches keep peak memory flat on iPhone307        for start in stride(from: 0, to: texts.count, by: 16) {308            let batch = Array(texts[start..<min(start + 16, texts.count)])309            let vectors = await container.perform { context -> [[Float]] in310                let inputs = batch.map { Array(context.tokenizer.encode(text: $0, addSpecialTokens: true).prefix(limit)) }311                let maxLength = inputs.reduce(into: 16) { $0 = max($0, $1.count) }312                let pad = context.tokenizer.eosTokenId ?? 0313                let padded = stacked(inputs.map { MLXArray($0 + Array(repeating: pad, count: maxLength - $0.count)) })314                let mask = stacked(inputs.map { MLXArray(Array(repeating: true, count: $0.count) + Array(repeating: false, count: maxLength - $0.count)) })315                let result = context.pooling(context.model(padded, positionIds: nil, tokenTypeIds: MLXArray.zeros(like: padded), attentionMask: mask),316                                             normalize: true, applyLayerNorm: true)317                result.eval()318                return result.map { $0.asArray(Float.self) }319            }320            out.append(contentsOf: vectors)321        }322        return out323    }324325    func unload() async { container = nil; MLX.Memory.clearCache() }326}327
Files main
Menu

Theme