SPB Git

Repository Public MIT

MLXRuntime.swift 26.6 KB 520 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraLocal/Sources/MLXRuntime.swift10// Module           : Packages11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import CoreGraphics90import CoreImage91import Foundation92import MLX93import MLXLLM94import MLXLMCommon95import MLXVLM96import Tokenizers9798// MARK: - Availability and memory99100public enum MLXAvailability {101    /// nil when MLX can run here; otherwise the sentence to show.102    public static var unavailableReason: String? {103        #if targetEnvironment(simulator)104        "On-device models need a real Apple GPU. MLX does not run in the iOS Simulator; use a physical iPhone or iPad."105        #else106        nil107        #endif108    }109    public static var isAvailable: Bool { unavailableReason == nil }110}111112/// Figures read from the MLX allocator. nil in the Simulator, where MLX does not run.113public struct MLXMemoryFigures: Sendable, Hashable, Codable {114    public var activeBytes: Int64115    public var cacheBytes: Int64116    public var peakBytes: Int64117118    public static func read() -> MLXMemoryFigures? {119        #if targetEnvironment(simulator)120        return nil121        #else122        let s = MLX.Memory.snapshot()123        return MLXMemoryFigures(activeBytes: Int64(s.activeMemory), cacheBytes: Int64(s.cacheMemory), peakBytes: Int64(s.peakMemory))124        #endif125    }126    static func resetPeak() {127        #if !targetEnvironment(simulator)128        MLX.Memory.peakMemory = 0129        #endif130    }131    static func clearCache() {132        #if !targetEnvironment(simulator)133        MLX.Memory.clearCache()134        #endif135    }136    /// A large buffer cache speeds a Mac up and gets an iPhone app terminated: keep it small on iOS.137    static func configureCache(_ facts: DeviceFacts) {138        #if !targetEnvironment(simulator)139        MLX.Memory.cacheLimit = facts.platform == .macOS ? Int(min(Int64(2) << 30, facts.physicalMemoryBytes / 16)) : 48 << 20140        #endif141    }142}143144// MARK: - Tokenizer bridge145146/// Adapts swift-transformers tokenizers to the mlx-swift-lm protocol.147struct TransformersTokenizerLoader: MLXLMCommon.TokenizerLoader {148    struct Bridge: MLXLMCommon.Tokenizer {149        let upstream: any Tokenizers.Tokenizer150        func encode(text: String, addSpecialTokens: Bool) -> [Int] { upstream.encode(text: text, addSpecialTokens: addSpecialTokens) }151        func decode(tokenIds: [Int], skipSpecialTokens: Bool) -> String { upstream.decode(tokens: tokenIds, skipSpecialTokens: skipSpecialTokens) }152        func convertTokenToId(_ token: String) -> Int? { upstream.convertTokenToId(token) }153        func convertIdToToken(_ id: Int) -> String? { upstream.convertIdToToken(id) }154        var bosToken: String? { upstream.bosToken }155        var eosToken: String? { upstream.eosToken }156        var unknownToken: String? { upstream.unknownToken }157        func applyChatTemplate(messages: [[String: any Sendable]], tools: [[String: any Sendable]]?, additionalContext: [String: any Sendable]?) throws -> [Int] {158            do { return try upstream.applyChatTemplate(messages: messages, tools: tools, additionalContext: additionalContext) }159            catch Tokenizers.TokenizerError.missingChatTemplate { throw MLXLMCommon.TokenizerError.missingChatTemplate }160        }161    }162    func load(from directory: URL) async throws -> any MLXLMCommon.Tokenizer {163        Bridge(upstream: try await AutoTokenizer.from(modelFolder: directory))164    }165}166167// MARK: - Loaded model facts168169/// Facts about a loaded model, read from its files and from the runtime.170public struct LoadedModelInfo: Sendable, Hashable, Codable {171    public var modelID: String172    public var directory: URL173    /// `model_type` in config.json.174    public var architecture: String?175    public var contextLength: Int?176    public var hasVision: Bool177    public var hasChatTemplate: Bool178    /// True when the chat template mentions tools, i.e. the model was trained with a tool-calling format.179    public var templateSupportsTools: Bool180    /// Bytes of weights as MLX reports them after loading.181    public var weightsMemoryBytes: Int64?182    public var loadTime: TimeInterval183    public var loadedAt: Date184185    public var capabilities: Set<Capability> {186        var c: Set<Capability> = [.text]187        if hasVision { c.insert(.vision) }188        if templateSupportsTools { c.insert(.tools) }189        return c190    }191}192193struct ModelConfigSummary {194    var hasVision = false195    var contextLength: Int?196    var modelType: String?197    var templateSupportsTools = false198    var templateSupportsThinking = false199200    static func read(_ directory: URL) -> ModelConfigSummary {201        var out = ModelConfigSummary()202        func json(_ name: String) -> [String: Any]? {203            (try? Data(contentsOf: directory.appendingPathComponent(name))).flatMap { try? JSONSerialization.jsonObject(with: $0) as? [String: Any] }204        }205        if let config = json("config.json") {206            out.hasVision = config["vision_config"] != nil || config["vision_tower"] != nil207            out.modelType = config["model_type"] as? String208            let text = config["text_config"] as? [String: Any]209            out.contextLength = (config["max_position_embeddings"] as? Int) ?? (text?["max_position_embeddings"] as? Int)210        }211        let tokenizer = json("tokenizer_config.json")212        if out.contextLength == nil, let n = tokenizer?["model_max_length"] as? Int, n < 10_000_000 { out.contextLength = n }213        var template = tokenizer?["chat_template"] as? String214        if template == nil { template = try? String(contentsOf: directory.appendingPathComponent("chat_template.jinja"), encoding: .utf8) }215        if template == nil { template = json("chat_template.json")?["chat_template"] as? String }216        if let template {217            out.templateSupportsTools = template.contains("tools") || template.contains("tool_call")218            out.templateSupportsThinking = template.contains("enable_thinking") || template.contains("<think>")219        }220        return out221    }222}223224// MARK: - Runtime225226/// Outcome of one generation: the measured metrics, and the error that ended it when it did not complete.227public struct LocalGenerationResult: Sendable {228    public var metrics: LocalGenerationMetrics229    public var error: AstraError?230    public init(metrics: LocalGenerationMetrics, error: AstraError? = nil) { self.metrics = metrics; self.error = error }231}232233/// What `LocalEngine` needs from a loaded model. `MLXRuntime` is the real one; tests substitute a scripted one.234public protocol LocalModelRuntime: Sendable {235    var info: LoadedModelInfo { get }236    /// Emits the events of one answer and returns when generation has fully stopped. On cancellation of the calling task it stops237    /// promptly and returns `.cancelled` as the error; a completed answer has emitted exactly one `.messageCompleted`.238    func perform(_ request: GenerationRequest, emit: @Sendable (StreamEvent) -> Void) async -> LocalGenerationResult239    func countTokens(_ text: String) async -> Int?240    func resetSession(_ key: String?) async241    func unload() async242}243244/// One loaded MLX chat model. Generations are serialized by `LocalEngine`; the runtime itself holds the KV-cache sessions.245///246/// Cancellation contract: when the consuming task is cancelled (or the stream is dropped), generation stops at the next token,247/// the GPU work is awaited, and the stream ends by **throwing `AstraError.cancelled`**. No `.messageCompleted` is emitted for a248/// cancelled generation, and the KV cache of that conversation is discarded (the next turn re-evaluates its prompt).249public actor MLXRuntime: LocalModelRuntime {250    public nonisolated let info: LoadedModelInfo251    private var container: ModelContainer?252    private var generationCount = 0253254    /// A live ChatSession and the transcript it has already consumed, so the next turn only evaluates new tokens.255    private struct Session { var chat: ChatSession; var transcript: [Fingerprint]; var system: String?; var toolsKey: String; var thinking: Bool }256    struct Fingerprint: Equatable {257        var role: PromptMessage.Role; var text: String; var extras: Int258        init(_ m: PromptMessage) { role = m.role; text = m.text.trimmingCharacters(in: .whitespacesAndNewlines); extras = m.images.count &+ m.toolCalls.count &* 31 }259        init(role: PromptMessage.Role, text: String, toolCalls: Int = 0) { self.role = role; self.text = text.trimmingCharacters(in: .whitespacesAndNewlines); extras = toolCalls &* 31 }260    }261    private var sessions: [String: Session] = [:]262    private var sessionOrder: [String] = []263    private let maxSessions: Int264265    private init(container: ModelContainer, info: LoadedModelInfo, maxSessions: Int) {266        self.container = container; self.info = info; self.maxSessions = maxSessions267    }268269    /// Loads a model directory (config.json, tokenizer files, safetensors). `progress` reports the fraction of weights read.270    public static func load(modelID: String, directory: URL, maxSessions: Int = 2,271                            progress: @escaping @Sendable (Double) -> Void = { _ in }) async throws -> MLXRuntime {272        if let reason = MLXAvailability.unavailableReason { throw AstraError.unsupported(reason) }273        guard FileManager.default.fileExists(atPath: directory.appendingPathComponent("config.json").path) else {274            throw AstraError.local("This model is incomplete: config.json is missing from “\(directory.lastPathComponent)”.")275        }276        let started = Date()277        let config = ModelConfigSummary.read(directory)278        let handlers = LoadProgressHandlers.weights { p in progress(p.fractionCompleted) }279        let loader = TransformersTokenizerLoader()280        let container: ModelContainer281        do {282            if config.hasVision {283                do { container = try await VLMModelFactory.shared.loadContainer(from: directory, using: loader, progress: handlers) }284                catch { container = try await LLMModelFactory.shared.loadContainer(from: directory, using: loader, progress: handlers) }285            } else {286                container = try await LLMModelFactory.shared.loadContainer(from: directory, using: loader, progress: handlers)287            }288        } catch { throw AstraError.local(describe(error)) }289        let bytes = (try? await container.memory().total).map(Int64.init)290        let tokenizer = await container.tokenizer291        let hasTemplate = (try? tokenizer.applyChatTemplate(messages: [["role": "user", "content": "hi"]])) != nil292        progress(1)293        let info = LoadedModelInfo(modelID: modelID, directory: directory, architecture: config.modelType, contextLength: config.contextLength,294                                   hasVision: config.hasVision, hasChatTemplate: hasTemplate, templateSupportsTools: hasTemplate && config.templateSupportsTools,295                                   weightsMemoryBytes: bytes, loadTime: Date().timeIntervalSince(started), loadedAt: Date())296        return MLXRuntime(container: container, info: info, maxSessions: maxSessions)297    }298299    static func describe(_ error: Error) -> String {300        let text = (error as? LocalizedError)?.errorDescription ?? String(describing: error)301        if text.contains("unsupportedModelType") || text.lowercased().contains("unsupported model type") {302            return "This model's architecture is not implemented in this version of MLX Swift (\(text))."303        }304        return "The model could not be loaded: \(text)"305    }306307    /// Streams one answer as a stream. Convenience over `perform`; metrics go to `onMetrics` once, whatever the outcome.308    public nonisolated func generate(_ request: GenerationRequest,309                                     onMetrics: @escaping @Sendable (LocalGenerationMetrics) -> Void = { _ in }) -> AsyncThrowingStream<StreamEvent, Error> {310        AsyncThrowingStream { continuation in311            let task = Task {312                let result = await self.perform(request) { continuation.yield($0) }313                onMetrics(result.metrics)314                if let error = result.error { continuation.finish(throwing: error) } else { continuation.finish() }315            }316            continuation.onTermination = { _ in task.cancel() }317        }318    }319320    private func refused(_ error: AstraError) -> LocalGenerationResult {321        LocalGenerationResult(metrics: LocalGenerationMetrics(modelID: info.modelID, stopReason: .error), error: error)322    }323324    /// Runs one generation to its end. Returns only after the GPU work has stopped, including when the calling task is cancelled.325    public func perform(_ request: GenerationRequest, emit: @Sendable (StreamEvent) -> Void) async -> LocalGenerationResult {326        guard let container else { return refused(.local("The model is not loaded.")) }327        guard info.hasChatTemplate else { return refused(.unsupported("This model has no chat template, so it cannot hold a conversation.")) }328        let messages = request.messages329        guard let last = messages.last, last.role != .assistant else {330            return refused(.invalidRequest("The conversation must end with a user message or a tool result."))331        }332        if !info.hasVision, messages.contains(where: { !$0.images.isEmpty }) { return refused(.unsupported("This model cannot read images.")) }333        if !request.tools.isEmpty, !info.templateSupportsTools {334            return refused(.unsupported("This model's chat template has no tool-calling format."))335        }336        let started = Date()337        let thermalBefore = ThermalLevel.current()338        let cold = generationCount == 0339        generationCount += 1340341        let p = request.parameters342        let parameters = GenerateParameters(343            maxTokens: p.maxOutputTokens ?? ThermalGovernor.defaultMaxTokens, maxKVSize: p.contextLength.flatMap { $0 > 0 ? $0 : nil },344            temperature: Float(p.temperature ?? 0.7), topP: Float(p.topP ?? 0.95), seed: p.seed.map { UInt64(truncatingIfNeeded: $0) })345        let toolSpecs: [ToolSpec]? = request.tools.isEmpty ? nil : request.tools.map(Self.toolSpec)346        let toolsKey = request.tools.map { $0.name + ":" + $0.parameters.serialized() }.joined(separator: ",")347        // Thinking is off unless asked for: on a phone a small model that thinks first makes every answer slow.348        let thinking = (p.reasoning ?? .off) != .off349        let context: [String: any Sendable] = ["enable_thinking": thinking]350        let system = request.system?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? request.system : nil351352        // Reuse the live session when this request only appends turns to what it already evaluated.353        let chat: ChatSession; let pending: [PromptMessage]; var reused = false354        let prints = messages.map(Fingerprint.init)355        if let key = request.sessionKey, let live = sessions[key], live.system == system, live.toolsKey == toolsKey, live.thinking == thinking,356           live.transcript.count < prints.count, Array(prints.prefix(live.transcript.count)) == live.transcript {357            chat = live.chat; pending = Array(messages.suffix(prints.count - live.transcript.count)); reused = true358        } else {359            let cut = Self.pendingStart(messages)360            pending = Array(messages.suffix(from: cut))361            chat = ChatSession(container, instructions: system, history: messages.prefix(cut).map(Self.chatMessage), generateParameters: parameters,362                               processing: .init(resize: CGSize(width: 768, height: 768)), additionalContext: context, tools: toolSpecs)363        }364        chat.generateParameters = parameters365        chat.additionalContext = context366        chat.tools = toolSpecs367        // The session leaves the table while it runs: a failed or cancelled turn must not be reused.368        if let key = request.sessionKey { sessions[key] = nil; sessionOrder.removeAll { $0 == key } }369370        MLXMemoryFigures.resetPeak()371        emit(.messageStarted(MessageStart(reportedModel: info.modelID, servedBy: "MLX")))372        var text = ""; var firstToken: Date?; var completion: GenerateCompletionInfo?; var toolCalls = 0; var stopped = false; var failure: Error?373        do {374            for try await event in chat.streamDetails(to: pending.map(Self.chatMessage)) {375                if Task.isCancelled { break }376                switch event {377                case .chunk(let piece):378                    if firstToken == nil { firstToken = Date() }379                    text += piece380                    if let hit = p.stop.first(where: { !$0.isEmpty && text.contains($0) }), let range = text.range(of: hit) {381                        let keep = String(text[..<range.lowerBound]); let already = text.count - piece.count382                        if keep.count > already { emit(.textDelta(String(keep.dropFirst(already)))) }383                        text = keep; stopped = true384                    } else { emit(.textDelta(piece)) }385                case .reasoning(let piece):386                    if firstToken == nil { firstToken = Date() }387                    emit(.reasoningSummaryDelta(piece))388                case .toolCall(let call):389                    toolCalls += 1390                    let id = call.id ?? "call_" + UUID().uuidString.prefix(12).lowercased()391                    let arguments = Self.jsonValue(call.function.arguments)392                    emit(.toolCallStarted(id: id, name: call.function.name))393                    emit(.toolCallDelta(id: id, argumentsFragment: arguments.serialized()))394                    emit(.toolCallCompleted(AstraCore.ToolCall(id: id, name: call.function.name, arguments: arguments)))395                case .info(let i): completion = i396                case .rejectedToolCall, .steering: break397                }398                if stopped { break }399            }400        } catch { failure = error }401402        let cancelled = Task.isCancelled403        if cancelled || stopped || failure != nil {404            // The consumer left mid-stream: wait for the GPU to be released before anything else touches the model.405            await chat.synchronize()406        }407        let reason: StopReason408        if cancelled { reason = .cancelled }409        else if failure != nil { reason = .error }410        else if stopped { reason = .stopSequence }411        else if toolCalls > 0 { reason = .toolCalls }412        else if let completion, case .length = completion.stopReason { reason = .maxTokens }413        else { reason = .endOfTurn }414415        // Keep the session only when the turn completed normally.416        if let key = request.sessionKey, reason == .endOfTurn || reason == .toolCalls || reason == .maxTokens, completion != nil {417            var transcript = prints418            transcript.append(Fingerprint(role: .assistant, text: text, toolCalls: toolCalls))419            sessions[key] = Session(chat: chat, transcript: transcript, system: system, toolsKey: toolsKey, thinking: thinking)420            sessionOrder.append(key)421            while sessionOrder.count > maxSessions { sessions[sessionOrder.removeFirst()] = nil }422        }423424        let memory = MLXMemoryFigures.read()425        var metrics = LocalGenerationMetrics(modelID: info.modelID, startedAt: started, totalTime: Date().timeIntervalSince(started), stopReason: reason)426        metrics.timeToFirstToken = firstToken.map { $0.timeIntervalSince(started) }427        metrics.isColdStart = cold428        metrics.reusedSession = reused429        metrics.peakMemoryBytes = memory?.peakBytes430        metrics.activeMemoryBytes = memory?.activeBytes431        metrics.thermalBefore = thermalBefore432        metrics.thermalAfter = .current()433        if let c = completion {434            metrics.promptTokens = c.promptTokenCount435            metrics.cachedPromptTokens = c.cachedPromptTokenCount436            metrics.completionTokens = c.generationTokenCount437            metrics.reasoningTokens = c.reasoningTokenCount438            metrics.promptTime = c.promptTime439            metrics.generationTime = c.generateTime440            let evaluated = max(0, c.promptTokenCount - c.cachedPromptTokenCount)441            if c.promptTime > 0, evaluated > 0 { metrics.promptTokensPerSecond = Double(evaluated) / c.promptTime }442            if c.generateTime > 0, c.generationTokenCount > 0 { metrics.decodeTokensPerSecond = Double(c.generationTokenCount) / c.generateTime }443        }444        if let failure, !cancelled { return LocalGenerationResult(metrics: metrics, error: .local("Generation failed: \((failure as? LocalizedError)?.errorDescription ?? String(describing: failure))")) }445        if cancelled { return LocalGenerationResult(metrics: metrics, error: .cancelled) }446        if let c = completion {447            emit(.usage(TokenUsage(inputTokens: c.promptTokenCount, outputTokens: c.generationTokenCount,448                                   cachedInputTokens: c.cachedPromptTokenCount > 0 ? c.cachedPromptTokenCount : nil,449                                   reasoningTokens: c.reasoningTokenCount)))450        }451        emit(.messageCompleted(reason))452        return LocalGenerationResult(metrics: metrics, error: nil)453    }454455    // MARK: Conversions456457    /// Index of the first message of the trailing run the model has not answered yet (the last user turn, or trailing tool results).458    static func pendingStart(_ messages: [PromptMessage]) -> Int {459        var i = messages.count - 1460        if messages[i].role == .tool { while i > 0, messages[i - 1].role == .tool { i -= 1 } }461        return i462    }463464    static func chatMessage(_ m: PromptMessage) -> Chat.Message {465        switch m.role {466        case .user:467            let images: [UserInput.Image] = m.images.compactMap { CIImage(data: $0.data).map { .ciImage($0) } }468            return .user(m.text, images: images)469        case .assistant:470            let calls = m.toolCalls.map { call in471                MLXLMCommon.ToolCall(function: .init(name: call.name, arguments: (sendable(call.arguments) as? [String: any Sendable]) ?? [:]), id: call.id)472            }473            return .assistant(m.text, toolCalls: calls.isEmpty ? nil : calls)474        case .tool:475            return .tool(m.text, id: m.toolCallID, name: m.toolName)476        }477    }478479    static func toolSpec(_ tool: ToolDefinition) -> ToolSpec {480        ["type": "function",481         "function": ["name": tool.name, "description": tool.summary, "parameters": sendable(tool.parameters)] as [String: any Sendable]]482    }483484    static func sendable(_ value: AstraCore.JSONValue) -> any Sendable {485        switch value {486        case .null: return NSNull()487        case .bool(let b): return b488        case .number(let n): return n.rounded() == n && abs(n) < 9_007_199_254_740_992 ? Int(n) : n489        case .string(let s): return s490        case .array(let a): return a.map(sendable)491        case .object(let o): return o.mapValues(sendable)492        }493    }494495    static func jsonValue(_ arguments: [String: MLXLMCommon.JSONValue]) -> AstraCore.JSONValue {496        guard let data = try? JSONEncoder().encode(arguments), let value = AstraCore.JSONValue.parse(data) else { return .object([:]) }497        return value498    }499500    // MARK: Services501502    /// Exact token count with the model's own tokenizer.503    public func countTokens(_ text: String) async -> Int? {504        guard let container else { return nil }505        return await container.encode(text).count506    }507508    public var liveSessionKeys: [String] { sessionOrder }509510    public func resetSession(_ key: String?) {511        if let key { sessions[key] = nil; sessionOrder.removeAll { $0 == key } } else { sessions.removeAll(); sessionOrder.removeAll() }512    }513514    /// Releases the weights and every KV cache.515    public func unload() {516        sessions.removeAll(); sessionOrder.removeAll(); container = nil517        MLXMemoryFigures.clearCache()518    }519}520
Files main
Menu

Theme