Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Packages/AstraLocal/Sources/MLXRuntime.swift10// Module : Packages11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import CoreGraphics90import CoreImage91import Foundation92import MLX93import MLXLLM94import MLXLMCommon95import MLXVLM96import Tokenizers9798// MARK: - Availability and memory99100public enum MLXAvailability {101 /// nil when MLX can run here; otherwise the sentence to show.102 public static var unavailableReason: String? {103 #if targetEnvironment(simulator)104 "On-device models need a real Apple GPU. MLX does not run in the iOS Simulator; use a physical iPhone or iPad."105 #else106 nil107 #endif108 }109 public static var isAvailable: Bool { unavailableReason == nil }110}111112/// Figures read from the MLX allocator. nil in the Simulator, where MLX does not run.113public struct MLXMemoryFigures: Sendable, Hashable, Codable {114 public var activeBytes: Int64115 public var cacheBytes: Int64116 public var peakBytes: Int64117118 public static func read() -> MLXMemoryFigures? {119 #if targetEnvironment(simulator)120 return nil121 #else122 let s = MLX.Memory.snapshot()123 return MLXMemoryFigures(activeBytes: Int64(s.activeMemory), cacheBytes: Int64(s.cacheMemory), peakBytes: Int64(s.peakMemory))124 #endif125 }126 static func resetPeak() {127 #if !targetEnvironment(simulator)128 MLX.Memory.peakMemory = 0129 #endif130 }131 static func clearCache() {132 #if !targetEnvironment(simulator)133 MLX.Memory.clearCache()134 #endif135 }136 /// A large buffer cache speeds a Mac up and gets an iPhone app terminated: keep it small on iOS.137 static func configureCache(_ facts: DeviceFacts) {138 #if !targetEnvironment(simulator)139 MLX.Memory.cacheLimit = facts.platform == .macOS ? Int(min(Int64(2) << 30, facts.physicalMemoryBytes / 16)) : 48 << 20140 #endif141 }142}143144// MARK: - Tokenizer bridge145146/// Adapts swift-transformers tokenizers to the mlx-swift-lm protocol.147struct TransformersTokenizerLoader: MLXLMCommon.TokenizerLoader {148 struct Bridge: MLXLMCommon.Tokenizer {149 let upstream: any Tokenizers.Tokenizer150 func encode(text: String, addSpecialTokens: Bool) -> [Int] { upstream.encode(text: text, addSpecialTokens: addSpecialTokens) }151 func decode(tokenIds: [Int], skipSpecialTokens: Bool) -> String { upstream.decode(tokens: tokenIds, skipSpecialTokens: skipSpecialTokens) }152 func convertTokenToId(_ token: String) -> Int? { upstream.convertTokenToId(token) }153 func convertIdToToken(_ id: Int) -> String? { upstream.convertIdToToken(id) }154 var bosToken: String? { upstream.bosToken }155 var eosToken: String? { upstream.eosToken }156 var unknownToken: String? { upstream.unknownToken }157 func applyChatTemplate(messages: [[String: any Sendable]], tools: [[String: any Sendable]]?, additionalContext: [String: any Sendable]?) throws -> [Int] {158 do { return try upstream.applyChatTemplate(messages: messages, tools: tools, additionalContext: additionalContext) }159 catch Tokenizers.TokenizerError.missingChatTemplate { throw MLXLMCommon.TokenizerError.missingChatTemplate }160 }161 }162 func load(from directory: URL) async throws -> any MLXLMCommon.Tokenizer {163 Bridge(upstream: try await AutoTokenizer.from(modelFolder: directory))164 }165}166167// MARK: - Loaded model facts168169/// Facts about a loaded model, read from its files and from the runtime.170public struct LoadedModelInfo: Sendable, Hashable, Codable {171 public var modelID: String172 public var directory: URL173 /// `model_type` in config.json.174 public var architecture: String?175 public var contextLength: Int?176 public var hasVision: Bool177 public var hasChatTemplate: Bool178 /// True when the chat template mentions tools, i.e. the model was trained with a tool-calling format.179 public var templateSupportsTools: Bool180 /// Bytes of weights as MLX reports them after loading.181 public var weightsMemoryBytes: Int64?182 public var loadTime: TimeInterval183 public var loadedAt: Date184185 public var capabilities: Set<Capability> {186 var c: Set<Capability> = [.text]187 if hasVision { c.insert(.vision) }188 if templateSupportsTools { c.insert(.tools) }189 return c190 }191}192193struct ModelConfigSummary {194 var hasVision = false195 var contextLength: Int?196 var modelType: String?197 var templateSupportsTools = false198 var templateSupportsThinking = false199200 static func read(_ directory: URL) -> ModelConfigSummary {201 var out = ModelConfigSummary()202 func json(_ name: String) -> [String: Any]? {203 (try? Data(contentsOf: directory.appendingPathComponent(name))).flatMap { try? JSONSerialization.jsonObject(with: $0) as? [String: Any] }204 }205 if let config = json("config.json") {206 out.hasVision = config["vision_config"] != nil || config["vision_tower"] != nil207 out.modelType = config["model_type"] as? String208 let text = config["text_config"] as? [String: Any]209 out.contextLength = (config["max_position_embeddings"] as? Int) ?? (text?["max_position_embeddings"] as? Int)210 }211 let tokenizer = json("tokenizer_config.json")212 if out.contextLength == nil, let n = tokenizer?["model_max_length"] as? Int, n < 10_000_000 { out.contextLength = n }213 var template = tokenizer?["chat_template"] as? String214 if template == nil { template = try? String(contentsOf: directory.appendingPathComponent("chat_template.jinja"), encoding: .utf8) }215 if template == nil { template = json("chat_template.json")?["chat_template"] as? String }216 if let template {217 out.templateSupportsTools = template.contains("tools") || template.contains("tool_call")218 out.templateSupportsThinking = template.contains("enable_thinking") || template.contains("<think>")219 }220 return out221 }222}223224// MARK: - Runtime225226/// Outcome of one generation: the measured metrics, and the error that ended it when it did not complete.227public struct LocalGenerationResult: Sendable {228 public var metrics: LocalGenerationMetrics229 public var error: AstraError?230 public init(metrics: LocalGenerationMetrics, error: AstraError? = nil) { self.metrics = metrics; self.error = error }231}232233/// What `LocalEngine` needs from a loaded model. `MLXRuntime` is the real one; tests substitute a scripted one.234public protocol LocalModelRuntime: Sendable {235 var info: LoadedModelInfo { get }236 /// Emits the events of one answer and returns when generation has fully stopped. On cancellation of the calling task it stops237 /// promptly and returns `.cancelled` as the error; a completed answer has emitted exactly one `.messageCompleted`.238 func perform(_ request: GenerationRequest, emit: @Sendable (StreamEvent) -> Void) async -> LocalGenerationResult239 func countTokens(_ text: String) async -> Int?240 func resetSession(_ key: String?) async241 func unload() async242}243244/// One loaded MLX chat model. Generations are serialized by `LocalEngine`; the runtime itself holds the KV-cache sessions.245///246/// Cancellation contract: when the consuming task is cancelled (or the stream is dropped), generation stops at the next token,247/// the GPU work is awaited, and the stream ends by **throwing `AstraError.cancelled`**. No `.messageCompleted` is emitted for a248/// cancelled generation, and the KV cache of that conversation is discarded (the next turn re-evaluates its prompt).249public actor MLXRuntime: LocalModelRuntime {250 public nonisolated let info: LoadedModelInfo251 private var container: ModelContainer?252 private var generationCount = 0253254 /// A live ChatSession and the transcript it has already consumed, so the next turn only evaluates new tokens.255 private struct Session { var chat: ChatSession; var transcript: [Fingerprint]; var system: String?; var toolsKey: String; var thinking: Bool }256 struct Fingerprint: Equatable {257 var role: PromptMessage.Role; var text: String; var extras: Int258 init(_ m: PromptMessage) { role = m.role; text = m.text.trimmingCharacters(in: .whitespacesAndNewlines); extras = m.images.count &+ m.toolCalls.count &* 31 }259 init(role: PromptMessage.Role, text: String, toolCalls: Int = 0) { self.role = role; self.text = text.trimmingCharacters(in: .whitespacesAndNewlines); extras = toolCalls &* 31 }260 }261 private var sessions: [String: Session] = [:]262 private var sessionOrder: [String] = []263 private let maxSessions: Int264265 private init(container: ModelContainer, info: LoadedModelInfo, maxSessions: Int) {266 self.container = container; self.info = info; self.maxSessions = maxSessions267 }268269 /// Loads a model directory (config.json, tokenizer files, safetensors). `progress` reports the fraction of weights read.270 public static func load(modelID: String, directory: URL, maxSessions: Int = 2,271 progress: @escaping @Sendable (Double) -> Void = { _ in }) async throws -> MLXRuntime {272 if let reason = MLXAvailability.unavailableReason { throw AstraError.unsupported(reason) }273 guard FileManager.default.fileExists(atPath: directory.appendingPathComponent("config.json").path) else {274 throw AstraError.local("This model is incomplete: config.json is missing from “\(directory.lastPathComponent)”.")275 }276 let started = Date()277 let config = ModelConfigSummary.read(directory)278 let handlers = LoadProgressHandlers.weights { p in progress(p.fractionCompleted) }279 let loader = TransformersTokenizerLoader()280 let container: ModelContainer281 do {282 if config.hasVision {283 do { container = try await VLMModelFactory.shared.loadContainer(from: directory, using: loader, progress: handlers) }284 catch { container = try await LLMModelFactory.shared.loadContainer(from: directory, using: loader, progress: handlers) }285 } else {286 container = try await LLMModelFactory.shared.loadContainer(from: directory, using: loader, progress: handlers)287 }288 } catch { throw AstraError.local(describe(error)) }289 let bytes = (try? await container.memory().total).map(Int64.init)290 let tokenizer = await container.tokenizer291 let hasTemplate = (try? tokenizer.applyChatTemplate(messages: [["role": "user", "content": "hi"]])) != nil292 progress(1)293 let info = LoadedModelInfo(modelID: modelID, directory: directory, architecture: config.modelType, contextLength: config.contextLength,294 hasVision: config.hasVision, hasChatTemplate: hasTemplate, templateSupportsTools: hasTemplate && config.templateSupportsTools,295 weightsMemoryBytes: bytes, loadTime: Date().timeIntervalSince(started), loadedAt: Date())296 return MLXRuntime(container: container, info: info, maxSessions: maxSessions)297 }298299 static func describe(_ error: Error) -> String {300 let text = (error as? LocalizedError)?.errorDescription ?? String(describing: error)301 if text.contains("unsupportedModelType") || text.lowercased().contains("unsupported model type") {302 return "This model's architecture is not implemented in this version of MLX Swift (\(text))."303 }304 return "The model could not be loaded: \(text)"305 }306307 /// Streams one answer as a stream. Convenience over `perform`; metrics go to `onMetrics` once, whatever the outcome.308 public nonisolated func generate(_ request: GenerationRequest,309 onMetrics: @escaping @Sendable (LocalGenerationMetrics) -> Void = { _ in }) -> AsyncThrowingStream<StreamEvent, Error> {310 AsyncThrowingStream { continuation in311 let task = Task {312 let result = await self.perform(request) { continuation.yield($0) }313 onMetrics(result.metrics)314 if let error = result.error { continuation.finish(throwing: error) } else { continuation.finish() }315 }316 continuation.onTermination = { _ in task.cancel() }317 }318 }319320 private func refused(_ error: AstraError) -> LocalGenerationResult {321 LocalGenerationResult(metrics: LocalGenerationMetrics(modelID: info.modelID, stopReason: .error), error: error)322 }323324 /// Runs one generation to its end. Returns only after the GPU work has stopped, including when the calling task is cancelled.325 public func perform(_ request: GenerationRequest, emit: @Sendable (StreamEvent) -> Void) async -> LocalGenerationResult {326 guard let container else { return refused(.local("The model is not loaded.")) }327 guard info.hasChatTemplate else { return refused(.unsupported("This model has no chat template, so it cannot hold a conversation.")) }328 let messages = request.messages329 guard let last = messages.last, last.role != .assistant else {330 return refused(.invalidRequest("The conversation must end with a user message or a tool result."))331 }332 if !info.hasVision, messages.contains(where: { !$0.images.isEmpty }) { return refused(.unsupported("This model cannot read images.")) }333 if !request.tools.isEmpty, !info.templateSupportsTools {334 return refused(.unsupported("This model's chat template has no tool-calling format."))335 }336 let started = Date()337 let thermalBefore = ThermalLevel.current()338 let cold = generationCount == 0339 generationCount += 1340341 let p = request.parameters342 let parameters = GenerateParameters(343 maxTokens: p.maxOutputTokens ?? ThermalGovernor.defaultMaxTokens, maxKVSize: p.contextLength.flatMap { $0 > 0 ? $0 : nil },344 temperature: Float(p.temperature ?? 0.7), topP: Float(p.topP ?? 0.95), seed: p.seed.map { UInt64(truncatingIfNeeded: $0) })345 let toolSpecs: [ToolSpec]? = request.tools.isEmpty ? nil : request.tools.map(Self.toolSpec)346 let toolsKey = request.tools.map { $0.name + ":" + $0.parameters.serialized() }.joined(separator: ",")347 // Thinking is off unless asked for: on a phone a small model that thinks first makes every answer slow.348 let thinking = (p.reasoning ?? .off) != .off349 let context: [String: any Sendable] = ["enable_thinking": thinking]350 let system = request.system?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? request.system : nil351352 // Reuse the live session when this request only appends turns to what it already evaluated.353 let chat: ChatSession; let pending: [PromptMessage]; var reused = false354 let prints = messages.map(Fingerprint.init)355 if let key = request.sessionKey, let live = sessions[key], live.system == system, live.toolsKey == toolsKey, live.thinking == thinking,356 live.transcript.count < prints.count, Array(prints.prefix(live.transcript.count)) == live.transcript {357 chat = live.chat; pending = Array(messages.suffix(prints.count - live.transcript.count)); reused = true358 } else {359 let cut = Self.pendingStart(messages)360 pending = Array(messages.suffix(from: cut))361 chat = ChatSession(container, instructions: system, history: messages.prefix(cut).map(Self.chatMessage), generateParameters: parameters,362 processing: .init(resize: CGSize(width: 768, height: 768)), additionalContext: context, tools: toolSpecs)363 }364 chat.generateParameters = parameters365 chat.additionalContext = context366 chat.tools = toolSpecs367 // The session leaves the table while it runs: a failed or cancelled turn must not be reused.368 if let key = request.sessionKey { sessions[key] = nil; sessionOrder.removeAll { $0 == key } }369370 MLXMemoryFigures.resetPeak()371 emit(.messageStarted(MessageStart(reportedModel: info.modelID, servedBy: "MLX")))372 var text = ""; var firstToken: Date?; var completion: GenerateCompletionInfo?; var toolCalls = 0; var stopped = false; var failure: Error?373 do {374 for try await event in chat.streamDetails(to: pending.map(Self.chatMessage)) {375 if Task.isCancelled { break }376 switch event {377 case .chunk(let piece):378 if firstToken == nil { firstToken = Date() }379 text += piece380 if let hit = p.stop.first(where: { !$0.isEmpty && text.contains($0) }), let range = text.range(of: hit) {381 let keep = String(text[..<range.lowerBound]); let already = text.count - piece.count382 if keep.count > already { emit(.textDelta(String(keep.dropFirst(already)))) }383 text = keep; stopped = true384 } else { emit(.textDelta(piece)) }385 case .reasoning(let piece):386 if firstToken == nil { firstToken = Date() }387 emit(.reasoningSummaryDelta(piece))388 case .toolCall(let call):389 toolCalls += 1390 let id = call.id ?? "call_" + UUID().uuidString.prefix(12).lowercased()391 let arguments = Self.jsonValue(call.function.arguments)392 emit(.toolCallStarted(id: id, name: call.function.name))393 emit(.toolCallDelta(id: id, argumentsFragment: arguments.serialized()))394 emit(.toolCallCompleted(AstraCore.ToolCall(id: id, name: call.function.name, arguments: arguments)))395 case .info(let i): completion = i396 case .rejectedToolCall, .steering: break397 }398 if stopped { break }399 }400 } catch { failure = error }401402 let cancelled = Task.isCancelled403 if cancelled || stopped || failure != nil {404 // The consumer left mid-stream: wait for the GPU to be released before anything else touches the model.405 await chat.synchronize()406 }407 let reason: StopReason408 if cancelled { reason = .cancelled }409 else if failure != nil { reason = .error }410 else if stopped { reason = .stopSequence }411 else if toolCalls > 0 { reason = .toolCalls }412 else if let completion, case .length = completion.stopReason { reason = .maxTokens }413 else { reason = .endOfTurn }414415 // Keep the session only when the turn completed normally.416 if let key = request.sessionKey, reason == .endOfTurn || reason == .toolCalls || reason == .maxTokens, completion != nil {417 var transcript = prints418 transcript.append(Fingerprint(role: .assistant, text: text, toolCalls: toolCalls))419 sessions[key] = Session(chat: chat, transcript: transcript, system: system, toolsKey: toolsKey, thinking: thinking)420 sessionOrder.append(key)421 while sessionOrder.count > maxSessions { sessions[sessionOrder.removeFirst()] = nil }422 }423424 let memory = MLXMemoryFigures.read()425 var metrics = LocalGenerationMetrics(modelID: info.modelID, startedAt: started, totalTime: Date().timeIntervalSince(started), stopReason: reason)426 metrics.timeToFirstToken = firstToken.map { $0.timeIntervalSince(started) }427 metrics.isColdStart = cold428 metrics.reusedSession = reused429 metrics.peakMemoryBytes = memory?.peakBytes430 metrics.activeMemoryBytes = memory?.activeBytes431 metrics.thermalBefore = thermalBefore432 metrics.thermalAfter = .current()433 if let c = completion {434 metrics.promptTokens = c.promptTokenCount435 metrics.cachedPromptTokens = c.cachedPromptTokenCount436 metrics.completionTokens = c.generationTokenCount437 metrics.reasoningTokens = c.reasoningTokenCount438 metrics.promptTime = c.promptTime439 metrics.generationTime = c.generateTime440 let evaluated = max(0, c.promptTokenCount - c.cachedPromptTokenCount)441 if c.promptTime > 0, evaluated > 0 { metrics.promptTokensPerSecond = Double(evaluated) / c.promptTime }442 if c.generateTime > 0, c.generationTokenCount > 0 { metrics.decodeTokensPerSecond = Double(c.generationTokenCount) / c.generateTime }443 }444 if let failure, !cancelled { return LocalGenerationResult(metrics: metrics, error: .local("Generation failed: \((failure as? LocalizedError)?.errorDescription ?? String(describing: failure))")) }445 if cancelled { return LocalGenerationResult(metrics: metrics, error: .cancelled) }446 if let c = completion {447 emit(.usage(TokenUsage(inputTokens: c.promptTokenCount, outputTokens: c.generationTokenCount,448 cachedInputTokens: c.cachedPromptTokenCount > 0 ? c.cachedPromptTokenCount : nil,449 reasoningTokens: c.reasoningTokenCount)))450 }451 emit(.messageCompleted(reason))452 return LocalGenerationResult(metrics: metrics, error: nil)453 }454455 // MARK: Conversions456457 /// Index of the first message of the trailing run the model has not answered yet (the last user turn, or trailing tool results).458 static func pendingStart(_ messages: [PromptMessage]) -> Int {459 var i = messages.count - 1460 if messages[i].role == .tool { while i > 0, messages[i - 1].role == .tool { i -= 1 } }461 return i462 }463464 static func chatMessage(_ m: PromptMessage) -> Chat.Message {465 switch m.role {466 case .user:467 let images: [UserInput.Image] = m.images.compactMap { CIImage(data: $0.data).map { .ciImage($0) } }468 return .user(m.text, images: images)469 case .assistant:470 let calls = m.toolCalls.map { call in471 MLXLMCommon.ToolCall(function: .init(name: call.name, arguments: (sendable(call.arguments) as? [String: any Sendable]) ?? [:]), id: call.id)472 }473 return .assistant(m.text, toolCalls: calls.isEmpty ? nil : calls)474 case .tool:475 return .tool(m.text, id: m.toolCallID, name: m.toolName)476 }477 }478479 static func toolSpec(_ tool: ToolDefinition) -> ToolSpec {480 ["type": "function",481 "function": ["name": tool.name, "description": tool.summary, "parameters": sendable(tool.parameters)] as [String: any Sendable]]482 }483484 static func sendable(_ value: AstraCore.JSONValue) -> any Sendable {485 switch value {486 case .null: return NSNull()487 case .bool(let b): return b488 case .number(let n): return n.rounded() == n && abs(n) < 9_007_199_254_740_992 ? Int(n) : n489 case .string(let s): return s490 case .array(let a): return a.map(sendable)491 case .object(let o): return o.mapValues(sendable)492 }493 }494495 static func jsonValue(_ arguments: [String: MLXLMCommon.JSONValue]) -> AstraCore.JSONValue {496 guard let data = try? JSONEncoder().encode(arguments), let value = AstraCore.JSONValue.parse(data) else { return .object([:]) }497 return value498 }499500 // MARK: Services501502 /// Exact token count with the model's own tokenizer.503 public func countTokens(_ text: String) async -> Int? {504 guard let container else { return nil }505 return await container.encode(text).count506 }507508 public var liveSessionKeys: [String] { sessionOrder }509510 public func resetSession(_ key: String?) {511 if let key { sessions[key] = nil; sessionOrder.removeAll { $0 == key } } else { sessions.removeAll(); sessionOrder.removeAll() }512 }513514 /// Releases the weights and every KV cache.515 public func unload() {516 sessions.removeAll(); sessionOrder.removeAll(); container = nil517 MLXMemoryFigures.clearCache()518 }519}520