Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Packages/AstraLocal/Sources/LocalEngine.swift10// Module : Packages11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation9091/// Owns the loaded on-device models: explicit load and unload, one generation at a time, automatic unloading under memory or thermal pressure.92public actor LocalEngine {93 public typealias Loader = @Sendable (_ modelID: String, _ directory: URL, _ progress: @escaping @Sendable (Double) -> Void) async throws -> any LocalModelRuntime9495 public struct Configuration: Sendable {96 /// Models kept in memory at once. 1 on iPhone and iPad.97 public var maxLoadedModels: Int98 /// Load an installed model on first use instead of failing with "not loaded".99 public var autoLoad = true100 /// Refuse a load the memory governor judges unlikely or too large (callers can force it).101 public var enforceMemoryVerdict = true102 public var defaultContextLength: Int?103 public init(maxLoadedModels: Int? = nil) {104 self.maxLoadedModels = max(1, maxLoadedModels ?? (DeviceFacts.current().platform == .macOS ? 2 : 1))105 }106 }107108 /// Injected readings and loader: the real ones by default, scripted ones in tests.109 public struct Environment: Sendable {110 public var memory: @Sendable () -> MemoryReading = { .current() }111 public var thermal: @Sendable () -> ThermalLevel = { .current() }112 public var mlxMemory: @Sendable () -> MLXMemoryFigures? = { MLXMemoryFigures.read() }113 public var loader: Loader = { id, directory, progress in try await MLXRuntime.load(modelID: id, directory: directory, progress: progress) }114 public init() {}115 }116117 public struct ActiveGeneration: Sendable, Hashable {118 public var id: UUID119 public var modelID: String120 public var sessionKey: String?121 public var startedAt: Date122 public var firstTokenAt: Date?123 /// Stream events carrying text so far: a live approximation, replaced by exact token counts in the final metrics.124 public var streamedChunks: Int125 }126127 public struct LoadingState: Sendable, Hashable {128 public var modelID: String129 /// Fraction of the weights read, 0…1.130 public var fraction: Double131 }132133 public struct Snapshot: Sendable, Hashable {134 public var loaded: [LoadedModelInfo] = []135 public var loading: LoadingState?136 public var active: ActiveGeneration?137 /// Generations waiting for the running one to finish.138 public var queued = 0139 public var memory: MemoryReading140 public var mlx: MLXMemoryFigures?141 public var thermal: ThermalLevel142 /// Current thermal warning, if any.143 public var warning: String?144 /// Why the engine last unloaded a model on its own ("memory pressure", "device too hot"); nil when it never did.145 public var lastAutomaticUnload: String?146 public func isLoaded(_ id: String) -> Bool { loaded.contains { $0.modelID == id } }147 }148149 private struct Entry { var runtime: any LocalModelRuntime; var lastUsed: Date }150151 public nonisolated let store: ModelStore152 public nonisolated let monitor: LocalRuntimeMonitor153 public nonisolated let memoryGovernor: MemoryGovernor154 public nonisolated let thermalGovernor: ThermalGovernor155 private var configuration: Configuration156 private let environment: Environment157 private var runtimes: [String: Entry] = [:]158 private var loadTasks: [String: Task<any LocalModelRuntime, Error>] = [:]159 private var loading: LoadingState?160 private var active: ActiveGeneration?161 private var activeWork: Task<LocalGenerationResult, Never>?162 private var queued = 0163 private var busy = false164 private var waiters: [CheckedContinuation<Void, Never>] = []165 private var lastAutomaticUnload: String?166 private var observers: [UUID: AsyncStream<Snapshot>.Continuation] = [:]167 private var pressureSource: DispatchSourceMemoryPressure?168 private var monitorTasks: [Task<Void, Never>] = []169170 public init(store: ModelStore, configuration: Configuration = Configuration(), monitor: LocalRuntimeMonitor = LocalRuntimeMonitor(),171 memoryGovernor: MemoryGovernor = MemoryGovernor(), thermalGovernor: ThermalGovernor = ThermalGovernor(), environment: Environment = Environment()) {172 self.store = store; self.configuration = configuration; self.monitor = monitor173 self.memoryGovernor = memoryGovernor; self.thermalGovernor = thermalGovernor; self.environment = environment174 }175176 public func setMaxLoadedModels(_ n: Int) { configuration.maxLoadedModels = max(1, n) }177178 // MARK: Observation179180 public func snapshot() -> Snapshot {181 let level = environment.thermal()182 return Snapshot(loaded: runtimes.values.map(\.runtime.info).sorted { $0.loadedAt < $1.loadedAt }, loading: loading, active: active, queued: queued,183 memory: environment.memory(), mlx: environment.mlxMemory(), thermal: level,184 warning: thermalGovernor.decision(for: level).warning, lastAutomaticUnload: lastAutomaticUnload)185 }186187 /// The current state first, then every change (load progress, generation start and end, unloads, pressure events).188 public func snapshots() -> AsyncStream<Snapshot> {189 let key = UUID()190 return AsyncStream(bufferingPolicy: .bufferingNewest(1)) { continuation in191 observers[key] = continuation192 continuation.yield(snapshot())193 continuation.onTermination = { _ in Task { await self.removeObserver(key) } }194 }195 }196 private func removeObserver(_ key: UUID) { observers[key] = nil }197 private func publish() { guard !observers.isEmpty else { return }; let s = snapshot(); for o in observers.values { o.yield(s) } }198199 // MARK: Fit200201 /// The governor's verdict for an installed or catalog model at the given context, with current readings.202 public nonisolated func verdict(for card: LocalModelCard, context: Int? = nil) -> FitVerdict {203 let reading = environment.memory()204 let context = context ?? memoryGovernor.recommendedMaxContext(for: card.memoryProfile, reading: reading) ?? min(card.contextLength, 2_048)205 return memoryGovernor.verdict(for: card.memoryProfile, context: context, reading: reading)206 }207208 // MARK: Loading209210 /// Loads an installed model. Progress is visible in `snapshots()`. Other models are unloaded first when the limit is reached.211 @discardableResult212 public func load(_ modelID: String, force: Bool = false) async throws -> LoadedModelInfo {213 if let entry = runtimes[modelID] { return entry.runtime.info }214 guard let directory = store.loadableDirectory(for: modelID) else {215 throw AstraError.local(store.card(id: modelID) == nil ? "“\(modelID)” is not downloaded." : "“\(modelID)” is not completely downloaded yet.")216 }217 return try await load(modelID: modelID, directory: directory, card: store.card(id: modelID), force: force)218 }219220 /// Loads a model from an explicit directory (developer tools, verification).221 @discardableResult222 public func load(modelID: String, directory: URL, card: LocalModelCard? = nil, force: Bool = false) async throws -> LoadedModelInfo {223 if let entry = runtimes[modelID] { return entry.runtime.info }224 if let pending = loadTasks[modelID] { return try await pending.value.info }225 let decision = thermalGovernor.decision(for: environment.thermal())226 guard decision.allowsNewGeneration else { throw AstraError.local(decision.warning ?? "The device is too hot to load a model.") }227228 // Make room first: the verdict is about what the device can give once the others are gone.229 while runtimes.count >= configuration.maxLoadedModels, let victim = idleVictim() { await unload(victim) }230 if runtimes.count >= configuration.maxLoadedModels {231 throw AstraError.local("Another model is generating. Wait for it to finish or stop it before loading “\(modelID)”.")232 }233 if let card, configuration.enforceMemoryVerdict, !force {234 let v = verdict(for: card, context: configuration.defaultContextLength)235 guard v.mayLoad else { throw AstraError.local(v.explanation) }236 }237 MLXMemoryFigures.configureCache(.current())238 let loader = environment.loader239 let report: @Sendable (Double) -> Void = { [weak self] fraction in Task { await self?.noteProgress(modelID, fraction) } }240 let task = Task<any LocalModelRuntime, Error> { try await loader(modelID, directory, report) }241 loadTasks[modelID] = task242 loading = LoadingState(modelID: modelID, fraction: 0); publish()243 do {244 let runtime = try await task.value245 loadTasks[modelID] = nil; loading = nil246 runtimes[modelID] = Entry(runtime: runtime, lastUsed: Date())247 publish()248 return runtime.info249 } catch {250 loadTasks[modelID] = nil; loading = nil; publish()251 throw AstraError.wrapLocal(error)252 }253 }254255 private func noteProgress(_ id: String, _ fraction: Double) {256 guard loadTasks[id] != nil, let current = loading, current.modelID == id, fraction > current.fraction else { return }257 loading = LoadingState(modelID: id, fraction: min(1, fraction)); publish()258 }259260 private func idleVictim() -> String? {261 runtimes.filter { $0.key != active?.modelID }.min { $0.value.lastUsed < $1.value.lastUsed }?.key262 }263264 public func unload(_ modelID: String) async {265 guard let entry = runtimes.removeValue(forKey: modelID) else { return }266 await entry.runtime.unload()267 publish()268 }269270 /// Unloads every model. A running generation is stopped first.271 public func unloadAll() async {272 await cancelActiveGeneration()273 for id in Array(runtimes.keys) { await unload(id) }274 }275276 public func loadedInfo(_ modelID: String) -> LoadedModelInfo? { runtimes[modelID]?.runtime.info }277278 /// Exact token count with a loaded model's tokenizer; nil when the model is not loaded (a model is never loaded just to count).279 public func countTokens(modelID: String, text: String) async -> Int? { await runtimes[modelID]?.runtime.countTokens(text) }280281 /// Drops the KV cache of one conversation (or all of them) for a loaded model.282 public func resetSession(modelID: String, key: String?) async { await runtimes[modelID]?.runtime.resetSession(key) }283284 // MARK: Generation285286 /// Streams one answer. Generations are serialized: a second request waits for the first.287 /// Cancelling the consumer (or `cancelActiveGeneration()`) stops generation promptly; the stream then ends by throwing `AstraError.cancelled`.288 public nonisolated func generate(_ request: GenerationRequest) -> AsyncThrowingStream<StreamEvent, Error> {289 AsyncThrowingStream { continuation in290 let task = Task { await self.run(request, continuation) }291 continuation.onTermination = { _ in task.cancel() }292 }293 }294295 /// Stops the running generation, if any, and waits until it has really stopped.296 public func cancelActiveGeneration() async {297 guard let work = activeWork else { return }298 work.cancel()299 _ = await work.value300 while active != nil { await Task.yield() } // let the generation's own bookkeeping finish301 }302303 private func run(_ request: GenerationRequest, _ continuation: AsyncThrowingStream<StreamEvent, Error>.Continuation) async {304 await acquire()305 defer { release() }306 if Task.isCancelled { continuation.finish(throwing: AstraError.cancelled); return }307308 let modelID = request.model.modelID309 let decision = thermalGovernor.decision(for: environment.thermal())310 if !decision.allowsNewGeneration {311 await automaticUnload(reason: "device too hot")312 continuation.finish(throwing: AstraError.local(decision.warning ?? "The device is too hot to run a model.")); return313 }314 var request = request315 let limited = decision.limit(maxTokens: request.parameters.maxOutputTokens, context: request.parameters.contextLength ?? configuration.defaultContextLength)316 request.parameters.maxOutputTokens = limited.maxTokens317 request.parameters.contextLength = limited.context318319 let runtime: any LocalModelRuntime320 if let entry = runtimes[modelID] { runtime = entry.runtime }321 else if configuration.autoLoad {322 do { _ = try await load(modelID); runtime = runtimes[modelID]!.runtime }323 catch { continuation.finish(throwing: AstraError.wrapLocal(error)); return }324 } else { continuation.finish(throwing: AstraError.local("“\(modelID)” is not loaded.")); return }325 if Task.isCancelled { continuation.finish(throwing: AstraError.cancelled); return }326327 let generation = ActiveGeneration(id: UUID(), modelID: modelID, sessionKey: request.sessionKey, startedAt: Date(), streamedChunks: 0)328 active = generation329 let counter = ChunkCounter()330 let work = Task { [request] () -> LocalGenerationResult in331 await runtime.perform(request) { event in332 switch event {333 case .textDelta, .reasoningSummaryDelta: counter.note()334 default: break335 }336 continuation.yield(event)337 }338 }339 activeWork = work340 publish()341 // Live progress for observers while the runtime works.342 let ticker = Task { [weak self] in343 while !Task.isCancelled {344 try? await Task.sleep(nanoseconds: 250_000_000)345 await self?.noteChunks(generation.id, counter.read())346 }347 }348 let result = await withTaskCancellationHandler { await work.value } onCancel: { work.cancel() }349 ticker.cancel()350 active = nil; activeWork = nil351 if var entry = runtimes[modelID] { entry.lastUsed = Date(); runtimes[modelID] = entry }352 monitor.record(result.metrics)353 publish()354 if let error = result.error { continuation.finish(throwing: error) } else { continuation.finish() }355 }356357 private func noteChunks(_ id: UUID, _ reading: (count: Int, first: Date?)) {358 guard var a = active, a.id == id, a.streamedChunks != reading.count else { return }359 a.streamedChunks = reading.count; a.firstTokenAt = reading.first360 active = a; publish()361 }362363 private func acquire() async {364 if !busy { busy = true; return }365 queued += 1; publish()366 await withCheckedContinuation { waiters.append($0) }367 queued = max(0, queued - 1)368 }369 private func release() {370 if waiters.isEmpty { busy = false } else { waiters.removeFirst().resume() }371 }372373 // MARK: Pressure374375 /// Starts listening to the system: memory pressure, memory warnings (iOS) and thermal state changes. Call once at launch.376 public func startMonitoring() {377 guard pressureSource == nil else { return }378 let source = DispatchSource.makeMemoryPressureSource(eventMask: [.warning, .critical], queue: .global(qos: .utility))379 source.setEventHandler { [weak self] in380 let critical = source.data.contains(.critical)381 Task { await self?.handleMemoryPressure(critical: critical) }382 }383 source.resume(); pressureSource = source384 monitorTasks.append(Task { [weak self] in385 // UIKit posts this by name; observing the name avoids importing UIKit into this module.386 let name = Notification.Name("UIApplicationDidReceiveMemoryWarningNotification")387 for await _ in NotificationCenter.default.notifications(named: name).map({ _ in () }) { await self?.handleMemoryPressure(critical: false) }388 })389 monitorTasks.append(Task { [weak self] in390 for await level in ThermalGovernor.levels() { await self?.handleThermal(level) }391 })392 }393394 public func stopMonitoring() {395 pressureSource?.cancel(); pressureSource = nil396 monitorTasks.forEach { $0.cancel() }; monitorTasks.removeAll()397 }398399 /// Memory warning: release MLX's buffer cache and unload models that are not generating.400 /// Critical pressure: also stop the running generation and unload everything, because the alternative is the system killing the app.401 public func handleMemoryPressure(critical: Bool) async {402 MLXMemoryFigures.clearCache()403 if critical { await cancelActiveGeneration() }404 let idle = runtimes.keys.filter { $0 != active?.modelID }405 guard !idle.isEmpty else { publish(); return }406 for id in idle { await unload(id) }407 lastAutomaticUnload = critical ? "critical memory pressure" : "memory warning"408 publish()409 }410411 /// Critical heat: stop and unload. Other levels only change the snapshot (limits are applied when a generation starts).412 public func handleThermal(_ level: ThermalLevel) async {413 if thermalGovernor.decision(for: level).shouldUnload { await cancelActiveGeneration(); await automaticUnload(reason: "device too hot") }414 publish()415 }416417 private func automaticUnload(reason: String) async {418 guard !runtimes.isEmpty else { return }419 for id in Array(runtimes.keys) where id != active?.modelID { await unload(id) }420 lastAutomaticUnload = reason421 publish()422 }423}424425final class ChunkCounter: @unchecked Sendable {426 private let lock = NSLock()427 private var count = 0428 private var first: Date?429 func note() { lock.withLock { count += 1; if first == nil { first = Date() } } }430 func read() -> (count: Int, first: Date?) { lock.withLock { (count, first) } }431}432433extension AstraError {434 /// Like `wrap`, but an unknown error becomes `.local` rather than a network error: nothing here touches the network.435 static func wrapLocal(_ error: Error) -> AstraError {436 if let e = error as? AstraError { return e }437 if error is CancellationError { return .cancelled }438 return .local((error as? LocalizedError)?.errorDescription ?? String(describing: error))439 }440}441