SPB Git

Repository Public MIT

LocalEngine.swift 21.2 KB 441 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraLocal/Sources/LocalEngine.swift10// Module           : Packages11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation9091/// Owns the loaded on-device models: explicit load and unload, one generation at a time, automatic unloading under memory or thermal pressure.92public actor LocalEngine {93    public typealias Loader = @Sendable (_ modelID: String, _ directory: URL, _ progress: @escaping @Sendable (Double) -> Void) async throws -> any LocalModelRuntime9495    public struct Configuration: Sendable {96        /// Models kept in memory at once. 1 on iPhone and iPad.97        public var maxLoadedModels: Int98        /// Load an installed model on first use instead of failing with "not loaded".99        public var autoLoad = true100        /// Refuse a load the memory governor judges unlikely or too large (callers can force it).101        public var enforceMemoryVerdict = true102        public var defaultContextLength: Int?103        public init(maxLoadedModels: Int? = nil) {104            self.maxLoadedModels = max(1, maxLoadedModels ?? (DeviceFacts.current().platform == .macOS ? 2 : 1))105        }106    }107108    /// Injected readings and loader: the real ones by default, scripted ones in tests.109    public struct Environment: Sendable {110        public var memory: @Sendable () -> MemoryReading = { .current() }111        public var thermal: @Sendable () -> ThermalLevel = { .current() }112        public var mlxMemory: @Sendable () -> MLXMemoryFigures? = { MLXMemoryFigures.read() }113        public var loader: Loader = { id, directory, progress in try await MLXRuntime.load(modelID: id, directory: directory, progress: progress) }114        public init() {}115    }116117    public struct ActiveGeneration: Sendable, Hashable {118        public var id: UUID119        public var modelID: String120        public var sessionKey: String?121        public var startedAt: Date122        public var firstTokenAt: Date?123        /// Stream events carrying text so far: a live approximation, replaced by exact token counts in the final metrics.124        public var streamedChunks: Int125    }126127    public struct LoadingState: Sendable, Hashable {128        public var modelID: String129        /// Fraction of the weights read, 0…1.130        public var fraction: Double131    }132133    public struct Snapshot: Sendable, Hashable {134        public var loaded: [LoadedModelInfo] = []135        public var loading: LoadingState?136        public var active: ActiveGeneration?137        /// Generations waiting for the running one to finish.138        public var queued = 0139        public var memory: MemoryReading140        public var mlx: MLXMemoryFigures?141        public var thermal: ThermalLevel142        /// Current thermal warning, if any.143        public var warning: String?144        /// Why the engine last unloaded a model on its own ("memory pressure", "device too hot"); nil when it never did.145        public var lastAutomaticUnload: String?146        public func isLoaded(_ id: String) -> Bool { loaded.contains { $0.modelID == id } }147    }148149    private struct Entry { var runtime: any LocalModelRuntime; var lastUsed: Date }150151    public nonisolated let store: ModelStore152    public nonisolated let monitor: LocalRuntimeMonitor153    public nonisolated let memoryGovernor: MemoryGovernor154    public nonisolated let thermalGovernor: ThermalGovernor155    private var configuration: Configuration156    private let environment: Environment157    private var runtimes: [String: Entry] = [:]158    private var loadTasks: [String: Task<any LocalModelRuntime, Error>] = [:]159    private var loading: LoadingState?160    private var active: ActiveGeneration?161    private var activeWork: Task<LocalGenerationResult, Never>?162    private var queued = 0163    private var busy = false164    private var waiters: [CheckedContinuation<Void, Never>] = []165    private var lastAutomaticUnload: String?166    private var observers: [UUID: AsyncStream<Snapshot>.Continuation] = [:]167    private var pressureSource: DispatchSourceMemoryPressure?168    private var monitorTasks: [Task<Void, Never>] = []169170    public init(store: ModelStore, configuration: Configuration = Configuration(), monitor: LocalRuntimeMonitor = LocalRuntimeMonitor(),171                memoryGovernor: MemoryGovernor = MemoryGovernor(), thermalGovernor: ThermalGovernor = ThermalGovernor(), environment: Environment = Environment()) {172        self.store = store; self.configuration = configuration; self.monitor = monitor173        self.memoryGovernor = memoryGovernor; self.thermalGovernor = thermalGovernor; self.environment = environment174    }175176    public func setMaxLoadedModels(_ n: Int) { configuration.maxLoadedModels = max(1, n) }177178    // MARK: Observation179180    public func snapshot() -> Snapshot {181        let level = environment.thermal()182        return Snapshot(loaded: runtimes.values.map(\.runtime.info).sorted { $0.loadedAt < $1.loadedAt }, loading: loading, active: active, queued: queued,183                        memory: environment.memory(), mlx: environment.mlxMemory(), thermal: level,184                        warning: thermalGovernor.decision(for: level).warning, lastAutomaticUnload: lastAutomaticUnload)185    }186187    /// The current state first, then every change (load progress, generation start and end, unloads, pressure events).188    public func snapshots() -> AsyncStream<Snapshot> {189        let key = UUID()190        return AsyncStream(bufferingPolicy: .bufferingNewest(1)) { continuation in191            observers[key] = continuation192            continuation.yield(snapshot())193            continuation.onTermination = { _ in Task { await self.removeObserver(key) } }194        }195    }196    private func removeObserver(_ key: UUID) { observers[key] = nil }197    private func publish() { guard !observers.isEmpty else { return }; let s = snapshot(); for o in observers.values { o.yield(s) } }198199    // MARK: Fit200201    /// The governor's verdict for an installed or catalog model at the given context, with current readings.202    public nonisolated func verdict(for card: LocalModelCard, context: Int? = nil) -> FitVerdict {203        let reading = environment.memory()204        let context = context ?? memoryGovernor.recommendedMaxContext(for: card.memoryProfile, reading: reading) ?? min(card.contextLength, 2_048)205        return memoryGovernor.verdict(for: card.memoryProfile, context: context, reading: reading)206    }207208    // MARK: Loading209210    /// Loads an installed model. Progress is visible in `snapshots()`. Other models are unloaded first when the limit is reached.211    @discardableResult212    public func load(_ modelID: String, force: Bool = false) async throws -> LoadedModelInfo {213        if let entry = runtimes[modelID] { return entry.runtime.info }214        guard let directory = store.loadableDirectory(for: modelID) else {215            throw AstraError.local(store.card(id: modelID) == nil ? "“\(modelID)” is not downloaded." : "“\(modelID)” is not completely downloaded yet.")216        }217        return try await load(modelID: modelID, directory: directory, card: store.card(id: modelID), force: force)218    }219220    /// Loads a model from an explicit directory (developer tools, verification).221    @discardableResult222    public func load(modelID: String, directory: URL, card: LocalModelCard? = nil, force: Bool = false) async throws -> LoadedModelInfo {223        if let entry = runtimes[modelID] { return entry.runtime.info }224        if let pending = loadTasks[modelID] { return try await pending.value.info }225        let decision = thermalGovernor.decision(for: environment.thermal())226        guard decision.allowsNewGeneration else { throw AstraError.local(decision.warning ?? "The device is too hot to load a model.") }227228        // Make room first: the verdict is about what the device can give once the others are gone.229        while runtimes.count >= configuration.maxLoadedModels, let victim = idleVictim() { await unload(victim) }230        if runtimes.count >= configuration.maxLoadedModels {231            throw AstraError.local("Another model is generating. Wait for it to finish or stop it before loading “\(modelID)”.")232        }233        if let card, configuration.enforceMemoryVerdict, !force {234            let v = verdict(for: card, context: configuration.defaultContextLength)235            guard v.mayLoad else { throw AstraError.local(v.explanation) }236        }237        MLXMemoryFigures.configureCache(.current())238        let loader = environment.loader239        let report: @Sendable (Double) -> Void = { [weak self] fraction in Task { await self?.noteProgress(modelID, fraction) } }240        let task = Task<any LocalModelRuntime, Error> { try await loader(modelID, directory, report) }241        loadTasks[modelID] = task242        loading = LoadingState(modelID: modelID, fraction: 0); publish()243        do {244            let runtime = try await task.value245            loadTasks[modelID] = nil; loading = nil246            runtimes[modelID] = Entry(runtime: runtime, lastUsed: Date())247            publish()248            return runtime.info249        } catch {250            loadTasks[modelID] = nil; loading = nil; publish()251            throw AstraError.wrapLocal(error)252        }253    }254255    private func noteProgress(_ id: String, _ fraction: Double) {256        guard loadTasks[id] != nil, let current = loading, current.modelID == id, fraction > current.fraction else { return }257        loading = LoadingState(modelID: id, fraction: min(1, fraction)); publish()258    }259260    private func idleVictim() -> String? {261        runtimes.filter { $0.key != active?.modelID }.min { $0.value.lastUsed < $1.value.lastUsed }?.key262    }263264    public func unload(_ modelID: String) async {265        guard let entry = runtimes.removeValue(forKey: modelID) else { return }266        await entry.runtime.unload()267        publish()268    }269270    /// Unloads every model. A running generation is stopped first.271    public func unloadAll() async {272        await cancelActiveGeneration()273        for id in Array(runtimes.keys) { await unload(id) }274    }275276    public func loadedInfo(_ modelID: String) -> LoadedModelInfo? { runtimes[modelID]?.runtime.info }277278    /// Exact token count with a loaded model's tokenizer; nil when the model is not loaded (a model is never loaded just to count).279    public func countTokens(modelID: String, text: String) async -> Int? { await runtimes[modelID]?.runtime.countTokens(text) }280281    /// Drops the KV cache of one conversation (or all of them) for a loaded model.282    public func resetSession(modelID: String, key: String?) async { await runtimes[modelID]?.runtime.resetSession(key) }283284    // MARK: Generation285286    /// Streams one answer. Generations are serialized: a second request waits for the first.287    /// Cancelling the consumer (or `cancelActiveGeneration()`) stops generation promptly; the stream then ends by throwing `AstraError.cancelled`.288    public nonisolated func generate(_ request: GenerationRequest) -> AsyncThrowingStream<StreamEvent, Error> {289        AsyncThrowingStream { continuation in290            let task = Task { await self.run(request, continuation) }291            continuation.onTermination = { _ in task.cancel() }292        }293    }294295    /// Stops the running generation, if any, and waits until it has really stopped.296    public func cancelActiveGeneration() async {297        guard let work = activeWork else { return }298        work.cancel()299        _ = await work.value300        while active != nil { await Task.yield() }   // let the generation's own bookkeeping finish301    }302303    private func run(_ request: GenerationRequest, _ continuation: AsyncThrowingStream<StreamEvent, Error>.Continuation) async {304        await acquire()305        defer { release() }306        if Task.isCancelled { continuation.finish(throwing: AstraError.cancelled); return }307308        let modelID = request.model.modelID309        let decision = thermalGovernor.decision(for: environment.thermal())310        if !decision.allowsNewGeneration {311            await automaticUnload(reason: "device too hot")312            continuation.finish(throwing: AstraError.local(decision.warning ?? "The device is too hot to run a model.")); return313        }314        var request = request315        let limited = decision.limit(maxTokens: request.parameters.maxOutputTokens, context: request.parameters.contextLength ?? configuration.defaultContextLength)316        request.parameters.maxOutputTokens = limited.maxTokens317        request.parameters.contextLength = limited.context318319        let runtime: any LocalModelRuntime320        if let entry = runtimes[modelID] { runtime = entry.runtime }321        else if configuration.autoLoad {322            do { _ = try await load(modelID); runtime = runtimes[modelID]!.runtime }323            catch { continuation.finish(throwing: AstraError.wrapLocal(error)); return }324        } else { continuation.finish(throwing: AstraError.local("“\(modelID)” is not loaded.")); return }325        if Task.isCancelled { continuation.finish(throwing: AstraError.cancelled); return }326327        let generation = ActiveGeneration(id: UUID(), modelID: modelID, sessionKey: request.sessionKey, startedAt: Date(), streamedChunks: 0)328        active = generation329        let counter = ChunkCounter()330        let work = Task { [request] () -> LocalGenerationResult in331            await runtime.perform(request) { event in332                switch event {333                case .textDelta, .reasoningSummaryDelta: counter.note()334                default: break335                }336                continuation.yield(event)337            }338        }339        activeWork = work340        publish()341        // Live progress for observers while the runtime works.342        let ticker = Task { [weak self] in343            while !Task.isCancelled {344                try? await Task.sleep(nanoseconds: 250_000_000)345                await self?.noteChunks(generation.id, counter.read())346            }347        }348        let result = await withTaskCancellationHandler { await work.value } onCancel: { work.cancel() }349        ticker.cancel()350        active = nil; activeWork = nil351        if var entry = runtimes[modelID] { entry.lastUsed = Date(); runtimes[modelID] = entry }352        monitor.record(result.metrics)353        publish()354        if let error = result.error { continuation.finish(throwing: error) } else { continuation.finish() }355    }356357    private func noteChunks(_ id: UUID, _ reading: (count: Int, first: Date?)) {358        guard var a = active, a.id == id, a.streamedChunks != reading.count else { return }359        a.streamedChunks = reading.count; a.firstTokenAt = reading.first360        active = a; publish()361    }362363    private func acquire() async {364        if !busy { busy = true; return }365        queued += 1; publish()366        await withCheckedContinuation { waiters.append($0) }367        queued = max(0, queued - 1)368    }369    private func release() {370        if waiters.isEmpty { busy = false } else { waiters.removeFirst().resume() }371    }372373    // MARK: Pressure374375    /// Starts listening to the system: memory pressure, memory warnings (iOS) and thermal state changes. Call once at launch.376    public func startMonitoring() {377        guard pressureSource == nil else { return }378        let source = DispatchSource.makeMemoryPressureSource(eventMask: [.warning, .critical], queue: .global(qos: .utility))379        source.setEventHandler { [weak self] in380            let critical = source.data.contains(.critical)381            Task { await self?.handleMemoryPressure(critical: critical) }382        }383        source.resume(); pressureSource = source384        monitorTasks.append(Task { [weak self] in385            // UIKit posts this by name; observing the name avoids importing UIKit into this module.386            let name = Notification.Name("UIApplicationDidReceiveMemoryWarningNotification")387            for await _ in NotificationCenter.default.notifications(named: name).map({ _ in () }) { await self?.handleMemoryPressure(critical: false) }388        })389        monitorTasks.append(Task { [weak self] in390            for await level in ThermalGovernor.levels() { await self?.handleThermal(level) }391        })392    }393394    public func stopMonitoring() {395        pressureSource?.cancel(); pressureSource = nil396        monitorTasks.forEach { $0.cancel() }; monitorTasks.removeAll()397    }398399    /// Memory warning: release MLX's buffer cache and unload models that are not generating.400    /// Critical pressure: also stop the running generation and unload everything, because the alternative is the system killing the app.401    public func handleMemoryPressure(critical: Bool) async {402        MLXMemoryFigures.clearCache()403        if critical { await cancelActiveGeneration() }404        let idle = runtimes.keys.filter { $0 != active?.modelID }405        guard !idle.isEmpty else { publish(); return }406        for id in idle { await unload(id) }407        lastAutomaticUnload = critical ? "critical memory pressure" : "memory warning"408        publish()409    }410411    /// Critical heat: stop and unload. Other levels only change the snapshot (limits are applied when a generation starts).412    public func handleThermal(_ level: ThermalLevel) async {413        if thermalGovernor.decision(for: level).shouldUnload { await cancelActiveGeneration(); await automaticUnload(reason: "device too hot") }414        publish()415    }416417    private func automaticUnload(reason: String) async {418        guard !runtimes.isEmpty else { return }419        for id in Array(runtimes.keys) where id != active?.modelID { await unload(id) }420        lastAutomaticUnload = reason421        publish()422    }423}424425final class ChunkCounter: @unchecked Sendable {426    private let lock = NSLock()427    private var count = 0428    private var first: Date?429    func note() { lock.withLock { count += 1; if first == nil { first = Date() } } }430    func read() -> (count: Int, first: Date?) { lock.withLock { (count, first) } }431}432433extension AstraError {434    /// Like `wrap`, but an unknown error becomes `.local` rather than a network error: nothing here touches the network.435    static func wrapLocal(_ error: Error) -> AstraError {436        if let e = error as? AstraError { return e }437        if error is CancellationError { return .cancelled }438        return .local((error as? LocalizedError)?.errorDescription ?? String(describing: error))439    }440}441
Files main
Menu

Theme