SPB Git

Repository Public

InferenceEngine.swift 15.9 KB 321 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatInference/Sources/InferenceEngine.swift3// Description: The engine every feature goes through: memory-aware loading and unloading, one on-device generation at a time, live measured telemetry.4import Foundation5import LocalChatCore67public struct LoadedModel: Sendable, Hashable, Identifiable {8    public var descriptor: ModelDescriptor9    public var info: RuntimeInfo10    public var loadedAt: Date11    public var lastUsedAt: Date12    public var id: String { descriptor.id }13}1415public struct ActiveGeneration: Sendable, Hashable, Identifiable {16    public var id: UUID17    public var modelID: String18    public var modelName: String19    public var backend: BackendKind20    public var startedAt: Date21    public var firstTokenAt: Date?22    public var generatedTokens: Int          // counted events while streaming; replaced by the backend's exact count at the end23    public var promptTokens: Int?24    public var contextLength: Int25    public var purpose: String26    /// Measured over the stream so far (text events per second, a live approximation of tokens/s until the final metrics arrive).27    public var liveRate: Double? {28        guard let firstTokenAt, generatedTokens > 1 else { return nil }29        let t = Date().timeIntervalSince(firstTokenAt); return t > 0 ? Double(generatedTokens) / t : nil30    }31}3233public struct EngineSnapshot: Sendable, Hashable {34    public var loaded: [LoadedModel] = []35    public var loading: [String: LoadProgress] = [:]36    public var active: [ActiveGeneration] = []37    public var queued: Int = 038    public var lastCompleted: CompletedGeneration?39    public var processFootprintBytes: Int64?40    public var mlxActiveBytes: Int64?41    public var mlxCacheBytes: Int64?42    public init() {}43    public func isLoaded(_ id: String) -> Bool { loaded.contains { $0.id == id } }44}4546public struct CompletedGeneration: Sendable, Hashable, Identifiable {47    public var id: UUID48    public var descriptor: ModelDescriptor49    public var metrics: GenerationMetrics50    public var loadTime: TimeInterval?51    public var contextLength: Int52    public var purpose: String53    public var finishedAt: Date54}5556public actor InferenceEngine: InferenceService {57    public typealias Catalog = @Sendable () async -> [ModelDescriptor]5859    private var backends: [BackendKind: any InferenceBackend] = [:]60    private let modelsRoot: URL61    private var catalog: Catalog62    private var runtimes: [String: (runtime: any ModelRuntime, model: LoadedModel)] = [:]63    private var loadTasks: [String: Task<any ModelRuntime, Error>] = [:]64    private var snapshot = EngineSnapshot()65    private var observers: [UUID: AsyncStream<EngineSnapshot>.Continuation] = [:]66    private var completionObservers: [UUID: AsyncStream<CompletedGeneration>.Continuation] = [:]67    private var localBusy = false68    private var waiters: [CheckedContinuation<Void, Never>] = []69    private var pressureSource: DispatchSourceMemoryPressure?70    /// Maximum number of on-device chat models kept in memory (1 on iPhone, more on a Mac when memory allows).71    public var maxLoadedLocalModels: Int72    public var device: DeviceProfile7374    public init(backends: [any InferenceBackend], modelsRoot: URL, device: DeviceProfile = .current(), catalog: @escaping Catalog = { [] }) {75        for b in backends { self.backends[b.kind] = b }76        self.modelsRoot = modelsRoot; self.catalog = catalog; self.device = device77        self.maxLoadedLocalModels = device.platform == .mac ? 3 : 178    }7980    public func setCatalog(_ catalog: @escaping Catalog) { self.catalog = catalog }81    public func register(_ backend: any InferenceBackend) { backends[backend.kind] = backend }8283    /// nil when the backend can run here, otherwise the reason it cannot.84    public nonisolated func staticUnavailableReason(_ kind: BackendKind, backends: [any InferenceBackend]) -> String? {85        backends.first { $0.kind == kind }.map { $0.unavailableReason } ?? "This backend is not part of this build."86    }87    public func unavailableReason(_ kind: BackendKind) -> String? {88        guard let b = backends[kind] else { return "The \(kind.displayName) backend is not part of this build." }89        return b.unavailableReason90    }91    public func availability() -> [BackendKind: String?] {92        var out: [BackendKind: String?] = [:]93        for kind in BackendKind.allCases { out[kind] = unavailableReason(kind) }94        return out95    }9697    // MARK: Observation9899    public func snapshots() -> AsyncStream<EngineSnapshot> {100        let id = UUID()101        return AsyncStream(bufferingPolicy: .bufferingNewest(1)) { continuation in102            observers[id] = continuation103            continuation.yield(refreshed())104            continuation.onTermination = { _ in Task { await self.removeObserver(id) } }105        }106    }107    public func completions() -> AsyncStream<CompletedGeneration> {108        let id = UUID()109        return AsyncStream { continuation in110            completionObservers[id] = continuation111            continuation.onTermination = { _ in Task { await self.removeCompletionObserver(id) } }112        }113    }114    private func removeObserver(_ id: UUID) { observers[id] = nil }115    private func removeCompletionObserver(_ id: UUID) { completionObservers[id] = nil }116    public func currentSnapshot() -> EngineSnapshot { refreshed() }117118    private func refreshed() -> EngineSnapshot {119        snapshot.loaded = runtimes.values.map(\.model).sorted { $0.loadedAt < $1.loadedAt }120        snapshot.processFootprintBytes = DeviceProfile.processFootprintBytes()121        let memory = MLXMemoryProbe.read()122        snapshot.mlxActiveBytes = memory?.active; snapshot.mlxCacheBytes = memory?.cache123        return snapshot124    }125    private func publish() { let s = refreshed(); for o in observers.values { o.yield(s) } }126127    // MARK: Loading128129    public func descriptor(for modelID: String) async -> ModelDescriptor? {130        if let loaded = runtimes[modelID] { return loaded.model.descriptor }131        return await catalog().first { $0.id == modelID }132    }133134    @discardableResult135    public func load(_ model: ModelDescriptor, force: Bool = false) async throws -> RuntimeInfo {136        try await runtime(for: model, force: force).info137    }138139    private func runtime(for model: ModelDescriptor, force: Bool = false) async throws -> any ModelRuntime {140        if var entry = runtimes[model.id] { entry.model.lastUsedAt = Date(); runtimes[model.id] = entry; return entry.runtime }141        if let pending = loadTasks[model.id] { return try await pending.value }142        guard let backend = backends[model.backend] else { throw InferenceError.backendUnavailable("The \(model.backend.displayName) backend is not part of this build.") }143        if let reason = backend.unavailableReason { throw InferenceError.backendUnavailable(reason) }144145        if model.backend.isOnDevice && model.backend != .appleFoundation {146            device = .current()147            try await makeRoom(for: model, force: force)148        }149        let root = modelsRoot150        let report: @Sendable (LoadProgress) -> Void = { [weak self] progress in Task { await self?.noteProgress(model.id, progress) } }151        let task = Task<any ModelRuntime, Error> { try await backend.load(model, modelsRoot: root, progress: report) }152        loadTasks[model.id] = task153        snapshot.loading[model.id] = LoadProgress(stage: .preparing); publish()154        do {155            let runtime = try await task.value156            loadTasks[model.id] = nil; snapshot.loading[model.id] = nil157            runtimes[model.id] = (runtime, LoadedModel(descriptor: model, info: runtime.info, loadedAt: Date(), lastUsedAt: Date()))158            startPressureMonitor(); publish()159            return runtime160        } catch {161            loadTasks[model.id] = nil; snapshot.loading[model.id] = nil; publish()162            throw error163        }164    }165166    private func noteProgress(_ id: String, _ progress: LoadProgress) {167        guard loadTasks[id] != nil else { return }168        snapshot.loading[id] = progress; publish()169    }170171    /// Unloads idle on-device models (least recently used first) until the new model fits; refuses when it cannot fit at all.172    private func makeRoom(for model: ModelDescriptor, force: Bool) async throws {173        let usable = device.usableModelMemoryBytes174        let need = model.estimatedMemoryBytes() ?? 0175        if !force, need > usable { throw InferenceError.insufficientMemory(needed: need, available: usable) }176        func localEntries() -> [LoadedModel] {177            runtimes.values.map(\.model).filter { $0.descriptor.backend == .mlx || $0.descriptor.backend == .gguf }.sorted { $0.lastUsedAt < $1.lastUsedAt }178        }179        func loadedBytes() -> Int64 { localEntries().reduce(0) { $0 + ($1.info.weightsMemoryBytes ?? $1.descriptor.sizeBytes ?? 0) } }180        let busy = Set(snapshot.active.map(\.modelID))181        while let victim = localEntries().first(where: { !busy.contains($0.id) }),182              localEntries().count >= maxLoadedLocalModels || loadedBytes() + need > usable {183            await unload(victim.id)184        }185    }186187    public func unload(_ modelID: String) async {188        guard let entry = runtimes.removeValue(forKey: modelID) else { return }189        await entry.runtime.unload(); publish()190    }191192    public func unloadAll() async { for id in Array(runtimes.keys) { await unload(id) } }193194    public func unloadIdle(olderThan interval: TimeInterval) async {195        let busy = Set(snapshot.active.map(\.modelID)); let now = Date()196        for entry in runtimes.values.map(\.model) where !busy.contains(entry.id) && now.timeIntervalSince(entry.lastUsedAt) > interval && entry.descriptor.backend != .remote {197            await unload(entry.id)198        }199    }200201    private func startPressureMonitor() {202        guard pressureSource == nil else { return }203        let source = DispatchSource.makeMemoryPressureSource(eventMask: [.warning, .critical], queue: .global(qos: .utility))204        source.setEventHandler { [weak self] in205            let critical = source.data.contains(.critical)206            Task { await self?.handleMemoryPressure(critical: critical) }207        }208        source.resume(); pressureSource = source209    }210211    private func handleMemoryPressure(critical: Bool) async {212        MLXMemoryProbe.clearCache()213        guard critical else { return }214        // never interrupt a running generation; release everything that is idle215        await unloadIdle(olderThan: 0)216    }217218    // MARK: Generation219220    public nonisolated func generate(modelID: String, request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error> {221        generate(modelID: modelID, request: request, purpose: "chat")222    }223224    public nonisolated func generate(modelID: String, request: GenerationRequest, purpose: String) -> AsyncThrowingStream<GenerationEvent, Error> {225        AsyncThrowingStream { continuation in226            let task = Task { await self.run(modelID: modelID, request: request, purpose: purpose, continuation: continuation) }227            continuation.onTermination = { _ in task.cancel() }228        }229    }230231    private func run(modelID: String, request: GenerationRequest, purpose: String, continuation: AsyncThrowingStream<GenerationEvent, Error>.Continuation) async {232        guard let model = await descriptor(for: modelID) else { continuation.finish(throwing: InferenceError.modelNotFound(modelID)); return }233        let serialize = model.backend != .remote234        if serialize { await acquireLocal() }235        defer { if serialize { releaseLocal() } }236        if Task.isCancelled { continuation.finish(throwing: InferenceError.cancelled); return }237        let runtime: any ModelRuntime238        do { runtime = try await self.runtime(for: model) } catch { continuation.finish(throwing: error); return }239240        let id = UUID()241        snapshot.active.append(ActiveGeneration(id: id, modelID: model.id, modelName: model.name, backend: model.backend, startedAt: Date(),242                                                generatedTokens: 0, contextLength: request.settings.contextLength, purpose: purpose))243        publish()244        var lastPublish = Date(); var final: GenerationMetrics?; var failure: Error?245        do {246            for try await event in runtime.generate(request) {247                switch event {248                case .text, .reasoning:249                    if let i = snapshot.active.firstIndex(where: { $0.id == id }) {250                        if snapshot.active[i].firstTokenAt == nil { snapshot.active[i].firstTokenAt = Date() }251                        snapshot.active[i].generatedTokens += 1252                    }253                    if Date().timeIntervalSince(lastPublish) > 0.2 { lastPublish = Date(); publish() }254                case .finished(let m): final = m255                case .toolCall: break256                }257                continuation.yield(event)258                if Task.isCancelled { break }259            }260        } catch { failure = error }261        snapshot.active.removeAll { $0.id == id }262        if var entry = runtimes[model.id] { entry.model.lastUsedAt = Date(); runtimes[model.id] = entry }263        if let final {264            let done = CompletedGeneration(id: id, descriptor: model, metrics: final, loadTime: runtime.info.loadTime,265                                           contextLength: request.settings.contextLength, purpose: purpose, finishedAt: Date())266            snapshot.lastCompleted = done267            for o in completionObservers.values { o.yield(done) }268        }269        publish()270        if let failure { continuation.finish(throwing: failure) } else { continuation.finish() }271    }272273    private func acquireLocal() async {274        if !localBusy { localBusy = true; return }275        snapshot.queued += 1; publish()276        await withCheckedContinuation { waiters.append($0) }277        snapshot.queued = max(0, snapshot.queued - 1)278    }279    private func releaseLocal() {280        if waiters.isEmpty { localBusy = false } else { waiters.removeFirst().resume() }281    }282283    public func countTokens(modelID: String, text: String) async -> Int? {284        guard let runtime = runtimes[modelID]?.runtime else { return nil }   // never load a model just to count285        return await runtime.countTokens(text)286    }287288    public func cacheStatistics(modelID: String) async -> CacheStatistics? { await runtimes[modelID]?.runtime.cacheStatistics() }289    public func resetSession(modelID: String, key: String?) async { await runtimes[modelID]?.runtime.resetSession(key) }290291    // MARK: InferenceService292293    public func listModels() async -> [ModelDescriptor] { await catalog() }294295    public func embed(modelID: String, texts: [String]) async throws -> [[Float]] {296        guard let model = await descriptor(for: modelID) else { throw InferenceError.modelNotFound(modelID) }297        let serialize = model.backend != .remote298        if serialize { await acquireLocal() }299        defer { if serialize { releaseLocal() } }300        return try await runtime(for: model).embed(texts)301    }302}303304/// An embedding model served by the engine, as the Knowledge Studio sees it.305public struct EngineEmbeddingProvider: EmbeddingProvider {306    public let identifier: String307    public let dimension: Int308    public let isRemote: Bool309    let engine: InferenceEngine310    public init(engine: InferenceEngine, model: ModelDescriptor, dimension: Int) {311        self.engine = engine; identifier = model.id; self.dimension = dimension; isRemote = model.backend == .remote312    }313    public func embed(_ texts: [String]) async throws -> [[Float]] { try await engine.embed(modelID: identifier, texts: texts) }314    /// Loads the model and measures its vector size with one probe sentence.315    public static func make(engine: InferenceEngine, model: ModelDescriptor) async throws -> EngineEmbeddingProvider {316        let probe = try await engine.embed(modelID: model.id, texts: ["dimension probe"])317        guard let n = probe.first?.count, n > 0 else { throw InferenceError.unsupported("This model returned no embedding.") }318        return EngineEmbeddingProvider(engine: engine, model: model, dimension: n)319    }320}321
Files main
Menu

Theme