Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatInference/Sources/InferenceEngine.swift3// Description: The engine every feature goes through: memory-aware loading and unloading, one on-device generation at a time, live measured telemetry.4import Foundation5import LocalChatCore67public struct LoadedModel: Sendable, Hashable, Identifiable {8 public var descriptor: ModelDescriptor9 public var info: RuntimeInfo10 public var loadedAt: Date11 public var lastUsedAt: Date12 public var id: String { descriptor.id }13}1415public struct ActiveGeneration: Sendable, Hashable, Identifiable {16 public var id: UUID17 public var modelID: String18 public var modelName: String19 public var backend: BackendKind20 public var startedAt: Date21 public var firstTokenAt: Date?22 public var generatedTokens: Int // counted events while streaming; replaced by the backend's exact count at the end23 public var promptTokens: Int?24 public var contextLength: Int25 public var purpose: String26 /// Measured over the stream so far (text events per second, a live approximation of tokens/s until the final metrics arrive).27 public var liveRate: Double? {28 guard let firstTokenAt, generatedTokens > 1 else { return nil }29 let t = Date().timeIntervalSince(firstTokenAt); return t > 0 ? Double(generatedTokens) / t : nil30 }31}3233public struct EngineSnapshot: Sendable, Hashable {34 public var loaded: [LoadedModel] = []35 public var loading: [String: LoadProgress] = [:]36 public var active: [ActiveGeneration] = []37 public var queued: Int = 038 public var lastCompleted: CompletedGeneration?39 public var processFootprintBytes: Int64?40 public var mlxActiveBytes: Int64?41 public var mlxCacheBytes: Int64?42 public init() {}43 public func isLoaded(_ id: String) -> Bool { loaded.contains { $0.id == id } }44}4546public struct CompletedGeneration: Sendable, Hashable, Identifiable {47 public var id: UUID48 public var descriptor: ModelDescriptor49 public var metrics: GenerationMetrics50 public var loadTime: TimeInterval?51 public var contextLength: Int52 public var purpose: String53 public var finishedAt: Date54}5556public actor InferenceEngine: InferenceService {57 public typealias Catalog = @Sendable () async -> [ModelDescriptor]5859 private var backends: [BackendKind: any InferenceBackend] = [:]60 private let modelsRoot: URL61 private var catalog: Catalog62 private var runtimes: [String: (runtime: any ModelRuntime, model: LoadedModel)] = [:]63 private var loadTasks: [String: Task<any ModelRuntime, Error>] = [:]64 private var snapshot = EngineSnapshot()65 private var observers: [UUID: AsyncStream<EngineSnapshot>.Continuation] = [:]66 private var completionObservers: [UUID: AsyncStream<CompletedGeneration>.Continuation] = [:]67 private var localBusy = false68 private var waiters: [CheckedContinuation<Void, Never>] = []69 private var pressureSource: DispatchSourceMemoryPressure?70 /// Maximum number of on-device chat models kept in memory (1 on iPhone, more on a Mac when memory allows).71 public var maxLoadedLocalModels: Int72 public var device: DeviceProfile7374 public init(backends: [any InferenceBackend], modelsRoot: URL, device: DeviceProfile = .current(), catalog: @escaping Catalog = { [] }) {75 for b in backends { self.backends[b.kind] = b }76 self.modelsRoot = modelsRoot; self.catalog = catalog; self.device = device77 self.maxLoadedLocalModels = device.platform == .mac ? 3 : 178 }7980 public func setCatalog(_ catalog: @escaping Catalog) { self.catalog = catalog }81 public func register(_ backend: any InferenceBackend) { backends[backend.kind] = backend }8283 /// nil when the backend can run here, otherwise the reason it cannot.84 public nonisolated func staticUnavailableReason(_ kind: BackendKind, backends: [any InferenceBackend]) -> String? {85 backends.first { $0.kind == kind }.map { $0.unavailableReason } ?? "This backend is not part of this build."86 }87 public func unavailableReason(_ kind: BackendKind) -> String? {88 guard let b = backends[kind] else { return "The \(kind.displayName) backend is not part of this build." }89 return b.unavailableReason90 }91 public func availability() -> [BackendKind: String?] {92 var out: [BackendKind: String?] = [:]93 for kind in BackendKind.allCases { out[kind] = unavailableReason(kind) }94 return out95 }9697 // MARK: Observation9899 public func snapshots() -> AsyncStream<EngineSnapshot> {100 let id = UUID()101 return AsyncStream(bufferingPolicy: .bufferingNewest(1)) { continuation in102 observers[id] = continuation103 continuation.yield(refreshed())104 continuation.onTermination = { _ in Task { await self.removeObserver(id) } }105 }106 }107 public func completions() -> AsyncStream<CompletedGeneration> {108 let id = UUID()109 return AsyncStream { continuation in110 completionObservers[id] = continuation111 continuation.onTermination = { _ in Task { await self.removeCompletionObserver(id) } }112 }113 }114 private func removeObserver(_ id: UUID) { observers[id] = nil }115 private func removeCompletionObserver(_ id: UUID) { completionObservers[id] = nil }116 public func currentSnapshot() -> EngineSnapshot { refreshed() }117118 private func refreshed() -> EngineSnapshot {119 snapshot.loaded = runtimes.values.map(\.model).sorted { $0.loadedAt < $1.loadedAt }120 snapshot.processFootprintBytes = DeviceProfile.processFootprintBytes()121 let memory = MLXMemoryProbe.read()122 snapshot.mlxActiveBytes = memory?.active; snapshot.mlxCacheBytes = memory?.cache123 return snapshot124 }125 private func publish() { let s = refreshed(); for o in observers.values { o.yield(s) } }126127 // MARK: Loading128129 public func descriptor(for modelID: String) async -> ModelDescriptor? {130 if let loaded = runtimes[modelID] { return loaded.model.descriptor }131 return await catalog().first { $0.id == modelID }132 }133134 @discardableResult135 public func load(_ model: ModelDescriptor, force: Bool = false) async throws -> RuntimeInfo {136 try await runtime(for: model, force: force).info137 }138139 private func runtime(for model: ModelDescriptor, force: Bool = false) async throws -> any ModelRuntime {140 if var entry = runtimes[model.id] { entry.model.lastUsedAt = Date(); runtimes[model.id] = entry; return entry.runtime }141 if let pending = loadTasks[model.id] { return try await pending.value }142 guard let backend = backends[model.backend] else { throw InferenceError.backendUnavailable("The \(model.backend.displayName) backend is not part of this build.") }143 if let reason = backend.unavailableReason { throw InferenceError.backendUnavailable(reason) }144145 if model.backend.isOnDevice && model.backend != .appleFoundation {146 device = .current()147 try await makeRoom(for: model, force: force)148 }149 let root = modelsRoot150 let report: @Sendable (LoadProgress) -> Void = { [weak self] progress in Task { await self?.noteProgress(model.id, progress) } }151 let task = Task<any ModelRuntime, Error> { try await backend.load(model, modelsRoot: root, progress: report) }152 loadTasks[model.id] = task153 snapshot.loading[model.id] = LoadProgress(stage: .preparing); publish()154 do {155 let runtime = try await task.value156 loadTasks[model.id] = nil; snapshot.loading[model.id] = nil157 runtimes[model.id] = (runtime, LoadedModel(descriptor: model, info: runtime.info, loadedAt: Date(), lastUsedAt: Date()))158 startPressureMonitor(); publish()159 return runtime160 } catch {161 loadTasks[model.id] = nil; snapshot.loading[model.id] = nil; publish()162 throw error163 }164 }165166 private func noteProgress(_ id: String, _ progress: LoadProgress) {167 guard loadTasks[id] != nil else { return }168 snapshot.loading[id] = progress; publish()169 }170171 /// Unloads idle on-device models (least recently used first) until the new model fits; refuses when it cannot fit at all.172 private func makeRoom(for model: ModelDescriptor, force: Bool) async throws {173 let usable = device.usableModelMemoryBytes174 let need = model.estimatedMemoryBytes() ?? 0175 if !force, need > usable { throw InferenceError.insufficientMemory(needed: need, available: usable) }176 func localEntries() -> [LoadedModel] {177 runtimes.values.map(\.model).filter { $0.descriptor.backend == .mlx || $0.descriptor.backend == .gguf }.sorted { $0.lastUsedAt < $1.lastUsedAt }178 }179 func loadedBytes() -> Int64 { localEntries().reduce(0) { $0 + ($1.info.weightsMemoryBytes ?? $1.descriptor.sizeBytes ?? 0) } }180 let busy = Set(snapshot.active.map(\.modelID))181 while let victim = localEntries().first(where: { !busy.contains($0.id) }),182 localEntries().count >= maxLoadedLocalModels || loadedBytes() + need > usable {183 await unload(victim.id)184 }185 }186187 public func unload(_ modelID: String) async {188 guard let entry = runtimes.removeValue(forKey: modelID) else { return }189 await entry.runtime.unload(); publish()190 }191192 public func unloadAll() async { for id in Array(runtimes.keys) { await unload(id) } }193194 public func unloadIdle(olderThan interval: TimeInterval) async {195 let busy = Set(snapshot.active.map(\.modelID)); let now = Date()196 for entry in runtimes.values.map(\.model) where !busy.contains(entry.id) && now.timeIntervalSince(entry.lastUsedAt) > interval && entry.descriptor.backend != .remote {197 await unload(entry.id)198 }199 }200201 private func startPressureMonitor() {202 guard pressureSource == nil else { return }203 let source = DispatchSource.makeMemoryPressureSource(eventMask: [.warning, .critical], queue: .global(qos: .utility))204 source.setEventHandler { [weak self] in205 let critical = source.data.contains(.critical)206 Task { await self?.handleMemoryPressure(critical: critical) }207 }208 source.resume(); pressureSource = source209 }210211 private func handleMemoryPressure(critical: Bool) async {212 MLXMemoryProbe.clearCache()213 guard critical else { return }214 // never interrupt a running generation; release everything that is idle215 await unloadIdle(olderThan: 0)216 }217218 // MARK: Generation219220 public nonisolated func generate(modelID: String, request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error> {221 generate(modelID: modelID, request: request, purpose: "chat")222 }223224 public nonisolated func generate(modelID: String, request: GenerationRequest, purpose: String) -> AsyncThrowingStream<GenerationEvent, Error> {225 AsyncThrowingStream { continuation in226 let task = Task { await self.run(modelID: modelID, request: request, purpose: purpose, continuation: continuation) }227 continuation.onTermination = { _ in task.cancel() }228 }229 }230231 private func run(modelID: String, request: GenerationRequest, purpose: String, continuation: AsyncThrowingStream<GenerationEvent, Error>.Continuation) async {232 guard let model = await descriptor(for: modelID) else { continuation.finish(throwing: InferenceError.modelNotFound(modelID)); return }233 let serialize = model.backend != .remote234 if serialize { await acquireLocal() }235 defer { if serialize { releaseLocal() } }236 if Task.isCancelled { continuation.finish(throwing: InferenceError.cancelled); return }237 let runtime: any ModelRuntime238 do { runtime = try await self.runtime(for: model) } catch { continuation.finish(throwing: error); return }239240 let id = UUID()241 snapshot.active.append(ActiveGeneration(id: id, modelID: model.id, modelName: model.name, backend: model.backend, startedAt: Date(),242 generatedTokens: 0, contextLength: request.settings.contextLength, purpose: purpose))243 publish()244 var lastPublish = Date(); var final: GenerationMetrics?; var failure: Error?245 do {246 for try await event in runtime.generate(request) {247 switch event {248 case .text, .reasoning:249 if let i = snapshot.active.firstIndex(where: { $0.id == id }) {250 if snapshot.active[i].firstTokenAt == nil { snapshot.active[i].firstTokenAt = Date() }251 snapshot.active[i].generatedTokens += 1252 }253 if Date().timeIntervalSince(lastPublish) > 0.2 { lastPublish = Date(); publish() }254 case .finished(let m): final = m255 case .toolCall: break256 }257 continuation.yield(event)258 if Task.isCancelled { break }259 }260 } catch { failure = error }261 snapshot.active.removeAll { $0.id == id }262 if var entry = runtimes[model.id] { entry.model.lastUsedAt = Date(); runtimes[model.id] = entry }263 if let final {264 let done = CompletedGeneration(id: id, descriptor: model, metrics: final, loadTime: runtime.info.loadTime,265 contextLength: request.settings.contextLength, purpose: purpose, finishedAt: Date())266 snapshot.lastCompleted = done267 for o in completionObservers.values { o.yield(done) }268 }269 publish()270 if let failure { continuation.finish(throwing: failure) } else { continuation.finish() }271 }272273 private func acquireLocal() async {274 if !localBusy { localBusy = true; return }275 snapshot.queued += 1; publish()276 await withCheckedContinuation { waiters.append($0) }277 snapshot.queued = max(0, snapshot.queued - 1)278 }279 private func releaseLocal() {280 if waiters.isEmpty { localBusy = false } else { waiters.removeFirst().resume() }281 }282283 public func countTokens(modelID: String, text: String) async -> Int? {284 guard let runtime = runtimes[modelID]?.runtime else { return nil } // never load a model just to count285 return await runtime.countTokens(text)286 }287288 public func cacheStatistics(modelID: String) async -> CacheStatistics? { await runtimes[modelID]?.runtime.cacheStatistics() }289 public func resetSession(modelID: String, key: String?) async { await runtimes[modelID]?.runtime.resetSession(key) }290291 // MARK: InferenceService292293 public func listModels() async -> [ModelDescriptor] { await catalog() }294295 public func embed(modelID: String, texts: [String]) async throws -> [[Float]] {296 guard let model = await descriptor(for: modelID) else { throw InferenceError.modelNotFound(modelID) }297 let serialize = model.backend != .remote298 if serialize { await acquireLocal() }299 defer { if serialize { releaseLocal() } }300 return try await runtime(for: model).embed(texts)301 }302}303304/// An embedding model served by the engine, as the Knowledge Studio sees it.305public struct EngineEmbeddingProvider: EmbeddingProvider {306 public let identifier: String307 public let dimension: Int308 public let isRemote: Bool309 let engine: InferenceEngine310 public init(engine: InferenceEngine, model: ModelDescriptor, dimension: Int) {311 self.engine = engine; identifier = model.id; self.dimension = dimension; isRemote = model.backend == .remote312 }313 public func embed(_ texts: [String]) async throws -> [[Float]] { try await engine.embed(modelID: identifier, texts: texts) }314 /// Loads the model and measures its vector size with one probe sentence.315 public static func make(engine: InferenceEngine, model: ModelDescriptor) async throws -> EngineEmbeddingProvider {316 let probe = try await engine.embed(modelID: model.id, texts: ["dimension probe"])317 guard let n = probe.first?.count, n > 0 else { throw InferenceError.unsupported("This model returned no embedding.") }318 return EngineEmbeddingProvider(engine: engine, model: model, dimension: n)319 }320}321