Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatInference/Sources/AppleFoundationBackend.swift3// Description: The system language model (Apple Intelligence, FoundationModels) as a zero-download backend on OS 26+ when the device supports it.4import Foundation5import LocalChatCore6#if canImport(FoundationModels)7import FoundationModels8#endif910public struct AppleFoundationBackend: InferenceBackend {11 public let kind: BackendKind = .appleFoundation12 public static let modelID = "apple:system"13 public init() {}1415 public var isAvailable: Bool { unavailableReason == nil }1617 public var unavailableReason: String? {18 #if canImport(FoundationModels)19 if #available(macOS 26.0, iOS 26.0, *) {20 switch SystemLanguageModel.default.availability {21 case .available: return nil22 case .unavailable(.deviceNotEligible): return "This device does not support Apple Intelligence."23 case .unavailable(.appleIntelligenceNotEnabled): return "Apple Intelligence is turned off in Settings."24 case .unavailable(.modelNotReady): return "The Apple Intelligence model is still downloading."25 case .unavailable: return "The Apple Intelligence model is unavailable."26 }27 }28 return "Apple Intelligence needs iOS 26 / macOS 26 or later."29 #else30 return "Apple Intelligence is not available on this platform."31 #endif32 }3334 /// The descriptor to list in the library when the system model can be used.35 public var descriptor: ModelDescriptor? {36 guard isAvailable else { return nil }37 return ModelDescriptor(id: Self.modelID, name: "Apple Intelligence (on-device)", backend: .appleFoundation, format: .system,38 architecture: "system", contextLength: contextSize, capabilities: [.chat], license: "Apple system model")39 }4041 var contextSize: Int? {42 #if canImport(FoundationModels)43 if #available(macOS 27.0, iOS 27.0, *) { return SystemLanguageModel.default.contextSize }44 #endif45 return nil46 }4748 public func load(_ model: ModelDescriptor, modelsRoot: URL, progress: @escaping @Sendable (LoadProgress) -> Void) async throws -> any ModelRuntime {49 if let reason = unavailableReason { throw InferenceError.backendUnavailable(reason) }50 #if canImport(FoundationModels)51 if #available(macOS 26.0, iOS 26.0, *) {52 let start = Date()53 progress(LoadProgress(stage: .ready, fraction: 1))54 return AppleFoundationRuntime(info: RuntimeInfo(modelID: model.id, backend: .appleFoundation, loadTime: Date().timeIntervalSince(start),55 contextLength: contextSize, capabilities: [.chat]))56 }57 #endif58 throw InferenceError.backendUnavailable("Apple Intelligence is unavailable.")59 }60}6162#if canImport(FoundationModels)63@available(macOS 26.0, iOS 26.0, *)64struct AppleFoundationRuntime: ModelRuntime {65 let info: RuntimeInfo6667 func generate(_ request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error> {68 AsyncThrowingStream { continuation in69 let task = Task {70 let started = Date()71 var messages = request.messages72 var instructions = messages.first?.role == .system ? messages.removeFirst().content : ""73 guard let last = messages.popLast() else { continuation.finish(throwing: InferenceError.generationFailed("Empty prompt.")); return }74 // The system model has no public multi-turn replay API we rely on here: earlier turns are given as context in the instructions.75 if !messages.isEmpty {76 let history = messages.map { "\($0.role == .user ? "User" : "Assistant"): \($0.content)" }.joined(separator: "\n\n")77 instructions += (instructions.isEmpty ? "" : "\n\n") + "Conversation so far:\n" + history78 }79 let session = LanguageModelSession(instructions: instructions.isEmpty ? nil : instructions)80 let options = GenerationOptions(temperature: request.settings.temperature, maximumResponseTokens: request.settings.maxTokens)81 var emitted = ""; var first: Date?82 do {83 for try await snapshot in session.streamResponse(to: last.content, options: options) {84 if Task.isCancelled { break }85 let full = snapshot.content86 guard full.count > emitted.count, full.hasPrefix(emitted) else { continue }87 if first == nil { first = Date() }88 continuation.yield(.text(String(full.dropFirst(emitted.count)))); emitted = full89 }90 } catch {91 continuation.finish(throwing: InferenceError.generationFailed(error.localizedDescription)); return92 }93 // The framework reports no token counts or timings for the system model: only wall-clock figures are measured here.94 var metrics = GenerationMetrics(timeToFirstToken: first.map { $0.timeIntervalSince(started) }, totalTime: Date().timeIntervalSince(started))95 if Task.isCancelled { metrics.stopReason = .cancelled }96 continuation.yield(.finished(metrics)); continuation.finish()97 }98 continuation.onTermination = { _ in task.cancel() }99 }100 }101102 func countTokens(_ text: String) async -> Int? {103 if #available(macOS 27.0, iOS 27.0, *) { return try? await SystemLanguageModel.default.tokenCount(for: text) }104 return nil105 }106 func unload() async {}107}108#endif109