SPB Git

Repository Public

AppleFoundationBackend.swift 5.7 KB 109 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatInference/Sources/AppleFoundationBackend.swift3// Description: The system language model (Apple Intelligence, FoundationModels) as a zero-download backend on OS 26+ when the device supports it.4import Foundation5import LocalChatCore6#if canImport(FoundationModels)7import FoundationModels8#endif910public struct AppleFoundationBackend: InferenceBackend {11    public let kind: BackendKind = .appleFoundation12    public static let modelID = "apple:system"13    public init() {}1415    public var isAvailable: Bool { unavailableReason == nil }1617    public var unavailableReason: String? {18        #if canImport(FoundationModels)19        if #available(macOS 26.0, iOS 26.0, *) {20            switch SystemLanguageModel.default.availability {21            case .available: return nil22            case .unavailable(.deviceNotEligible): return "This device does not support Apple Intelligence."23            case .unavailable(.appleIntelligenceNotEnabled): return "Apple Intelligence is turned off in Settings."24            case .unavailable(.modelNotReady): return "The Apple Intelligence model is still downloading."25            case .unavailable: return "The Apple Intelligence model is unavailable."26            }27        }28        return "Apple Intelligence needs iOS 26 / macOS 26 or later."29        #else30        return "Apple Intelligence is not available on this platform."31        #endif32    }3334    /// The descriptor to list in the library when the system model can be used.35    public var descriptor: ModelDescriptor? {36        guard isAvailable else { return nil }37        return ModelDescriptor(id: Self.modelID, name: "Apple Intelligence (on-device)", backend: .appleFoundation, format: .system,38                               architecture: "system", contextLength: contextSize, capabilities: [.chat], license: "Apple system model")39    }4041    var contextSize: Int? {42        #if canImport(FoundationModels)43        if #available(macOS 27.0, iOS 27.0, *) { return SystemLanguageModel.default.contextSize }44        #endif45        return nil46    }4748    public func load(_ model: ModelDescriptor, modelsRoot: URL, progress: @escaping @Sendable (LoadProgress) -> Void) async throws -> any ModelRuntime {49        if let reason = unavailableReason { throw InferenceError.backendUnavailable(reason) }50        #if canImport(FoundationModels)51        if #available(macOS 26.0, iOS 26.0, *) {52            let start = Date()53            progress(LoadProgress(stage: .ready, fraction: 1))54            return AppleFoundationRuntime(info: RuntimeInfo(modelID: model.id, backend: .appleFoundation, loadTime: Date().timeIntervalSince(start),55                                                           contextLength: contextSize, capabilities: [.chat]))56        }57        #endif58        throw InferenceError.backendUnavailable("Apple Intelligence is unavailable.")59    }60}6162#if canImport(FoundationModels)63@available(macOS 26.0, iOS 26.0, *)64struct AppleFoundationRuntime: ModelRuntime {65    let info: RuntimeInfo6667    func generate(_ request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error> {68        AsyncThrowingStream { continuation in69            let task = Task {70                let started = Date()71                var messages = request.messages72                var instructions = messages.first?.role == .system ? messages.removeFirst().content : ""73                guard let last = messages.popLast() else { continuation.finish(throwing: InferenceError.generationFailed("Empty prompt.")); return }74                // The system model has no public multi-turn replay API we rely on here: earlier turns are given as context in the instructions.75                if !messages.isEmpty {76                    let history = messages.map { "\($0.role == .user ? "User" : "Assistant"): \($0.content)" }.joined(separator: "\n\n")77                    instructions += (instructions.isEmpty ? "" : "\n\n") + "Conversation so far:\n" + history78                }79                let session = LanguageModelSession(instructions: instructions.isEmpty ? nil : instructions)80                let options = GenerationOptions(temperature: request.settings.temperature, maximumResponseTokens: request.settings.maxTokens)81                var emitted = ""; var first: Date?82                do {83                    for try await snapshot in session.streamResponse(to: last.content, options: options) {84                        if Task.isCancelled { break }85                        let full = snapshot.content86                        guard full.count > emitted.count, full.hasPrefix(emitted) else { continue }87                        if first == nil { first = Date() }88                        continuation.yield(.text(String(full.dropFirst(emitted.count)))); emitted = full89                    }90                } catch {91                    continuation.finish(throwing: InferenceError.generationFailed(error.localizedDescription)); return92                }93                // The framework reports no token counts or timings for the system model: only wall-clock figures are measured here.94                var metrics = GenerationMetrics(timeToFirstToken: first.map { $0.timeIntervalSince(started) }, totalTime: Date().timeIntervalSince(started))95                if Task.isCancelled { metrics.stopReason = .cancelled }96                continuation.yield(.finished(metrics)); continuation.finish()97            }98            continuation.onTermination = { _ in task.cancel() }99        }100    }101102    func countTokens(_ text: String) async -> Int? {103        if #available(macOS 27.0, iOS 27.0, *) { return try? await SystemLanguageModel.default.tokenCount(for: text) }104        return nil105    }106    func unload() async {}107}108#endif109
Files main
Menu

Theme