SPB Git

Repository Public

Inference.swift 4.9 KB 99 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatCore/Sources/Inference.swift3// Description: Backend-agnostic inference contracts: a backend loads a model into a runtime; a runtime streams generations and embeddings.4import Foundation56public struct LoadProgress: Sendable, Hashable {7    public enum Stage: String, Sendable { case preparing, loadingWeights, warmingUp, ready }8    public var stage: Stage9    public var fraction: Double?   // nil when the backend reports no progress10    public init(stage: Stage, fraction: Double? = nil) { self.stage = stage; self.fraction = fraction }11}1213public struct RuntimeInfo: Sendable, Hashable, Codable {14    public var modelID: String15    public var backend: BackendKind16    public var loadTime: TimeInterval          // measured17    public var weightsMemoryBytes: Int64?      // measured by the backend when available18    public var contextLength: Int?             // maximum the loaded model declares19    public var capabilities: Set<ModelCapability>20    public var chatTemplateAvailable: Bool21    public init(modelID: String, backend: BackendKind, loadTime: TimeInterval, weightsMemoryBytes: Int64? = nil, contextLength: Int? = nil,22                capabilities: Set<ModelCapability> = [.chat], chatTemplateAvailable: Bool = true) {23        self.modelID = modelID; self.backend = backend; self.loadTime = loadTime; self.weightsMemoryBytes = weightsMemoryBytes24        self.contextLength = contextLength; self.capabilities = capabilities; self.chatTemplateAvailable = chatTemplateAvailable25    }26}2728public struct CacheStatistics: Sendable, Hashable, Codable {29    public var cachedTokens: Int30    public var bytes: Int64?31    public init(cachedTokens: Int, bytes: Int64? = nil) { self.cachedTokens = cachedTokens; self.bytes = bytes }32}3334/// A loaded model. Implementations own their weights and release them in `unload()`.35public protocol ModelRuntime: Sendable {36    var info: RuntimeInfo { get }37    func generate(_ request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error>38    /// Exact token count with the model's tokenizer, or nil when the backend cannot tokenize locally (remote nodes).39    func countTokens(_ text: String) async -> Int?40    func embed(_ texts: [String]) async throws -> [[Float]]41    func cacheStatistics() async -> CacheStatistics?42    func resetSession(_ key: String?) async43    func unload() async44}4546public extension ModelRuntime {47    func embed(_ texts: [String]) async throws -> [[Float]] { throw InferenceError.unsupported("This model does not produce embeddings.") }48    func cacheStatistics() async -> CacheStatistics? { nil }49    func resetSession(_ key: String?) async {}50}5152public protocol InferenceBackend: Sendable {53    var kind: BackendKind { get }54    /// False when the backend cannot run in this process at all (e.g. MLX in the iOS Simulator, GGUF on iOS in this build).55    var isAvailable: Bool { get }56    var unavailableReason: String? { get }57    func load(_ model: ModelDescriptor, modelsRoot: URL, progress: @escaping @Sendable (LoadProgress) -> Void) async throws -> any ModelRuntime58}5960/// Text → vectors, used by the Knowledge Studio. Always local unless the user picked a remote embedder explicitly.61public protocol EmbeddingProvider: Sendable {62    var identifier: String { get }63    var dimension: Int { get }64    var isRemote: Bool { get }65    func embed(_ texts: [String]) async throws -> [[Float]]66}6768/// The one entry point features use to generate text, whatever the backend (chat, arena, agents, API server, summaries).69public protocol TextGenerating: Sendable {70    func generate(modelID: String, request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error>71    func countTokens(modelID: String, text: String) async -> Int?72}7374public extension TextGenerating {75    /// Collects a full answer (used for titles, summaries, routing decisions).76    func complete(modelID: String, request: GenerationRequest) async throws -> (text: String, metrics: GenerationMetrics?) {77        var out = ""; var metrics: GenerationMetrics?78        for try await event in generate(modelID: modelID, request: request) {79            switch event {80            case .text(let t): out += t81            case .finished(let m): metrics = m82            default: break83            }84        }85        return (out, metrics)86    }87}8889public enum TokenEstimator {90    /// Fallback estimate when no tokenizer is available (≈ 4 characters per token for Latin scripts, 1.5 for CJK). Labelled "estimated" in the UI.91    public static func estimate(_ text: String) -> Int {92        var cjk = 0, other = 093        for scalar in text.unicodeScalars {94            if (0x3040...0x30FF).contains(scalar.value) || (0x4E00...0x9FFF).contains(scalar.value) || (0xAC00...0xD7AF).contains(scalar.value) { cjk += 1 } else { other += 1 }95        }96        return Int((Double(other) / 4.0).rounded(.up)) + Int((Double(cjk) / 1.5).rounded(.up))97    }98}99
Files main
Menu

Theme