Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatCore/Sources/Inference.swift3// Description: Backend-agnostic inference contracts: a backend loads a model into a runtime; a runtime streams generations and embeddings.4import Foundation56public struct LoadProgress: Sendable, Hashable {7 public enum Stage: String, Sendable { case preparing, loadingWeights, warmingUp, ready }8 public var stage: Stage9 public var fraction: Double? // nil when the backend reports no progress10 public init(stage: Stage, fraction: Double? = nil) { self.stage = stage; self.fraction = fraction }11}1213public struct RuntimeInfo: Sendable, Hashable, Codable {14 public var modelID: String15 public var backend: BackendKind16 public var loadTime: TimeInterval // measured17 public var weightsMemoryBytes: Int64? // measured by the backend when available18 public var contextLength: Int? // maximum the loaded model declares19 public var capabilities: Set<ModelCapability>20 public var chatTemplateAvailable: Bool21 public init(modelID: String, backend: BackendKind, loadTime: TimeInterval, weightsMemoryBytes: Int64? = nil, contextLength: Int? = nil,22 capabilities: Set<ModelCapability> = [.chat], chatTemplateAvailable: Bool = true) {23 self.modelID = modelID; self.backend = backend; self.loadTime = loadTime; self.weightsMemoryBytes = weightsMemoryBytes24 self.contextLength = contextLength; self.capabilities = capabilities; self.chatTemplateAvailable = chatTemplateAvailable25 }26}2728public struct CacheStatistics: Sendable, Hashable, Codable {29 public var cachedTokens: Int30 public var bytes: Int64?31 public init(cachedTokens: Int, bytes: Int64? = nil) { self.cachedTokens = cachedTokens; self.bytes = bytes }32}3334/// A loaded model. Implementations own their weights and release them in `unload()`.35public protocol ModelRuntime: Sendable {36 var info: RuntimeInfo { get }37 func generate(_ request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error>38 /// Exact token count with the model's tokenizer, or nil when the backend cannot tokenize locally (remote nodes).39 func countTokens(_ text: String) async -> Int?40 func embed(_ texts: [String]) async throws -> [[Float]]41 func cacheStatistics() async -> CacheStatistics?42 func resetSession(_ key: String?) async43 func unload() async44}4546public extension ModelRuntime {47 func embed(_ texts: [String]) async throws -> [[Float]] { throw InferenceError.unsupported("This model does not produce embeddings.") }48 func cacheStatistics() async -> CacheStatistics? { nil }49 func resetSession(_ key: String?) async {}50}5152public protocol InferenceBackend: Sendable {53 var kind: BackendKind { get }54 /// False when the backend cannot run in this process at all (e.g. MLX in the iOS Simulator, GGUF on iOS in this build).55 var isAvailable: Bool { get }56 var unavailableReason: String? { get }57 func load(_ model: ModelDescriptor, modelsRoot: URL, progress: @escaping @Sendable (LoadProgress) -> Void) async throws -> any ModelRuntime58}5960/// Text → vectors, used by the Knowledge Studio. Always local unless the user picked a remote embedder explicitly.61public protocol EmbeddingProvider: Sendable {62 var identifier: String { get }63 var dimension: Int { get }64 var isRemote: Bool { get }65 func embed(_ texts: [String]) async throws -> [[Float]]66}6768/// The one entry point features use to generate text, whatever the backend (chat, arena, agents, API server, summaries).69public protocol TextGenerating: Sendable {70 func generate(modelID: String, request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error>71 func countTokens(modelID: String, text: String) async -> Int?72}7374public extension TextGenerating {75 /// Collects a full answer (used for titles, summaries, routing decisions).76 func complete(modelID: String, request: GenerationRequest) async throws -> (text: String, metrics: GenerationMetrics?) {77 var out = ""; var metrics: GenerationMetrics?78 for try await event in generate(modelID: modelID, request: request) {79 switch event {80 case .text(let t): out += t81 case .finished(let m): metrics = m82 default: break83 }84 }85 return (out, metrics)86 }87}8889public enum TokenEstimator {90 /// Fallback estimate when no tokenizer is available (≈ 4 characters per token for Latin scripts, 1.5 for CJK). Labelled "estimated" in the UI.91 public static func estimate(_ text: String) -> Int {92 var cjk = 0, other = 093 for scalar in text.unicodeScalars {94 if (0x3040...0x30FF).contains(scalar.value) || (0x4E00...0x9FFF).contains(scalar.value) || (0xAC00...0xD7AF).contains(scalar.value) { cjk += 1 } else { other += 1 }95 }96 return Int((Double(other) / 4.0).rounded(.up)) + Int((Double(cjk) / 1.5).rounded(.up))97 }98}99