Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatCore/Sources/Generation.swift3// Description: Sampling settings, presets, generation requests, streamed events and measured metrics, common to every backend.4import Foundation56public struct GenerationSettings: Codable, Sendable, Hashable {7 public var temperature: Double8 public var topP: Double9 public var topK: Int // 0 = disabled10 public var minP: Double11 public var repetitionPenalty: Double // 1 = disabled12 public var repetitionContext: Int13 public var seed: UInt64? // nil = random14 public var contextLength: Int // tokens kept in the window (prompt + answer)15 public var maxTokens: Int // maximum generated tokens16 public var reasoningEnabled: Bool // ask reasoning-capable templates to think17 public var kvBits: Int? // KV-cache quantization (MLX), nil = full precision1819 public init(temperature: Double = 0.7, topP: Double = 0.95, topK: Int = 40, minP: Double = 0, repetitionPenalty: Double = 1.0,20 repetitionContext: Int = 64, seed: UInt64? = nil, contextLength: Int = 8192, maxTokens: Int = 2048,21 reasoningEnabled: Bool = true, kvBits: Int? = nil) {22 self.temperature = temperature; self.topP = topP; self.topK = topK; self.minP = minP23 self.repetitionPenalty = repetitionPenalty; self.repetitionContext = repetitionContext; self.seed = seed24 self.contextLength = contextLength; self.maxTokens = maxTokens; self.reasoningEnabled = reasoningEnabled; self.kvBits = kvBits25 }26 public static let `default` = GenerationSettings()27}2829public enum InferencePreset: String, Codable, Sendable, CaseIterable, Identifiable {30 case fast, balanced, reasoning, maximumQuality31 public var id: String { rawValue }32 public var title: String {33 switch self { case .fast: "Fast"; case .balanced: "Balanced"; case .reasoning: "Reasoning"; case .maximumQuality: "Maximum Quality" }34 }35 public var symbol: String {36 switch self { case .fast: "hare"; case .balanced: "scalemass"; case .reasoning: "brain"; case .maximumQuality: "sparkles" }37 }38 public var summary: String {39 switch self {40 case .fast: "Short answers, small window, no thinking."41 case .balanced: "Sensible defaults for everyday chat."42 case .reasoning: "Thinking enabled, long answers, low temperature."43 case .maximumQuality: "Largest window and output the device allows."44 }45 }46 /// Applies the preset on top of `base`, never exceeding `maxContext` (the model's or the device's limit).47 public func settings(base: GenerationSettings = .default, maxContext: Int = 32768) -> GenerationSettings {48 var s = base49 switch self {50 case .fast: s.temperature = 0.6; s.topP = 0.9; s.topK = 40; s.contextLength = min(4096, maxContext); s.maxTokens = 512; s.reasoningEnabled = false51 case .balanced: s.temperature = 0.7; s.topP = 0.95; s.topK = 40; s.contextLength = min(8192, maxContext); s.maxTokens = 2048; s.reasoningEnabled = true52 case .reasoning: s.temperature = 0.4; s.topP = 0.95; s.topK = 20; s.contextLength = min(16384, maxContext); s.maxTokens = 8192; s.reasoningEnabled = true53 case .maximumQuality: s.temperature = 0.5; s.topP = 0.98; s.topK = 0; s.contextLength = maxContext; s.maxTokens = min(16384, maxContext / 2); s.reasoningEnabled = true54 }55 return s56 }57}5859public struct ToolDefinition: Codable, Sendable, Hashable, Identifiable {60 public var name: String61 public var summary: String62 public var parameters: JSONValue // JSON Schema object63 public var id: String { name }64 public init(name: String, summary: String, parameters: JSONValue) { self.name = name; self.summary = summary; self.parameters = parameters }65 /// OpenAI function-tool form, the shape chat templates expect.66 public var openAISpec: JSONValue {67 ["type": "function", "function": ["name": .string(name), "description": .string(summary), "parameters": parameters]]68 }69}7071public struct ToolInvocation: Codable, Sendable, Hashable, Identifiable {72 public var id: String73 public var name: String74 public var arguments: JSONValue75 public init(id: String = UUID().uuidString, name: String, arguments: JSONValue) { self.id = id; self.name = name; self.arguments = arguments }76}7778public struct ImageInput: Sendable, Hashable, Codable {79 public var data: Data // encoded image (PNG/JPEG/HEIC)80 public var mimeType: String81 public init(data: Data, mimeType: String = "image/png") { self.data = data; self.mimeType = mimeType }82}8384/// One turn handed to a backend. History handling (branches, compaction, retrieval) happens before this point.85public struct PromptMessage: Sendable, Hashable, Codable {86 public enum Role: String, Sendable, Codable { case system, user, assistant, tool }87 public var role: Role88 public var content: String89 public var images: [ImageInput]90 public var toolCalls: [ToolInvocation] // assistant turns that called tools91 public var toolCallID: String? // tool turns92 public var toolName: String?93 public init(role: Role, content: String, images: [ImageInput] = [], toolCalls: [ToolInvocation] = [], toolCallID: String? = nil, toolName: String? = nil) {94 self.role = role; self.content = content; self.images = images; self.toolCalls = toolCalls; self.toolCallID = toolCallID; self.toolName = toolName95 }96 public static func system(_ t: String) -> Self { .init(role: .system, content: t) }97 public static func user(_ t: String, images: [ImageInput] = []) -> Self { .init(role: .user, content: t, images: images) }98 public static func assistant(_ t: String) -> Self { .init(role: .assistant, content: t) }99}100101public struct GenerationRequest: Sendable {102 public var messages: [PromptMessage]103 public var settings: GenerationSettings104 public var tools: [ToolDefinition]105 public var responseSchema: JSONValue? // JSON Schema for structured output, when the backend supports it106 public var stop: [String]107 /// Lets a backend reuse its KV cache across turns of the same conversation.108 public var sessionKey: String?109 public init(messages: [PromptMessage], settings: GenerationSettings = .default, tools: [ToolDefinition] = [], responseSchema: JSONValue? = nil,110 stop: [String] = [], sessionKey: String? = nil) {111 self.messages = messages; self.settings = settings; self.tools = tools; self.responseSchema = responseSchema; self.stop = stop; self.sessionKey = sessionKey112 }113}114115public enum StopReason: String, Codable, Sendable { case endOfTurn, maxTokens, cancelled, stopSequence, toolCall, error }116117/// Everything here is measured by the backend that produced the answer; a field is nil when that backend cannot measure it.118public struct GenerationMetrics: Codable, Sendable, Hashable {119 public var promptTokens: Int?120 public var cachedPromptTokens: Int?121 public var completionTokens: Int?122 public var reasoningTokens: Int?123 public var timeToFirstToken: TimeInterval? // request start → first streamed text124 public var promptTime: TimeInterval? // prompt evaluation125 public var generationTime: TimeInterval? // token generation126 public var totalTime: TimeInterval?127 public var peakMemoryBytes: Int64?128 public var activeMemoryBytes: Int64?129 public var stopReason: StopReason130 public init(promptTokens: Int? = nil, cachedPromptTokens: Int? = nil, completionTokens: Int? = nil, reasoningTokens: Int? = nil,131 timeToFirstToken: TimeInterval? = nil, promptTime: TimeInterval? = nil, generationTime: TimeInterval? = nil,132 totalTime: TimeInterval? = nil, peakMemoryBytes: Int64? = nil, activeMemoryBytes: Int64? = nil, stopReason: StopReason = .endOfTurn) {133 self.promptTokens = promptTokens; self.cachedPromptTokens = cachedPromptTokens; self.completionTokens = completionTokens134 self.reasoningTokens = reasoningTokens; self.timeToFirstToken = timeToFirstToken; self.promptTime = promptTime135 self.generationTime = generationTime; self.totalTime = totalTime; self.peakMemoryBytes = peakMemoryBytes136 self.activeMemoryBytes = activeMemoryBytes; self.stopReason = stopReason137 }138 public var tokensPerSecond: Double? {139 guard let n = completionTokens, let t = generationTime, t > 0, n > 0 else { return nil }140 return Double(n) / t141 }142 public var promptTokensPerSecond: Double? {143 guard let n = promptTokens, let t = promptTime, t > 0 else { return nil }144 let evaluated = n - (cachedPromptTokens ?? 0)145 return evaluated > 0 ? Double(evaluated) / t : nil146 }147}148149public enum GenerationEvent: Sendable {150 case text(String)151 case reasoning(String)152 case toolCall(ToolInvocation)153 case finished(GenerationMetrics)154}155156public enum InferenceError: LocalizedError, Sendable, Equatable {157 case modelNotLoaded158 case modelNotFound(String)159 case unsupported(String)160 case insufficientMemory(needed: Int64, available: Int64)161 case loadFailed(String)162 case generationFailed(String)163 case backendUnavailable(String)164 case cancelled165 public var errorDescription: String? {166 switch self {167 case .modelNotLoaded: "No model is loaded."168 case .modelNotFound(let m): "Model not found: \(m)"169 case .unsupported(let m): m170 case .insufficientMemory(let n, let a): "This model needs about \(ByteFormat.memory(n)) but only \(ByteFormat.memory(a)) can be used on this device."171 case .loadFailed(let m): "The model could not be loaded: \(m)"172 case .generationFailed(let m): "Generation failed: \(m)"173 case .backendUnavailable(let m): m174 case .cancelled: "Cancelled."175 }176 }177}178