Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatInference/Sources/GGUFBackend.swift3// Description: llama.cpp backend for GGUF models (Metal offload). Built where the upstream XCFramework has a slice: macOS in this project.4import Foundation5import LocalChatCore67#if canImport(LlamaSwift)8import Jinja9import LlamaSwift1011public struct GGUFBackend: InferenceBackend {12 public let kind: BackendKind = .gguf13 public init() {}14 public var isAvailable: Bool { true }15 public var unavailableReason: String? { nil }1617 public func load(_ model: ModelDescriptor, modelsRoot: URL, progress: @escaping @Sendable (LoadProgress) -> Void) async throws -> any ModelRuntime {18 guard let relative = model.localPath else { throw InferenceError.modelNotFound(model.name) }19 let url = relative.hasPrefix("/") ? URL(fileURLWithPath: relative) : modelsRoot.appendingPathComponent(relative)20 guard FileManager.default.fileExists(atPath: url.path) else { throw InferenceError.modelNotFound(url.lastPathComponent) }21 progress(LoadProgress(stage: .preparing))22 let start = Date()23 let session = try await Task.detached(priority: .userInitiated) { try LlamaSession(path: url.path, progress: progress) }.value24 var capabilities = model.capabilities; capabilities.remove(.vision); capabilities.remove(.embeddings)25 if session.templateMentionsTools { capabilities.insert(.tools) }26 progress(LoadProgress(stage: .ready, fraction: 1))27 return LlamaRuntime(session: session, info: RuntimeInfo(28 modelID: model.id, backend: .gguf, loadTime: Date().timeIntervalSince(start), weightsMemoryBytes: Int64(session.modelSize),29 contextLength: session.trainedContext, capabilities: capabilities, chatTemplateAvailable: session.chatTemplate != nil))30 }31}3233/// Owns the llama.cpp model and context. Every call into llama.cpp happens on `queue` (the C API is not thread-safe per context).34final class LlamaSession: @unchecked Sendable {35 private static let initBackend: Void = { llama_backend_init(); llama_log_set({ _, _, _ in }, nil) }()36 private final class ProgressBox { let report: @Sendable (LoadProgress) -> Void; init(_ r: @escaping @Sendable (LoadProgress) -> Void) { report = r } }3738 let queue = DispatchQueue(label: "ai.spboucher.localchat.llama", qos: .userInitiated)39 private var model: OpaquePointer?40 private var context: OpaquePointer?41 private var contextSize: UInt32 = 042 private var evaluated: [llama_token] = [] // tokens currently in the KV cache (sequence 0)43 private let vocab: OpaquePointer44 let chatTemplate: String?45 let trainedContext: Int46 let modelSize: UInt6447 let bosText: String48 let eosText: String49 private let lock = NSLock()50 private var cancelFlag = false5152 var templateMentionsTools: Bool { chatTemplate?.contains("tools") ?? false }5354 init(path: String, progress: @escaping @Sendable (LoadProgress) -> Void) throws {55 _ = Self.initBackend56 var params = llama_model_default_params()57 params.n_gpu_layers = -1 // offload every layer to Metal58 let box = Unmanaged.passRetained(ProgressBox(progress))59 params.progress_callback = { value, user in60 if let user { Unmanaged<ProgressBox>.fromOpaque(user).takeUnretainedValue().report(LoadProgress(stage: .loadingWeights, fraction: Double(value))) }61 return true62 }63 params.progress_callback_user_data = box.toOpaque()64 defer { box.release() }65 guard let model = llama_model_load_from_file(path, params) else {66 throw InferenceError.loadFailed("llama.cpp could not read this GGUF file (unsupported architecture or corrupt file).")67 }68 self.model = model69 vocab = llama_model_get_vocab(model)70 trainedContext = Int(llama_model_n_ctx_train(model))71 modelSize = llama_model_size(model)72 chatTemplate = llama_model_chat_template(model, nil).map { String(cString: $0) }73 bosText = Self.piece(vocab, llama_vocab_bos(vocab), special: true)74 eosText = Self.piece(vocab, llama_vocab_eos(vocab), special: true)75 }7677 deinit {78 if let context { llama_free(context) }79 if let model { llama_model_free(model) }80 }8182 func cancel() { lock.lock(); cancelFlag = true; lock.unlock() }83 private var isCancelled: Bool { lock.lock(); defer { lock.unlock() }; return cancelFlag }8485 func free() {86 queue.sync {87 if let context { llama_free(context) }; context = nil88 if let model { llama_model_free(model) }; model = nil89 evaluated = []90 }91 }9293 static func piece(_ vocab: OpaquePointer, _ token: llama_token, special: Bool) -> String {94 String(decoding: pieceBytes(vocab, token, special: special), as: UTF8.self)95 }96 static func pieceBytes(_ vocab: OpaquePointer, _ token: llama_token, special: Bool) -> [UInt8] {97 var buffer = [CChar](repeating: 0, count: 64)98 var n = llama_token_to_piece(vocab, token, &buffer, Int32(buffer.count), 0, special)99 if n < 0 { buffer = [CChar](repeating: 0, count: Int(-n)); n = llama_token_to_piece(vocab, token, &buffer, Int32(buffer.count), 0, special) }100 return buffer.prefix(Int(max(0, n))).map { UInt8(bitPattern: $0) }101 }102103 func tokenize(_ text: String, addSpecial: Bool, parseSpecial: Bool) -> [llama_token] {104 let utf8 = Array(text.utf8CString.dropLast())105 var tokens = [llama_token](repeating: 0, count: utf8.count + 16)106 var n = llama_tokenize(vocab, utf8, Int32(utf8.count), &tokens, Int32(tokens.count), addSpecial, parseSpecial)107 if n < 0 { tokens = [llama_token](repeating: 0, count: Int(-n)); n = llama_tokenize(vocab, utf8, Int32(utf8.count), &tokens, Int32(tokens.count), addSpecial, parseSpecial) }108 return Array(tokens.prefix(Int(max(0, n))))109 }110111 /// Renders the model's own Jinja chat template (same data model as transformers: messages, tools, add_generation_prompt…).112 func renderPrompt(_ request: GenerationRequest) throws -> String {113 guard let chatTemplate else { throw InferenceError.unsupported("This GGUF file has no chat template; it cannot be used for chat.") }114 let messages: [[String: Any]] = request.messages.map { m in115 var d: [String: Any] = ["role": m.role.rawValue, "content": m.content]116 if !m.toolCalls.isEmpty {117 d["tool_calls"] = m.toolCalls.map { ["id": $0.id, "type": "function", "function": ["name": $0.name, "arguments": $0.arguments.anyValue]] as [String: Any] }118 }119 if let id = m.toolCallID { d["tool_call_id"] = id }120 if let name = m.toolName { d["name"] = name }121 return d122 }123 var context: [String: Any] = ["messages": messages, "add_generation_prompt": true, "bos_token": bosText, "eos_token": eosText,124 "enable_thinking": request.settings.reasoningEnabled]125 if !request.tools.isEmpty { context["tools"] = request.tools.map { $0.openAISpec.anyValue } }126 do {127 let template = try Template(chatTemplate, with: .init(lstripBlocks: true, trimBlocks: true))128 return try template.render(try context.mapValues { try Value(any: $0) })129 } catch {130 throw InferenceError.generationFailed("The model's chat template could not be rendered: \(error)")131 }132 }133134 private func ensureContext(_ wanted: Int) throws {135 let size = UInt32(max(512, min(wanted, trainedContext > 0 ? trainedContext : wanted)))136 if context != nil, contextSize == size { return }137 if let context { llama_free(context) }138 context = nil; evaluated = []139 var params = llama_context_default_params()140 params.n_ctx = size; params.n_batch = 512141 let threads = Int32(max(2, DeviceProfile.sysctlIntPublic("hw.perflevel0.physicalcpu") ?? ProcessInfo.processInfo.activeProcessorCount / 2))142 params.n_threads = threads; params.n_threads_batch = threads143 guard let model, let created = llama_init_from_model(model, params) else {144 throw InferenceError.loadFailed("llama.cpp could not create a context of \(size) tokens (not enough memory).")145 }146 context = created; contextSize = size147 }148149 func generate(_ request: GenerationRequest, yield: (GenerationEvent) -> Void) throws {150 lock.lock(); cancelFlag = false; lock.unlock()151 let started = Date()152 let s = request.settings153 try ensureContext(s.contextLength)154 guard let context else { throw InferenceError.modelNotLoaded }155 let prompt = try renderPrompt(request)156 let tokens = tokenize(prompt, addSpecial: !prompt.hasPrefix(bosText) || bosText.isEmpty, parseSpecial: true)157 let room = Int(contextSize) - tokens.count158 guard room > 16 else {159 throw InferenceError.generationFailed("The prompt (\(tokens.count) tokens) does not fit in the context window (\(contextSize) tokens).")160 }161 let maxNew = min(s.maxTokens, room - 1)162163 // KV reuse: keep the longest common prefix with what is already evaluated, drop the rest.164 var common = 0165 while common < min(evaluated.count, tokens.count - 1), evaluated[common] == tokens[common] { common += 1 }166 let memory = llama_get_memory(context)167 if common < evaluated.count, !llama_memory_seq_rm(memory, 0, llama_pos(common), -1) { llama_memory_clear(memory, true); common = 0 }168 evaluated = Array(tokens.prefix(common))169170 let promptStart = Date()171 var index = common172 while index < tokens.count {173 if isCancelled { throw InferenceError.cancelled }174 var slice = Array(tokens[index..<min(index + 512, tokens.count)])175 let status = slice.withUnsafeMutableBufferPointer { llama_decode(context, llama_batch_get_one($0.baseAddress, Int32($0.count))) }176 guard status == 0 else { evaluated = []; llama_memory_clear(memory, true); throw InferenceError.generationFailed("llama_decode failed (\(status)).") }177 evaluated.append(contentsOf: slice); index += slice.count178 }179 let promptTime = Date().timeIntervalSince(promptStart)180181 let chain = llama_sampler_chain_init(llama_sampler_chain_default_params())!182 defer { llama_sampler_free(chain) }183 if s.repetitionPenalty != 1 { llama_sampler_chain_add(chain, llama_sampler_init_penalties(llama_vocab_n_tokens(vocab), Int32(s.repetitionContext), Float(s.repetitionPenalty), 0, 0)) }184 if s.temperature <= 0 { llama_sampler_chain_add(chain, llama_sampler_init_greedy()) }185 else {186 if s.topK > 0 { llama_sampler_chain_add(chain, llama_sampler_init_top_k(Int32(s.topK))) }187 if s.topP < 1 { llama_sampler_chain_add(chain, llama_sampler_init_top_p(Float(s.topP), 1)) }188 if s.minP > 0 { llama_sampler_chain_add(chain, llama_sampler_init_min_p(Float(s.minP), 1)) }189 llama_sampler_chain_add(chain, llama_sampler_init_temp(Float(s.temperature)))190 llama_sampler_chain_add(chain, llama_sampler_init_dist(s.seed.map { UInt32(truncatingIfNeeded: $0) } ?? UInt32.random(in: 0..<UInt32.max)))191 }192193 let opensThinking = prompt.hasSuffix("<think>\n") || prompt.hasSuffix("<think>")194 var splitter = ReasoningSplitter(startsInside: opensThinking)195 var pending = [UInt8](); var answer = ""; var held = ""; var generated = 0; var reasoningTokens = 0196 var firstToken: Date?; var reason: StopReason = .endOfTurn197 let wantsTools = !request.tools.isEmpty198 let generationStart = Date()199200 func emit(_ text: String) {201 for piece in splitter.feed(text) { route(piece) }202 }203 func route(_ piece: ReasoningSplitter.Piece) {204 switch piece {205 case .reasoning(let t): yield(.reasoning(t))206 case .text(let t):207 answer += t208 if wantsTools {209 // hold back anything that may be a textual tool call until generation ends210 held += t211 if !held.contains("<tool_call>") && !held.contains("[TOOL_CALLS]") {212 let safe = ReasoningSplitter.safePrefix(held, markers: ["<tool_call>", "[TOOL_CALLS]"])213 if !safe.isEmpty { yield(.text(safe)); held.removeFirst(safe.count) }214 }215 } else { yield(.text(t)) }216 }217 }218219 while generated < maxNew {220 if isCancelled { reason = .cancelled; break }221 var token = llama_sampler_sample(chain, context, -1)222 if llama_vocab_is_eog(vocab, token) { break }223 generated += 1224 if firstToken == nil { firstToken = Date() }225 pending.append(contentsOf: Self.pieceBytes(vocab, token, special: false))226 // emit only complete UTF-8 sequences227 if let text = String(validatingUTF8Bytes: pending) { pending.removeAll(keepingCapacity: true); if !text.isEmpty { emit(text) } }228 else if pending.count > 8 { emit(String(decoding: pending, as: UTF8.self)); pending.removeAll(keepingCapacity: true) }229 if let stop = request.stop.first(where: { !$0.isEmpty && answer.contains($0) }) {230 _ = stop; reason = .stopSequence; break231 }232 let status = withUnsafeMutablePointer(to: &token) { llama_decode(context, llama_batch_get_one($0, 1)) }233 guard status == 0 else { evaluated = []; llama_memory_clear(memory, true); throw InferenceError.generationFailed("llama_decode failed (\(status)).") }234 evaluated.append(token)235 _ = reasoningTokens236 }237 if generated >= maxNew, reason == .endOfTurn { reason = .maxTokens }238 if !pending.isEmpty { emit(String(decoding: pending, as: UTF8.self)) }239 for piece in splitter.finish() { route(piece) }240 reasoningTokens = 0241242 if wantsTools {243 let parsed = TextToolCallParser.extract(from: held, allowed: Set(request.tools.map(\.name)))244 if !parsed.clean.isEmpty { yield(.text(parsed.clean)) }245 for call in parsed.calls { yield(.toolCall(call)) }246 if !parsed.calls.isEmpty { reason = .toolCall }247 }248 let now = Date()249 yield(.finished(GenerationMetrics(250 promptTokens: tokens.count, cachedPromptTokens: common, completionTokens: generated, reasoningTokens: nil,251 timeToFirstToken: firstToken.map { $0.timeIntervalSince(started) }, promptTime: promptTime,252 generationTime: now.timeIntervalSince(generationStart), totalTime: now.timeIntervalSince(started),253 peakMemoryBytes: nil, activeMemoryBytes: DeviceProfile.processFootprintBytes(), stopReason: reason)))254 }255256 var cachedTokenCount: Int { queue.sync { evaluated.count } }257 func resetCache() { queue.sync { if let context { llama_memory_clear(llama_get_memory(context), true) }; evaluated = [] } }258}259260private extension String {261 init?(validatingUTF8Bytes bytes: [UInt8]) {262 guard let s = String(bytes: bytes, encoding: .utf8) else { return nil }263 self = s264 }265}266267struct LlamaRuntime: ModelRuntime {268 let session: LlamaSession269 let info: RuntimeInfo270271 func generate(_ request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error> {272 AsyncThrowingStream { continuation in273 session.queue.async {274 do { try session.generate(request) { continuation.yield($0) }; continuation.finish() }275 catch { continuation.finish(throwing: error) }276 }277 continuation.onTermination = { termination in if case .cancelled = termination { session.cancel() } }278 }279 }280 func countTokens(_ text: String) async -> Int? { session.tokenize(text, addSpecial: false, parseSpecial: true).count }281 func cacheStatistics() async -> CacheStatistics? { CacheStatistics(cachedTokens: session.cachedTokenCount) }282 func resetSession(_ key: String?) async { session.resetCache() }283 func unload() async { session.cancel(); session.free() }284}285#else286public struct GGUFBackend: InferenceBackend {287 public let kind: BackendKind = .gguf288 public init() {}289 public var isAvailable: Bool { false }290 public var unavailableReason: String? { "GGUF models run on the Mac in this version (the llama.cpp build used here has no iOS Simulator slice). Use an MLX model, or route to a Mac." }291 public func load(_ model: ModelDescriptor, modelsRoot: URL, progress: @escaping @Sendable (LoadProgress) -> Void) async throws -> any ModelRuntime {292 throw InferenceError.backendUnavailable(unavailableReason ?? "")293 }294}295#endif296