SPB Git

Repository Public

GGUFBackend.swift 16.4 KB 296 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatInference/Sources/GGUFBackend.swift3// Description: llama.cpp backend for GGUF models (Metal offload). Built where the upstream XCFramework has a slice: macOS in this project.4import Foundation5import LocalChatCore67#if canImport(LlamaSwift)8import Jinja9import LlamaSwift1011public struct GGUFBackend: InferenceBackend {12    public let kind: BackendKind = .gguf13    public init() {}14    public var isAvailable: Bool { true }15    public var unavailableReason: String? { nil }1617    public func load(_ model: ModelDescriptor, modelsRoot: URL, progress: @escaping @Sendable (LoadProgress) -> Void) async throws -> any ModelRuntime {18        guard let relative = model.localPath else { throw InferenceError.modelNotFound(model.name) }19        let url = relative.hasPrefix("/") ? URL(fileURLWithPath: relative) : modelsRoot.appendingPathComponent(relative)20        guard FileManager.default.fileExists(atPath: url.path) else { throw InferenceError.modelNotFound(url.lastPathComponent) }21        progress(LoadProgress(stage: .preparing))22        let start = Date()23        let session = try await Task.detached(priority: .userInitiated) { try LlamaSession(path: url.path, progress: progress) }.value24        var capabilities = model.capabilities; capabilities.remove(.vision); capabilities.remove(.embeddings)25        if session.templateMentionsTools { capabilities.insert(.tools) }26        progress(LoadProgress(stage: .ready, fraction: 1))27        return LlamaRuntime(session: session, info: RuntimeInfo(28            modelID: model.id, backend: .gguf, loadTime: Date().timeIntervalSince(start), weightsMemoryBytes: Int64(session.modelSize),29            contextLength: session.trainedContext, capabilities: capabilities, chatTemplateAvailable: session.chatTemplate != nil))30    }31}3233/// Owns the llama.cpp model and context. Every call into llama.cpp happens on `queue` (the C API is not thread-safe per context).34final class LlamaSession: @unchecked Sendable {35    private static let initBackend: Void = { llama_backend_init(); llama_log_set({ _, _, _ in }, nil) }()36    private final class ProgressBox { let report: @Sendable (LoadProgress) -> Void; init(_ r: @escaping @Sendable (LoadProgress) -> Void) { report = r } }3738    let queue = DispatchQueue(label: "ai.spboucher.localchat.llama", qos: .userInitiated)39    private var model: OpaquePointer?40    private var context: OpaquePointer?41    private var contextSize: UInt32 = 042    private var evaluated: [llama_token] = []      // tokens currently in the KV cache (sequence 0)43    private let vocab: OpaquePointer44    let chatTemplate: String?45    let trainedContext: Int46    let modelSize: UInt6447    let bosText: String48    let eosText: String49    private let lock = NSLock()50    private var cancelFlag = false5152    var templateMentionsTools: Bool { chatTemplate?.contains("tools") ?? false }5354    init(path: String, progress: @escaping @Sendable (LoadProgress) -> Void) throws {55        _ = Self.initBackend56        var params = llama_model_default_params()57        params.n_gpu_layers = -1   // offload every layer to Metal58        let box = Unmanaged.passRetained(ProgressBox(progress))59        params.progress_callback = { value, user in60            if let user { Unmanaged<ProgressBox>.fromOpaque(user).takeUnretainedValue().report(LoadProgress(stage: .loadingWeights, fraction: Double(value))) }61            return true62        }63        params.progress_callback_user_data = box.toOpaque()64        defer { box.release() }65        guard let model = llama_model_load_from_file(path, params) else {66            throw InferenceError.loadFailed("llama.cpp could not read this GGUF file (unsupported architecture or corrupt file).")67        }68        self.model = model69        vocab = llama_model_get_vocab(model)70        trainedContext = Int(llama_model_n_ctx_train(model))71        modelSize = llama_model_size(model)72        chatTemplate = llama_model_chat_template(model, nil).map { String(cString: $0) }73        bosText = Self.piece(vocab, llama_vocab_bos(vocab), special: true)74        eosText = Self.piece(vocab, llama_vocab_eos(vocab), special: true)75    }7677    deinit {78        if let context { llama_free(context) }79        if let model { llama_model_free(model) }80    }8182    func cancel() { lock.lock(); cancelFlag = true; lock.unlock() }83    private var isCancelled: Bool { lock.lock(); defer { lock.unlock() }; return cancelFlag }8485    func free() {86        queue.sync {87            if let context { llama_free(context) }; context = nil88            if let model { llama_model_free(model) }; model = nil89            evaluated = []90        }91    }9293    static func piece(_ vocab: OpaquePointer, _ token: llama_token, special: Bool) -> String {94        String(decoding: pieceBytes(vocab, token, special: special), as: UTF8.self)95    }96    static func pieceBytes(_ vocab: OpaquePointer, _ token: llama_token, special: Bool) -> [UInt8] {97        var buffer = [CChar](repeating: 0, count: 64)98        var n = llama_token_to_piece(vocab, token, &buffer, Int32(buffer.count), 0, special)99        if n < 0 { buffer = [CChar](repeating: 0, count: Int(-n)); n = llama_token_to_piece(vocab, token, &buffer, Int32(buffer.count), 0, special) }100        return buffer.prefix(Int(max(0, n))).map { UInt8(bitPattern: $0) }101    }102103    func tokenize(_ text: String, addSpecial: Bool, parseSpecial: Bool) -> [llama_token] {104        let utf8 = Array(text.utf8CString.dropLast())105        var tokens = [llama_token](repeating: 0, count: utf8.count + 16)106        var n = llama_tokenize(vocab, utf8, Int32(utf8.count), &tokens, Int32(tokens.count), addSpecial, parseSpecial)107        if n < 0 { tokens = [llama_token](repeating: 0, count: Int(-n)); n = llama_tokenize(vocab, utf8, Int32(utf8.count), &tokens, Int32(tokens.count), addSpecial, parseSpecial) }108        return Array(tokens.prefix(Int(max(0, n))))109    }110111    /// Renders the model's own Jinja chat template (same data model as transformers: messages, tools, add_generation_prompt…).112    func renderPrompt(_ request: GenerationRequest) throws -> String {113        guard let chatTemplate else { throw InferenceError.unsupported("This GGUF file has no chat template; it cannot be used for chat.") }114        let messages: [[String: Any]] = request.messages.map { m in115            var d: [String: Any] = ["role": m.role.rawValue, "content": m.content]116            if !m.toolCalls.isEmpty {117                d["tool_calls"] = m.toolCalls.map { ["id": $0.id, "type": "function", "function": ["name": $0.name, "arguments": $0.arguments.anyValue]] as [String: Any] }118            }119            if let id = m.toolCallID { d["tool_call_id"] = id }120            if let name = m.toolName { d["name"] = name }121            return d122        }123        var context: [String: Any] = ["messages": messages, "add_generation_prompt": true, "bos_token": bosText, "eos_token": eosText,124                                      "enable_thinking": request.settings.reasoningEnabled]125        if !request.tools.isEmpty { context["tools"] = request.tools.map { $0.openAISpec.anyValue } }126        do {127            let template = try Template(chatTemplate, with: .init(lstripBlocks: true, trimBlocks: true))128            return try template.render(try context.mapValues { try Value(any: $0) })129        } catch {130            throw InferenceError.generationFailed("The model's chat template could not be rendered: \(error)")131        }132    }133134    private func ensureContext(_ wanted: Int) throws {135        let size = UInt32(max(512, min(wanted, trainedContext > 0 ? trainedContext : wanted)))136        if context != nil, contextSize == size { return }137        if let context { llama_free(context) }138        context = nil; evaluated = []139        var params = llama_context_default_params()140        params.n_ctx = size; params.n_batch = 512141        let threads = Int32(max(2, DeviceProfile.sysctlIntPublic("hw.perflevel0.physicalcpu") ?? ProcessInfo.processInfo.activeProcessorCount / 2))142        params.n_threads = threads; params.n_threads_batch = threads143        guard let model, let created = llama_init_from_model(model, params) else {144            throw InferenceError.loadFailed("llama.cpp could not create a context of \(size) tokens (not enough memory).")145        }146        context = created; contextSize = size147    }148149    func generate(_ request: GenerationRequest, yield: (GenerationEvent) -> Void) throws {150        lock.lock(); cancelFlag = false; lock.unlock()151        let started = Date()152        let s = request.settings153        try ensureContext(s.contextLength)154        guard let context else { throw InferenceError.modelNotLoaded }155        let prompt = try renderPrompt(request)156        let tokens = tokenize(prompt, addSpecial: !prompt.hasPrefix(bosText) || bosText.isEmpty, parseSpecial: true)157        let room = Int(contextSize) - tokens.count158        guard room > 16 else {159            throw InferenceError.generationFailed("The prompt (\(tokens.count) tokens) does not fit in the context window (\(contextSize) tokens).")160        }161        let maxNew = min(s.maxTokens, room - 1)162163        // KV reuse: keep the longest common prefix with what is already evaluated, drop the rest.164        var common = 0165        while common < min(evaluated.count, tokens.count - 1), evaluated[common] == tokens[common] { common += 1 }166        let memory = llama_get_memory(context)167        if common < evaluated.count, !llama_memory_seq_rm(memory, 0, llama_pos(common), -1) { llama_memory_clear(memory, true); common = 0 }168        evaluated = Array(tokens.prefix(common))169170        let promptStart = Date()171        var index = common172        while index < tokens.count {173            if isCancelled { throw InferenceError.cancelled }174            var slice = Array(tokens[index..<min(index + 512, tokens.count)])175            let status = slice.withUnsafeMutableBufferPointer { llama_decode(context, llama_batch_get_one($0.baseAddress, Int32($0.count))) }176            guard status == 0 else { evaluated = []; llama_memory_clear(memory, true); throw InferenceError.generationFailed("llama_decode failed (\(status)).") }177            evaluated.append(contentsOf: slice); index += slice.count178        }179        let promptTime = Date().timeIntervalSince(promptStart)180181        let chain = llama_sampler_chain_init(llama_sampler_chain_default_params())!182        defer { llama_sampler_free(chain) }183        if s.repetitionPenalty != 1 { llama_sampler_chain_add(chain, llama_sampler_init_penalties(llama_vocab_n_tokens(vocab), Int32(s.repetitionContext), Float(s.repetitionPenalty), 0, 0)) }184        if s.temperature <= 0 { llama_sampler_chain_add(chain, llama_sampler_init_greedy()) }185        else {186            if s.topK > 0 { llama_sampler_chain_add(chain, llama_sampler_init_top_k(Int32(s.topK))) }187            if s.topP < 1 { llama_sampler_chain_add(chain, llama_sampler_init_top_p(Float(s.topP), 1)) }188            if s.minP > 0 { llama_sampler_chain_add(chain, llama_sampler_init_min_p(Float(s.minP), 1)) }189            llama_sampler_chain_add(chain, llama_sampler_init_temp(Float(s.temperature)))190            llama_sampler_chain_add(chain, llama_sampler_init_dist(s.seed.map { UInt32(truncatingIfNeeded: $0) } ?? UInt32.random(in: 0..<UInt32.max)))191        }192193        let opensThinking = prompt.hasSuffix("<think>\n") || prompt.hasSuffix("<think>")194        var splitter = ReasoningSplitter(startsInside: opensThinking)195        var pending = [UInt8](); var answer = ""; var held = ""; var generated = 0; var reasoningTokens = 0196        var firstToken: Date?; var reason: StopReason = .endOfTurn197        let wantsTools = !request.tools.isEmpty198        let generationStart = Date()199200        func emit(_ text: String) {201            for piece in splitter.feed(text) { route(piece) }202        }203        func route(_ piece: ReasoningSplitter.Piece) {204            switch piece {205            case .reasoning(let t): yield(.reasoning(t))206            case .text(let t):207                answer += t208                if wantsTools {209                    // hold back anything that may be a textual tool call until generation ends210                    held += t211                    if !held.contains("<tool_call>") && !held.contains("[TOOL_CALLS]") {212                        let safe = ReasoningSplitter.safePrefix(held, markers: ["<tool_call>", "[TOOL_CALLS]"])213                        if !safe.isEmpty { yield(.text(safe)); held.removeFirst(safe.count) }214                    }215                } else { yield(.text(t)) }216            }217        }218219        while generated < maxNew {220            if isCancelled { reason = .cancelled; break }221            var token = llama_sampler_sample(chain, context, -1)222            if llama_vocab_is_eog(vocab, token) { break }223            generated += 1224            if firstToken == nil { firstToken = Date() }225            pending.append(contentsOf: Self.pieceBytes(vocab, token, special: false))226            // emit only complete UTF-8 sequences227            if let text = String(validatingUTF8Bytes: pending) { pending.removeAll(keepingCapacity: true); if !text.isEmpty { emit(text) } }228            else if pending.count > 8 { emit(String(decoding: pending, as: UTF8.self)); pending.removeAll(keepingCapacity: true) }229            if let stop = request.stop.first(where: { !$0.isEmpty && answer.contains($0) }) {230                _ = stop; reason = .stopSequence; break231            }232            let status = withUnsafeMutablePointer(to: &token) { llama_decode(context, llama_batch_get_one($0, 1)) }233            guard status == 0 else { evaluated = []; llama_memory_clear(memory, true); throw InferenceError.generationFailed("llama_decode failed (\(status)).") }234            evaluated.append(token)235            _ = reasoningTokens236        }237        if generated >= maxNew, reason == .endOfTurn { reason = .maxTokens }238        if !pending.isEmpty { emit(String(decoding: pending, as: UTF8.self)) }239        for piece in splitter.finish() { route(piece) }240        reasoningTokens = 0241242        if wantsTools {243            let parsed = TextToolCallParser.extract(from: held, allowed: Set(request.tools.map(\.name)))244            if !parsed.clean.isEmpty { yield(.text(parsed.clean)) }245            for call in parsed.calls { yield(.toolCall(call)) }246            if !parsed.calls.isEmpty { reason = .toolCall }247        }248        let now = Date()249        yield(.finished(GenerationMetrics(250            promptTokens: tokens.count, cachedPromptTokens: common, completionTokens: generated, reasoningTokens: nil,251            timeToFirstToken: firstToken.map { $0.timeIntervalSince(started) }, promptTime: promptTime,252            generationTime: now.timeIntervalSince(generationStart), totalTime: now.timeIntervalSince(started),253            peakMemoryBytes: nil, activeMemoryBytes: DeviceProfile.processFootprintBytes(), stopReason: reason)))254    }255256    var cachedTokenCount: Int { queue.sync { evaluated.count } }257    func resetCache() { queue.sync { if let context { llama_memory_clear(llama_get_memory(context), true) }; evaluated = [] } }258}259260private extension String {261    init?(validatingUTF8Bytes bytes: [UInt8]) {262        guard let s = String(bytes: bytes, encoding: .utf8) else { return nil }263        self = s264    }265}266267struct LlamaRuntime: ModelRuntime {268    let session: LlamaSession269    let info: RuntimeInfo270271    func generate(_ request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error> {272        AsyncThrowingStream { continuation in273            session.queue.async {274                do { try session.generate(request) { continuation.yield($0) }; continuation.finish() }275                catch { continuation.finish(throwing: error) }276            }277            continuation.onTermination = { termination in if case .cancelled = termination { session.cancel() } }278        }279    }280    func countTokens(_ text: String) async -> Int? { session.tokenize(text, addSpecial: false, parseSpecial: true).count }281    func cacheStatistics() async -> CacheStatistics? { CacheStatistics(cachedTokens: session.cachedTokenCount) }282    func resetSession(_ key: String?) async { session.resetCache() }283    func unload() async { session.cancel(); session.free() }284}285#else286public struct GGUFBackend: InferenceBackend {287    public let kind: BackendKind = .gguf288    public init() {}289    public var isAvailable: Bool { false }290    public var unavailableReason: String? { "GGUF models run on the Mac in this version (the llama.cpp build used here has no iOS Simulator slice). Use an MLX model, or route to a Mac." }291    public func load(_ model: ModelDescriptor, modelsRoot: URL, progress: @escaping @Sendable (LoadProgress) -> Void) async throws -> any ModelRuntime {292        throw InferenceError.backendUnavailable(unavailableReason ?? "")293    }294}295#endif296
Files main
Menu

Theme