SPB Git

Repository Public

RealInferenceTests.swift 12.2 KB 204 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatInferenceTests/RealInferenceTests.swift3// Description: Real inference on this Mac's GPU with real weights (MLX and llama.cpp). Skipped, with the reason recorded, when the weights are not on disk.4import Foundation5import Testing6@testable import LocalChatInference7import LocalChatCore89enum TestModels {10    /// An MLX model already in the local Hugging Face cache (never downloaded by the tests).11    static var mlxDirectory: URL? {12        if let override = ProcessInfo.processInfo.environment["LC_TEST_MLX_DIR"] { return URL(fileURLWithPath: override) }13        let home = URL(fileURLWithPath: NSHomeDirectory().components(separatedBy: "/Library/Containers").first ?? NSHomeDirectory())14        let roots = ["Library/Caches/LocalChatTests/mlx", ".cache/huggingface/hub/models--mlx-community--Qwen2.5-1.5B-Instruct-bf16/snapshots"]15        for root in roots {16            let base = home.appendingPathComponent(root)17            let candidates = (try? FileManager.default.contentsOfDirectory(at: base, includingPropertiesForKeys: nil)) ?? []18            if let hit = candidates.first(where: { FileManager.default.fileExists(atPath: $0.appendingPathComponent("config.json").path) }) { return hit }19        }20        return nil21    }22    static var ggufFile: URL? {23        let home = URL(fileURLWithPath: NSHomeDirectory())24        let url = home.appendingPathComponent("Library/Caches/LocalChatTests/SmolLM2-360M-Instruct-Q4_K_M.gguf")25        return FileManager.default.fileExists(atPath: url.path) ? url : nil26    }27    static func mlxDescriptor(_ dir: URL) -> ModelDescriptor {28        ModelDescriptor(id: "mlx:test", name: "Test MLX", backend: .mlx, format: .mlx, parameterCount: 1_500_000_000,29                        capabilities: [.chat, .tools], sizeBytes: 3_100_000_000, localPath: dir.path)30    }31    static func ggufDescriptor(_ file: URL) -> ModelDescriptor {32        ModelDescriptor(id: "gguf:test", name: "Test GGUF", backend: .gguf, format: .gguf, parameterCount: 360_000_000,33                        capabilities: [.chat], sizeBytes: 270_590_880, localPath: file.path)34    }35}3637func collect(_ stream: AsyncThrowingStream<GenerationEvent, Error>) async throws -> (text: String, reasoning: String, calls: [ToolInvocation], metrics: GenerationMetrics?, chunks: Int) {38    var text = "", reasoning = ""; var calls: [ToolInvocation] = []; var metrics: GenerationMetrics?; var chunks = 039    for try await event in stream {40        switch event {41        case .text(let t): text += t; chunks += 142        case .reasoning(let t): reasoning += t43        case .toolCall(let c): calls.append(c)44        case .finished(let m): metrics = m45        }46    }47    return (text, reasoning, calls, metrics, chunks)48}4950@Suite(.serialized) struct RealMLXInference {51    @Test func streamsARealAnswerAndReusesTheCache() async throws {52        guard let dir = TestModels.mlxDirectory else { print("SKIPPED: no MLX weights on disk"); return }53        let runtime = try await MLXBackend().load(TestModels.mlxDescriptor(dir), modelsRoot: dir) { _ in }54        #expect(runtime.info.loadTime > 0)55        #expect((runtime.info.weightsMemoryBytes ?? 0) > 100_000_000)56        #expect(runtime.info.chatTemplateAvailable)5758        let settings = GenerationSettings(temperature: 0, topK: 0, seed: 7, contextLength: 4096, maxTokens: 48)59        var messages: [PromptMessage] = [.system("You are a terse assistant."), .user("What is the capital of France? Answer with one word.")]60        let first = try await collect(runtime.generate(GenerationRequest(messages: messages, settings: settings, sessionKey: "t")))61        print("MLX answer 1:", first.text, first.metrics as Any)62        #expect(first.text.lowercased().contains("paris"))63        #expect(first.chunks >= 1)64        let m1 = try #require(first.metrics)65        #expect((m1.promptTokens ?? 0) > 10)66        #expect((m1.completionTokens ?? 0) >= 1)67        #expect((m1.tokensPerSecond ?? 0) > 1)68        #expect((m1.timeToFirstToken ?? 0) > 0)69        #expect((m1.peakMemoryBytes ?? 0) > 0)7071        messages.append(.assistant(first.text)); messages.append(.user("And of Italy? One word."))72        let second = try await collect(runtime.generate(GenerationRequest(messages: messages, settings: settings, sessionKey: "t")))73        print("MLX answer 2:", second.text, second.metrics as Any)74        #expect(second.text.lowercased().contains("rome"))75        #expect((second.metrics?.cachedPromptTokens ?? 0) > 0, "the second turn must reuse the KV cache of the first")7677        // an edited history (different first question) cannot reuse the session and must still answer correctly78        let edited: [PromptMessage] = [.system("You are a terse assistant."), .user("What is the capital of Japan? Answer with one word.")]79        let third = try await collect(runtime.generate(GenerationRequest(messages: edited, settings: settings, sessionKey: "t")))80        #expect(third.text.lowercased().contains("tokyo"))8182        let count = await runtime.countTokens("Hello, world!")83        #expect((count ?? 0) >= 3)84        await runtime.unload()85    }8687    @Test func cancellationStopsGeneration() async throws {88        guard let dir = TestModels.mlxDirectory else { print("SKIPPED: no MLX weights on disk"); return }89        let runtime = try await MLXBackend().load(TestModels.mlxDescriptor(dir), modelsRoot: dir) { _ in }90        let request = GenerationRequest(messages: [.user("Write a very long story about a lighthouse keeper.")],91                                        settings: GenerationSettings(contextLength: 4096, maxTokens: 2000))92        let started = Date(); var pieces = 093        let task = Task { () -> Int in94            var n = 095            for try await event in runtime.generate(request) { if case .text = event { n += 1; if n == 12 { break } } }96            return n97        }98        pieces = try await task.value99        #expect(pieces == 12)100        #expect(Date().timeIntervalSince(started) < 30)101        // the runtime must be usable right after a cancelled generation102        let again = try await collect(runtime.generate(GenerationRequest(messages: [.user("Say OK.")], settings: GenerationSettings(temperature: 0, maxTokens: 8))))103        #expect(!again.text.isEmpty)104        await runtime.unload()105    }106107    @Test func toolCallIsEmittedForAToolPrompt() async throws {108        guard let dir = TestModels.mlxDirectory else { print("SKIPPED: no MLX weights on disk"); return }109        let runtime = try await MLXBackend().load(TestModels.mlxDescriptor(dir), modelsRoot: dir) { _ in }110        let tool = ToolDefinition(name: "get_weather", summary: "Get the current weather for a city.",111                                  parameters: ["type": "object", "properties": ["city": ["type": "string", "description": "City name"]], "required": ["city"]])112        let result = try await collect(runtime.generate(GenerationRequest(113            messages: [.system("Use tools when they help."), .user("What is the weather in Montreal right now?")],114            settings: GenerationSettings(temperature: 0, maxTokens: 96), tools: [tool])))115        print("MLX tool run:", result.text, result.calls)116        let call = try #require(result.calls.first, "the model should call the tool")117        #expect(call.name == "get_weather")118        #expect(call.arguments["city"]?.stringValue?.lowercased().contains("montr") == true)119        #expect(result.metrics?.stopReason == .toolCall)120        await runtime.unload()121    }122}123124@Suite(.serialized) struct RealEngine {125    @Test func engineLoadsGeneratesObservesAndUnloads() async throws {126        guard let dir = TestModels.mlxDirectory else { print("SKIPPED: no MLX weights on disk"); return }127        let model = TestModels.mlxDescriptor(dir)128        let engine = InferenceEngine(backends: [MLXBackend(), GGUFBackend()], modelsRoot: dir) { [model] }129        async let completed: CompletedGeneration? = { var it = await engine.completions().makeAsyncIterator(); return await it.next() }()130        try await Task.sleep(for: .milliseconds(50))131        let out = try await engine.complete(modelID: model.id, request: GenerationRequest(messages: [.user("Reply with the single word: ready")],132                                                                                         settings: GenerationSettings(temperature: 0, maxTokens: 8)))133        #expect(!out.text.isEmpty)134        let snapshot = await engine.currentSnapshot()135        #expect(snapshot.isLoaded(model.id))136        #expect(snapshot.active.isEmpty)137        #expect((snapshot.mlxActiveBytes ?? 0) > 100_000_000)138        let done = await completed139        #expect(done?.descriptor.id == model.id)140        #expect((done?.metrics.completionTokens ?? 0) >= 1)141        let tokens = await engine.countTokens(modelID: model.id, text: "one two three")142        #expect((tokens ?? 0) >= 3)143        await engine.unload(model.id)144        #expect(await engine.currentSnapshot().loaded.isEmpty)145    }146147    @Test func engineRefusesAModelThatCannotFit() async throws {148        let huge = ModelDescriptor(id: "mlx:huge", name: "Huge", backend: .mlx, format: .mlx, sizeBytes: 900 * 1_073_741_824, localPath: "/nonexistent")149        let engine = InferenceEngine(backends: [MLXBackend()], modelsRoot: URL(fileURLWithPath: "/tmp")) { [huge] }150        await #expect(throws: InferenceError.self) { try await engine.load(huge) }151    }152}153154@Suite(.serialized) struct RealGGUFInference {155    @Test func llamaCppStreamsARealAnswerWithPrefixReuse() async throws {156        guard let file = TestModels.ggufFile else { print("SKIPPED: no GGUF file on disk"); return }157        let backend = GGUFBackend()158        guard backend.isAvailable else { print("SKIPPED: GGUF backend unavailable on this platform"); return }159        let runtime = try await backend.load(TestModels.ggufDescriptor(file), modelsRoot: file.deletingLastPathComponent()) { _ in }160        #expect(runtime.info.chatTemplateAvailable)161        #expect((runtime.info.contextLength ?? 0) >= 2048)162        let settings = GenerationSettings(temperature: 0, contextLength: 2048, maxTokens: 32)163        var messages: [PromptMessage] = [.user("What is the capital of France? Answer briefly.")]164        let first = try await collect(runtime.generate(GenerationRequest(messages: messages, settings: settings)))165        print("GGUF answer 1:", first.text, first.metrics as Any)166        #expect(first.text.lowercased().contains("paris"))167        #expect((first.metrics?.completionTokens ?? 0) >= 1)168        #expect((first.metrics?.tokensPerSecond ?? 0) > 1)169        messages.append(.assistant(first.text)); messages.append(.user("And of Italy?"))170        let second = try await collect(runtime.generate(GenerationRequest(messages: messages, settings: settings)))171        print("GGUF answer 2:", second.text, second.metrics as Any)172        #expect((second.metrics?.cachedPromptTokens ?? 0) > 5)173        #expect(!second.text.isEmpty)174        #expect((await runtime.countTokens("Hello, world!") ?? 0) >= 3)175        await runtime.unload()176    }177}178179@Suite struct StreamParsingTests {180    @Test func reasoningSplitterHandlesSplitMarkers() {181        var s = ReasoningSplitter()182        var pieces: [ReasoningSplitter.Piece] = []183        for chunk in ["Hi <th", "ink>\nplan", " more</thi", "nk>\n\nAnswer", " done"] { pieces += s.feed(chunk) }184        pieces += s.finish()185        let text = pieces.compactMap { if case .text(let t) = $0 { t } else { nil } }.joined()186        let reasoning = pieces.compactMap { if case .reasoning(let t) = $0 { t } else { nil } }.joined()187        #expect(text == "Hi Answer done")188        #expect(reasoning == "plan more")189    }190    @Test func splitterStartingInsideThinking() {191        var s = ReasoningSplitter(startsInside: true)192        var pieces = s.feed("thinking…</think>\nfinal"); pieces += s.finish()193        #expect(pieces == [.reasoning("thinking…"), .text("final")])194    }195    @Test func textToolCallsAreRecovered() {196        let raw = "Let me check.\n<tool_call>\n{\"name\": \"calculator\", \"arguments\": {\"expression\": \"2+2\"}}\n</tool_call>"197        let parsed = TextToolCallParser.extract(from: raw, allowed: ["calculator"])198        #expect(parsed.clean == "Let me check.")199        #expect(parsed.calls.first?.name == "calculator")200        #expect(parsed.calls.first?.arguments["expression"]?.stringValue == "2+2")201        #expect(TextToolCallParser.extract(from: raw, allowed: ["other"]).calls.isEmpty)202    }203}204
Files main
Menu

Theme