Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatInferenceTests/RealInferenceTests.swift3// Description: Real inference on this Mac's GPU with real weights (MLX and llama.cpp). Skipped, with the reason recorded, when the weights are not on disk.4import Foundation5import Testing6@testable import LocalChatInference7import LocalChatCore89enum TestModels {10 /// An MLX model already in the local Hugging Face cache (never downloaded by the tests).11 static var mlxDirectory: URL? {12 if let override = ProcessInfo.processInfo.environment["LC_TEST_MLX_DIR"] { return URL(fileURLWithPath: override) }13 let home = URL(fileURLWithPath: NSHomeDirectory().components(separatedBy: "/Library/Containers").first ?? NSHomeDirectory())14 let roots = ["Library/Caches/LocalChatTests/mlx", ".cache/huggingface/hub/models--mlx-community--Qwen2.5-1.5B-Instruct-bf16/snapshots"]15 for root in roots {16 let base = home.appendingPathComponent(root)17 let candidates = (try? FileManager.default.contentsOfDirectory(at: base, includingPropertiesForKeys: nil)) ?? []18 if let hit = candidates.first(where: { FileManager.default.fileExists(atPath: $0.appendingPathComponent("config.json").path) }) { return hit }19 }20 return nil21 }22 static var ggufFile: URL? {23 let home = URL(fileURLWithPath: NSHomeDirectory())24 let url = home.appendingPathComponent("Library/Caches/LocalChatTests/SmolLM2-360M-Instruct-Q4_K_M.gguf")25 return FileManager.default.fileExists(atPath: url.path) ? url : nil26 }27 static func mlxDescriptor(_ dir: URL) -> ModelDescriptor {28 ModelDescriptor(id: "mlx:test", name: "Test MLX", backend: .mlx, format: .mlx, parameterCount: 1_500_000_000,29 capabilities: [.chat, .tools], sizeBytes: 3_100_000_000, localPath: dir.path)30 }31 static func ggufDescriptor(_ file: URL) -> ModelDescriptor {32 ModelDescriptor(id: "gguf:test", name: "Test GGUF", backend: .gguf, format: .gguf, parameterCount: 360_000_000,33 capabilities: [.chat], sizeBytes: 270_590_880, localPath: file.path)34 }35}3637func collect(_ stream: AsyncThrowingStream<GenerationEvent, Error>) async throws -> (text: String, reasoning: String, calls: [ToolInvocation], metrics: GenerationMetrics?, chunks: Int) {38 var text = "", reasoning = ""; var calls: [ToolInvocation] = []; var metrics: GenerationMetrics?; var chunks = 039 for try await event in stream {40 switch event {41 case .text(let t): text += t; chunks += 142 case .reasoning(let t): reasoning += t43 case .toolCall(let c): calls.append(c)44 case .finished(let m): metrics = m45 }46 }47 return (text, reasoning, calls, metrics, chunks)48}4950@Suite(.serialized) struct RealMLXInference {51 @Test func streamsARealAnswerAndReusesTheCache() async throws {52 guard let dir = TestModels.mlxDirectory else { print("SKIPPED: no MLX weights on disk"); return }53 let runtime = try await MLXBackend().load(TestModels.mlxDescriptor(dir), modelsRoot: dir) { _ in }54 #expect(runtime.info.loadTime > 0)55 #expect((runtime.info.weightsMemoryBytes ?? 0) > 100_000_000)56 #expect(runtime.info.chatTemplateAvailable)5758 let settings = GenerationSettings(temperature: 0, topK: 0, seed: 7, contextLength: 4096, maxTokens: 48)59 var messages: [PromptMessage] = [.system("You are a terse assistant."), .user("What is the capital of France? Answer with one word.")]60 let first = try await collect(runtime.generate(GenerationRequest(messages: messages, settings: settings, sessionKey: "t")))61 print("MLX answer 1:", first.text, first.metrics as Any)62 #expect(first.text.lowercased().contains("paris"))63 #expect(first.chunks >= 1)64 let m1 = try #require(first.metrics)65 #expect((m1.promptTokens ?? 0) > 10)66 #expect((m1.completionTokens ?? 0) >= 1)67 #expect((m1.tokensPerSecond ?? 0) > 1)68 #expect((m1.timeToFirstToken ?? 0) > 0)69 #expect((m1.peakMemoryBytes ?? 0) > 0)7071 messages.append(.assistant(first.text)); messages.append(.user("And of Italy? One word."))72 let second = try await collect(runtime.generate(GenerationRequest(messages: messages, settings: settings, sessionKey: "t")))73 print("MLX answer 2:", second.text, second.metrics as Any)74 #expect(second.text.lowercased().contains("rome"))75 #expect((second.metrics?.cachedPromptTokens ?? 0) > 0, "the second turn must reuse the KV cache of the first")7677 // an edited history (different first question) cannot reuse the session and must still answer correctly78 let edited: [PromptMessage] = [.system("You are a terse assistant."), .user("What is the capital of Japan? Answer with one word.")]79 let third = try await collect(runtime.generate(GenerationRequest(messages: edited, settings: settings, sessionKey: "t")))80 #expect(third.text.lowercased().contains("tokyo"))8182 let count = await runtime.countTokens("Hello, world!")83 #expect((count ?? 0) >= 3)84 await runtime.unload()85 }8687 @Test func cancellationStopsGeneration() async throws {88 guard let dir = TestModels.mlxDirectory else { print("SKIPPED: no MLX weights on disk"); return }89 let runtime = try await MLXBackend().load(TestModels.mlxDescriptor(dir), modelsRoot: dir) { _ in }90 let request = GenerationRequest(messages: [.user("Write a very long story about a lighthouse keeper.")],91 settings: GenerationSettings(contextLength: 4096, maxTokens: 2000))92 let started = Date(); var pieces = 093 let task = Task { () -> Int in94 var n = 095 for try await event in runtime.generate(request) { if case .text = event { n += 1; if n == 12 { break } } }96 return n97 }98 pieces = try await task.value99 #expect(pieces == 12)100 #expect(Date().timeIntervalSince(started) < 30)101 // the runtime must be usable right after a cancelled generation102 let again = try await collect(runtime.generate(GenerationRequest(messages: [.user("Say OK.")], settings: GenerationSettings(temperature: 0, maxTokens: 8))))103 #expect(!again.text.isEmpty)104 await runtime.unload()105 }106107 @Test func toolCallIsEmittedForAToolPrompt() async throws {108 guard let dir = TestModels.mlxDirectory else { print("SKIPPED: no MLX weights on disk"); return }109 let runtime = try await MLXBackend().load(TestModels.mlxDescriptor(dir), modelsRoot: dir) { _ in }110 let tool = ToolDefinition(name: "get_weather", summary: "Get the current weather for a city.",111 parameters: ["type": "object", "properties": ["city": ["type": "string", "description": "City name"]], "required": ["city"]])112 let result = try await collect(runtime.generate(GenerationRequest(113 messages: [.system("Use tools when they help."), .user("What is the weather in Montreal right now?")],114 settings: GenerationSettings(temperature: 0, maxTokens: 96), tools: [tool])))115 print("MLX tool run:", result.text, result.calls)116 let call = try #require(result.calls.first, "the model should call the tool")117 #expect(call.name == "get_weather")118 #expect(call.arguments["city"]?.stringValue?.lowercased().contains("montr") == true)119 #expect(result.metrics?.stopReason == .toolCall)120 await runtime.unload()121 }122}123124@Suite(.serialized) struct RealEngine {125 @Test func engineLoadsGeneratesObservesAndUnloads() async throws {126 guard let dir = TestModels.mlxDirectory else { print("SKIPPED: no MLX weights on disk"); return }127 let model = TestModels.mlxDescriptor(dir)128 let engine = InferenceEngine(backends: [MLXBackend(), GGUFBackend()], modelsRoot: dir) { [model] }129 async let completed: CompletedGeneration? = { var it = await engine.completions().makeAsyncIterator(); return await it.next() }()130 try await Task.sleep(for: .milliseconds(50))131 let out = try await engine.complete(modelID: model.id, request: GenerationRequest(messages: [.user("Reply with the single word: ready")],132 settings: GenerationSettings(temperature: 0, maxTokens: 8)))133 #expect(!out.text.isEmpty)134 let snapshot = await engine.currentSnapshot()135 #expect(snapshot.isLoaded(model.id))136 #expect(snapshot.active.isEmpty)137 #expect((snapshot.mlxActiveBytes ?? 0) > 100_000_000)138 let done = await completed139 #expect(done?.descriptor.id == model.id)140 #expect((done?.metrics.completionTokens ?? 0) >= 1)141 let tokens = await engine.countTokens(modelID: model.id, text: "one two three")142 #expect((tokens ?? 0) >= 3)143 await engine.unload(model.id)144 #expect(await engine.currentSnapshot().loaded.isEmpty)145 }146147 @Test func engineRefusesAModelThatCannotFit() async throws {148 let huge = ModelDescriptor(id: "mlx:huge", name: "Huge", backend: .mlx, format: .mlx, sizeBytes: 900 * 1_073_741_824, localPath: "/nonexistent")149 let engine = InferenceEngine(backends: [MLXBackend()], modelsRoot: URL(fileURLWithPath: "/tmp")) { [huge] }150 await #expect(throws: InferenceError.self) { try await engine.load(huge) }151 }152}153154@Suite(.serialized) struct RealGGUFInference {155 @Test func llamaCppStreamsARealAnswerWithPrefixReuse() async throws {156 guard let file = TestModels.ggufFile else { print("SKIPPED: no GGUF file on disk"); return }157 let backend = GGUFBackend()158 guard backend.isAvailable else { print("SKIPPED: GGUF backend unavailable on this platform"); return }159 let runtime = try await backend.load(TestModels.ggufDescriptor(file), modelsRoot: file.deletingLastPathComponent()) { _ in }160 #expect(runtime.info.chatTemplateAvailable)161 #expect((runtime.info.contextLength ?? 0) >= 2048)162 let settings = GenerationSettings(temperature: 0, contextLength: 2048, maxTokens: 32)163 var messages: [PromptMessage] = [.user("What is the capital of France? Answer briefly.")]164 let first = try await collect(runtime.generate(GenerationRequest(messages: messages, settings: settings)))165 print("GGUF answer 1:", first.text, first.metrics as Any)166 #expect(first.text.lowercased().contains("paris"))167 #expect((first.metrics?.completionTokens ?? 0) >= 1)168 #expect((first.metrics?.tokensPerSecond ?? 0) > 1)169 messages.append(.assistant(first.text)); messages.append(.user("And of Italy?"))170 let second = try await collect(runtime.generate(GenerationRequest(messages: messages, settings: settings)))171 print("GGUF answer 2:", second.text, second.metrics as Any)172 #expect((second.metrics?.cachedPromptTokens ?? 0) > 5)173 #expect(!second.text.isEmpty)174 #expect((await runtime.countTokens("Hello, world!") ?? 0) >= 3)175 await runtime.unload()176 }177}178179@Suite struct StreamParsingTests {180 @Test func reasoningSplitterHandlesSplitMarkers() {181 var s = ReasoningSplitter()182 var pieces: [ReasoningSplitter.Piece] = []183 for chunk in ["Hi <th", "ink>\nplan", " more</thi", "nk>\n\nAnswer", " done"] { pieces += s.feed(chunk) }184 pieces += s.finish()185 let text = pieces.compactMap { if case .text(let t) = $0 { t } else { nil } }.joined()186 let reasoning = pieces.compactMap { if case .reasoning(let t) = $0 { t } else { nil } }.joined()187 #expect(text == "Hi Answer done")188 #expect(reasoning == "plan more")189 }190 @Test func splitterStartingInsideThinking() {191 var s = ReasoningSplitter(startsInside: true)192 var pieces = s.feed("thinking…</think>\nfinal"); pieces += s.finish()193 #expect(pieces == [.reasoning("thinking…"), .text("final")])194 }195 @Test func textToolCallsAreRecovered() {196 let raw = "Let me check.\n<tool_call>\n{\"name\": \"calculator\", \"arguments\": {\"expression\": \"2+2\"}}\n</tool_call>"197 let parsed = TextToolCallParser.extract(from: raw, allowed: ["calculator"])198 #expect(parsed.clean == "Let me check.")199 #expect(parsed.calls.first?.name == "calculator")200 #expect(parsed.calls.first?.arguments["expression"]?.stringValue == "2+2")201 #expect(TextToolCallParser.extract(from: raw, allowed: ["other"]).calls.isEmpty)202 }203}204