SPB Git

Repository Public MIT

MLXInferenceTests.swift 16.6 KB 275 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraLocalTests/MLXInferenceTests.swift10// Module           : AstraLocalTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraLocal9293/// Real inference. Enabled with `ASTRA_MLX_TESTS=1` when the model directory exists94/// (`ASTRA_MLX_MODEL_DIR`, default `build/models/Qwen3-0.6B-4bit`, downloaded by Scripts/verify-models.sh).95enum MLXTestModel {96    static let directory: URL = ProjectPaths.env("ASTRA_MLX_MODEL_DIR").map { URL(fileURLWithPath: $0) } ?? ProjectPaths.models.appendingPathComponent("Qwen3-0.6B-4bit")97    static let enabled = ProjectPaths.env("ASTRA_MLX_TESTS") == "1" && FileManager.default.fileExists(atPath: directory.appendingPathComponent("config.json").path)98    static let id = "test-model"99    static let ref = ModelRef(provider: .mlx, modelID: id)100}101102@Suite(.enabled(if: MLXTestModel.enabled), .serialized) struct MLXInferenceTests {103    private func engine() async throws -> LocalEngine {104        var configuration = LocalEngine.Configuration(maxLoadedModels: 1); configuration.enforceMemoryVerdict = false105        let engine = LocalEngine(store: ModelStore(root: ProjectPaths.models), configuration: configuration)106        try await engine.load(modelID: MLXTestModel.id, directory: MLXTestModel.directory)107        return engine108    }109110    private func run(_ engine: LocalEngine, _ request: GenerationRequest) async throws -> GenerationOutcome {111        var validator = StreamValidator(); var outcome = GenerationOutcome()112        for try await event in engine.generate(request) { for e in try validator.accept(event) { outcome.apply(e) } }113        #expect(validator.finished)114        return outcome115    }116117    @Test func streamsARealAnswerWithMeasuredUsageAndMetrics() async throws {118        let engine = try await engine()119        let info = try #require(await engine.loadedInfo(MLXTestModel.id))120        #expect(info.hasChatTemplate && (info.weightsMemoryBytes ?? 0) > 0 && info.loadTime > 0)121        let request = GenerationRequest(model: MLXTestModel.ref, system: "You are terse.", messages: [.user("Say hello in five words.")],122                                        parameters: GenerationParameters(temperature: 0, maxOutputTokens: 48))123        let outcome = try await run(engine, request)124        #expect(!outcome.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)125        #expect(outcome.start.reportedModel == MLXTestModel.id)126        #expect((outcome.usage?.inputTokens ?? 0) > 0 && (outcome.usage?.outputTokens ?? 0) > 0)127        #expect(outcome.stopReason == .endOfTurn || outcome.stopReason == .maxTokens)128        let m = try #require(engine.monitor.lastMetrics)129        #expect(m.isColdStart && (m.timeToFirstToken ?? 0) > 0 && (m.decodeTokensPerSecond ?? 0) > 0 && (m.promptTokensPerSecond ?? 0) > 0)130        #expect((m.peakMemoryBytes ?? 0) > (info.weightsMemoryBytes ?? 0) / 2)131        #expect(m.completionTokens == outcome.usage?.outputTokens)132        #expect((await engine.countTokens(modelID: MLXTestModel.id, text: "hello world") ?? 0) >= 2)133        await engine.unloadAll()134    }135136    @Test func reusesTheKVCacheAcrossTurnsOfOneSession() async throws {137        let engine = try await engine()138        let p = GenerationParameters(temperature: 0, maxOutputTokens: 40)139        let first = PromptMessage.user("Remember the code word: zebra. Reply OK.")140        let a = try await run(engine, GenerationRequest(model: MLXTestModel.ref, messages: [first], parameters: p, sessionKey: "k"))141        let m1 = try #require(engine.monitor.lastMetrics)142        #expect(!m1.reusedSession && (m1.cachedPromptTokens ?? 0) == 0)143        let turn2 = [first, .assistant(a.text), PromptMessage.user("What was the code word?")]144        _ = try await run(engine, GenerationRequest(model: MLXTestModel.ref, messages: turn2, parameters: p, sessionKey: "k"))145        let m2 = try #require(engine.monitor.lastMetrics)146        #expect(m2.reusedSession && (m2.cachedPromptTokens ?? 0) >= (m1.promptTokens ?? .max) / 2 && !m2.isColdStart, "cached \(m2.cachedPromptTokens ?? -1) of \(m2.promptTokens ?? -1), turn 1 prompt \(m1.promptTokens ?? -1)")147        // a different history under the same key must not reuse the cache148        _ = try await run(engine, GenerationRequest(model: MLXTestModel.ref, messages: [.user("Unrelated question: 2+2?")], parameters: p, sessionKey: "k"))149        #expect(engine.monitor.lastMetrics?.reusedSession == false)150        await engine.unloadAll()151    }152153    @Test func cancellationStopsPromptlyAndTheEngineKeepsWorking() async throws {154        let engine = try await engine()155        let long = GenerationRequest(model: MLXTestModel.ref, messages: [.user("Count from 1 to 500, one number per line.")],156                                     parameters: GenerationParameters(temperature: 0, maxOutputTokens: 2000), sessionKey: "c")157        var chunks = 0; var cancelledAt: Date?; var thrown: AstraError?; var completed = false158        do {159            for try await event in engine.generate(long) {160                if case .textDelta = event { chunks += 1 }161                if case .messageCompleted = event { completed = true }162                if chunks == 10, cancelledAt == nil { cancelledAt = Date(); Task { await engine.cancelActiveGeneration() } }163            }164        } catch { thrown = error as? AstraError }165        let at = try #require(cancelledAt)166        #expect(thrown == .cancelled && !completed)167        #expect(Date().timeIntervalSince(at) < 2)168        let m = try #require(engine.monitor.lastMetrics)169        #expect(m.stopReason == .cancelled && (m.timeToFirstToken ?? 0) > 0)170        #expect(await engine.snapshot().active == nil)171172        // dropping the stream also stops generation173        var seen = 0174        for try await event in engine.generate(long) { if case .textDelta = event { seen += 1; if seen == 5 { break } } }175        try await waitFor(timeout: 5) { await engine.snapshot().active == nil }176        let after = try await run(engine, GenerationRequest(model: MLXTestModel.ref, messages: [.user("Say yes.")], parameters: GenerationParameters(temperature: 0, maxOutputTokens: 16)))177        #expect(!after.text.isEmpty)178        await engine.unloadAll()179        #expect(await engine.snapshot().loaded.isEmpty)180    }181182    @Test func thinkingBlockIsReportedAsReasoningWhenTheModelSupportsIt() async throws {183        let engine = try await engine()184        guard ModelConfigSummary.read(MLXTestModel.directory).templateSupportsThinking else { await engine.unloadAll(); return }185        let on = try await run(engine, GenerationRequest(model: MLXTestModel.ref, messages: [.user("What is 17 + 26?")],186                                                         parameters: GenerationParameters(temperature: 0, maxOutputTokens: 400, reasoning: .medium)))187        #expect(!on.reasoningSummary.isEmpty)188        #expect(!on.text.contains("<think>") && !on.reasoningSummary.contains("</think>"))189        let off = try await run(engine, GenerationRequest(model: MLXTestModel.ref, messages: [.user("What is 17 + 26?")],190                                                          parameters: GenerationParameters(temperature: 0, maxOutputTokens: 100, reasoning: .off)))191        #expect(off.reasoningSummary.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty && !off.text.isEmpty && !off.text.contains("<think>"))192        await engine.unloadAll()193    }194195    @Test func toolCallIsEmittedAsTypedEventsAndTheResultIsUsed() async throws {196        let engine = try await engine()197        guard await engine.loadedInfo(MLXTestModel.id)?.templateSupportsTools == true else { await engine.unloadAll(); return }198        let tool = ToolDefinition(name: "get_weather", summary: "Get the current weather for a city.",199                                  parameters: ["type": "object", "properties": ["city": ["type": "string", "description": "City name"]], "required": ["city"]])200        let ask = PromptMessage.user("What is the weather in Paris right now? Use the tool.")201        let p = GenerationParameters(temperature: 0, maxOutputTokens: 200)202        let first = try await run(engine, GenerationRequest(model: MLXTestModel.ref, messages: [ask], parameters: p, tools: [tool], sessionKey: "t"))203        let call = try #require(first.toolCalls.first)204        #expect(call.name == "get_weather" && first.stopReason == .toolCalls)205        #expect(call.arguments["city"]?.stringValue?.lowercased().contains("paris") == true)206        let messages = [ask, .assistant(first.text, toolCalls: first.toolCalls), PromptMessage.tool(#"{"temperature_c": 31, "sky": "sunny"}"#, callID: call.id, name: call.name)]207        let second = try await run(engine, GenerationRequest(model: MLXTestModel.ref, messages: messages, parameters: p, tools: [tool], sessionKey: "t"))208        #expect(second.text.contains("31") && second.toolCalls.isEmpty)209        #expect(engine.monitor.lastMetrics?.reusedSession == true)210        await engine.unloadAll()211    }212213    @Test func stopSequenceAndRefusedShapes() async throws {214        let engine = try await engine()215        let stopped = try await run(engine, GenerationRequest(model: MLXTestModel.ref, messages: [.user("Print exactly: alpha, beta, gamma, delta")],216                                                              parameters: GenerationParameters(temperature: 0, maxOutputTokens: 60, stop: ["gamma"])))217        #expect(stopped.stopReason == .stopSequence && !stopped.text.contains("gamma") && stopped.text.lowercased().contains("alpha"), "\(stopped.stopReason) “\(stopped.text)”")218        func error(_ request: GenerationRequest) async -> AstraError? {219            do { for try await _ in engine.generate(request) {}; return nil } catch { return error as? AstraError }220        }221        if await engine.loadedInfo(MLXTestModel.id)?.hasVision == false {222            #expect(await error(GenerationRequest(model: MLXTestModel.ref, messages: [.user("x", images: [ImageInput(data: Data([0]))])])) == .unsupported("This model cannot read images."))223        }224        #expect(await error(GenerationRequest(model: MLXTestModel.ref, messages: [.assistant("x")])) != nil)225        await engine.unloadAll()226    }227}228229@Test(.enabled(if: ProjectPaths.env("ASTRA_APPLE_FM_TESTS") == "1" && AppleFoundationAvailability.current().isAvailable))230func appleSystemModelStreamsAMultiTurnAnswer() async throws {231    let provider = AppleFoundationProvider()232    let request = GenerationRequest(model: AppleFoundationProvider.modelRef, system: "Answer briefly.",233                                    messages: [.user("My favourite number is 42."), .assistant("Noted."), .user("What is my favourite number?")],234                                    parameters: GenerationParameters(maxOutputTokens: 60))235    var validator = StreamValidator(); var outcome = GenerationOutcome()236    for try await event in provider.stream(request) { for e in try validator.accept(event) { outcome.apply(e) } }237    #expect(validator.finished && outcome.stopReason == .endOfTurn)238    #expect(outcome.text.contains("42"))239    FileHandle.standardError.write(Data("[apple-fm] \(outcome.text) usage=\(String(describing: outcome.usage))\n".utf8))240}241242/// Raw throughput of the URLSession transport against a real server: `ASTRA_TRANSPORT_PROBE=<url> swift test --filter transportThroughputProbe`.243@Test(.enabled(if: ProjectPaths.env("ASTRA_TRANSPORT_PROBE") != nil)) func transportThroughputProbe() async throws {244    let url = try #require(URL(string: ProjectPaths.env("ASTRA_TRANSPORT_PROBE") ?? ""))245    let response = try await URLSessionDownloadTransport().open(URLRequest(url: url))246    #expect(response.status == 200)247    let started = Date(); var total = 0; var chunks = 0248    for try await chunk in response.body {249        total += chunk.count; chunks += 1250        response.consumed?(chunk.count)251        if Date().timeIntervalSince(started) > 8 { break }252    }253    let seconds = Date().timeIntervalSince(started)254    FileHandle.standardError.write(Data(String(format: "[probe] %.1f MB/s, %d chunks, mean chunk %d bytes\n", Double(total) / 1_048_576 / seconds, chunks, total / max(1, chunks)).utf8))255}256257/// Throughput of the whole download path (transport, partial file, progress) against the Hub: `ASTRA_COORDINATOR_PROBE=<repo>`.258@Test(.enabled(if: ProjectPaths.env("ASTRA_COORDINATOR_PROBE") != nil)) func coordinatorThroughputProbe() async throws {259    let repo = try #require(ProjectPaths.env("ASTRA_COORDINATOR_PROBE"))260    let snapshot = try await HuggingFaceResolver(transport: URLSessionTransport()).snapshot(repo: repo)261    var plan = snapshot.plan(id: "probe")262    plan.files = plan.files.filter { $0.path.hasSuffix(".safetensors") }263    let dir = TempDirectory()264    let transport = URLSessionDownloadTransport()265    let coordinator = DownloadCoordinator(store: ModelStore(root: dir.url), transport: transport)266    try await coordinator.start(plan)267    let started = Date()268    try await Task.sleep(nanoseconds: 10_000_000_000)269    let status = try #require(await coordinator.status(id: "probe"))270    let seconds = Date().timeIntervalSince(started)271    FileHandle.standardError.write(Data(String(format: "[probe] coordinator %.1f MB/s over %.0f s (reported %.1f MB/s), phase \(status.phase), suspensions \(transport.suspensions)\n",272                                               Double(status.bytesDone) / 1_048_576 / seconds, seconds, (status.bytesPerSecond ?? 0) / 1_048_576).utf8))273    await coordinator.cancel(id: "probe")274}275
Files main
Menu

Theme