SPB Git

Repository Public MIT

EngineTests.swift 24.2 KB 446 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraLocalTests/EngineTests.swift10// Module           : AstraLocalTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraLocal9293/// A runtime that streams scripted words: lets the engine's policies be tested without loading weights.94final class ScriptedRuntime: LocalModelRuntime, @unchecked Sendable {95    let info: LoadedModelInfo96    let words: Int97    let delay: UInt6498    private let lock = NSLock()99    private(set) var unloaded = false100    private(set) var requests: [GenerationRequest] = []101    private(set) var concurrent = 0102    private(set) var maxConcurrent = 0103104    init(id: String, words: Int = 5, delayMs: UInt64 = 2) {105        info = LoadedModelInfo(modelID: id, directory: URL(fileURLWithPath: "/dev/null"), architecture: "scripted", contextLength: 2048, hasVision: false,106                               hasChatTemplate: true, templateSupportsTools: false, weightsMemoryBytes: 1 << 20, loadTime: 0.01, loadedAt: Date())107        self.words = words; delay = delayMs * 1_000_000108    }109110    func perform(_ request: GenerationRequest, emit: @Sendable (StreamEvent) -> Void) async -> LocalGenerationResult {111        lock.withLock { requests.append(request); concurrent += 1; maxConcurrent = max(maxConcurrent, concurrent) }112        defer { lock.withLock { concurrent -= 1 } }113        var metrics = LocalGenerationMetrics(modelID: info.modelID)114        emit(.messageStarted(MessageStart(reportedModel: info.modelID)))115        var produced = 0116        for i in 0..<words {117            if Task.isCancelled { metrics.stopReason = .cancelled; metrics.completionTokens = produced; return LocalGenerationResult(metrics: metrics, error: .cancelled) }118            emit(.textDelta("w\(i) ")); produced += 1119            try? await Task.sleep(nanoseconds: delay)120        }121        metrics.stopReason = .endOfTurn; metrics.completionTokens = produced; metrics.promptTokens = 3122        emit(.usage(TokenUsage(inputTokens: 3, outputTokens: produced)))123        emit(.messageCompleted(.endOfTurn))124        return LocalGenerationResult(metrics: metrics)125    }126    func countTokens(_ text: String) async -> Int? { text.split(separator: " ").count }127    func resetSession(_ key: String?) async {}128    func unload() async { lock.withLock { unloaded = true } }129    var snapshot: (unloaded: Bool, requests: [GenerationRequest], maxConcurrent: Int) { lock.withLock { (unloaded, requests, maxConcurrent) } }130}131132final class Readings: @unchecked Sendable {133    private let lock = NSLock()134    private var _thermal = ThermalLevel.nominal135    private var _memory = MemoryReading(physicalBytes: 8 << 30, availableBytes: 5 << 30)136    private var _runtimes: [String: ScriptedRuntime] = [:]137    var words = 5138    var delayMs: UInt64 = 2139    var thermal: ThermalLevel { get { lock.withLock { _thermal } } set { lock.withLock { _thermal = newValue } } }140    var memory: MemoryReading { get { lock.withLock { _memory } } set { lock.withLock { _memory = newValue } } }141    func runtime(_ id: String) -> ScriptedRuntime? { lock.withLock { _runtimes[id] } }142    func make(_ id: String) -> ScriptedRuntime { let r = ScriptedRuntime(id: id, words: words, delayMs: delayMs); lock.withLock { _runtimes[id] = r }; return r }143}144145@Suite struct LocalEngineTests {146    let dir = TempDirectory()147    let readings = Readings()148149    /// An engine over a store holding installed models "a" and "b" (and optionally a big one), with scripted runtimes.150    private func makeEngine(maxLoaded: Int = 1, bigWeights: Int? = nil) async throws -> LocalEngine {151        let server = FakeServer()152        let (coordinator, store) = Fixture.coordinator(dir, server)153        var ids: [(String, Int)] = [("a", 300_000), ("b", 300_000)]154        if let bigWeights { ids.append(("big", bigWeights)) }155        for (id, size) in ids {156            var files = Fixture.contents; files[2].1 = bytes(size, seed: UInt64(size))157            let base = "https://mirror.test/m/\(id)/"158            Fixture.publish(server, base: base, files: files)159            var card = Fixture.card(id: id, files: files); card.baseURL = base160            try await coordinator.start(DownloadPlan(card: card))161            #expect(await coordinator.waitUntilSettled(id: id)?.phase == .completed)162        }163        var environment = LocalEngine.Environment()164        let readings = readings165        environment.memory = { readings.memory }166        environment.thermal = { readings.thermal }167        environment.mlxMemory = { nil }168        environment.loader = { id, _, progress in progress(0.5); progress(1); return readings.make(id) }169        return LocalEngine(store: store, configuration: .init(maxLoadedModels: maxLoaded), memoryGovernor: MemoryGovernor(platform: .iOS), environment: environment)170    }171172    private func request(_ id: String, maxTokens: Int? = nil, context: Int? = nil) -> GenerationRequest {173        GenerationRequest(model: ModelRef(provider: .mlx, modelID: id), messages: [.user("hi")],174                          parameters: GenerationParameters(maxOutputTokens: maxTokens, contextLength: context), sessionKey: "s")175    }176177    private func collect(_ stream: AsyncThrowingStream<StreamEvent, Error>) async -> (events: [StreamEvent], error: AstraError?) {178        var events: [StreamEvent] = []179        do { for try await e in stream { events.append(e) }; return (events, nil) } catch { return (events, error as? AstraError) }180    }181182    @Test func loadsOneModelAtATimeAndStreams() async throws {183        let engine = try await makeEngine()184        let info = try await engine.load("a")185        #expect(info.modelID == "a")186        #expect(await engine.snapshot().loaded.map(\.modelID) == ["a"])187        let (events, error) = await collect(engine.generate(request("a")))188        #expect(error == nil)189        #expect(events.first == .messageStarted(MessageStart(reportedModel: "a")))190        #expect(events.filter { if case .messageCompleted = $0 { true } else { false } }.count == 1)191        #expect(events.last == .messageCompleted(.endOfTurn))192        #expect(engine.monitor.lastMetrics?.completionTokens == 5)193194        try await engine.load("b")   // limit is 1: "a" is unloaded first195        #expect(await engine.snapshot().loaded.map(\.modelID) == ["b"])196        #expect(readings.runtime("a")?.snapshot.unloaded == true)197        #expect(await engine.countTokens(modelID: "b", text: "one two three") == 3)198        #expect(await engine.countTokens(modelID: "a", text: "one") == nil)199    }200201    @Test func configurableLimitKeepsTwoModels() async throws {202        let engine = try await makeEngine(maxLoaded: 2)203        try await engine.load("a"); try await engine.load("b")204        #expect(await engine.snapshot().loaded.count == 2)205        await engine.setMaxLoadedModels(1)206        await engine.unload("a")207        #expect(await engine.snapshot().loaded.map(\.modelID) == ["b"])208    }209210    @Test func autoLoadsOnFirstUseAndRefusesUnknownModels() async throws {211        let engine = try await makeEngine()212        #expect(await collect(engine.generate(request("a"))).error == nil)213        #expect(await engine.snapshot().isLoaded("a"))214        let missing = await collect(engine.generate(request("nope")))215        #expect(missing.error == .local("“nope” is not downloaded.") && missing.events.isEmpty)216    }217218    @Test func generationsAreSerialized() async throws {219        readings.words = 20220        let engine = try await makeEngine()221        try await engine.load("a")222        async let one = collect(engine.generate(request("a")))223        async let two = collect(engine.generate(request("a")))224        async let three = collect(engine.generate(request("a")))225        let results = await [one, two, three]226        #expect(results.allSatisfy { $0.error == nil && $0.events.last == .messageCompleted(.endOfTurn) })227        #expect(readings.runtime("a")?.snapshot.maxConcurrent == 1)228        #expect(engine.monitor.history.count == 3)229    }230231    @Test func cancellingTheConsumerStopsPromptlyWithCancelled() async throws {232        readings.words = 5_000; readings.delayMs = 2233        let engine = try await makeEngine()234        try await engine.load("a")235        let started = Date()236        var seen = 0237        // leaving the loop releases the stream, which cancels the generation238        for try await event in engine.generate(request("a")) { if case .textDelta = event { seen += 1; if seen == 5 { break } } }239        try await waitFor { await engine.snapshot().active == nil }240        #expect(Date().timeIntervalSince(started) < 3)241        let metrics = try #require(engine.monitor.lastMetrics)242        #expect(metrics.stopReason == .cancelled && (metrics.completionTokens ?? 0) < 100)243244        // explicit cancellation: the stream ends by throwing .cancelled, with no .messageCompleted245        let task = Task { await collect(engine.generate(request("a"))) }246        try await waitFor { await engine.snapshot().active?.streamedChunks ?? 0 > 0 }247        await engine.cancelActiveGeneration()248        let result = await task.value249        #expect(result.error == .cancelled)250        #expect(!result.events.contains { $0.isTerminal })251        #expect(await engine.snapshot().active == nil)252        // the engine is usable again253        readings.words = 3254    }255256    @Test func snapshotStreamShowsLoadingAndActiveGeneration() async throws {257        readings.words = 200258        let engine = try await makeEngine()259        let seen = SnapshotLog()260        let watcher = Task { for await s in await engine.snapshots() { seen.add(s) } }261        _ = await collect(engine.generate(request("a")))262        try await waitFor { seen.all.last?.active == nil && seen.all.last?.isLoaded("a") == true }263        watcher.cancel()264        let all = seen.all265        #expect(all.first?.loaded.isEmpty == true)266        #expect(all.contains { $0.loading?.modelID == "a" })267        #expect(all.contains { $0.active?.modelID == "a" && $0.active?.sessionKey == "s" })268        #expect(all.contains { ($0.active?.streamedChunks ?? 0) > 0 })269        #expect(all.last?.memory.physicalBytes == 8 << 30 && all.last?.thermal == .nominal)270    }271272    @Test func memoryVerdictBlocksAModelThatDoesNotFitUnlessForced() async throws {273        let engine = try await makeEngine(bigWeights: 3_000_000)274        readings.memory = MemoryReading(physicalBytes: 300 << 20, availableBytes: 200 << 20)   // a pretend 300 MB device275        await #expect(throws: AstraError.self) { try await engine.load("big") }276        #expect(await engine.snapshot().loaded.isEmpty)277        do { try await engine.load("big"); Issue.record("should not load") }278        catch let e as AstraError { #expect(e.message.contains("estimate, not a guarantee")) }279        try await engine.load("big", force: true)280        #expect(await engine.snapshot().isLoaded("big"))281        let card = try #require(engine.store.card(id: "big"))282        #expect(!engine.verdict(for: card).mayLoad)283    }284285    @Test func memoryWarningUnloadsIdleModelsAndCriticalStopsGeneration() async throws {286        readings.words = 5_000287        let engine = try await makeEngine(maxLoaded: 2)288        try await engine.load("a"); try await engine.load("b")289        let task = Task { await collect(engine.generate(request("a"))) }290        try await waitFor { await engine.snapshot().active != nil }291        await engine.handleMemoryPressure(critical: false)292        var snapshot = await engine.snapshot()293        #expect(snapshot.loaded.map(\.modelID) == ["a"])            // the generating model stays294        #expect(snapshot.lastAutomaticUnload == "memory warning" && snapshot.active != nil)295        await engine.handleMemoryPressure(critical: true)296        snapshot = await engine.snapshot()297        #expect(snapshot.loaded.isEmpty && snapshot.active == nil && snapshot.lastAutomaticUnload == "critical memory pressure")298        #expect(await task.value.error == .cancelled)299        #expect(readings.runtime("a")?.snapshot.unloaded == true)300    }301302    @Test func seriousHeatReducesLimitsAndCriticalHeatRefusesAndUnloads() async throws {303        let engine = try await makeEngine()304        try await engine.load("a")305        readings.thermal = .serious306        #expect(await collect(engine.generate(request("a", maxTokens: 800, context: 8192))).error == nil)307        let sent = try #require(readings.runtime("a")?.snapshot.requests.last)308        #expect(sent.parameters.maxOutputTokens == 400 && sent.parameters.contextLength == 4096)309        #expect(await engine.snapshot().warning?.contains("hot") == true)310311        readings.thermal = .critical312        let refused = await collect(engine.generate(request("a")))313        #expect(refused.events.isEmpty)314        guard case .local(let message)? = refused.error else { Issue.record("expected a local error"); return }315        #expect(message.contains("too hot"))316        #expect(await engine.snapshot().loaded.isEmpty)317        #expect(await engine.snapshot().lastAutomaticUnload == "device too hot")318        await #expect(throws: AstraError.self) { try await engine.load("a") }319320        readings.thermal = .nominal321        #expect(await collect(engine.generate(request("a", maxTokens: 800))).error == nil)322        #expect(readings.runtime("a")?.snapshot.requests.last?.parameters.maxOutputTokens == 800)323    }324325    @Test func criticalHeatDuringGenerationStopsIt() async throws {326        readings.words = 5_000327        let engine = try await makeEngine()328        let task = Task { await collect(engine.generate(request("a"))) }329        try await waitFor { await engine.snapshot().active != nil }330        readings.thermal = .critical331        await engine.handleThermal(.critical)332        #expect(await task.value.error == .cancelled)333        #expect(await engine.snapshot().loaded.isEmpty)334    }335336    @Test func mlxProviderListsInstalledModelsAndRoutesToTheEngine() async throws {337        let engine = try await makeEngine()338        let provider = MLXProvider(engine: engine)339        #expect(provider.kind == .mlx)340        let models = try await provider.listModels()341        #expect(models.map(\.ref) == [ModelRef(provider: .mlx, modelID: "a"), ModelRef(provider: .mlx, modelID: "b")])342        #expect(models.allSatisfy { $0.source == .runtime && $0.pricing == nil && $0.contextLength == 2048 })343        let status = try await provider.verify()344        #expect(status.ok == MLXAvailability.isAvailable && status.modelCount == (MLXAvailability.isAvailable ? 2 : 0))345        let wrong = await collect(provider.stream(GenerationRequest(model: ModelRef(provider: .openAI, modelID: "a"), messages: [.user("x")])))346        #expect(wrong.error != nil && wrong.events.isEmpty)347        if MLXAvailability.isAvailable {348            let ok = await collect(provider.stream(request("a")))349            #expect(ok.error == nil && ok.events.last == .messageCompleted(.endOfTurn))350        }351    }352353    @Test func benchmarkRecordsColdAndWarmSamplesFromMeasuredMetrics() async throws {354        let engine = try await makeEngine()355        let result = try await LocalBenchmark(prompts: Array(LocalBenchmark.standardPrompts.prefix(2))).run(modelID: "a", engine: engine, revision: Fixture.revision)356        #expect(result.samples.count == 2 && result.modelRevision == Fixture.revision)357        #expect(result.samples.allSatisfy { $0.outputTokens == 5 && $0.outputPreview.hasPrefix("w0 w1") })358        #expect(result.device == DeviceFacts.current())359        let decoded = try JSONDecoder().decode(LocalBenchmarkResult.self, from: JSONEncoder().encode(result))360        #expect(decoded == result)361        // the scripted runtime measures no speed: the result must not invent one362        #expect(result.warmDecodeTokensPerSecond == nil && result.samples.allSatisfy { $0.decodeTokensPerSecond == nil })363    }364}365366final class SnapshotLog: @unchecked Sendable {367    private let lock = NSLock(); private var items: [LocalEngine.Snapshot] = []368    func add(_ s: LocalEngine.Snapshot) { lock.withLock { items.append(s) } }369    var all: [LocalEngine.Snapshot] { lock.withLock { items } }370}371372@Suite struct AppleFoundationProviderTests {373    private func collect(_ stream: AsyncThrowingStream<StreamEvent, Error>) async -> AstraError? {374        do { for try await _ in stream {}; return nil } catch { return error as? AstraError }375    }376377    @Test func unavailableReasonsArePreciseAndNothingIsListed() async throws {378        let cases: [(AppleFoundationAvailability, String)] = [379            (.deviceNotEligible, "does not support"), (.appleIntelligenceOff, "turned off"), (.modelNotReady, "not ready"),380            (.osTooOld, "iOS 26"), (.notInThisBuild, "platform"),381        ]382        for (state, fragment) in cases {383            let provider = AppleFoundationProvider(availability: { state })384            #expect(try await provider.listModels().isEmpty)385            let status = try await provider.verify()386            #expect(!status.ok && status.detail.contains(fragment) && status.modelCount == 0)387            let request = GenerationRequest(model: AppleFoundationProvider.modelRef, messages: [.user("hi")])388            #expect(await collect(provider.stream(request)) == .unsupported(state.reason!))389        }390        #expect(AppleFoundationAvailability.available.reason == nil)391        #expect(Set(cases.map(\.0.reason)).count == cases.count)392    }393394    @Test func listsTheSingleSystemModelOnlyWhenAvailable() async throws {395        let provider = AppleFoundationProvider(availability: { .available })396        let models = try await provider.listModels()397        #expect(models.count == 1 && models[0].ref == AppleFoundationProvider.modelRef && models[0].source == .runtime && models[0].pricing == nil)398        #expect(models[0].summary?.lowercased().contains("download") == false)399        #expect(try await provider.verify().ok)400        // unsupported request shapes are refused before the framework is touched401        let tool = ToolDefinition(name: "t", summary: "s", parameters: ["type": "object"])402        #expect(await collect(provider.stream(GenerationRequest(model: AppleFoundationProvider.modelRef, messages: [.user("x")], tools: [tool]))) != nil)403        let image = GenerationRequest(model: AppleFoundationProvider.modelRef, messages: [.user("x", images: [ImageInput(data: Data([1]))])])404        #expect(await collect(provider.stream(image)) != nil)405        #expect(await collect(provider.stream(GenerationRequest(model: AppleFoundationProvider.modelRef, messages: [.assistant("x")]))) != nil)406    }407408    @Test func generationFailuresMapToPlainMessages() {409        #expect(AppleGenerationFailure.contextWindowExceeded.astraError(simulator: false).title == "Conversation too long")410        for filtered in [AppleGenerationFailure.guardrailViolation, .refusal] {411            guard case .contentFiltered(let m?) = filtered.astraError(simulator: false) else { Issue.record("\(filtered) should be contentFiltered"); continue }412            #expect(m.contains("Apple Intelligence"))413        }414        #expect(AppleGenerationFailure.cancelled.astraError() == .cancelled)415        let locals: [AppleGenerationFailure] = [.assetsUnavailable, .unsupportedLanguage, .unsupportedGuide, .decodingFailure, .rateLimited, .concurrentRequests,416                                                .unsupportedCapability, .unsupportedTranscriptContent, .timeout, .unknown]417        for failure in locals {418            guard case .local = failure.astraError(simulator: false) else { Issue.record("\(failure) should be .local"); continue }419        }420        // every case has its own sentence, and none leaks a type name, an error code or a system-localized string421        let messages = AppleGenerationFailure.allCases.map { $0.astraError(simulator: false).message }422        #expect(Set(messages).count == messages.count)423        for m in messages + [AppleGenerationFailure.unknown.astraError(simulator: true).message] {424            #expect(!m.contains("GenerationError") && !m.contains("FoundationModels") && !m.contains("-1") && !m.contains("opération") && m.hasSuffix("."))425        }426        #expect(AppleGenerationFailure.unknown.astraError(simulator: false) == .local("Apple Intelligence could not answer this request."))427        let sim = AppleGenerationFailure.unknown.astraError(simulator: true).message428        #expect(sim.hasPrefix("Apple Intelligence could not answer this request.") && sim.contains("Simulator") && sim.contains("physical device"))429        // only the unrecognised case carries the Simulator note430        #expect(AppleGenerationFailure.rateLimited.astraError(simulator: true) == AppleGenerationFailure.rateLimited.astraError(simulator: false))431        #if canImport(FoundationModels)432        if #available(iOS 26.0, macOS 26.0, *) {433            struct Other: Error {}434            #expect(AppleFoundationProvider.classify(Other()) == .unknown)435            #expect(AppleFoundationProvider.classify(CancellationError()) == .cancelled)436            #expect(AppleFoundationProvider.map(NSError(domain: "FoundationModels.LanguageModelSession.GenerationError", code: -1)) == AppleGenerationFailure.unknown.astraError())437        }438        #endif439    }440441    @Test func liveAvailabilityIsOneOfTheDocumentedStates() {442        let live = AppleFoundationAvailability.current()443        #expect(live.isAvailable == (live.reason == nil))444    }445}446
Files main
Menu

Theme