Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Tests/AstraLocalTests/EngineTests.swift10// Module : AstraLocalTests11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraLocal9293/// A runtime that streams scripted words: lets the engine's policies be tested without loading weights.94final class ScriptedRuntime: LocalModelRuntime, @unchecked Sendable {95 let info: LoadedModelInfo96 let words: Int97 let delay: UInt6498 private let lock = NSLock()99 private(set) var unloaded = false100 private(set) var requests: [GenerationRequest] = []101 private(set) var concurrent = 0102 private(set) var maxConcurrent = 0103104 init(id: String, words: Int = 5, delayMs: UInt64 = 2) {105 info = LoadedModelInfo(modelID: id, directory: URL(fileURLWithPath: "/dev/null"), architecture: "scripted", contextLength: 2048, hasVision: false,106 hasChatTemplate: true, templateSupportsTools: false, weightsMemoryBytes: 1 << 20, loadTime: 0.01, loadedAt: Date())107 self.words = words; delay = delayMs * 1_000_000108 }109110 func perform(_ request: GenerationRequest, emit: @Sendable (StreamEvent) -> Void) async -> LocalGenerationResult {111 lock.withLock { requests.append(request); concurrent += 1; maxConcurrent = max(maxConcurrent, concurrent) }112 defer { lock.withLock { concurrent -= 1 } }113 var metrics = LocalGenerationMetrics(modelID: info.modelID)114 emit(.messageStarted(MessageStart(reportedModel: info.modelID)))115 var produced = 0116 for i in 0..<words {117 if Task.isCancelled { metrics.stopReason = .cancelled; metrics.completionTokens = produced; return LocalGenerationResult(metrics: metrics, error: .cancelled) }118 emit(.textDelta("w\(i) ")); produced += 1119 try? await Task.sleep(nanoseconds: delay)120 }121 metrics.stopReason = .endOfTurn; metrics.completionTokens = produced; metrics.promptTokens = 3122 emit(.usage(TokenUsage(inputTokens: 3, outputTokens: produced)))123 emit(.messageCompleted(.endOfTurn))124 return LocalGenerationResult(metrics: metrics)125 }126 func countTokens(_ text: String) async -> Int? { text.split(separator: " ").count }127 func resetSession(_ key: String?) async {}128 func unload() async { lock.withLock { unloaded = true } }129 var snapshot: (unloaded: Bool, requests: [GenerationRequest], maxConcurrent: Int) { lock.withLock { (unloaded, requests, maxConcurrent) } }130}131132final class Readings: @unchecked Sendable {133 private let lock = NSLock()134 private var _thermal = ThermalLevel.nominal135 private var _memory = MemoryReading(physicalBytes: 8 << 30, availableBytes: 5 << 30)136 private var _runtimes: [String: ScriptedRuntime] = [:]137 var words = 5138 var delayMs: UInt64 = 2139 var thermal: ThermalLevel { get { lock.withLock { _thermal } } set { lock.withLock { _thermal = newValue } } }140 var memory: MemoryReading { get { lock.withLock { _memory } } set { lock.withLock { _memory = newValue } } }141 func runtime(_ id: String) -> ScriptedRuntime? { lock.withLock { _runtimes[id] } }142 func make(_ id: String) -> ScriptedRuntime { let r = ScriptedRuntime(id: id, words: words, delayMs: delayMs); lock.withLock { _runtimes[id] = r }; return r }143}144145@Suite struct LocalEngineTests {146 let dir = TempDirectory()147 let readings = Readings()148149 /// An engine over a store holding installed models "a" and "b" (and optionally a big one), with scripted runtimes.150 private func makeEngine(maxLoaded: Int = 1, bigWeights: Int? = nil) async throws -> LocalEngine {151 let server = FakeServer()152 let (coordinator, store) = Fixture.coordinator(dir, server)153 var ids: [(String, Int)] = [("a", 300_000), ("b", 300_000)]154 if let bigWeights { ids.append(("big", bigWeights)) }155 for (id, size) in ids {156 var files = Fixture.contents; files[2].1 = bytes(size, seed: UInt64(size))157 let base = "https://mirror.test/m/\(id)/"158 Fixture.publish(server, base: base, files: files)159 var card = Fixture.card(id: id, files: files); card.baseURL = base160 try await coordinator.start(DownloadPlan(card: card))161 #expect(await coordinator.waitUntilSettled(id: id)?.phase == .completed)162 }163 var environment = LocalEngine.Environment()164 let readings = readings165 environment.memory = { readings.memory }166 environment.thermal = { readings.thermal }167 environment.mlxMemory = { nil }168 environment.loader = { id, _, progress in progress(0.5); progress(1); return readings.make(id) }169 return LocalEngine(store: store, configuration: .init(maxLoadedModels: maxLoaded), memoryGovernor: MemoryGovernor(platform: .iOS), environment: environment)170 }171172 private func request(_ id: String, maxTokens: Int? = nil, context: Int? = nil) -> GenerationRequest {173 GenerationRequest(model: ModelRef(provider: .mlx, modelID: id), messages: [.user("hi")],174 parameters: GenerationParameters(maxOutputTokens: maxTokens, contextLength: context), sessionKey: "s")175 }176177 private func collect(_ stream: AsyncThrowingStream<StreamEvent, Error>) async -> (events: [StreamEvent], error: AstraError?) {178 var events: [StreamEvent] = []179 do { for try await e in stream { events.append(e) }; return (events, nil) } catch { return (events, error as? AstraError) }180 }181182 @Test func loadsOneModelAtATimeAndStreams() async throws {183 let engine = try await makeEngine()184 let info = try await engine.load("a")185 #expect(info.modelID == "a")186 #expect(await engine.snapshot().loaded.map(\.modelID) == ["a"])187 let (events, error) = await collect(engine.generate(request("a")))188 #expect(error == nil)189 #expect(events.first == .messageStarted(MessageStart(reportedModel: "a")))190 #expect(events.filter { if case .messageCompleted = $0 { true } else { false } }.count == 1)191 #expect(events.last == .messageCompleted(.endOfTurn))192 #expect(engine.monitor.lastMetrics?.completionTokens == 5)193194 try await engine.load("b") // limit is 1: "a" is unloaded first195 #expect(await engine.snapshot().loaded.map(\.modelID) == ["b"])196 #expect(readings.runtime("a")?.snapshot.unloaded == true)197 #expect(await engine.countTokens(modelID: "b", text: "one two three") == 3)198 #expect(await engine.countTokens(modelID: "a", text: "one") == nil)199 }200201 @Test func configurableLimitKeepsTwoModels() async throws {202 let engine = try await makeEngine(maxLoaded: 2)203 try await engine.load("a"); try await engine.load("b")204 #expect(await engine.snapshot().loaded.count == 2)205 await engine.setMaxLoadedModels(1)206 await engine.unload("a")207 #expect(await engine.snapshot().loaded.map(\.modelID) == ["b"])208 }209210 @Test func autoLoadsOnFirstUseAndRefusesUnknownModels() async throws {211 let engine = try await makeEngine()212 #expect(await collect(engine.generate(request("a"))).error == nil)213 #expect(await engine.snapshot().isLoaded("a"))214 let missing = await collect(engine.generate(request("nope")))215 #expect(missing.error == .local("“nope” is not downloaded.") && missing.events.isEmpty)216 }217218 @Test func generationsAreSerialized() async throws {219 readings.words = 20220 let engine = try await makeEngine()221 try await engine.load("a")222 async let one = collect(engine.generate(request("a")))223 async let two = collect(engine.generate(request("a")))224 async let three = collect(engine.generate(request("a")))225 let results = await [one, two, three]226 #expect(results.allSatisfy { $0.error == nil && $0.events.last == .messageCompleted(.endOfTurn) })227 #expect(readings.runtime("a")?.snapshot.maxConcurrent == 1)228 #expect(engine.monitor.history.count == 3)229 }230231 @Test func cancellingTheConsumerStopsPromptlyWithCancelled() async throws {232 readings.words = 5_000; readings.delayMs = 2233 let engine = try await makeEngine()234 try await engine.load("a")235 let started = Date()236 var seen = 0237 // leaving the loop releases the stream, which cancels the generation238 for try await event in engine.generate(request("a")) { if case .textDelta = event { seen += 1; if seen == 5 { break } } }239 try await waitFor { await engine.snapshot().active == nil }240 #expect(Date().timeIntervalSince(started) < 3)241 let metrics = try #require(engine.monitor.lastMetrics)242 #expect(metrics.stopReason == .cancelled && (metrics.completionTokens ?? 0) < 100)243244 // explicit cancellation: the stream ends by throwing .cancelled, with no .messageCompleted245 let task = Task { await collect(engine.generate(request("a"))) }246 try await waitFor { await engine.snapshot().active?.streamedChunks ?? 0 > 0 }247 await engine.cancelActiveGeneration()248 let result = await task.value249 #expect(result.error == .cancelled)250 #expect(!result.events.contains { $0.isTerminal })251 #expect(await engine.snapshot().active == nil)252 // the engine is usable again253 readings.words = 3254 }255256 @Test func snapshotStreamShowsLoadingAndActiveGeneration() async throws {257 readings.words = 200258 let engine = try await makeEngine()259 let seen = SnapshotLog()260 let watcher = Task { for await s in await engine.snapshots() { seen.add(s) } }261 _ = await collect(engine.generate(request("a")))262 try await waitFor { seen.all.last?.active == nil && seen.all.last?.isLoaded("a") == true }263 watcher.cancel()264 let all = seen.all265 #expect(all.first?.loaded.isEmpty == true)266 #expect(all.contains { $0.loading?.modelID == "a" })267 #expect(all.contains { $0.active?.modelID == "a" && $0.active?.sessionKey == "s" })268 #expect(all.contains { ($0.active?.streamedChunks ?? 0) > 0 })269 #expect(all.last?.memory.physicalBytes == 8 << 30 && all.last?.thermal == .nominal)270 }271272 @Test func memoryVerdictBlocksAModelThatDoesNotFitUnlessForced() async throws {273 let engine = try await makeEngine(bigWeights: 3_000_000)274 readings.memory = MemoryReading(physicalBytes: 300 << 20, availableBytes: 200 << 20) // a pretend 300 MB device275 await #expect(throws: AstraError.self) { try await engine.load("big") }276 #expect(await engine.snapshot().loaded.isEmpty)277 do { try await engine.load("big"); Issue.record("should not load") }278 catch let e as AstraError { #expect(e.message.contains("estimate, not a guarantee")) }279 try await engine.load("big", force: true)280 #expect(await engine.snapshot().isLoaded("big"))281 let card = try #require(engine.store.card(id: "big"))282 #expect(!engine.verdict(for: card).mayLoad)283 }284285 @Test func memoryWarningUnloadsIdleModelsAndCriticalStopsGeneration() async throws {286 readings.words = 5_000287 let engine = try await makeEngine(maxLoaded: 2)288 try await engine.load("a"); try await engine.load("b")289 let task = Task { await collect(engine.generate(request("a"))) }290 try await waitFor { await engine.snapshot().active != nil }291 await engine.handleMemoryPressure(critical: false)292 var snapshot = await engine.snapshot()293 #expect(snapshot.loaded.map(\.modelID) == ["a"]) // the generating model stays294 #expect(snapshot.lastAutomaticUnload == "memory warning" && snapshot.active != nil)295 await engine.handleMemoryPressure(critical: true)296 snapshot = await engine.snapshot()297 #expect(snapshot.loaded.isEmpty && snapshot.active == nil && snapshot.lastAutomaticUnload == "critical memory pressure")298 #expect(await task.value.error == .cancelled)299 #expect(readings.runtime("a")?.snapshot.unloaded == true)300 }301302 @Test func seriousHeatReducesLimitsAndCriticalHeatRefusesAndUnloads() async throws {303 let engine = try await makeEngine()304 try await engine.load("a")305 readings.thermal = .serious306 #expect(await collect(engine.generate(request("a", maxTokens: 800, context: 8192))).error == nil)307 let sent = try #require(readings.runtime("a")?.snapshot.requests.last)308 #expect(sent.parameters.maxOutputTokens == 400 && sent.parameters.contextLength == 4096)309 #expect(await engine.snapshot().warning?.contains("hot") == true)310311 readings.thermal = .critical312 let refused = await collect(engine.generate(request("a")))313 #expect(refused.events.isEmpty)314 guard case .local(let message)? = refused.error else { Issue.record("expected a local error"); return }315 #expect(message.contains("too hot"))316 #expect(await engine.snapshot().loaded.isEmpty)317 #expect(await engine.snapshot().lastAutomaticUnload == "device too hot")318 await #expect(throws: AstraError.self) { try await engine.load("a") }319320 readings.thermal = .nominal321 #expect(await collect(engine.generate(request("a", maxTokens: 800))).error == nil)322 #expect(readings.runtime("a")?.snapshot.requests.last?.parameters.maxOutputTokens == 800)323 }324325 @Test func criticalHeatDuringGenerationStopsIt() async throws {326 readings.words = 5_000327 let engine = try await makeEngine()328 let task = Task { await collect(engine.generate(request("a"))) }329 try await waitFor { await engine.snapshot().active != nil }330 readings.thermal = .critical331 await engine.handleThermal(.critical)332 #expect(await task.value.error == .cancelled)333 #expect(await engine.snapshot().loaded.isEmpty)334 }335336 @Test func mlxProviderListsInstalledModelsAndRoutesToTheEngine() async throws {337 let engine = try await makeEngine()338 let provider = MLXProvider(engine: engine)339 #expect(provider.kind == .mlx)340 let models = try await provider.listModels()341 #expect(models.map(\.ref) == [ModelRef(provider: .mlx, modelID: "a"), ModelRef(provider: .mlx, modelID: "b")])342 #expect(models.allSatisfy { $0.source == .runtime && $0.pricing == nil && $0.contextLength == 2048 })343 let status = try await provider.verify()344 #expect(status.ok == MLXAvailability.isAvailable && status.modelCount == (MLXAvailability.isAvailable ? 2 : 0))345 let wrong = await collect(provider.stream(GenerationRequest(model: ModelRef(provider: .openAI, modelID: "a"), messages: [.user("x")])))346 #expect(wrong.error != nil && wrong.events.isEmpty)347 if MLXAvailability.isAvailable {348 let ok = await collect(provider.stream(request("a")))349 #expect(ok.error == nil && ok.events.last == .messageCompleted(.endOfTurn))350 }351 }352353 @Test func benchmarkRecordsColdAndWarmSamplesFromMeasuredMetrics() async throws {354 let engine = try await makeEngine()355 let result = try await LocalBenchmark(prompts: Array(LocalBenchmark.standardPrompts.prefix(2))).run(modelID: "a", engine: engine, revision: Fixture.revision)356 #expect(result.samples.count == 2 && result.modelRevision == Fixture.revision)357 #expect(result.samples.allSatisfy { $0.outputTokens == 5 && $0.outputPreview.hasPrefix("w0 w1") })358 #expect(result.device == DeviceFacts.current())359 let decoded = try JSONDecoder().decode(LocalBenchmarkResult.self, from: JSONEncoder().encode(result))360 #expect(decoded == result)361 // the scripted runtime measures no speed: the result must not invent one362 #expect(result.warmDecodeTokensPerSecond == nil && result.samples.allSatisfy { $0.decodeTokensPerSecond == nil })363 }364}365366final class SnapshotLog: @unchecked Sendable {367 private let lock = NSLock(); private var items: [LocalEngine.Snapshot] = []368 func add(_ s: LocalEngine.Snapshot) { lock.withLock { items.append(s) } }369 var all: [LocalEngine.Snapshot] { lock.withLock { items } }370}371372@Suite struct AppleFoundationProviderTests {373 private func collect(_ stream: AsyncThrowingStream<StreamEvent, Error>) async -> AstraError? {374 do { for try await _ in stream {}; return nil } catch { return error as? AstraError }375 }376377 @Test func unavailableReasonsArePreciseAndNothingIsListed() async throws {378 let cases: [(AppleFoundationAvailability, String)] = [379 (.deviceNotEligible, "does not support"), (.appleIntelligenceOff, "turned off"), (.modelNotReady, "not ready"),380 (.osTooOld, "iOS 26"), (.notInThisBuild, "platform"),381 ]382 for (state, fragment) in cases {383 let provider = AppleFoundationProvider(availability: { state })384 #expect(try await provider.listModels().isEmpty)385 let status = try await provider.verify()386 #expect(!status.ok && status.detail.contains(fragment) && status.modelCount == 0)387 let request = GenerationRequest(model: AppleFoundationProvider.modelRef, messages: [.user("hi")])388 #expect(await collect(provider.stream(request)) == .unsupported(state.reason!))389 }390 #expect(AppleFoundationAvailability.available.reason == nil)391 #expect(Set(cases.map(\.0.reason)).count == cases.count)392 }393394 @Test func listsTheSingleSystemModelOnlyWhenAvailable() async throws {395 let provider = AppleFoundationProvider(availability: { .available })396 let models = try await provider.listModels()397 #expect(models.count == 1 && models[0].ref == AppleFoundationProvider.modelRef && models[0].source == .runtime && models[0].pricing == nil)398 #expect(models[0].summary?.lowercased().contains("download") == false)399 #expect(try await provider.verify().ok)400 // unsupported request shapes are refused before the framework is touched401 let tool = ToolDefinition(name: "t", summary: "s", parameters: ["type": "object"])402 #expect(await collect(provider.stream(GenerationRequest(model: AppleFoundationProvider.modelRef, messages: [.user("x")], tools: [tool]))) != nil)403 let image = GenerationRequest(model: AppleFoundationProvider.modelRef, messages: [.user("x", images: [ImageInput(data: Data([1]))])])404 #expect(await collect(provider.stream(image)) != nil)405 #expect(await collect(provider.stream(GenerationRequest(model: AppleFoundationProvider.modelRef, messages: [.assistant("x")]))) != nil)406 }407408 @Test func generationFailuresMapToPlainMessages() {409 #expect(AppleGenerationFailure.contextWindowExceeded.astraError(simulator: false).title == "Conversation too long")410 for filtered in [AppleGenerationFailure.guardrailViolation, .refusal] {411 guard case .contentFiltered(let m?) = filtered.astraError(simulator: false) else { Issue.record("\(filtered) should be contentFiltered"); continue }412 #expect(m.contains("Apple Intelligence"))413 }414 #expect(AppleGenerationFailure.cancelled.astraError() == .cancelled)415 let locals: [AppleGenerationFailure] = [.assetsUnavailable, .unsupportedLanguage, .unsupportedGuide, .decodingFailure, .rateLimited, .concurrentRequests,416 .unsupportedCapability, .unsupportedTranscriptContent, .timeout, .unknown]417 for failure in locals {418 guard case .local = failure.astraError(simulator: false) else { Issue.record("\(failure) should be .local"); continue }419 }420 // every case has its own sentence, and none leaks a type name, an error code or a system-localized string421 let messages = AppleGenerationFailure.allCases.map { $0.astraError(simulator: false).message }422 #expect(Set(messages).count == messages.count)423 for m in messages + [AppleGenerationFailure.unknown.astraError(simulator: true).message] {424 #expect(!m.contains("GenerationError") && !m.contains("FoundationModels") && !m.contains("-1") && !m.contains("opération") && m.hasSuffix("."))425 }426 #expect(AppleGenerationFailure.unknown.astraError(simulator: false) == .local("Apple Intelligence could not answer this request."))427 let sim = AppleGenerationFailure.unknown.astraError(simulator: true).message428 #expect(sim.hasPrefix("Apple Intelligence could not answer this request.") && sim.contains("Simulator") && sim.contains("physical device"))429 // only the unrecognised case carries the Simulator note430 #expect(AppleGenerationFailure.rateLimited.astraError(simulator: true) == AppleGenerationFailure.rateLimited.astraError(simulator: false))431 #if canImport(FoundationModels)432 if #available(iOS 26.0, macOS 26.0, *) {433 struct Other: Error {}434 #expect(AppleFoundationProvider.classify(Other()) == .unknown)435 #expect(AppleFoundationProvider.classify(CancellationError()) == .cancelled)436 #expect(AppleFoundationProvider.map(NSError(domain: "FoundationModels.LanguageModelSession.GenerationError", code: -1)) == AppleGenerationFailure.unknown.astraError())437 }438 #endif439 }440441 @Test func liveAvailabilityIsOneOfTheDocumentedStates() {442 let live = AppleFoundationAvailability.current()443 #expect(live.isAvailable == (live.reason == nil))444 }445}446