Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Tests/AstraLocalTests/ModelVerification.swift10// Module : AstraLocalTests11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2214//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import CoreGraphics90import Foundation91import ImageIO92import Testing93@testable import AstraLocal9495enum ProjectPaths {96 /// Repository root, derived from this file's location (Tests/AstraLocalTests/…).97 static let root: URL = {98 if let override = ProcessInfo.processInfo.environment["ASTRA_PROJECT_ROOT"], !override.isEmpty { return URL(fileURLWithPath: override) }99 return URL(fileURLWithPath: #filePath).deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()100 }()101 static let models = root.appendingPathComponent("build/models", isDirectory: true)102 static let results = root.appendingPathComponent("build/verify", isDirectory: true)103 static let catalog = root.appendingPathComponent("docs/catalog/catalog.json")104 static let compatibility = root.appendingPathComponent("docs/MODEL_COMPATIBILITY.md")105 static func env(_ name: String) -> String? { ProcessInfo.processInfo.environment[name].flatMap { $0.isEmpty ? nil : $0 } }106}107108/// Everything recorded while verifying one candidate repository on this machine.109struct VerificationResult: Codable {110 struct Turn: Codable {111 var text: String; var reasoning: String112 var promptTokens: Int?; var cachedPromptTokens: Int?; var completionTokens: Int?113 var timeToFirstToken: Double?; var decodeTokensPerSecond: Double?; var stopReason: String114 }115 struct Cancellation: Codable { var chunksBeforeCancel: Int; var secondsToStop: Double; var endedWithCancelledError: Bool; var completedEventSeen: Bool }116 var repo: String117 var id: String118 var date: String119 var passed = false120 var failure: String?121 var stage = "resolve"122 var revision: String?123 var hubLicense: String?124 var hubTags: [String] = []125 var hubBaseModels: [String] = []126 var hubPipeline: String?127 var files: [LocalModelFile] = []128 var totalBytes: Int64 = 0129 var downloadSeconds: Double?130 var downloadRetries: Int?131 var modelType: String?132 var quantizationBits: Int?133 var quantizationMode: String?134 var contextLength: Int?135 var hasVision = false136 var licenseFiles: [String] = []137 var licenseFileHead: String?138 var loadSeconds: Double?139 var weightsMemoryBytes: Int64?140 var templateSupportsTools: Bool?141 var turn1: Turn?142 var turn2: Turn?143 var recalledFact: Bool?144 var cancellation: Cancellation?145 var reloadSeconds: Double?146 var reloadAnswer: String?147 var benchmark: LocalBenchmarkResult?148 var tokensPerSecond: Double?149 var timeToFirstToken: Double?150 var coldTimeToFirstToken: Double?151 var peakMemoryBytes: Int64?152 var device: DeviceFacts = .current()153 var chip: String?154 /// Probes that do not decide pass or fail; nil = not attempted because the model does not declare the feature.155 var reasoningVerified: Bool?156 var toolCallVerified: Bool?157 var visionVerified: Bool?158 var visionAnswer: String?159 var buildConfiguration: String?160 /// One-minute load average of the machine when the benchmark ran: other work on the Mac lowers every speed figure.161 var loadAverage: Double?162}163164struct ModelVerifier {165 let repo: String166 var id: String { String(repo.split(separator: "/").last ?? "") }167168 static func today() -> String { let f = DateFormatter(); f.locale = Locale(identifier: "en_US_POSIX"); f.dateFormat = "yyyy-MM-dd"; return f.string(from: Date()) }169170 static func chip() -> String? {171 var size = 0172 guard sysctlbyname("machdep.cpu.brand_string", nil, &size, nil, 0) == 0, size > 0 else { return nil }173 var buffer = [CChar](repeating: 0, count: size)174 guard sysctlbyname("machdep.cpu.brand_string", &buffer, &size, nil, 0) == 0 else { return nil }175 return String(decoding: buffer.prefix { $0 != 0 }.map { UInt8(bitPattern: $0) }, as: UTF8.self)176 }177178 /// A 96 × 96 plain red PNG, made here so the vision probe needs no fixture file.179 static func redSquarePNG() -> Data? {180 let side = 96181 guard let context = CGContext(data: nil, width: side, height: side, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(),182 bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return nil }183 context.setFillColor(red: 0.9, green: 0.05, blue: 0.05, alpha: 1); context.fill(CGRect(x: 0, y: 0, width: side, height: side))184 guard let image = context.makeImage() else { return nil }185 let data = NSMutableData()186 guard let destination = CGImageDestinationCreateWithData(data, "public.png" as CFString, 1, nil) else { return nil }187 CGImageDestinationAddImage(destination, image, nil)188 return CGImageDestinationFinalize(destination) ? data as Data : nil189 }190191 func log(_ message: String) { FileHandle.standardError.write(Data("[verify \(id)] \(message)\n".utf8)) }192193 func run() async -> VerificationResult {194 var result = VerificationResult(repo: repo, id: id, date: Self.today())195 result.chip = Self.chip()196 #if DEBUG197 result.buildConfiguration = "debug"198 #else199 result.buildConfiguration = "release"200 #endif201 do { try await steps(&result); result.passed = true; result.stage = "done" }202 catch { result.failure = (error as? LocalizedError)?.errorDescription ?? String(describing: error) }203 log(result.passed ? "PASS" : "FAIL at \(result.stage): \(result.failure ?? "")")204 return result205 }206207 private func collect(_ engine: LocalEngine, _ request: GenerationRequest) async throws -> VerificationResult.Turn {208 var validator = StreamValidator(); var text = ""; var reasoning = ""; var completed: StopReason?209 for try await event in engine.generate(request) {210 for e in try validator.accept(event) {211 switch e {212 case .textDelta(let t): text += t213 case .reasoningSummaryDelta(let t): reasoning += t214 case .messageCompleted(let r): completed = r215 default: break216 }217 }218 }219 guard let completed else { throw AstraError.protocolViolation("the stream ended without .messageCompleted") }220 guard let m = engine.monitor.lastMetrics else { throw AstraError.local("no metrics were recorded") }221 return .init(text: text, reasoning: reasoning, promptTokens: m.promptTokens, cachedPromptTokens: m.cachedPromptTokens, completionTokens: m.completionTokens,222 timeToFirstToken: m.timeToFirstToken, decodeTokensPerSecond: m.decodeTokensPerSecond, stopReason: completed.rawValue)223 }224225 private func steps(_ r: inout VerificationResult) async throws {226 // 1. Resolve the current commit and file list from the public Hub API.227 r.stage = "resolve"228 let snapshot = try await HuggingFaceResolver(transport: URLSessionTransport()).snapshot(repo: repo)229 r.revision = snapshot.revision; r.hubLicense = snapshot.license; r.hubTags = snapshot.tags230 r.hubBaseModels = snapshot.baseModels; r.hubPipeline = snapshot.pipelineTag231 let plan = snapshot.plan(id: id)232 log("revision \(snapshot.revision), \(plan.files.count) files, \(MemoryGovernor.format(plan.totalBytes))")233234 // 2. Download through the app's own coordinator, with Range resume and per-file verification.235 r.stage = "download"236 let store = ModelStore(root: ProjectPaths.models)237 let coordinator = DownloadCoordinator(store: store, transport: URLSessionDownloadTransport())238 let started = Date()239 try await coordinator.start(plan)240 let progress = Task {241 while !Task.isCancelled {242 try? await Task.sleep(nanoseconds: 15_000_000_000)243 if let s = await coordinator.status(id: id), s.phase.isActive {244 log(String(format: "%.0f %% %.1f MB/s", s.fraction * 100, (s.bytesPerSecond ?? 0) / 1_048_576))245 }246 }247 }248 let status = await coordinator.waitUntilSettled(id: id)249 progress.cancel()250 guard let status, status.phase == .completed else {251 if case .failed(let e)? = status?.phase { throw e }252 throw AstraError.local("download ended in state \(String(describing: status?.phase))")253 }254 r.downloadSeconds = Date().timeIntervalSince(started); r.downloadRetries = status.retries255 let directory = try store.directory(for: id)256 // SHA-256 of every file, recomputed here from disk, independently of the coordinator's own record.257 var files: [LocalModelFile] = []258 for f in plan.files {259 let url = try store.fileURL(modelID: id, path: f.path)260 let hash = try FileDigest.sha256(of: url)261 guard hash == status.sha256[f.path] else { throw AstraError.local("hash of \(f.path) changed between download and re-read") }262 if let published = f.sha256, published != hash { throw AstraError.local("\(f.path) does not match the SHA-256 published by the Hub") }263 guard ModelStore.fileSize(url) == f.size else { throw AstraError.local("size of \(f.path) differs from the Hub's listing") }264 files.append(LocalModelFile(path: f.path, size: f.size, sha256: hash))265 }266 r.files = files; r.totalBytes = files.reduce(0) { $0 + $1.size }267268 // 3. What the repository actually is, read from its own files.269 r.stage = "inspect"270 if let data = try? Data(contentsOf: directory.appendingPathComponent("config.json")), let config = JSONValue.parse(data) {271 r.modelType = config["model_type"]?.stringValue272 r.quantizationBits = config["quantization"]?["bits"]?.intValue273 r.quantizationMode = config["quantization"]?["mode"]?.stringValue274 r.hasVision = config["vision_config"] != nil275 }276 r.contextLength = ModelConfigSummary.read(directory).contextLength277 r.licenseFiles = snapshot.files.map(\.path).filter { $0.lowercased().contains("license") || $0.lowercased().contains("notice") || $0.lowercased().contains("use_policy") }278 if let first = r.licenseFiles.first, let text = try? String(contentsOf: store.fileURL(modelID: id, path: first), encoding: .utf8) {279 r.licenseFileHead = String(text.trimmingCharacters(in: .whitespacesAndNewlines).prefix(200))280 }281282 // 4. Load with the MLX runtime on this machine.283 r.stage = "load"284 var configuration = LocalEngine.Configuration(maxLoadedModels: 1); configuration.enforceMemoryVerdict = false285 let engine = LocalEngine(store: store, configuration: configuration)286 let info = try await engine.load(modelID: id, directory: directory)287 r.loadSeconds = info.loadTime; r.weightsMemoryBytes = info.weightsMemoryBytes; r.templateSupportsTools = info.templateSupportsTools288 log(String(format: "loaded in %.2f s, %@ of weights", info.loadTime, MemoryGovernor.format(info.weightsMemoryBytes ?? 0)))289 let ref = ModelRef(provider: .mlx, modelID: id)290 let parameters = GenerationParameters(temperature: 0, maxOutputTokens: 96, seed: 11)291292 // 5. A real two-turn conversation on one KV-cache session.293 r.stage = "chat turn 1"294 let first = PromptMessage.user("My name is Sam and my favourite number is 42. Greet me by name in one short sentence.")295 let t1 = try await collect(engine, GenerationRequest(model: ref, messages: [first], parameters: parameters, sessionKey: "verify"))296 r.turn1 = t1297 guard !t1.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, (t1.completionTokens ?? 0) > 0, (t1.promptTokens ?? 0) > 0 else {298 throw AstraError.local("turn 1 produced no text or no measured token counts")299 }300 r.stage = "chat turn 2"301 let t2 = try await collect(engine, GenerationRequest(model: ref, messages: [first, .assistant(t1.text), .user("What is my favourite number? Answer with the number only.")],302 parameters: parameters, sessionKey: "verify"))303 r.turn2 = t2304 guard !t2.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, (t2.completionTokens ?? 0) > 0 else { throw AstraError.local("turn 2 produced no text") }305 guard (t2.cachedPromptTokens ?? 0) > 0 else { throw AstraError.local("turn 2 did not reuse the KV cache of turn 1") }306 r.recalledFact = t2.text.contains("42")307 log("turn 2: \(t2.text.prefix(60).replacingOccurrences(of: "\n", with: " ")) (cached \(t2.cachedPromptTokens ?? 0) prompt tokens)")308309 // 6. Cancel in the middle of a long answer.310 r.stage = "cancellation"311 var long = parameters; long.maxOutputTokens = 800312 let request = GenerationRequest(model: ref, messages: [.user("Write a detailed 600-word story about a lighthouse keeper.")], parameters: long)313 var chunks = 0; var cancelledAt: Date?; var sawCompleted = false; var endedCancelled = false314 do {315 for try await event in engine.generate(request) {316 if case .textDelta = event { chunks += 1 }317 if case .messageCompleted = event { sawCompleted = true }318 if chunks == 12, cancelledAt == nil { cancelledAt = Date(); Task { await engine.cancelActiveGeneration() } }319 }320 } catch { endedCancelled = (error as? AstraError) == .cancelled }321 guard let cancelledAt else { throw AstraError.local("the model stopped before the cancellation point (\(chunks) chunks)") }322 let stop = Date().timeIntervalSince(cancelledAt)323 r.cancellation = .init(chunksBeforeCancel: chunks, secondsToStop: stop, endedWithCancelledError: endedCancelled, completedEventSeen: sawCompleted)324 guard endedCancelled, !sawCompleted, stop < 3 else { throw AstraError.local("cancellation did not end the stream promptly with .cancelled (\(stop) s)") }325 guard engine.monitor.lastMetrics?.stopReason == .cancelled else { throw AstraError.local("cancelled generation was not recorded as cancelled") }326327 // 7. Unload, then load again from disk only: no network object exists in this path.328 r.stage = "reload from disk"329 await engine.unload(id)330 guard await engine.snapshot().loaded.isEmpty else { throw AstraError.local("the model stayed loaded after unload") }331 let again = try await engine.load(modelID: id, directory: directory)332 r.reloadSeconds = again.loadTime333 let t3 = try await collect(engine, GenerationRequest(model: ref, messages: [.user("Name the capital of France in one word.")], parameters: parameters))334 r.reloadAnswer = t3.text335 guard !t3.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw AstraError.local("no text after reloading from disk") }336337 // Probes (recorded, never decisive): thinking block, tool call, image input. Each runs only when the model declares the feature.338 r.stage = "probes"339 if ModelConfigSummary.read(directory).templateSupportsThinking {340 var p = parameters; p.maxOutputTokens = 300; p.reasoning = .medium341 let t = try? await collect(engine, GenerationRequest(model: ref, messages: [.user("What is 17 + 26? Think first.")], parameters: p))342 r.reasoningVerified = !(t?.reasoning.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ?? true)343 }344 if again.templateSupportsTools {345 let tool = ToolDefinition(name: "get_weather", summary: "Get the current weather for a city.",346 parameters: ["type": "object", "properties": ["city": ["type": "string", "description": "City name"]], "required": ["city"]])347 var p = parameters; p.maxOutputTokens = 200348 var called = false349 do {350 for try await event in engine.generate(GenerationRequest(model: ref, messages: [.user("What is the weather in Paris right now? Use the tool.")], parameters: p, tools: [tool])) {351 if case .toolCallCompleted(let call) = event, call.name == "get_weather" { called = true }352 }353 } catch {}354 r.toolCallVerified = called355 }356 if again.hasVision, let image = Self.redSquarePNG() {357 let t = try? await collect(engine, GenerationRequest(model: ref, messages: [.user("What colour fills this image? Answer with one word.", images: [ImageInput(data: image, mimeType: "image/png")])], parameters: parameters))358 r.visionAnswer = t?.text359 r.visionVerified = t?.text.lowercased().contains("red") ?? false360 }361362 // 8. Measured speed: reload once more so the first benchmark sample is a true cold start.363 r.stage = "benchmark"364 var load = [Double](repeating: 0, count: 3)365 if getloadavg(&load, 3) > 0 { r.loadAverage = load[0] }366 await engine.unload(id)367 _ = try await engine.load(modelID: id, directory: directory)368 let benchmark = try await LocalBenchmark().run(modelID: id, engine: engine, revision: snapshot.revision)369 r.benchmark = benchmark370 r.tokensPerSecond = benchmark.warmDecodeTokensPerSecond; r.timeToFirstToken = benchmark.warmTimeToFirstToken371 r.coldTimeToFirstToken = benchmark.cold?.timeToFirstToken; r.peakMemoryBytes = benchmark.peakMemoryBytes372 guard let rate = r.tokensPerSecond, rate > 0, r.timeToFirstToken != nil else { throw AstraError.local("the benchmark produced no measured speed") }373 log(String(format: "%.1f tok/s warm, ttft %.3f s warm / %.3f s cold, peak %@", rate, r.timeToFirstToken ?? 0, r.coldTimeToFirstToken ?? 0,374 MemoryGovernor.format(r.peakMemoryBytes ?? 0)))375 await engine.unloadAll()376 }377}378379/// `ASTRA_VERIFY_REPO=mlx-community/<name> swift test --filter verifyOneModel` (driven by Scripts/verify-models.sh).380@Test(.enabled(if: ProjectPaths.env("ASTRA_VERIFY_REPO") != nil)) func verifyOneModel() async throws {381 let repo = try #require(ProjectPaths.env("ASTRA_VERIFY_REPO"))382 let result = await ModelVerifier(repo: repo).run()383 try FileManager.default.createDirectory(at: ProjectPaths.results, withIntermediateDirectories: true)384 let e = JSONEncoder(); e.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes]; e.dateEncodingStrategy = .iso8601385 try e.encode(result).write(to: ProjectPaths.results.appendingPathComponent(result.id + ".json"), options: .atomic)386 #expect(result.passed, "\(repo): \(result.failure ?? "")")387}388389/// Downloads several repositories at once through the DownloadCoordinator, without loading them: `ASTRA_PREFETCH_REPOS=a,b,c`.390/// The Hub limits each connection to about 2 MB/s after an initial burst, so fetching the candidates one after the other takes hours;391/// verification itself stays sequential and finds the files already verified on disk.392@Test(.enabled(if: ProjectPaths.env("ASTRA_PREFETCH_REPOS") != nil)) func prefetchModels() async throws {393 let repos = (ProjectPaths.env("ASTRA_PREFETCH_REPOS") ?? "").split(separator: ",").map(String.init)394 let store = ModelStore(root: ProjectPaths.models)395 let coordinator = DownloadCoordinator(store: store, transport: URLSessionDownloadTransport())396 let resolver = HuggingFaceResolver(transport: URLSessionTransport())397 var ids: [String] = []398 for repo in repos {399 let id = String(repo.split(separator: "/").last ?? "")400 do { try await coordinator.start(try await resolver.snapshot(repo: repo).plan(id: id)); ids.append(id) }401 catch { FileHandle.standardError.write(Data("[prefetch] \(repo): \(error)\n".utf8)) }402 }403 let reporter = Task {404 while !Task.isCancelled {405 try? await Task.sleep(nanoseconds: 60_000_000_000)406 let line = await coordinator.statuses().map { String(format: "%@ %.0f%%", $0.id, $0.fraction * 100) }.joined(separator: " ")407 FileHandle.standardError.write(Data("[prefetch] \(line)\n".utf8))408 }409 }410 for id in ids {411 let status = await coordinator.waitUntilSettled(id: id)412 FileHandle.standardError.write(Data("[prefetch] \(id): \(String(describing: status?.phase)) retries \(status?.retries ?? 0)\n".utf8))413 #expect(status?.phase == .completed)414 }415 reporter.cancel()416}417