SPB Git

Repository Public MIT

ModelVerification.swift 23.4 KB 417 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraLocalTests/ModelVerification.swift10// Module           : AstraLocalTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2214//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import CoreGraphics90import Foundation91import ImageIO92import Testing93@testable import AstraLocal9495enum ProjectPaths {96    /// Repository root, derived from this file's location (Tests/AstraLocalTests/…).97    static let root: URL = {98        if let override = ProcessInfo.processInfo.environment["ASTRA_PROJECT_ROOT"], !override.isEmpty { return URL(fileURLWithPath: override) }99        return URL(fileURLWithPath: #filePath).deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()100    }()101    static let models = root.appendingPathComponent("build/models", isDirectory: true)102    static let results = root.appendingPathComponent("build/verify", isDirectory: true)103    static let catalog = root.appendingPathComponent("docs/catalog/catalog.json")104    static let compatibility = root.appendingPathComponent("docs/MODEL_COMPATIBILITY.md")105    static func env(_ name: String) -> String? { ProcessInfo.processInfo.environment[name].flatMap { $0.isEmpty ? nil : $0 } }106}107108/// Everything recorded while verifying one candidate repository on this machine.109struct VerificationResult: Codable {110    struct Turn: Codable {111        var text: String; var reasoning: String112        var promptTokens: Int?; var cachedPromptTokens: Int?; var completionTokens: Int?113        var timeToFirstToken: Double?; var decodeTokensPerSecond: Double?; var stopReason: String114    }115    struct Cancellation: Codable { var chunksBeforeCancel: Int; var secondsToStop: Double; var endedWithCancelledError: Bool; var completedEventSeen: Bool }116    var repo: String117    var id: String118    var date: String119    var passed = false120    var failure: String?121    var stage = "resolve"122    var revision: String?123    var hubLicense: String?124    var hubTags: [String] = []125    var hubBaseModels: [String] = []126    var hubPipeline: String?127    var files: [LocalModelFile] = []128    var totalBytes: Int64 = 0129    var downloadSeconds: Double?130    var downloadRetries: Int?131    var modelType: String?132    var quantizationBits: Int?133    var quantizationMode: String?134    var contextLength: Int?135    var hasVision = false136    var licenseFiles: [String] = []137    var licenseFileHead: String?138    var loadSeconds: Double?139    var weightsMemoryBytes: Int64?140    var templateSupportsTools: Bool?141    var turn1: Turn?142    var turn2: Turn?143    var recalledFact: Bool?144    var cancellation: Cancellation?145    var reloadSeconds: Double?146    var reloadAnswer: String?147    var benchmark: LocalBenchmarkResult?148    var tokensPerSecond: Double?149    var timeToFirstToken: Double?150    var coldTimeToFirstToken: Double?151    var peakMemoryBytes: Int64?152    var device: DeviceFacts = .current()153    var chip: String?154    /// Probes that do not decide pass or fail; nil = not attempted because the model does not declare the feature.155    var reasoningVerified: Bool?156    var toolCallVerified: Bool?157    var visionVerified: Bool?158    var visionAnswer: String?159    var buildConfiguration: String?160    /// One-minute load average of the machine when the benchmark ran: other work on the Mac lowers every speed figure.161    var loadAverage: Double?162}163164struct ModelVerifier {165    let repo: String166    var id: String { String(repo.split(separator: "/").last ?? "") }167168    static func today() -> String { let f = DateFormatter(); f.locale = Locale(identifier: "en_US_POSIX"); f.dateFormat = "yyyy-MM-dd"; return f.string(from: Date()) }169170    static func chip() -> String? {171        var size = 0172        guard sysctlbyname("machdep.cpu.brand_string", nil, &size, nil, 0) == 0, size > 0 else { return nil }173        var buffer = [CChar](repeating: 0, count: size)174        guard sysctlbyname("machdep.cpu.brand_string", &buffer, &size, nil, 0) == 0 else { return nil }175        return String(decoding: buffer.prefix { $0 != 0 }.map { UInt8(bitPattern: $0) }, as: UTF8.self)176    }177178    /// A 96 × 96 plain red PNG, made here so the vision probe needs no fixture file.179    static func redSquarePNG() -> Data? {180        let side = 96181        guard let context = CGContext(data: nil, width: side, height: side, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(),182                                      bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return nil }183        context.setFillColor(red: 0.9, green: 0.05, blue: 0.05, alpha: 1); context.fill(CGRect(x: 0, y: 0, width: side, height: side))184        guard let image = context.makeImage() else { return nil }185        let data = NSMutableData()186        guard let destination = CGImageDestinationCreateWithData(data, "public.png" as CFString, 1, nil) else { return nil }187        CGImageDestinationAddImage(destination, image, nil)188        return CGImageDestinationFinalize(destination) ? data as Data : nil189    }190191    func log(_ message: String) { FileHandle.standardError.write(Data("[verify \(id)] \(message)\n".utf8)) }192193    func run() async -> VerificationResult {194        var result = VerificationResult(repo: repo, id: id, date: Self.today())195        result.chip = Self.chip()196        #if DEBUG197        result.buildConfiguration = "debug"198        #else199        result.buildConfiguration = "release"200        #endif201        do { try await steps(&result); result.passed = true; result.stage = "done" }202        catch { result.failure = (error as? LocalizedError)?.errorDescription ?? String(describing: error) }203        log(result.passed ? "PASS" : "FAIL at \(result.stage): \(result.failure ?? "")")204        return result205    }206207    private func collect(_ engine: LocalEngine, _ request: GenerationRequest) async throws -> VerificationResult.Turn {208        var validator = StreamValidator(); var text = ""; var reasoning = ""; var completed: StopReason?209        for try await event in engine.generate(request) {210            for e in try validator.accept(event) {211                switch e {212                case .textDelta(let t): text += t213                case .reasoningSummaryDelta(let t): reasoning += t214                case .messageCompleted(let r): completed = r215                default: break216                }217            }218        }219        guard let completed else { throw AstraError.protocolViolation("the stream ended without .messageCompleted") }220        guard let m = engine.monitor.lastMetrics else { throw AstraError.local("no metrics were recorded") }221        return .init(text: text, reasoning: reasoning, promptTokens: m.promptTokens, cachedPromptTokens: m.cachedPromptTokens, completionTokens: m.completionTokens,222                     timeToFirstToken: m.timeToFirstToken, decodeTokensPerSecond: m.decodeTokensPerSecond, stopReason: completed.rawValue)223    }224225    private func steps(_ r: inout VerificationResult) async throws {226        // 1. Resolve the current commit and file list from the public Hub API.227        r.stage = "resolve"228        let snapshot = try await HuggingFaceResolver(transport: URLSessionTransport()).snapshot(repo: repo)229        r.revision = snapshot.revision; r.hubLicense = snapshot.license; r.hubTags = snapshot.tags230        r.hubBaseModels = snapshot.baseModels; r.hubPipeline = snapshot.pipelineTag231        let plan = snapshot.plan(id: id)232        log("revision \(snapshot.revision), \(plan.files.count) files, \(MemoryGovernor.format(plan.totalBytes))")233234        // 2. Download through the app's own coordinator, with Range resume and per-file verification.235        r.stage = "download"236        let store = ModelStore(root: ProjectPaths.models)237        let coordinator = DownloadCoordinator(store: store, transport: URLSessionDownloadTransport())238        let started = Date()239        try await coordinator.start(plan)240        let progress = Task {241            while !Task.isCancelled {242                try? await Task.sleep(nanoseconds: 15_000_000_000)243                if let s = await coordinator.status(id: id), s.phase.isActive {244                    log(String(format: "%.0f %%  %.1f MB/s", s.fraction * 100, (s.bytesPerSecond ?? 0) / 1_048_576))245                }246            }247        }248        let status = await coordinator.waitUntilSettled(id: id)249        progress.cancel()250        guard let status, status.phase == .completed else {251            if case .failed(let e)? = status?.phase { throw e }252            throw AstraError.local("download ended in state \(String(describing: status?.phase))")253        }254        r.downloadSeconds = Date().timeIntervalSince(started); r.downloadRetries = status.retries255        let directory = try store.directory(for: id)256        // SHA-256 of every file, recomputed here from disk, independently of the coordinator's own record.257        var files: [LocalModelFile] = []258        for f in plan.files {259            let url = try store.fileURL(modelID: id, path: f.path)260            let hash = try FileDigest.sha256(of: url)261            guard hash == status.sha256[f.path] else { throw AstraError.local("hash of \(f.path) changed between download and re-read") }262            if let published = f.sha256, published != hash { throw AstraError.local("\(f.path) does not match the SHA-256 published by the Hub") }263            guard ModelStore.fileSize(url) == f.size else { throw AstraError.local("size of \(f.path) differs from the Hub's listing") }264            files.append(LocalModelFile(path: f.path, size: f.size, sha256: hash))265        }266        r.files = files; r.totalBytes = files.reduce(0) { $0 + $1.size }267268        // 3. What the repository actually is, read from its own files.269        r.stage = "inspect"270        if let data = try? Data(contentsOf: directory.appendingPathComponent("config.json")), let config = JSONValue.parse(data) {271            r.modelType = config["model_type"]?.stringValue272            r.quantizationBits = config["quantization"]?["bits"]?.intValue273            r.quantizationMode = config["quantization"]?["mode"]?.stringValue274            r.hasVision = config["vision_config"] != nil275        }276        r.contextLength = ModelConfigSummary.read(directory).contextLength277        r.licenseFiles = snapshot.files.map(\.path).filter { $0.lowercased().contains("license") || $0.lowercased().contains("notice") || $0.lowercased().contains("use_policy") }278        if let first = r.licenseFiles.first, let text = try? String(contentsOf: store.fileURL(modelID: id, path: first), encoding: .utf8) {279            r.licenseFileHead = String(text.trimmingCharacters(in: .whitespacesAndNewlines).prefix(200))280        }281282        // 4. Load with the MLX runtime on this machine.283        r.stage = "load"284        var configuration = LocalEngine.Configuration(maxLoadedModels: 1); configuration.enforceMemoryVerdict = false285        let engine = LocalEngine(store: store, configuration: configuration)286        let info = try await engine.load(modelID: id, directory: directory)287        r.loadSeconds = info.loadTime; r.weightsMemoryBytes = info.weightsMemoryBytes; r.templateSupportsTools = info.templateSupportsTools288        log(String(format: "loaded in %.2f s, %@ of weights", info.loadTime, MemoryGovernor.format(info.weightsMemoryBytes ?? 0)))289        let ref = ModelRef(provider: .mlx, modelID: id)290        let parameters = GenerationParameters(temperature: 0, maxOutputTokens: 96, seed: 11)291292        // 5. A real two-turn conversation on one KV-cache session.293        r.stage = "chat turn 1"294        let first = PromptMessage.user("My name is Sam and my favourite number is 42. Greet me by name in one short sentence.")295        let t1 = try await collect(engine, GenerationRequest(model: ref, messages: [first], parameters: parameters, sessionKey: "verify"))296        r.turn1 = t1297        guard !t1.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, (t1.completionTokens ?? 0) > 0, (t1.promptTokens ?? 0) > 0 else {298            throw AstraError.local("turn 1 produced no text or no measured token counts")299        }300        r.stage = "chat turn 2"301        let t2 = try await collect(engine, GenerationRequest(model: ref, messages: [first, .assistant(t1.text), .user("What is my favourite number? Answer with the number only.")],302                                                           parameters: parameters, sessionKey: "verify"))303        r.turn2 = t2304        guard !t2.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, (t2.completionTokens ?? 0) > 0 else { throw AstraError.local("turn 2 produced no text") }305        guard (t2.cachedPromptTokens ?? 0) > 0 else { throw AstraError.local("turn 2 did not reuse the KV cache of turn 1") }306        r.recalledFact = t2.text.contains("42")307        log("turn 2: \(t2.text.prefix(60).replacingOccurrences(of: "\n", with: " ")) (cached \(t2.cachedPromptTokens ?? 0) prompt tokens)")308309        // 6. Cancel in the middle of a long answer.310        r.stage = "cancellation"311        var long = parameters; long.maxOutputTokens = 800312        let request = GenerationRequest(model: ref, messages: [.user("Write a detailed 600-word story about a lighthouse keeper.")], parameters: long)313        var chunks = 0; var cancelledAt: Date?; var sawCompleted = false; var endedCancelled = false314        do {315            for try await event in engine.generate(request) {316                if case .textDelta = event { chunks += 1 }317                if case .messageCompleted = event { sawCompleted = true }318                if chunks == 12, cancelledAt == nil { cancelledAt = Date(); Task { await engine.cancelActiveGeneration() } }319            }320        } catch { endedCancelled = (error as? AstraError) == .cancelled }321        guard let cancelledAt else { throw AstraError.local("the model stopped before the cancellation point (\(chunks) chunks)") }322        let stop = Date().timeIntervalSince(cancelledAt)323        r.cancellation = .init(chunksBeforeCancel: chunks, secondsToStop: stop, endedWithCancelledError: endedCancelled, completedEventSeen: sawCompleted)324        guard endedCancelled, !sawCompleted, stop < 3 else { throw AstraError.local("cancellation did not end the stream promptly with .cancelled (\(stop) s)") }325        guard engine.monitor.lastMetrics?.stopReason == .cancelled else { throw AstraError.local("cancelled generation was not recorded as cancelled") }326327        // 7. Unload, then load again from disk only: no network object exists in this path.328        r.stage = "reload from disk"329        await engine.unload(id)330        guard await engine.snapshot().loaded.isEmpty else { throw AstraError.local("the model stayed loaded after unload") }331        let again = try await engine.load(modelID: id, directory: directory)332        r.reloadSeconds = again.loadTime333        let t3 = try await collect(engine, GenerationRequest(model: ref, messages: [.user("Name the capital of France in one word.")], parameters: parameters))334        r.reloadAnswer = t3.text335        guard !t3.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw AstraError.local("no text after reloading from disk") }336337        // Probes (recorded, never decisive): thinking block, tool call, image input. Each runs only when the model declares the feature.338        r.stage = "probes"339        if ModelConfigSummary.read(directory).templateSupportsThinking {340            var p = parameters; p.maxOutputTokens = 300; p.reasoning = .medium341            let t = try? await collect(engine, GenerationRequest(model: ref, messages: [.user("What is 17 + 26? Think first.")], parameters: p))342            r.reasoningVerified = !(t?.reasoning.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ?? true)343        }344        if again.templateSupportsTools {345            let tool = ToolDefinition(name: "get_weather", summary: "Get the current weather for a city.",346                                      parameters: ["type": "object", "properties": ["city": ["type": "string", "description": "City name"]], "required": ["city"]])347            var p = parameters; p.maxOutputTokens = 200348            var called = false349            do {350                for try await event in engine.generate(GenerationRequest(model: ref, messages: [.user("What is the weather in Paris right now? Use the tool.")], parameters: p, tools: [tool])) {351                    if case .toolCallCompleted(let call) = event, call.name == "get_weather" { called = true }352                }353            } catch {}354            r.toolCallVerified = called355        }356        if again.hasVision, let image = Self.redSquarePNG() {357            let t = try? await collect(engine, GenerationRequest(model: ref, messages: [.user("What colour fills this image? Answer with one word.", images: [ImageInput(data: image, mimeType: "image/png")])], parameters: parameters))358            r.visionAnswer = t?.text359            r.visionVerified = t?.text.lowercased().contains("red") ?? false360        }361362        // 8. Measured speed: reload once more so the first benchmark sample is a true cold start.363        r.stage = "benchmark"364        var load = [Double](repeating: 0, count: 3)365        if getloadavg(&load, 3) > 0 { r.loadAverage = load[0] }366        await engine.unload(id)367        _ = try await engine.load(modelID: id, directory: directory)368        let benchmark = try await LocalBenchmark().run(modelID: id, engine: engine, revision: snapshot.revision)369        r.benchmark = benchmark370        r.tokensPerSecond = benchmark.warmDecodeTokensPerSecond; r.timeToFirstToken = benchmark.warmTimeToFirstToken371        r.coldTimeToFirstToken = benchmark.cold?.timeToFirstToken; r.peakMemoryBytes = benchmark.peakMemoryBytes372        guard let rate = r.tokensPerSecond, rate > 0, r.timeToFirstToken != nil else { throw AstraError.local("the benchmark produced no measured speed") }373        log(String(format: "%.1f tok/s warm, ttft %.3f s warm / %.3f s cold, peak %@", rate, r.timeToFirstToken ?? 0, r.coldTimeToFirstToken ?? 0,374                   MemoryGovernor.format(r.peakMemoryBytes ?? 0)))375        await engine.unloadAll()376    }377}378379/// `ASTRA_VERIFY_REPO=mlx-community/<name> swift test --filter verifyOneModel` (driven by Scripts/verify-models.sh).380@Test(.enabled(if: ProjectPaths.env("ASTRA_VERIFY_REPO") != nil)) func verifyOneModel() async throws {381    let repo = try #require(ProjectPaths.env("ASTRA_VERIFY_REPO"))382    let result = await ModelVerifier(repo: repo).run()383    try FileManager.default.createDirectory(at: ProjectPaths.results, withIntermediateDirectories: true)384    let e = JSONEncoder(); e.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes]; e.dateEncodingStrategy = .iso8601385    try e.encode(result).write(to: ProjectPaths.results.appendingPathComponent(result.id + ".json"), options: .atomic)386    #expect(result.passed, "\(repo): \(result.failure ?? "")")387}388389/// Downloads several repositories at once through the DownloadCoordinator, without loading them: `ASTRA_PREFETCH_REPOS=a,b,c`.390/// The Hub limits each connection to about 2 MB/s after an initial burst, so fetching the candidates one after the other takes hours;391/// verification itself stays sequential and finds the files already verified on disk.392@Test(.enabled(if: ProjectPaths.env("ASTRA_PREFETCH_REPOS") != nil)) func prefetchModels() async throws {393    let repos = (ProjectPaths.env("ASTRA_PREFETCH_REPOS") ?? "").split(separator: ",").map(String.init)394    let store = ModelStore(root: ProjectPaths.models)395    let coordinator = DownloadCoordinator(store: store, transport: URLSessionDownloadTransport())396    let resolver = HuggingFaceResolver(transport: URLSessionTransport())397    var ids: [String] = []398    for repo in repos {399        let id = String(repo.split(separator: "/").last ?? "")400        do { try await coordinator.start(try await resolver.snapshot(repo: repo).plan(id: id)); ids.append(id) }401        catch { FileHandle.standardError.write(Data("[prefetch] \(repo): \(error)\n".utf8)) }402    }403    let reporter = Task {404        while !Task.isCancelled {405            try? await Task.sleep(nanoseconds: 60_000_000_000)406            let line = await coordinator.statuses().map { String(format: "%@ %.0f%%", $0.id, $0.fraction * 100) }.joined(separator: "  ")407            FileHandle.standardError.write(Data("[prefetch] \(line)\n".utf8))408        }409    }410    for id in ids {411        let status = await coordinator.waitUntilSettled(id: id)412        FileHandle.standardError.write(Data("[prefetch] \(id): \(String(describing: status?.phase)) retries \(status?.retries ?? 0)\n".utf8))413        #expect(status?.phase == .completed)414    }415    reporter.cancel()416}417
Files main
Menu

Theme