SPB Git

Repository Public MIT

VerificationReport.swift 23.6 KB 339 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraLocalTests/VerificationReport.swift10// Module           : AstraLocalTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2214//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraLocal9293/// Facts about each candidate that do not come from running it: who made the original model, its nominal size and its license terms.94/// License conditions are summarized from the license texts linked here; they are not legal advice.95struct CandidateFacts {96    enum Rehost: String { case yes = "Yes", conditional = "Conditional", unclear = "Unclear" }97    var repo: String98    var displayName: String99    var family: String100    var publisher: String101    /// Nominal parameter count of the original model, from its publisher's model card.102    var parameters: Int64103    var licenseID: String104    var licenseName: String105    var licenseURL: String106    var rehost: Rehost107    var conditions: String108    var id: String { String(repo.split(separator: "/").last ?? "") }109110    static let apache = ("apache-2.0", "Apache-2.0", "https://www.apache.org/licenses/LICENSE-2.0",111                         "Keep the license text and any NOTICE with the files; state changes if the weights are modified.")112    static let mit = ("mit", "MIT", "https://opensource.org/license/mit", "Keep the copyright and permission notice with the files.")113    static let gemma = ("gemma", "Gemma Terms of Use", "https://ai.google.dev/gemma/terms",114                        "Redistribution is allowed only if recipients are bound by the same use restrictions (Prohibited Use Policy), receive a copy of the Gemma Terms, and modified files carry a notice. Served from Hugging Face only until the owner decides.")115    static let llama = ("llama3.2", "Llama 3.2 Community License", "https://www.llama.com/llama3_2/license/",116                        "Redistribution is allowed only with a copy of the license, the “Built with Llama” attribution shown prominently, the notice file kept, and the Acceptable Use Policy respected. Served from Hugging Face only until the owner decides.")117118    static func make(_ name: String, _ display: String, _ family: String, _ publisher: String, _ parameters: Int64,119                     _ license: (String, String, String, String), _ rehost: Rehost) -> CandidateFacts {120        CandidateFacts(repo: "mlx-community/" + name, displayName: display, family: family, publisher: publisher, parameters: parameters,121                       licenseID: license.0, licenseName: license.1, licenseURL: license.2, rehost: rehost, conditions: license.3)122    }123124    static let all: [CandidateFacts] = [125        make("Qwen3-0.6B-4bit", "Qwen3 0.6B", "Qwen3", "Alibaba Cloud (Qwen)", 600_000_000, apache, .yes),126        make("Qwen3-1.7B-4bit", "Qwen3 1.7B", "Qwen3", "Alibaba Cloud (Qwen)", 1_700_000_000, apache, .yes),127        make("Qwen3-4B-4bit", "Qwen3 4B", "Qwen3", "Alibaba Cloud (Qwen)", 4_000_000_000, apache, .yes),128        make("Qwen2.5-0.5B-Instruct-4bit", "Qwen2.5 0.5B Instruct", "Qwen2.5", "Alibaba Cloud (Qwen)", 494_000_000, apache, .yes),129        make("gemma-3-1b-it-qat-4bit", "Gemma 3 1B (QAT)", "Gemma 3", "Google", 1_000_000_000, gemma, .conditional),130        make("gemma-3-4b-it-qat-4bit", "Gemma 3 4B (QAT)", "Gemma 3", "Google", 4_300_000_000, gemma, .conditional),131        make("Llama-3.2-1B-Instruct-4bit", "Llama 3.2 1B Instruct", "Llama 3.2", "Meta", 1_240_000_000, llama, .conditional),132        make("Llama-3.2-3B-Instruct-4bit", "Llama 3.2 3B Instruct", "Llama 3.2", "Meta", 3_210_000_000, llama, .conditional),133        make("Phi-4-mini-instruct-4bit", "Phi-4 mini Instruct", "Phi-4", "Microsoft", 3_840_000_000, mit, .yes),134        make("SmolLM2-135M-Instruct", "SmolLM2 135M Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 135_000_000, apache, .yes),135        make("SmolLM2-135M-Instruct-8bit", "SmolLM2 135M Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 135_000_000, apache, .yes),136        make("SmolLM2-360M-Instruct", "SmolLM2 360M Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 362_000_000, apache, .yes),137        make("SmolLM2-360M-Instruct-6bit", "SmolLM2 360M Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 362_000_000, apache, .yes),138        make("SmolLM2-1.7B-Instruct", "SmolLM2 1.7B Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 1_710_000_000, apache, .yes),139        make("SmolLM2-1.7B-Instruct-4bit", "SmolLM2 1.7B Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 1_710_000_000, apache, .yes),140        make("SmolLM2-360M-Instruct-4bit", "SmolLM2 360M Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 362_000_000, apache, .yes),141    ]142}143144enum VerificationReport {145    static func load() throws -> [String: VerificationResult] {146        let d = JSONDecoder(); d.dateDecodingStrategy = .iso8601147        var out: [String: VerificationResult] = [:]148        for url in (try? FileManager.default.contentsOfDirectory(at: ProjectPaths.results, includingPropertiesForKeys: nil)) ?? [] where url.pathExtension == "json" {149            let r = try d.decode(VerificationResult.self, from: Data(contentsOf: url))150            out[r.id] = r151        }152        return out153    }154155    /// Quantization as the repository's own config states it; a repository without a quantization block holds unquantized weights.156    static func quantization(_ r: VerificationResult) -> String {157        if let bits = r.quantizationBits { return "\(bits)-bit" + (r.id.lowercased().contains("qat") ? " (QAT)" : "") }158        return "none (16-bit float)"159    }160161    /// Why a model that ran is still kept out of the catalog; nil when nothing objects.162    static func exclusion(_ r: VerificationResult, _ facts: CandidateFacts) -> String? {163        guard r.passed else { return r.failure ?? "failed" }164        // The Hub card must agree with what the files are: same family as the config, a license tag consistent with the original model.165        let tag = (r.hubLicense ?? "").lowercased()166        if tag != facts.licenseID {167            return "Provenance unclear: the Hub card declares license “\(r.hubLicense ?? "none")” and base model “\(r.hubBaseModels.joined(separator: ", "))”, which does not match a \(facts.family) model (config model_type “\(r.modelType ?? "?")”)."168        }169        return nil170    }171172    static func card(_ r: VerificationResult, _ facts: CandidateFacts, machine: String) -> LocalModelCard? {173        guard r.passed, exclusion(r, facts) == nil, let revision = r.revision, let rate = r.tokensPerSecond, let ttft = r.timeToFirstToken else { return nil }174        let governor = MemoryGovernor(platform: .iOS)175        let weights = r.files.filter { $0.path.hasSuffix(".safetensors") }.reduce(Int64(0)) { $0 + $1.size }176        let profile = ModelMemoryProfile(weightsBytes: weights, parameterCount: facts.parameters, modelContextLength: r.contextLength)177        let need = weights + governor.kvCacheBytes(profile, context: 4_096) + governor.overheadBytes178        // Smallest common device memory on which the estimate stays inside the governor's 50 % share. An estimate, like the verdict itself.179        let minimum = [4.0, 6, 8, 12, 16, 24, 32].first { Double(need) <= $0 * 1_073_741_824 * governor.physicalShare } ?? 64180        var capabilities: [Capability] = [.text]181        var tasks = ["chat"]182        if r.reasoningVerified == true { capabilities.append(.reasoning); tasks.append("reasoning") }183        if r.toolCallVerified == true { capabilities.append(.tools); tasks.append("tool-use") }184        if r.visionVerified == true { capabilities.append(.vision); tasks.append("vision") }185        var notes = ["Measured on a Mac, not on an iPhone."]186        if r.recalledFact == false { notes.append("Did not recall a fact from the previous turn in the verification chat.") }187        if r.hasVision, r.visionVerified != true { notes.append("The repository contains a vision tower; image input was not verified.") }188        if facts.rehost != .yes { notes.append("Not re-hosted: \(facts.licenseName).") }189        let redistributable = facts.rehost == .yes190        return LocalModelCard(191            id: r.id, displayName: facts.displayName, family: facts.family, publisher: facts.publisher, repo: r.repo, revision: revision,192            architecture: r.modelType ?? "unknown", quantization: quantization(r), parameterCount: facts.parameters,193            licenseID: facts.licenseID, licenseURL: facts.licenseURL, redistributable: redistributable,194            totalBytes: r.totalBytes, files: r.files, contextLength: r.contextLength ?? 2_048, capabilities: capabilities,195            minimumDeviceMemoryGB: minimum, estimatedRuntimeMemoryBytes: need, supportedTasks: tasks,196            verified: VerifiedRecord(date: r.date, machine: machine, platform: "macOS", osVersion: r.device.osVersion, tokensPerSecond: (rate * 10).rounded() / 10,197                                     timeToFirstToken: (ttft * 1000).rounded() / 1000, peakMemoryBytes: r.peakMemoryBytes,198                                     runtime: "mlx-swift-lm 3.33.0, \(r.buildConfiguration ?? "debug") build"),199            notes: notes.joined(separator: " "),200            baseURL: redistributable ? ModelCatalog.defaultBase.absoluteString + "/files/\(r.id)/" : nil,201            fallbackBaseURL: LocalModelCard.huggingFaceBase(repo: r.repo, revision: revision))202    }203204    static func mb(_ bytes: Int64?) -> String { bytes.map { String(format: "%.0f MB", Double($0) / 1_048_576) } ?? "–" }205    static func gb(_ bytes: Int64) -> String { String(format: "%.2f GB", Double(bytes) / 1_073_741_824) }206    static func num(_ v: Double?, _ format: String) -> String { v.map { String(format: format, $0) } ?? "–" }207    static func yesNo(_ v: Bool?) -> String { v == nil ? "not run" : (v! ? "yes" : "no") }208209    /// States plainly how busy the Mac was, because that bounds how far the speed figures can be trusted.210    static func loadNote(_ results: [String: VerificationResult]) -> String {211        let loads = results.values.compactMap(\.loadAverage)212        guard let low = loads.min(), let high = loads.max() else { return "The load of the machine during the run was not recorded." }213        let cores = ProcessInfo.processInfo.activeProcessorCount214        let range = String(format: "%.0f to %.0f", low, high)215        if high > Double(cores) * 1.5 {216            return "**The Mac was busy with other work during this run** (one-minute load average \(range) on \(cores) cores), so the speed and time-to-first-token figures are lower than what the same Mac does when idle. Treat them as conservative; run the script again on a quiet machine for representative figures."217        }218        return "One-minute load average during the run: \(range) on \(cores) cores."219    }220221    static func dates(_ results: [String: VerificationResult]) -> String {222        let all = results.values.filter(\.passed).map(\.date).sorted()223        guard let first = all.first, let last = all.last else { return "?" }224        return first == last ? first : "\(first) to \(last)"225    }226227    static func markdown(results: [String: VerificationResult], cards: [LocalModelCard], machine: String) -> String {228        let any = results.values.first229        var md = """230        # Model compatibility231232        Generated by `Scripts/verify-models.sh` from the records in `build/verify/`. Every number below was measured by running the model;233        nothing is copied from a model card.234235        **All measurements were taken on a Mac: \(machine), macOS \(any?.device.osVersion ?? "?"), with mlx-swift-lm 3.33.0 (\(any?.buildConfiguration ?? "debug") build), on \(dates(results)).236        They are not iPhone measurements.** No iPhone was attached to this Mac; see "Not yet verified on a physical iPhone" below.237238        \(loadNote(results))239240        What "pass" means: the repository's current commit was resolved from the Hugging Face API, every file was downloaded through241        AstraLocal's `DownloadCoordinator` and its SHA-256 recomputed from disk (and matched against the hash the Hub publishes for242        LFS files), the model was loaded by AstraLocal's MLX runtime, answered a two-turn conversation with the second turn reusing243        the KV cache, stopped within 3 s of a mid-stream cancellation with `AstraError.cancelled`, loaded again from disk and answered,244        and completed the fixed four-prompt benchmark. Speed is the median decode rate of the three warm prompts; time to first token245        is the warm median; peak memory is the highest MLX allocator peak during the benchmark (weights included, short prompts).246247        ## Results248249        | Model (repository) | Result | Revision | Quantization | Download | Speed | TTFT warm / cold | Peak memory | In catalog |250        |---|---|---|---|---|---|---|---|---|251252        """253        let order = CandidateFacts.all254        let inCatalog = Set(cards.map(\.id))255        for facts in order {256            guard let r = results[facts.id] else {257                md += "| `\(facts.repo)` | verification in progress | – | – | – | – | – | – | not yet |\n"; continue258            }259            let revision = r.revision.map { "`\($0)`" } ?? "–"260            if r.passed {261                md += "| `\(r.repo)` | **pass** | \(revision) | \(quantization(r)) | \(gb(r.totalBytes)) (\(r.files.count) files) | \(num(r.tokensPerSecond, "%.1f tok/s")) | \(num(r.timeToFirstToken, "%.3f s")) / \(num(r.coldTimeToFirstToken, "%.3f s")) | \(mb(r.peakMemoryBytes)) | \(inCatalog.contains(r.id) ? "yes" : "**no**") |\n"262            } else {263                md += "| `\(r.repo)` | **fail** (\(r.stage)) | \(revision) | \(r.revision == nil ? "–" : quantization(r)) | \(r.totalBytes > 0 ? gb(r.totalBytes) : "–") | – | – | – | no |\n"264            }265        }266        md += "\n## Licenses and re-hosting\n\n`redistributable` in the catalog is `true` only for \"Yes\". The others are downloaded from Hugging Face only, pending the owner's decision.\n\n"267        md += "| Model | License | May the owner re-host the weights? | Conditions (summary, not legal advice) | License tag on the Hub card |\n|---|---|---|---|---|\n"268        for facts in order {269            guard let r = results[facts.id], r.revision != nil else { continue }270            md += "| `\(facts.id)` | [\(facts.licenseName)](\(facts.licenseURL)) | \(facts.rehost.rawValue) | \(facts.conditions) | `\(r.hubLicense ?? "none")` |\n"271        }272        md += "\n## What each repository actually is\n\nRead from the downloaded `config.json` and from the Hub API, not assumed from the name.\n\n"273        md += "| Model | `model_type` | Context (config) | Vision tower | Base model on the Hub card | Recalled the fact in turn 2 | Thinking block | Tool call | Image input | Load time | Weights in memory |\n|---|---|---|---|---|---|---|---|---|---|---|\n"274        for facts in order {275            guard let r = results[facts.id], r.revision != nil, r.modelType != nil else { continue }276            md += "| `\(facts.id)` | `\(r.modelType ?? "?")` | \(r.contextLength.map(String.init) ?? "–") | \(r.hasVision ? "yes" : "no") | \(r.hubBaseModels.joined(separator: ", ")) | \(yesNo(r.recalledFact)) | \(yesNo(r.reasoningVerified)) | \(yesNo(r.toolCallVerified)) | \(yesNo(r.visionVerified)) | \(num(r.loadSeconds, "%.2f s")) | \(mb(r.weightsMemoryBytes)) |\n"277        }278        md += "\n\"not run\" in the three probe columns means the chat template does not declare the feature (or there is no vision tower), so nothing was attempted. \"no\" means it was attempted and did not work.\n"279280        let failures = order.compactMap { facts -> String? in281            guard let r = results[facts.id] else { return nil }282            if !r.passed { return "- `\(r.repo)`: failed at *\(r.stage)*: \(r.failure ?? "unknown")" }283            if let why = exclusion(r, facts) { return "- `\(r.repo)`: ran correctly but is **kept out of the catalog**. \(why)" }284            return nil285        }286        md += "\n## Failures and exclusions\n\n" + (failures.isEmpty ? "None.\n" : failures.joined(separator: "\n") + "\n")287288        md += "\n## Sample output\n\nFirst turn of the verification chat (\"My name is Sam and my favourite number is 42. Greet me by name in one short sentence.\"), temperature 0.\n\n"289        for facts in order {290            guard let r = results[facts.id], let t = r.turn1 else { continue }291            md += "- `\(facts.id)`: \(t.text.trimmingCharacters(in: .whitespacesAndNewlines).replacingOccurrences(of: "\n", with: " ").prefix(200))\n"292        }293294        md += """295296        ## Not yet verified on a physical iPhone297298        Nothing in this document was run on an iPhone or iPad. In particular, the following are unknown until a device run:299300        - Whether each model loads and generates on iOS at all (MLX does not run in the iOS Simulator, so there is no simulator result either).301        - Tokens per second, time to first token and peak memory on any iPhone. The Mac figures above come from a machine with far more memory bandwidth and must not be shown as iPhone figures.302        - Behaviour under the iOS memory limit (jetsam): the `minimumDeviceMemoryGB` and `estimatedRuntimeMemoryBytes` fields of the catalog are estimates computed from the weight size, a KV-cache formula at 4 096 tokens and a fixed overhead. They are not measurements.303        - Thermal behaviour during long generations, and the effect of the thermal policy.304        - Downloads on a cellular connection, and resuming after iOS suspends the app (downloads use a foreground session).305        - Whether the larger models (2 GB and more of weights) are usable on 8 GB devices in practice.306307        ## Apple Foundation Models (system model): verified on this Mac only308309        - **macOS, verified.** On this Mac (macOS 27.0) the live test `appleSystemModelStreamsAMultiTurnAnswer` (`ASTRA_APPLE_FM_TESTS=1`) ran on 2026-11-22: a three-turn conversation replayed through a `Transcript` streamed an answer that recalled a fact from the first turn, ended with one `.messageCompleted`, and the framework reported token usage (86 input, 7 output), which the adapter passed on. Cancellation with the system model was not exercised.310        - **iOS Simulator, failed.** A generation was attempted in the iOS Simulator (iOS 26.5 runtime, Mac host on macOS 27) on 2026-11-21. `SystemLanguageModel.default.availability` reported `.available`, yet the generation failed with `LanguageModelSession.GenerationError` code -1. Availability in the Simulator is therefore not a reliable signal; the app now says so in its error message when an unrecognised failure happens there.311        - **iPhone / iPad, NOT verified.** Nothing was run on an Apple Intelligence-eligible device. It needs one.312313        The `verified` record of each catalog entry says `"platform": "macOS"` for this reason. It should be replaced by an iOS record only after a real run on the device it names.314315        """316        return md317    }318}319320/// `ASTRA_VERIFY_REPORT=1 swift test --filter generateCatalogAndCompatibilityReport`321@Test(.enabled(if: ProjectPaths.env("ASTRA_VERIFY_REPORT") != nil)) func generateCatalogAndCompatibilityReport() throws {322    let results = try VerificationReport.load()323    try #require(!results.isEmpty, "no verification records in build/verify")324    let sample = try #require(results.values.first { $0.passed } ?? results.values.first)325    let machine = "\(sample.device.modelIdentifier) (\(sample.chip ?? "Apple silicon"), \(sample.device.physicalMemoryBytes >> 30) GB)"326    let cards = CandidateFacts.all.compactMap { facts in results[facts.id].flatMap { VerificationReport.card($0, facts, machine: machine) } }327    for card in cards { try card.validate() }328    let catalog = ModelCatalog(generatedAt: Date(timeIntervalSince1970: Date().timeIntervalSince1970.rounded()), entries: cards.map { .init(card: $0, origin: .bundled) })329    let data = try catalog.encoded()330    // what is written must be exactly what the app will accept331    let decoded = try ModelCatalog.decode(data)332    #expect(decoded.models == cards)333    #expect(decoded.models.allSatisfy { $0.verified?.platform == "macOS" && $0.verified?.machine.hasPrefix("Mac") == true })334    try FileManager.default.createDirectory(at: ProjectPaths.catalog.deletingLastPathComponent(), withIntermediateDirectories: true)335    try data.write(to: ProjectPaths.catalog, options: .atomic)336    try Data(VerificationReport.markdown(results: results, cards: cards, machine: machine).utf8).write(to: ProjectPaths.compatibility, options: .atomic)337    FileHandle.standardError.write(Data("[report] \(cards.count) of \(results.count) candidates in the catalog → \(ProjectPaths.catalog.path)\n".utf8))338}339
Files main
Menu

Theme