Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Tests/AstraLocalTests/VerificationReport.swift10// Module : AstraLocalTests11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2214//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraLocal9293/// Facts about each candidate that do not come from running it: who made the original model, its nominal size and its license terms.94/// License conditions are summarized from the license texts linked here; they are not legal advice.95struct CandidateFacts {96 enum Rehost: String { case yes = "Yes", conditional = "Conditional", unclear = "Unclear" }97 var repo: String98 var displayName: String99 var family: String100 var publisher: String101 /// Nominal parameter count of the original model, from its publisher's model card.102 var parameters: Int64103 var licenseID: String104 var licenseName: String105 var licenseURL: String106 var rehost: Rehost107 var conditions: String108 var id: String { String(repo.split(separator: "/").last ?? "") }109110 static let apache = ("apache-2.0", "Apache-2.0", "https://www.apache.org/licenses/LICENSE-2.0",111 "Keep the license text and any NOTICE with the files; state changes if the weights are modified.")112 static let mit = ("mit", "MIT", "https://opensource.org/license/mit", "Keep the copyright and permission notice with the files.")113 static let gemma = ("gemma", "Gemma Terms of Use", "https://ai.google.dev/gemma/terms",114 "Redistribution is allowed only if recipients are bound by the same use restrictions (Prohibited Use Policy), receive a copy of the Gemma Terms, and modified files carry a notice. Served from Hugging Face only until the owner decides.")115 static let llama = ("llama3.2", "Llama 3.2 Community License", "https://www.llama.com/llama3_2/license/",116 "Redistribution is allowed only with a copy of the license, the “Built with Llama” attribution shown prominently, the notice file kept, and the Acceptable Use Policy respected. Served from Hugging Face only until the owner decides.")117118 static func make(_ name: String, _ display: String, _ family: String, _ publisher: String, _ parameters: Int64,119 _ license: (String, String, String, String), _ rehost: Rehost) -> CandidateFacts {120 CandidateFacts(repo: "mlx-community/" + name, displayName: display, family: family, publisher: publisher, parameters: parameters,121 licenseID: license.0, licenseName: license.1, licenseURL: license.2, rehost: rehost, conditions: license.3)122 }123124 static let all: [CandidateFacts] = [125 make("Qwen3-0.6B-4bit", "Qwen3 0.6B", "Qwen3", "Alibaba Cloud (Qwen)", 600_000_000, apache, .yes),126 make("Qwen3-1.7B-4bit", "Qwen3 1.7B", "Qwen3", "Alibaba Cloud (Qwen)", 1_700_000_000, apache, .yes),127 make("Qwen3-4B-4bit", "Qwen3 4B", "Qwen3", "Alibaba Cloud (Qwen)", 4_000_000_000, apache, .yes),128 make("Qwen2.5-0.5B-Instruct-4bit", "Qwen2.5 0.5B Instruct", "Qwen2.5", "Alibaba Cloud (Qwen)", 494_000_000, apache, .yes),129 make("gemma-3-1b-it-qat-4bit", "Gemma 3 1B (QAT)", "Gemma 3", "Google", 1_000_000_000, gemma, .conditional),130 make("gemma-3-4b-it-qat-4bit", "Gemma 3 4B (QAT)", "Gemma 3", "Google", 4_300_000_000, gemma, .conditional),131 make("Llama-3.2-1B-Instruct-4bit", "Llama 3.2 1B Instruct", "Llama 3.2", "Meta", 1_240_000_000, llama, .conditional),132 make("Llama-3.2-3B-Instruct-4bit", "Llama 3.2 3B Instruct", "Llama 3.2", "Meta", 3_210_000_000, llama, .conditional),133 make("Phi-4-mini-instruct-4bit", "Phi-4 mini Instruct", "Phi-4", "Microsoft", 3_840_000_000, mit, .yes),134 make("SmolLM2-135M-Instruct", "SmolLM2 135M Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 135_000_000, apache, .yes),135 make("SmolLM2-135M-Instruct-8bit", "SmolLM2 135M Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 135_000_000, apache, .yes),136 make("SmolLM2-360M-Instruct", "SmolLM2 360M Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 362_000_000, apache, .yes),137 make("SmolLM2-360M-Instruct-6bit", "SmolLM2 360M Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 362_000_000, apache, .yes),138 make("SmolLM2-1.7B-Instruct", "SmolLM2 1.7B Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 1_710_000_000, apache, .yes),139 make("SmolLM2-1.7B-Instruct-4bit", "SmolLM2 1.7B Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 1_710_000_000, apache, .yes),140 make("SmolLM2-360M-Instruct-4bit", "SmolLM2 360M Instruct", "SmolLM2", "Hugging Face (HuggingFaceTB)", 362_000_000, apache, .yes),141 ]142}143144enum VerificationReport {145 static func load() throws -> [String: VerificationResult] {146 let d = JSONDecoder(); d.dateDecodingStrategy = .iso8601147 var out: [String: VerificationResult] = [:]148 for url in (try? FileManager.default.contentsOfDirectory(at: ProjectPaths.results, includingPropertiesForKeys: nil)) ?? [] where url.pathExtension == "json" {149 let r = try d.decode(VerificationResult.self, from: Data(contentsOf: url))150 out[r.id] = r151 }152 return out153 }154155 /// Quantization as the repository's own config states it; a repository without a quantization block holds unquantized weights.156 static func quantization(_ r: VerificationResult) -> String {157 if let bits = r.quantizationBits { return "\(bits)-bit" + (r.id.lowercased().contains("qat") ? " (QAT)" : "") }158 return "none (16-bit float)"159 }160161 /// Why a model that ran is still kept out of the catalog; nil when nothing objects.162 static func exclusion(_ r: VerificationResult, _ facts: CandidateFacts) -> String? {163 guard r.passed else { return r.failure ?? "failed" }164 // The Hub card must agree with what the files are: same family as the config, a license tag consistent with the original model.165 let tag = (r.hubLicense ?? "").lowercased()166 if tag != facts.licenseID {167 return "Provenance unclear: the Hub card declares license “\(r.hubLicense ?? "none")” and base model “\(r.hubBaseModels.joined(separator: ", "))”, which does not match a \(facts.family) model (config model_type “\(r.modelType ?? "?")”)."168 }169 return nil170 }171172 static func card(_ r: VerificationResult, _ facts: CandidateFacts, machine: String) -> LocalModelCard? {173 guard r.passed, exclusion(r, facts) == nil, let revision = r.revision, let rate = r.tokensPerSecond, let ttft = r.timeToFirstToken else { return nil }174 let governor = MemoryGovernor(platform: .iOS)175 let weights = r.files.filter { $0.path.hasSuffix(".safetensors") }.reduce(Int64(0)) { $0 + $1.size }176 let profile = ModelMemoryProfile(weightsBytes: weights, parameterCount: facts.parameters, modelContextLength: r.contextLength)177 let need = weights + governor.kvCacheBytes(profile, context: 4_096) + governor.overheadBytes178 // Smallest common device memory on which the estimate stays inside the governor's 50 % share. An estimate, like the verdict itself.179 let minimum = [4.0, 6, 8, 12, 16, 24, 32].first { Double(need) <= $0 * 1_073_741_824 * governor.physicalShare } ?? 64180 var capabilities: [Capability] = [.text]181 var tasks = ["chat"]182 if r.reasoningVerified == true { capabilities.append(.reasoning); tasks.append("reasoning") }183 if r.toolCallVerified == true { capabilities.append(.tools); tasks.append("tool-use") }184 if r.visionVerified == true { capabilities.append(.vision); tasks.append("vision") }185 var notes = ["Measured on a Mac, not on an iPhone."]186 if r.recalledFact == false { notes.append("Did not recall a fact from the previous turn in the verification chat.") }187 if r.hasVision, r.visionVerified != true { notes.append("The repository contains a vision tower; image input was not verified.") }188 if facts.rehost != .yes { notes.append("Not re-hosted: \(facts.licenseName).") }189 let redistributable = facts.rehost == .yes190 return LocalModelCard(191 id: r.id, displayName: facts.displayName, family: facts.family, publisher: facts.publisher, repo: r.repo, revision: revision,192 architecture: r.modelType ?? "unknown", quantization: quantization(r), parameterCount: facts.parameters,193 licenseID: facts.licenseID, licenseURL: facts.licenseURL, redistributable: redistributable,194 totalBytes: r.totalBytes, files: r.files, contextLength: r.contextLength ?? 2_048, capabilities: capabilities,195 minimumDeviceMemoryGB: minimum, estimatedRuntimeMemoryBytes: need, supportedTasks: tasks,196 verified: VerifiedRecord(date: r.date, machine: machine, platform: "macOS", osVersion: r.device.osVersion, tokensPerSecond: (rate * 10).rounded() / 10,197 timeToFirstToken: (ttft * 1000).rounded() / 1000, peakMemoryBytes: r.peakMemoryBytes,198 runtime: "mlx-swift-lm 3.33.0, \(r.buildConfiguration ?? "debug") build"),199 notes: notes.joined(separator: " "),200 baseURL: redistributable ? ModelCatalog.defaultBase.absoluteString + "/files/\(r.id)/" : nil,201 fallbackBaseURL: LocalModelCard.huggingFaceBase(repo: r.repo, revision: revision))202 }203204 static func mb(_ bytes: Int64?) -> String { bytes.map { String(format: "%.0f MB", Double($0) / 1_048_576) } ?? "–" }205 static func gb(_ bytes: Int64) -> String { String(format: "%.2f GB", Double(bytes) / 1_073_741_824) }206 static func num(_ v: Double?, _ format: String) -> String { v.map { String(format: format, $0) } ?? "–" }207 static func yesNo(_ v: Bool?) -> String { v == nil ? "not run" : (v! ? "yes" : "no") }208209 /// States plainly how busy the Mac was, because that bounds how far the speed figures can be trusted.210 static func loadNote(_ results: [String: VerificationResult]) -> String {211 let loads = results.values.compactMap(\.loadAverage)212 guard let low = loads.min(), let high = loads.max() else { return "The load of the machine during the run was not recorded." }213 let cores = ProcessInfo.processInfo.activeProcessorCount214 let range = String(format: "%.0f to %.0f", low, high)215 if high > Double(cores) * 1.5 {216 return "**The Mac was busy with other work during this run** (one-minute load average \(range) on \(cores) cores), so the speed and time-to-first-token figures are lower than what the same Mac does when idle. Treat them as conservative; run the script again on a quiet machine for representative figures."217 }218 return "One-minute load average during the run: \(range) on \(cores) cores."219 }220221 static func dates(_ results: [String: VerificationResult]) -> String {222 let all = results.values.filter(\.passed).map(\.date).sorted()223 guard let first = all.first, let last = all.last else { return "?" }224 return first == last ? first : "\(first) to \(last)"225 }226227 static func markdown(results: [String: VerificationResult], cards: [LocalModelCard], machine: String) -> String {228 let any = results.values.first229 var md = """230 # Model compatibility231232 Generated by `Scripts/verify-models.sh` from the records in `build/verify/`. Every number below was measured by running the model;233 nothing is copied from a model card.234235 **All measurements were taken on a Mac: \(machine), macOS \(any?.device.osVersion ?? "?"), with mlx-swift-lm 3.33.0 (\(any?.buildConfiguration ?? "debug") build), on \(dates(results)).236 They are not iPhone measurements.** No iPhone was attached to this Mac; see "Not yet verified on a physical iPhone" below.237238 \(loadNote(results))239240 What "pass" means: the repository's current commit was resolved from the Hugging Face API, every file was downloaded through241 AstraLocal's `DownloadCoordinator` and its SHA-256 recomputed from disk (and matched against the hash the Hub publishes for242 LFS files), the model was loaded by AstraLocal's MLX runtime, answered a two-turn conversation with the second turn reusing243 the KV cache, stopped within 3 s of a mid-stream cancellation with `AstraError.cancelled`, loaded again from disk and answered,244 and completed the fixed four-prompt benchmark. Speed is the median decode rate of the three warm prompts; time to first token245 is the warm median; peak memory is the highest MLX allocator peak during the benchmark (weights included, short prompts).246247 ## Results248249 | Model (repository) | Result | Revision | Quantization | Download | Speed | TTFT warm / cold | Peak memory | In catalog |250 |---|---|---|---|---|---|---|---|---|251252 """253 let order = CandidateFacts.all254 let inCatalog = Set(cards.map(\.id))255 for facts in order {256 guard let r = results[facts.id] else {257 md += "| `\(facts.repo)` | verification in progress | – | – | – | – | – | – | not yet |\n"; continue258 }259 let revision = r.revision.map { "`\($0)`" } ?? "–"260 if r.passed {261 md += "| `\(r.repo)` | **pass** | \(revision) | \(quantization(r)) | \(gb(r.totalBytes)) (\(r.files.count) files) | \(num(r.tokensPerSecond, "%.1f tok/s")) | \(num(r.timeToFirstToken, "%.3f s")) / \(num(r.coldTimeToFirstToken, "%.3f s")) | \(mb(r.peakMemoryBytes)) | \(inCatalog.contains(r.id) ? "yes" : "**no**") |\n"262 } else {263 md += "| `\(r.repo)` | **fail** (\(r.stage)) | \(revision) | \(r.revision == nil ? "–" : quantization(r)) | \(r.totalBytes > 0 ? gb(r.totalBytes) : "–") | – | – | – | no |\n"264 }265 }266 md += "\n## Licenses and re-hosting\n\n`redistributable` in the catalog is `true` only for \"Yes\". The others are downloaded from Hugging Face only, pending the owner's decision.\n\n"267 md += "| Model | License | May the owner re-host the weights? | Conditions (summary, not legal advice) | License tag on the Hub card |\n|---|---|---|---|---|\n"268 for facts in order {269 guard let r = results[facts.id], r.revision != nil else { continue }270 md += "| `\(facts.id)` | [\(facts.licenseName)](\(facts.licenseURL)) | \(facts.rehost.rawValue) | \(facts.conditions) | `\(r.hubLicense ?? "none")` |\n"271 }272 md += "\n## What each repository actually is\n\nRead from the downloaded `config.json` and from the Hub API, not assumed from the name.\n\n"273 md += "| Model | `model_type` | Context (config) | Vision tower | Base model on the Hub card | Recalled the fact in turn 2 | Thinking block | Tool call | Image input | Load time | Weights in memory |\n|---|---|---|---|---|---|---|---|---|---|---|\n"274 for facts in order {275 guard let r = results[facts.id], r.revision != nil, r.modelType != nil else { continue }276 md += "| `\(facts.id)` | `\(r.modelType ?? "?")` | \(r.contextLength.map(String.init) ?? "–") | \(r.hasVision ? "yes" : "no") | \(r.hubBaseModels.joined(separator: ", ")) | \(yesNo(r.recalledFact)) | \(yesNo(r.reasoningVerified)) | \(yesNo(r.toolCallVerified)) | \(yesNo(r.visionVerified)) | \(num(r.loadSeconds, "%.2f s")) | \(mb(r.weightsMemoryBytes)) |\n"277 }278 md += "\n\"not run\" in the three probe columns means the chat template does not declare the feature (or there is no vision tower), so nothing was attempted. \"no\" means it was attempted and did not work.\n"279280 let failures = order.compactMap { facts -> String? in281 guard let r = results[facts.id] else { return nil }282 if !r.passed { return "- `\(r.repo)`: failed at *\(r.stage)*: \(r.failure ?? "unknown")" }283 if let why = exclusion(r, facts) { return "- `\(r.repo)`: ran correctly but is **kept out of the catalog**. \(why)" }284 return nil285 }286 md += "\n## Failures and exclusions\n\n" + (failures.isEmpty ? "None.\n" : failures.joined(separator: "\n") + "\n")287288 md += "\n## Sample output\n\nFirst turn of the verification chat (\"My name is Sam and my favourite number is 42. Greet me by name in one short sentence.\"), temperature 0.\n\n"289 for facts in order {290 guard let r = results[facts.id], let t = r.turn1 else { continue }291 md += "- `\(facts.id)`: \(t.text.trimmingCharacters(in: .whitespacesAndNewlines).replacingOccurrences(of: "\n", with: " ").prefix(200))\n"292 }293294 md += """295296 ## Not yet verified on a physical iPhone297298 Nothing in this document was run on an iPhone or iPad. In particular, the following are unknown until a device run:299300 - Whether each model loads and generates on iOS at all (MLX does not run in the iOS Simulator, so there is no simulator result either).301 - Tokens per second, time to first token and peak memory on any iPhone. The Mac figures above come from a machine with far more memory bandwidth and must not be shown as iPhone figures.302 - Behaviour under the iOS memory limit (jetsam): the `minimumDeviceMemoryGB` and `estimatedRuntimeMemoryBytes` fields of the catalog are estimates computed from the weight size, a KV-cache formula at 4 096 tokens and a fixed overhead. They are not measurements.303 - Thermal behaviour during long generations, and the effect of the thermal policy.304 - Downloads on a cellular connection, and resuming after iOS suspends the app (downloads use a foreground session).305 - Whether the larger models (2 GB and more of weights) are usable on 8 GB devices in practice.306307 ## Apple Foundation Models (system model): verified on this Mac only308309 - **macOS, verified.** On this Mac (macOS 27.0) the live test `appleSystemModelStreamsAMultiTurnAnswer` (`ASTRA_APPLE_FM_TESTS=1`) ran on 2026-11-22: a three-turn conversation replayed through a `Transcript` streamed an answer that recalled a fact from the first turn, ended with one `.messageCompleted`, and the framework reported token usage (86 input, 7 output), which the adapter passed on. Cancellation with the system model was not exercised.310 - **iOS Simulator, failed.** A generation was attempted in the iOS Simulator (iOS 26.5 runtime, Mac host on macOS 27) on 2026-11-21. `SystemLanguageModel.default.availability` reported `.available`, yet the generation failed with `LanguageModelSession.GenerationError` code -1. Availability in the Simulator is therefore not a reliable signal; the app now says so in its error message when an unrecognised failure happens there.311 - **iPhone / iPad, NOT verified.** Nothing was run on an Apple Intelligence-eligible device. It needs one.312313 The `verified` record of each catalog entry says `"platform": "macOS"` for this reason. It should be replaced by an iOS record only after a real run on the device it names.314315 """316 return md317 }318}319320/// `ASTRA_VERIFY_REPORT=1 swift test --filter generateCatalogAndCompatibilityReport`321@Test(.enabled(if: ProjectPaths.env("ASTRA_VERIFY_REPORT") != nil)) func generateCatalogAndCompatibilityReport() throws {322 let results = try VerificationReport.load()323 try #require(!results.isEmpty, "no verification records in build/verify")324 let sample = try #require(results.values.first { $0.passed } ?? results.values.first)325 let machine = "\(sample.device.modelIdentifier) (\(sample.chip ?? "Apple silicon"), \(sample.device.physicalMemoryBytes >> 30) GB)"326 let cards = CandidateFacts.all.compactMap { facts in results[facts.id].flatMap { VerificationReport.card($0, facts, machine: machine) } }327 for card in cards { try card.validate() }328 let catalog = ModelCatalog(generatedAt: Date(timeIntervalSince1970: Date().timeIntervalSince1970.rounded()), entries: cards.map { .init(card: $0, origin: .bundled) })329 let data = try catalog.encoded()330 // what is written must be exactly what the app will accept331 let decoded = try ModelCatalog.decode(data)332 #expect(decoded.models == cards)333 #expect(decoded.models.allSatisfy { $0.verified?.platform == "macOS" && $0.verified?.machine.hasPrefix("Mac") == true })334 try FileManager.default.createDirectory(at: ProjectPaths.catalog.deletingLastPathComponent(), withIntermediateDirectories: true)335 try data.write(to: ProjectPaths.catalog, options: .atomic)336 try Data(VerificationReport.markdown(results: results, cards: cards, machine: machine).utf8).write(to: ProjectPaths.compatibility, options: .atomic)337 FileHandle.standardError.write(Data("[report] \(cards.count) of \(results.count) candidates in the catalog → \(ProjectPaths.catalog.path)\n".utf8))338}339