Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Tests/AstraLocalTests/CatalogStoreGovernorTests.swift10// Module : AstraLocalTests11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraLocal9293private func catalogData(_ cards: [LocalModelCard], schema: Int = 1, generatedAt: String = "2026-11-21T12:00:00Z") throws -> Data {94 let models = try JSONSerialization.jsonObject(with: JSONEncoder().encode(cards))95 return try JSONSerialization.data(withJSONObject: ["schema": schema, "generatedAt": generatedAt, "models": models])96}9798@Suite struct CatalogTests {99 @Test func decodesAValidCatalogAndRoundTrips() throws {100 var card = Fixture.card()101 card.verified = VerifiedRecord(date: "2026-11-21", machine: "Mac17,6", platform: "macOS", osVersion: "27.0", tokensPerSecond: 120, timeToFirstToken: 0.08, peakMemoryBytes: 1 << 30)102 let catalog = try ModelCatalog.decode(try catalogData([card]))103 #expect(catalog.entries.count == 1 && catalog.entries[0].origin == .bundled)104 #expect(catalog.card(id: "tiny") == card)105 let again = try ModelCatalog.decode(try catalog.encoded())106 #expect(again.models == catalog.models && again.generatedAt == catalog.generatedAt)107 #expect(card.sourceURLs.map(\.absoluteString) == [Fixture.mirror, Fixture.hub])108 #expect(Fixture.card(redistributable: false).sourceURLs.map(\.absoluteString) == [Fixture.hub])109 #expect(card.modelInfo().ref == ModelRef(provider: .mlx, modelID: "tiny"))110 }111112 @Test func acceptsFractionalSecondsAndRejectsBadDatesAndSchemas() throws {113 _ = try ModelCatalog.decode(try catalogData([Fixture.card()], generatedAt: "2026-11-21T12:00:00.123Z"))114 #expect(throws: CatalogError.unsupportedSchema(2)) { try ModelCatalog.decode(try catalogData([Fixture.card()], schema: 2)) }115 #expect(throws: CatalogError.self) { try ModelCatalog.decode(try catalogData([Fixture.card()], generatedAt: "yesterday")) }116 #expect(throws: CatalogError.self) { try ModelCatalog.decode(Data("not json".utf8)) }117 #expect(throws: CatalogError.duplicateID("tiny")) { try ModelCatalog.decode(try catalogData([Fixture.card(), Fixture.card()])) }118 }119120 @Test func rejectsUnsafeOrInconsistentCards() throws {121 func invalid(_ mutate: (inout LocalModelCard) -> Void) -> Bool {122 var card = Fixture.card(); mutate(&card)123 do { try card.validate(); return false } catch { return true }124 }125 #expect(invalid { $0.files[1].path = "../tokenizer.json" })126 #expect(invalid { $0.files[1].path = "/abs/tokenizer.json" })127 #expect(invalid { $0.files[1].path = "a/./b" })128 #expect(invalid { $0.files[1].sha256 = "abc" })129 #expect(invalid { $0.files[1].sha256 = String(repeating: "G", count: 64) })130 #expect(invalid { $0.files[1].size = -1 })131 #expect(invalid { $0.files[1].size = ModelPath.maxFileBytes + 1; $0.totalBytes = $0.files.reduce(0) { $0 + $1.size } })132 #expect(invalid { $0.totalBytes += 1 })133 #expect(invalid { $0.revision = "main" })134 #expect(invalid { $0.id = "../x" })135 #expect(invalid { $0.id = "a/b" })136 #expect(invalid { $0.baseURL = "http://mirror.test/x/" })137 #expect(invalid { $0.baseURL = "https://mirror.test/x" })138 #expect(invalid { $0.fallbackBaseURL = "ftp://x/" })139 #expect(invalid { $0.files.removeFirst(); $0.totalBytes = $0.files.reduce(0) { $0 + $1.size } }) // no config.json140 #expect(invalid { $0.files.append($0.files[0]); $0.totalBytes = $0.files.reduce(0) { $0 + $1.size } })141 #expect(invalid { $0.verified = VerifiedRecord(date: "2026-11-21", machine: "x", platform: "iPhone (assumed)", osVersion: "27", tokensPerSecond: 1, timeToFirstToken: 0) })142 #expect(!invalid { _ in })143 }144145 @Test func hashPinningRule() throws {146 let bundled = try ModelCatalog.decode(try catalogData([Fixture.card()]))147148 // same files: accepted, descriptive fields and mirror may change149 var refreshed = Fixture.card(); refreshed.displayName = "Tiny (refreshed)"; refreshed.baseURL = "https://cdn.test/tiny/"150 var online = try ModelCatalog.decode(try catalogData([refreshed], generatedAt: "2026-12-01T00:00:00Z"), origin: .onlineCatalog)151 var merged = bundled.merging(online: online)152 #expect(merged.rejected.isEmpty)153 #expect(merged.catalog.entries[0].origin == .bundledRefreshed && merged.catalog.card(id: "tiny")?.displayName == "Tiny (refreshed)")154 #expect(merged.catalog.generatedAt == online.generatedAt)155156 // a different hash for a known id: rejected, bundled card kept157 var tampered = Fixture.card(); tampered.files[2].sha256 = String(repeating: "b", count: 64); tampered.displayName = "Evil"158 online = try ModelCatalog.decode(try catalogData([tampered]), origin: .onlineCatalog)159 merged = bundled.merging(online: online)160 #expect(merged.rejected.map(\.id) == ["tiny"])161 #expect(merged.catalog.card(id: "tiny") == Fixture.card() && merged.catalog.entries[0].origin == .bundled)162163 // a different revision or size is a different content too164 var moved = Fixture.card(); moved.revision = String(repeating: "f", count: 40)165 #expect(bundled.merging(online: try ModelCatalog.decode(try catalogData([moved]), origin: .onlineCatalog)).rejected.count == 1)166167 // a new id: accepted and labelled168 let fresh = Fixture.card(id: "fresh")169 merged = bundled.merging(online: try ModelCatalog.decode(try catalogData([fresh, Fixture.card()]), origin: .onlineCatalog))170 #expect(merged.catalog.entries.map(\.id) == ["tiny", "fresh"])171 #expect(merged.catalog.entries[1].origin == .onlineCatalog && merged.catalog.entries[1].origin.label == "From the online catalog")172 }173174 @Test func clientFetchesMergesAndSurvivesFailures() async throws {175 let http = FakeHTTP()176 let bundled = try ModelCatalog.decode(try catalogData([Fixture.card()]))177 let client = CatalogClient(transport: http)178 #expect(client.catalogURL.absoluteString == "https://www.orixlab.dev/astra/models/v1/catalog.json")179 var result = await client.refreshed(bundled: bundled) // offline180 #expect(result.catalog == bundled && result.failure != nil)181 http.respond(client.catalogURL.absoluteString, try catalogData([Fixture.card(), Fixture.card(id: "fresh")]))182 result = await client.refreshed(bundled: bundled)183 #expect(result.failure == nil && result.catalog.entries.count == 2)184 http.respond(client.catalogURL.absoluteString, Data(#"{"schema":1,"generatedAt":"2026-11-21T00:00:00Z","models":[{"id":"x"}]}"#.utf8))185 result = await client.refreshed(bundled: bundled)186 #expect(result.catalog == bundled && result.failure != nil)187 http.respond(client.catalogURL.absoluteString, status: 503, Data())188 #expect(await client.refreshed(bundled: bundled).failure == .server(status: 503, message: "The model catalog is not available right now (HTTP 503)."))189 }190191 @Test func hubResolverReadsCommitFilesAndHashes() async throws {192 let http = FakeHTTP()193 let sha = Fixture.revision194 http.respond("https://huggingface.co/api/models/acme/tiny", Data(#"{"sha":"\#(sha)","tags":["mlx","license:apache-2.0"],"pipeline_tag":"text-generation","cardData":{"license":"apache-2.0","base_model":"acme/base"},"safetensors":{"total":1234}}"#.utf8))195 http.respond("https://huggingface.co/api/models/acme/tiny/tree/\(sha)?recursive=true&limit=1000", Data(#"""196 [{"type":"file","path":"config.json","size":52,"oid":"1111111111111111111111111111111111111111"},197 {"type":"file","path":"README.md","size":900,"oid":"2222222222222222222222222222222222222222"},198 {"type":"file","path":".gitattributes","size":100,"oid":"3333333333333333333333333333333333333333"},199 {"type":"directory","path":"sub","size":0,"oid":"x"},200 {"type":"file","path":"model.safetensors","size":134,"oid":"4444444444444444444444444444444444444444","lfs":{"oid":"\#(String(repeating: "c", count: 64))","size":3000000,"pointerSize":134}}]201 """#.utf8))202 let snapshot = try await HuggingFaceResolver(transport: http).snapshot(repo: "acme/tiny")203 #expect(snapshot.revision == sha && snapshot.license == "apache-2.0" && snapshot.baseModels == ["acme/base"])204 #expect(snapshot.runtimeFiles.map(\.path) == ["config.json", "model.safetensors"])205 let plan = snapshot.plan(id: "tiny")206 #expect(plan.files[1].size == 3_000_000 && plan.files[1].sha256 == String(repeating: "c", count: 64) && plan.files[0].gitBlobSHA1 != nil)207 #expect(plan.sources.map(\.absoluteString) == [Fixture.hub])208 http.respond("https://huggingface.co/api/models/acme/missing", status: 401, Data())209 await #expect(throws: AstraError.local("Hugging Face has no public repository named “acme/missing” (HTTP 401).")) { try await HuggingFaceResolver(transport: http).snapshot(repo: "acme/missing") }210 }211}212213@Suite struct ModelStoreTests {214 let dir = TempDirectory()215216 @Test func listsUsageAndRemovesCompletely() async throws {217 let server = FakeServer(); Fixture.publish(server, base: Fixture.mirror)218 let (coordinator, store) = Fixture.coordinator(dir, server)219 try await coordinator.start(DownloadPlan(card: Fixture.card()))220 _ = await coordinator.waitUntilSettled(id: "tiny")221222 // a second model stopped midway: partial file + download state223 var other = Fixture.contents; other[2].1 = bytes(2_000_000, seed: 8)224 Fixture.publish(server, base: "https://mirror.test/astra/models/v1/other/", files: other)225 server.script("https://mirror.test/astra/models/v1/other/model.safetensors", [.stall(1_000_000)])226 var otherCard = Fixture.card(id: "other", files: other); otherCard.baseURL = "https://mirror.test/astra/models/v1/other/"227 try await coordinator.start(DownloadPlan(card: otherCard))228 let partial = URL(fileURLWithPath: try store.fileURL(modelID: "other", path: "model.safetensors").path + ".partial")229 try await waitFor { (ModelStore.fileSize(partial) ?? 0) >= 1_000_000 }230 await coordinator.pause(id: "other")231232 let models = store.models()233 #expect(models.map(\.id) == ["other", "tiny"])234 #expect(models[0].isComplete == false && models[1].isComplete == true)235 #expect(store.installed().map(\.id) == ["tiny"])236 #expect(models[1].bytesOnDisk >= Fixture.card().totalBytes)237 #expect(models[0].bytesOnDisk >= 1_000_000)238 #expect(store.totalBytesOnDisk() == models[0].bytesOnDisk + models[1].bytesOnDisk)239 #expect(store.loadableDirectory(for: "other") == nil && store.loadableDirectory(for: "tiny") != nil)240241 let freed = try store.remove(id: "other")242 #expect(freed == models[0].bytesOnDisk)243 #expect(!FileManager.default.fileExists(atPath: try store.directory(for: "other").path))244 try store.remove(id: "tiny")245 #expect(store.models().isEmpty && store.totalBytesOnDisk() == 0)246 #expect(try store.remove(id: "tiny") == 0)247 }248249 @Test func installedRequiresEveryFileAtItsDeclaredSize() async throws {250 let server = FakeServer(); Fixture.publish(server, base: Fixture.mirror)251 let (coordinator, store) = Fixture.coordinator(dir, server)252 try await coordinator.start(DownloadPlan(card: Fixture.card()))253 _ = await coordinator.waitUntilSettled(id: "tiny")254 #expect(store.isInstalled("tiny"))255 try FileManager.default.removeItem(at: try store.fileURL(modelID: "tiny", path: "tokenizer.json"))256 #expect(!store.isInstalled("tiny") && store.installed().isEmpty)257 }258259 @Test func orphansAndBadIdentifiers() throws {260 let store = ModelStore(root: dir.url.appendingPathComponent("Models"))261 try store.prepare()262 #expect(store.isExcludedFromBackup())263 let stray = store.root.appendingPathComponent("leftover"); try FileManager.default.createDirectory(at: stray, withIntermediateDirectories: true)264 try bytes(5000).write(to: stray.appendingPathComponent("x.partial"))265 #expect(store.models().isEmpty)266 #expect(store.removeOrphans() >= 5000)267 #expect(store.totalBytesOnDisk() == 0)268 #expect(throws: (any Error).self) { try store.directory(for: "..") }269 #expect(throws: (any Error).self) { try store.directory(for: "a/b") }270 #expect(throws: (any Error).self) { try store.remove(id: "../..") }271 #expect(store.availableDiskBytes() ?? 0 > 0)272 }273274 @Test func streamingDigestsMatchKnownValues() throws {275 let url = dir.url.appendingPathComponent("f.bin")276 try Data("hello\n".utf8).write(to: url)277 #expect(try FileDigest.sha256(of: url) == "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03")278 #expect(try FileDigest.gitBlobSHA1(of: url) == "ce013625030ba8dba906f756967f9e9ca394464a") // `git hash-object`279 let big = bytes(9_000_000, seed: 4); try big.write(to: url)280 #expect(try FileDigest.sha256(of: url) == sha256Hex(big))281 }282}283284@Suite struct GovernorTests {285 let gb: Int64 = 1 << 30286287 @Test func verdictStatesOnAnEightGigabytePhone() {288 let governor = MemoryGovernor(platform: .iOS)289 let reading = MemoryReading(physicalBytes: 8 * gb, availableBytes: 5 * gb)290 #expect(governor.budget(reading) == 4 * gb) // min(50 % of 8, 85 % of 5)291 let small = ModelMemoryProfile(weightsBytes: 400 << 20, kvBytesPerToken: 30_000, modelContextLength: 32_768)292 let v = governor.verdict(for: small, context: 4096, reading: reading)293 #expect(v.state == .comfortable && v.isEstimate && v.mayLoad)294 #expect(v.estimatedNeedBytes == (400 << 20) + 30_000 * 4096 + (350 << 20))295 #expect(v.explanation.contains("estimate, not a guarantee") && v.explanation.contains("4096"))296 let medium = ModelMemoryProfile(weightsBytes: 2_500 << 20, kvBytesPerToken: 60_000)297 #expect(governor.verdict(for: medium, context: 4096, reading: reading).state == .tight)298 let large = ModelMemoryProfile(weightsBytes: 4_300 << 20, kvBytesPerToken: 60_000)299 let unlikely = governor.verdict(for: large, context: 4096, reading: reading)300 #expect(unlikely.state == .unlikely && !unlikely.mayLoad)301 let huge = ModelMemoryProfile(weightsBytes: 9 * gb)302 #expect(governor.verdict(for: huge, context: 2048, reading: reading).state == .tooLarge)303 }304305 @Test func lowAvailableMemoryTightensTheBudget() {306 let governor = MemoryGovernor(platform: .iOS)307 let model = ModelMemoryProfile(weightsBytes: 1_800 << 20, kvBytesPerToken: 40_000)308 #expect(governor.verdict(for: model, context: 2048, reading: MemoryReading(physicalBytes: 8 * gb, availableBytes: 5 * gb)).mayLoad)309 #expect(governor.verdict(for: model, context: 2048, reading: MemoryReading(physicalBytes: 8 * gb, availableBytes: 2 * gb)).state == .unlikely)310 // macOS gives no per-process figure: the physical share alone decides311 #expect(MemoryGovernor(platform: .macOS).budget(MemoryReading(physicalBytes: 48 * gb)) == Int64(Double(48 * gb) * 0.7))312 }313314 @Test func recommendedContextShrinksWithMemoryAndRespectsTheModel() {315 let governor = MemoryGovernor(platform: .iOS)316 let model = ModelMemoryProfile(weightsBytes: 2_300 << 20, kvBytesPerToken: 150_000, modelContextLength: 32_768)317 #expect(governor.recommendedMaxContext(for: model, reading: MemoryReading(physicalBytes: 12 * gb, availableBytes: 8 * gb)) == 8_192) // phone cap318 #expect(governor.recommendedMaxContext(for: model, reading: MemoryReading(physicalBytes: 8 * gb, availableBytes: 4 * gb)) == 4_096)319 #expect(governor.recommendedMaxContext(for: model, reading: MemoryReading(physicalBytes: 6 * gb, availableBytes: 3 * gb)) == nil)320 let short = ModelMemoryProfile(weightsBytes: 300 << 20, kvBytesPerToken: 20_000, modelContextLength: 2_048)321 #expect(governor.recommendedMaxContext(for: short, reading: MemoryReading(physicalBytes: 8 * gb, availableBytes: 5 * gb)) == 2_048)322 #expect(MemoryGovernor(platform: .macOS).recommendedMaxContext(for: model, reading: MemoryReading(physicalBytes: 48 * gb)) == 32_768)323 #expect(governor.defaultContext(reading: MemoryReading(physicalBytes: 6 * gb)) == 2_048)324 #expect(governor.defaultContext(reading: MemoryReading(physicalBytes: 12 * gb)) == 8_192)325 }326327 @Test func kvEstimateWithoutArchitectureNumbersIsBoundedAndGrowsWithSize() {328 let governor = MemoryGovernor(platform: .iOS)329 let small = governor.kvCacheBytes(ModelMemoryProfile(weightsBytes: 300 << 20, parameterCount: 500_000_000), context: 1000)330 let large = governor.kvCacheBytes(ModelMemoryProfile(weightsBytes: 2_300 << 20, parameterCount: 4_000_000_000), context: 1000)331 #expect(small == 35_000 * 1000 && large == 280_000 * 1000)332 #expect(governor.kvCacheBytes(ModelMemoryProfile(weightsBytes: 10, parameterCount: 1), context: 10) == 24_000 * 10)333 }334335 @Test func thermalPolicy() {336 let governor = ThermalGovernor()337 for level in [ThermalLevel.nominal, .fair] {338 let d = governor.decision(for: level)339 #expect(d.allowsNewGeneration && !d.shouldUnload && d.warning == nil)340 #expect(d.limit(maxTokens: 800, context: 8192) == (800, 8192))341 #expect(d.limit(maxTokens: nil, context: nil) == (nil, nil))342 }343 let serious = governor.decision(for: .serious)344 #expect(serious.allowsNewGeneration && !serious.shouldUnload && serious.warning != nil)345 #expect(serious.limit(maxTokens: 800, context: 8192) == (400, 4096))346 #expect(serious.limit(maxTokens: nil, context: nil) == (512, nil))347 #expect(serious.limit(maxTokens: 100, context: 1500) == (64, 1024))348 let critical = governor.decision(for: .critical)349 #expect(!critical.allowsNewGeneration && critical.shouldUnload && critical.warning != nil)350 #expect(ThermalLevel.nominal < .fair && ThermalLevel.serious < .critical)351 #expect(ThermalLevel(.critical) == .critical && ThermalLevel(.nominal) == .nominal)352 }353354 @Test func deviceFactsAreReadFromTheSystem() {355 let facts = DeviceFacts.current()356 #expect(facts.physicalMemoryBytes > 0 && !facts.modelIdentifier.isEmpty && facts.modelIdentifier != "unknown")357 let reading = MemoryReading.current()358 #expect(reading.physicalBytes == facts.physicalMemoryBytes && (reading.footprintBytes ?? 0) > 0)359 #if os(macOS)360 #expect(facts.platform == .macOS && reading.availableBytes == nil)361 #endif362 }363}364