SPB Git

Repository Public MIT

CatalogStoreGovernorTests.swift 21.7 KB 364 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraLocalTests/CatalogStoreGovernorTests.swift10// Module           : AstraLocalTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90import Testing91@testable import AstraLocal9293private func catalogData(_ cards: [LocalModelCard], schema: Int = 1, generatedAt: String = "2026-11-21T12:00:00Z") throws -> Data {94    let models = try JSONSerialization.jsonObject(with: JSONEncoder().encode(cards))95    return try JSONSerialization.data(withJSONObject: ["schema": schema, "generatedAt": generatedAt, "models": models])96}9798@Suite struct CatalogTests {99    @Test func decodesAValidCatalogAndRoundTrips() throws {100        var card = Fixture.card()101        card.verified = VerifiedRecord(date: "2026-11-21", machine: "Mac17,6", platform: "macOS", osVersion: "27.0", tokensPerSecond: 120, timeToFirstToken: 0.08, peakMemoryBytes: 1 << 30)102        let catalog = try ModelCatalog.decode(try catalogData([card]))103        #expect(catalog.entries.count == 1 && catalog.entries[0].origin == .bundled)104        #expect(catalog.card(id: "tiny") == card)105        let again = try ModelCatalog.decode(try catalog.encoded())106        #expect(again.models == catalog.models && again.generatedAt == catalog.generatedAt)107        #expect(card.sourceURLs.map(\.absoluteString) == [Fixture.mirror, Fixture.hub])108        #expect(Fixture.card(redistributable: false).sourceURLs.map(\.absoluteString) == [Fixture.hub])109        #expect(card.modelInfo().ref == ModelRef(provider: .mlx, modelID: "tiny"))110    }111112    @Test func acceptsFractionalSecondsAndRejectsBadDatesAndSchemas() throws {113        _ = try ModelCatalog.decode(try catalogData([Fixture.card()], generatedAt: "2026-11-21T12:00:00.123Z"))114        #expect(throws: CatalogError.unsupportedSchema(2)) { try ModelCatalog.decode(try catalogData([Fixture.card()], schema: 2)) }115        #expect(throws: CatalogError.self) { try ModelCatalog.decode(try catalogData([Fixture.card()], generatedAt: "yesterday")) }116        #expect(throws: CatalogError.self) { try ModelCatalog.decode(Data("not json".utf8)) }117        #expect(throws: CatalogError.duplicateID("tiny")) { try ModelCatalog.decode(try catalogData([Fixture.card(), Fixture.card()])) }118    }119120    @Test func rejectsUnsafeOrInconsistentCards() throws {121        func invalid(_ mutate: (inout LocalModelCard) -> Void) -> Bool {122            var card = Fixture.card(); mutate(&card)123            do { try card.validate(); return false } catch { return true }124        }125        #expect(invalid { $0.files[1].path = "../tokenizer.json" })126        #expect(invalid { $0.files[1].path = "/abs/tokenizer.json" })127        #expect(invalid { $0.files[1].path = "a/./b" })128        #expect(invalid { $0.files[1].sha256 = "abc" })129        #expect(invalid { $0.files[1].sha256 = String(repeating: "G", count: 64) })130        #expect(invalid { $0.files[1].size = -1 })131        #expect(invalid { $0.files[1].size = ModelPath.maxFileBytes + 1; $0.totalBytes = $0.files.reduce(0) { $0 + $1.size } })132        #expect(invalid { $0.totalBytes += 1 })133        #expect(invalid { $0.revision = "main" })134        #expect(invalid { $0.id = "../x" })135        #expect(invalid { $0.id = "a/b" })136        #expect(invalid { $0.baseURL = "http://mirror.test/x/" })137        #expect(invalid { $0.baseURL = "https://mirror.test/x" })138        #expect(invalid { $0.fallbackBaseURL = "ftp://x/" })139        #expect(invalid { $0.files.removeFirst(); $0.totalBytes = $0.files.reduce(0) { $0 + $1.size } })   // no config.json140        #expect(invalid { $0.files.append($0.files[0]); $0.totalBytes = $0.files.reduce(0) { $0 + $1.size } })141        #expect(invalid { $0.verified = VerifiedRecord(date: "2026-11-21", machine: "x", platform: "iPhone (assumed)", osVersion: "27", tokensPerSecond: 1, timeToFirstToken: 0) })142        #expect(!invalid { _ in })143    }144145    @Test func hashPinningRule() throws {146        let bundled = try ModelCatalog.decode(try catalogData([Fixture.card()]))147148        // same files: accepted, descriptive fields and mirror may change149        var refreshed = Fixture.card(); refreshed.displayName = "Tiny (refreshed)"; refreshed.baseURL = "https://cdn.test/tiny/"150        var online = try ModelCatalog.decode(try catalogData([refreshed], generatedAt: "2026-12-01T00:00:00Z"), origin: .onlineCatalog)151        var merged = bundled.merging(online: online)152        #expect(merged.rejected.isEmpty)153        #expect(merged.catalog.entries[0].origin == .bundledRefreshed && merged.catalog.card(id: "tiny")?.displayName == "Tiny (refreshed)")154        #expect(merged.catalog.generatedAt == online.generatedAt)155156        // a different hash for a known id: rejected, bundled card kept157        var tampered = Fixture.card(); tampered.files[2].sha256 = String(repeating: "b", count: 64); tampered.displayName = "Evil"158        online = try ModelCatalog.decode(try catalogData([tampered]), origin: .onlineCatalog)159        merged = bundled.merging(online: online)160        #expect(merged.rejected.map(\.id) == ["tiny"])161        #expect(merged.catalog.card(id: "tiny") == Fixture.card() && merged.catalog.entries[0].origin == .bundled)162163        // a different revision or size is a different content too164        var moved = Fixture.card(); moved.revision = String(repeating: "f", count: 40)165        #expect(bundled.merging(online: try ModelCatalog.decode(try catalogData([moved]), origin: .onlineCatalog)).rejected.count == 1)166167        // a new id: accepted and labelled168        let fresh = Fixture.card(id: "fresh")169        merged = bundled.merging(online: try ModelCatalog.decode(try catalogData([fresh, Fixture.card()]), origin: .onlineCatalog))170        #expect(merged.catalog.entries.map(\.id) == ["tiny", "fresh"])171        #expect(merged.catalog.entries[1].origin == .onlineCatalog && merged.catalog.entries[1].origin.label == "From the online catalog")172    }173174    @Test func clientFetchesMergesAndSurvivesFailures() async throws {175        let http = FakeHTTP()176        let bundled = try ModelCatalog.decode(try catalogData([Fixture.card()]))177        let client = CatalogClient(transport: http)178        #expect(client.catalogURL.absoluteString == "https://www.orixlab.dev/astra/models/v1/catalog.json")179        var result = await client.refreshed(bundled: bundled)   // offline180        #expect(result.catalog == bundled && result.failure != nil)181        http.respond(client.catalogURL.absoluteString, try catalogData([Fixture.card(), Fixture.card(id: "fresh")]))182        result = await client.refreshed(bundled: bundled)183        #expect(result.failure == nil && result.catalog.entries.count == 2)184        http.respond(client.catalogURL.absoluteString, Data(#"{"schema":1,"generatedAt":"2026-11-21T00:00:00Z","models":[{"id":"x"}]}"#.utf8))185        result = await client.refreshed(bundled: bundled)186        #expect(result.catalog == bundled && result.failure != nil)187        http.respond(client.catalogURL.absoluteString, status: 503, Data())188        #expect(await client.refreshed(bundled: bundled).failure == .server(status: 503, message: "The model catalog is not available right now (HTTP 503)."))189    }190191    @Test func hubResolverReadsCommitFilesAndHashes() async throws {192        let http = FakeHTTP()193        let sha = Fixture.revision194        http.respond("https://huggingface.co/api/models/acme/tiny", Data(#"{"sha":"\#(sha)","tags":["mlx","license:apache-2.0"],"pipeline_tag":"text-generation","cardData":{"license":"apache-2.0","base_model":"acme/base"},"safetensors":{"total":1234}}"#.utf8))195        http.respond("https://huggingface.co/api/models/acme/tiny/tree/\(sha)?recursive=true&limit=1000", Data(#"""196        [{"type":"file","path":"config.json","size":52,"oid":"1111111111111111111111111111111111111111"},197         {"type":"file","path":"README.md","size":900,"oid":"2222222222222222222222222222222222222222"},198         {"type":"file","path":".gitattributes","size":100,"oid":"3333333333333333333333333333333333333333"},199         {"type":"directory","path":"sub","size":0,"oid":"x"},200         {"type":"file","path":"model.safetensors","size":134,"oid":"4444444444444444444444444444444444444444","lfs":{"oid":"\#(String(repeating: "c", count: 64))","size":3000000,"pointerSize":134}}]201        """#.utf8))202        let snapshot = try await HuggingFaceResolver(transport: http).snapshot(repo: "acme/tiny")203        #expect(snapshot.revision == sha && snapshot.license == "apache-2.0" && snapshot.baseModels == ["acme/base"])204        #expect(snapshot.runtimeFiles.map(\.path) == ["config.json", "model.safetensors"])205        let plan = snapshot.plan(id: "tiny")206        #expect(plan.files[1].size == 3_000_000 && plan.files[1].sha256 == String(repeating: "c", count: 64) && plan.files[0].gitBlobSHA1 != nil)207        #expect(plan.sources.map(\.absoluteString) == [Fixture.hub])208        http.respond("https://huggingface.co/api/models/acme/missing", status: 401, Data())209        await #expect(throws: AstraError.local("Hugging Face has no public repository named “acme/missing” (HTTP 401).")) { try await HuggingFaceResolver(transport: http).snapshot(repo: "acme/missing") }210    }211}212213@Suite struct ModelStoreTests {214    let dir = TempDirectory()215216    @Test func listsUsageAndRemovesCompletely() async throws {217        let server = FakeServer(); Fixture.publish(server, base: Fixture.mirror)218        let (coordinator, store) = Fixture.coordinator(dir, server)219        try await coordinator.start(DownloadPlan(card: Fixture.card()))220        _ = await coordinator.waitUntilSettled(id: "tiny")221222        // a second model stopped midway: partial file + download state223        var other = Fixture.contents; other[2].1 = bytes(2_000_000, seed: 8)224        Fixture.publish(server, base: "https://mirror.test/astra/models/v1/other/", files: other)225        server.script("https://mirror.test/astra/models/v1/other/model.safetensors", [.stall(1_000_000)])226        var otherCard = Fixture.card(id: "other", files: other); otherCard.baseURL = "https://mirror.test/astra/models/v1/other/"227        try await coordinator.start(DownloadPlan(card: otherCard))228        let partial = URL(fileURLWithPath: try store.fileURL(modelID: "other", path: "model.safetensors").path + ".partial")229        try await waitFor { (ModelStore.fileSize(partial) ?? 0) >= 1_000_000 }230        await coordinator.pause(id: "other")231232        let models = store.models()233        #expect(models.map(\.id) == ["other", "tiny"])234        #expect(models[0].isComplete == false && models[1].isComplete == true)235        #expect(store.installed().map(\.id) == ["tiny"])236        #expect(models[1].bytesOnDisk >= Fixture.card().totalBytes)237        #expect(models[0].bytesOnDisk >= 1_000_000)238        #expect(store.totalBytesOnDisk() == models[0].bytesOnDisk + models[1].bytesOnDisk)239        #expect(store.loadableDirectory(for: "other") == nil && store.loadableDirectory(for: "tiny") != nil)240241        let freed = try store.remove(id: "other")242        #expect(freed == models[0].bytesOnDisk)243        #expect(!FileManager.default.fileExists(atPath: try store.directory(for: "other").path))244        try store.remove(id: "tiny")245        #expect(store.models().isEmpty && store.totalBytesOnDisk() == 0)246        #expect(try store.remove(id: "tiny") == 0)247    }248249    @Test func installedRequiresEveryFileAtItsDeclaredSize() async throws {250        let server = FakeServer(); Fixture.publish(server, base: Fixture.mirror)251        let (coordinator, store) = Fixture.coordinator(dir, server)252        try await coordinator.start(DownloadPlan(card: Fixture.card()))253        _ = await coordinator.waitUntilSettled(id: "tiny")254        #expect(store.isInstalled("tiny"))255        try FileManager.default.removeItem(at: try store.fileURL(modelID: "tiny", path: "tokenizer.json"))256        #expect(!store.isInstalled("tiny") && store.installed().isEmpty)257    }258259    @Test func orphansAndBadIdentifiers() throws {260        let store = ModelStore(root: dir.url.appendingPathComponent("Models"))261        try store.prepare()262        #expect(store.isExcludedFromBackup())263        let stray = store.root.appendingPathComponent("leftover"); try FileManager.default.createDirectory(at: stray, withIntermediateDirectories: true)264        try bytes(5000).write(to: stray.appendingPathComponent("x.partial"))265        #expect(store.models().isEmpty)266        #expect(store.removeOrphans() >= 5000)267        #expect(store.totalBytesOnDisk() == 0)268        #expect(throws: (any Error).self) { try store.directory(for: "..") }269        #expect(throws: (any Error).self) { try store.directory(for: "a/b") }270        #expect(throws: (any Error).self) { try store.remove(id: "../..") }271        #expect(store.availableDiskBytes() ?? 0 > 0)272    }273274    @Test func streamingDigestsMatchKnownValues() throws {275        let url = dir.url.appendingPathComponent("f.bin")276        try Data("hello\n".utf8).write(to: url)277        #expect(try FileDigest.sha256(of: url) == "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03")278        #expect(try FileDigest.gitBlobSHA1(of: url) == "ce013625030ba8dba906f756967f9e9ca394464a")   // `git hash-object`279        let big = bytes(9_000_000, seed: 4); try big.write(to: url)280        #expect(try FileDigest.sha256(of: url) == sha256Hex(big))281    }282}283284@Suite struct GovernorTests {285    let gb: Int64 = 1 << 30286287    @Test func verdictStatesOnAnEightGigabytePhone() {288        let governor = MemoryGovernor(platform: .iOS)289        let reading = MemoryReading(physicalBytes: 8 * gb, availableBytes: 5 * gb)290        #expect(governor.budget(reading) == 4 * gb)   // min(50 % of 8, 85 % of 5)291        let small = ModelMemoryProfile(weightsBytes: 400 << 20, kvBytesPerToken: 30_000, modelContextLength: 32_768)292        let v = governor.verdict(for: small, context: 4096, reading: reading)293        #expect(v.state == .comfortable && v.isEstimate && v.mayLoad)294        #expect(v.estimatedNeedBytes == (400 << 20) + 30_000 * 4096 + (350 << 20))295        #expect(v.explanation.contains("estimate, not a guarantee") && v.explanation.contains("4096"))296        let medium = ModelMemoryProfile(weightsBytes: 2_500 << 20, kvBytesPerToken: 60_000)297        #expect(governor.verdict(for: medium, context: 4096, reading: reading).state == .tight)298        let large = ModelMemoryProfile(weightsBytes: 4_300 << 20, kvBytesPerToken: 60_000)299        let unlikely = governor.verdict(for: large, context: 4096, reading: reading)300        #expect(unlikely.state == .unlikely && !unlikely.mayLoad)301        let huge = ModelMemoryProfile(weightsBytes: 9 * gb)302        #expect(governor.verdict(for: huge, context: 2048, reading: reading).state == .tooLarge)303    }304305    @Test func lowAvailableMemoryTightensTheBudget() {306        let governor = MemoryGovernor(platform: .iOS)307        let model = ModelMemoryProfile(weightsBytes: 1_800 << 20, kvBytesPerToken: 40_000)308        #expect(governor.verdict(for: model, context: 2048, reading: MemoryReading(physicalBytes: 8 * gb, availableBytes: 5 * gb)).mayLoad)309        #expect(governor.verdict(for: model, context: 2048, reading: MemoryReading(physicalBytes: 8 * gb, availableBytes: 2 * gb)).state == .unlikely)310        // macOS gives no per-process figure: the physical share alone decides311        #expect(MemoryGovernor(platform: .macOS).budget(MemoryReading(physicalBytes: 48 * gb)) == Int64(Double(48 * gb) * 0.7))312    }313314    @Test func recommendedContextShrinksWithMemoryAndRespectsTheModel() {315        let governor = MemoryGovernor(platform: .iOS)316        let model = ModelMemoryProfile(weightsBytes: 2_300 << 20, kvBytesPerToken: 150_000, modelContextLength: 32_768)317        #expect(governor.recommendedMaxContext(for: model, reading: MemoryReading(physicalBytes: 12 * gb, availableBytes: 8 * gb)) == 8_192)   // phone cap318        #expect(governor.recommendedMaxContext(for: model, reading: MemoryReading(physicalBytes: 8 * gb, availableBytes: 4 * gb)) == 4_096)319        #expect(governor.recommendedMaxContext(for: model, reading: MemoryReading(physicalBytes: 6 * gb, availableBytes: 3 * gb)) == nil)320        let short = ModelMemoryProfile(weightsBytes: 300 << 20, kvBytesPerToken: 20_000, modelContextLength: 2_048)321        #expect(governor.recommendedMaxContext(for: short, reading: MemoryReading(physicalBytes: 8 * gb, availableBytes: 5 * gb)) == 2_048)322        #expect(MemoryGovernor(platform: .macOS).recommendedMaxContext(for: model, reading: MemoryReading(physicalBytes: 48 * gb)) == 32_768)323        #expect(governor.defaultContext(reading: MemoryReading(physicalBytes: 6 * gb)) == 2_048)324        #expect(governor.defaultContext(reading: MemoryReading(physicalBytes: 12 * gb)) == 8_192)325    }326327    @Test func kvEstimateWithoutArchitectureNumbersIsBoundedAndGrowsWithSize() {328        let governor = MemoryGovernor(platform: .iOS)329        let small = governor.kvCacheBytes(ModelMemoryProfile(weightsBytes: 300 << 20, parameterCount: 500_000_000), context: 1000)330        let large = governor.kvCacheBytes(ModelMemoryProfile(weightsBytes: 2_300 << 20, parameterCount: 4_000_000_000), context: 1000)331        #expect(small == 35_000 * 1000 && large == 280_000 * 1000)332        #expect(governor.kvCacheBytes(ModelMemoryProfile(weightsBytes: 10, parameterCount: 1), context: 10) == 24_000 * 10)333    }334335    @Test func thermalPolicy() {336        let governor = ThermalGovernor()337        for level in [ThermalLevel.nominal, .fair] {338            let d = governor.decision(for: level)339            #expect(d.allowsNewGeneration && !d.shouldUnload && d.warning == nil)340            #expect(d.limit(maxTokens: 800, context: 8192) == (800, 8192))341            #expect(d.limit(maxTokens: nil, context: nil) == (nil, nil))342        }343        let serious = governor.decision(for: .serious)344        #expect(serious.allowsNewGeneration && !serious.shouldUnload && serious.warning != nil)345        #expect(serious.limit(maxTokens: 800, context: 8192) == (400, 4096))346        #expect(serious.limit(maxTokens: nil, context: nil) == (512, nil))347        #expect(serious.limit(maxTokens: 100, context: 1500) == (64, 1024))348        let critical = governor.decision(for: .critical)349        #expect(!critical.allowsNewGeneration && critical.shouldUnload && critical.warning != nil)350        #expect(ThermalLevel.nominal < .fair && ThermalLevel.serious < .critical)351        #expect(ThermalLevel(.critical) == .critical && ThermalLevel(.nominal) == .nominal)352    }353354    @Test func deviceFactsAreReadFromTheSystem() {355        let facts = DeviceFacts.current()356        #expect(facts.physicalMemoryBytes > 0 && !facts.modelIdentifier.isEmpty && facts.modelIdentifier != "unknown")357        let reading = MemoryReading.current()358        #expect(reading.physicalBytes == facts.physicalMemoryBytes && (reading.footprintBytes ?? 0) > 0)359        #if os(macOS)360        #expect(facts.platform == .macOS && reading.availableBytes == nil)361        #endif362    }363}364
Files main
Menu

Theme