SPB Git

Repository Public

ModelHubDownloadTests.swift 17.7 KB 261 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatDataTests/ModelHubDownloadTests.swift3// Description: Download manager against a stubbed Hub: install, Range resume after a cut connection, integrity failures, restore after relaunch, pause, cancel, gated repositories.4import Foundation5import LocalChatCore6import Testing7@testable import LocalChatModelHub89@Suite struct ModelHubDownloadTests {10    static let weights = HubFixtures.bytes(900_000, seed: 3)11    static var files: [String: Data] { ["config.json": HubFixtures.mlxConfig, "tokenizer.json": HubFixtures.bytes(40_000, seed: 9), "model.safetensors": weights] }1213    private func ranges(_ base: URL, _ file: String) -> [String?] {14        HubStub.requests(base).filter { $0.url!.path.hasSuffix("/" + file) }.map { $0.value(forHTTPHeaderField: "Range") }15    }1617    @Test func downloadsVerifiesAndInstallsIntoTheLibrary() async throws {18        let paths = HubFixtures.tempPaths(); defer { HubFixtures.cleanup(paths) }19        let base = HubStub.registerFiles(Self.files)20        let library = ModelLibrary(paths: paths)21        let manager = DownloadManager(paths: paths, configuration: HubFixtures.configuration(base), library: library, tokenProvider: { "hf_secret" })22        let request = HubFixtures.mlxRequest(files: Self.files, lfs: ["model.safetensors", "tokenizer.json"])23        try await manager.enqueue(request)24        let state = try #require(await manager.waitUntilSettled(id: request.id))25        #expect(state.phase == .completed)26        #expect(state.filesDone == 3 && state.filesTotal == 3 && state.fraction == 1)2728        let installed = try #require(state.installed)29        #expect(installed.localPath == "acme/tiny-4bit" && installed.revision == "abc123" && installed.addedAt != nil)30        #expect(installed.architecture == "qwen3" && installed.contextLength == 40960 && installed.quantization == "4bit")31        let dir = paths.models.appendingPathComponent("acme/tiny-4bit")32        for (name, data) in Self.files { #expect(try Data(contentsOf: dir.appendingPathComponent(name)) == data) }33        #expect(await library.all().map(\.id) == ["mlx:acme/tiny-4bit"])34        #expect(installed.sizeBytes == Int64(Self.files.values.reduce(0) { $0 + $1.count }))35        // nothing is left in the downloads folder, the revision is in the URL, the token was sent36        #expect(((try? FileManager.default.contentsOfDirectory(atPath: paths.downloads.path)) ?? []).isEmpty)37        let requests = HubStub.requests(base)38        #expect(requests.count == 3 && requests.allSatisfy { $0.url!.path.hasPrefix("/acme/tiny-4bit/resolve/abc123/") })39        #expect(requests.allSatisfy { $0.value(forHTTPHeaderField: "Authorization") == "Bearer hf_secret" })40        #expect(requests.allSatisfy { $0.value(forHTTPHeaderField: "Range") == nil })41    }4243    @Test func resumesWithARangeRequestAfterTheConnectionDrops() async throws {44        let paths = HubFixtures.tempPaths(); defer { HubFixtures.cleanup(paths) }45        let base = HubStub.registerFiles(Self.files) { _, name, attempt, response in46            if name == "model.safetensors" && attempt == 1 { response.cutAfter = 300_000 }47        }48        let manager = DownloadManager(paths: paths, configuration: HubFixtures.configuration(base))49        let request = HubFixtures.mlxRequest(files: Self.files)50        try await manager.enqueue(request)51        let state = try #require(await manager.waitUntilSettled(id: request.id))52        #expect(state.phase == .completed)53        // the second request asked for exactly the missing bytes; the result is byte-identical and passed the SHA-256 check54        #expect(ranges(base, "model.safetensors") == [nil, "bytes=300000-"])55        #expect(try Data(contentsOf: paths.models.appendingPathComponent("acme/tiny-4bit/model.safetensors")) == Self.weights)56    }5758    @Test func corruptedFileFailsVerificationAndIsRemoved() async throws {59        let paths = HubFixtures.tempPaths(); defer { HubFixtures.cleanup(paths) }60        var corrupted = Self.weights; corrupted[123_456] ^= 0xFF   // same size, one byte differs61        var served = Self.files; served["model.safetensors"] = corrupted62        let base = HubStub.registerFiles(served)63        let library = ModelLibrary(paths: paths)64        let manager = DownloadManager(paths: paths, configuration: HubFixtures.configuration(base), library: library)65        let request = HubFixtures.mlxRequest(files: Self.files)   // hashes of the genuine files66        try await manager.enqueue(request)67        let state = try #require(await manager.waitUntilSettled(id: request.id))68        guard case .failed(let reason) = state.phase else { Issue.record("expected a failure, got \(state.phase)"); return }69        #expect(reason.contains("model.safetensors") && reason.contains("integrity"))70        #expect(!FileManager.default.fileExists(atPath: paths.models.appendingPathComponent("acme/tiny-4bit").path))71        #expect(await library.all().isEmpty)72        // the bad bytes are gone and no longer counted; the small files that verified are kept73        let jobDir = paths.downloads.appendingPathComponent(DownloadManager.folderName(request.id)).appendingPathComponent("files")74        #expect(!FileManager.default.fileExists(atPath: jobDir.appendingPathComponent("model.safetensors.partial").path))75        #expect(FileManager.default.fileExists(atPath: jobDir.appendingPathComponent("config.json").path))76        #expect(state.bytesDone == Int64(Self.files["config.json"]!.count + Self.files["tokenizer.json"]!.count))77    }7879    @Test func wrongSizeIsAnIntegrityFailure() async throws {80        let paths = HubFixtures.tempPaths(); defer { HubFixtures.cleanup(paths) }81        var served = Self.files; served["config.json"] = Data("{}".utf8)82        let base = HubStub.registerFiles(served)83        let manager = DownloadManager(paths: paths, configuration: HubFixtures.configuration(base))84        let request = HubFixtures.mlxRequest(files: Self.files)85        try await manager.enqueue(request)86        let state = try #require(await manager.waitUntilSettled(id: request.id))87        guard case .failed(let reason) = state.phase else { Issue.record("expected a failure, got \(state.phase)"); return }88        #expect(reason.contains("config.json"))89    }9091    @Test func survivesARelaunchAndResumesFromThePartialFile() async throws {92        let paths = HubFixtures.tempPaths(); defer { HubFixtures.cleanup(paths) }93        let healthy = HubStub.Counter()94        let base = HubStub.registerFiles(Self.files) { _, name, attempt, response in95            // the connection drops after 200 kB, then the network stays down for the big file until the "relaunch"96            if name == "model.safetensors" && healthy.value("healthy") == 0 { response.cutAfter = attempt == 1 ? 200_000 : 0 }97        }98        var configuration = HubFixtures.configuration(base); configuration.maxRetries = 099        let request = HubFixtures.mlxRequest(files: Self.files)100        do {101            let first = DownloadManager(paths: paths, configuration: configuration)102            try await first.enqueue(request)103            let state = try #require(await first.waitUntilSettled(id: request.id))104            guard case .failed = state.phase else { Issue.record("expected a failure, got \(state.phase)"); return }105        }106        let partial = paths.downloads.appendingPathComponent(DownloadManager.folderName(request.id)).appendingPathComponent("files/model.safetensors.partial")107        let kept = LocalModelInspector.fileSize(partial)108        #expect(kept == 200_000)109110        // "relaunch": a new manager finds the job on disk, with its bytes counted111        _ = healthy.next("healthy")112        let second = DownloadManager(paths: paths, configuration: configuration)113        await second.restore()114        let restored = try #require(await second.state(id: request.id))115        guard case .failed = restored.phase else { Issue.record("expected the failed job back, got \(restored.phase)"); return }116        #expect(restored.bytesDone == kept + Int64(Self.files["config.json"]!.count + Self.files["tokenizer.json"]!.count))117        #expect(restored.filesDone == 2)118        await second.resume(id: request.id)119        let done = try #require(await second.waitUntilSettled(id: request.id))120        #expect(done.phase == .completed)121        #expect(ranges(base, "model.safetensors").last == "bytes=\(kept)-")122        #expect(try Data(contentsOf: paths.models.appendingPathComponent("acme/tiny-4bit/model.safetensors")) == Self.weights)123        // the small files were not fetched again124        #expect(ranges(base, "config.json").count == 1)125    }126127    @Test func pauseKeepsBytesAndCancelDeletesThem() async throws {128        let paths = HubFixtures.tempPaths(); defer { HubFixtures.cleanup(paths) }129        let base = HubStub.registerFiles(Self.files) { _, name, _, response in130            if name == "model.safetensors" { response.chunkSize = 20_000; response.delayPerChunk = 0.01 }   // ~0.45 s for the whole file131        }132        let manager = DownloadManager(paths: paths, configuration: HubFixtures.configuration(base))133        let request = HubFixtures.mlxRequest(files: Self.files)134        var seenSpeed = false135        let watcher = Task { for await states in await manager.updates() { if (states.first?.bytesPerSecond ?? 0) > 0 { return true } }; return false }136        try await manager.enqueue(request)137        // wait until the big file is flowing, then pause138        while (await manager.state(id: request.id)?.bytesDone ?? 0) < 150_000 { try await Task.sleep(for: .milliseconds(5)) }139        try await Task.sleep(for: .milliseconds(120))140        seenSpeed = (await manager.state(id: request.id)?.bytesPerSecond ?? 0) > 0141        await manager.pause(id: request.id)142        let paused = try #require(await manager.state(id: request.id))143        #expect(paused.phase == .paused && paused.bytesPerSecond == nil)144        #expect(seenSpeed, "a measured speed was published while transferring")145        watcher.cancel()146        try await Task.sleep(for: .milliseconds(100))147        let partial = paths.downloads.appendingPathComponent(DownloadManager.folderName(request.id)).appendingPathComponent("files/model.safetensors.partial")148        let kept = LocalModelInspector.fileSize(partial)149        #expect(kept > 0 && kept < Int64(Self.weights.count))150151        await manager.resume(id: request.id)152        let done = try #require(await manager.waitUntilSettled(id: request.id))153        #expect(done.phase == .completed)154        #expect(ranges(base, "model.safetensors").count == 2 && ranges(base, "model.safetensors").last == "bytes=\(kept)-")155        #expect(try Data(contentsOf: paths.models.appendingPathComponent("acme/tiny-4bit/model.safetensors")) == Self.weights)156157        // cancel on a second, slow job removes its folder158        let other = HubFixtures.mlxRequest(repo: "acme/other-4bit", files: Self.files)159        try await manager.enqueue(other)160        while (await manager.state(id: other.id)?.bytesDone ?? 0) < 100_000 { try await Task.sleep(for: .milliseconds(5)) }161        await manager.cancel(id: other.id)162        #expect(await manager.state(id: other.id)?.phase == .cancelled)163        try await Task.sleep(for: .milliseconds(100))164        #expect(!FileManager.default.fileExists(atPath: paths.downloads.appendingPathComponent(DownloadManager.folderName(other.id)).path))165        #expect(!FileManager.default.fileExists(atPath: paths.models.appendingPathComponent("acme/other-4bit").path))166        await manager.clearFinished()167        #expect(await manager.states().isEmpty)168    }169170    @Test func gatedRepositoryFailsWithAClearMessageAndNoRetry() async throws {171        let paths = HubFixtures.tempPaths(); defer { HubFixtures.cleanup(paths) }172        let base = HubStub.register { _ in HubStubResponse(status: 401, headers: ["X-Error-Code": "GatedRepo"]) }173        let manager = DownloadManager(paths: paths, configuration: HubFixtures.configuration(base))174        let request = HubFixtures.mlxRequest(files: Self.files)175        try await manager.enqueue(request)176        let state = try #require(await manager.waitUntilSettled(id: request.id))177        guard case .failed(let reason) = state.phase else { Issue.record("expected a failure, got \(state.phase)"); return }178        #expect(reason.contains("gated") && reason.contains("license") && reason.contains("token"))179        #expect(HubStub.requests(base).count <= 2, "a refusal is not retried")180    }181182    @Test func transientServerErrorsAreRetriedWithBackoff() async throws {183        let paths = HubFixtures.tempPaths(); defer { HubFixtures.cleanup(paths) }184        let base = HubStub.registerFiles(Self.files) { _, name, attempt, response in185            if name == "config.json" && attempt <= 2 { response = HubStubResponse(status: 503) }186        }187        let manager = DownloadManager(paths: paths, configuration: HubFixtures.configuration(base))188        let request = HubFixtures.mlxRequest(files: Self.files)189        try await manager.enqueue(request)190        #expect(await manager.waitUntilSettled(id: request.id)?.phase == .completed)191        #expect(ranges(base, "config.json").count == 3)192    }193194    @Test func refusesWhenStorageIsInsufficientOrTheRequestIsUnsafe() async throws {195        let paths = HubFixtures.tempPaths(); defer { HubFixtures.cleanup(paths) }196        let base = HubStub.registerFiles(Self.files)197        var configuration = HubFixtures.configuration(base); configuration.storageMarginBytes = .max / 4198        let manager = DownloadManager(paths: paths, configuration: configuration)199        let request = HubFixtures.mlxRequest(files: Self.files)200        try await manager.enqueue(request)201        let state = try #require(await manager.waitUntilSettled(id: request.id))202        guard case .failed(let reason) = state.phase else { Issue.record("expected a failure, got \(state.phase)"); return }203        #expect(reason.contains("storage"))204        #expect(HubStub.requests(base).isEmpty, "nothing is fetched when there is no room")205206        var unsafe = request; unsafe.id = "x"; unsafe.files = [DownloadFile(path: "../../escape.txt", size: 1)]207        await #expect(throws: HubError.self) { try await manager.enqueue(unsafe) }208        var empty = request; empty.id = "y"; empty.files = []209        await #expect(throws: HubError.nothingToDownload) { try await manager.enqueue(empty) }210    }211212    @Test func ggufFilesInstallNextToEachOther() async throws {213        let paths = HubFixtures.tempPaths(); defer { HubFixtures.cleanup(paths) }214        let model = HubFixtures.chatGGUF(), projector = HubFixtures.gguf([("general.architecture", .string("clip"))])215        let files = ["tiny-Q4_K_M.gguf": model, "mmproj-tiny-f16.gguf": projector]216        let base = HubStub.registerFiles(files)217        let library = ModelLibrary(paths: paths)218        let manager = DownloadManager(paths: paths, configuration: HubFixtures.configuration(base), library: library)219        let descriptor = ModelDescriptor(id: "gguf:acme/tiny-GGUF:tiny-Q4_K_M.gguf", name: "tiny-Q4_K_M", repoID: "acme/tiny-GGUF", fileName: "tiny-Q4_K_M.gguf",220                                         backend: .gguf, format: .gguf)221        let request = DownloadRequest(descriptor: descriptor, repoID: "acme/tiny-GGUF", revision: "main",222                                      files: files.map { DownloadFile(path: $0.key, size: Int64($0.value.count), sha256: HubFixtures.sha256($0.value)) })223        try await manager.enqueue(request)224        let state = try #require(await manager.waitUntilSettled(id: request.id))225        #expect(state.phase == .completed)226        let installed = try #require(state.installed)227        #expect(installed.localPath == "acme/tiny-GGUF/tiny-Q4_K_M.gguf" && installed.quantization == "Q4_K_M" && installed.contextLength == 40960)228        #expect(installed.capabilities.contains(.vision), "the projector was installed next to the model")229        #expect(FileManager.default.fileExists(atPath: paths.models.appendingPathComponent("acme/tiny-GGUF/mmproj-tiny-f16.gguf").path))230    }231232    /// Real end-to-end download from huggingface.co. Run with LC_LIVE=1.233    @Test(.enabled(if: ProcessInfo.processInfo.environment["LC_LIVE"] == "1"))234    func liveDownloadOfATinyRepository() async throws {235        let paths = HubFixtures.tempPaths(); defer { HubFixtures.cleanup(paths) }236        let api = HuggingFaceAPI()237        let model = try await api.model("TaylorAI/bge-micro-v2")238        let candidate = try #require(ModelMetadataResolver().mlxCandidate(for: model))239        #expect(candidate.descriptor.capabilities == [.embeddings])240        #expect(candidate.downloadBytes > 1_000_000 && candidate.downloadBytes < 40_000_000)241        #expect(candidate.files.contains { $0.sha256 != nil })242        let library = ModelLibrary(paths: paths)243        let manager = DownloadManager(paths: paths, library: library)244        let request = try #require(DownloadRequest(candidate: candidate))245        try await manager.enqueue(request)246        let state = try #require(await manager.waitUntilSettled(id: request.id))247        #expect(state.phase == .completed, "\(state.phase)")248        let installed = try #require(state.installed)249        #expect(installed.sizeBytes == candidate.downloadBytes)250        let dir = try #require(await library.url(for: installed))251        #expect(LocalModelInspector.isMLXModelDirectory(dir))252        let weights = try #require(candidate.files.first { $0.path == "model.safetensors" })253        #expect(try FileHasher.sha256(of: dir.appendingPathComponent("model.safetensors")) == weights.sha256)254        // search works against the live API too255        let page = try await api.search(HubSearchQuery(text: "qwen3", format: .mlx, limit: 5))256        #expect(!page.models.isEmpty && page.models.allSatisfy { $0.tags.contains("mlx") })257        let starters = await StarterCatalog.resolve(api: api)258        #expect(starters.count == StarterCatalog.entries.count, "every starter repository resolves: \(starters.map(\.entry.repoID))")259    }260}261
Files main
Menu

Theme