SPB Git

Repository Public

DownloadManager.swift 25.9 KB 482 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatModelHub/Sources/DownloadManager.swift3// Description: Resumable, verified model downloads: persisted jobs, HTTP Range resume, SHA-256 checks, measured speed, atomic install into the models folder.4import CryptoKit5import Foundation6import LocalChatCore78public struct DownloadFile: Codable, Sendable, Hashable {9    public var path: String10    public var size: Int64?11    public var sha256: String?12    public init(path: String, size: Int64? = nil, sha256: String? = nil) { self.path = path; self.size = size; self.sha256 = sha256 }13}1415public struct DownloadRequest: Codable, Sendable, Hashable, Identifiable {16    public var id: String                 // the descriptor id17    public var descriptor: ModelDescriptor18    public var repoID: String19    public var revision: String           // a commit hash when known, so a resumed download never mixes two versions20    public var files: [DownloadFile]2122    public init(descriptor: ModelDescriptor, repoID: String, revision: String, files: [DownloadFile]) {23        id = descriptor.id; self.descriptor = descriptor; self.repoID = repoID; self.revision = revision; self.files = files24    }25    public init?(candidate: ModelCandidate) {26        guard let repo = candidate.descriptor.repoID, !candidate.files.isEmpty else { return nil }27        self.init(descriptor: candidate.descriptor, repoID: repo, revision: candidate.descriptor.revision ?? "main",28                  files: candidate.files.map { DownloadFile(path: $0.path, size: $0.size, sha256: $0.sha256) })29    }30    public var totalBytes: Int64 { files.reduce(0) { $0 + ($1.size ?? 0) } }31}3233public struct DownloadState: Sendable, Hashable, Identifiable {34    public enum Phase: Sendable, Hashable {35        case queued, downloading, verifying, paused, failed(String), completed, cancelled36        public var isActive: Bool { self == .queued || self == .downloading || self == .verifying }37        public var isFinished: Bool { self == .completed || self == .cancelled }38    }39    public var id: String40    public var name: String41    public var phase: Phase42    public var bytesDone: Int6443    public var bytesTotal: Int6444    public var bytesPerSecond: Double?    // measured over the last few seconds; nil when not transferring45    public var eta: TimeInterval?46    public var currentFile: String?47    public var filesDone: Int48    public var filesTotal: Int49    public var installed: ModelDescriptor?   // set once the model is in the models folder50    public var fraction: Double { bytesTotal > 0 ? min(1, Double(bytesDone) / Double(bytesTotal)) : (phase == .completed ? 1 : 0) }51}5253public actor DownloadManager {54    public struct Configuration: @unchecked Sendable {55        public var baseURL = URL(string: "https://huggingface.co")!56        public var maxConcurrentJobs = 157        public var maxConcurrentFiles = 258        public var maxRetries = 4                       // consecutive failed attempts per file without progress59        public var retryBaseDelay: TimeInterval = 1.5   // doubled at every attempt60        public var storageMarginBytes: Int64 = 512 * 1_048_57661        public var publishInterval: TimeInterval = 0.562        public var speedWindow: TimeInterval = 563        /// Set on iOS to keep transferring while the app is suspended. Files are then fetched in Range segments of `backgroundSegmentBytes`.64        public var backgroundSessionIdentifier: String?65        public var backgroundSegmentBytes: Int64 = 64 * 1_048_57666        public var protocolClasses: [AnyClass]?         // URLProtocol stubs for tests67        public init() {}68    }6970    private struct Manifest: Codable { var request: DownloadRequest; var paused: Bool; var failure: String? }7172    private struct Job {73        var request: DownloadRequest74        var phase: DownloadState.Phase75        var counter = ByteCounter()76        var task: Task<Void, Never>?77        var samples: [(time: Date, bytes: Int64)] = []78        var speed: Double?79        var transferring: Set<String> = []80        var verifying: Set<String> = []81        var filesDone = 082        var installed: ModelDescriptor?83        var order: Int84        var lastRun: Task<Void, Never>? // the previous run; a new run waits for it so two runs never write the same partial file85        var generation = 0      // bumped at every start, so a late callback of a cancelled run is ignored86    }8788    public let paths: AppPaths89    public let configuration: Configuration90    private let tokenProvider: HuggingFaceAPI.TokenProvider91    private let library: ModelLibrary?92    private let driver: TransferDriver93    private var jobs: [String: Job] = [:]94    private var nextOrder = 095    private var observers: [UUID: AsyncStream<[DownloadState]>.Continuation] = [:]96    private var ticker: Task<Void, Never>?97    private var restored = false9899    public init(paths: AppPaths, configuration: Configuration = .init(), library: ModelLibrary? = nil,100                tokenProvider: @escaping HuggingFaceAPI.TokenProvider = { nil }) {101        self.paths = paths; self.configuration = configuration; self.library = library; self.tokenProvider = tokenProvider102        driver = TransferDriver(backgroundIdentifier: configuration.backgroundSessionIdentifier, protocolClasses: configuration.protocolClasses)103    }104105    deinit { driver.invalidate() }106107    // MARK: layout108109    static func folderName(_ id: String) -> String {110        let safe = String(id.map { $0.isLetter || $0.isNumber || $0 == "-" || $0 == "." ? $0 : "_" }.suffix(60))111        let hash = SHA256.hash(data: Data(id.utf8)).prefix(4).map { String(format: "%02x", $0) }.joined()112        return "\(safe)-\(hash)"113    }114    private func jobDirectory(_ id: String) -> URL { paths.downloads.appendingPathComponent(Self.folderName(id), isDirectory: true) }115    private static func staged(_ dir: URL, _ path: String) -> URL { dir.appendingPathComponent("files").appendingPathComponent(path) }116    private static func partial(_ dir: URL, _ path: String) -> URL { dir.appendingPathComponent("files").appendingPathComponent(path + ".partial") }117118    /// Where a model ends up, relative to the models folder.119    public static func installPath(for request: DownloadRequest) -> String {120        request.descriptor.format == .gguf ? "\(request.repoID)/\(request.descriptor.fileName ?? request.files.first?.path ?? "model.gguf")" : request.repoID121    }122123    private func saveManifest(_ job: Job) {124        var failure: String?125        if case .failed(let reason) = job.phase { failure = reason }126        let manifest = Manifest(request: job.request, paused: job.phase == .paused || failure != nil, failure: failure)127        let dir = jobDirectory(job.request.id)128        try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)129        if let data = try? CoreCoding.encoder.encode(manifest) { try? data.write(to: dir.appendingPathComponent("job.json"), options: .atomic) }130    }131132    /// Bytes already on disk for a job (verified files + partial files).133    private static func bytesOnDisk(_ request: DownloadRequest, dir: URL) -> (bytes: Int64, files: Int) {134        var bytes: Int64 = 0; var done = 0135        for f in request.files {136            let final = staged(dir, f.path)137            if FileManager.default.fileExists(atPath: final.path) { bytes += LocalModelInspector.fileSize(final); done += 1 }138            else { bytes += LocalModelInspector.fileSize(partial(dir, f.path)) }139        }140        return (bytes, done)141    }142143    // MARK: observation144145    public func states() -> [DownloadState] {146        jobs.values.sorted { $0.order < $1.order }.map { job in147            let done = job.counter.current, total = max(job.request.totalBytes, job.phase == .completed ? done : 0)148            let speed = job.phase == .downloading ? job.speed : nil149            let eta = speed.flatMap { $0 > 1 && total > done ? Double(total - done) / $0 : nil }150            return DownloadState(id: job.request.id, name: job.request.descriptor.name, phase: job.phase, bytesDone: min(done, max(total, done)),151                                 bytesTotal: total, bytesPerSecond: speed, eta: eta, currentFile: job.transferring.sorted().first ?? job.verifying.sorted().first,152                                 filesDone: job.filesDone, filesTotal: job.request.files.count, installed: job.installed)153        }154    }155156    public func state(id: String) -> DownloadState? { states().first { $0.id == id } }157158    /// Emits every job's state now, on every change, and twice a second while something is transferring.159    public func updates() -> AsyncStream<[DownloadState]> {160        let (stream, continuation) = AsyncStream<[DownloadState]>.makeStream(bufferingPolicy: .bufferingNewest(1))161        let key = UUID()162        observers[key] = continuation163        continuation.onTermination = { [weak self] _ in Task { await self?.removeObserver(key) } }164        continuation.yield(states())165        return stream166    }167    private func removeObserver(_ key: UUID) { observers[key] = nil }168    private func publish() { let s = states(); for o in observers.values { o.yield(s) } }169170    /// Suspends until the job is no longer active and returns its last state (nil when the id is unknown).171    public func waitUntilSettled(id: String) async -> DownloadState? {172        while let s = state(id: id), s.phase.isActive { try? await Task.sleep(for: .milliseconds(20)) }173        return state(id: id)174    }175176    /// iOS: forward the handler from `application(_:handleEventsForBackgroundURLSession:completionHandler:)`.177    public func setBackgroundCompletionHandler(_ handler: @escaping @Sendable () -> Void) { driver.setBackgroundCompletion(handler) }178179    // MARK: commands180181    /// Reloads the jobs left by a previous launch. They come back paused (or failed), with their bytes on disk counted; call `resume` to continue.182    public func restore() {183        guard !restored else { return }184        restored = true185        let fm = FileManager.default186        for name in ((try? fm.contentsOfDirectory(atPath: paths.downloads.path)) ?? []).sorted() {187            let dir = paths.downloads.appendingPathComponent(name)188            guard let data = try? Data(contentsOf: dir.appendingPathComponent("job.json")),189                  let manifest = try? CoreCoding.decoder.decode(Manifest.self, from: data), jobs[manifest.request.id] == nil else { continue }190            var job = Job(request: manifest.request, phase: manifest.failure.map { .failed($0) } ?? .paused, order: nextOrder)191            nextOrder += 1192            let disk = Self.bytesOnDisk(manifest.request, dir: dir)193            job.counter.set(disk.bytes); job.filesDone = disk.files194            jobs[manifest.request.id] = job195        }196        publish()197    }198199    /// Queues a download. Enqueueing an id that is paused or failed resumes it; an active one is left alone.200    public func enqueue(_ request: DownloadRequest) throws {201        restore()202        guard !request.files.isEmpty else { throw HubError.nothingToDownload }203        guard HuggingFaceAPI.isValidRepositoryID(request.repoID) else { throw HubError.invalidRepository(request.repoID) }204        for f in request.files where f.path.hasPrefix("/") || f.path.split(separator: "/").contains("..") { throw HubError.invalidResponse("unsafe file path \(f.path)") }205        if let existing = jobs[request.id] {206            if existing.phase.isActive { return }207            // a new revision or file list restarts from scratch: partial bytes of another version must never be reused208            if existing.request.revision != request.revision || existing.request.files != request.files {209                try? FileManager.default.removeItem(at: jobDirectory(request.id))210            }211        }212        var job = Job(request: request, phase: .queued, order: jobs[request.id]?.order ?? nextOrder)213        nextOrder += 1214        job.generation = jobs[request.id]?.generation ?? 0; job.lastRun = jobs[request.id]?.lastRun215        let disk = Self.bytesOnDisk(request, dir: jobDirectory(request.id))216        job.counter.set(disk.bytes); job.filesDone = disk.files217        jobs[request.id] = job218        saveManifest(job)219        pump()220    }221222    public func pause(id: String) {223        guard var job = jobs[id], job.phase.isActive else { return }224        job.task?.cancel(); job.task = nil225        job.phase = .paused; job.transferring = []; job.verifying = []; job.speed = nil; job.samples = []226        jobs[id] = job227        saveManifest(job)228        pump()229    }230231    public func resume(id: String) {232        restore()233        guard var job = jobs[id], !job.phase.isActive, job.phase != .completed else { return }234        job.phase = .queued235        let disk = Self.bytesOnDisk(job.request, dir: jobDirectory(id))236        job.counter.set(disk.bytes); job.filesDone = disk.files237        jobs[id] = job238        saveManifest(job)239        pump()240    }241242    /// Stops the job and deletes everything it downloaded so far.243    public func cancel(id: String) {244        guard var job = jobs[id], job.phase != .completed else { return }245        job.task?.cancel(); job.task = nil246        job.phase = .cancelled; job.transferring = []; job.verifying = []; job.speed = nil247        job.counter.set(0); job.filesDone = 0248        jobs[id] = job249        try? FileManager.default.removeItem(at: jobDirectory(id))250        pump()251    }252253    /// Forgets completed and cancelled jobs.254    public func clearFinished() {255        for (id, job) in jobs where job.phase.isFinished { jobs[id] = nil }256        publish()257    }258259    // MARK: scheduling260261    private func pump() {262        var running = jobs.values.filter { $0.task != nil }.count263        for id in jobs.values.filter({ $0.phase == .queued && $0.task == nil }).sorted(by: { $0.order < $1.order }).map(\.request.id) {264            guard running < max(1, configuration.maxConcurrentJobs) else { break }265            start(id); running += 1266        }267        publish()268        if ticker == nil, jobs.values.contains(where: { $0.task != nil }) {269            ticker = Task { [weak self] in270                while !Task.isCancelled {271                    guard let self else { return }272                    try? await Task.sleep(for: .seconds(self.configuration.publishInterval))273                    if await !self.tick() { return }274                }275            }276        }277    }278279    /// Samples the counters, recomputes speeds and publishes. Returns false when nothing is running any more.280    private func tick() -> Bool {281        let now = Date()282        for (id, var job) in jobs where job.task != nil {283            job.samples.append((now, job.counter.current))284            job.samples.removeAll { now.timeIntervalSince($0.time) > configuration.speedWindow }285            if let first = job.samples.first, let last = job.samples.last, last.time.timeIntervalSince(first.time) >= configuration.publishInterval * 0.9 {286                job.speed = max(0, Double(last.bytes - first.bytes) / last.time.timeIntervalSince(first.time))287            }288            jobs[id] = job289        }290        publish()291        if jobs.values.contains(where: { $0.task != nil }) { return true }292        ticker = nil293        return false294    }295296    private func start(_ id: String) {297        guard var job = jobs[id] else { return }298        job.phase = .downloading; job.samples = []; job.speed = nil; job.generation += 1299        let generation = job.generation300        let request = job.request, dir = jobDirectory(id), counter = job.counter301        let driver = driver, configuration = configuration, tokenProvider = tokenProvider, modelsRoot = paths.models302        let previous = job.lastRun303        job.task = Task { [weak self] in304            await previous?.value305            do {306                let token = await tokenProvider()307                try await Self.fetchAll(request: request, dir: dir, driver: driver, configuration: configuration, token: token, counter: counter) { [weak self] event in308                    await self?.note(id, generation, event)309                }310                try Task.checkCancellation()311                let relative = try Self.install(request: request, dir: dir, modelsRoot: modelsRoot)312                await self?.finish(id, generation, relativePath: relative, error: nil)313            } catch {314                await self?.finish(id, generation, relativePath: nil, error: error)315            }316        }317        job.lastRun = job.task318        jobs[id] = job319    }320321    enum FileEvent: Sendable { case transferring(String), verifying(String), done(String), released(String) }322323    private func note(_ id: String, _ generation: Int, _ event: FileEvent) {324        guard var job = jobs[id], job.task != nil, job.generation == generation else { return }325        switch event {326        case .transferring(let f): job.transferring.insert(f); job.verifying.remove(f)327        case .verifying(let f): job.transferring.remove(f); job.verifying.insert(f)328        case .done(let f): job.transferring.remove(f); job.verifying.remove(f); job.filesDone += 1329        case .released(let f): job.transferring.remove(f); job.verifying.remove(f)330        }331        job.phase = job.transferring.isEmpty && !job.verifying.isEmpty ? .verifying : .downloading332        jobs[id] = job333        publish()334    }335336    private func finish(_ id: String, _ generation: Int, relativePath: String?, error: Error?) async {337        guard var job = jobs[id], job.task != nil, job.generation == generation else { return }   // paused or cancelled meanwhile: that command already set the state338        job.task = nil; job.transferring = []; job.verifying = []; job.speed = nil; job.samples = []339        if let relativePath {340            var descriptor = job.request.descriptor341            descriptor.localPath = relativePath; descriptor.revision = job.request.revision; descriptor.addedAt = Date()342            if let library { descriptor = (try? await library.register(descriptor)) ?? descriptor }343            // the library call suspended this actor: only complete the job if nobody replaced it meanwhile344            guard jobs[id]?.generation == generation, jobs[id]?.task != nil else { return }345            job.installed = descriptor; job.phase = .completed; job.filesDone = job.request.files.count346            if let size = descriptor.sizeBytes { job.counter.set(max(size, job.request.totalBytes)) }347            try? FileManager.default.removeItem(at: jobDirectory(id))348            jobs[id] = job349        } else {350            let hubError = error as? HubError351            if error is CancellationError || hubError == .cancelled { job.phase = .paused }352            else { job.phase = .failed(hubError?.errorDescription ?? error?.localizedDescription ?? "Unknown error") }353            let disk = Self.bytesOnDisk(job.request, dir: jobDirectory(id))354            job.counter.set(disk.bytes); job.filesDone = disk.files355            jobs[id] = job356            saveManifest(job)357        }358        pump()359    }360361    // MARK: transfer (runs outside the actor)362363    private static func fetchAll(request: DownloadRequest, dir: URL, driver: TransferDriver, configuration: Configuration, token: String?,364                                 counter: ByteCounter, event: @escaping @Sendable (FileEvent) async -> Void) async throws {365        let fm = FileManager.default366        try fm.createDirectory(at: dir.appendingPathComponent("files"), withIntermediateDirectories: true)367        // free space: what is still to fetch, plus a margin (the install is a move on the same volume, it needs no extra room)368        let remaining = max(0, request.totalBytes - counter.current)369        if let free = try? dir.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]).volumeAvailableCapacityForImportantUsage,370           free < remaining + configuration.storageMarginBytes {371            throw HubError.insufficientStorage(needed: remaining + configuration.storageMarginBytes, available: free)372        }373        // small files first: a broken repository or a gated one fails in the first second, not after gigabytes374        var pending = request.files.sorted { ($0.size ?? .max, $0.path) < ($1.size ?? .max, $1.path) }[...]375        try await withThrowingTaskGroup(of: Void.self) { group in376            var active = 0377            func launch(_ file: DownloadFile) {378                active += 1379                group.addTask {380                    do { try await fetchFile(file, request: request, dir: dir, driver: driver, configuration: configuration, token: token, counter: counter, event: event) }381                    catch { await event(.released(file.path)); throw error }382                }383            }384            while active < max(1, configuration.maxConcurrentFiles), let file = pending.popFirst() { launch(file) }385            while active > 0 {386                try await group.next()387                active -= 1388                if let file = pending.popFirst() { launch(file) }389            }390        }391    }392393    private static func fetchFile(_ file: DownloadFile, request: DownloadRequest, dir: URL, driver: TransferDriver, configuration: Configuration,394                                  token: String?, counter: ByteCounter, event: @escaping @Sendable (FileEvent) async -> Void) async throws {395        let fm = FileManager.default396        let final = staged(dir, file.path), partial = partial(dir, file.path)397        if fm.fileExists(atPath: final.path) { return }   // verified in a previous run (already counted)398        try fm.createDirectory(at: final.deletingLastPathComponent(), withIntermediateDirectories: true)399        let url = HuggingFaceAPI.resolveURL(base: configuration.baseURL, repoID: request.repoID, revision: request.revision, path: file.path)400        var failures = 0401402        func discardPartial() {403            counter.add(-LocalModelInspector.fileSize(partial))404            try? fm.removeItem(at: partial)405        }406407        while true {408            try Task.checkCancellation()409            var offset = LocalModelInspector.fileSize(partial)410            if let size = file.size, offset > size { discardPartial(); offset = 0 }411            let complete = file.size.map { offset == $0 && (offset > 0 || fm.fileExists(atPath: partial.path)) } ?? false412            if !complete {413                await event(.transferring(file.path))414                do {415                    var end: Int64?416                    if driver.isBackground, let size = file.size { end = min(size, offset + configuration.backgroundSegmentBytes) - 1 }417                    let outcome = try await driver.fetch(url: url, partial: partial, offset: offset, end: end, token: token, repo: request.repoID) { counter.add($0) }418                    if outcome == .rangeNotSatisfiable {419                        // the partial file is at or beyond the end of the remote file: if the size is unknown it may simply be complete420                        if file.size != nil || offset == 0 { discardPartial(); failures += 1; if failures > configuration.maxRetries { throw HubError.http(416) }; continue }421                    } else {422                        let now = LocalModelInspector.fileSize(partial)423                        if let size = file.size, now < size {424                            // a Range segment ended and more remain: go on. A full answer that is shorter than announced is a different file.425                            if let end, end < size - 1, now > offset { failures = 0; continue }426                            discardPartial()427                            throw HubError.integrityFailure(file: file.path, detail: "size \(now) instead of \(size)")428                        }429                    }430                } catch let error as HubError where error.isTransient {431                    if LocalModelInspector.fileSize(partial) > offset { failures = 0 } else { failures += 1 }432                    if failures > configuration.maxRetries { throw error }433                    let delay = configuration.retryBaseDelay * pow(2, Double(max(0, failures - 1)))434                    try await Task.sleep(for: .seconds(min(60, delay)))435                    continue436                }437            }438            // integrity: exact size, then SHA-256 for LFS files439            await event(.verifying(file.path))440            let size = LocalModelInspector.fileSize(partial)441            if let expected = file.size, size != expected {442                discardPartial()443                throw HubError.integrityFailure(file: file.path, detail: "size \(size) instead of \(expected)")444            }445            if let expected = file.sha256?.lowercased() {446                let actual = try await Task.detached(priority: .utility) { try FileHasher.sha256(of: partial) }.value447                if actual != expected {448                    discardPartial()449                    throw HubError.integrityFailure(file: file.path, detail: "SHA-256 mismatch")450                }451            }452            try fm.moveItem(at: partial, to: final)453            await event(.done(file.path))454            return455        }456    }457458    /// Moves the verified files into the models folder. Returns the model path relative to that folder.459    private static func install(request: DownloadRequest, dir: URL, modelsRoot: URL) throws -> String {460        let fm = FileManager.default461        let staging = dir.appendingPathComponent("files")462        let relative = installPath(for: request)463        if request.descriptor.format == .gguf {464            let target = modelsRoot.appendingPathComponent(request.repoID, isDirectory: true)465            for f in request.files {466                let destination = target.appendingPathComponent(f.path)467                try fm.createDirectory(at: destination.deletingLastPathComponent(), withIntermediateDirectories: true)468                if fm.fileExists(atPath: destination.path) { try fm.removeItem(at: destination) }469                try fm.moveItem(at: staged(dir, f.path), to: destination)470            }471        } else {472            let target = modelsRoot.appendingPathComponent(request.repoID, isDirectory: true)473            try fm.createDirectory(at: target.deletingLastPathComponent(), withIntermediateDirectories: true)474            if fm.fileExists(atPath: target.path) {475                // replace an older copy without ever leaving the folder half-written476                _ = try fm.replaceItemAt(target, withItemAt: staging)477            } else { try fm.moveItem(at: staging, to: target) }478        }479        return relative480    }481}482
Files main
Menu

Theme