SPB Git

Repository Public

TransferQueue.swift 20.7 KB 440 lines swift

1// SP Drive iOS2// File: Sync/TransferQueue.swift3// Description: Persistent queue of uploads and downloads: priorities, bounded parallelism, pause, cancel, retry with back-off,4//              and resumption after the process was stopped. The bytes travel through a `TransferDriver`.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import Synchronization1011/// Moves bytes for the queue and reports back through events. The app uses a background URLSession (transfers continue while it is12/// suspended, within what iOS allows); extensions and tests use an in-process session.13public protocol TransferDriver: Sendable {14    func start(handler: @escaping @Sendable (BackgroundEvent) -> Void)15    func download(_ request: URLRequest, tag: String, resumeData: Data?, expectedBytes: Int64)16    func upload(_ request: URLRequest, fromFile file: URL, tag: String)17    func activeTags() async -> Set<String>18    func cancel(tag: String) async19    func cancelAll() async20    /// True when results of steps finished while the process was not running may be delivered some time after `start` (background URLSession).21    var resultsMayArriveLate: Bool { get }22}2324extension TransferDriver {25    public var resultsMayArriveLate: Bool { false }26}2728extension BackgroundSession: TransferDriver {29    public var resultsMayArriveLate: Bool { true }30}3132/// In-process driver: same contract as the background session, tasks die with the process.33public final class InlineTransferDriver: TransferDriver {34    private let session: URLSession35    private let staging: URL36    private let state = Mutex<(handler: (@Sendable (BackgroundEvent) -> Void)?, tasks: [String: Task<Void, Never>])>((nil, [:]))3738    public init(staging: URL, configuration: URLSessionConfiguration = .ephemeral) {39        configuration.httpCookieStorage = nil40        configuration.urlCache = nil41        session = URLSession(configuration: configuration)42        self.staging = staging43        try? FileManager.default.createDirectory(at: staging, withIntermediateDirectories: true)44    }4546    public func start(handler: @escaping @Sendable (BackgroundEvent) -> Void) { state.withLock { $0.handler = handler } }47    private func emit(_ e: BackgroundEvent) { state.withLock { $0.handler }?(e) }48    private func track(_ tag: String, _ task: Task<Void, Never>) { state.withLock { $0.tasks[tag] = task } }49    private func finish(_ tag: String, _ result: BackgroundResult) {50        state.withLock { _ = $0.tasks.removeValue(forKey: tag) }51        emit(.finished(tag: tag, result: result))52    }53    private static func headers(_ http: HTTPURLResponse?) -> [String: String] {54        var h: [String: String] = [:]55        for (k, v) in http?.allHeaderFields ?? [:] { if let k = k as? String, let v = v as? String { h[k.lowercased()] = v } }56        return h57    }58    private static func failure(_ error: Error) -> BackgroundResult {59        BackgroundResult(status: nil, headers: [:], body: Data(), file: nil, error: (error as? URLError) ?? URLError(error is CancellationError ? .cancelled : .unknown), resumeData: nil)60    }6162    public func download(_ request: URLRequest, tag: String, resumeData: Data?, expectedBytes: Int64) {63        let session = session, staging = staging64        track(tag, Task { [weak self] in65            do {66                let (tmp, response) = try await session.download(for: request)67                let http = response as? HTTPURLResponse68                var result = BackgroundResult(status: http?.statusCode, headers: Self.headers(http), body: Data(), file: nil, error: nil, resumeData: nil)69                if let s = http?.statusCode, (200..<300).contains(s) {70                    let dest = staging.appendingPathComponent(UUID().uuidString)71                    try FileManager.default.moveItem(at: tmp, to: dest)72                    result.file = dest73                } else {74                    result.body = (try? Data(contentsOf: tmp).prefix(64 * 1024)).map { Data($0) } ?? Data()75                    try? FileManager.default.removeItem(at: tmp)76                }77                self?.finish(tag, result)78            } catch { self?.finish(tag, Self.failure(error)) }79        })80    }8182    public func upload(_ request: URLRequest, fromFile file: URL, tag: String) {83        let session = session84        track(tag, Task { [weak self] in85            do {86                let (data, response) = try await session.upload(for: request, fromFile: file)87                let http = response as? HTTPURLResponse88                self?.finish(tag, BackgroundResult(status: http?.statusCode, headers: Self.headers(http), body: data, file: nil, error: nil, resumeData: nil))89            } catch { self?.finish(tag, Self.failure(error)) }90        })91    }9293    public func activeTags() async -> Set<String> { state.withLock { Set($0.tasks.keys) } }94    public func cancel(tag: String) async { state.withLock { $0.tasks[tag] }?.cancel() }95    public func cancelAll() async { state.withLock { Array($0.tasks.values) }.forEach { $0.cancel() } }96}9798extension SyncEngine {99    static let maxAttempts = 8100101    // MARK: Public controls102103    public nonisolated func transfers() -> [TransferRecord] { store.transfers() }104105    /// Bytes done of a running transfer, fresher than what is persisted.106    public func progress(of id: String) -> Int64? { liveProgress[id] }107108    public func pause(_ id: String) async {109        guard var t = store.transfer(id), t.isActive else { return }110        t.state = .paused111        try? store.saveTransfer(t)112        inFlight.remove(id)113        await driver.cancel(tag: id)114        emit(.transfersChanged)115        await pump()116    }117118    public func resume(_ id: String) async {119        guard var t = store.transfer(id), t.state == .paused || t.state == .failed else { return }120        t.state = .queued121        t.errorCode = nil122        t.attempts = 0123        notBefore[id] = nil124        backoff[id] = nil125        try? store.saveTransfer(t)126        if let op = t.opID, store.op(op)?.state == .failed { store.setOpState(op, .pending) }127        emit(.transfersChanged)128        await pump()129    }130131    public func resumeAll() async {132        for t in store.transfers(states: [.paused, .failed]) { await resume(t.id) }133    }134135    /// Cancels a transfer. A cancelled download keeps nothing; a cancelled upload of a file that only exists on this device136    /// drops the provisional item (the user chose not to send it), a cancelled new version leaves the local edit in place.137    public func cancel(_ id: String) async {138        guard var t = store.transfer(id) else { return }139        inFlight.remove(id)140        await driver.cancel(tag: id)141        if let up = t.uploadID { try? await remote.abortUpload(id: up) }142        files.removeTransferFiles(id)143        t.state = .cancelled144        try? store.saveTransfer(t)145        if t.kind == .upload, let opID = t.opID, let op = store.op(opID) {146            // a cancelled new version keeps the edit on the device: the operation waits, visibly, for « retry » or « give up »147            if op.kind == .uploadVersion { store.setOpState(opID, .failed, errorCode: "cancelled") } else { store.deleteOp(opID) }148            if op.kind == .uploadNew, t.itemID.hasPrefix(DriveItem.localPrefix) {149                files.removeCopy(itemID: t.itemID, location: .offline)150                try? store.deleteLocalState(t.itemID)151                try? store.deleteItem(t.itemID)152            }153        }154        finishWaiters(id, .failure(EngineError.cancelled))155        emit(.transfersChanged)156        emit(.itemsChanged)157        await pump()158    }159160    public func clearFinished() {161        store.clearFinishedTransfers()162        emit(.transfersChanged)163    }164165    public func setPriority(_ id: String, _ priority: Int) async {166        guard var t = store.transfer(id) else { return }167        t.priority = priority168        try? store.saveTransfer(t)169        emit(.transfersChanged)170        await pump()171    }172173    // MARK: Scheduling174175    /// Starts as many queued transfers as conditions allow, highest priority first. User-initiated transfers go before background ones176    /// (offline pins), and may take the place of one that is running when every slot is busy.177    public func pump() async {178        let allowance = policy().evaluate(conditions())179        let limit = min(concurrency(), allowance.limit, adaptive.limit)180        let now = Date.now181        var soonest: Date?182        var newHolds: [String: TransferPolicy.Hold] = [:]183        var waitingUser: [TransferRecord] = []184        for t in store.transfers(states: [.queued]) {185            if inFlight.contains(t.id) { continue }186            let user = Self.isUserInitiated(t)187            if let hold = allowance.hold(userInitiated: user, exemptFromWiFiOnly: wifiOnlyExempt.contains(t.id)) { newHolds[t.id] = hold; continue }188            if let nb = notBefore[t.id], nb > now { soonest = min(soonest ?? nb, nb); continue }189            if limit <= 0 { continue }190            if inFlight.count >= limit { if user { waitingUser.append(t) }; continue }191            launch(t)192        }193        // every slot is busy: a transfer the user is waiting for takes the place of a background one194        for t in waitingUser {195            guard let victim = store.transfers(states: [.running]).filter({ inFlight.contains($0.id) && !Self.isUserInitiated($0) }).min(by: { $0.priority < $1.priority }) else { break }196            await requeue(victim)197            launch(t)198        }199        if newHolds != holds { holds = newHolds; emit(.transfersChanged) }200        if let soonest, retryTask == nil {201            retryTask = Task { [weak self] in202                try? await Task.sleep(for: .seconds(max(0.05, soonest.timeIntervalSinceNow)))203                await self?.retryFired()204            }205        }206    }207208    static func isUserInitiated(_ t: TransferRecord) -> Bool { t.priority >= DownloadPriority.user.rawValue }209210    private func launch(_ t: TransferRecord) {211        inFlight.insert(t.id)212        var running = t213        running.state = .running214        try? store.saveTransfer(running)215        emit(.transfersChanged)216        // the first step of a transfer talks to the server: it must not hold the queue217        Task { await self.begin(running.id) }218    }219220    /// Takes a running transfer off the network without losing what it moved: it waits in the queue (not « paused »: the user did not ask).221    func requeue(_ t: TransferRecord) async {222        guard var t = store.transfer(t.id), t.state == .running else { return }223        t.state = .queued224        try? store.saveTransfer(t)225        inFlight.remove(t.id)226        stepStarted[t.id] = nil227        await driver.cancel(tag: t.id)228        emit(.transfersChanged)229    }230231    /// Running transfers the policy no longer allows (the path became cellular with « Wi-Fi only » on, Low Power Mode came on…) are requeued.232    func enforcePolicyOnRunning() async {233        let allowance = policy().evaluate(conditions())234        for t in store.transfers(states: [.running]) where inFlight.contains(t.id) {235            if allowance.hold(userInitiated: Self.isUserInitiated(t), exemptFromWiFiOnly: wifiOnlyExempt.contains(t.id)) != nil { await requeue(t) }236        }237    }238239    /// Why a queued transfer is not running, when it is the policy (not the queue) that keeps it waiting.240    public func hold(of id: String) -> TransferPolicy.Hold? { holds[id] }241    /// The reason shared by held transfers, for a banner (« En attente du Wi-Fi »).242    public func currentHolds() -> [String: TransferPolicy.Hold] { holds }243    /// How many transfers may run at once right now, and what the adaptive controller alone would allow.244    public func concurrencyState() -> (effective: Int, adaptive: Int) {245        (min(concurrency(), policy().evaluate(conditions()).limit, adaptive.limit), adaptive.limit)246    }247248    /// « Use cellular data this once »: lets held transfers through although « Wi-Fi only » is on. Forgotten when they finish.249    public func allowOnCellular(_ ids: [String]? = nil) async {250        let targets = ids ?? holds.filter { $0.value == .waitingForWiFi }.map(\.key)251        wifiOnlyExempt.formUnion(targets)252        await pump()253    }254255    /// Applies the policy to a request about to leave: even with the app suspended, the system then refuses a path the user excluded.256    func shape(_ request: URLRequest, for t: TransferRecord) -> URLRequest {257        var r = request258        let p = policy()259        let exempt = wifiOnlyExempt.contains(t.id)260        r.allowsExpensiveNetworkAccess = p.allowCellular || exempt261        r.allowsCellularAccess = p.allowCellular || exempt262        r.allowsConstrainedNetworkAccess = Self.isUserInitiated(t) || p.backgroundInLowDataMode263        return r264    }265266    /// Bandwidth cap: after a segment, the transfer waits in the queue for as long as the cap asks. Returns true when it was put to wait.267    func paceIfCapped(_ t: TransferRecord, bytes: Int64, started: Date?) async -> Bool {268        let wait = policy().pause(afterBytes: bytes, took: started.map { Date.now.timeIntervalSince($0) } ?? 0)269        guard wait > 0.02, var current = store.transfer(t.id), current.state == .running else { return false }270        current.state = .queued271        try? store.saveTransfer(current)272        inFlight.remove(t.id)273        notBefore[t.id] = Date.now.addingTimeInterval(wait)274        emit(.transfersChanged)275        await pump()276        return true277    }278279    func noteStepStarted(_ id: String, bytes: Int64) { stepStarted[id] = (.now, bytes) }280    func noteStepSucceeded(_ id: String) {281        guard let s = stepStarted.removeValue(forKey: id) else { return }282        adaptive.recordSuccess(bytes: s.bytes, seconds: Date.now.timeIntervalSince(s.at), active: max(1, inFlight.count))283    }284    /// `error` nil = the server answered with a 5xx or 429.285    func noteStepFailed(_ id: String, error: URLError?) {286        stepStarted[id] = nil287        if let error, error.code.meansNoNetwork { return }288        adaptive.recordFailure()289    }290    func retryFired() async { retryTask = nil; await pump() }291292    func begin(_ id: String) async {293        guard let t = store.transfer(id), t.state == .running else { inFlight.remove(id); return }294        switch t.kind {295        case .download: await startDownload(t)296        case .upload: await advanceUpload(t)297        }298    }299300    func handle(_ event: BackgroundEvent) async {301        switch event {302        case .progress(let tag, let done, _):303            guard var t = store.transfer(tag), t.state == .running else { return }304            let absolute = t.kind == .upload ? t.done + done : (progressBase[tag] ?? 0) + done305            liveProgress[tag] = absolute306            if Date.now.timeIntervalSince(lastProgressSave[tag] ?? .distantPast) > 0.5 {307                lastProgressSave[tag] = .now308                if t.kind == .download { t.done = min(absolute, t.size); try? store.saveTransfer(t) }309                emit(.transfersChanged)310            }311        case .finished(let tag, let result):312            liveProgress[tag] = nil313            guard var t = store.transfer(tag) else { if let f = result.file { try? FileManager.default.removeItem(at: f) }; return }314            if t.state == .queued, !inFlight.contains(tag), result.isSuccess {315                // crash-safe recovery: the system finished this step while the app was not running (or before the queue was read again316                // after a relaunch). The result is adopted instead of being thrown away and fetched a second time.317                t.state = .running318                try? store.saveTransfer(t)319                inFlight.insert(tag)320                Log.transfer.notice("adopted a step finished while the app was away: \(tag, privacy: .public)")321            }322            if t.state == .running { inFlight.insert(tag) } // « running » from a previous life of the process: this result is its outcome323            guard t.state == .running else {324                // paused, requeued or cancelled while the bytes were in flight325                if let f = result.file { try? FileManager.default.removeItem(at: f) }326                if t.state == .paused || t.state == .queued, let r = result.resumeData { try? r.write(to: files.resumeData(tag)) }327                return328            }329            switch t.kind {330            case .download: await downloadFinished(t, result)331            case .upload: await chunkFinished(t, result)332            }333        }334    }335336    // MARK: Outcomes337338    func succeed(_ t: TransferRecord) async {339        // a step that was talking to the server may come back after the transfer was paused, cancelled or withdrawn: its outcome is void340        guard store.transfer(t.id)?.state == .running else { inFlight.remove(t.id); return }341        var t = t342        t.state = .done343        t.done = t.size344        t.errorCode = nil345        try? store.saveTransfer(t)346        files.removeTransferFiles(t.id)347        inFlight.remove(t.id)348        notBefore[t.id] = nil349        backoff[t.id] = nil350        progressBase[t.id] = nil351        wifiOnlyExempt.remove(t.id)352        stepStarted[t.id] = nil353        finishWaiters(t.id, .success(()))354        emit(.transferFinished(t))355        emit(.transfersChanged)356        emit(.itemsChanged)357        await pump()358    }359360    /// A definitive failure: the transfer stops and waits for the user (retry or cancel). Local data is kept.361    func fail(_ t: TransferRecord, _ code: String) async {362        // a step that was talking to the server may come back after the transfer was paused, cancelled or withdrawn: its outcome is void363        guard store.transfer(t.id)?.state == .running else { inFlight.remove(t.id); return }364        var t = t365        t.state = .failed366        t.errorCode = code367        try? store.saveTransfer(t)368        inFlight.remove(t.id)369        progressBase[t.id] = nil370        finishWaiters(t.id, .failure(EngineError.transferFailed(code)))371        if code == "unauthenticated" { emit(.signedOut) }372        emit(.transferFinished(t))373        emit(.transfersChanged)374        await pump()375    }376377    /// The delay a transfer still has to wait before its next attempt (nil = none).378    public func retryDelay(of id: String) -> TimeInterval? { notBefore[id].map { $0.timeIntervalSinceNow }.flatMap { $0 > 0 ? $0 : nil } }379    func setJitter(_ j: @escaping @Sendable () -> Double) { jitter = j }380381    /// A temporary failure: back in the queue, retried later with an increasing delay (exponential, capped at 60 s, with jitter). Being offline is not counted against the transfer382    /// (it simply waits for the network); repeated server-side failures eventually stop it.383    func retryLater(_ t: TransferRecord, countAttempt: Bool = true) async {384        // a step that was talking to the server may come back after the transfer was paused, cancelled or withdrawn: its outcome is void385        guard store.transfer(t.id)?.state == .running else { inFlight.remove(t.id); return }386        var t = t387        if countAttempt { t.attempts += 1 }388        if t.attempts > Self.maxAttempts { return await fail(t, "network") }389        t.state = .queued390        try? store.saveTransfer(t)391        inFlight.remove(t.id)392        let n = backoff[t.id, default: 0]393        backoff[t.id] = n + 1394        notBefore[t.id] = Date.now.addingTimeInterval(Backoff.delay(attempt: n, unit: jitter()))395        emit(.transfersChanged)396        await pump()397    }398399    func classify(_ error: Error) -> RemoteError {400        (error as? RemoteError) ?? (error is CancellationError ? .offline : .server(status: 0, code: "local", message: String(describing: error)))401    }402403    func code(for error: RemoteError) -> String {404        switch error {405        case .unauthenticated: "unauthenticated"406        case .notFound: "not_found"407        case .quotaExceeded: "quota"408        case .fileTooLarge: "too_large"409        case .infected: "infected"410        case .versionConflict: "conflict"411        case .checksumMismatch: "checksum"412        case .nameConflict: "name_conflict"413        case .server(_, let c, _): c.isEmpty ? "server" : c414        default: "network"415        }416    }417418    /// Persists the progress of a step only if the transfer is still the running one (see `succeed`).419    @discardableResult420    func saveIfRunning(_ t: TransferRecord) -> Bool {421        guard store.transfer(t.id)?.state == .running else { inFlight.remove(t.id); return false }422        try? store.saveTransfer(t)423        return true424    }425426    // MARK: Waiting for a transfer427428    func finishWaiters(_ id: String, _ result: Result<Void, Error>) {429        guard let list = waiters.removeValue(forKey: id) else { return }430        for c in list { c.resume(with: result) }431    }432433    func wait(for transferID: String) async throws {434        guard let t = store.transfer(transferID) else { throw EngineError.notFound }435        if t.state == .done { return }436        if t.state == .failed || t.state == .cancelled { throw EngineError.transferFailed(t.errorCode ?? "failed") }437        try await withCheckedThrowingContinuation { (c: CheckedContinuation<Void, Error>) in waiters[transferID, default: []].append(c) }438    }439}440
Files main
Menu

Theme