SPB Git

Repository Public

UploadManager.swift 12.4 KB 220 lines swift

1// SP Drive iOS2// File: Sync/UploadManager.swift3// Description: Resumable uploads in blocks. Each block carries its own SHA-256, the whole file is verified by the server at the end,4//              and an interrupted upload continues from the offset the server reports. A new version always names the version it was5//              derived from: if the server moved on meanwhile, nothing is overwritten and a conflict is raised.6//7// Author: Simon-Pierre Boucher8// Contact: contact@spboucher.ai9import Foundation1011extension SyncEngine {12    static let chunkSize: Int64 = 8 * 1024 * 10241314    /// Makes sure the upload operation has a transfer in the queue.15    func ensureUploadTransfer(for op: PendingOp) {16        if let t = store.activeTransfer(itemID: op.itemID, kind: .upload), t.opID == op.id { return }17        guard let item = store.item(op.itemID), let ls = store.localState(op.itemID), let source = files.url(for: ls), FileManager.default.fileExists(atPath: source.path) else {18            store.setOpState(op.id, .failed, errorCode: "missing_file")19            return20        }21        let id = UUID().uuidString.lowercased()22        // a snapshot: the user may keep editing the file while this content travels (APFS clones it without copying bytes)23        let snapshot = files.outboxFile(id)24        try? FileManager.default.removeItem(at: snapshot)25        guard (try? FileManager.default.copyItem(at: source, to: snapshot)) != nil else { store.setOpState(op.id, .failed, errorCode: "disk"); return }26        let t = TransferRecord(id: id, kind: .upload, itemID: item.id, opID: op.id, name: item.name, size: IntegrityVerifier.size(of: snapshot), done: 0, state: .queued, priority: DownloadPriority.user.rawValue,27                               uploadID: nil, localPath: nil, sha256: nil, errorCode: nil, attempts: 0, createdAt: .now, updatedAt: .now)28        try? store.saveTransfer(t)29        store.setOpState(op.id, .running)30        emit(.transfersChanged)31    }3233    /// The local file changed again: the upload under way carries stale bytes, drop it (the operation stays and will start a fresh one).34    func restartUpload(for itemID: String) {35        guard let t = store.activeTransfer(itemID: itemID, kind: .upload) else { return }36        inFlight.remove(t.id)37        let driver = driver, remote = remote, upload = t.uploadID, id = t.id38        Task { await driver.cancel(tag: id); if let upload { try? await remote.abortUpload(id: upload) } }39        files.removeTransferFiles(t.id)40        store.deleteTransfer(t.id)41        if let op = t.opID { store.setOpState(op, .pending) }42        emit(.transfersChanged)43    }4445    /// One step of an upload: open or resynchronise the server-side session, send the next block, or seal the file.46    func advanceUpload(_ t: TransferRecord) async {47        var t = t48        guard let opID = t.opID, let op = store.op(opID), let item = store.item(t.itemID) else { files.removeTransferFiles(t.id); return await fail(t, "not_found") }49        let source = files.outboxFile(t.id)50        guard FileManager.default.fileExists(atPath: source.path) else { return await fail(t, "missing_file") }51        do {52            if t.sha256 == nil {53                t.sha256 = try await Task.detached(priority: .utility) { try IntegrityVerifier.sha256(of: source) }.value54                guard saveIfRunning(t) else { return }55            }56            var received: Int6457            if let uploadID = t.uploadID {58                let status = try await remote.uploadStatus(id: uploadID)59                if status.status == "done" { return await finishUploadAlreadySealed(t, op) }60                guard status.status == "active" else { t.uploadID = nil; t.done = 0; guard saveIfRunning(t) else { return }; return await retryLater(t, countAttempt: false) }61                received = status.received62            } else {63                let created: RemoteUpload64                if op.kind == .uploadVersion {65                    let base = store.localState(item.id)?.baseVersionID66                    created = try await remote.createUpload(parentID: item.parentID ?? "root", name: item.name, size: t.size, onConflict: .version, fileID: item.id, baseVersionID: base)67                } else {68                    var parent = op.targetID ?? item.parentID ?? "root"69                    if parent.hasPrefix(DriveItem.localPrefix) { return await retryLater(t, countAttempt: false) }70                    do { created = try await remote.createUpload(parentID: parent, name: item.name, size: t.size, onConflict: .rename, fileID: nil, baseVersionID: nil) }71                    catch RemoteError.notFound {72                        parent = try await rootIdentifier() // destination gone on the server: keep the file at the root rather than lose it73                        created = try await remote.createUpload(parentID: parent, name: item.name, size: t.size, onConflict: .rename, fileID: nil, baseVersionID: nil)74                    }75                }76                t.uploadID = created.id77                received = created.received78            }79            t.done = received80            guard saveIfRunning(t) else {81                // withdrawn while the server was opening the session: do not leave that session behind82                if let orphan = t.uploadID, store.transfer(t.id) == nil || store.transfer(t.id)?.uploadID == nil { try? await remote.abortUpload(id: orphan) }83                return84            }85            emit(.transfersChanged)86            guard let uploadID = t.uploadID else { return await retryLater(t) }87            if received < t.size {88                let length = min(policy().segment(max: Self.chunkSize), t.size - received)89                let chunk = files.chunk(t.id)90                let sha = try await Task.detached(priority: .utility) { try IntegrityVerifier.slice(source, offset: received, length: length, to: chunk) }.value91                let request = try await remote.chunkRequest(uploadID: uploadID, offset: received, sha256: sha)92                guard store.transfer(t.id)?.state == .running else { return } // paused or cancelled meanwhile93                noteStepStarted(t.id, bytes: length)94                driver.upload(shape(request, for: t), fromFile: chunk, tag: t.id)95                return96            }97            guard store.transfer(t.id)?.state == .running else { inFlight.remove(t.id); return }98            let sealed = try await remote.completeUpload(id: uploadID, sha256: t.sha256 ?? "")99            await uploadSealed(t, op, sealed)100        } catch {101            await uploadFailed(t, op, classify(error))102        }103    }104105    func chunkFinished(_ t: TransferRecord, _ r: BackgroundResult) async {106        var t = t107        if let e = r.error { noteStepFailed(t.id, error: e); return await retryLater(t, countAttempt: false) }108        guard let status = r.status else { return await retryLater(t, countAttempt: false) }109        if status >= 500 || status == 429 { noteStepFailed(t.id, error: nil) }110        if (200..<300).contains(status) {111            let step = stepStarted[t.id]112            noteStepSucceeded(t.id)113            if let up = try? remote.decodeUpload(r.body) { t.done = up.received } // otherwise the next step asks the server where it stands114            t.attempts = 0115            backoff[t.id] = nil116            guard saveIfRunning(t) else { return }117            emit(.transfersChanged)118            if t.done < t.size, await paceIfCapped(t, bytes: step?.bytes ?? 0, started: step?.at) { return }119            return await advanceUpload(t)120        }121        let e = remote.error(status: status, body: r.body)122        switch e {123        case .offsetMismatch, .checksumMismatch:124            // the server tells where it really stands; the next step starts from there125            t.attempts += 1126            try? store.saveTransfer(t)127            if t.attempts > Self.maxAttempts { return await fail(t, "checksum") }128            return await advanceUpload(t)129        case .uploadClosed, .notFound:130            t.uploadID = nil; t.done = 0131            try? store.saveTransfer(t)132            return await retryLater(t)133        default:134            guard let opID = t.opID, let op = store.op(opID) else { return await fail(t, code(for: e)) }135            await uploadFailed(t, op, e)136        }137    }138139    func uploadFailed(_ t: TransferRecord, _ op: PendingOp, _ e: RemoteError) async {140        // the transfer was withdrawn while this step was talking to the server (cancelled, conflict resolved): its outcome no longer matters141        guard store.transfer(t.id) != nil, store.op(op.id) != nil else { inFlight.remove(t.id); return }142        if e.isTransient { return await retryLater(t, countAttempt: e != .offline) }143        if e == .versionConflict { return await raiseConflict(t, op) }144        if e == .notFound, op.kind == .uploadVersion {145            // the file no longer exists on the server while it was edited here146            if store.conflict(itemID: t.itemID) == nil {147                try? store.addConflict(itemID: t.itemID, name: t.name, reason: .remoteDeleted, uploadID: nil)148                emit(.conflictsChanged)149                emit(.conflictDetected(name: t.name))150            }151        }152        store.setOpState(op.id, .failed, errorCode: code(for: e))153        await fail(t, code(for: e))154    }155156    /// The server's current version is not the one this content was derived from: both are kept until the user decides.157    func raiseConflict(_ t: TransferRecord, _ op: PendingOp) async {158        try? store.addConflict(itemID: t.itemID, name: t.name, reason: .bothModified, uploadID: t.uploadID)159        store.setOpState(op.id, .failed, errorCode: "conflict")160        // the bytes already on the server stay there (the conflict copy can reuse them); this transfer is over161        var t = t162        t.state = .failed163        t.errorCode = "conflict"164        try? store.saveTransfer(t)165        inFlight.remove(t.id)166        try? FileManager.default.removeItem(at: files.chunk(t.id))167        finishWaiters(t.id, .failure(EngineError.transferFailed("conflict")))168        emit(.conflictsChanged)169        emit(.conflictDetected(name: t.name))170        emit(.transfersChanged)171        emit(.itemsChanged)172        await pump()173    }174175    func finishUploadAlreadySealed(_ t: TransferRecord, _ op: PendingOp) async {176        // sealed by an earlier attempt whose answer never arrived177        do {178            let status = try await remote.uploadStatus(id: t.uploadID ?? "")179            guard let fileID = status.fileID else { return await fail(t, "server") }180            await uploadSealed(t, op, try await remote.item(id: fileID, isFolder: false))181        } catch { await uploadFailed(t, op, classify(error)) }182    }183184    /// The server holds the content and verified its hash: the device's records follow.185    func uploadSealed(_ t: TransferRecord, _ op: PendingOp, _ sealed: DriveItem) async {186        guard store.transfer(t.id) != nil else { inFlight.remove(t.id); return }187        var t = t188        let oldID = t.itemID189        do {190            if oldID != sealed.id {191                try store.remap(localID: oldID, to: sealed.id)192                moveLocalDirectory(from: oldID, to: sealed.id)193                t.itemID = sealed.id194            }195            try store.upsert(sealed)196            if var ls = store.localState(sealed.id) {197                ls.baseSHA = sealed.sha256198                ls.baseVersionID = sealed.versionID199                // edited again while this version travelled? then the copy is still ahead of the server200                let live = files.url(for: ls)201                let liveSHA = live.flatMap { try? IntegrityVerifier.sha256(of: $0) }202                ls.dirty = liveSHA != nil && liveSHA != sealed.sha256203                if !ls.dirty, !isEffectivelyPinned(sealed.id), ls.location == .offline, let live, let name = ls.fileName {204                    // not pinned: the copy becomes an ordinary cached copy the system may reclaim205                    if (try? files.install(live, itemID: sealed.id, name: name, location: .cache, move: true)) != nil { files.removeCopy(itemID: sealed.id, location: .offline); ls.location = .cache }206                }207                try store.saveLocalState(ls)208                store.deleteOp(op.id)209                if ls.dirty { _ = try store.appendOp(.uploadVersion, itemID: sealed.id) }210            } else {211                store.deleteOp(op.id)212            }213        } catch {214            Log.sync.error("could not record a finished upload: \(String(describing: error), privacy: .public)")215        }216        await succeed(t)217        kick()218    }219}220
Files main
Menu

Theme