Repository Public
1// SP Drive iOS2// File: Sync/UploadManager.swift3// Description: Resumable uploads in blocks. Each block carries its own SHA-256, the whole file is verified by the server at the end,4// and an interrupted upload continues from the offset the server reports. A new version always names the version it was5// derived from: if the server moved on meanwhile, nothing is overwritten and a conflict is raised.6//7// Author: Simon-Pierre Boucher8// Contact: contact@spboucher.ai9import Foundation1011extension SyncEngine {12 static let chunkSize: Int64 = 8 * 1024 * 10241314 /// Makes sure the upload operation has a transfer in the queue.15 func ensureUploadTransfer(for op: PendingOp) {16 if let t = store.activeTransfer(itemID: op.itemID, kind: .upload), t.opID == op.id { return }17 guard let item = store.item(op.itemID), let ls = store.localState(op.itemID), let source = files.url(for: ls), FileManager.default.fileExists(atPath: source.path) else {18 store.setOpState(op.id, .failed, errorCode: "missing_file")19 return20 }21 let id = UUID().uuidString.lowercased()22 // a snapshot: the user may keep editing the file while this content travels (APFS clones it without copying bytes)23 let snapshot = files.outboxFile(id)24 try? FileManager.default.removeItem(at: snapshot)25 guard (try? FileManager.default.copyItem(at: source, to: snapshot)) != nil else { store.setOpState(op.id, .failed, errorCode: "disk"); return }26 let t = TransferRecord(id: id, kind: .upload, itemID: item.id, opID: op.id, name: item.name, size: IntegrityVerifier.size(of: snapshot), done: 0, state: .queued, priority: DownloadPriority.user.rawValue,27 uploadID: nil, localPath: nil, sha256: nil, errorCode: nil, attempts: 0, createdAt: .now, updatedAt: .now)28 try? store.saveTransfer(t)29 store.setOpState(op.id, .running)30 emit(.transfersChanged)31 }3233 /// The local file changed again: the upload under way carries stale bytes, drop it (the operation stays and will start a fresh one).34 func restartUpload(for itemID: String) {35 guard let t = store.activeTransfer(itemID: itemID, kind: .upload) else { return }36 inFlight.remove(t.id)37 let driver = driver, remote = remote, upload = t.uploadID, id = t.id38 Task { await driver.cancel(tag: id); if let upload { try? await remote.abortUpload(id: upload) } }39 files.removeTransferFiles(t.id)40 store.deleteTransfer(t.id)41 if let op = t.opID { store.setOpState(op, .pending) }42 emit(.transfersChanged)43 }4445 /// One step of an upload: open or resynchronise the server-side session, send the next block, or seal the file.46 func advanceUpload(_ t: TransferRecord) async {47 var t = t48 guard let opID = t.opID, let op = store.op(opID), let item = store.item(t.itemID) else { files.removeTransferFiles(t.id); return await fail(t, "not_found") }49 let source = files.outboxFile(t.id)50 guard FileManager.default.fileExists(atPath: source.path) else { return await fail(t, "missing_file") }51 do {52 if t.sha256 == nil {53 t.sha256 = try await Task.detached(priority: .utility) { try IntegrityVerifier.sha256(of: source) }.value54 guard saveIfRunning(t) else { return }55 }56 var received: Int6457 if let uploadID = t.uploadID {58 let status = try await remote.uploadStatus(id: uploadID)59 if status.status == "done" { return await finishUploadAlreadySealed(t, op) }60 guard status.status == "active" else { t.uploadID = nil; t.done = 0; guard saveIfRunning(t) else { return }; return await retryLater(t, countAttempt: false) }61 received = status.received62 } else {63 let created: RemoteUpload64 if op.kind == .uploadVersion {65 let base = store.localState(item.id)?.baseVersionID66 created = try await remote.createUpload(parentID: item.parentID ?? "root", name: item.name, size: t.size, onConflict: .version, fileID: item.id, baseVersionID: base)67 } else {68 var parent = op.targetID ?? item.parentID ?? "root"69 if parent.hasPrefix(DriveItem.localPrefix) { return await retryLater(t, countAttempt: false) }70 do { created = try await remote.createUpload(parentID: parent, name: item.name, size: t.size, onConflict: .rename, fileID: nil, baseVersionID: nil) }71 catch RemoteError.notFound {72 parent = try await rootIdentifier() // destination gone on the server: keep the file at the root rather than lose it73 created = try await remote.createUpload(parentID: parent, name: item.name, size: t.size, onConflict: .rename, fileID: nil, baseVersionID: nil)74 }75 }76 t.uploadID = created.id77 received = created.received78 }79 t.done = received80 guard saveIfRunning(t) else {81 // withdrawn while the server was opening the session: do not leave that session behind82 if let orphan = t.uploadID, store.transfer(t.id) == nil || store.transfer(t.id)?.uploadID == nil { try? await remote.abortUpload(id: orphan) }83 return84 }85 emit(.transfersChanged)86 guard let uploadID = t.uploadID else { return await retryLater(t) }87 if received < t.size {88 let length = min(policy().segment(max: Self.chunkSize), t.size - received)89 let chunk = files.chunk(t.id)90 let sha = try await Task.detached(priority: .utility) { try IntegrityVerifier.slice(source, offset: received, length: length, to: chunk) }.value91 let request = try await remote.chunkRequest(uploadID: uploadID, offset: received, sha256: sha)92 guard store.transfer(t.id)?.state == .running else { return } // paused or cancelled meanwhile93 noteStepStarted(t.id, bytes: length)94 driver.upload(shape(request, for: t), fromFile: chunk, tag: t.id)95 return96 }97 guard store.transfer(t.id)?.state == .running else { inFlight.remove(t.id); return }98 let sealed = try await remote.completeUpload(id: uploadID, sha256: t.sha256 ?? "")99 await uploadSealed(t, op, sealed)100 } catch {101 await uploadFailed(t, op, classify(error))102 }103 }104105 func chunkFinished(_ t: TransferRecord, _ r: BackgroundResult) async {106 var t = t107 if let e = r.error { noteStepFailed(t.id, error: e); return await retryLater(t, countAttempt: false) }108 guard let status = r.status else { return await retryLater(t, countAttempt: false) }109 if status >= 500 || status == 429 { noteStepFailed(t.id, error: nil) }110 if (200..<300).contains(status) {111 let step = stepStarted[t.id]112 noteStepSucceeded(t.id)113 if let up = try? remote.decodeUpload(r.body) { t.done = up.received } // otherwise the next step asks the server where it stands114 t.attempts = 0115 backoff[t.id] = nil116 guard saveIfRunning(t) else { return }117 emit(.transfersChanged)118 if t.done < t.size, await paceIfCapped(t, bytes: step?.bytes ?? 0, started: step?.at) { return }119 return await advanceUpload(t)120 }121 let e = remote.error(status: status, body: r.body)122 switch e {123 case .offsetMismatch, .checksumMismatch:124 // the server tells where it really stands; the next step starts from there125 t.attempts += 1126 try? store.saveTransfer(t)127 if t.attempts > Self.maxAttempts { return await fail(t, "checksum") }128 return await advanceUpload(t)129 case .uploadClosed, .notFound:130 t.uploadID = nil; t.done = 0131 try? store.saveTransfer(t)132 return await retryLater(t)133 default:134 guard let opID = t.opID, let op = store.op(opID) else { return await fail(t, code(for: e)) }135 await uploadFailed(t, op, e)136 }137 }138139 func uploadFailed(_ t: TransferRecord, _ op: PendingOp, _ e: RemoteError) async {140 // the transfer was withdrawn while this step was talking to the server (cancelled, conflict resolved): its outcome no longer matters141 guard store.transfer(t.id) != nil, store.op(op.id) != nil else { inFlight.remove(t.id); return }142 if e.isTransient { return await retryLater(t, countAttempt: e != .offline) }143 if e == .versionConflict { return await raiseConflict(t, op) }144 if e == .notFound, op.kind == .uploadVersion {145 // the file no longer exists on the server while it was edited here146 if store.conflict(itemID: t.itemID) == nil {147 try? store.addConflict(itemID: t.itemID, name: t.name, reason: .remoteDeleted, uploadID: nil)148 emit(.conflictsChanged)149 emit(.conflictDetected(name: t.name))150 }151 }152 store.setOpState(op.id, .failed, errorCode: code(for: e))153 await fail(t, code(for: e))154 }155156 /// The server's current version is not the one this content was derived from: both are kept until the user decides.157 func raiseConflict(_ t: TransferRecord, _ op: PendingOp) async {158 try? store.addConflict(itemID: t.itemID, name: t.name, reason: .bothModified, uploadID: t.uploadID)159 store.setOpState(op.id, .failed, errorCode: "conflict")160 // the bytes already on the server stay there (the conflict copy can reuse them); this transfer is over161 var t = t162 t.state = .failed163 t.errorCode = "conflict"164 try? store.saveTransfer(t)165 inFlight.remove(t.id)166 try? FileManager.default.removeItem(at: files.chunk(t.id))167 finishWaiters(t.id, .failure(EngineError.transferFailed("conflict")))168 emit(.conflictsChanged)169 emit(.conflictDetected(name: t.name))170 emit(.transfersChanged)171 emit(.itemsChanged)172 await pump()173 }174175 func finishUploadAlreadySealed(_ t: TransferRecord, _ op: PendingOp) async {176 // sealed by an earlier attempt whose answer never arrived177 do {178 let status = try await remote.uploadStatus(id: t.uploadID ?? "")179 guard let fileID = status.fileID else { return await fail(t, "server") }180 await uploadSealed(t, op, try await remote.item(id: fileID, isFolder: false))181 } catch { await uploadFailed(t, op, classify(error)) }182 }183184 /// The server holds the content and verified its hash: the device's records follow.185 func uploadSealed(_ t: TransferRecord, _ op: PendingOp, _ sealed: DriveItem) async {186 guard store.transfer(t.id) != nil else { inFlight.remove(t.id); return }187 var t = t188 let oldID = t.itemID189 do {190 if oldID != sealed.id {191 try store.remap(localID: oldID, to: sealed.id)192 moveLocalDirectory(from: oldID, to: sealed.id)193 t.itemID = sealed.id194 }195 try store.upsert(sealed)196 if var ls = store.localState(sealed.id) {197 ls.baseSHA = sealed.sha256198 ls.baseVersionID = sealed.versionID199 // edited again while this version travelled? then the copy is still ahead of the server200 let live = files.url(for: ls)201 let liveSHA = live.flatMap { try? IntegrityVerifier.sha256(of: $0) }202 ls.dirty = liveSHA != nil && liveSHA != sealed.sha256203 if !ls.dirty, !isEffectivelyPinned(sealed.id), ls.location == .offline, let live, let name = ls.fileName {204 // not pinned: the copy becomes an ordinary cached copy the system may reclaim205 if (try? files.install(live, itemID: sealed.id, name: name, location: .cache, move: true)) != nil { files.removeCopy(itemID: sealed.id, location: .offline); ls.location = .cache }206 }207 try store.saveLocalState(ls)208 store.deleteOp(op.id)209 if ls.dirty { _ = try store.appendOp(.uploadVersion, itemID: sealed.id) }210 } else {211 store.deleteOp(op.id)212 }213 } catch {214 Log.sync.error("could not record a finished upload: \(String(describing: error), privacy: .public)")215 }216 await succeed(t)217 kick()218 }219}220