Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Sync/UploadManager.swift3// Description: Resumable, verified uploads: the file is read block by block (never whole in memory), each block carries its SHA-256, an interrupted session resumes at the offset the server reports, and the server confirms the whole-file hash before anything counts as stored. A changed server version never gets overwritten: the bytes become a conflict copy.4import CryptoKit5import Foundation6import SPDriveCore78public enum UploadTarget: Sendable, Equatable {9 /// A new file in a folder.10 case newFile(folderID: String, name: String, onConflict: ConflictPolicy)11 /// A new version of an existing file, derived from `baseVersionID` (nil = replace whatever is there, decided by the person).12 case newVersion(fileID: String, folderID: String, name: String, baseVersionID: String?)13 var key: String {14 switch self {15 case .newFile(let f, let n, _): return "new|\(f)|\(NameRules.key(n))"16 case .newVersion(let id, _, _, let base): return "ver|\(id)|\(base ?? "-")"17 }18 }19}20public enum UploadOutcome: Sendable {21 case stored(DriveItem)22 /// The file had changed on the server: nothing was overwritten, the local content was stored next to it as `copy`.23 case conflictCopy(copy: DriveItem, originalID: String)24 public var item: DriveItem { switch self { case .stored(let i): return i; case .conflictCopy(let c, _): return c } }25}2627public final class UploadEngine: Sendable {28 public let api: APIClient29 /// Where resumable sessions are remembered across restarts (nil = no resumption after a restart).30 public let store: MetadataStore?31 public let throttle: Throttle?32 /// Name of this device, used in the name of conflict copies.33 public let deviceName: String34 public static let minBlock = 256 * 1024, maxBlock = 16 * 1024 * 10243536 public init(api: APIClient, store: MetadataStore?, throttle: Throttle? = nil, deviceName: String) { self.api = api; self.store = store; self.throttle = throttle; self.deviceName = deviceName }3738 private func savedSession(_ key: String) -> String? { try? store?.db.query("SELECT upload_id FROM upload_sessions WHERE key = ?", [.text(key)]).first?["upload_id"]?.text }39 private func save(_ key: String, _ id: String?) {40 if let id { _ = try? store?.db.run("INSERT OR REPLACE INTO upload_sessions (key, upload_id, created_at) VALUES (?, ?, ?)", [.text(key), .text(id), .double(Date().timeIntervalSince1970)]) }41 else { _ = try? store?.db.run("DELETE FROM upload_sessions WHERE key = ?", [.text(key)]) }42 }4344 /// Sends the file. `progress` receives bytes confirmed by the server and the total.45 public func upload(file url: URL, to target: UploadTarget, progress: @escaping @Sendable (Int64, Int64) -> Void = { _, _ in }) async throws -> UploadOutcome {46 let before = try fingerprint(url)47 let (sha, size) = try Hashing.sha256Hex(of: url)48 let key = "\(target.key)|\(sha)|\(size)"49 var restarts = 050 while true {51 do {52 let outcome = try await attempt(url: url, target: target, sha: sha, size: size, key: key, progress: progress)53 // the file must not have changed while it travelled; if it did, what was stored is a past state: say so54 if try fingerprint(url) != before { throw DriveError.local("le fichier a été modifié pendant l’envoi") }55 return outcome56 } catch let e as DriveError where ["upload_closed", "not_found", "checksum_mismatch", "upload_incomplete"].contains(e.code) && restarts < 2 {57 restarts += 1; save(key, nil) // the server session is gone or damaged: open a new one58 }59 }60 }61 private func fingerprint(_ url: URL) throws -> String {62 let a = try FileManager.default.attributesOfItem(atPath: url.path)63 return "\((a[.size] as? NSNumber)?.int64Value ?? -1)|\((a[.modificationDate] as? Date)?.timeIntervalSince1970 ?? 0)"64 }6566 private func attempt(url: URL, target: UploadTarget, sha: String, size: Int64, key: String, progress: @escaping @Sendable (Int64, Int64) -> Void) async throws -> UploadOutcome {67 var conflicted: String? // identifier of the original when the bytes are being stored as a conflict copy68 var session: UploadSession69 if let id = savedSession(key), let s = try? await api.uploadStatus(id), s.size == size, ["active", "done", "completing"].contains(s.status) { session = s }70 else {71 do {72 switch target {73 case .newFile(let folder, let name, let policy): session = try await api.createUpload(folderID: folder, name: name, size: size, onConflict: policy)74 case .newVersion(let id, let folder, let name, let base): session = try await api.createUpload(folderID: folder, name: name, size: size, fileID: id, baseVersionID: base)75 }76 } catch let e as DriveError where e.code == "version_conflict" {77 guard case .newVersion(let id, let folder, let name, _) = target else { throw e }78 if e.extra["sha256"]?.string == sha, let cur = try await api.item(id, kind: .file) { return .stored(cur) } // same bytes on both sides: nothing to reconcile79 conflicted = id80 session = try await api.createUpload(folderID: folder, name: NameRules.conflictCopyName(name, device: deviceName), size: size, onConflict: .rename)81 }82 save(key, session.id)83 }84 progress(session.received, size)8586 if session.status == "active" {87 let handle = try FileHandle(forReadingFrom: url)88 defer { try? handle.close() }89 var offset = session.received, block = 1 << 20, failures = 090 while offset < size {91 try Task.checkCancellation()92 let want = await throttle?.preferredBlock(block) ?? block93 try handle.seek(toOffset: UInt64(offset))94 guard let data = try handle.read(upToCount: Int(min(Int64(want), size - offset))), !data.isEmpty else { throw DriveError.local("le fichier a raccourci pendant l’envoi") }95 try await throttle?.consume(data.count)96 let request = try api.chunkRequest(uploadID: session.id, offset: offset, sha256: Hashing.sha256Hex(data))97 let started = ContinuousClock.now98 do {99 let (body, response) = try await api.session.upload(for: request, from: data)100 let s: UploadSession = try APIClient.decoder.decode(UploadSession.self, from: try APIClient.check(body, response))101 offset = s.received; failures = 0102 // aim for blocks of about two seconds: small on a slow link (little to resend), large on a fast one103 let d = ContinuousClock.now - started104 let seconds = max(0.05, Double(d.components.seconds) + Double(d.components.attoseconds) / 1e18)105 block = min(UploadEngine.maxBlock, max(UploadEngine.minBlock, Int(Double(data.count) / seconds * 2)))106 progress(offset, size)107 } catch {108 let e = DriveError.wrap(error)109 if e.code == "offset_mismatch", let r = e.extra["received"]?.int { offset = r; continue } // the server knows where it is: follow it110 if e.code == "chunk_corrupt" { failures += 1; if failures > 5 { throw e }; continue }111 if e.code == "rate_limited" { try await Task.sleep(nanoseconds: UInt64(min(max(e.extra["retryAfterS"]?.int ?? 2, 1), 30)) * 1_000_000_000); continue }112 guard e.isTransient else { throw e }113 failures += 1114 if failures > 6 { throw e } // gives up for now; the session stays on the server and is resumed later115 block = max(UploadEngine.minBlock, block / 2)116 try await Task.sleep(nanoseconds: UInt64(min(30.0, pow(2.0, Double(failures)) * 0.4) * 1e9))117 if let s = try? await api.uploadStatus(session.id) { offset = s.received } // a block may have landed although its answer was lost118 }119 }120 }121 // completion is idempotent: asking again after a lost answer returns the file that was created122 do {123 let r = try await api.completeUpload(session.id, sha256: sha)124 save(key, nil)125 if let conflicted { return .conflictCopy(copy: r.file, originalID: conflicted) }126 return .stored(r.file)127 } catch let e as DriveError where e.code == "version_conflict" {128 guard case .newVersion(let id, _, let name, _) = target else { throw e }129 // the file changed while the bytes travelled: keep both, without sending the content again130 try await api.convertUploadToNewFile(session.id, name: NameRules.conflictCopyName(name, device: deviceName))131 let r = try await api.completeUpload(session.id, sha256: sha)132 save(key, nil)133 return .conflictCopy(copy: r.file, originalID: id)134 }135 }136}137