SPB Git

Repository Public

UploadManager.swift 9.0 KB 137 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Sync/UploadManager.swift3// Description: Resumable, verified uploads: the file is read block by block (never whole in memory), each block carries its SHA-256, an interrupted session resumes at the offset the server reports, and the server confirms the whole-file hash before anything counts as stored. A changed server version never gets overwritten: the bytes become a conflict copy.4import CryptoKit5import Foundation6import SPDriveCore78public enum UploadTarget: Sendable, Equatable {9    /// A new file in a folder.10    case newFile(folderID: String, name: String, onConflict: ConflictPolicy)11    /// A new version of an existing file, derived from `baseVersionID` (nil = replace whatever is there, decided by the person).12    case newVersion(fileID: String, folderID: String, name: String, baseVersionID: String?)13    var key: String {14        switch self {15        case .newFile(let f, let n, _): return "new|\(f)|\(NameRules.key(n))"16        case .newVersion(let id, _, _, let base): return "ver|\(id)|\(base ?? "-")"17        }18    }19}20public enum UploadOutcome: Sendable {21    case stored(DriveItem)22    /// The file had changed on the server: nothing was overwritten, the local content was stored next to it as `copy`.23    case conflictCopy(copy: DriveItem, originalID: String)24    public var item: DriveItem { switch self { case .stored(let i): return i; case .conflictCopy(let c, _): return c } }25}2627public final class UploadEngine: Sendable {28    public let api: APIClient29    /// Where resumable sessions are remembered across restarts (nil = no resumption after a restart).30    public let store: MetadataStore?31    public let throttle: Throttle?32    /// Name of this device, used in the name of conflict copies.33    public let deviceName: String34    public static let minBlock = 256 * 1024, maxBlock = 16 * 1024 * 10243536    public init(api: APIClient, store: MetadataStore?, throttle: Throttle? = nil, deviceName: String) { self.api = api; self.store = store; self.throttle = throttle; self.deviceName = deviceName }3738    private func savedSession(_ key: String) -> String? { try? store?.db.query("SELECT upload_id FROM upload_sessions WHERE key = ?", [.text(key)]).first?["upload_id"]?.text }39    private func save(_ key: String, _ id: String?) {40        if let id { _ = try? store?.db.run("INSERT OR REPLACE INTO upload_sessions (key, upload_id, created_at) VALUES (?, ?, ?)", [.text(key), .text(id), .double(Date().timeIntervalSince1970)]) }41        else { _ = try? store?.db.run("DELETE FROM upload_sessions WHERE key = ?", [.text(key)]) }42    }4344    /// Sends the file. `progress` receives bytes confirmed by the server and the total.45    public func upload(file url: URL, to target: UploadTarget, progress: @escaping @Sendable (Int64, Int64) -> Void = { _, _ in }) async throws -> UploadOutcome {46        let before = try fingerprint(url)47        let (sha, size) = try Hashing.sha256Hex(of: url)48        let key = "\(target.key)|\(sha)|\(size)"49        var restarts = 050        while true {51            do {52                let outcome = try await attempt(url: url, target: target, sha: sha, size: size, key: key, progress: progress)53                // the file must not have changed while it travelled; if it did, what was stored is a past state: say so54                if try fingerprint(url) != before { throw DriveError.local("le fichier a été modifié pendant l’envoi") }55                return outcome56            } catch let e as DriveError where ["upload_closed", "not_found", "checksum_mismatch", "upload_incomplete"].contains(e.code) && restarts < 2 {57                restarts += 1; save(key, nil) // the server session is gone or damaged: open a new one58            }59        }60    }61    private func fingerprint(_ url: URL) throws -> String {62        let a = try FileManager.default.attributesOfItem(atPath: url.path)63        return "\((a[.size] as? NSNumber)?.int64Value ?? -1)|\((a[.modificationDate] as? Date)?.timeIntervalSince1970 ?? 0)"64    }6566    private func attempt(url: URL, target: UploadTarget, sha: String, size: Int64, key: String, progress: @escaping @Sendable (Int64, Int64) -> Void) async throws -> UploadOutcome {67        var conflicted: String? // identifier of the original when the bytes are being stored as a conflict copy68        var session: UploadSession69        if let id = savedSession(key), let s = try? await api.uploadStatus(id), s.size == size, ["active", "done", "completing"].contains(s.status) { session = s }70        else {71            do {72                switch target {73                case .newFile(let folder, let name, let policy): session = try await api.createUpload(folderID: folder, name: name, size: size, onConflict: policy)74                case .newVersion(let id, let folder, let name, let base): session = try await api.createUpload(folderID: folder, name: name, size: size, fileID: id, baseVersionID: base)75                }76            } catch let e as DriveError where e.code == "version_conflict" {77                guard case .newVersion(let id, let folder, let name, _) = target else { throw e }78                if e.extra["sha256"]?.string == sha, let cur = try await api.item(id, kind: .file) { return .stored(cur) } // same bytes on both sides: nothing to reconcile79                conflicted = id80                session = try await api.createUpload(folderID: folder, name: NameRules.conflictCopyName(name, device: deviceName), size: size, onConflict: .rename)81            }82            save(key, session.id)83        }84        progress(session.received, size)8586        if session.status == "active" {87            let handle = try FileHandle(forReadingFrom: url)88            defer { try? handle.close() }89            var offset = session.received, block = 1 << 20, failures = 090            while offset < size {91                try Task.checkCancellation()92                let want = await throttle?.preferredBlock(block) ?? block93                try handle.seek(toOffset: UInt64(offset))94                guard let data = try handle.read(upToCount: Int(min(Int64(want), size - offset))), !data.isEmpty else { throw DriveError.local("le fichier a raccourci pendant l’envoi") }95                try await throttle?.consume(data.count)96                let request = try api.chunkRequest(uploadID: session.id, offset: offset, sha256: Hashing.sha256Hex(data))97                let started = ContinuousClock.now98                do {99                    let (body, response) = try await api.session.upload(for: request, from: data)100                    let s: UploadSession = try APIClient.decoder.decode(UploadSession.self, from: try APIClient.check(body, response))101                    offset = s.received; failures = 0102                    // aim for blocks of about two seconds: small on a slow link (little to resend), large on a fast one103                    let d = ContinuousClock.now - started104                    let seconds = max(0.05, Double(d.components.seconds) + Double(d.components.attoseconds) / 1e18)105                    block = min(UploadEngine.maxBlock, max(UploadEngine.minBlock, Int(Double(data.count) / seconds * 2)))106                    progress(offset, size)107                } catch {108                    let e = DriveError.wrap(error)109                    if e.code == "offset_mismatch", let r = e.extra["received"]?.int { offset = r; continue } // the server knows where it is: follow it110                    if e.code == "chunk_corrupt" { failures += 1; if failures > 5 { throw e }; continue }111                    if e.code == "rate_limited" { try await Task.sleep(nanoseconds: UInt64(min(max(e.extra["retryAfterS"]?.int ?? 2, 1), 30)) * 1_000_000_000); continue }112                    guard e.isTransient else { throw e }113                    failures += 1114                    if failures > 6 { throw e } // gives up for now; the session stays on the server and is resumed later115                    block = max(UploadEngine.minBlock, block / 2)116                    try await Task.sleep(nanoseconds: UInt64(min(30.0, pow(2.0, Double(failures)) * 0.4) * 1e9))117                    if let s = try? await api.uploadStatus(session.id) { offset = s.received } // a block may have landed although its answer was lost118                }119            }120        }121        // completion is idempotent: asking again after a lost answer returns the file that was created122        do {123            let r = try await api.completeUpload(session.id, sha256: sha)124            save(key, nil)125            if let conflicted { return .conflictCopy(copy: r.file, originalID: conflicted) }126            return .stored(r.file)127        } catch let e as DriveError where e.code == "version_conflict" {128            guard case .newVersion(let id, _, let name, _) = target else { throw e }129            // the file changed while the bytes travelled: keep both, without sending the content again130            try await api.convertUploadToNewFile(session.id, name: NameRules.conflictCopyName(name, device: deviceName))131            let r = try await api.completeUpload(session.id, sha256: sha)132            save(key, nil)133            return .conflictCopy(copy: r.file, originalID: id)134        }135    }136}137
Files main
Menu

Theme