SPB Git

Repository Public

NextEngineTests.swift 37.2 KB 495 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/CoreTests/NextEngineTests.swift3// Description: Tests of the 2.0 sync engine policies: retry back-off with jitter, power awareness, error-rate concurrency, priorities and pre-emption, bandwidth cap, global pause, automatic retries across an outage, crash-safe resume after a real SIGKILL, the event listener with its polling fallback, and "keep mine as a new version". Integration tests run against the real local backend.4import Foundation5import XCTest6@testable import SPDriveCore7@testable import SPDriveSync89final class PolicyTests: XCTestCase {10    func testRetryPolicyGrowsIsCappedAndStaysInsideItsJitterBand() {11        let p = RetryPolicy(base: 2, cap: 600, jitter: 0.25)12        XCTAssertEqual((1...6).map { p.nominal(attempt: $0) }, [2, 4, 8, 16, 32, 64])13        XCTAssertEqual(p.nominal(attempt: 30), 600); XCTAssertEqual(p.nominal(attempt: 0), 2)14        XCTAssertEqual(p.delay(attempt: 3, unit: 0), 6, accuracy: 1e-9); XCTAssertEqual(p.delay(attempt: 3, unit: 1), 10, accuracy: 1e-9); XCTAssertEqual(p.delay(attempt: 3, unit: 0.5), 8, accuracy: 1e-9)15        var seen = Set<Int>()16        for _ in 0..<400 { let d = p.delay(attempt: 4); XCTAssertGreaterThanOrEqual(d, 12); XCTAssertLessThanOrEqual(d, 20); seen.insert(Int(d * 10)) }17        XCTAssertGreaterThan(seen.count, 20, "delays must actually be spread") // jitter is real, not a constant18        XCTAssertLessThanOrEqual(p.delay(attempt: 50, unit: 1), 600)19        XCTAssertEqual(RetryPolicy.reconnect.nominal(attempt: 1), 1); XCTAssertEqual(RetryPolicy.reconnect.nominal(attempt: 20), 60)20    }21    func testPowerStateShapesConcurrencyAndDefersBackground() {22        let net = NetworkStatus(online: true, expensive: false, constrained: false)23        let a = ConcurrencyAdvisor()24        XCTAssertEqual(a.allowed(network: net, power: .unconstrained), 3)25        XCTAssertEqual(a.allowed(network: net, power: PowerState(onBattery: true, lowPowerMode: false, level: 0.8)), 3) // battery alone changes nothing26        XCTAssertEqual(a.allowed(network: net, power: PowerState(onBattery: true, lowPowerMode: false, level: 0.18)), 2)27        XCTAssertEqual(a.allowed(network: net, power: PowerState(onBattery: true, lowPowerMode: false, level: 0.07)), 1)28        XCTAssertEqual(a.allowed(network: net, power: PowerState(onBattery: false, lowPowerMode: true, level: 1)), 1)29        XCTAssertEqual(a.allowed(network: net, power: PowerState(onBattery: false, lowPowerMode: false, level: 0.05)), 3) // plugged in: the level does not matter30        XCTAssertTrue(PowerState(lowPowerMode: true).defersBackground); XCTAssertTrue(PowerState(onBattery: true, level: 0.1).defersBackground)31        XCTAssertFalse(PowerState(onBattery: true, level: 0.5).defersBackground)32        XCTAssertEqual(ConcurrencyAdvisor(manual: 4).allowed(network: net, power: PowerState(lowPowerMode: true)), 4) // an explicit choice of the person stands33        let real = SystemPowerSource().state // whatever this Mac says, it must be coherent34        if let l = real.level { XCTAssertTrue((0...1).contains(l)) }35        XCTAssertEqual(real.lowPowerMode, ProcessInfo.processInfo.isLowPowerModeEnabled)36    }37    func testAdvisorGivesSlotsBackWhenTransfersKeepFailing() {38        var a = ConcurrencyAdvisor()39        let net = NetworkStatus(online: true, expensive: false, constrained: false)40        for t in [10.0, 10, 20, 20, 30, 30, 40, 40, 50, 50] { a.observe(throughput: t * 1e6, saturated: true) }41        let grown = a.limit; XCTAssertGreaterThanOrEqual(grown, 5)42        a.record(success: true); a.record(success: false); a.record(success: false)43        XCTAssertEqual(a.limit, grown, "two failures are not a trend")44        a.record(success: false)45        XCTAssertEqual(a.limit, grown - 1); XCTAssertEqual(a.errorRate, 0) // window restarted after the decision46        for _ in 0..<40 { a.record(success: false) }47        XCTAssertEqual(a.limit, 1); XCTAssertEqual(a.allowed(network: net), 1) // never below one48        let before = a.limit49        a.observe(throughput: 90e6, saturated: true); a.observe(throughput: 90e6, saturated: true)50        XCTAssertEqual(a.limit, before, "growth waits for calm windows after errors")51        for t in [10.0, 10, 10, 20, 20, 30, 30] { a.observe(throughput: t * 1e6, saturated: true) }52        XCTAssertGreaterThan(a.limit, before) // and comes back when the link behaves53        var m = ConcurrencyAdvisor(manual: 5); for _ in 0..<20 { m.record(success: false) }54        XCTAssertEqual(m.allowed(network: net), 5)55    }56    func testListing1000ItemsFromTheMirrorIsFast() throws {57        let dir = FileManager.default.temporaryDirectory.appendingPathComponent("spdrive-perf-\(UUID().uuidString)"); defer { try? FileManager.default.removeItem(at: dir) }58        let s = try MetadataStore(url: dir.appendingPathComponent("m.sqlite"))59        try s.upsert([item("root", "", parent: nil, kind: .folder, rev: 1), item("d", "Mille", parent: "root", kind: .folder, rev: 2)])60        try s.upsert((0..<1000).map { item("f\($0)", "Document \($0).pdf", parent: "d", rev: Int64($0 + 3), size: 1000) })61        var best = Double.infinity62        for _ in 0..<10 { let t = Date(); let rows = try s.children(of: "d"); best = min(best, Date().timeIntervalSince(t)); XCTAssertEqual(rows.count, 1000) }63        print(String(format: "MEASURE mirror-list-1000: %.2f ms (best of 10)", best * 1000))64        XCTAssertLessThan(best, 0.1)65    }66}6768/// A scripted event source: each `connect()` hands out the next prepared connection.69final class FakeEventSource: RemoteEventSource, @unchecked Sendable {70    private let lock = NSLock()71    private var live: AsyncThrowingStream<RemoteEvent, Error>.Continuation?72    private(set) var connections = 073    var refuse = false74    func connect() -> AsyncThrowingStream<RemoteEvent, Error> {75        AsyncThrowingStream { c in76            lock.withLock { connections += 1 }77            if refuse { c.finish(throwing: DriveError.offline("refused")); return }78            lock.withLock { live = c }79        }80    }81    func send(_ e: RemoteEvent) { _ = lock.withLock { live }?.yield(e) }82    func drop() { let c = lock.withLock { let c = live; live = nil; return c }; c?.finish(throwing: DriveError.offline("dropped")) }83    var count: Int { lock.withLock { connections } }84}8586final class EventListenerTests: XCTestCase {87    func testPollingOnlyReportsHeadMovesOnce() async throws {88        let head = Box<Int64>(5), seen = Box<[Int64]>([])89        let l = EventListener(source: nil, pollInterval: 0.05, head: { head.value }, onChange: { h in seen.update { $0.append(h) } })90        await l.start()91        try await waitUntil { seen.value == [5] }92        try await Task.sleep(nanoseconds: 300_000_000)93        XCTAssertEqual(seen.value, [5], "an unchanged head is not news")94        head.value = 995        try await waitUntil { seen.value == [5, 9] }96        let ch = await l.channel; XCTAssertEqual(ch, .polling)97        await l.stop()98        head.value = 12; try await Task.sleep(nanoseconds: 250_000_000)99        XCTAssertEqual(seen.value, [5, 9]); let off = await l.channel; XCTAssertEqual(off, .off)100    }101    func testStreamTakesOverThenPollingCoversADropAndReconnectBacksOff() async throws {102        let src = FakeEventSource(), head = Box<Int64>(1), seen = Box<[Int64]>([]), notes = Box<[String]>([])103        let l = EventListener(source: src, pollInterval: 0.05, retry: RetryPolicy(base: 0.3, cap: 1, jitter: 0.2), head: { head.value },104                              onNotify: { k in notes.update { $0.append(k) } }, onChange: { h in seen.update { $0.append(h) } })105        await l.start()106        try await waitUntil { src.count == 1 }107        src.send(.hello(head: 1))108        try await waitUntil { await l.channel == .stream }109        try await waitUntil { seen.value == [1] }110        let pollsWhenLive = await l.polls111        src.send(.change(head: 2)); src.send(.change(head: 2)); src.send(.notify(kind: "request.submission")); src.send(.change(head: 3))112        try await waitUntil { seen.value == [1, 2, 3] }113        XCTAssertEqual(notes.value, ["request.submission"])114        try await Task.sleep(nanoseconds: 250_000_000)115        let pollsStill = await l.polls; XCTAssertEqual(pollsStill, pollsWhenLive, "no polling while the stream is live")116        // the stream drops: polling takes over at once, and reports what the stream can no longer say117        head.value = 3 // what the server really has at that point118        src.refuse = true; src.drop()119        try await waitUntil { await l.channel == .polling }120        head.value = 7121        try await waitUntil { seen.value == [1, 2, 3, 7] }122        // reconnection attempts are spaced by the back-off (0.3 s, 0.6 s, 1 s…): far fewer than one per poll tick123        try await Task.sleep(nanoseconds: 1_200_000_000)124        let attempts = src.count; XCTAssertGreaterThanOrEqual(attempts, 3); XCTAssertLessThanOrEqual(attempts, 5)125        // the server is back: the stream is live again and polling stops126        src.refuse = false127        try await waitUntil(timeout: 5) { src.count > attempts }128        src.send(.hello(head: 8))129        try await waitUntil { await l.channel == .stream }130        try await waitUntil { seen.value.last == 8 }131        let reconnects = await l.reconnects; XCTAssertGreaterThanOrEqual(reconnects, 2)132        await l.stop()133    }134}135136final class EngineEventTests: LiveCase {137    func testEnginePicksUpRemoteChangesFromThePollingFallbackWithoutBeingPoked() async throws {138        let s = try store()139        let engine = SyncEngine(api: api, store: s, network: NetworkMonitor(), workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests", eventPollInterval: 1)140        await engine.start()141        let fid = folder.id142        try await waitUntil { (try? s.item(fid)) != nil }143        try await waitUntil { let st = await engine.currentStatus(); return st.phase == .idle && st.events == .polling }144        try await Task.sleep(nanoseconds: 1_500_000_000) // let the first poll settle145        let made = try await api.createFolder(parent: folder.id, name: "vu par le sondage")146        let t0 = Date()147        try await waitUntil(timeout: 6, "the engine's own loop rests at least 10 s: only the listener can bring this in time") { (try? s.item(made.id)) != nil }148        XCTAssertLessThan(Date().timeIntervalSince(t0), 6)149        // paused: the listener is off; resumed: it is back150        await engine.pause()151        try await waitUntil { await engine.currentStatus().events == .off }152        await engine.resume()153        try await waitUntil { await engine.currentStatus().events == .polling }154        await engine.stop()155    }156    func testEngineFollowsAPluggedInEventStream() async throws {157        let s = try store(), src = FakeEventSource()158        let engine = SyncEngine(api: api, store: s, network: NetworkMonitor(), workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests", eventPollInterval: 600)159        await engine.setEventSource(src)160        await engine.start()161        let fid = folder.id162        try await waitUntil { (try? s.item(fid)) != nil }163        try await waitUntil { src.count >= 1 }164        src.send(.hello(head: try await api.syncCursor().head))165        try await waitUntil { await engine.currentStatus().events == .stream }166        try await waitUntil { await engine.currentStatus().phase == .idle }167        let made = try await api.createFolder(parent: folder.id, name: "annoncé par le flux")168        try await Task.sleep(nanoseconds: 300_000_000)169        XCTAssertNil(try s.item(made.id)) // nobody said anything yet (polling is 10 minutes away)170        src.send(.change(head: try await api.syncCursor().head))171        try await waitUntil(timeout: 5) { (try? s.item(made.id)) != nil }172        await engine.stop()173    }174}175176/// The real stream of the 2.0 server (SSE through SPDriveNext) plugged into the engine.177final class LiveEventStreamTests: LiveCase {178    func testRealStreamBringsARemoteChangeAtOnceAndPollingCoversAKilledStream() async throws {179        let s = try store()180        let engine = SyncEngine(api: api, store: s, network: NetworkMonitor(), workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests", eventPollInterval: 2)181        await engine.setEventSource(NextEventSource(api: api, idleTimeout: 30)) // above the server's 25 s ping, so a healthy stream is never cut182        await engine.start()183        let fid = folder.id184        try await waitUntil { (try? s.item(fid)) != nil }185        try await waitUntil(timeout: 10, "stream live") { await engine.currentStatus().events == .stream }186        try await waitUntil { await engine.currentStatus().phase == .idle }187        try await Task.sleep(nanoseconds: 1_200_000_000)188        // another client of the account changes something: the Mac learns of it without being poked and well before any poll189        let other = try Backend.api()190        var t0 = Date()191        let made = try await other.createFolder(parent: folder.id, name: "poussé par le serveur")192        try await waitUntil(timeout: 4, "pushed change mirrored") { (try? s.item(made.id)) != nil }193        let pushed = Date().timeIntervalSince(t0)194        XCTAssertLessThan(pushed, 1.9, "faster than the 2 s poll of this test (and than the 30 s poll of the app): it came from the stream")195        // the server goes away. A server stopped with SIGTERM may leave the open stream hanging without closing it: only the196        // silence (no ping) gives it away. Within the idle timeout the listener declares the stream dead and polls.197        try Backend.serverDown()198        var up = false; defer { if !up { try? Backend.serverUp() } }199        let tDown = Date()200        try await waitUntil(timeout: 45, "fallback to polling") { await engine.currentStatus().events == .polling }201        let noticed = Date().timeIntervalSince(tDown)202        try Backend.serverUp(); up = true203        // while the stream is still reconnecting (back-off), a change is found by the poll204        t0 = Date()205        let later = try await other.createFolder(parent: folder.id, name: "vu après la coupure")206        try await waitUntil(timeout: 15, "change after the outage mirrored") { (try? s.item(later.id)) != nil }207        // and the stream comes back by itself208        try await waitUntil(timeout: 70, "stream live again") { await engine.currentStatus().events == .stream }209        let again = try await other.createFolder(parent: folder.id, name: "poussé de nouveau")210        try await waitUntil(timeout: 4) { (try? s.item(again.id)) != nil }211        print(String(format: "MEASURE sse: pushed change mirrored in %.2f s; dead stream noticed after %.1f s", pushed, noticed))212        await engine.stop()213    }214}215216final class QueuePolicyTests: LiveCase {217    private func finishOrder(_ q: TransferQueue) -> (Box<[String]>, Task<Void, Never>) {218        let order = Box<[String]>([])219        let w = Task { for await e in await q.events() { if case .finished(let t, _) = e { order.update { $0.append(t.name) } } } }220        return (order, w)221    }222    func testUserInitiatedTransfersPreemptAndPrecedeBackgroundOnes() async throws {223        let s = try store(), q = TransferQueue(api: api, store: s, workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests")224        let (order, w) = finishOrder(q); defer { w.cancel() }225        await q.setManualConcurrency(1)226        await q.setLimits(uploadBytesPerSecond: 1024 * 1024, downloadBytesPerSecond: 0)227        var shas: [String: String] = [:]228        for i in 1...3 { let f = try randomFile("fond\(i).bin", bytes: 2 * 1024 * 1024 + i); shas["fond\(i).bin"] = f.sha; _ = try await q.enqueueUpload(file: f.url, parent: folder.id, priority: TransferPriority.background) }229        await q.start()230        try await waitUntil { await q.list().contains { $0.name == "fond1.bin" && $0.state == .running && $0.done > 0 } }231        let user = try randomFile("demandé.bin", bytes: 700 * 1024); shas["demandé.bin"] = user.sha232        let u = try await q.enqueueUpload(file: user.url, parent: folder.id, priority: TransferPriority.userInitiated)233        // the running background transfer is put back in the queue (not failed, progress kept) and the user's goes first234        try await waitUntil { await q.get(u.id)?.state == .running }235        let stopped = await q.list().first { $0.name == "fond1.bin" }; XCTAssertEqual(stopped?.state, .queued); XCTAssertNil(stopped?.error)236        try await waitUntil(timeout: 60) { order.value.count == 4 }237        XCTAssertEqual(order.value.first, "demandé.bin")238        // "send this one first" on a queued transfer239        let files = try await api.listFolder(folder.id).files240        XCTAssertEqual(files.count, 4); for f in files { XCTAssertEqual(f.sha256, shas[f.name], f.name) } // pre-emption lost and duplicated nothing241        await q.stop()242    }243    func testSendFirstReordersTheQueue() async throws {244        let s = try store(), q = TransferQueue(api: api, store: s, workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests")245        let (order, w) = finishOrder(q); defer { w.cancel() }246        await q.setManualConcurrency(1)247        var ids: [String] = []248        for i in 1...4 { ids.append(try await q.enqueueUpload(file: try text("f\(i).txt", "contenu \(i)"), parent: folder.id).id) }249        await q.setPriority(ids[3], TransferPriority.urgent)250        await q.start()251        try await waitUntil { order.value.count == 4 }252        XCTAssertEqual(order.value, ["f4.txt", "f1.txt", "f2.txt", "f3.txt"])253        await q.stop()254    }255    func testLowPowerModeRunsOnlyWhatThePersonAskedForOneAtATime() async throws {256        let power = FixedPowerSource(PowerState(onBattery: true, lowPowerMode: true, level: 0.5))257        let s = try store(), q = TransferQueue(api: api, store: s, workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests", power: power)258        let bg = try await q.enqueueUpload(file: try text("fond.txt", "arrière-plan"), parent: folder.id, priority: TransferPriority.background)259        let u1 = try await q.enqueueUpload(file: try text("a.txt", "demandé a"), parent: folder.id, priority: TransferPriority.userInitiated)260        let u2 = try await q.enqueueUpload(file: try text("b.txt", "demandé b"), parent: folder.id, priority: TransferPriority.userInitiated)261        await q.start()262        let pol = await q.policy(); XCTAssertEqual(pol.concurrency, 1); XCTAssertTrue(pol.deferringBackground)263        try await waitUntil { let a = await q.get(u1.id)?.state, b = await q.get(u2.id)?.state; return a == .done && b == .done }264        try await Task.sleep(nanoseconds: 2_500_000_000) // more than one queue tick265        let waiting = await q.get(bg.id); XCTAssertEqual(waiting?.state, .queued); XCTAssertNil(waiting?.error) // deferred, not failed266        var names = try await api.listFolder(folder.id).files.map(\.name).sorted(); XCTAssertEqual(names, ["a.txt", "b.txt"])267        // the person turns the power policy off for this queue: the deferred transfer runs268        await q.setPowerAware(false)269        try await waitUntil { await q.get(bg.id)?.state == .done }270        await q.setPowerAware(true)271        // back on mains: nothing is held any more272        let bg2 = try await q.enqueueUpload(file: try text("fond2.txt", "arrière-plan 2"), parent: folder.id, priority: TransferPriority.background)273        try await Task.sleep(nanoseconds: 600_000_000)274        let held = await q.get(bg2.id); XCTAssertEqual(held?.state, .queued)275        power.state = .unconstrained276        try await waitUntil(timeout: 10) { await q.get(bg2.id)?.state == .done } // picked up at the next tick, without anyone asking277        names = try await api.listFolder(folder.id).files.map(\.name).sorted(); XCTAssertEqual(names, ["a.txt", "b.txt", "fond.txt", "fond2.txt"])278        await q.stop()279    }280    func testBandwidthCapHoldsARealUploadAndDownload() async throws {281        let s = try store(), q = TransferQueue(api: api, store: s, workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests")282        let f = try randomFile("plafonné.bin", bytes: 4 * 1024 * 1024)283        await q.setLimits(uploadBytesPerSecond: 1024 * 1024, downloadBytesPerSecond: 1024 * 1024)284        await q.start()285        var t0 = Date()286        let up = try await q.enqueueUpload(file: f.url, parent: folder.id)287        try await waitUntil(timeout: 30) { await q.get(up.id)?.state == .done }288        let upSeconds = Date().timeIntervalSince(t0)289        XCTAssertGreaterThan(upSeconds, 2.6, "4 MiB at 1 MiB/s cannot take less than ~3 s (one second of burst allowed)"); XCTAssertLessThan(upSeconds, 9)290        let item = try await api.listFolder(folder.id).files.first!291        XCTAssertEqual(item.sha256, f.sha)292        // downloads are paced on the delivery of the bytes to the engine: what the loader buffered does not escape the cap293        let g = try randomFile("retour-source.bin", bytes: 6 * 1024 * 1024)294        let big = try await upload(g.url)295        t0 = Date()296        let dest = dir.appendingPathComponent("retour.bin")297        let down = try await q.enqueueDownload(big, to: dest)298        try await waitUntil(timeout: 30) { await q.get(down.id)?.state == .done }299        let downSeconds = Date().timeIntervalSince(t0)300        XCTAssertGreaterThan(downSeconds, 4.5, "6 MiB at 1 MiB/s, minus one second of burst"); XCTAssertEqual(try Hashing.sha256Hex(of: dest).sha256, g.sha)301        // cap lifted: the same upload is several times faster302        await q.setLimits(uploadBytesPerSecond: 0, downloadBytesPerSecond: 0)303        t0 = Date()304        let free = try await q.enqueueUpload(file: f.url, parent: folder.id, name: "libre.bin")305        try await waitUntil(timeout: 30) { await q.get(free.id)?.state == .done }306        XCTAssertLessThan(Date().timeIntervalSince(t0), upSeconds / 2)307        print(String(format: "MEASURE cap 1 MiB/s: upload 4 MiB in %.2f s, download 6 MiB in %.2f s; uncapped upload %.2f s", upSeconds, downSeconds, Date().timeIntervalSince(t0)))308        await q.stop()309    }310    func testGlobalPauseFreezesTheServerSideOffsetAndResumeFinishes() async throws {311        let s = try store(), q = TransferQueue(api: api, store: s, workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests")312        let f = try randomFile("pause.bin", bytes: 8 * 1024 * 1024)313        await q.setLimits(uploadBytesPerSecond: 2 * 1024 * 1024, downloadBytesPerSecond: 0)314        let t = try await q.enqueueUpload(file: f.url, parent: folder.id)315        await q.start()316        try await waitUntil { (await q.get(t.id)?.done ?? 0) > 1024 * 1024 }317        await q.pauseAll()318        let session = try XCTUnwrap(try s.db.query("SELECT upload_id FROM upload_sessions").first?["upload_id"]?.text)319        try await Task.sleep(nanoseconds: 700_000_000) // a block in flight may still land320        let a = try await api.uploadStatus(session).received321        try await Task.sleep(nanoseconds: 1_500_000_000)322        let b = try await api.uploadStatus(session).received323        XCTAssertEqual(a, b, "nothing travels while paused"); XCTAssertGreaterThan(a, 0); XCTAssertLessThan(a, Int64(8 * 1024 * 1024))324        let st = await q.get(t.id); XCTAssertEqual(st?.state, .queued); let paused = await q.isPaused; XCTAssertTrue(paused)325        let late = try await q.enqueueUpload(file: try text("pendant.txt", "ajouté pendant la pause"), parent: folder.id)326        try await Task.sleep(nanoseconds: 500_000_000)327        let lateState = await q.get(late.id)?.state; XCTAssertEqual(lateState, .queued) // nothing starts either328        await q.setLimits(uploadBytesPerSecond: 0, downloadBytesPerSecond: 0)329        await q.resumeAll()330        try await waitUntil(timeout: 30) { let a = await q.get(t.id)?.state, b = await q.get(late.id)?.state; return a == .done && b == .done }331        let done = try await api.uploadStatus(session); XCTAssertEqual(done.status, "done") // the same server session was continued332        let sha = try await api.listFolder(folder.id).files.first { $0.name == "pause.bin" }?.sha256; XCTAssertEqual(sha, f.sha)333        await q.stop()334    }335    func testOutageRetriesBackOffWithJitterGiveSlotsBackAndRecoverAlone() async throws {336        let s = try store(), q = TransferQueue(api: api, store: s, workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests")337        await q.setRetryPolicy(RetryPolicy(base: 1, cap: 4, jitter: 0.5))338        var ids: [String] = []339        for i in 1...4 { ids.append(try await q.enqueueUpload(file: try text("r\(i).txt", "retry \(i)"), parent: folder.id).id) }340        try Backend.serverDown()341        var up = false; defer { if !up { try? Backend.serverUp() } }342        await q.start()343        // every transfer fails for a transient reason: none is declared failed, each gets a later next try, delays differ (jitter)344        try await waitUntil(timeout: 30) { ((try? s.db.query("SELECT COUNT(*) AS n FROM transfers WHERE attempts >= 2 AND state = 'queued'").first?["n"]?.int) ?? 0) == 4 }345        let rows = try s.db.query("SELECT attempts, updated_at, error, state FROM transfers")346        let now = Date().timeIntervalSince1970347        var waits = Set<Int>()348        for r in rows {349            XCTAssertEqual(r["state"]?.text, "queued"); XCTAssertNotNil(r["error"]?.text)350            let attempts = Int(r["attempts"]?.int ?? 0), next = (r["updated_at"]?.double ?? 0) - now351            XCTAssertLessThanOrEqual(next, RetryPolicy(base: 1, cap: 4, jitter: 0.5).nominal(attempt: attempts) * 1.5 + 0.05) // never beyond the band352            waits.insert(Int((r["updated_at"]?.double ?? 0) * 1000))353        }354        XCTAssertGreaterThan(waits.count, 1, "retries must not be synchronised")355        let pol = await q.policy()356        XCTAssertLessThan(pol.measuredLimit, 3, "repeated failures give slots back"); XCTAssertEqual(pol.concurrency, pol.measuredLimit)357        // the server comes back; nobody kicks the queue: the back-off itself brings every transfer in358        try Backend.serverUp(); up = true359        try await waitUntil(timeout: 30) { let m = await q.summary(); return m.queued + m.running == 0 }360        for id in ids { let st = await q.get(id)?.state; XCTAssertEqual(st, .done) }361        let names = try await api.listFolder(folder.id).files.map(\.name).sorted(); XCTAssertEqual(names, ["r1.txt", "r2.txt", "r3.txt", "r4.txt"]) // each exactly once362        await q.stop()363    }364}365366/// The transfer queue runs in another process (spdrive-cli), which is killed with SIGKILL in the middle of a transfer.367final class CrashResumeTests: LiveCase {368    private func cli() throws -> URL {369        let u = Bundle(for: CrashResumeTests.self).bundleURL.deletingLastPathComponent().appendingPathComponent("spdrive-cli")370        guard FileManager.default.isExecutableFile(atPath: u.path) else { XCTFail("spdrive-cli not built next to the tests: \(u.path)"); throw DriveError.local("no cli") }371        return u372    }373    // `Process.waitUntilExit()` is not used: called from a Swift concurrency thread it can wait forever (seen once in this suite).374    private func spawn(_ args: [String]) throws -> Process {375        let b = try Backend.require()376        let p = Process(); p.executableURL = try cli(); p.arguments = args377        p.environment = ["SPDRIVE_URL": b.url.absoluteString, "SPDRIVE_TOKEN": b.token]378        p.standardOutput = FileHandle.nullDevice; p.standardError = FileHandle.nullDevice379        try p.run(); return p380    }381    func testUploadKilledMidTransferResumesOnTheSameServerSession() async throws {382        let db = dir.appendingPathComponent("meta.sqlite"), work = dir.appendingPathComponent("work")383        let big = try randomFile("tué.bin", bytes: 12 * 1024 * 1024)384        let p = try spawn(["queue-put", db.path, work.path, big.url.path, folder.id, String(2 * 1024 * 1024)])385        defer { if p.isRunning { kill(p.processIdentifier, SIGKILL) } }386        let s = try MetadataStore(url: db)387        let api = self.api!388        try await waitUntil(timeout: 30, "the other process is uploading") {389            guard let id = try? s.db.query("SELECT upload_id FROM upload_sessions").first?["upload_id"]?.text else { return false }390            return ((try? await api.uploadStatus(id).received) ?? 0) > 3 * 1024 * 1024391        }392        let session = try XCTUnwrap(try s.db.query("SELECT upload_id FROM upload_sessions").first?["upload_id"]?.text)393        XCTAssertEqual(kill(p.processIdentifier, SIGKILL), 0); try await waitUntil(timeout: 10, "the killed process is gone") { !p.isRunning }394        XCTAssertEqual(p.terminationReason, .uncaughtSignal); XCTAssertEqual(p.terminationStatus, SIGKILL) // really killed, no clean-up ran395        let row = try XCTUnwrap(try s.db.query("SELECT id, state FROM transfers").first)396        XCTAssertEqual(row["state"]?.text, "running", "the crash leaves the row as it was")397        let before = try await api.uploadStatus(session)398        XCTAssertEqual(before.status, "active"); XCTAssertGreaterThan(before.received, 3 * 1024 * 1024); XCTAssertLessThan(before.received, Int64(12 * 1024 * 1024))399        let none = try await api.listFolder(folder.id).files; XCTAssertTrue(none.isEmpty) // nothing half-written is visible400        // "relaunch": a new queue on the same database and work directory401        let q = TransferQueue(api: api, store: s, workDirectory: work, deviceName: "Tests")402        let firstProgress = Box<Int64?>(nil)403        let id = row["id"]?.text ?? ""404        let w = Task { for await e in await q.events() { if case .changed(let t) = e, t.id == id, t.state == .running, t.done > 0, firstProgress.value == nil { firstProgress.value = t.done } } }405        defer { w.cancel() }406        await q.start()407        try await waitUntil(timeout: 60) { await q.get(id)?.state == .done }408        let after = try await api.uploadStatus(session); XCTAssertEqual(after.status, "done") // the same session was finished, not a new one409        let files = try await api.listFolder(folder.id).files410        XCTAssertEqual(files.map(\.name), ["tué.bin"]); XCTAssertEqual(files.first?.sha256, big.sha) // once, intact411        XCTAssertEqual(try s.db.query("SELECT COUNT(*) AS n FROM upload_sessions").first?["n"]?.int, 0)412        print("MEASURE kill-resume upload: server had \(before.received) of \(12 * 1024 * 1024) bytes at SIGKILL; first progress after relaunch \(firstProgress.value ?? -1)")413        await q.stop()414    }415    func testDownloadKilledMidTransferResumesFromItsPartialFile() async throws {416        let db = dir.appendingPathComponent("meta.sqlite"), work = dir.appendingPathComponent("work"), dest = dir.appendingPathComponent("out/reçu.bin")417        let big = try randomFile("source.bin", bytes: 12 * 1024 * 1024)418        let item = try await upload(big.url)419        let p = try spawn(["queue-get", db.path, work.path, item.id, dest.path, String(2 * 1024 * 1024)])420        defer { if p.isRunning { kill(p.processIdentifier, SIGKILL) } }421        let partials = work.appendingPathComponent("partials")422        @Sendable func partialSize() -> Int64 {423            guard let f = (try? FileManager.default.contentsOfDirectory(at: partials, includingPropertiesForKeys: nil))?.first else { return 0 }424            return ((try? FileManager.default.attributesOfItem(atPath: f.path)[.size]) as? NSNumber)?.int64Value ?? 0425        }426        try await waitUntil(timeout: 30, "the other process is downloading") { partialSize() > 3 * 1024 * 1024 }427        XCTAssertEqual(kill(p.processIdentifier, SIGKILL), 0); try await waitUntil(timeout: 10, "the killed process is gone") { !p.isRunning }428        XCTAssertEqual(p.terminationReason, .uncaughtSignal)429        let had = partialSize()430        XCTAssertGreaterThan(had, 3 * 1024 * 1024); XCTAssertLessThan(had, Int64(12 * 1024 * 1024))431        XCTAssertFalse(FileManager.default.fileExists(atPath: dest.path), "an unfinished download is never placed at its destination")432        let s = try MetadataStore(url: db)433        let id = try XCTUnwrap(try s.db.query("SELECT id FROM transfers").first?["id"]?.text)434        let q = TransferQueue(api: api, store: s, workDirectory: work, deviceName: "Tests")435        let lowest = Box<Int64>(.max)436        let w = Task { for await e in await q.events() { if case .changed(let t) = e, t.id == id, t.state == .running, t.done > 0 { lowest.update { $0 = min($0, t.done) } } } }437        defer { w.cancel() }438        await q.setLimits(uploadBytesPerSecond: 0, downloadBytesPerSecond: 2 * 1024 * 1024)439        await q.start()440        try await waitUntil(timeout: 60) { await q.get(id)?.state == .done }441        XCTAssertEqual(try Hashing.sha256Hex(of: dest).sha256, big.sha)442        XCTAssertEqual(partialSize(), 0) // the partial became the file443        if lowest.value != .max { XCTAssertGreaterThanOrEqual(lowest.value, had, "progress restarts from the partial file, not from zero") }444        print("MEASURE kill-resume download: partial had \(had) bytes at SIGKILL; lowest progress reported after relaunch \(lowest.value == .max ? -1 : lowest.value)")445        await q.stop()446    }447}448449final class ConflictVersionTests: LiveCase {450    func testKeepMineAsNewVersionKeepsTheServerVersionInHistoryAndOverwritesNothingSilently() async throws {451        let s = try store(), work = dir.appendingPathComponent("work")452        let base = try await upload(try text("contrat.txt", "version de départ"))453        // a real conflict: another device replaces the content, then this device sends an edit derived from the old version454        let other = try await UploadEngine(api: api, store: nil, deviceName: "Autre").upload(file: try text("o.txt", "version de l'autre appareil"), to: .newVersion(fileID: base.id, folderID: folder.id, name: "contrat.txt", baseVersionID: base.versionID)).item455        let outcome = try await UploadEngine(api: api, store: s, deviceName: "Mac de test").upload(file: try text("m.txt", "version de ce Mac"), to: .newVersion(fileID: base.id, folderID: folder.id, name: "contrat.txt", baseVersionID: base.versionID))456        guard case .conflictCopy(let copy, _) = outcome else { return XCTFail("expected a conflict") }457        let untouched = try await serverBytes(base.id); XCTAssertEqual(String(decoding: untouched, as: UTF8.self), "version de l'autre appareil")458        let resolver = ConflictResolver(api: api, store: s)459        let c = try resolver.record(itemID: base.id, itemName: "contrat.txt", copy: copy)460        // while the person hesitates, the file changes a third time: the resolution must refuse, not overwrite461        let stale = SyncConflict(id: c.id, itemID: c.itemID, itemName: c.itemName, copyID: c.copyID, copyName: c.copyName, kind: c.kind, detail: c.detail, detectedAt: c.detectedAt, resolvedAt: nil, resolution: nil)462        let updated = try await resolver.resolveKeepingMineAsNewVersion(stale, workDirectory: work, deviceName: "Mac de test")463        XCTAssertEqual(updated.id, base.id); XCTAssertEqual(updated.versionSeq, 3)464        let now = try await serverBytes(base.id); XCTAssertEqual(String(decoding: now, as: UTF8.self), "version de ce Mac")465        let details = try await api.fileDetails(base.id)466        XCTAssertEqual(details.versions.count, 3) // start, other device, this Mac: the whole history is there467        let v2 = try XCTUnwrap(details.versions.first { $0.seq == 2 }); XCTAssertEqual(v2.id, other.versionID)468        let kept = try await serverBytes(base.id, version: v2.id); XCTAssertEqual(String(decoding: kept, as: UTF8.self), "version de l'autre appareil") // set aside, still readable469        let live = try await api.listFolder(folder.id).files; XCTAssertEqual(live.map(\.id), [base.id]) // the copy left the folder…470        let trashedCopy = try await api.item(copy.id, kind: .file); XCTAssertNotNil(trashedCopy?.trashedAt) // …for the trash, not for nowhere471        XCTAssertEqual(try resolver.openCount(), 0); XCTAssertEqual(try resolver.conflicts().first?.resolution, "keepMineVersion")472        XCTAssertEqual(try s.item(base.id)?.versionSeq, 3) // the mirror knows473        XCTAssertFalse(FileManager.default.fileExists(atPath: work.appendingPathComponent("conflicts/\(c.id)").path)) // nothing left behind474    }475    func testKeepMineAsNewVersionRefusesWhenTheFileChangedAgain() async throws {476        let s = try store(), work = dir.appendingPathComponent("work")477        let base = try await upload(try text("plan.txt", "v1"))478        _ = try await UploadEngine(api: api, store: nil, deviceName: "Autre").upload(file: try text("o.txt", "v2 autre"), to: .newVersion(fileID: base.id, folderID: folder.id, name: "plan.txt", baseVersionID: base.versionID))479        let outcome = try await UploadEngine(api: api, store: s, deviceName: "Mac").upload(file: try text("m.txt", "v2 mienne"), to: .newVersion(fileID: base.id, folderID: folder.id, name: "plan.txt", baseVersionID: base.versionID))480        guard case .conflictCopy(let copy, _) = outcome else { return XCTFail("expected a conflict") }481        let resolver = ConflictResolver(api: api, store: s)482        let c = try resolver.record(itemID: base.id, itemName: "plan.txt", copy: copy)483        // identical content on both sides resolves without sending a byte and without a new version484        let twin = try await upload(try text("twin.txt", "même contenu"))485        let twinCopy = try await upload(try text("twin (copie).txt", "même contenu"))486        let c2 = try resolver.record(itemID: twin.id, itemName: "twin.txt", copy: twinCopy)487        let same = try await resolver.resolveKeepingMineAsNewVersion(c2, workDirectory: work, deviceName: "Mac")488        XCTAssertEqual(same.versionSeq, 1); let gone = try await api.item(twinCopy.id, kind: .file); XCTAssertNotNil(gone?.trashedAt)489        // the first conflict is still open and resolves normally490        _ = try await resolver.resolveKeepingMineAsNewVersion(c, workDirectory: work, deviceName: "Mac")491        let bytes = try await serverBytes(base.id); XCTAssertEqual(String(decoding: bytes, as: UTF8.self), "v2 mienne")492        XCTAssertEqual(try resolver.openCount(), 0)493    }494}495
Files main
Menu

Theme