Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/CoreTests/NextEngineTests.swift3// Description: Tests of the 2.0 sync engine policies: retry back-off with jitter, power awareness, error-rate concurrency, priorities and pre-emption, bandwidth cap, global pause, automatic retries across an outage, crash-safe resume after a real SIGKILL, the event listener with its polling fallback, and "keep mine as a new version". Integration tests run against the real local backend.4import Foundation5import XCTest6@testable import SPDriveCore7@testable import SPDriveSync89final class PolicyTests: XCTestCase {10 func testRetryPolicyGrowsIsCappedAndStaysInsideItsJitterBand() {11 let p = RetryPolicy(base: 2, cap: 600, jitter: 0.25)12 XCTAssertEqual((1...6).map { p.nominal(attempt: $0) }, [2, 4, 8, 16, 32, 64])13 XCTAssertEqual(p.nominal(attempt: 30), 600); XCTAssertEqual(p.nominal(attempt: 0), 2)14 XCTAssertEqual(p.delay(attempt: 3, unit: 0), 6, accuracy: 1e-9); XCTAssertEqual(p.delay(attempt: 3, unit: 1), 10, accuracy: 1e-9); XCTAssertEqual(p.delay(attempt: 3, unit: 0.5), 8, accuracy: 1e-9)15 var seen = Set<Int>()16 for _ in 0..<400 { let d = p.delay(attempt: 4); XCTAssertGreaterThanOrEqual(d, 12); XCTAssertLessThanOrEqual(d, 20); seen.insert(Int(d * 10)) }17 XCTAssertGreaterThan(seen.count, 20, "delays must actually be spread") // jitter is real, not a constant18 XCTAssertLessThanOrEqual(p.delay(attempt: 50, unit: 1), 600)19 XCTAssertEqual(RetryPolicy.reconnect.nominal(attempt: 1), 1); XCTAssertEqual(RetryPolicy.reconnect.nominal(attempt: 20), 60)20 }21 func testPowerStateShapesConcurrencyAndDefersBackground() {22 let net = NetworkStatus(online: true, expensive: false, constrained: false)23 let a = ConcurrencyAdvisor()24 XCTAssertEqual(a.allowed(network: net, power: .unconstrained), 3)25 XCTAssertEqual(a.allowed(network: net, power: PowerState(onBattery: true, lowPowerMode: false, level: 0.8)), 3) // battery alone changes nothing26 XCTAssertEqual(a.allowed(network: net, power: PowerState(onBattery: true, lowPowerMode: false, level: 0.18)), 2)27 XCTAssertEqual(a.allowed(network: net, power: PowerState(onBattery: true, lowPowerMode: false, level: 0.07)), 1)28 XCTAssertEqual(a.allowed(network: net, power: PowerState(onBattery: false, lowPowerMode: true, level: 1)), 1)29 XCTAssertEqual(a.allowed(network: net, power: PowerState(onBattery: false, lowPowerMode: false, level: 0.05)), 3) // plugged in: the level does not matter30 XCTAssertTrue(PowerState(lowPowerMode: true).defersBackground); XCTAssertTrue(PowerState(onBattery: true, level: 0.1).defersBackground)31 XCTAssertFalse(PowerState(onBattery: true, level: 0.5).defersBackground)32 XCTAssertEqual(ConcurrencyAdvisor(manual: 4).allowed(network: net, power: PowerState(lowPowerMode: true)), 4) // an explicit choice of the person stands33 let real = SystemPowerSource().state // whatever this Mac says, it must be coherent34 if let l = real.level { XCTAssertTrue((0...1).contains(l)) }35 XCTAssertEqual(real.lowPowerMode, ProcessInfo.processInfo.isLowPowerModeEnabled)36 }37 func testAdvisorGivesSlotsBackWhenTransfersKeepFailing() {38 var a = ConcurrencyAdvisor()39 let net = NetworkStatus(online: true, expensive: false, constrained: false)40 for t in [10.0, 10, 20, 20, 30, 30, 40, 40, 50, 50] { a.observe(throughput: t * 1e6, saturated: true) }41 let grown = a.limit; XCTAssertGreaterThanOrEqual(grown, 5)42 a.record(success: true); a.record(success: false); a.record(success: false)43 XCTAssertEqual(a.limit, grown, "two failures are not a trend")44 a.record(success: false)45 XCTAssertEqual(a.limit, grown - 1); XCTAssertEqual(a.errorRate, 0) // window restarted after the decision46 for _ in 0..<40 { a.record(success: false) }47 XCTAssertEqual(a.limit, 1); XCTAssertEqual(a.allowed(network: net), 1) // never below one48 let before = a.limit49 a.observe(throughput: 90e6, saturated: true); a.observe(throughput: 90e6, saturated: true)50 XCTAssertEqual(a.limit, before, "growth waits for calm windows after errors")51 for t in [10.0, 10, 10, 20, 20, 30, 30] { a.observe(throughput: t * 1e6, saturated: true) }52 XCTAssertGreaterThan(a.limit, before) // and comes back when the link behaves53 var m = ConcurrencyAdvisor(manual: 5); for _ in 0..<20 { m.record(success: false) }54 XCTAssertEqual(m.allowed(network: net), 5)55 }56 func testListing1000ItemsFromTheMirrorIsFast() throws {57 let dir = FileManager.default.temporaryDirectory.appendingPathComponent("spdrive-perf-\(UUID().uuidString)"); defer { try? FileManager.default.removeItem(at: dir) }58 let s = try MetadataStore(url: dir.appendingPathComponent("m.sqlite"))59 try s.upsert([item("root", "", parent: nil, kind: .folder, rev: 1), item("d", "Mille", parent: "root", kind: .folder, rev: 2)])60 try s.upsert((0..<1000).map { item("f\($0)", "Document \($0).pdf", parent: "d", rev: Int64($0 + 3), size: 1000) })61 var best = Double.infinity62 for _ in 0..<10 { let t = Date(); let rows = try s.children(of: "d"); best = min(best, Date().timeIntervalSince(t)); XCTAssertEqual(rows.count, 1000) }63 print(String(format: "MEASURE mirror-list-1000: %.2f ms (best of 10)", best * 1000))64 XCTAssertLessThan(best, 0.1)65 }66}6768/// A scripted event source: each `connect()` hands out the next prepared connection.69final class FakeEventSource: RemoteEventSource, @unchecked Sendable {70 private let lock = NSLock()71 private var live: AsyncThrowingStream<RemoteEvent, Error>.Continuation?72 private(set) var connections = 073 var refuse = false74 func connect() -> AsyncThrowingStream<RemoteEvent, Error> {75 AsyncThrowingStream { c in76 lock.withLock { connections += 1 }77 if refuse { c.finish(throwing: DriveError.offline("refused")); return }78 lock.withLock { live = c }79 }80 }81 func send(_ e: RemoteEvent) { _ = lock.withLock { live }?.yield(e) }82 func drop() { let c = lock.withLock { let c = live; live = nil; return c }; c?.finish(throwing: DriveError.offline("dropped")) }83 var count: Int { lock.withLock { connections } }84}8586final class EventListenerTests: XCTestCase {87 func testPollingOnlyReportsHeadMovesOnce() async throws {88 let head = Box<Int64>(5), seen = Box<[Int64]>([])89 let l = EventListener(source: nil, pollInterval: 0.05, head: { head.value }, onChange: { h in seen.update { $0.append(h) } })90 await l.start()91 try await waitUntil { seen.value == [5] }92 try await Task.sleep(nanoseconds: 300_000_000)93 XCTAssertEqual(seen.value, [5], "an unchanged head is not news")94 head.value = 995 try await waitUntil { seen.value == [5, 9] }96 let ch = await l.channel; XCTAssertEqual(ch, .polling)97 await l.stop()98 head.value = 12; try await Task.sleep(nanoseconds: 250_000_000)99 XCTAssertEqual(seen.value, [5, 9]); let off = await l.channel; XCTAssertEqual(off, .off)100 }101 func testStreamTakesOverThenPollingCoversADropAndReconnectBacksOff() async throws {102 let src = FakeEventSource(), head = Box<Int64>(1), seen = Box<[Int64]>([]), notes = Box<[String]>([])103 let l = EventListener(source: src, pollInterval: 0.05, retry: RetryPolicy(base: 0.3, cap: 1, jitter: 0.2), head: { head.value },104 onNotify: { k in notes.update { $0.append(k) } }, onChange: { h in seen.update { $0.append(h) } })105 await l.start()106 try await waitUntil { src.count == 1 }107 src.send(.hello(head: 1))108 try await waitUntil { await l.channel == .stream }109 try await waitUntil { seen.value == [1] }110 let pollsWhenLive = await l.polls111 src.send(.change(head: 2)); src.send(.change(head: 2)); src.send(.notify(kind: "request.submission")); src.send(.change(head: 3))112 try await waitUntil { seen.value == [1, 2, 3] }113 XCTAssertEqual(notes.value, ["request.submission"])114 try await Task.sleep(nanoseconds: 250_000_000)115 let pollsStill = await l.polls; XCTAssertEqual(pollsStill, pollsWhenLive, "no polling while the stream is live")116 // the stream drops: polling takes over at once, and reports what the stream can no longer say117 head.value = 3 // what the server really has at that point118 src.refuse = true; src.drop()119 try await waitUntil { await l.channel == .polling }120 head.value = 7121 try await waitUntil { seen.value == [1, 2, 3, 7] }122 // reconnection attempts are spaced by the back-off (0.3 s, 0.6 s, 1 s…): far fewer than one per poll tick123 try await Task.sleep(nanoseconds: 1_200_000_000)124 let attempts = src.count; XCTAssertGreaterThanOrEqual(attempts, 3); XCTAssertLessThanOrEqual(attempts, 5)125 // the server is back: the stream is live again and polling stops126 src.refuse = false127 try await waitUntil(timeout: 5) { src.count > attempts }128 src.send(.hello(head: 8))129 try await waitUntil { await l.channel == .stream }130 try await waitUntil { seen.value.last == 8 }131 let reconnects = await l.reconnects; XCTAssertGreaterThanOrEqual(reconnects, 2)132 await l.stop()133 }134}135136final class EngineEventTests: LiveCase {137 func testEnginePicksUpRemoteChangesFromThePollingFallbackWithoutBeingPoked() async throws {138 let s = try store()139 let engine = SyncEngine(api: api, store: s, network: NetworkMonitor(), workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests", eventPollInterval: 1)140 await engine.start()141 let fid = folder.id142 try await waitUntil { (try? s.item(fid)) != nil }143 try await waitUntil { let st = await engine.currentStatus(); return st.phase == .idle && st.events == .polling }144 try await Task.sleep(nanoseconds: 1_500_000_000) // let the first poll settle145 let made = try await api.createFolder(parent: folder.id, name: "vu par le sondage")146 let t0 = Date()147 try await waitUntil(timeout: 6, "the engine's own loop rests at least 10 s: only the listener can bring this in time") { (try? s.item(made.id)) != nil }148 XCTAssertLessThan(Date().timeIntervalSince(t0), 6)149 // paused: the listener is off; resumed: it is back150 await engine.pause()151 try await waitUntil { await engine.currentStatus().events == .off }152 await engine.resume()153 try await waitUntil { await engine.currentStatus().events == .polling }154 await engine.stop()155 }156 func testEngineFollowsAPluggedInEventStream() async throws {157 let s = try store(), src = FakeEventSource()158 let engine = SyncEngine(api: api, store: s, network: NetworkMonitor(), workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests", eventPollInterval: 600)159 await engine.setEventSource(src)160 await engine.start()161 let fid = folder.id162 try await waitUntil { (try? s.item(fid)) != nil }163 try await waitUntil { src.count >= 1 }164 src.send(.hello(head: try await api.syncCursor().head))165 try await waitUntil { await engine.currentStatus().events == .stream }166 try await waitUntil { await engine.currentStatus().phase == .idle }167 let made = try await api.createFolder(parent: folder.id, name: "annoncé par le flux")168 try await Task.sleep(nanoseconds: 300_000_000)169 XCTAssertNil(try s.item(made.id)) // nobody said anything yet (polling is 10 minutes away)170 src.send(.change(head: try await api.syncCursor().head))171 try await waitUntil(timeout: 5) { (try? s.item(made.id)) != nil }172 await engine.stop()173 }174}175176/// The real stream of the 2.0 server (SSE through SPDriveNext) plugged into the engine.177final class LiveEventStreamTests: LiveCase {178 func testRealStreamBringsARemoteChangeAtOnceAndPollingCoversAKilledStream() async throws {179 let s = try store()180 let engine = SyncEngine(api: api, store: s, network: NetworkMonitor(), workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests", eventPollInterval: 2)181 await engine.setEventSource(NextEventSource(api: api, idleTimeout: 30)) // above the server's 25 s ping, so a healthy stream is never cut182 await engine.start()183 let fid = folder.id184 try await waitUntil { (try? s.item(fid)) != nil }185 try await waitUntil(timeout: 10, "stream live") { await engine.currentStatus().events == .stream }186 try await waitUntil { await engine.currentStatus().phase == .idle }187 try await Task.sleep(nanoseconds: 1_200_000_000)188 // another client of the account changes something: the Mac learns of it without being poked and well before any poll189 let other = try Backend.api()190 var t0 = Date()191 let made = try await other.createFolder(parent: folder.id, name: "poussé par le serveur")192 try await waitUntil(timeout: 4, "pushed change mirrored") { (try? s.item(made.id)) != nil }193 let pushed = Date().timeIntervalSince(t0)194 XCTAssertLessThan(pushed, 1.9, "faster than the 2 s poll of this test (and than the 30 s poll of the app): it came from the stream")195 // the server goes away. A server stopped with SIGTERM may leave the open stream hanging without closing it: only the196 // silence (no ping) gives it away. Within the idle timeout the listener declares the stream dead and polls.197 try Backend.serverDown()198 var up = false; defer { if !up { try? Backend.serverUp() } }199 let tDown = Date()200 try await waitUntil(timeout: 45, "fallback to polling") { await engine.currentStatus().events == .polling }201 let noticed = Date().timeIntervalSince(tDown)202 try Backend.serverUp(); up = true203 // while the stream is still reconnecting (back-off), a change is found by the poll204 t0 = Date()205 let later = try await other.createFolder(parent: folder.id, name: "vu après la coupure")206 try await waitUntil(timeout: 15, "change after the outage mirrored") { (try? s.item(later.id)) != nil }207 // and the stream comes back by itself208 try await waitUntil(timeout: 70, "stream live again") { await engine.currentStatus().events == .stream }209 let again = try await other.createFolder(parent: folder.id, name: "poussé de nouveau")210 try await waitUntil(timeout: 4) { (try? s.item(again.id)) != nil }211 print(String(format: "MEASURE sse: pushed change mirrored in %.2f s; dead stream noticed after %.1f s", pushed, noticed))212 await engine.stop()213 }214}215216final class QueuePolicyTests: LiveCase {217 private func finishOrder(_ q: TransferQueue) -> (Box<[String]>, Task<Void, Never>) {218 let order = Box<[String]>([])219 let w = Task { for await e in await q.events() { if case .finished(let t, _) = e { order.update { $0.append(t.name) } } } }220 return (order, w)221 }222 func testUserInitiatedTransfersPreemptAndPrecedeBackgroundOnes() async throws {223 let s = try store(), q = TransferQueue(api: api, store: s, workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests")224 let (order, w) = finishOrder(q); defer { w.cancel() }225 await q.setManualConcurrency(1)226 await q.setLimits(uploadBytesPerSecond: 1024 * 1024, downloadBytesPerSecond: 0)227 var shas: [String: String] = [:]228 for i in 1...3 { let f = try randomFile("fond\(i).bin", bytes: 2 * 1024 * 1024 + i); shas["fond\(i).bin"] = f.sha; _ = try await q.enqueueUpload(file: f.url, parent: folder.id, priority: TransferPriority.background) }229 await q.start()230 try await waitUntil { await q.list().contains { $0.name == "fond1.bin" && $0.state == .running && $0.done > 0 } }231 let user = try randomFile("demandé.bin", bytes: 700 * 1024); shas["demandé.bin"] = user.sha232 let u = try await q.enqueueUpload(file: user.url, parent: folder.id, priority: TransferPriority.userInitiated)233 // the running background transfer is put back in the queue (not failed, progress kept) and the user's goes first234 try await waitUntil { await q.get(u.id)?.state == .running }235 let stopped = await q.list().first { $0.name == "fond1.bin" }; XCTAssertEqual(stopped?.state, .queued); XCTAssertNil(stopped?.error)236 try await waitUntil(timeout: 60) { order.value.count == 4 }237 XCTAssertEqual(order.value.first, "demandé.bin")238 // "send this one first" on a queued transfer239 let files = try await api.listFolder(folder.id).files240 XCTAssertEqual(files.count, 4); for f in files { XCTAssertEqual(f.sha256, shas[f.name], f.name) } // pre-emption lost and duplicated nothing241 await q.stop()242 }243 func testSendFirstReordersTheQueue() async throws {244 let s = try store(), q = TransferQueue(api: api, store: s, workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests")245 let (order, w) = finishOrder(q); defer { w.cancel() }246 await q.setManualConcurrency(1)247 var ids: [String] = []248 for i in 1...4 { ids.append(try await q.enqueueUpload(file: try text("f\(i).txt", "contenu \(i)"), parent: folder.id).id) }249 await q.setPriority(ids[3], TransferPriority.urgent)250 await q.start()251 try await waitUntil { order.value.count == 4 }252 XCTAssertEqual(order.value, ["f4.txt", "f1.txt", "f2.txt", "f3.txt"])253 await q.stop()254 }255 func testLowPowerModeRunsOnlyWhatThePersonAskedForOneAtATime() async throws {256 let power = FixedPowerSource(PowerState(onBattery: true, lowPowerMode: true, level: 0.5))257 let s = try store(), q = TransferQueue(api: api, store: s, workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests", power: power)258 let bg = try await q.enqueueUpload(file: try text("fond.txt", "arrière-plan"), parent: folder.id, priority: TransferPriority.background)259 let u1 = try await q.enqueueUpload(file: try text("a.txt", "demandé a"), parent: folder.id, priority: TransferPriority.userInitiated)260 let u2 = try await q.enqueueUpload(file: try text("b.txt", "demandé b"), parent: folder.id, priority: TransferPriority.userInitiated)261 await q.start()262 let pol = await q.policy(); XCTAssertEqual(pol.concurrency, 1); XCTAssertTrue(pol.deferringBackground)263 try await waitUntil { let a = await q.get(u1.id)?.state, b = await q.get(u2.id)?.state; return a == .done && b == .done }264 try await Task.sleep(nanoseconds: 2_500_000_000) // more than one queue tick265 let waiting = await q.get(bg.id); XCTAssertEqual(waiting?.state, .queued); XCTAssertNil(waiting?.error) // deferred, not failed266 var names = try await api.listFolder(folder.id).files.map(\.name).sorted(); XCTAssertEqual(names, ["a.txt", "b.txt"])267 // the person turns the power policy off for this queue: the deferred transfer runs268 await q.setPowerAware(false)269 try await waitUntil { await q.get(bg.id)?.state == .done }270 await q.setPowerAware(true)271 // back on mains: nothing is held any more272 let bg2 = try await q.enqueueUpload(file: try text("fond2.txt", "arrière-plan 2"), parent: folder.id, priority: TransferPriority.background)273 try await Task.sleep(nanoseconds: 600_000_000)274 let held = await q.get(bg2.id); XCTAssertEqual(held?.state, .queued)275 power.state = .unconstrained276 try await waitUntil(timeout: 10) { await q.get(bg2.id)?.state == .done } // picked up at the next tick, without anyone asking277 names = try await api.listFolder(folder.id).files.map(\.name).sorted(); XCTAssertEqual(names, ["a.txt", "b.txt", "fond.txt", "fond2.txt"])278 await q.stop()279 }280 func testBandwidthCapHoldsARealUploadAndDownload() async throws {281 let s = try store(), q = TransferQueue(api: api, store: s, workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests")282 let f = try randomFile("plafonné.bin", bytes: 4 * 1024 * 1024)283 await q.setLimits(uploadBytesPerSecond: 1024 * 1024, downloadBytesPerSecond: 1024 * 1024)284 await q.start()285 var t0 = Date()286 let up = try await q.enqueueUpload(file: f.url, parent: folder.id)287 try await waitUntil(timeout: 30) { await q.get(up.id)?.state == .done }288 let upSeconds = Date().timeIntervalSince(t0)289 XCTAssertGreaterThan(upSeconds, 2.6, "4 MiB at 1 MiB/s cannot take less than ~3 s (one second of burst allowed)"); XCTAssertLessThan(upSeconds, 9)290 let item = try await api.listFolder(folder.id).files.first!291 XCTAssertEqual(item.sha256, f.sha)292 // downloads are paced on the delivery of the bytes to the engine: what the loader buffered does not escape the cap293 let g = try randomFile("retour-source.bin", bytes: 6 * 1024 * 1024)294 let big = try await upload(g.url)295 t0 = Date()296 let dest = dir.appendingPathComponent("retour.bin")297 let down = try await q.enqueueDownload(big, to: dest)298 try await waitUntil(timeout: 30) { await q.get(down.id)?.state == .done }299 let downSeconds = Date().timeIntervalSince(t0)300 XCTAssertGreaterThan(downSeconds, 4.5, "6 MiB at 1 MiB/s, minus one second of burst"); XCTAssertEqual(try Hashing.sha256Hex(of: dest).sha256, g.sha)301 // cap lifted: the same upload is several times faster302 await q.setLimits(uploadBytesPerSecond: 0, downloadBytesPerSecond: 0)303 t0 = Date()304 let free = try await q.enqueueUpload(file: f.url, parent: folder.id, name: "libre.bin")305 try await waitUntil(timeout: 30) { await q.get(free.id)?.state == .done }306 XCTAssertLessThan(Date().timeIntervalSince(t0), upSeconds / 2)307 print(String(format: "MEASURE cap 1 MiB/s: upload 4 MiB in %.2f s, download 6 MiB in %.2f s; uncapped upload %.2f s", upSeconds, downSeconds, Date().timeIntervalSince(t0)))308 await q.stop()309 }310 func testGlobalPauseFreezesTheServerSideOffsetAndResumeFinishes() async throws {311 let s = try store(), q = TransferQueue(api: api, store: s, workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests")312 let f = try randomFile("pause.bin", bytes: 8 * 1024 * 1024)313 await q.setLimits(uploadBytesPerSecond: 2 * 1024 * 1024, downloadBytesPerSecond: 0)314 let t = try await q.enqueueUpload(file: f.url, parent: folder.id)315 await q.start()316 try await waitUntil { (await q.get(t.id)?.done ?? 0) > 1024 * 1024 }317 await q.pauseAll()318 let session = try XCTUnwrap(try s.db.query("SELECT upload_id FROM upload_sessions").first?["upload_id"]?.text)319 try await Task.sleep(nanoseconds: 700_000_000) // a block in flight may still land320 let a = try await api.uploadStatus(session).received321 try await Task.sleep(nanoseconds: 1_500_000_000)322 let b = try await api.uploadStatus(session).received323 XCTAssertEqual(a, b, "nothing travels while paused"); XCTAssertGreaterThan(a, 0); XCTAssertLessThan(a, Int64(8 * 1024 * 1024))324 let st = await q.get(t.id); XCTAssertEqual(st?.state, .queued); let paused = await q.isPaused; XCTAssertTrue(paused)325 let late = try await q.enqueueUpload(file: try text("pendant.txt", "ajouté pendant la pause"), parent: folder.id)326 try await Task.sleep(nanoseconds: 500_000_000)327 let lateState = await q.get(late.id)?.state; XCTAssertEqual(lateState, .queued) // nothing starts either328 await q.setLimits(uploadBytesPerSecond: 0, downloadBytesPerSecond: 0)329 await q.resumeAll()330 try await waitUntil(timeout: 30) { let a = await q.get(t.id)?.state, b = await q.get(late.id)?.state; return a == .done && b == .done }331 let done = try await api.uploadStatus(session); XCTAssertEqual(done.status, "done") // the same server session was continued332 let sha = try await api.listFolder(folder.id).files.first { $0.name == "pause.bin" }?.sha256; XCTAssertEqual(sha, f.sha)333 await q.stop()334 }335 func testOutageRetriesBackOffWithJitterGiveSlotsBackAndRecoverAlone() async throws {336 let s = try store(), q = TransferQueue(api: api, store: s, workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests")337 await q.setRetryPolicy(RetryPolicy(base: 1, cap: 4, jitter: 0.5))338 var ids: [String] = []339 for i in 1...4 { ids.append(try await q.enqueueUpload(file: try text("r\(i).txt", "retry \(i)"), parent: folder.id).id) }340 try Backend.serverDown()341 var up = false; defer { if !up { try? Backend.serverUp() } }342 await q.start()343 // every transfer fails for a transient reason: none is declared failed, each gets a later next try, delays differ (jitter)344 try await waitUntil(timeout: 30) { ((try? s.db.query("SELECT COUNT(*) AS n FROM transfers WHERE attempts >= 2 AND state = 'queued'").first?["n"]?.int) ?? 0) == 4 }345 let rows = try s.db.query("SELECT attempts, updated_at, error, state FROM transfers")346 let now = Date().timeIntervalSince1970347 var waits = Set<Int>()348 for r in rows {349 XCTAssertEqual(r["state"]?.text, "queued"); XCTAssertNotNil(r["error"]?.text)350 let attempts = Int(r["attempts"]?.int ?? 0), next = (r["updated_at"]?.double ?? 0) - now351 XCTAssertLessThanOrEqual(next, RetryPolicy(base: 1, cap: 4, jitter: 0.5).nominal(attempt: attempts) * 1.5 + 0.05) // never beyond the band352 waits.insert(Int((r["updated_at"]?.double ?? 0) * 1000))353 }354 XCTAssertGreaterThan(waits.count, 1, "retries must not be synchronised")355 let pol = await q.policy()356 XCTAssertLessThan(pol.measuredLimit, 3, "repeated failures give slots back"); XCTAssertEqual(pol.concurrency, pol.measuredLimit)357 // the server comes back; nobody kicks the queue: the back-off itself brings every transfer in358 try Backend.serverUp(); up = true359 try await waitUntil(timeout: 30) { let m = await q.summary(); return m.queued + m.running == 0 }360 for id in ids { let st = await q.get(id)?.state; XCTAssertEqual(st, .done) }361 let names = try await api.listFolder(folder.id).files.map(\.name).sorted(); XCTAssertEqual(names, ["r1.txt", "r2.txt", "r3.txt", "r4.txt"]) // each exactly once362 await q.stop()363 }364}365366/// The transfer queue runs in another process (spdrive-cli), which is killed with SIGKILL in the middle of a transfer.367final class CrashResumeTests: LiveCase {368 private func cli() throws -> URL {369 let u = Bundle(for: CrashResumeTests.self).bundleURL.deletingLastPathComponent().appendingPathComponent("spdrive-cli")370 guard FileManager.default.isExecutableFile(atPath: u.path) else { XCTFail("spdrive-cli not built next to the tests: \(u.path)"); throw DriveError.local("no cli") }371 return u372 }373 // `Process.waitUntilExit()` is not used: called from a Swift concurrency thread it can wait forever (seen once in this suite).374 private func spawn(_ args: [String]) throws -> Process {375 let b = try Backend.require()376 let p = Process(); p.executableURL = try cli(); p.arguments = args377 p.environment = ["SPDRIVE_URL": b.url.absoluteString, "SPDRIVE_TOKEN": b.token]378 p.standardOutput = FileHandle.nullDevice; p.standardError = FileHandle.nullDevice379 try p.run(); return p380 }381 func testUploadKilledMidTransferResumesOnTheSameServerSession() async throws {382 let db = dir.appendingPathComponent("meta.sqlite"), work = dir.appendingPathComponent("work")383 let big = try randomFile("tué.bin", bytes: 12 * 1024 * 1024)384 let p = try spawn(["queue-put", db.path, work.path, big.url.path, folder.id, String(2 * 1024 * 1024)])385 defer { if p.isRunning { kill(p.processIdentifier, SIGKILL) } }386 let s = try MetadataStore(url: db)387 let api = self.api!388 try await waitUntil(timeout: 30, "the other process is uploading") {389 guard let id = try? s.db.query("SELECT upload_id FROM upload_sessions").first?["upload_id"]?.text else { return false }390 return ((try? await api.uploadStatus(id).received) ?? 0) > 3 * 1024 * 1024391 }392 let session = try XCTUnwrap(try s.db.query("SELECT upload_id FROM upload_sessions").first?["upload_id"]?.text)393 XCTAssertEqual(kill(p.processIdentifier, SIGKILL), 0); try await waitUntil(timeout: 10, "the killed process is gone") { !p.isRunning }394 XCTAssertEqual(p.terminationReason, .uncaughtSignal); XCTAssertEqual(p.terminationStatus, SIGKILL) // really killed, no clean-up ran395 let row = try XCTUnwrap(try s.db.query("SELECT id, state FROM transfers").first)396 XCTAssertEqual(row["state"]?.text, "running", "the crash leaves the row as it was")397 let before = try await api.uploadStatus(session)398 XCTAssertEqual(before.status, "active"); XCTAssertGreaterThan(before.received, 3 * 1024 * 1024); XCTAssertLessThan(before.received, Int64(12 * 1024 * 1024))399 let none = try await api.listFolder(folder.id).files; XCTAssertTrue(none.isEmpty) // nothing half-written is visible400 // "relaunch": a new queue on the same database and work directory401 let q = TransferQueue(api: api, store: s, workDirectory: work, deviceName: "Tests")402 let firstProgress = Box<Int64?>(nil)403 let id = row["id"]?.text ?? ""404 let w = Task { for await e in await q.events() { if case .changed(let t) = e, t.id == id, t.state == .running, t.done > 0, firstProgress.value == nil { firstProgress.value = t.done } } }405 defer { w.cancel() }406 await q.start()407 try await waitUntil(timeout: 60) { await q.get(id)?.state == .done }408 let after = try await api.uploadStatus(session); XCTAssertEqual(after.status, "done") // the same session was finished, not a new one409 let files = try await api.listFolder(folder.id).files410 XCTAssertEqual(files.map(\.name), ["tué.bin"]); XCTAssertEqual(files.first?.sha256, big.sha) // once, intact411 XCTAssertEqual(try s.db.query("SELECT COUNT(*) AS n FROM upload_sessions").first?["n"]?.int, 0)412 print("MEASURE kill-resume upload: server had \(before.received) of \(12 * 1024 * 1024) bytes at SIGKILL; first progress after relaunch \(firstProgress.value ?? -1)")413 await q.stop()414 }415 func testDownloadKilledMidTransferResumesFromItsPartialFile() async throws {416 let db = dir.appendingPathComponent("meta.sqlite"), work = dir.appendingPathComponent("work"), dest = dir.appendingPathComponent("out/reçu.bin")417 let big = try randomFile("source.bin", bytes: 12 * 1024 * 1024)418 let item = try await upload(big.url)419 let p = try spawn(["queue-get", db.path, work.path, item.id, dest.path, String(2 * 1024 * 1024)])420 defer { if p.isRunning { kill(p.processIdentifier, SIGKILL) } }421 let partials = work.appendingPathComponent("partials")422 @Sendable func partialSize() -> Int64 {423 guard let f = (try? FileManager.default.contentsOfDirectory(at: partials, includingPropertiesForKeys: nil))?.first else { return 0 }424 return ((try? FileManager.default.attributesOfItem(atPath: f.path)[.size]) as? NSNumber)?.int64Value ?? 0425 }426 try await waitUntil(timeout: 30, "the other process is downloading") { partialSize() > 3 * 1024 * 1024 }427 XCTAssertEqual(kill(p.processIdentifier, SIGKILL), 0); try await waitUntil(timeout: 10, "the killed process is gone") { !p.isRunning }428 XCTAssertEqual(p.terminationReason, .uncaughtSignal)429 let had = partialSize()430 XCTAssertGreaterThan(had, 3 * 1024 * 1024); XCTAssertLessThan(had, Int64(12 * 1024 * 1024))431 XCTAssertFalse(FileManager.default.fileExists(atPath: dest.path), "an unfinished download is never placed at its destination")432 let s = try MetadataStore(url: db)433 let id = try XCTUnwrap(try s.db.query("SELECT id FROM transfers").first?["id"]?.text)434 let q = TransferQueue(api: api, store: s, workDirectory: work, deviceName: "Tests")435 let lowest = Box<Int64>(.max)436 let w = Task { for await e in await q.events() { if case .changed(let t) = e, t.id == id, t.state == .running, t.done > 0 { lowest.update { $0 = min($0, t.done) } } } }437 defer { w.cancel() }438 await q.setLimits(uploadBytesPerSecond: 0, downloadBytesPerSecond: 2 * 1024 * 1024)439 await q.start()440 try await waitUntil(timeout: 60) { await q.get(id)?.state == .done }441 XCTAssertEqual(try Hashing.sha256Hex(of: dest).sha256, big.sha)442 XCTAssertEqual(partialSize(), 0) // the partial became the file443 if lowest.value != .max { XCTAssertGreaterThanOrEqual(lowest.value, had, "progress restarts from the partial file, not from zero") }444 print("MEASURE kill-resume download: partial had \(had) bytes at SIGKILL; lowest progress reported after relaunch \(lowest.value == .max ? -1 : lowest.value)")445 await q.stop()446 }447}448449final class ConflictVersionTests: LiveCase {450 func testKeepMineAsNewVersionKeepsTheServerVersionInHistoryAndOverwritesNothingSilently() async throws {451 let s = try store(), work = dir.appendingPathComponent("work")452 let base = try await upload(try text("contrat.txt", "version de départ"))453 // a real conflict: another device replaces the content, then this device sends an edit derived from the old version454 let other = try await UploadEngine(api: api, store: nil, deviceName: "Autre").upload(file: try text("o.txt", "version de l'autre appareil"), to: .newVersion(fileID: base.id, folderID: folder.id, name: "contrat.txt", baseVersionID: base.versionID)).item455 let outcome = try await UploadEngine(api: api, store: s, deviceName: "Mac de test").upload(file: try text("m.txt", "version de ce Mac"), to: .newVersion(fileID: base.id, folderID: folder.id, name: "contrat.txt", baseVersionID: base.versionID))456 guard case .conflictCopy(let copy, _) = outcome else { return XCTFail("expected a conflict") }457 let untouched = try await serverBytes(base.id); XCTAssertEqual(String(decoding: untouched, as: UTF8.self), "version de l'autre appareil")458 let resolver = ConflictResolver(api: api, store: s)459 let c = try resolver.record(itemID: base.id, itemName: "contrat.txt", copy: copy)460 // while the person hesitates, the file changes a third time: the resolution must refuse, not overwrite461 let stale = SyncConflict(id: c.id, itemID: c.itemID, itemName: c.itemName, copyID: c.copyID, copyName: c.copyName, kind: c.kind, detail: c.detail, detectedAt: c.detectedAt, resolvedAt: nil, resolution: nil)462 let updated = try await resolver.resolveKeepingMineAsNewVersion(stale, workDirectory: work, deviceName: "Mac de test")463 XCTAssertEqual(updated.id, base.id); XCTAssertEqual(updated.versionSeq, 3)464 let now = try await serverBytes(base.id); XCTAssertEqual(String(decoding: now, as: UTF8.self), "version de ce Mac")465 let details = try await api.fileDetails(base.id)466 XCTAssertEqual(details.versions.count, 3) // start, other device, this Mac: the whole history is there467 let v2 = try XCTUnwrap(details.versions.first { $0.seq == 2 }); XCTAssertEqual(v2.id, other.versionID)468 let kept = try await serverBytes(base.id, version: v2.id); XCTAssertEqual(String(decoding: kept, as: UTF8.self), "version de l'autre appareil") // set aside, still readable469 let live = try await api.listFolder(folder.id).files; XCTAssertEqual(live.map(\.id), [base.id]) // the copy left the folder…470 let trashedCopy = try await api.item(copy.id, kind: .file); XCTAssertNotNil(trashedCopy?.trashedAt) // …for the trash, not for nowhere471 XCTAssertEqual(try resolver.openCount(), 0); XCTAssertEqual(try resolver.conflicts().first?.resolution, "keepMineVersion")472 XCTAssertEqual(try s.item(base.id)?.versionSeq, 3) // the mirror knows473 XCTAssertFalse(FileManager.default.fileExists(atPath: work.appendingPathComponent("conflicts/\(c.id)").path)) // nothing left behind474 }475 func testKeepMineAsNewVersionRefusesWhenTheFileChangedAgain() async throws {476 let s = try store(), work = dir.appendingPathComponent("work")477 let base = try await upload(try text("plan.txt", "v1"))478 _ = try await UploadEngine(api: api, store: nil, deviceName: "Autre").upload(file: try text("o.txt", "v2 autre"), to: .newVersion(fileID: base.id, folderID: folder.id, name: "plan.txt", baseVersionID: base.versionID))479 let outcome = try await UploadEngine(api: api, store: s, deviceName: "Mac").upload(file: try text("m.txt", "v2 mienne"), to: .newVersion(fileID: base.id, folderID: folder.id, name: "plan.txt", baseVersionID: base.versionID))480 guard case .conflictCopy(let copy, _) = outcome else { return XCTFail("expected a conflict") }481 let resolver = ConflictResolver(api: api, store: s)482 let c = try resolver.record(itemID: base.id, itemName: "plan.txt", copy: copy)483 // identical content on both sides resolves without sending a byte and without a new version484 let twin = try await upload(try text("twin.txt", "même contenu"))485 let twinCopy = try await upload(try text("twin (copie).txt", "même contenu"))486 let c2 = try resolver.record(itemID: twin.id, itemName: "twin.txt", copy: twinCopy)487 let same = try await resolver.resolveKeepingMineAsNewVersion(c2, workDirectory: work, deviceName: "Mac")488 XCTAssertEqual(same.versionSeq, 1); let gone = try await api.item(twinCopy.id, kind: .file); XCTAssertNotNil(gone?.trashedAt)489 // the first conflict is still open and resolves normally490 _ = try await resolver.resolveKeepingMineAsNewVersion(c, workDirectory: work, deviceName: "Mac")491 let bytes = try await serverBytes(base.id); XCTAssertEqual(String(decoding: bytes, as: UTF8.self), "v2 mienne")492 XCTAssertEqual(try resolver.openCount(), 0)493 }494}495