Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/CoreTests/SyncIntegrationTests.swift3// Description: Integration tests of the sync layer against the real local backend: incremental change feed, offline journal replayed after a "restart", persistent transfer queue, the sync loop through a server outage.4import Foundation5import XCTest6@testable import SPDriveCore7@testable import SPDriveSync89final class ChangeFeedTests: LiveCase {10 func testPullMirrorsRemoteChangesIncrementally() async throws {11 let s = try store(), d = ChangeDetector(api: api, store: s)12 _ = try await d.pull(pageSize: 3) // first sync, in small pages13 XCTAssertNotNil(s.rootID); XCTAssertEqual(try s.item(folder.id)?.name, folder.name)14 let head = try await api.syncCursor().head; XCTAssertEqual(s.cursor, head)15 let none = try await d.pull(); XCTAssertTrue(none.isEmpty)16 let dirty = try await d.hasRemoteChanges(); XCTAssertFalse(dirty)1718 // "another device" works on the same account19 let other = try Backend.api()20 let sub = try await other.createFolder(parent: folder.id, name: "Venu d'ailleurs")21 let f = try await UploadEngine(api: other, store: nil, deviceName: "Autre").upload(file: try text("a.txt", "un"), to: .newFile(folderID: sub.id, name: "a.txt", onConflict: .fail)).item22 var changes = try await d.pull()23 XCTAssertEqual(changes.first { $0.id == sub.id }?.kind, .created); XCTAssertEqual(changes.first { $0.id == f.id }?.kind, .created)24 XCTAssertEqual(try s.children(of: sub.id).map(\.name), ["a.txt"])25 _ = try await other.updateFile(f.id, name: "b.txt")26 changes = try await d.pull(); XCTAssertEqual(changes.first { $0.id == f.id }?.kind, .renamed)27 _ = try await UploadEngine(api: other, store: nil, deviceName: "Autre").upload(file: try text("a2.txt", "deux"), to: .newVersion(fileID: f.id, folderID: sub.id, name: "b.txt", baseVersionID: f.versionID))28 changes = try await d.pull(); XCTAssertEqual(changes.first { $0.id == f.id }?.kind, .contentChanged); XCTAssertEqual(try s.item(f.id)?.versionSeq, 2)29 try await other.move(Selection(fileIds: [f.id]), to: folder.id)30 changes = try await d.pull(); XCTAssertEqual(changes.first { $0.id == f.id }?.kind, .moved); XCTAssertEqual(try s.item(f.id)?.parentID, folder.id)31 try await other.trash(Selection(fileIds: [f.id]))32 changes = try await d.pull(); XCTAssertEqual(changes.first { $0.id == f.id }?.kind, .trashed)33 XCTAssertTrue(try s.children(of: folder.id).allSatisfy { $0.id != f.id }); XCTAssertTrue(try s.trashRoots().contains { $0.id == f.id })34 try await other.restore(Selection(fileIds: [f.id]))35 changes = try await d.pull(); XCTAssertEqual(changes.first { $0.id == f.id }?.kind, .restored)36 try await other.trash(Selection(fileIds: [f.id])); try await other.purge(Selection(fileIds: [f.id]))37 let anchor = s.seq38 changes = try await d.pull(); XCTAssertEqual(changes.last { $0.id == f.id }?.kind, .deleted)39 XCTAssertNil(try s.item(f.id)); XCTAssertFalse(try s.changes(since: anchor).deleted.isEmpty) // a tombstone for the File Provider40 }41 func testInterruptedPullLosesNothing() async throws {42 let s = try store(), d = ChangeDetector(api: api, store: s)43 _ = try await d.pull()44 var made: Set<String> = []45 for i in 0..<9 { made.insert(try await api.createFolder(parent: folder.id, name: "d\(i)").id) }46 _ = try await d.pull(pageSize: 2, maxPages: 1) // the process stops after one page47 let mid = s.cursor48 let reopened = try store() // "restart"49 XCTAssertEqual(reopened.cursor, mid)50 _ = try await ChangeDetector(api: api, store: reopened).pull(pageSize: 2)51 XCTAssertTrue(made.isSubset(of: Set(try reopened.children(of: folder.id).map(\.id))))52 let head = try await api.syncCursor().head; XCTAssertEqual(reopened.cursor, head)53 }54 func testOwnWritesAreNotReportedAsRemoteChanges() async throws {55 let s = try store(), d = ChangeDetector(api: api, store: s)56 _ = try await d.pull()57 let made = try await api.createFolder(parent: folder.id, name: "local")58 try s.upsert([made]) // what the app does with the answer of its own request59 let changes = try await d.pull()60 XCTAssertNil(changes.first { $0.id == made.id })61 }62}6364final class OfflineJournalTests: LiveCase {65 func testOfflineOperationsSurviveRestartAndReplayInOrder() async throws {66 var s: MetadataStore? = try store()67 _ = try await ChangeDetector(api: api, store: s!).pull()68 let keep = try await upload(try text("garde.txt", "contenu")), toss = try await upload(try text("jette.txt", "x")), ren = try await upload(try text("ancien nom.txt", "y"))69 _ = try await ChangeDetector(api: api, store: s!).pull()70 try Backend.serverDown()71 var up = false; defer { if !up { try? Backend.serverUp() } }72 // --- offline: everything below is decided without a server ---73 var j = OperationJournal(api: api, store: s!)74 let local = try j.createFolder(parent: folder.id, name: "Créé hors ligne")75 XCTAssertTrue(OperationJournal.isLocal(local.id))76 let nested = try j.createFolder(parent: local.id, name: "Sous-dossier")77 try j.move([try XCTUnwrap(s!.item(keep.id))], to: nested.id)78 try j.rename(try XCTUnwrap(s!.item(ren.id)), to: "nouveau nom.txt")79 try j.setStarred(try XCTUnwrap(s!.item(ren.id)), true)80 try j.trash([try XCTUnwrap(s!.item(toss.id))])81 // the mirror already shows the intent82 XCTAssertEqual(try s!.children(of: nested.id).map(\.id), [keep.id]); XCTAssertEqual(try s!.item(ren.id)?.name, "nouveau nom.txt")83 XCTAssertNotNil(try s!.item(toss.id)?.trashedAt); var inFolder = LocalSearch(text: "nouveau"); inFolder.folderID = folder.id // the account is shared with other runs: look in this test's folder84 XCTAssertEqual(try s!.search(inFolder).map(\.id), [ren.id])85 do { try j.rename(try XCTUnwrap(s!.item(keep.id)), to: "garde.txt"); try j.move([local], to: nested.id); XCTFail("a folder cannot go into itself") } catch { XCTAssertEqual((error as? DriveError)?.code, "move_into_self") }86 // a file added offline into the folder created offline87 let work = dir.appendingPathComponent("work")88 var q: TransferQueue? = TransferQueue(api: api, store: s!, workDirectory: work, deviceName: "Tests")89 let added = try text("ajouté hors ligne.txt", "ajouté sans réseau")90 let t = try await q!.enqueueUpload(file: added, parent: local.id)91 try FileManager.default.removeItem(at: added) // the original may disappear: the queue owns a staged copy92 let r = await j.run()93 XCTAssertTrue(r.blocked); XCTAssertEqual(r.done, 0); XCTAssertEqual(try j.pendingCount(), 6)94 // --- restart of the Mac: every object is dropped, the database is reopened ---95 q = nil; s = nil96 let s2 = try store()97 j = OperationJournal(api: api, store: s2)98 XCTAssertEqual(try j.pendingCount(), 6); XCTAssertEqual(try s2.item(ren.id)?.name, "nouveau nom.txt")99 let q2 = TransferQueue(api: api, store: s2, workDirectory: work, deviceName: "Tests")100 let pendingTransfer = await q2.get(t.id); XCTAssertEqual(pendingTransfer?.state, .queued)101 // --- back online ---102 try Backend.serverUp(); up = true103 let done = await j.run()104 XCTAssertEqual(done, .init(done: 6, failed: 0, blocked: false)); XCTAssertEqual(try j.pendingCount(), 0)105 let realFolder = j.resolve(local.id), realNested = j.resolve(nested.id)106 XCTAssertFalse(OperationJournal.isLocal(realFolder)); XCTAssertNil(try s2.item(local.id))107 await q2.start()108 try await waitUntil { await q2.get(t.id)?.state == .done }109 // the server now agrees with everything decided offline110 let top = try await api.listFolder(folder.id)111 XCTAssertEqual(top.folders.map(\.name), ["Créé hors ligne"]); XCTAssertEqual(top.folders.first?.id, realFolder)112 XCTAssertEqual(top.files.map(\.name), ["nouveau nom.txt"]); XCTAssertEqual(top.files.first?.starred, true)113 let inside = try await api.listFolder(realFolder)114 XCTAssertEqual(inside.folders.map(\.id), [realNested]); XCTAssertEqual(inside.files.map(\.name), ["ajouté hors ligne.txt"])115 let bytes = try await serverBytes(inside.files[0].id); XCTAssertEqual(String(decoding: bytes, as: UTF8.self), "ajouté sans réseau")116 let deep = try await api.listFolder(realNested); XCTAssertEqual(deep.files.map(\.id), [keep.id])117 let tossed = try await api.item(toss.id, kind: .file); XCTAssertNotNil(tossed?.trashedAt) // in the trash, not destroyed118 await q2.stop()119 // and the mirror converges on the server's description120 _ = try await ChangeDetector(api: api, store: s2).pull()121 XCTAssertEqual(try s2.children(of: realFolder).map(\.name).sorted(), ["Sous-dossier", "ajouté hors ligne.txt"])122 XCTAssertEqual(try j.history().filter { $0.state == "done" }.count, 6)123 }124 func testRefusedOperationIsReportedAndMirrorRealigned() async throws {125 let s = try store()126 let a = try await upload(try text("a.txt", "a"))127 _ = try await ChangeDetector(api: api, store: s).pull()128 let j = OperationJournal(api: api, store: s)129 try j.rename(try XCTUnwrap(s.item(a.id)), to: "pris.txt") // free as far as this device knows…130 _ = try await upload(try text("pris.txt", "b")) // …but another device took the name meanwhile131 let other = try await upload(try text("c.txt", "c"))132 try s.upsert([other]); try j.setStarred(other, true) // a later operation must still go through133 let r = await j.run()134 XCTAssertEqual(r, .init(done: 1, failed: 1, blocked: false))135 XCTAssertEqual(try s.item(a.id)?.name, "a.txt") // the optimistic name is withdrawn136 let failed = try j.history().first { $0.state == "failed" }; XCTAssertEqual(failed?.kind, .rename); XCTAssertNotNil(failed?.error)137 let server = try await api.item(a.id, kind: .file); XCTAssertEqual(server?.name, "a.txt")138 let starred = try await api.item(other.id, kind: .file); XCTAssertEqual(starred?.starred, true)139 }140}141142extension XCTestCase {143 func waitUntil(timeout: Double = 30, _ what: String = "condition", file: StaticString = #filePath, line: UInt = #line, _ cond: @escaping @Sendable () async -> Bool) async throws {144 let end = Date().addingTimeInterval(timeout)145 while Date() < end { if await cond() { return }; try await Task.sleep(nanoseconds: 100_000_000) }146 XCTFail("timeout: \(what)", file: file, line: line); throw DriveError.local("timeout")147 }148}149150final class TransferQueueTests: LiveCase {151 func testQueueRunsPausesResumesCancelsAndVerifies() async throws {152 let s = try store(), work = dir.appendingPathComponent("work")153 let q = TransferQueue(api: api, store: s, workDirectory: work, deviceName: "Tests")154 let events = Box<[String]>([])155 let watcher = Task { for await e in await q.events() { if case .finished(let t, _) = e { events.update { $0.append(t.name) } } } }156 defer { watcher.cancel() }157 await q.setLimits(uploadBytesPerSecond: 3 * 1024 * 1024, downloadBytesPerSecond: 0)158 let big = try randomFile("gros.bin", bytes: 12 * 1024 * 1024), big2 = try randomFile("annulé.bin", bytes: 12 * 1024 * 1024)159 var ids: [String] = []160 for i in 0..<5 { ids.append(try await q.enqueueUpload(file: try text("petit\(i).txt", "contenu \(i)"), parent: folder.id).id) }161 let slow = try await q.enqueueUpload(file: big.url, parent: folder.id), doomed = try await q.enqueueUpload(file: big2.url, parent: folder.id)162 await q.start()163 try await waitUntil { (await q.get(slow.id)?.done ?? 0) > 1024 * 1024 }164 await q.pause(slow.id); await q.cancel(doomed.id)165 let paused = await q.get(slow.id); XCTAssertEqual(paused?.state, .paused)166 for id in ids { try await waitUntil { await q.get(id)?.state == .done } }167 try await Task.sleep(nanoseconds: 600_000_000)168 let still = await q.get(slow.id); XCTAssertEqual(still?.state, .paused) // a paused transfer stays paused169 var listed = try await api.listFolder(folder.id); XCTAssertEqual(listed.files.count, 5)170 await q.setLimits(uploadBytesPerSecond: 0, downloadBytesPerSecond: 0)171 await q.resume(slow.id)172 try await waitUntil { await q.get(slow.id)?.state == .done }173 listed = try await api.listFolder(folder.id)174 XCTAssertEqual(listed.files.first { $0.name == "gros.bin" }?.sha256, big.sha)175 XCTAssertNil(listed.files.first { $0.name == "annulé.bin" }) // cancelled: nothing appeared on the server176 let cancelled = await q.get(doomed.id); XCTAssertEqual(cancelled?.state, .cancelled)177 XCTAssertFalse(FileManager.default.fileExists(atPath: doomed.localPath)); XCTAssertFalse(FileManager.default.fileExists(atPath: slow.localPath)) // staging cleaned178 XCTAssertEqual(try s.item(listed.files.first { $0.name == "gros.bin" }!.id)?.sha256, big.sha) // the mirror learnt the new file179 // downloads through the queue, verified180 let dest = dir.appendingPathComponent("out/gros.bin")181 let dl = try await q.enqueueDownload(listed.files.first { $0.name == "gros.bin" }!, to: dest)182 try await waitUntil { await q.get(dl.id)?.state == .done }183 XCTAssertEqual(try Hashing.sha256Hex(of: dest).sha256, big.sha)184 XCTAssertTrue(events.value.contains("gros.bin")); XCTAssertEqual(events.value.filter { $0.hasPrefix("petit") }.count, 5)185 let sum = await q.summary(); XCTAssertEqual(sum.running + sum.queued + sum.paused + sum.failed, 0)186 await q.clearFinished()187 let left = await q.list(); XCTAssertTrue(left.isEmpty)188 await q.stop()189 }190 func testQueueResumesAfterProcessDeathAndServerOutage() async throws {191 let work = dir.appendingPathComponent("work")192 var s: MetadataStore? = try store()193 let big = try randomFile("reprise.bin", bytes: 14 * 1024 * 1024)194 let t: Transfer195 do {196 let q = TransferQueue(api: api, store: s!, workDirectory: work, deviceName: "Tests")197 await q.setLimits(uploadBytesPerSecond: 3 * 1024 * 1024, downloadBytesPerSecond: 0)198 let queued = try await q.enqueueUpload(file: big.url, parent: folder.id); t = queued199 await q.start()200 try await waitUntil { (await q.get(queued.id)?.done ?? 0) > 2 * 1024 * 1024 }201 await q.stop()202 }203 s = nil // the app is killed in the middle204 try Backend.serverDown()205 var up = false; defer { if !up { try? Backend.serverUp() } }206 let s2 = try store()207 try s2.db.run("UPDATE transfers SET state = 'running'") // what a crash leaves behind: a row still marked running208 let q2 = TransferQueue(api: api, store: s2, workDirectory: work, deviceName: "Tests")209 await q2.start() // server down: the transfer waits, it is not declared failed210 try await Task.sleep(nanoseconds: 2_000_000_000)211 let waiting = await q2.get(t.id); XCTAssertEqual(waiting?.state, .queued); XCTAssertNotNil(waiting?.error)212 try Backend.serverUp(); up = true213 await q2.kick()214 try await waitUntil(timeout: 60) { await q2.get(t.id)?.state == .done }215 let listed = try await api.listFolder(folder.id)216 XCTAssertEqual(listed.files.map(\.name), ["reprise.bin"]); XCTAssertEqual(listed.files.first?.sha256, big.sha) // once, intact217 await q2.stop()218 }219 func testReplacingAStaleFileThroughTheQueueRaisesAConflict() async throws {220 let s = try store(), q = TransferQueue(api: api, store: s, workDirectory: dir.appendingPathComponent("work"), deviceName: "Mac de test")221 let base = try await upload(try text("plan.txt", "base"))222 try s.upsert([base])223 _ = try await UploadEngine(api: api, store: nil, deviceName: "Autre").upload(file: try text("x.txt", "autre appareil"), to: .newVersion(fileID: base.id, folderID: folder.id, name: "plan.txt", baseVersionID: base.versionID))224 let got = Box<SyncConflict?>(nil)225 let watcher = Task { for await e in await q.events() { if case .conflict(let c) = e { got.value = c } } }226 defer { watcher.cancel() }227 // the queue sends a new version derived from the version this device knew228 let t = try await q.enqueueUpload(file: try text("y.txt", "ce Mac"), parent: folder.id, replacing: base)229 try s.db.run("UPDATE transfers SET base_version_id = ? WHERE id = ?", [.text(base.versionID!), .text(t.id)])230 await q.start()231 try await waitUntil { await q.get(t.id)?.state == .done }232 try await waitUntil { got.value != nil }233 XCTAssertEqual(got.value?.itemID, base.id); XCTAssertTrue(got.value?.copyName.contains("conflit Mac de test") ?? false)234 let files = try await api.listFolder(folder.id).files; XCTAssertEqual(files.count, 2)235 let original = try await serverBytes(base.id); XCTAssertEqual(String(decoding: original, as: UTF8.self), "autre appareil")236 XCTAssertEqual(try ConflictResolver(api: api, store: s).openCount(), 1)237 await q.stop()238 }239}240241final class SyncEngineTests: LiveCase {242 func testLoopSyncsGoesOfflineAndCatchesUp() async throws {243 let s = try store()244 let engine = SyncEngine(api: api, store: s, network: NetworkMonitor(), workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests")245 let remote = Box<[DetectedChange]>([])246 await engine.setRemoteChangeHandler { c in remote.update { $0.append(contentsOf: c) } }247 let phases = Box<[SyncPhase]>([])248 let watcher = Task { for await st in await engine.statusStream() { phases.update { if $0.last != st.phase { $0.append(st.phase) } } } }249 defer { watcher.cancel() }250 await engine.start()251 let folderID = folder.id252 try await waitUntil { s.rootID != nil && (try? s.item(folderID)) != nil }253 try await waitUntil { await engine.currentStatus().phase == .idle }254 // a remote change is picked up after a poke255 let made = try await api.createFolder(parent: folder.id, name: "distant")256 await engine.poke()257 try await waitUntil { (try? s.item(made.id)) != nil }258 XCTAssertTrue(remote.value.contains { $0.id == made.id && $0.kind == .created })259 // server down: local decisions are journaled, the engine reports offline and keeps them260 try Backend.serverDown()261 var up = false; defer { if !up { try? Backend.serverUp() } }262 let local = try await engine.journal.createFolder(parent: folder.id, name: "pendant la panne")263 await engine.poke()264 try await waitUntil { await engine.currentStatus().phase == .offline }265 let st = await engine.currentStatus(); XCTAssertEqual(st.pendingOperations, 1); XCTAssertNotNil(st.lastError)266 try Backend.serverUp(); up = true267 await engine.poke()268 try await waitUntil { await engine.currentStatus().pendingOperations == 0 }269 try await waitUntil { await engine.currentStatus().phase == .idle }270 let names = try await api.listFolder(folder.id).folders.map(\.name).sorted(); XCTAssertEqual(names, ["distant", "pendant la panne"])271 XCTAssertNil(try s.item(local.id)); let last = await engine.currentStatus().lastSync; XCTAssertNotNil(last)272 // paused: nothing is pulled until resumed273 await engine.pause()274 let quiet = try await api.createFolder(parent: folder.id, name: "pendant la pause")275 await engine.poke(); try await Task.sleep(nanoseconds: 1_200_000_000)276 XCTAssertNil(try s.item(quiet.id)); let ph = await engine.currentStatus().phase; XCTAssertEqual(ph, .paused)277 await engine.resume()278 try await waitUntil { (try? s.item(quiet.id)) != nil }279 XCTAssertTrue(phases.value.contains(.syncing)); XCTAssertTrue(phases.value.contains(.offline)); XCTAssertTrue(phases.value.contains(.paused))280 await engine.stop()281 }282 func testRevokedSessionStopsTheEngineWithoutLosingTheJournal() async throws {283 let token = try Backend.script("token", "Session à révoquer")284 let dev = try Backend.api(token: token)285 let s = try store()286 let engine = SyncEngine(api: dev, store: s, network: NetworkMonitor(), workDirectory: dir.appendingPathComponent("work"), deviceName: "Tests")287 await engine.start()288 try await waitUntil { await engine.currentStatus().phase == .idle }289 let id = try await api.sessions().first { $0.deviceName == "Session à révoquer" }!.id290 try await api.revokeSession(id) // the owner revokes this device from another one291 _ = try await engine.journal.createFolder(parent: folder.id, name: "après révocation")292 await engine.poke()293 try await waitUntil { await engine.currentStatus().phase == .needsLogin }294 XCTAssertEqual(try engine.journal.pendingCount(), 1) // kept for after the next sign-in (read from the journal itself: the status snapshot was seen at 0 once in ~6 full runs, cause not established)295 let names = try await api.listFolder(folder.id).folders.map(\.name); XCTAssertFalse(names.contains("après révocation"))296 await engine.stop()297 }298}299