Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatSync/Sources/SyncEngine.swift3// Description: Optional end-to-end encrypted synchronization of a SyncableStore: pull, decrypt, apply (last writer wins), then push local changes as encrypted batches.4import CryptoKit5import Foundation6import LocalChatCore7import LocalChatSecurity89/// Non-secret setup data shared through the transport so every device derives the same key from the passphrase.10public struct SyncMetadata: Codable, Sendable, Equatable {11 public var version: Int12 public var salt: Data13 public var iterations: Int14 public var verifier: Data // HMAC-SHA256 with a subkey: tells a wrong passphrase apart from damaged data15 public var createdAt: Date16}1718/// The clear header of a batch. Its exact bytes are the AEAD associated data, so it cannot be altered or swapped.19public struct SyncBatchHeader: Codable, Sendable, Equatable {20 public var version: Int21 public var id: String22 public var deviceID: String23 public var sequence: Int24 public var createdAt: Date25 public var recordCount: Int26}2728/// Wire form of a batch. `ciphertext` is a SecretBox envelope (version byte, random nonce, ChaChaPoly ciphertext, tag).29struct SyncBatchEnvelope: Codable, Sendable {30 var header: Data31 var ciphertext: Data32}3334public struct SyncSummary: Sendable, Hashable, Codable {35 public var pulledBatches = 036 public var appliedRecords = 037 public var pushedBatches = 038 public var pushedRecords = 039 public var skippedBatches: [String] = [] // damaged batches that were ignored40 public var finishedAt = Date()41}4243public enum SyncStatus: Sendable, Equatable {44 case localOnly45 case idle(last: SyncSummary?)46 case syncing47 case failed(String)48}4950public actor SyncEngine {51 public static let formatVersion = 152 static let verifierMessage = Data("localchat-sync-verifier-v1".utf8)53 static let maxRecordsPerBatch = 40054 static let maxBatchPlaintextBytes = 3 * 1_048_5765556 struct State: Codable {57 var deviceID: String = UUID().uuidString58 var cursor: String?59 var appliedBatchIDs: Set<String> = []60 var lastPushedAt: Date?61 var nextSequence: Int = 162 /// Version (updatedAt) of each record received from other devices, so it is not pushed back (no echo).63 var pulledVersions: [String: Double] = [:]64 var lastSummary: SyncSummary?65 }6667 private let store: any SyncableStore68 private let stateURL: URL69 private var transport: (any SyncTransport)?70 private var masterKey: SymmetricKey?71 private var recordKey: SymmetricKey?72 private var state: State73 private var isSyncing = false74 private var status: SyncStatus = .localOnly75 private var observers: [UUID: AsyncStream<SyncStatus>.Continuation] = [:]76 private let iterations: Int7778 /// `stateURL` holds this device's sync bookkeeping (no secrets). With no transport configured nothing ever leaves the device.79 public init(store: any SyncableStore, stateURL: URL, pbkdfIterations: Int = KeyDerivation.defaultIterations) {80 self.store = store; self.stateURL = stateURL; self.iterations = pbkdfIterations81 state = (try? Data(contentsOf: stateURL)).flatMap { try? CoreCoding.decoder.decode(State.self, from: $0) } ?? State()82 }8384 public var deviceID: String { state.deviceID }85 public var isConfigured: Bool { transport != nil && recordKey != nil }86 public var currentStatus: SyncStatus { status }87 public var lastSummary: SyncSummary? { state.lastSummary }8889 public func statusUpdates() -> AsyncStream<SyncStatus> {90 AsyncStream { continuation in91 let id = UUID()92 observers[id] = continuation93 continuation.yield(status)94 continuation.onTermination = { _ in Task { await self.removeObserver(id) } }95 }96 }97 private func removeObserver(_ id: UUID) { observers[id] = nil }98 private func set(_ new: SyncStatus) { status = new; observers.values.forEach { $0.yield(new) } }99100 private func persist() {101 try? FileManager.default.createDirectory(at: stateURL.deletingLastPathComponent(), withIntermediateDirectories: true)102 try? CoreCoding.encoder.encode(state).write(to: stateURL, options: .atomic)103 }104105 /// Turns sync on. The first device creates the shared metadata; the others verify the passphrase against it.106 /// Returns the derived master key so the app can keep it in the Keychain and call `configure(transport:masterKey:)` at next launch.107 @discardableResult108 public func configure(transport: any SyncTransport, passphrase: String) async throws -> Data {109 let key: SymmetricKey110 if let data = try await transport.fetchMetadata() {111 guard let meta = try? CoreCoding.decoder.decode(SyncMetadata.self, from: data) else { throw SyncError.tampered("metadata") }112 guard meta.version <= Self.formatVersion else { throw SyncError.unsupportedVersion(meta.version) }113 key = try KeyDerivation.key(fromPassphrase: passphrase, salt: meta.salt, iterations: meta.iterations)114 guard Self.verifier(for: key) == meta.verifier else { throw SyncError.wrongPassphrase }115 } else {116 let salt = KeyDerivation.randomSalt()117 key = try KeyDerivation.key(fromPassphrase: passphrase, salt: salt, iterations: iterations)118 let meta = SyncMetadata(version: Self.formatVersion, salt: salt, iterations: iterations, verifier: Self.verifier(for: key), createdAt: Date())119 try await transport.storeMetadata(try CoreCoding.encoder.encode(meta))120 }121 let raw = key.withUnsafeBytes { Data($0) }122 try await configure(transport: transport, masterKey: raw)123 return raw124 }125126 /// Resumes sync with a key kept in the Keychain. The key is checked against the shared metadata.127 public func configure(transport: any SyncTransport, masterKey raw: Data) async throws {128 let key = SymmetricKey(data: raw)129 if let data = try await transport.fetchMetadata(), let meta = try? CoreCoding.decoder.decode(SyncMetadata.self, from: data) {130 guard Self.verifier(for: key) == meta.verifier else { throw SyncError.wrongPassphrase }131 }132 self.transport = transport; masterKey = key133 recordKey = KeyDerivation.subkey(from: key, purpose: "sync-records")134 set(.idle(last: state.lastSummary))135 }136137 /// Back to local-only mode. `forgetProgress` also resets the bookkeeping (use when leaving a sync group for good).138 public func disable(forgetProgress: Bool = false) {139 transport = nil; masterKey = nil; recordKey = nil140 if forgetProgress { let id = state.deviceID; state = State(); state.deviceID = id; persist() }141 set(.localOnly)142 }143144 static func verifier(for key: SymmetricKey) -> Data {145 Data(HMAC<SHA256>.authenticationCode(for: verifierMessage, using: KeyDerivation.subkey(from: key, purpose: "sync-verify")))146 }147148 static func versionKey(_ r: SyncRecord) -> String { r.kind.rawValue + ":" + r.id }149150 // MARK: batches151152 static func seal(_ records: [SyncRecord], header: SyncBatchHeader, key: SymmetricKey) throws -> Data {153 let headerData = try CoreCoding.encoder.encode(header)154 let cipher = try SecretBox.seal(try CoreCoding.encoder.encode(records), key: key, associatedData: headerData)155 return try JSONEncoder().encode(SyncBatchEnvelope(header: headerData, ciphertext: cipher))156 }157158 static func open(_ data: Data, expectedID: String, key: SymmetricKey) throws -> (SyncBatchHeader, [SyncRecord]) {159 guard let envelope = try? JSONDecoder().decode(SyncBatchEnvelope.self, from: data),160 let header = try? CoreCoding.decoder.decode(SyncBatchHeader.self, from: envelope.header) else { throw SyncError.tampered(expectedID) }161 guard header.version <= formatVersion else { throw SyncError.unsupportedVersion(header.version) }162 guard header.id == expectedID else { throw SyncError.tampered(expectedID) } // a batch renamed or replayed under another id163 guard let plain = try? SecretBox.open(envelope.ciphertext, key: key, associatedData: envelope.header),164 let records = try? CoreCoding.decoder.decode([SyncRecord].self, from: plain) else { throw SyncError.tampered(expectedID) }165 return (header, records)166 }167168 // MARK: sync169170 /// Pull, apply, push. Local changes are found with `store.changes(since:)`, so the store must stamp `updatedAt` with the171 /// current time on every local write (an edit stamped in the past, before the previous sync, would not be uploaded).172 /// Safe to call again after an interruption: applied batches are remembered and uploads are idempotent.173 @discardableResult174 public func sync() async throws -> SyncSummary {175 guard let transport, let recordKey else { throw SyncError.notConfigured }176 guard !isSyncing else { throw SyncError.alreadySyncing }177 isSyncing = true; set(.syncing)178 defer { isSyncing = false }179 var summary = SyncSummary()180 do {181 // 1. pull182 let refs = try await transport.listBatches(after: state.cursor)183 for ref in refs {184 try Task.checkCancellation()185 if state.appliedBatchIDs.contains(ref.id) || ref.id.hasPrefix(state.deviceID + "-") {186 if let c = ref.cursor { state.cursor = c }187 continue188 }189 let data = try await transport.fetchBatch(ref.id) // a failure here leaves the cursor before this batch190 let records: [SyncRecord]191 do { records = try Self.open(data, expectedID: ref.id, key: recordKey).1 }192 catch SyncError.tampered {193 // nothing from a damaged batch is applied; it is skipped for good194 summary.skippedBatches.append(ref.id); state.appliedBatchIDs.insert(ref.id)195 if let c = ref.cursor { state.cursor = c }196 continue197 }198 summary.appliedRecords += try await store.apply(records)199 for r in records {200 let k = Self.versionKey(r)201 state.pulledVersions[k] = max(state.pulledVersions[k] ?? 0, r.updatedAt.timeIntervalSince1970)202 }203 state.appliedBatchIDs.insert(ref.id); summary.pulledBatches += 1204 if let c = ref.cursor { state.cursor = c }205 persist()206 }207 // 2. push what changed here and was not just received208 let pushStart = Date()209 let local = try await store.changes(since: state.lastPushedAt).filter { r in210 guard let seen = state.pulledVersions[Self.versionKey(r)] else { return true }211 return r.updatedAt.timeIntervalSince1970 > seen + 0.0005212 }213 var index = 0214 while index < local.count {215 try Task.checkCancellation()216 var chunk: [SyncRecord] = []; var bytes = 0217 while index < local.count, chunk.count < Self.maxRecordsPerBatch, chunk.isEmpty || bytes + local[index].payload.count <= Self.maxBatchPlaintextBytes {218 bytes += local[index].payload.count; chunk.append(local[index]); index += 1219 }220 let sequence = state.nextSequence221 let id = "\(state.deviceID)-\(String(format: "%08d", sequence))"222 let header = SyncBatchHeader(version: Self.formatVersion, id: id, deviceID: state.deviceID, sequence: sequence, createdAt: Date(), recordCount: chunk.count)223 try await transport.uploadBatch(id: id, data: try Self.seal(chunk, header: header, key: recordKey))224 state.nextSequence = sequence + 1; state.appliedBatchIDs.insert(id)225 // these versions are now on the transport: a crash before lastPushedAt moves must not upload them twice226 for r in chunk { state.pulledVersions[Self.versionKey(r)] = r.updatedAt.timeIntervalSince1970 }227 summary.pushedBatches += 1; summary.pushedRecords += chunk.count228 persist()229 }230 // a small overlap covers writes that landed while the query ran; duplicates are filtered by pulledVersions231 state.lastPushedAt = pushStart.addingTimeInterval(-2)232 summary.finishedAt = Date(); state.lastSummary = summary233 persist(); set(.idle(last: summary))234 return summary235 } catch {236 persist()237 let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)238 set(error is CancellationError ? .idle(last: state.lastSummary) : .failed(message))239 throw error240 }241 }242243 /// Uploads everything again (e.g. after the relay pruned old batches and a new device must be seeded).244 public func resetPushProgress() { state.lastPushedAt = nil; state.pulledVersions = [:]; persist() }245}246247public extension SyncRecord.Kind {248 /// Model weights, downloads and indexed document contents are never synchronized: no record kind can carry them.249 static var carriesModelWeights: Bool { false }250}251252/// Hooks used by the unit tests to exercise batch sealing without a transport.253enum SyncEngineInternals {254 static func sealForTest(header: (id: String, device: String), text: String) throws -> (data: Data, key: SymmetricKey) {255 let key = SymmetricKey(size: .bits256)256 let h = SyncBatchHeader(version: SyncEngine.formatVersion, id: header.id, deviceID: header.device, sequence: 1, createdAt: Date(), recordCount: 1)257 let record = SyncRecord(kind: .conversation, id: "c", updatedAt: Date(), payload: Data(text.utf8))258 let data = try SyncEngine.seal([record], header: h, key: key)259 _ = try SyncEngine.open(data, expectedID: header.id, key: key) // sanity: the untouched batch opens260 return (data, key)261 }262 /// Rewrites the clear header (sequence 1 → 2) and tries to open: must throw because the header is the AEAD associated data.263 static func openWithAlteredHeader(_ sealed: (data: Data, key: SymmetricKey)) throws {264 var envelope = try JSONDecoder().decode(SyncBatchEnvelope.self, from: sealed.data)265 var header = try CoreCoding.decoder.decode(SyncBatchHeader.self, from: envelope.header)266 let id = header.id; header.sequence = 2267 envelope.header = try CoreCoding.encoder.encode(header)268 _ = try SyncEngine.open(try JSONEncoder().encode(envelope), expectedID: id, key: sealed.key)269 }270}271