SPB Git

Repository Public

SyncEngine.swift 14.3 KB 271 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatSync/Sources/SyncEngine.swift3// Description: Optional end-to-end encrypted synchronization of a SyncableStore: pull, decrypt, apply (last writer wins), then push local changes as encrypted batches.4import CryptoKit5import Foundation6import LocalChatCore7import LocalChatSecurity89/// Non-secret setup data shared through the transport so every device derives the same key from the passphrase.10public struct SyncMetadata: Codable, Sendable, Equatable {11    public var version: Int12    public var salt: Data13    public var iterations: Int14    public var verifier: Data          // HMAC-SHA256 with a subkey: tells a wrong passphrase apart from damaged data15    public var createdAt: Date16}1718/// The clear header of a batch. Its exact bytes are the AEAD associated data, so it cannot be altered or swapped.19public struct SyncBatchHeader: Codable, Sendable, Equatable {20    public var version: Int21    public var id: String22    public var deviceID: String23    public var sequence: Int24    public var createdAt: Date25    public var recordCount: Int26}2728/// Wire form of a batch. `ciphertext` is a SecretBox envelope (version byte, random nonce, ChaChaPoly ciphertext, tag).29struct SyncBatchEnvelope: Codable, Sendable {30    var header: Data31    var ciphertext: Data32}3334public struct SyncSummary: Sendable, Hashable, Codable {35    public var pulledBatches = 036    public var appliedRecords = 037    public var pushedBatches = 038    public var pushedRecords = 039    public var skippedBatches: [String] = []     // damaged batches that were ignored40    public var finishedAt = Date()41}4243public enum SyncStatus: Sendable, Equatable {44    case localOnly45    case idle(last: SyncSummary?)46    case syncing47    case failed(String)48}4950public actor SyncEngine {51    public static let formatVersion = 152    static let verifierMessage = Data("localchat-sync-verifier-v1".utf8)53    static let maxRecordsPerBatch = 40054    static let maxBatchPlaintextBytes = 3 * 1_048_5765556    struct State: Codable {57        var deviceID: String = UUID().uuidString58        var cursor: String?59        var appliedBatchIDs: Set<String> = []60        var lastPushedAt: Date?61        var nextSequence: Int = 162        /// Version (updatedAt) of each record received from other devices, so it is not pushed back (no echo).63        var pulledVersions: [String: Double] = [:]64        var lastSummary: SyncSummary?65    }6667    private let store: any SyncableStore68    private let stateURL: URL69    private var transport: (any SyncTransport)?70    private var masterKey: SymmetricKey?71    private var recordKey: SymmetricKey?72    private var state: State73    private var isSyncing = false74    private var status: SyncStatus = .localOnly75    private var observers: [UUID: AsyncStream<SyncStatus>.Continuation] = [:]76    private let iterations: Int7778    /// `stateURL` holds this device's sync bookkeeping (no secrets). With no transport configured nothing ever leaves the device.79    public init(store: any SyncableStore, stateURL: URL, pbkdfIterations: Int = KeyDerivation.defaultIterations) {80        self.store = store; self.stateURL = stateURL; self.iterations = pbkdfIterations81        state = (try? Data(contentsOf: stateURL)).flatMap { try? CoreCoding.decoder.decode(State.self, from: $0) } ?? State()82    }8384    public var deviceID: String { state.deviceID }85    public var isConfigured: Bool { transport != nil && recordKey != nil }86    public var currentStatus: SyncStatus { status }87    public var lastSummary: SyncSummary? { state.lastSummary }8889    public func statusUpdates() -> AsyncStream<SyncStatus> {90        AsyncStream { continuation in91            let id = UUID()92            observers[id] = continuation93            continuation.yield(status)94            continuation.onTermination = { _ in Task { await self.removeObserver(id) } }95        }96    }97    private func removeObserver(_ id: UUID) { observers[id] = nil }98    private func set(_ new: SyncStatus) { status = new; observers.values.forEach { $0.yield(new) } }99100    private func persist() {101        try? FileManager.default.createDirectory(at: stateURL.deletingLastPathComponent(), withIntermediateDirectories: true)102        try? CoreCoding.encoder.encode(state).write(to: stateURL, options: .atomic)103    }104105    /// Turns sync on. The first device creates the shared metadata; the others verify the passphrase against it.106    /// Returns the derived master key so the app can keep it in the Keychain and call `configure(transport:masterKey:)` at next launch.107    @discardableResult108    public func configure(transport: any SyncTransport, passphrase: String) async throws -> Data {109        let key: SymmetricKey110        if let data = try await transport.fetchMetadata() {111            guard let meta = try? CoreCoding.decoder.decode(SyncMetadata.self, from: data) else { throw SyncError.tampered("metadata") }112            guard meta.version <= Self.formatVersion else { throw SyncError.unsupportedVersion(meta.version) }113            key = try KeyDerivation.key(fromPassphrase: passphrase, salt: meta.salt, iterations: meta.iterations)114            guard Self.verifier(for: key) == meta.verifier else { throw SyncError.wrongPassphrase }115        } else {116            let salt = KeyDerivation.randomSalt()117            key = try KeyDerivation.key(fromPassphrase: passphrase, salt: salt, iterations: iterations)118            let meta = SyncMetadata(version: Self.formatVersion, salt: salt, iterations: iterations, verifier: Self.verifier(for: key), createdAt: Date())119            try await transport.storeMetadata(try CoreCoding.encoder.encode(meta))120        }121        let raw = key.withUnsafeBytes { Data($0) }122        try await configure(transport: transport, masterKey: raw)123        return raw124    }125126    /// Resumes sync with a key kept in the Keychain. The key is checked against the shared metadata.127    public func configure(transport: any SyncTransport, masterKey raw: Data) async throws {128        let key = SymmetricKey(data: raw)129        if let data = try await transport.fetchMetadata(), let meta = try? CoreCoding.decoder.decode(SyncMetadata.self, from: data) {130            guard Self.verifier(for: key) == meta.verifier else { throw SyncError.wrongPassphrase }131        }132        self.transport = transport; masterKey = key133        recordKey = KeyDerivation.subkey(from: key, purpose: "sync-records")134        set(.idle(last: state.lastSummary))135    }136137    /// Back to local-only mode. `forgetProgress` also resets the bookkeeping (use when leaving a sync group for good).138    public func disable(forgetProgress: Bool = false) {139        transport = nil; masterKey = nil; recordKey = nil140        if forgetProgress { let id = state.deviceID; state = State(); state.deviceID = id; persist() }141        set(.localOnly)142    }143144    static func verifier(for key: SymmetricKey) -> Data {145        Data(HMAC<SHA256>.authenticationCode(for: verifierMessage, using: KeyDerivation.subkey(from: key, purpose: "sync-verify")))146    }147148    static func versionKey(_ r: SyncRecord) -> String { r.kind.rawValue + ":" + r.id }149150    // MARK: batches151152    static func seal(_ records: [SyncRecord], header: SyncBatchHeader, key: SymmetricKey) throws -> Data {153        let headerData = try CoreCoding.encoder.encode(header)154        let cipher = try SecretBox.seal(try CoreCoding.encoder.encode(records), key: key, associatedData: headerData)155        return try JSONEncoder().encode(SyncBatchEnvelope(header: headerData, ciphertext: cipher))156    }157158    static func open(_ data: Data, expectedID: String, key: SymmetricKey) throws -> (SyncBatchHeader, [SyncRecord]) {159        guard let envelope = try? JSONDecoder().decode(SyncBatchEnvelope.self, from: data),160              let header = try? CoreCoding.decoder.decode(SyncBatchHeader.self, from: envelope.header) else { throw SyncError.tampered(expectedID) }161        guard header.version <= formatVersion else { throw SyncError.unsupportedVersion(header.version) }162        guard header.id == expectedID else { throw SyncError.tampered(expectedID) }     // a batch renamed or replayed under another id163        guard let plain = try? SecretBox.open(envelope.ciphertext, key: key, associatedData: envelope.header),164              let records = try? CoreCoding.decoder.decode([SyncRecord].self, from: plain) else { throw SyncError.tampered(expectedID) }165        return (header, records)166    }167168    // MARK: sync169170    /// Pull, apply, push. Local changes are found with `store.changes(since:)`, so the store must stamp `updatedAt` with the171    /// current time on every local write (an edit stamped in the past, before the previous sync, would not be uploaded).172    /// Safe to call again after an interruption: applied batches are remembered and uploads are idempotent.173    @discardableResult174    public func sync() async throws -> SyncSummary {175        guard let transport, let recordKey else { throw SyncError.notConfigured }176        guard !isSyncing else { throw SyncError.alreadySyncing }177        isSyncing = true; set(.syncing)178        defer { isSyncing = false }179        var summary = SyncSummary()180        do {181            // 1. pull182            let refs = try await transport.listBatches(after: state.cursor)183            for ref in refs {184                try Task.checkCancellation()185                if state.appliedBatchIDs.contains(ref.id) || ref.id.hasPrefix(state.deviceID + "-") {186                    if let c = ref.cursor { state.cursor = c }187                    continue188                }189                let data = try await transport.fetchBatch(ref.id)   // a failure here leaves the cursor before this batch190                let records: [SyncRecord]191                do { records = try Self.open(data, expectedID: ref.id, key: recordKey).1 }192                catch SyncError.tampered {193                    // nothing from a damaged batch is applied; it is skipped for good194                    summary.skippedBatches.append(ref.id); state.appliedBatchIDs.insert(ref.id)195                    if let c = ref.cursor { state.cursor = c }196                    continue197                }198                summary.appliedRecords += try await store.apply(records)199                for r in records {200                    let k = Self.versionKey(r)201                    state.pulledVersions[k] = max(state.pulledVersions[k] ?? 0, r.updatedAt.timeIntervalSince1970)202                }203                state.appliedBatchIDs.insert(ref.id); summary.pulledBatches += 1204                if let c = ref.cursor { state.cursor = c }205                persist()206            }207            // 2. push what changed here and was not just received208            let pushStart = Date()209            let local = try await store.changes(since: state.lastPushedAt).filter { r in210                guard let seen = state.pulledVersions[Self.versionKey(r)] else { return true }211                return r.updatedAt.timeIntervalSince1970 > seen + 0.0005212            }213            var index = 0214            while index < local.count {215                try Task.checkCancellation()216                var chunk: [SyncRecord] = []; var bytes = 0217                while index < local.count, chunk.count < Self.maxRecordsPerBatch, chunk.isEmpty || bytes + local[index].payload.count <= Self.maxBatchPlaintextBytes {218                    bytes += local[index].payload.count; chunk.append(local[index]); index += 1219                }220                let sequence = state.nextSequence221                let id = "\(state.deviceID)-\(String(format: "%08d", sequence))"222                let header = SyncBatchHeader(version: Self.formatVersion, id: id, deviceID: state.deviceID, sequence: sequence, createdAt: Date(), recordCount: chunk.count)223                try await transport.uploadBatch(id: id, data: try Self.seal(chunk, header: header, key: recordKey))224                state.nextSequence = sequence + 1; state.appliedBatchIDs.insert(id)225                // these versions are now on the transport: a crash before lastPushedAt moves must not upload them twice226                for r in chunk { state.pulledVersions[Self.versionKey(r)] = r.updatedAt.timeIntervalSince1970 }227                summary.pushedBatches += 1; summary.pushedRecords += chunk.count228                persist()229            }230            // a small overlap covers writes that landed while the query ran; duplicates are filtered by pulledVersions231            state.lastPushedAt = pushStart.addingTimeInterval(-2)232            summary.finishedAt = Date(); state.lastSummary = summary233            persist(); set(.idle(last: summary))234            return summary235        } catch {236            persist()237            let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)238            set(error is CancellationError ? .idle(last: state.lastSummary) : .failed(message))239            throw error240        }241    }242243    /// Uploads everything again (e.g. after the relay pruned old batches and a new device must be seeded).244    public func resetPushProgress() { state.lastPushedAt = nil; state.pulledVersions = [:]; persist() }245}246247public extension SyncRecord.Kind {248    /// Model weights, downloads and indexed document contents are never synchronized: no record kind can carry them.249    static var carriesModelWeights: Bool { false }250}251252/// Hooks used by the unit tests to exercise batch sealing without a transport.253enum SyncEngineInternals {254    static func sealForTest(header: (id: String, device: String), text: String) throws -> (data: Data, key: SymmetricKey) {255        let key = SymmetricKey(size: .bits256)256        let h = SyncBatchHeader(version: SyncEngine.formatVersion, id: header.id, deviceID: header.device, sequence: 1, createdAt: Date(), recordCount: 1)257        let record = SyncRecord(kind: .conversation, id: "c", updatedAt: Date(), payload: Data(text.utf8))258        let data = try SyncEngine.seal([record], header: h, key: key)259        _ = try SyncEngine.open(data, expectedID: header.id, key: key)   // sanity: the untouched batch opens260        return (data, key)261    }262    /// Rewrites the clear header (sequence 1 → 2) and tries to open: must throw because the header is the AEAD associated data.263    static func openWithAlteredHeader(_ sealed: (data: Data, key: SymmetricKey)) throws {264        var envelope = try JSONDecoder().decode(SyncBatchEnvelope.self, from: sealed.data)265        var header = try CoreCoding.decoder.decode(SyncBatchHeader.self, from: envelope.header)266        let id = header.id; header.sequence = 2267        envelope.header = try CoreCoding.encoder.encode(header)268        _ = try SyncEngine.open(try JSONEncoder().encode(envelope), expectedID: id, key: sealed.key)269    }270}271
Files main
Menu

Theme