Repository Public
1// SP Notes Core2// File: Sources/SPNotesSync/SyncEngine+Entities.swift3// Description: The 2.0 part of the engine (API-NEXT §3, §4.2, §4.8): server features, the entity change feed applied to4// the generic `entities` table, the task outbox (create / update / delete with client ids), and canvases5// (optimistic PUT, three-way merge by node / edge id on divergence, conflicts kept visible). Against a 2.x6// server none of this runs and the engine behaves exactly as 1.0.7//8// Author: Simon-Pierre Boucher9// Contact: contact@spboucher.ai10import Foundation11import SPNotesAPI12import SPNotesDocument13import SPNotesStore1415/// Payload of a queued task creation (`entity_ops`, kind `task`, verb `create`). The task id is the entity id.16public struct QueuedTaskDraft: Codable, Sendable, Equatable {17 public var title: String18 public var status: String?19 public var priority: String?20 public var dueAt: Date?21 /// A bare calendar day `YYYY-MM-DD` (the server stores it at 12:00 UTC).22 public var due: String?23 public var project: String?24 public var pageId: String?25 public var blockId: String?2627 public init(title: String, status: String? = nil, priority: String? = nil, dueAt: Date? = nil, due: String? = nil, project: String? = nil,28 pageId: String? = nil, blockId: String? = nil) {29 self.title = title; self.status = status; self.priority = priority; self.dueAt = dueAt; self.due = due30 self.project = project; self.pageId = pageId; self.blockId = blockId31 }32}3334/// Payload of a queued task update (verb `update`): only the fields the user changed. A field listed in `clear` is35/// sent as JSON `null` (`dueAt`, `project`, `pageId`, `blockId`).36public struct QueuedTaskChanges: Codable, Sendable, Equatable {37 public var title: String?38 public var status: String?39 public var priority: String?40 public var dueAt: Date?41 public var due: String?42 public var project: String?43 public var pageId: String?44 public var blockId: String?45 public var clear: [String]?4647 public init(title: String? = nil, status: String? = nil, priority: String? = nil, dueAt: Date? = nil, due: String? = nil, project: String? = nil,48 pageId: String? = nil, blockId: String? = nil, clear: [String]? = nil) {49 self.title = title; self.status = status; self.priority = priority; self.dueAt = dueAt; self.due = due50 self.project = project; self.pageId = pageId; self.blockId = blockId; self.clear = clear51 }5253 var changes: TaskChanges {54 func patch<V>(_ value: V?, _ name: String) -> PatchValue<V>? {55 if let value { return .value(value) }56 return clear?.contains(name) == true ? .null : nil57 }58 return TaskChanges(title: title, status: status.flatMap(TaskStatus.init(rawValue:)), priority: priority.flatMap(TaskPriority.init(rawValue:)),59 dueAt: patch(dueAt, "dueAt"), dueDay: due.map { .value($0) }, project: patch(project, "project"),60 pageId: patch(pageId, "pageId"), blockId: patch(blockId, "blockId"))61 }62}6364extension SyncEngine {65 // MARK: - Server features6667 /// What the server supports, as last answered by `GET /api/meta` (nil before the first cycle; `.legacy` for 2.x).68 public func serverFeatures() -> ServerMeta? { serverMeta }6970 /// Asks `/api/meta` when the stored answer is missing or older than `options.metaMaxAge`. A copy is kept in the71 /// store cache (`CacheKey.meta`) so that the repositories know the features offline.72 func refreshServerMeta(force: Bool = false) async throws {73 if !force, serverMeta != nil, let at = serverMetaAt, Date().timeIntervalSince(at) < options.metaMaxAge { return }74 let meta = try await api.meta()75 serverMeta = meta76 serverMetaAt = Date()77 if let data = try? JSONEncoder().encode(meta) { try? await store.setCached(CacheKey.meta, json: data) }78 }7980 func supports(_ feature: String) -> Bool { serverMeta?.has(feature) ?? false }8182 /// The `kinds` to ask the feed for, in the server's order; empty against a server without the `entities` feature.83 func feedKinds() -> [String] {84 guard supports(ServerFeature.entities) else { return [] }85 let wanted = Set(options.entityKinds.map(\.rawValue))86 return SyncKind.all.filter(wanted.contains)87 }8889 // MARK: - Pull: one window of the entity feed9091 private struct CanvasRow: Decodable {92 var data: RawJSON93 var version: Int94 }9596 /// Applies the entities and the entity tombstones of one answer. Rows the server marks deleted (`deletedAt`:97 /// tasks, databases, rows, references) leave the local cache like tombstones do. Canvases go to their own table,98 /// where a local edit meets the server copy through the three-way merge. Returns true when a canvas merge left99 /// something to push.100 func applyEntityWindow(_ feed: SyncPage, kinds: [String], collecting seen: inout [EntityKind: Set<String>]?) async throws -> Bool {101 var rows: [RemoteEntity] = []102 var stones: [EntityTombstone] = []103 var canvases: [SyncEntityRow] = []104 for (name, list) in feed.entities ?? [:] {105 let kind = EntityKind(name)106 if name == SyncKind.canvas { canvases = list; continue }107 for row in list {108 if row.deletedAt != nil { stones.append(EntityTombstone(kind: kind, id: row.id)); continue }109 seen?[kind, default: []].insert(row.id)110 rows.append(RemoteEntity(kind: kind, id: row.id, rev: row.rev, json: row.json.data, updatedAt: row.updatedAt))111 }112 }113 var purgedCanvases: [String] = []114 for stone in feed.tombstones where stone.kind != SPNotesAPI.Tombstone.pageKind && stone.kind != SPNotesAPI.Tombstone.attachmentKind {115 if stone.kind == SyncKind.canvas { purgedCanvases.append(stone.id) } else { stones.append(EntityTombstone(kind: EntityKind(stone.kind), id: stone.id)) }116 }117 var cursors: [EntityKind: Int64] = [:]118 for name in kinds { cursors[EntityKind(name)] = feed.cursor }119 if !rows.isEmpty || !stones.isEmpty || !cursors.isEmpty { try await store.applyRemoteEntities(rows, tombstones: stones, cursors: cursors) }120121 var merged = false122 for row in canvases {123 guard let canvas = try? row.decode(CanvasRow.self) else { continue }124 let remote = Self.canonicalCanvas(canvas.data.data)125 if case .diverged(let base, _, _, let version) = try await store.applyRemoteCanvas(pageId: row.id, json: remote, version: canvas.version, rev: row.rev) {126 if try await reconcileCanvas(pageId: row.id, base: base, remote: remote, remoteVersion: version) { merged = true }127 }128 }129 // The page of a purged canvas is purged too (its tombstone is in the same feed): a canvas without local changes goes;130 // one with unsent changes is left to the store's page purge, which decides with the page.131 for id in purgedCanvases {132 if let record = try await store.canvas(pageId: id), !record.dirty { try await store.deleteCanvas(pageId: id) }133 }134 return merged135 }136137 // MARK: - Push: the task outbox138139 /// The entity operations this build executes; anything else in the outbox is marked unsupported (kept, surfaced).140 public static let supportedEntityOps: [EntityKind: Set<String>] = [.task: [EntityOpVerb.create, EntityOpVerb.update, EntityOpVerb.delete]]141142 private enum EntityDeferral: Error { case pageNotCreated }143 /// Not refused for good: tried again with the outbox backoff (5 s … 15 min).144 private struct EntityRetryLater: Error { var message: String }145146 /// Sends the queued task operations in order. The order of one task is never broken: when an operation does not147 /// go through, the later ones of the same task wait. Nothing runs against a server without `tasks-v2` (the148 /// operations stay queued).149 func pushEntityOps() async throws {150 guard supports(ServerFeature.tasksV2) else { return }151 // Operations set aside by an older build that this one knows go back in line.152 try await store.retryEntityOps(state: .unsupported, where: { kind, verb in Self.supportedEntityOps[kind]?.contains(verb) ?? false })153 var blocked = Set<String>()154 for op in try await store.nextEntityOps(limit: 200) {155 let entity = op.kind.rawValue + ":" + op.entityId156 guard !blocked.contains(entity) else { continue }157 guard Self.supportedEntityOps[op.kind]?.contains(op.op) == true else {158 try await store.markEntityOpUnsupported(id: op.id, reason: "unsupported entity op: \(op.kind.rawValue) \(op.op)")159 continue160 }161 do {162 let answer = try await sendTaskOp(op)163 try await store.completeEntityOp(id: op.id)164 try await settleTask(id: op.entityId, answer: answer)165 } catch is EntityDeferral {166 blocked.insert(entity)167 } catch let later as EntityRetryLater {168 blocked.insert(entity)169 _ = try await store.failEntityOp(id: op.id, error: later.message)170 } catch {171 if stopsCycle(error) { throw error }172 blocked.insert(entity)173 if let api = error as? APIError, api.isBadRequest || api.status == 409 || api.status == 403 || api.status == 404 {174 // Refused for what it is: kept with its payload and surfaced (`SyncStatus.rejected`), never dropped.175 try await store.rejectEntityOp(id: op.id, error: Self.describe(error))176 } else {177 _ = try await store.failEntityOp(id: op.id, error: Self.describe(error))178 }179 }180 }181 try await settleAcknowledgedTasks()182 }183184 /// A task whose last operation was acknowledged but whose local row could not take the server's copy then (the185 /// fresh read failed, the app was killed between the two steps) is still flagged as having local changes, with186 /// an empty outbox. It is settled here, from the server.187 private func settleAcknowledgedTasks() async throws {188 for record in try await store.dirtyEntities(kind: .task) {189 guard try await store.entityOps(kind: .task, entityId: record.id).isEmpty else { continue }190 if record.deleted { _ = try await store.markEntitySynced(kind: .task, id: record.id); continue }191 do {192 let task: TaskItem?193 do { task = try await api.task(id: record.id) } catch let error as APIError where error.isNotFound { task = nil }194 try await adopt(taskId: record.id, server: task)195 } catch {196 if stopsCycle(error) { throw error }197 }198 }199 }200201 /// The local row of a task with an empty outbox takes the server's state (nil or deleted there: it leaves).202 private func adopt(taskId id: String, server: TaskItem?) async throws {203 guard let task = server, task.deletedAt == nil else {204 _ = try await store.markEntitySynced(kind: .task, id: id)205 try await store.applyRemoteEntities([], tombstones: [EntityTombstone(kind: .task, id: id)])206 return207 }208 _ = try await store.markEntitySynced(kind: .task, id: id, json: try APIJSON.encoder().encode(task), rev: task.rev)209 }210211 /// Runs one task operation. Returns the task as the server holds it afterwards (nil when it no longer exists there).212 private func sendTaskOp(_ op: EntityOp) async throws -> TaskItem? {213 let decoder = APIJSON.decoder()214 switch op.op {215 case EntityOpVerb.create:216 guard let payload = op.payload, let draft = try? decoder.decode(QueuedTaskDraft.self, from: payload) else {217 throw APIError.http(status: 400, code: nil, message: "Unreadable queued task", body: Data())218 }219 var pageId = draft.pageId220 if let id = pageId {221 // The page of a task must exist on the server first; a page that is gone from this device no longer anchors it.222 if let page = try await store.page(id: id) { if page.needsCreate { throw EntityDeferral.pageNotCreated } } else { pageId = nil }223 }224 // No idempotency key: the client-chosen id makes the creation safe to repeat, and the answer is always the stored row.225 let body = TaskDraft(title: draft.title, dueAt: draft.dueAt, priority: draft.priority.flatMap(TaskPriority.init(rawValue:)), project: draft.project, pageId: pageId,226 id: op.entityId, blockId: pageId == nil ? nil : draft.blockId, status: draft.status.flatMap(TaskStatus.init(rawValue:)), due: draft.due)227 do {228 return try await api.createTaskIdempotent(body).value229 } catch let error as APIError where error.code == APIError.NextCode.badPage && pageId != nil {230 // The page is in the server's trash or was purged there. While this device still shows it live the creation231 // waits (the page may come back); the engine never invents another anchor.232 throw EntityRetryLater(message: error.message)233 }234 case EntityOpVerb.update:235 guard let payload = op.payload, let queued = try? decoder.decode(QueuedTaskChanges.self, from: payload) else {236 throw APIError.http(status: 400, code: nil, message: "Unreadable queued task changes", body: Data())237 }238 let changes = queued.changes239 do {240 if changes.isEmpty { return try await api.task(id: op.entityId) }241 // The op's own key: an update whose answer was lost is applied once, not again on top of a later change.242 return try await api.updateTask(id: op.entityId, changes, idempotencyKey: op.idempotencyKey)243 } catch let error as APIError where error.isNotFound {244 return nil // deleted on the server: the deletion wins, the change has nothing left to apply to245 }246 default:247 do { _ = try await api.removeTask(id: op.entityId) } catch let error as APIError where error.isNotFound {}248 return nil249 }250 }251252 /// After an operation of a task went through. While other operations of the task wait, the local row stays as the253 /// user left it. Once the outbox is empty the row takes the server's copy — the answer just received, or a fresh254 /// read when the feed brought a newer revision meanwhile (an answer replayed from an idempotency key describes the255 /// first attempt, not the present).256 private func settleTask(id: String, answer: TaskItem?) async throws {257 guard try await store.entityOps(kind: .task, entityId: id).isEmpty else { return }258 guard let record = try await store.entity(kind: .task, id: id) else { return }259 var current = answer260 if let task = current, record.rev > (task.rev ?? 0) || record.remoteJSON != nil {261 do { current = try await api.task(id: id) } catch let error as APIError where error.isNotFound { current = nil }262 }263 try await adopt(taskId: id, server: current)264 }265266 // MARK: - Push: canvases267268 /// Sends the canvases edited on this device. Returns true when a merge left something for the next round.269 func pushCanvases() async throws -> Bool {270 guard supports(ServerFeature.canvas) else { return false }271 var again = false272 for record in try await store.canvasesNeedingPush(limit: 50) {273 // The canvas row is born with its page: the page goes first.274 if let page = try await store.page(id: record.pageId), page.needsCreate { continue }275 do {276 if record.version == 0 {277 // Never synced (drawn offline on a new canvas page): the server copy — normally the empty canvas created with278 // the page — is read first and merged like any other, then the result is sent by the next round.279 let server = try await api.canvas(pageID: record.pageId)280 let remote = Self.canonicalCanvas(server.data.data)281 if try await reconcileCanvas(pageId: record.pageId, base: nil, remote: remote, remoteVersion: server.version) { again = true }282 continue283 }284 let saved = try await api.saveCanvas(pageID: record.pageId, data: RawJSON(data: record.json), baseVersion: record.version)285 try await store.markCanvasPushed(pageId: record.pageId, version: saved.version, pushedEditSeq: record.editSeq, pushed: record.json)286 } catch let error as APIError where error.isConflict {287 guard let server = error.canvasConflict else {288 try? await store.markCanvasPushFailed(pageId: record.pageId, error: Self.describe(error))289 continue290 }291 if try await reconcileCanvas(pageId: record.pageId, base: record.baseJSON, remote: Self.canonicalCanvas(server.data.data), remoteVersion: server.version) { again = true }292 } catch {293 if stopsCycle(error) { throw error }294 // 404: the page is in the server's trash (or purged — the feed then says so). 400 / 413: refused for what it is.295 // Either way the canvas keeps its local changes and is tried again at the next cycle.296 try? await store.markCanvasPushFailed(pageId: record.pageId, error: Self.describe(error))297 }298 }299 return again300 }301302 /// A local canvas with unsent changes meets another server copy. Identical → adopted. Otherwise the three-way merge303 /// by node / edge id (`CanvasDocument.merge`): a clean merge replaces the local canvas and is pushed on top of the304 /// server version; a merge with conflicts changes NOTHING locally — the server copy is stored beside the local305 /// canvas, which leaves the push list until the user decides (`CanvasRepository`). Neither side is ever lost.306 /// The local row is read again here: the decision is made on what the user sees now.307 func reconcileCanvas(pageId: String, base: Data?, remote: Data, remoteVersion: Int) async throws -> Bool {308 // The merge is written only if the canvas was not edited while it was computed (compare-and-set on `editSeq`);309 // otherwise it is computed again on what the user has now.310 for _ in 0..<6 {311 guard let record = try await store.canvas(pageId: pageId) else { return false }312 guard record.dirty else {313 _ = try await store.applyRemoteCanvas(pageId: pageId, json: remote, version: remoteVersion)314 return false315 }316 guard let local = try? CanvasDocument(data: record.json), let theirs = try? CanvasDocument(data: remote) else {317 try await store.recordCanvasConflict(pageId: pageId, remote: remote, remoteVersion: remoteVersion)318 return false319 }320 if local == theirs {321 // The server already holds this canvas (a save whose answer was lost).322 if try await store.resolveCanvas(pageId: pageId, merged: remote, remote: remote, remoteVersion: remoteVersion, ifEditSeq: record.editSeq) { return false }323 continue324 }325 let ancestor = (base ?? record.baseJSON).flatMap { try? CanvasDocument(data: $0) } ?? CanvasDocument()326 let result = CanvasDocument.merge(base: ancestor, local: local, remote: theirs)327 guard result.isClean else {328 try await store.recordCanvasConflict(pageId: pageId, remote: remote, remoteVersion: remoteVersion)329 return false330 }331 let merged = result.merged == theirs ? remote : result.merged.jsonData()332 if try await store.resolveCanvas(pageId: pageId, merged: merged, remote: remote, remoteVersion: remoteVersion, ifEditSeq: record.editSeq) { return merged != remote }333 }334 // The user edits faster than the merge can be written: the server copy is kept aside, nothing is lost, the next cycle merges.335 try await store.recordCanvasConflict(pageId: pageId, remote: remote, remoteVersion: remoteVersion)336 return false337 }338339 /// The canonical bytes of a canvas (what `CanvasDocument.jsonData()` writes), so that copies compare byte for byte340 /// whatever key order the server used. Unreadable JSON is kept as it is.341 static func canonicalCanvas(_ data: Data) -> Data {342 (try? CanvasDocument(data: data))?.jsonData() ?? data343 }344}345