SPB Git

Repository Public

SyncHardeningTests.swift 34.6 KB 456 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : SP Mail9// File             : Packages/SPMailCore/Tests/SPMailCoreTests/SyncHardeningTests.swift10// Description      : Unit tests of the SP Mail Next synchronisation hardening11//                    that need no server: quote-mode sanitising, lossless12//                    drafts, schema migration from a 1.1.0 database, statement13//                    cache and batching, attachment cache, clear-text decision,14//                    pending-patch application.15// Module           : SPMailCoreTests16// Repository       : https://git.spboucher.ai/sp-mail-apple17// Created          : 2026-11-2018// Last Modified    : 2026-11-2019//20// =============================================================================21// AUTHOR & PROFESSIONAL AFFILIATION22// =============================================================================23//24// Author           : Simon-Pierre Boucher25// Position         : Professor of Finance26// Institution      : Université du Québec en Outaouais (UQO)27// Department       : Département des sciences administratives28// Organization     : SP Boucher29//30// Website          : https://spboucher.ai31// Git              : https://git.spboucher.ai32//33// =============================================================================34// OFFICIAL CONTACT DIRECTORY35// =============================================================================36//37// General          : info@spboucher.ai38// Professional     : contact@spboucher.ai39// Direct           : simon@spboucher.ai40// Research         : research@spboucher.ai41// Development      : dev@spboucher.ai42// Support          : support@spboucher.ai43// Security/Admin   : admin@spboucher.ai44// Notifications    : notifications@spboucher.ai45//46// Preferred Contact: support@spboucher.ai47//48// =============================================================================49// COPYRIGHT & INTELLECTUAL PROPERTY50// =============================================================================51//52// Copyright (c) 202653// Simon-Pierre Boucher54//55// All rights reserved where applicable.56//57// This file is governed by the applicable project license,58// copyright notices, institutional agreements and relevant law.59//60// Unless expressly authorized by the applicable license or61// rights holder, copying, redistribution, sublicensing,62// publication and commercial exploitation are not permitted.63//64// Existing third-party rights and notices remain unaffected.65//66// =============================================================================67// CONFIDENTIALITY & SECURITY68// =============================================================================69//70// Classification   : Public71// Distribution     : Public repository; use is governed by the project license72// Security Contact : admin@spboucher.ai73//74// Access to this source does not independently grant permission75// to redistribute or disclose confidential materials.76//77// Do not embed credentials, private keys, access tokens,78// certificates or confidential personal information.79//80// =============================================================================81// LICENSE & MAINTENANCE82// =============================================================================83//84// License          : None specified (no license file in this repository)85// Maintainer       : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Foundation93import Testing94@testable import SPMailCore9596/// Collects what a stream delivers while a test runs.97final class Recorder<T: Sendable>: @unchecked Sendable {98    private let lock = NSLock()99    private var items: [T] = []100    private var task: Task<Void, Never>?101    init(_ stream: AsyncStream<T>) { task = Task { [weak self] in for await v in stream { self?.lock.withLock { self?.items.append(v) } } } }102    var all: [T] { lock.withLock { items } }103    deinit { task?.cancel() }104}105106@Suite struct QuoteSanitizerTests {107    static let original = #"<p>Bonjour <b>vous</b></p><img src="cid:logo@x" alt="Logo"><img src="cid:absent@x" alt="Perdu"><img src="cid:muet@x"><img src="https://tracker.example/p.gif?id=1" width="1" height="1"><img src="https://cdn.example/photo.jpg" alt="Photo"><img src="data:image/png;base64,iVBORw0KGgo="><script>x()</script><a href="https://example.org/a">lien <img src="//cdn.example/b.png" alt="B"></a>"#108109    // Item 1 — before the fix the reply builder ran the DISPLAY sanitiser: `spmail-cid://part/…` and `data-spm-remote` went out in the mail.110    @Test func quoteModeNeverEmitsAppInternalSchemesOrAttributes() {111        let display = HTMLSanitizer.sanitize(Self.original).html112        #expect(display.contains("spmail-cid://part/") && display.contains("data-spm-remote")) // what the old code quoted113        let q = HTMLSanitizer.sanitizeForQuoting(Self.original, availableCIDs: ["<logo@x>"])114        for marker in ["spmail-cid", "spmail-img", "data-spm", "tracker.example"] { #expect(!q.html.contains(marker), "\(marker) in quoted HTML: \(q.html)") }115        #expect(q.html.contains(#"<img src="cid:logo@x" alt="Logo">"#) && q.cids == ["logo@x"])116        #expect(q.html.contains("[Perdu]") && !q.html.contains("cid:absent@x") && !q.html.contains("muet@x")) // inline images that cannot travel are dropped cleanly117        #expect(q.html.contains(#"<a href="https://cdn.example/photo.jpg" rel="noopener noreferrer">[Photo]</a>"#)) // a real, non-loading reference118        #expect(!q.html.contains("<img src=\"http") && !q.html.contains("src=\"//")) // nothing loads from the sender119        #expect(q.html.contains("data:image/png;base64,iVBORw0KGgo=") && !q.html.contains("script"))120        #expect(q.html.contains(#"<a href="https://example.org/a" rel="noopener noreferrer">lien [B]</a>"#)) // no link nested in a link121        #expect(q.remoteImageCount == 3)122    }123124    @Test func replyKeepsCidReferencesAndReattachesTheirParts() {125        var e = sample("1", date: 1_700_000_000, subject: "Projet")126        e.to = [EmailAddress(email: "moi@stg.spboucher.ai")]127        let identity = Identity(id: "i", name: "Moi", email: "moi@stg.spboucher.ai")128        let body = EmailBody(text: "Bonjour", html: Self.original, attachments: [129            BodyPart(blobId: "B-logo", size: 10, name: "logo.png", type: "image/png", disposition: "inline", cid: "logo@x"),130            BodyPart(blobId: "B-unused", size: 10, name: "autre.png", type: "image/png", disposition: "inline", cid: "inutilise@x"),131            BodyPart(blobId: "B-pdf", size: 10, name: "a.pdf", type: "application/pdf", disposition: "attachment"),132        ])133        let reply = ReplyBuilder.build(kind: .reply, original: e, body: body, identity: identity, me: [])134        let html = reply.jmapEmail(mailboxId: "d", draft: true)["bodyValues"]["h"]["value"].stringValue ?? ""135        for marker in ["spmail-cid", "spmail-img", "data-spm"] { #expect(!html.contains(marker), "\(marker) leaves in a reply") }136        #expect(html.contains(#"src="cid:logo@x""#))137        #expect(reply.attachments.map(\.blobId) == ["B-logo"] && reply.attachments[0].inline && reply.attachments[0].cid == "logo@x") // only what the quote references138        let fwd = ReplyBuilder.build(kind: .forward, original: e, body: body, identity: identity, me: [])139        #expect(Set(fwd.attachments.compactMap(\.blobId)) == ["B-logo", "B-unused", "B-pdf"])140        #expect(fwd.attachments.first { $0.blobId == "B-logo" }?.inline == true && fwd.attachments.first { $0.blobId == "B-unused" }?.inline == false)141        #expect(fwd.fullHTML?.contains("spmail-") == false && fwd.fullHTML?.contains("data-spm") == false)142        // the quoted original is a block of its own that the draft model can split back143        #expect(reply.quotedHTML?.hasPrefix(OutgoingMessage.quoteBlockOpen) == true && reply.fullHTML?.contains("<blockquote") == true)144        // an inline part without a blob cannot be re-attached: its image is dropped rather than left dangling145        let orphan = EmailBody(html: #"<img src="cid:x@y" alt="X">"#, attachments: [BodyPart(size: 1, type: "image/png", cid: "x@y")])146        #expect(ReplyBuilder.build(kind: .reply, original: e, body: orphan, identity: identity, me: []).fullHTML?.contains("cid:") == false)147    }148}149150@Suite struct DraftFormattingTests {151    // Item 2 — formatting survives a save / reopen cycle.152    @Test func richTextRoundTripsThroughTheStoredHTML() throws {153        var bold = AttributedString("gras"); bold.inlinePresentationIntent = .stronglyEmphasized154        var both = AttributedString("les deux"); both.inlinePresentationIntent = [.stronglyEmphasized, .emphasized]155        var code = AttributedString("x < y"); code.inlinePresentationIntent = .code156        var struck = AttributedString("barré"); struck.inlinePresentationIntent = .strikethrough157        var link = AttributedString("lien & co"); link.link = URL(string: "https://example.org/?a=1&b=2")158        let typed = AttributedString("Bonjour,\nvoici du ") + bold + AttributedString(", ") + both + AttributedString(", ") + code + AttributedString(", ") + struck + AttributedString(" et un ") + link + AttributedString(".\n\n-- \nMoi")159        let html = HTMLText.html(fromAttributed: typed)160        let back = try #require(HTMLText.attributed(fromHTML: html))161        #expect(HTMLText.html(fromAttributed: back) == html)162        #expect(String(back.characters) == String(typed.characters))163        // what the composer must use instead of `AttributedString(message.text)`164        var m = OutgoingMessage(from: EmailAddress(email: "a@stg.spboucher.ai"), text: String(typed.characters), html: html)165        #expect(HTMLText.html(fromAttributed: m.editableBody) == html)166        m.html = nil167        #expect(m.editableBody == AttributedString(m.text))168        // HTML the composer cannot represent is never offered as editable rich text169        for h in ["<table><tr><td>x</td></tr></table>", "<p style=\"color:red\">x</p>", "<ul><li>a</li></ul>", "<img src=\"cid:a\">", "<div class=\"x\">y</div>"] { #expect(HTMLText.attributed(fromHTML: h) == nil, "\(h)") }170        #expect(HTMLText.attributed(fromHTML: "<p>un</p><p>deux<br>trois</p>").map { String($0.characters) } == "un\ndeux\ntrois")171    }172173    @Test func quotedBlockSplitsBackByteForByte() {174        let quote = OutgoingMessage.wrapQuote(#"<p>Le 1 janvier, A a écrit :</p><blockquote type="cite"><table><tr><td style="color:red">cellule</td></tr></table><img src="cid:logo@x"></blockquote>"#)175        let stored = "<div>Ma <b>réponse</b></div>" + quote176        let (top, quoted) = OutgoingMessage.splitQuoted(html: stored)177        #expect(top == "<div>Ma <b>réponse</b></div>" && quoted == quote && top + (quoted ?? "") == stored)178        #expect(OutgoingMessage.splitQuoted(html: "<div>seul</div>").quoted == nil)179        var m = OutgoingMessage(from: EmailAddress(email: "a@stg.spboucher.ai"))180        SyncEngine.fill(&m, text: "Ma réponse\n\nLe 1 janvier, A a écrit :\n> cellule", html: stored)181        #expect(m.fullHTML == stored) // lossless182        #expect(m.quotedHTML == quote && m.html == top && m.text == "Ma réponse" && m.quotedText == "Le 1 janvier, A a écrit :\n> cellule")183        #expect(m.editableBody.runs.contains { $0.inlinePresentationIntent == .stronglyEmphasized })184        // the composer edits the top as text; the quoted block stays untouched185        m.text = "Ma réponse corrigée"; m.html = HTMLText.html(fromAttributed: AttributedString(m.text))186        #expect(m.fullHTML == "<div>Ma réponse corrigée</div>" + quote)187        // a draft whose HTML the composer cannot represent is kept whole as the untouched block188        var foreign = OutgoingMessage(from: EmailAddress(email: "a@stg.spboucher.ai"))189        let rich = #"<table><tr><td>Offre</td></tr></table><p style="color:#333">Texte</p>"#190        SyncEngine.fill(&foreign, text: "Offre\nTexte", html: rich)191        #expect(foreign.fullHTML == rich && foreign.quotedHTML == rich && foreign.text.isEmpty && foreign.quotedText == "Offre\nTexte" && !foreign.isEmpty)192    }193194    @Test func messageIdAndDigest() {195        var m = OutgoingMessage(from: EmailAddress(name: "A", email: "a@stg.spboucher.ai"), to: [EmailAddress(email: "b@stg.spboucher.ai")], subject: "S", text: "T")196        #expect(m.jmapEmail(mailboxId: "d", draft: true)["messageId"].isNull) // a draft carries none197        let id = OutgoingMessage.makeMessageId(for: m.from)198        #expect(id.hasSuffix("@stg.spboucher.ai") && !id.contains("<") && id != OutgoingMessage.makeMessageId(for: m.from))199        let digest = m.contentDigest200        m.messageId = id201        m.attachments = []202        #expect(m.contentDigest == digest) // queueing does not change the content203        #expect(m.jmapEmail(mailboxId: "d", draft: true)["messageId"].stringArray == [id])204        m.text = "T2"205        #expect(m.contentDigest != digest)206        // an item written by 1.1.0 (no new field) still decodes207        let legacy = #"{"id":"L1","account":"A","message":{"from":{"email":"a@stg.spboucher.ai"},"to":[],"cc":[],"bcc":[],"subject":"Ancien","text":"t","attachments":[],"inReplyTo":[],"references":[]},"state":"failed","attempts":2,"createdAt":0,"updatedAt":0,"needsDraftSync":false,"delivery":{}}"#208        let old = try? JSONDecoder().decode(OutboxItem.self, from: Data(legacy.utf8))209        #expect(old?.message.subject == "Ancien" && old?.failureKind == nil && old?.isPermanentFailure == true && old?.nextAttemptAt == nil && old?.message.messageId == nil)210    }211212    @Test func pendingPatchesApplyLikeTheServerWould() {213        let e = sample("1", box: "in", date: 1, seen: false)214        var p = PendingPatch.apply(["keywords/$pinned": true, "keywords/lbl_1a2b3c4d": true, "keywords/$seen": true], to: e)215        #expect(p?.keywords == ["$pinned", "$seen", "lbl_1a2b3c4d"])216        p = PendingPatch.apply(["keywords/$pinned": nil, "mailboxIds": ["snz": true], "keywords/$snoozed": true, "keywords/$followup": true], to: p!)217        #expect(p?.keywords == ["$followup", "$seen", "$snoozed", "lbl_1a2b3c4d"] && p?.mailboxIds == ["snz"])218        p = PendingPatch.apply(["mailboxIds/in": true, "mailboxIds/snz": nil], to: p!)219        #expect(p?.mailboxIds == ["in"])220        #expect(PendingPatch.apply(["@destroy": true], to: e) == nil)221        #expect(FailureClass.of(APIError.offline) == .transport && FailureClass.of(APIError(status: 503, code: "x")) == .transient && FailureClass.of(APIError(status: 401, code: "x")) == .unauthorized)222        #expect(FailureClass.of(APIError(status: 400, code: "x")) == .rejected && FailureClass.ofJMAP(type: "invalidPatch") == .rejected && FailureClass.ofJMAP(type: "notFound") == .rejected)223        #expect(FailureClass.ofJMAP(type: "serverFail") == .transient && FailureClass.ofJMAP(type: "stateMismatch") == .rejected)224    }225226    // Item 17227    @Test func coreHoldsNoTranslatedFallbackName() {228        #expect(BodyPart(type: "application/pdf").displayName == "attachment")229        #expect(BodyPart(name: "rapport.pdf", type: "application/pdf").displayName == "rapport.pdf")230    }231}232233@Suite struct SchemaTests {234    /// The schema exactly as SP Mail 1.1.0 created it (no `user_version`).235    static let schema110 = """236    CREATE TABLE IF NOT EXISTS mailboxes (account TEXT NOT NULL, id TEXT NOT NULL, name TEXT NOT NULL, parent_id TEXT, role TEXT, sort_order INTEGER NOT NULL DEFAULT 0,237        total_emails INTEGER NOT NULL DEFAULT 0, unread_emails INTEGER NOT NULL DEFAULT 0, total_threads INTEGER NOT NULL DEFAULT 0, unread_threads INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (account, id));238    CREATE TABLE IF NOT EXISTS emails (account TEXT NOT NULL, id TEXT NOT NULL, thread_id TEXT NOT NULL, received_at REAL NOT NULL, is_seen INTEGER NOT NULL, is_flagged INTEGER NOT NULL,239        is_draft INTEGER NOT NULL, has_attachment INTEGER NOT NULL, search_text TEXT NOT NULL, meta TEXT NOT NULL, preview BLOB, PRIMARY KEY (account, id));240    CREATE INDEX IF NOT EXISTS emails_by_date ON emails (account, received_at DESC);241    CREATE INDEX IF NOT EXISTS emails_by_thread ON emails (account, thread_id);242    CREATE TABLE IF NOT EXISTS email_mailboxes (account TEXT NOT NULL, email_id TEXT NOT NULL, mailbox_id TEXT NOT NULL, PRIMARY KEY (account, mailbox_id, email_id));243    CREATE INDEX IF NOT EXISTS email_mailboxes_by_email ON email_mailboxes (account, email_id);244    CREATE TABLE IF NOT EXISTS bodies (account TEXT NOT NULL, email_id TEXT NOT NULL, sealed BLOB NOT NULL, PRIMARY KEY (account, email_id));245    CREATE TABLE IF NOT EXISTS sync_state (account TEXT NOT NULL, type TEXT NOT NULL, state TEXT NOT NULL, PRIMARY KEY (account, type));246    CREATE TABLE IF NOT EXISTS kv (key TEXT PRIMARY KEY, sealed BLOB NOT NULL);247    CREATE TABLE IF NOT EXISTS outbox (id TEXT PRIMARY KEY, account TEXT NOT NULL, state TEXT NOT NULL, updated_at REAL NOT NULL, sealed BLOB NOT NULL);248    CREATE TABLE IF NOT EXISTS outbox_blobs (id TEXT PRIMARY KEY, outbox_id TEXT NOT NULL, sealed BLOB NOT NULL);249    CREATE TABLE IF NOT EXISTS pending_ops (id INTEGER PRIMARY KEY AUTOINCREMENT, account TEXT NOT NULL, email_id TEXT NOT NULL, patch TEXT NOT NULL);250    CREATE TABLE IF NOT EXISTS correspondents (email TEXT PRIMARY KEY, name TEXT, uses INTEGER NOT NULL DEFAULT 1, last_used REAL NOT NULL);251    """252253    // Item 12 — a database created by the old schema, with data in every table, opens under the versioned schema without losing anything.254    @Test func migratesADatabaseWrittenBy110() async throws {255        let dir = FileManager.default.temporaryDirectory.appendingPathComponent("spmail-legacy-\(UUID().uuidString)")256        try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)257        let secrets = InMemorySecretStore()258        let cipher = try CacheCipher.load(from: secrets)259        do {260            let db = try SQLiteDatabase(path: dir.appendingPathComponent("mail.sqlite").path)261            try db.exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; PRAGMA secure_delete=ON; PRAGMA foreign_keys=OFF;")262            try db.exec(Self.schema110)263            #expect(db.userVersion == 0)264            var e = sample("1", date: 1000, subject: "Ancien message")265            e.preview = ""266            try db.run("INSERT INTO mailboxes (account,id,name,role) VALUES ('A','in','Inbox','inbox')")267            try db.run("INSERT INTO emails (account,id,thread_id,received_at,is_seen,is_flagged,is_draft,has_attachment,search_text,meta,preview) VALUES (?,?,?,?,?,?,?,?,?,?,?)",268                       [.text("A"), .text("1"), .text("t1"), .real(1000), .int(0), .int(0), .int(0), .int(0), .text("ancien message"), .text(String(decoding: try JSONEncoder().encode(e), as: UTF8.self)),269                        .blob(try cipher.seal("aperçu ancien", context: "A/1"))])270            try db.run("INSERT INTO email_mailboxes (account,email_id,mailbox_id) VALUES ('A','1','in')")271            try db.run("INSERT INTO bodies (account,email_id,sealed) VALUES (?,?,?)", [.text("A"), .text("1"), .blob(try cipher.sealJSON(EmailBody(text: "corps ancien"), context: "A/1"))])272            try db.run("INSERT INTO sync_state (account,type,state) VALUES ('A','Email','s42')")273            try db.run("INSERT INTO pending_ops (account,email_id,patch) VALUES ('A','1','{\"keywords/$flagged\":true}')")274            let legacy = #"{"id":"L1","account":"A","message":{"from":{"email":"a@stg.spboucher.ai"},"to":[{"email":"b@stg.spboucher.ai"}],"cc":[],"bcc":[],"subject":"File 1.1.0","text":"t","attachments":[],"inReplyTo":[],"references":[]},"state":"queuedOffline","attempts":1,"createdAt":0,"updatedAt":0,"needsDraftSync":false,"delivery":{}}"#275            try db.run("INSERT INTO outbox (id,account,state,updated_at,sealed) VALUES (?,?,?,?,?)", [.text("L1"), .text("A"), .text("queuedOffline"), .real(0), .blob(try cipher.seal(Data(legacy.utf8), context: "L1"))])276            try db.run("INSERT INTO correspondents (email,name,uses,last_used) VALUES ('bob@example.org','Bob',2,0)")277            db.close()278        }279        let store = try MailStore(directory: dir, secrets: secrets, temporaryFilesDirectory: dir.appendingPathComponent("tmp-files"))280        #expect(await store.schemaVersion() == MailStore.schemaVersion && MailStore.schemaVersion == 2)281        #expect(try await store.email("A", id: "1")?.subject == "Ancien message")282        #expect(try await store.email("A", id: "1")?.preview == "aperçu ancien")283        #expect(try await store.body("A", id: "1")?.text == "corps ancien")284        #expect(try await store.state("A", "Email") == "s42")285        let ops = try await store.pending("A")286        #expect(ops.count == 1 && ops[0].patch["keywords/$flagged"].boolValue == true && ops[0].attempts == 0 && ops[0].lastError == nil)287        let queued = try #require(try await store.outboxItem("L1"))288        #expect(queued.state == .queuedOffline && queued.message.subject == "File 1.1.0" && queued.failureKind == nil && queued.message.messageId == nil && queued.serverCopyIsDraft == nil)289        #expect(try await store.suggestCorrespondents("bob").count == 1)290        #expect(try await store.threads("A", scope: .mailbox("in")).map(\.email.id) == ["1"])291        // the tables of version 2 work on the migrated file292        #expect(try await store.folderSync("A", mailboxId: "in") == nil)293        try await store.setFolderSync("A", mailboxId: "in", FolderSyncState(anchorReceivedAt: Date(timeIntervalSince1970: 1000), anchorId: "1"))294        try await store.notePendingFailure(ids: ops.map(\.id), error: "serverFail")295        #expect(try await store.pending("A").first?.attempts == 1)296        try await store.cacheAttachment("A", blobId: "b", Data("x".utf8))297        #expect(try await store.cachedAttachment("A", blobId: "b") == Data("x".utf8))298        // reopening is a no-op (migrations do not run twice), and a database from the future is refused instead of being damaged299        let again = try MailStore(directory: dir, secrets: secrets, temporaryFilesDirectory: dir.appendingPathComponent("tmp-files"))300        #expect(await again.schemaVersion() == 2)301        #expect(try await again.pending("A").first?.attempts == 1)302        let future = FileManager.default.temporaryDirectory.appendingPathComponent("spmail-future-\(UUID().uuidString)")303        try FileManager.default.createDirectory(at: future, withIntermediateDirectories: true)304        do { let db = try SQLiteDatabase(path: future.appendingPathComponent("mail.sqlite").path); db.userVersion = 99; db.close() }305        #expect(throws: StoreError.self) { _ = try MailStore(directory: future, secrets: InMemorySecretStore()) }306    }307308    // Item 12 — the same on a GENUINE 1.1.0 cache: Resources/legacy-1.1.0/mail.sqlite was written by the unchanged 1.1.0 core (git 102facc,309    // its own MailStore API: three messages, a body, two pending operations, four outbox items, identities, a correspondent), key in key.bin.310    @Test func opensACacheFileWrittenByTheReal110Core() async throws {311        let fixture = try #require(Bundle.module.url(forResource: "mail", withExtension: "sqlite", subdirectory: "Resources/legacy-1.1.0"))312        let dir = FileManager.default.temporaryDirectory.appendingPathComponent("spmail-110-\(UUID().uuidString)")313        try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)314        try FileManager.default.copyItem(at: fixture, to: dir.appendingPathComponent("mail.sqlite")) // always a copy: the fixture stays at version 0315        let secrets = InMemorySecretStore()316        secrets.write(SecretKeys.cacheKey, try Data(contentsOf: fixture.deletingLastPathComponent().appendingPathComponent("key.bin")))317        do { let raw = try SQLiteDatabase(path: dir.appendingPathComponent("mail.sqlite").path); #expect(raw.userVersion == 0); raw.close() }318        let store = try MailStore(directory: dir, secrets: secrets, temporaryFilesDirectory: dir.appendingPathComponent("tmp-files"))319        #expect(await store.schemaVersion() == 2)320        // everything 1.1.0 wrote is still there and readable321        #expect(try await store.mailboxes("A").map(\.id) == ["in", "tr"])322        #expect(try await store.threads("A", scope: .mailbox("in")).map(\.email.id) == ["2", "3"])323        #expect(try await store.email("A", id: "1")?.preview == "aperçu un")324        #expect(try await store.body("A", id: "1")?.html == "<p>corps <b>un</b></p>")325        #expect(try await store.state("A", "Email") == "s-110")326        let ops = try await store.pending("A")327        #expect(ops.map(\.emailId) == ["2", "3"] && ops.allSatisfy { $0.attempts == 0 && $0.lastError == nil } && ops[1].patch["mailboxIds"]["tr"].boolValue == true)328        let items = Dictionary(try await store.outbox(account: "A").map { ($0.id, $0) }, uniquingKeysWith: { a, _ in a })329        #expect(Set(items.keys) == ["draft-110", "queued-110", "failed-110", "accepted-110"])330        #expect(items["draft-110"]?.serverEmailId == "srv-draft" && items["draft-110"]?.serverCopyIsDraft == nil && items["draft-110"]?.message.html == "<div>texte</div>")331        #expect(try await store.attachmentData("att-110") == Data([1, 2, 3, 4]))332        #expect(items["queued-110"]?.state == .queuedOffline && items["queued-110"]?.attempts == 2 && items["queued-110"]?.nextAttemptAt == nil && items["queued-110"]?.message.messageId == nil)333        #expect(items["failed-110"]?.isPermanentFailure == true && items["failed-110"]?.isRetryableFailure == false) // a 1.1.0 failure is never retried by itself334        #expect(items["accepted-110"]?.isFullyDelivered == true && items["accepted-110"]?.submissionId == "sub-1")335        #expect(try await store.value("identities/A", as: [Identity].self)?.first?.textSignature == "Signature")336        #expect(try await store.suggestCorrespondents("bob").map(\.email) == ["bob@example.org"])337        #expect(try await store.search("A", text: "deuxième").map(\.id) == ["2"])338        // the new tables are usable on it, and an engine runs on the migrated cache: the 1.1.0 queue is replayed and sent339        #expect(try await store.folderSync("A", mailboxId: "in") == nil)340        #expect(try await store.rejectedOperations("A").isEmpty)341        let server = FakeMailServer(address: "a@stg.spboucher.ai")342        let engine = server.engine(store)343        try await engine.syncNow() // stored state "s-110" is unknown to this server: full resync; the two 1.1.0 operations name messages it does not hold344        #expect(try await store.pending("A").isEmpty)345        #expect(try await store.rejectedOperations("A").count == 2)346        let after = Dictionary(try await store.outbox(account: "A").map { ($0.id, $0) }, uniquingKeysWith: { a, _ in a })347        #expect(after["queued-110"]?.state == .accepted && after["queued-110"]?.message.messageId != nil && after["failed-110"]?.state == .failed && after["accepted-110"]?.serverEmailId == "srv-sent")348        #expect(server.submissionCount == 1 && server.emails(in: "se").first?["subject"].stringValue == "En file 1.1.0")349        // reopening changes nothing350        let again = try MailStore(directory: dir, secrets: secrets, temporaryFilesDirectory: dir.appendingPathComponent("tmp-files"))351        #expect(await again.schemaVersion() == 2)352        #expect(try await again.outbox(account: "A").count == 4)353    }354355    @Test func freshDatabaseIsCreatedAtTheCurrentVersion() async throws {356        let (store, _, _) = try tempStore()357        #expect(await store.schemaVersion() == 2)358    }359360    @Test func preparedStatementsAreReusedAndLookupsBatched() async throws {361        let (store, _, _) = try tempStore()362        var batch: [EmailSummary] = []363        for n in 0..<1500 { batch.append(sample("m\(n)", date: Double(n))) }364        try await store.upsertEmails("A", batch)365        let before = await store.statementCacheStats()366        try await store.upsertEmails("A", batch) // 1500 × (insert + delete + insert) with three prepared statements367        for n in 0..<50 { _ = try await store.email("A", id: "m\(n)") }368        let after = await store.statementCacheStats()369        #expect(after.prepared - before.prepared <= 2, "statements prepared again: \(after.prepared - before.prepared)")370        #expect(after.cached > 0 && after.cached <= 160)371        // batched id lookups: one statement per 400 ids instead of one per id372        let ids = (0..<1500).map { "m\($0)" } + ["absent1", "absent2"]373        let p0 = await store.statementCacheStats().prepared374        #expect(try await store.knownIds("A", among: ids).count == 1500)375        #expect(try await store.emails("A", ids: ids).count == 1500)376        let p1 = await store.statementCacheStats().prepared377        #expect(p1 - p0 == 8, "\(p1 - p0) statements for two lookups of 1502 ids")378        try await store.saveBody("A", id: "m1", EmailBody(text: "x"))379        #expect(try await store.bodyIds("A", among: ids) == ["m1"])380        try await store.deleteEmails("A", ids: Array(ids.prefix(1400)))381        #expect(try await store.emailCount("A") == 100)382        #expect(try await store.body("A", id: "m1") == nil)383        try await store.addPending("A", emailIds: (0..<900).map { "m\($0)" }, patch: ["keywords/$seen": true])384        let ops = try await store.pending("A")385        #expect(ops.count == 900)386        #expect(try await store.pendingCount("A") == 900)387        try await store.removePending(ids: ops.map(\.id))388        #expect(try await store.pending("A").isEmpty)389    }390391    // Item 12 — the clear-text columns are a decision, and this test pins it: exactly these values are readable on disk, nothing else.392    @Test func clearTextColumnsAreExactlyTheDocumentedOnes() async throws {393        let (store, _, dir) = try tempStore()394        var e = sample("1", date: 1, subject: "SUJET-EN-CLAIR-77", preview: "APERCU-SCELLE-77")395        e.from = [EmailAddress(name: "NOM-EN-CLAIR-77", email: "adresse-en-clair-77@example.org")]396        try await store.upsertEmails("A", [e])397        try await store.saveBody("A", id: "1", EmailBody(text: "CORPS-SCELLE-77"))398        try await store.cacheAttachment("A", blobId: "blob-1", Data("PIECE-SCELLEE-77 contenu".utf8))399        try await store.checkpoint()400        // offline search works BECAUSE these are in clear…401        #expect(try await store.search("A", text: "sujet-en-clair").map(\.id) == ["1"])402        #expect(try await store.search("A", text: "nom-en-clair").count == 1)403        for clear in ["SUJET-EN-CLAIR-77", "sujet-en-clair-77", "NOM-EN-CLAIR-77", "adresse-en-clair-77@example.org"] { #expect(fileContains(dir, clear), "\(clear) should be searchable in clear") }404        // …and content never is405        for sealed in ["APERCU-SCELLE-77", "CORPS-SCELLE-77", "PIECE-SCELLEE-77"] {406            #expect(!fileContains(dir, sealed) && !fileContains(store.attachmentCacheDirectory, sealed), "\(sealed) is readable on disk")407        }408        let files = try FileManager.default.contentsOfDirectory(atPath: store.attachmentCacheDirectory.path)409        #expect(files.count == 1 && !files[0].contains("blob") && UUID(uuidString: files[0]) != nil) // the file name says nothing410    }411412    // Item 13 — sealed attachment cache: least recently used first out, size budget, purged by every wipe.413    @Test func attachmentCacheIsSealedBudgetedAndPurged() async throws {414        let (store, _, dir) = try tempStore()415        func blob(_ n: Int) -> Data { Data(repeating: UInt8(n), count: 1000) }416        try await store.setAttachmentCacheBudget(3500) // three sealed files of ~1028 bytes fit, a fourth does not417        for n in 1...3 { try await store.cacheAttachment("A", blobId: "b\(n)", blob(n)) }418        #expect(try await store.cachedAttachment("A", blobId: "b1") == blob(1)) // b1 becomes the most recently used419        try await store.cacheAttachment("A", blobId: "b4", blob(4))420        #expect(try await store.cachedAttachment("A", blobId: "b2") == nil) // the least recently used went421        #expect(try await store.cachedAttachment("A", blobId: "b1") == blob(1))422        #expect(try await store.cachedAttachment("A", blobId: "b4") == blob(4))423        #expect(try await store.attachmentCacheSize() <= 3500)424        #expect(try FileManager.default.contentsOfDirectory(atPath: store.attachmentCacheDirectory.path).count == 3)425        try await store.cacheAttachment("A", blobId: "huge", Data(count: 5000)) // larger than the whole budget: not cached426        #expect(try await store.cachedAttachment("A", blobId: "huge") == nil)427        try await store.cacheAttachment("B", blobId: "b1", blob(9)) // same blob id, other account: separate entries428        #expect(try await store.cachedAttachment("B", blobId: "b1") == blob(9))429        #expect(try await store.cachedAttachment("A", blobId: "b1") == blob(1))430        // another install key cannot open the files431        let other = try MailStore(directory: dir, secrets: InMemorySecretStore(), temporaryFilesDirectory: dir.appendingPathComponent("tmp-files"))432        #expect(try await other.cachedAttachment("A", blobId: "b1") == nil)433        try await store.cacheAttachment("A", blobId: "b5", blob(5))434        // decrypted temporary files and the cache both go with the 401 wipe435        let tmp = try store.writeTemporaryFile(named: "secret/../x.pdf", data: Data("CLAIR".utf8))436        #expect(FileManager.default.fileExists(atPath: tmp.path) && tmp.path.hasPrefix(store.temporaryFilesDirectory.path) && tmp.lastPathComponent == "secret-..-x.pdf")437        try await store.wipeKeepingOutbox()438        #expect(!FileManager.default.fileExists(atPath: tmp.path))439        #expect(try FileManager.default.contentsOfDirectory(atPath: store.attachmentCacheDirectory.path).isEmpty)440        #expect(try await store.attachmentCacheSize() == 0)441    }442443    @Test func horizonFiltersSparseOldRows() async throws {444        let (store, _, _) = try tempStore()445        try await store.upsertMailboxes("A", [Mailbox(id: "in", name: "Inbox", role: "inbox")])446        try await store.upsertEmails("A", [sample("new1", date: 500), sample("new2", date: 400), sample("old-sparse", date: 10)])447        #expect(try await store.threads("A", scope: .mailbox("in"), withinHorizon: true).count == 3) // no horizon known: nothing hidden448        try await store.setFolderSync("A", mailboxId: "in", FolderSyncState(anchorReceivedAt: Date(timeIntervalSince1970: 400), anchorId: "new2"))449        #expect(try await store.threads("A", scope: .mailbox("in"), withinHorizon: true).map(\.email.id) == ["new1", "new2"])450        #expect(try await store.threads("A", scope: .mailbox("in"), threaded: false, withinHorizon: true).map(\.email.id) == ["new1", "new2"])451        #expect(try await store.threads("A", scope: .mailbox("in")).count == 3) // default unchanged452        try await store.setFolderSync("A", mailboxId: "in", FolderSyncState(anchorReceivedAt: Date(timeIntervalSince1970: 10), anchorId: "old-sparse", isComplete: true))453        #expect(try await store.threads("A", scope: .mailbox("in"), withinHorizon: true).count == 3)454    }455}456
Files main
Menu

Theme