SPB Git

Repository Public

SearchTests.swift 16.3 KB 238 lines swift

1// SP Notes Core2// File: Tests/SPNotesStoreTests/SearchTests.swift3// Description: Full-text search (accents, prefix, hostile input, ranking, snippets, filters, incremental4//              index maintenance) and the performance sanity check on 5,000 notes.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import Testing10@testable import SPNotesStore1112@Suite struct SearchTests {13    private func ids(_ store: NotesStore, _ query: String, _ filters: SearchFilters = SearchFilters()) async throws -> [String] {14        try await store.search(query: query, filters: filters).map(\.id)15    }1617    @Test func accentsCaseAndPrefix() async throws {18        let store = try NotesStore.inMemory()19        try await store.createPages([20            draft("Été à Québec", "Les vacances d'été commencent bientôt près du fleuve.", id: "ete"),21            draft("Winter", "Nothing about summer here, only étendue and naïve façade.", id: "winter"),22            draft("Programmation", "Swift concurrency and actors", id: "prog", tags: ["développement"]),23        ])24        #expect(try await ids(store, "éte").first == "ete") // "éte" finds "Été"25        #expect(Set(try await ids(store, "éte")) == ["ete", "winter"]) // prefix: also "étendue"26        #expect(try await ids(store, "ete ") == ["ete", "winter"])27        #expect(try await ids(store, "ETE quebec") == ["ete"])28        #expect(try await ids(store, "QUÉBEC") == ["ete"])29        #expect(try await ids(store, "naive facade") == ["winter"])30        #expect(try await ids(store, "prog") == ["prog"]) // prefix on the last term31        #expect(try await ids(store, "conc act") == []) // only the LAST term is a prefix32        #expect(try await ids(store, "concurrency act") == ["prog"])33        #expect(try await ids(store, "developpement") == ["prog"]) // tags are indexed, accents folded34        #expect(try await ids(store, "vacances fleuve") == ["ete"]) // all terms, any order35        #expect(try await ids(store, "vacances hiver") == [])36    }3738    @Test func hostileInputNeverFails() async throws {39        let store = try NotesStore.inMemory()40        try await store.createPages([draft("foo bar", "foo OR baz near something", id: "a"), draft("other", "nothing", id: "b")])41        let hostile = [42            "\"foo OR (", "foo OR (", "\"", "\"\"", "'", "(", ")", "((()))", "*", "foo*", "*foo", "foo AND", "AND", "OR", "NOT", "NOT foo",43            "NEAR(foo bar, 2)", "NEAR(", "title:foo", "title : foo", "{title body}: foo", "foo:bar", "^foo", "foo ^ bar", "-foo", "+foo",44            "foo -bar", "a\"b\"c", "foo\\", "\\", "foo;DROP TABLE pages;--", "' OR 1=1 --", "%", "_", "foo\u{0}bar", "\u{0}", "😀", "😀 foo",45            "foo", "foo\n\tbar", "   ", "", "...", "c++", "#tag", "@user", "x:y:z", "rank", "bm25(", "pages_fts", "\u{E000}foo\u{E001}",46            String(repeating: "\"", count: 501), String(repeating: "(foo ", count: 200), String(repeating: "a ", count: 500),47        ]48        for input in hostile {49            let hits = try await store.search(query: input)50            #expect(hits.count <= 2, "input: \(input)")51            // With filters and a folder subtree too.52            _ = try await store.search(query: input, filters: SearchFilters(section: "notes", folderId: "a", tags: [input], favoritesOnly: true))53        }54        #expect(try await ids(store, "\"foo OR (") == ["a"]) // operators are plain words: "foo" AND "or"*55        #expect(try await ids(store, "foo AND") == []) // the document has no word starting with "and"56        #expect(try await ids(store, "NEAR(foo bar, 2)") == [])57        #expect(try await ids(store, "foo -bar") == ["a"]) // not an exclusion: both words are required58        #expect(try await ids(store, "title:other") == []) // not a column filter59        #expect(Set(try await ids(store, "*")) == ["a", "b"]) // no term = listing60        #expect(try await store.pageCount() == 2)61        #expect(try await store.integrityCheck().isEmpty)62        #expect(NotesStore.ftsQuery("say \"hi\" (now)") == "\"say\" \"hi\" \"now\"*")63        #expect(NotesStore.ftsQuery(" \"\" () ") == nil)64    }6566    @Test func rankingPutsTitleFirstThenTagsThenBody() async throws {67        let store = try NotesStore.inMemory()68        let filler = String(repeating: "lorem ipsum dolor sit amet ", count: 20)69        try await store.createPages([70            draft("Notes diverses", filler + "budget " + filler, id: "body"),71            draft("Autre chose", filler, id: "tag", tags: ["budget"]),72            draft("Budget 2027", filler, id: "title"),73            draft("Sans rapport", filler, id: "none"),74        ])75        let hits = try await store.search(query: "budget")76        #expect(hits.map(\.id) == ["title", "tag", "body"])77        #expect(hits[0].score < hits[1].score && hits[1].score < hits[2].score) // bm25: lower is better78        // Highlighting.79        #expect(SearchHit.plain(hits[0].highlightedTitle) == "Budget 2027")80        let titleSegments = SearchHit.segments(of: hits[0].highlightedTitle)81        #expect(titleSegments.map(\.text) == ["Budget", " 2027"] && titleSegments.map(\.highlighted) == [true, false])82        let snippet = hits[2].snippet83        #expect(SearchHit.segments(of: snippet).contains { $0.highlighted && $0.text == "budget" })84        #expect(snippet.count < filler.count && snippet.contains("…"))85        #expect(try await store.search(query: "budget", limit: 1).map(\.id) == ["title"])86        #expect(try await store.search(query: "budget", limit: 0).isEmpty)87    }8889    @Test func filtersCombine() async throws {90        let store = try NotesStore.inMemory()91        let t = { (days: Double) in epoch.addingTimeInterval(days * 86_400) }92        try await store.applyRemote(pages: [93            remote("f", "Dossier projets", "", section: "projects", properties: #"{"folder":true}"#, updatedAt: t(0)),94            remote("f1", "Sous-dossier", "", parent: "f", section: "projects", properties: #"{"folder":true}"#, updatedAt: t(0)),95            remote("n1", "Rapport alpha", "analyse du marché", parent: "f", section: "projects", tags: ["work", "q1"], properties: #"{"favorite":true}"#, pinned: true, updatedAt: t(1)),96            remote("n2", "Rapport beta", "analyse des coûts", parent: "f1", section: "projects", tags: ["work"], updatedAt: t(2)),97            remote("n3", "Rapport gamma", "analyse personnelle", section: "notes", tags: ["home", "q1"], updatedAt: t(3)),98            remote("n4", "Rapport archivé", "analyse ancienne", section: "notes", tags: ["work"], archived: true, updatedAt: t(4)),99            remote("n5", "Rapport supprimé", "analyse jetée", parent: "f", section: "projects", tags: ["work"], updatedAt: t(5), deletedAt: t(6)),100        ])101        #expect(Set(try await ids(store, "analyse")) == ["n1", "n2", "n3"]) // trash and archive excluded by default102        #expect(Set(try await ids(store, "analyse", SearchFilters(includeArchived: true))) == ["n1", "n2", "n3", "n4"])103        #expect(Set(try await ids(store, "analyse", SearchFilters(includeTrash: true))) == ["n1", "n2", "n3", "n5"])104        #expect(Set(try await ids(store, "analyse", SearchFilters(includeArchived: true, includeTrash: true))).count == 5)105        #expect(Set(try await ids(store, "analyse", SearchFilters(section: "projects"))) == ["n1", "n2"])106        #expect(Set(try await ids(store, "analyse", SearchFilters(folderId: "f"))) == ["n1", "n2"]) // whole subtree107        #expect(try await ids(store, "analyse", SearchFilters(folderId: "f1")) == ["n2"])108        #expect(Set(try await ids(store, "analyse", SearchFilters(folderId: "f", includeTrash: true))) == ["n1", "n2", "n5"])109        #expect(Set(try await ids(store, "analyse", SearchFilters(tags: ["work"]))) == ["n1", "n2"])110        #expect(try await ids(store, "analyse", SearchFilters(tags: ["#Work", "Q1"])) == ["n1"]) // all of111        #expect(try await ids(store, "analyse", SearchFilters(tags: ["work", "home"])) == [])112        #expect(Set(try await ids(store, "analyse", SearchFilters(updatedAfter: t(2)))) == ["n2", "n3"])113        #expect(try await ids(store, "analyse", SearchFilters(updatedAfter: t(2), updatedBefore: t(2))) == ["n2"])114        #expect(try await ids(store, "analyse", SearchFilters(favoritesOnly: true)) == ["n1"])115        #expect(try await ids(store, "analyse", SearchFilters(pinnedOnly: true)) == ["n1"])116        #expect(try await ids(store, "analyse", SearchFilters(section: "projects", folderId: "f", tags: ["work"], updatedAfter: t(2), updatedBefore: t(9))) == ["n2"])117        #expect(try await ids(store, "analyse", SearchFilters(section: "notes", folderId: "f")) == [])118119        // Empty query + filters = filtered listing, newest first.120        #expect(try await ids(store, "") == ["n3", "n2", "n1", "f1", "f"])121        #expect(try await ids(store, "  ", SearchFilters(tags: ["q1"])) == ["n3", "n1"])122        #expect(try await ids(store, "", SearchFilters(folderId: "f", includeTrash: true)) == ["n5", "n2", "n1", "f1"])123        #expect(try await ids(store, "", SearchFilters(section: "notes", includeArchived: true)) == ["n4", "n3"])124        let listing = try await store.search(query: "", filters: SearchFilters(favoritesOnly: true))125        #expect(listing.map(\.snippet) == ["analyse du marché"] && listing[0].score == 0 && listing[0].highlightedTitle == "Rapport alpha")126        #expect(try await store.search(query: "", limit: 2).count == 2)127    }128129    @Test func indexFollowsEveryChange() async throws {130        let store = try NotesStore.inMemory()131        let page = try await store.createPage(draft("Recette", "tarte aux pommes"))132        #expect(try await ids(store, "pommes") == [page.id])133        // Edit: the old words are gone, the new ones are found.134        try await store.edit(page.id, title: "Recette de saison", text: "tarte aux poires")135        #expect(try await ids(store, "pommes") == [])136        #expect(try await ids(store, "poires") == [page.id])137        #expect(try await ids(store, "saison") == [page.id])138        // Tags.139        #expect(try await ids(store, "dessert") == [])140        try await store.setTags(id: page.id, names: ["dessert"])141        #expect(try await ids(store, "dessert") == [page.id])142        try await store.setTags(id: page.id, names: ["cuisine"])143        #expect(try await ids(store, "dessert") == [])144        #expect(try await ids(store, "cuisine") == [page.id])145        // Trash and restore.146        try await store.trash(id: page.id)147        #expect(try await ids(store, "poires") == [])148        #expect(try await ids(store, "poires", SearchFilters(includeTrash: true)) == [page.id])149        try await store.restore(id: page.id)150        #expect(try await ids(store, "poires") == [page.id])151        // Archive.152        try await store.setArchived(id: page.id, true)153        #expect(try await ids(store, "poires") == [])154        #expect(try await ids(store, "poires", SearchFilters(includeArchived: true)) == [page.id])155        try await store.setArchived(id: page.id, false)156        // Remote overwrite of a clean page, then purge.157        try await store.markPushed(id: page.id, serverVersion: 1, updatedAt: epoch, pushedEditSeq: try await store.must(page.id).editSeq)158        try await store.applyRemote(pages: [remote(page.id, "Recette", "clafoutis aux cerises", version: 2, rev: 2)])159        #expect(try await ids(store, "poires") == [])160        #expect(try await ids(store, "cerises") == [page.id])161        // Exactly one index row per page, whatever happened; a rebuild gives the same answers.162        #expect(try await store.debugValue("SELECT count(*) FROM pages_fts") == "1")163        try await store.rebuildSearchIndex()164        #expect(try await ids(store, "cerises") == [page.id])165        #expect(try await ids(store, "poires") == [])166        try await store.purge(id: page.id)167        #expect(try await ids(store, "cerises", SearchFilters(includeArchived: true, includeTrash: true)) == [])168        #expect(try await store.debugValue("SELECT count(*) FROM pages_fts") == "0")169        #expect(try await store.debugValue("INSERT INTO pages_fts(pages_fts) VALUES('integrity-check')") == nil)170    }171172    @Test func performanceWithFiveThousandNotes() async throws {173        let temp = TempDatabase()174        let store = try NotesStore(url: temp.url)175        let vocabulary = ["marché", "analyse", "budget", "réunion", "projet", "client", "rapport", "idée", "été", "hiver", "donnée", "modèle",176                          "valeur", "prix", "maison", "terrain", "revenu", "coût", "méthode", "comparaison", "évaluation", "immeuble",177                          "quartier", "ville", "région", "taux", "prêt", "banque", "notaire", "contrat"]178        var generator = SystemRandomNumberGenerator()179        var drafts: [NewPage] = []180        drafts.reserveCapacity(5_000)181        for index in 0..<5_000 {182            var text = ""183            while text.utf8.count < 2_000 { text += vocabulary.randomElement(using: &generator)! + " " }184            if index % 500 == 0 { text += "aiguille" }185            drafts.append(draft("Note \(index) \(vocabulary[index % vocabulary.count])", text, tags: ["t\(index % 25)"]))186        }187        let clock = ContinuousClock()188        func milliseconds(_ duration: Duration) -> Double { Double(duration.components.attoseconds) / 1e15 + Double(duration.components.seconds) * 1e3 }189        func best<T: Equatable>(_ expected: T, _ body: () async throws -> T) async throws -> Double {190            var result = Double.infinity191            for _ in 0..<5 {192                let start = clock.now193                let value = try await body()194                result = min(result, milliseconds(clock.now - start))195                #expect(value == expected)196            }197            return result198        }199200        let insertStart = clock.now201        let ids = try await store.createPages(drafts)202        let insert = milliseconds(clock.now - insertStart)203        #expect(ids.count == 5_000)204        #expect(try await store.pageCount() == 5_000)205206        let list = try await best(100) { try await store.summaries(list: .all, limit: 100).count }207        let recent = try await best(100) { try await store.summaries(list: .recent, limit: 100).count }208        let counts = try await best(5_000) { try await store.counts().all }209        let common = try await best(50) { try await store.search(query: "marché analyse", limit: 50).count }210        let rare = try await best(10) { try await store.search(query: "aiguil", limit: 50).count }211        let filtered = try await best(50) { try await store.search(query: "budget", filters: SearchFilters(tags: ["t3"]), limit: 50).count }212        let byTag = try await best(100) { try await store.summaries(tag: "t7", limit: 100).count }213        let one = try await best(ids[2_500]) { try await store.page(id: ids[2_500])?.id }214        let editStart = clock.now215        try await store.edit(ids[10], text: "une modification unique zanzibar")216        let edit = milliseconds(clock.now - editStart)217        #expect(try await store.search(query: "zanzibar").map(\.id) == [ids[10]])218219        print(String(format: """220        [SPNotesStore performance, 5,000 notes of ~2 KB] batch insert %.0f ms | list(all, 100) %.2f ms | list(recent, 100) %.2f ms | \221        counts %.2f ms | search common (2 terms, 50) %.2f ms | search rare prefix %.2f ms | search + tag filter %.2f ms | by tag %.2f ms | \222        single page %.2f ms | one edit (durable commit + reindex) %.2f ms223        """, insert, list, recent, counts, common, rare, filtered, byTag, one, edit))224        for sql in ["SELECT p.id FROM pages p WHERE p.deleted_at IS NULL AND p.archived=0 ORDER BY p.updated_at DESC, p.rid DESC LIMIT 100",225                    "SELECT p.id FROM pages p WHERE p.deleted_at IS NULL AND p.archived=0 ORDER BY max(p.updated_at, coalesce(p.last_opened_at, 0)) DESC, p.rid DESC LIMIT 100",226                    "SELECT p.id FROM pages p WHERE p.pinned=1 AND p.deleted_at IS NULL AND p.archived=0 ORDER BY p.updated_at DESC, p.rid DESC LIMIT 100",227                    "SELECT p.id FROM pages p WHERE p.deleted_at IS NOT NULL ORDER BY p.deleted_at DESC, p.rid DESC LIMIT 100"] {228            // Lists are served straight from an index: no full scan, no sort.229            let plan = try await store.queryPlan(sql)230            #expect(plan.contains("USING INDEX") && !plan.contains("TEMP B-TREE"), "plan: \(plan)")231        }232        #expect(list < 50 && recent < 50 && counts < 50)233        #expect(common < 50 && rare < 50 && filtered < 50 && byTag < 50)234        #expect(insert < 15_000)235        #expect(try await store.integrityCheck().isEmpty)236    }237}238
Files main
Menu

Theme