Repository Public
1// SP Notes Core2// File: Tests/SPNotesStoreTests/SearchTests.swift3// Description: Full-text search (accents, prefix, hostile input, ranking, snippets, filters, incremental4// index maintenance) and the performance sanity check on 5,000 notes.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import Testing10@testable import SPNotesStore1112@Suite struct SearchTests {13 private func ids(_ store: NotesStore, _ query: String, _ filters: SearchFilters = SearchFilters()) async throws -> [String] {14 try await store.search(query: query, filters: filters).map(\.id)15 }1617 @Test func accentsCaseAndPrefix() async throws {18 let store = try NotesStore.inMemory()19 try await store.createPages([20 draft("Été à Québec", "Les vacances d'été commencent bientôt près du fleuve.", id: "ete"),21 draft("Winter", "Nothing about summer here, only étendue and naïve façade.", id: "winter"),22 draft("Programmation", "Swift concurrency and actors", id: "prog", tags: ["développement"]),23 ])24 #expect(try await ids(store, "éte").first == "ete") // "éte" finds "Été"25 #expect(Set(try await ids(store, "éte")) == ["ete", "winter"]) // prefix: also "étendue"26 #expect(try await ids(store, "ete ") == ["ete", "winter"])27 #expect(try await ids(store, "ETE quebec") == ["ete"])28 #expect(try await ids(store, "QUÉBEC") == ["ete"])29 #expect(try await ids(store, "naive facade") == ["winter"])30 #expect(try await ids(store, "prog") == ["prog"]) // prefix on the last term31 #expect(try await ids(store, "conc act") == []) // only the LAST term is a prefix32 #expect(try await ids(store, "concurrency act") == ["prog"])33 #expect(try await ids(store, "developpement") == ["prog"]) // tags are indexed, accents folded34 #expect(try await ids(store, "vacances fleuve") == ["ete"]) // all terms, any order35 #expect(try await ids(store, "vacances hiver") == [])36 }3738 @Test func hostileInputNeverFails() async throws {39 let store = try NotesStore.inMemory()40 try await store.createPages([draft("foo bar", "foo OR baz near something", id: "a"), draft("other", "nothing", id: "b")])41 let hostile = [42 "\"foo OR (", "foo OR (", "\"", "\"\"", "'", "(", ")", "((()))", "*", "foo*", "*foo", "foo AND", "AND", "OR", "NOT", "NOT foo",43 "NEAR(foo bar, 2)", "NEAR(", "title:foo", "title : foo", "{title body}: foo", "foo:bar", "^foo", "foo ^ bar", "-foo", "+foo",44 "foo -bar", "a\"b\"c", "foo\\", "\\", "foo;DROP TABLE pages;--", "' OR 1=1 --", "%", "_", "foo\u{0}bar", "\u{0}", "😀", "😀 foo",45 "foo", "foo\n\tbar", " ", "", "...", "c++", "#tag", "@user", "x:y:z", "rank", "bm25(", "pages_fts", "\u{E000}foo\u{E001}",46 String(repeating: "\"", count: 501), String(repeating: "(foo ", count: 200), String(repeating: "a ", count: 500),47 ]48 for input in hostile {49 let hits = try await store.search(query: input)50 #expect(hits.count <= 2, "input: \(input)")51 // With filters and a folder subtree too.52 _ = try await store.search(query: input, filters: SearchFilters(section: "notes", folderId: "a", tags: [input], favoritesOnly: true))53 }54 #expect(try await ids(store, "\"foo OR (") == ["a"]) // operators are plain words: "foo" AND "or"*55 #expect(try await ids(store, "foo AND") == []) // the document has no word starting with "and"56 #expect(try await ids(store, "NEAR(foo bar, 2)") == [])57 #expect(try await ids(store, "foo -bar") == ["a"]) // not an exclusion: both words are required58 #expect(try await ids(store, "title:other") == []) // not a column filter59 #expect(Set(try await ids(store, "*")) == ["a", "b"]) // no term = listing60 #expect(try await store.pageCount() == 2)61 #expect(try await store.integrityCheck().isEmpty)62 #expect(NotesStore.ftsQuery("say \"hi\" (now)") == "\"say\" \"hi\" \"now\"*")63 #expect(NotesStore.ftsQuery(" \"\" () ") == nil)64 }6566 @Test func rankingPutsTitleFirstThenTagsThenBody() async throws {67 let store = try NotesStore.inMemory()68 let filler = String(repeating: "lorem ipsum dolor sit amet ", count: 20)69 try await store.createPages([70 draft("Notes diverses", filler + "budget " + filler, id: "body"),71 draft("Autre chose", filler, id: "tag", tags: ["budget"]),72 draft("Budget 2027", filler, id: "title"),73 draft("Sans rapport", filler, id: "none"),74 ])75 let hits = try await store.search(query: "budget")76 #expect(hits.map(\.id) == ["title", "tag", "body"])77 #expect(hits[0].score < hits[1].score && hits[1].score < hits[2].score) // bm25: lower is better78 // Highlighting.79 #expect(SearchHit.plain(hits[0].highlightedTitle) == "Budget 2027")80 let titleSegments = SearchHit.segments(of: hits[0].highlightedTitle)81 #expect(titleSegments.map(\.text) == ["Budget", " 2027"] && titleSegments.map(\.highlighted) == [true, false])82 let snippet = hits[2].snippet83 #expect(SearchHit.segments(of: snippet).contains { $0.highlighted && $0.text == "budget" })84 #expect(snippet.count < filler.count && snippet.contains("…"))85 #expect(try await store.search(query: "budget", limit: 1).map(\.id) == ["title"])86 #expect(try await store.search(query: "budget", limit: 0).isEmpty)87 }8889 @Test func filtersCombine() async throws {90 let store = try NotesStore.inMemory()91 let t = { (days: Double) in epoch.addingTimeInterval(days * 86_400) }92 try await store.applyRemote(pages: [93 remote("f", "Dossier projets", "", section: "projects", properties: #"{"folder":true}"#, updatedAt: t(0)),94 remote("f1", "Sous-dossier", "", parent: "f", section: "projects", properties: #"{"folder":true}"#, updatedAt: t(0)),95 remote("n1", "Rapport alpha", "analyse du marché", parent: "f", section: "projects", tags: ["work", "q1"], properties: #"{"favorite":true}"#, pinned: true, updatedAt: t(1)),96 remote("n2", "Rapport beta", "analyse des coûts", parent: "f1", section: "projects", tags: ["work"], updatedAt: t(2)),97 remote("n3", "Rapport gamma", "analyse personnelle", section: "notes", tags: ["home", "q1"], updatedAt: t(3)),98 remote("n4", "Rapport archivé", "analyse ancienne", section: "notes", tags: ["work"], archived: true, updatedAt: t(4)),99 remote("n5", "Rapport supprimé", "analyse jetée", parent: "f", section: "projects", tags: ["work"], updatedAt: t(5), deletedAt: t(6)),100 ])101 #expect(Set(try await ids(store, "analyse")) == ["n1", "n2", "n3"]) // trash and archive excluded by default102 #expect(Set(try await ids(store, "analyse", SearchFilters(includeArchived: true))) == ["n1", "n2", "n3", "n4"])103 #expect(Set(try await ids(store, "analyse", SearchFilters(includeTrash: true))) == ["n1", "n2", "n3", "n5"])104 #expect(Set(try await ids(store, "analyse", SearchFilters(includeArchived: true, includeTrash: true))).count == 5)105 #expect(Set(try await ids(store, "analyse", SearchFilters(section: "projects"))) == ["n1", "n2"])106 #expect(Set(try await ids(store, "analyse", SearchFilters(folderId: "f"))) == ["n1", "n2"]) // whole subtree107 #expect(try await ids(store, "analyse", SearchFilters(folderId: "f1")) == ["n2"])108 #expect(Set(try await ids(store, "analyse", SearchFilters(folderId: "f", includeTrash: true))) == ["n1", "n2", "n5"])109 #expect(Set(try await ids(store, "analyse", SearchFilters(tags: ["work"]))) == ["n1", "n2"])110 #expect(try await ids(store, "analyse", SearchFilters(tags: ["#Work", "Q1"])) == ["n1"]) // all of111 #expect(try await ids(store, "analyse", SearchFilters(tags: ["work", "home"])) == [])112 #expect(Set(try await ids(store, "analyse", SearchFilters(updatedAfter: t(2)))) == ["n2", "n3"])113 #expect(try await ids(store, "analyse", SearchFilters(updatedAfter: t(2), updatedBefore: t(2))) == ["n2"])114 #expect(try await ids(store, "analyse", SearchFilters(favoritesOnly: true)) == ["n1"])115 #expect(try await ids(store, "analyse", SearchFilters(pinnedOnly: true)) == ["n1"])116 #expect(try await ids(store, "analyse", SearchFilters(section: "projects", folderId: "f", tags: ["work"], updatedAfter: t(2), updatedBefore: t(9))) == ["n2"])117 #expect(try await ids(store, "analyse", SearchFilters(section: "notes", folderId: "f")) == [])118119 // Empty query + filters = filtered listing, newest first.120 #expect(try await ids(store, "") == ["n3", "n2", "n1", "f1", "f"])121 #expect(try await ids(store, " ", SearchFilters(tags: ["q1"])) == ["n3", "n1"])122 #expect(try await ids(store, "", SearchFilters(folderId: "f", includeTrash: true)) == ["n5", "n2", "n1", "f1"])123 #expect(try await ids(store, "", SearchFilters(section: "notes", includeArchived: true)) == ["n4", "n3"])124 let listing = try await store.search(query: "", filters: SearchFilters(favoritesOnly: true))125 #expect(listing.map(\.snippet) == ["analyse du marché"] && listing[0].score == 0 && listing[0].highlightedTitle == "Rapport alpha")126 #expect(try await store.search(query: "", limit: 2).count == 2)127 }128129 @Test func indexFollowsEveryChange() async throws {130 let store = try NotesStore.inMemory()131 let page = try await store.createPage(draft("Recette", "tarte aux pommes"))132 #expect(try await ids(store, "pommes") == [page.id])133 // Edit: the old words are gone, the new ones are found.134 try await store.edit(page.id, title: "Recette de saison", text: "tarte aux poires")135 #expect(try await ids(store, "pommes") == [])136 #expect(try await ids(store, "poires") == [page.id])137 #expect(try await ids(store, "saison") == [page.id])138 // Tags.139 #expect(try await ids(store, "dessert") == [])140 try await store.setTags(id: page.id, names: ["dessert"])141 #expect(try await ids(store, "dessert") == [page.id])142 try await store.setTags(id: page.id, names: ["cuisine"])143 #expect(try await ids(store, "dessert") == [])144 #expect(try await ids(store, "cuisine") == [page.id])145 // Trash and restore.146 try await store.trash(id: page.id)147 #expect(try await ids(store, "poires") == [])148 #expect(try await ids(store, "poires", SearchFilters(includeTrash: true)) == [page.id])149 try await store.restore(id: page.id)150 #expect(try await ids(store, "poires") == [page.id])151 // Archive.152 try await store.setArchived(id: page.id, true)153 #expect(try await ids(store, "poires") == [])154 #expect(try await ids(store, "poires", SearchFilters(includeArchived: true)) == [page.id])155 try await store.setArchived(id: page.id, false)156 // Remote overwrite of a clean page, then purge.157 try await store.markPushed(id: page.id, serverVersion: 1, updatedAt: epoch, pushedEditSeq: try await store.must(page.id).editSeq)158 try await store.applyRemote(pages: [remote(page.id, "Recette", "clafoutis aux cerises", version: 2, rev: 2)])159 #expect(try await ids(store, "poires") == [])160 #expect(try await ids(store, "cerises") == [page.id])161 // Exactly one index row per page, whatever happened; a rebuild gives the same answers.162 #expect(try await store.debugValue("SELECT count(*) FROM pages_fts") == "1")163 try await store.rebuildSearchIndex()164 #expect(try await ids(store, "cerises") == [page.id])165 #expect(try await ids(store, "poires") == [])166 try await store.purge(id: page.id)167 #expect(try await ids(store, "cerises", SearchFilters(includeArchived: true, includeTrash: true)) == [])168 #expect(try await store.debugValue("SELECT count(*) FROM pages_fts") == "0")169 #expect(try await store.debugValue("INSERT INTO pages_fts(pages_fts) VALUES('integrity-check')") == nil)170 }171172 @Test func performanceWithFiveThousandNotes() async throws {173 let temp = TempDatabase()174 let store = try NotesStore(url: temp.url)175 let vocabulary = ["marché", "analyse", "budget", "réunion", "projet", "client", "rapport", "idée", "été", "hiver", "donnée", "modèle",176 "valeur", "prix", "maison", "terrain", "revenu", "coût", "méthode", "comparaison", "évaluation", "immeuble",177 "quartier", "ville", "région", "taux", "prêt", "banque", "notaire", "contrat"]178 var generator = SystemRandomNumberGenerator()179 var drafts: [NewPage] = []180 drafts.reserveCapacity(5_000)181 for index in 0..<5_000 {182 var text = ""183 while text.utf8.count < 2_000 { text += vocabulary.randomElement(using: &generator)! + " " }184 if index % 500 == 0 { text += "aiguille" }185 drafts.append(draft("Note \(index) \(vocabulary[index % vocabulary.count])", text, tags: ["t\(index % 25)"]))186 }187 let clock = ContinuousClock()188 func milliseconds(_ duration: Duration) -> Double { Double(duration.components.attoseconds) / 1e15 + Double(duration.components.seconds) * 1e3 }189 func best<T: Equatable>(_ expected: T, _ body: () async throws -> T) async throws -> Double {190 var result = Double.infinity191 for _ in 0..<5 {192 let start = clock.now193 let value = try await body()194 result = min(result, milliseconds(clock.now - start))195 #expect(value == expected)196 }197 return result198 }199200 let insertStart = clock.now201 let ids = try await store.createPages(drafts)202 let insert = milliseconds(clock.now - insertStart)203 #expect(ids.count == 5_000)204 #expect(try await store.pageCount() == 5_000)205206 let list = try await best(100) { try await store.summaries(list: .all, limit: 100).count }207 let recent = try await best(100) { try await store.summaries(list: .recent, limit: 100).count }208 let counts = try await best(5_000) { try await store.counts().all }209 let common = try await best(50) { try await store.search(query: "marché analyse", limit: 50).count }210 let rare = try await best(10) { try await store.search(query: "aiguil", limit: 50).count }211 let filtered = try await best(50) { try await store.search(query: "budget", filters: SearchFilters(tags: ["t3"]), limit: 50).count }212 let byTag = try await best(100) { try await store.summaries(tag: "t7", limit: 100).count }213 let one = try await best(ids[2_500]) { try await store.page(id: ids[2_500])?.id }214 let editStart = clock.now215 try await store.edit(ids[10], text: "une modification unique zanzibar")216 let edit = milliseconds(clock.now - editStart)217 #expect(try await store.search(query: "zanzibar").map(\.id) == [ids[10]])218219 print(String(format: """220 [SPNotesStore performance, 5,000 notes of ~2 KB] batch insert %.0f ms | list(all, 100) %.2f ms | list(recent, 100) %.2f ms | \221 counts %.2f ms | search common (2 terms, 50) %.2f ms | search rare prefix %.2f ms | search + tag filter %.2f ms | by tag %.2f ms | \222 single page %.2f ms | one edit (durable commit + reindex) %.2f ms223 """, insert, list, recent, counts, common, rare, filtered, byTag, one, edit))224 for sql in ["SELECT p.id FROM pages p WHERE p.deleted_at IS NULL AND p.archived=0 ORDER BY p.updated_at DESC, p.rid DESC LIMIT 100",225 "SELECT p.id FROM pages p WHERE p.deleted_at IS NULL AND p.archived=0 ORDER BY max(p.updated_at, coalesce(p.last_opened_at, 0)) DESC, p.rid DESC LIMIT 100",226 "SELECT p.id FROM pages p WHERE p.pinned=1 AND p.deleted_at IS NULL AND p.archived=0 ORDER BY p.updated_at DESC, p.rid DESC LIMIT 100",227 "SELECT p.id FROM pages p WHERE p.deleted_at IS NOT NULL ORDER BY p.deleted_at DESC, p.rid DESC LIMIT 100"] {228 // Lists are served straight from an index: no full scan, no sort.229 let plan = try await store.queryPlan(sql)230 #expect(plan.contains("USING INDEX") && !plan.contains("TEMP B-TREE"), "plan: \(plan)")231 }232 #expect(list < 50 && recent < 50 && counts < 50)233 #expect(common < 50 && rare < 50 && filtered < 50 && byTag < 50)234 #expect(insert < 15_000)235 #expect(try await store.integrityCheck().isEmpty)236 }237}238