Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Tests/AstraPersistenceTests/SearchAndAttachmentTests.swift10// Module : AstraPersistenceTests11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import Testing90import AstraCore91@testable import AstraPersistence9293@Suite struct SearchTests {94 static let hostile = [95 "\"", "\"\"", "\"unbalanced", "it's", "*", "***", "a*", "NEAR", "NEAR(a b, 3)", "a NEAR b", "AND", "OR", "NOT", "a AND", "OR b", "NOT NOT",96 "title:foo", "text : bar", "{text title}: x", "^start", "a + b", "(", ")", "((", "a) OR (b", "-", "--", "- -x", "col:\"x\"*", "\\", "%", "_", "%_%",97 "'; DROP TABLE message; --", "🙂", "🙂🙂 tides", "👨👩👧👦", "潮", "潮汐", "日本語のテキスト", "한국어", "ไทย", "Ⓐ", "①", "\u{0}", "a\u{0}b", "\n\t", " ",98 "fullwidth", "é", "e\u{301}", "ß", "İ", String(repeating: "x", count: 5000), String(repeating: "\" OR \"", count: 200), "a\"b\"c", "\"*\"", "*\"*",99 ]100101 @Test func findsTitlesAndMessagesWithSnippets() async throws {102 let store = try makeStore()103 let tides = try await store.createConversation(title: "Tide tables", model: gpt, at: t(0))104 let cooking = try await store.createConversation(title: "Cooking", model: gpt, at: t(1))105 let archived = try await store.createConversation(title: "Old harbour notes", model: gpt, at: t(2))106 try await store.exchange(tides.id, "When is high water in Halifax?", "High water is around noon; the harbour gauge confirms it.", at: 10)107 let soup = try await store.exchange(cooking.id, "A soup recipe please", "Simmer leeks and potatoes for twenty minutes, then blend. " + String(repeating: "Filler sentence. ", count: 40) + "Serve near the harbour if you like.", at: 20)108 try await store.setConversationArchived(id: archived.id, true)109110 let title = try await store.searchConversations("tide")111 #expect(title.map(\.conversation.id) == [tides.id])112 #expect(title.first?.matchedTitle == true)113114 let hits = try await store.searchConversations("harbour")115 #expect(Set(hits.map(\.conversation.id)) == [tides.id, cooking.id, archived.id])116 #expect(hits.first?.conversation.id == archived.id) // title matches come first117 let cookingHit = try #require(hits.first { $0.conversation.id == cooking.id })118 #expect(cookingHit.messageID == soup.assistant.id)119 #expect(cookingHit.snippet.localizedCaseInsensitiveContains("harbour"))120 #expect(cookingHit.snippet.count < soup.assistant.text.count)121 #expect(!cookingHit.matchedTitle)122 #expect(try await store.searchConversations("harbour", includeArchived: false).count == 2)123 #expect(try await store.searchConversations("harbour", limit: 1).count == 1)124125 // Prefixes, several terms (all required), case and diacritics.126 #expect(try await store.searchConversations("harb").count == 3)127 #expect(try await store.searchConversations("LEEKS potatoes").map(\.conversation.id) == [cooking.id])128 #expect(try await store.searchConversations("leeks halifax").isEmpty)129 try await store.exchange(cooking.id, "Et la crème brûlée ?", "Caraméliser le sucre.", at: 30)130 #expect(try await store.searchConversations("creme brulee").map(\.conversation.id) == [cooking.id])131 // Substring fallback when no word starts with the text.132 #expect(try await store.searchConversations("arbou").count == 3)133 #expect(try await store.searchConversations("").isEmpty)134 #expect(try await store.searchConversations(" ").isEmpty)135 #expect(try await store.searchConversations("nothingmatchesthis").isEmpty)136 }137138 @Test func indexFollowsRenamesEditsAndDeletes() async throws {139 let store = try makeStore()140 let conversation = try await store.createConversation(title: "Albatross", model: gpt)141 let pair = try await store.exchange(conversation.id, "about pelicans", "pelicans dive")142 #expect(try await store.searchConversations("albatross").count == 1)143 try await store.renameConversation(id: conversation.id, title: "Cormorant")144 #expect(try await store.searchConversations("albatross").isEmpty)145 #expect(try await store.searchConversations("cormorant").count == 1)146 try await store.setMessageText(id: pair.assistant.id, text: "gannets dive")147 #expect(try await store.searchConversations("gannets").first?.messageID == pair.assistant.id)148 #expect(try await store.searchConversations("pelicans").first?.messageID == pair.user.id)149 try await store.deleteMessageSubtree(messageID: pair.user.id)150 #expect(try await store.searchConversations("pelicans").isEmpty)151 #expect(try await store.searchConversations("gannets").isEmpty)152 }153154 @Test func hostileQueriesNeverBreakSearch() async throws {155 let store = try makeStore()156 let conversation = try await store.createConversation(title: "NEAR AND OR NOT \"quoted\" * title:x", model: gpt)157 try await store.exchange(conversation.id, "100% sure_thing \\ back\\slash 🙂 潮汐 tides", "it's a \"quote\" (paren) col:on a*b ^caret")158 try await store.addDocument(name: "doc", kind: "text", text: "NEAR the * harbour: \"quoted\" 100% 🙂 潮汐表 AND OR NOT")159 for query in Self.hostile {160 // Must not throw, whatever comes back.161 _ = try await store.searchConversations(query)162 _ = try await store.searchChunks(query)163 if let pattern = FTSQuery.pattern(query) {164 // Everything outside the quoted terms is a space, a star or OR: no operator, column filter or parenthesis can appear.165 let outside = pattern.split(separator: "\"", omittingEmptySubsequences: false).enumerated().filter { $0.offset % 2 == 0 }.map(\.element).joined()166 #expect(outside.allSatisfy { $0 == " " || $0 == "*" }, "Unsafe pattern for \(query.prefix(40)): \(pattern.prefix(80))")167 }168 }169 let plain = try await store.createConversation(title: "Plain", model: gpt)170 try await store.exchange(plain.id, "nothing special here", "quote without the other word")171 // Operators typed by a person are plain words: every term must be present, including "AND", "OR", "NOT" and "NEAR".172 #expect(try await store.searchConversations("NEAR").map(\.conversation.id) == [conversation.id])173 #expect(try await store.searchConversations("quote paren").map(\.conversation.id) == [conversation.id])174 #expect(try await store.searchConversations("quote AND paren").isEmpty)175 #expect(try await store.searchConversations("paren OR special").isEmpty)176 #expect(try await store.searchConversations("quote NOT paren").isEmpty)177 #expect(try await store.searchConversations("\"quote\"").count == 2)178 #expect(try await store.searchConversations("text:quote").isEmpty)179 #expect(try await store.searchConversations("it's").map(\.conversation.id) == [conversation.id])180 #expect(try await store.searchConversations("quot*").count == 2)181 // Literal punctuation, emoji and CJK are found by substring; `%` and `_` are literal there, not wildcards.182 #expect(try await store.searchConversations("100%").map(\.conversation.id) == [conversation.id])183 #expect(try await store.searchConversations("%").map(\.conversation.id) == [conversation.id])184 #expect(try await store.searchConversations("_").map(\.conversation.id) == [conversation.id])185 #expect(try await store.searchConversations("\\").map(\.conversation.id) == [conversation.id])186 #expect(try await store.searchConversations("%%").isEmpty)187 #expect(try await store.searchConversations("back\\slash").map(\.conversation.id) == [conversation.id])188 #expect(try await store.searchConversations("🙂").first?.snippet.contains("🙂") == true)189 #expect(try await store.searchConversations("潮").map(\.conversation.id) == [conversation.id])190 #expect(try await store.searchConversations("潮汐").map(\.conversation.id) == [conversation.id])191 #expect(try await store.searchConversations("汐 t").map(\.conversation.id) == [conversation.id])192 #expect(try await store.searchConversations("津波").isEmpty)193 #expect(try await store.searchChunks("潮汐").count == 1)194 #expect(try await store.searchChunks("🙂").count == 1)195 #expect(try await store.searchChunks("NEAR(harbour quoted, 2)").count == 1)196 #expect(try await store.read { db in try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM message") } == 4)197 }198199 @Test func patternBuilder() {200 #expect(FTSQuery.pattern("hello world") == "\"hello\"* \"world\"*")201 #expect(FTSQuery.pattern("Hello hello HELLO") == "\"Hello\"*")202 #expect(FTSQuery.pattern("a \"b\" c*") == "\"a\"* \"b\"* \"c\"*")203 #expect(FTSQuery.pattern("NEAR(a b)", mode: .anyTerm) == "\"NEAR\"")204 #expect(FTSQuery.pattern("what is the tide", mode: .anyTerm) == "\"what\" OR \"is\" OR \"the\" OR \"tide\"")205 #expect(FTSQuery.pattern("* \" ( ) : ^ - 🙂") == nil)206 #expect(FTSQuery.pattern("") == nil)207 #expect(FTSQuery.terms(String(repeating: "word ", count: 100)).count == 24)208 #expect(FTSQuery.terms(String(repeating: "x", count: 500)).first?.count == 64)209 #expect(FTSQuery.likePattern(" 50%_\\ ") == "%50\\%\\_\\\\%")210 #expect(FTSQuery.containsUnsegmentedScript("潮汐"))211 #expect(!FTSQuery.containsUnsegmentedScript("tide"))212 }213}214215@Suite struct AttachmentTests {216 @Test func storesMetadataAndBytes() async throws {217 let store = try makeStore()218 let conversation = try await store.createConversation(model: gpt)219 let bytes = Data("%PDF-1.7 pretend".utf8)220 let draft = AttachmentDraft(kind: .pdf, filename: "Report 2026.pdf", mimeType: "Application/PDF", data: bytes, extractedText: "pretend text")221 let message = try await store.appendUserMessage(conversationID: conversation.id, text: "see file", attachments: [draft], at: t(1))222 let attachment = try #require(try await store.attachments(messageID: message.id).first)223 #expect(attachment.kind == .pdf)224 #expect(attachment.filename == "Report 2026.pdf")225 #expect(attachment.mimeType == "application/pdf")226 #expect(attachment.byteSize == bytes.count)227 #expect(attachment.sha256 == Hashing.sha256(bytes))228 #expect(Hashing.sha256(Data("abc".utf8)) == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")229 #expect(attachment.extractedText == "pretend text")230 #expect(attachment.createdAt == t(1))231 #expect(try await store.attachmentData(attachment) == bytes)232 #expect(try store.attachmentFileURL(sha256: attachment.sha256).lastPathComponent == attachment.sha256)233 #expect(try await store.attachments(conversationID: conversation.id)[message.id]?.count == 1)234 let usage = try await store.storageUsage()235 #expect(usage.attachmentBytes == Int64(bytes.count))236 #expect(usage.attachmentFileCount == 1)237 #expect(usage.databaseBytes > 0)238 await #expect(throws: AstraStoreError.self) { try await store.addAttachment(messageID: "missing", draft) }239 }240241 @Test func identicalContentIsStoredOnce() async throws {242 let store = try makeStore()243 let conversation = try await store.createConversation(model: gpt)244 let data = Data(repeating: 42, count: 4096)245 let a = try await store.appendUserMessage(conversationID: conversation.id, text: "a",246 attachments: [AttachmentDraft(kind: .image, filename: "one.png", mimeType: "image/png", data: data)])247 let b = try await store.appendUserMessage(conversationID: conversation.id, text: "b",248 attachments: [AttachmentDraft(kind: .image, filename: "two.png", mimeType: "image/png", data: data),249 AttachmentDraft(kind: .image, filename: "three.png", mimeType: "image/png", data: data)])250 let first = try #require(try await store.attachments(messageID: a.id).first)251 let others = try await store.attachments(messageID: b.id)252 #expect(Set(others.map(\.sha256)) == [first.sha256])253 #expect(Set(others.map(\.filename)) == ["two.png", "three.png"])254 #expect(try await store.attachmentStorageBytes() == 4096)255 #expect(try FileManager.default.contentsOfDirectory(atPath: store.attachmentsDirectory.path).count == 1)256 // The file stays while any row uses it, and goes with the last one.257 #expect(try await store.deleteAttachment(id: first.id).bytesFreed == 0)258 #expect(try await store.deleteAttachment(id: others[0].id).bytesFreed == 0)259 #expect(try await store.attachmentData(sha256: first.sha256) == data)260 let last = try await store.deleteAttachment(id: others[1].id)261 #expect(last == DeletionResult(rowsDeleted: 1, bytesFreed: 4096))262 await #expect(throws: AstraStoreError.missingAttachmentData(first.sha256)) { try await store.attachmentData(sha256: first.sha256) }263 #expect(try await store.deleteAttachment(id: "missing").rowsDeleted == 0)264 }265266 @Test func sizeCapIsEnforcedAndNothingIsWritten() async throws {267 let store = try makeStore(.init(maxAttachmentBytes: 1000))268 let conversation = try await store.createConversation(model: gpt)269 let exact = AttachmentDraft(kind: .file, filename: "ok.bin", mimeType: "application/octet-stream", data: Data(repeating: 1, count: 1000))270 let tooBig = AttachmentDraft(kind: .file, filename: "big.bin", mimeType: "application/octet-stream", data: Data(repeating: 2, count: 1001))271 try await store.appendUserMessage(conversationID: conversation.id, text: "fits", attachments: [exact])272 await #expect(throws: AstraStoreError.attachmentTooLarge(limit: 1000, actual: 1001)) {273 try await store.appendUserMessage(conversationID: conversation.id, text: "does not fit", attachments: [exact, tooBig])274 }275 // The whole write rolled back: no message, no row, no extra file.276 #expect(try await store.activePath(conversationID: conversation.id).map(\.text) == ["fits"])277 #expect(try await store.attachmentStorageBytes() == 1000)278 #expect(AstraStoreError.attachmentTooLarge(limit: 1000, actual: 1001).errorDescription?.contains("too large") == true)279 }280281 @Test func namesCanNeverEscapeTheAttachmentsDirectory() async throws {282 let outer = try temporaryDirectory()283 defer { try? FileManager.default.removeItem(at: outer) }284 let attachments = outer.appendingPathComponent("inner/Attachments", isDirectory: true)285 let store = try AstraStore(inMemory: .init(), attachmentsDirectory: attachments)286 let conversation = try await store.createConversation(model: gpt)287 let names = ["../../escape.txt", "/etc/passwd", "..", "../", "..\\..\\windows\\system32\\evil.dll", "a/b/../../../c.txt", "\u{0}null.txt",288 "~/.ssh/authorized_keys", ".hidden", "name\nwith\nnewlines.txt", "", " ", String(repeating: "long", count: 200) + ".txt",289 "file:///etc/hosts", "CON", "normal.txt", "%2e%2e%2fencoded.txt", "....//....//x"]290 var messageIDs: [String] = []291 for (index, name) in names.enumerated() {292 let draft = AttachmentDraft(kind: .file, filename: name, mimeType: "../../text/plain", data: Data("content \(index)".utf8))293 messageIDs.append(try await store.appendUserMessage(conversationID: conversation.id, text: "m\(index)", attachments: [draft]).id)294 }295 // Every byte landed directly inside the attachments directory, under a 64-hex-digit name.296 let stored = try FileManager.default.contentsOfDirectory(atPath: attachments.path)297 #expect(stored.count == names.count)298 #expect(stored.allSatisfy { Hashing.isSHA256($0) })299 #expect(directoriesHoldingFiles(under: outer) == [attachments.standardizedFileURL.path])300 // Display names are sanitized: no separators, no control characters, no leading dot, never empty, bounded.301 for id in messageIDs {302 let record = try #require(try await store.attachments(messageID: id).first)303 #expect(!record.filename.isEmpty)304 #expect(!record.filename.contains("/"))305 #expect(!record.filename.contains("\\"))306 #expect(!record.filename.hasPrefix("."))307 #expect(record.filename.unicodeScalars.allSatisfy { !CharacterSet.controlCharacters.contains($0) })308 #expect(record.filename.count <= 200)309 #expect(record.mimeType == "application/octet-stream")310 }311 #expect(BlobDirectory.displayName("../../escape.txt") == "escape.txt")312 #expect(BlobDirectory.displayName("..") == "attachment")313 #expect(BlobDirectory.displayName("C:\\Users\\me\\photo.jpg") == "photo.jpg")314 // Reading only accepts hashes.315 for reference in ["../../etc/passwd", "/etc/passwd", "..", "", "abc", String(repeating: "g", count: 64), String(repeating: "A", count: 64),316 String(repeating: "a", count: 63) + "/", String(repeating: "a", count: 65)] {317 await #expect(throws: AstraStoreError.invalidAttachmentReference) { try await store.attachmentData(sha256: reference) }318 #expect(throws: AstraStoreError.invalidAttachmentReference) { try store.attachmentFileURL(sha256: reference) }319 }320 }321322 @Test func orphanFilesAreCleanedUp() async throws {323 let store = try makeStore()324 let conversation = try await store.createConversation(model: gpt)325 let kept = AttachmentDraft(kind: .text, filename: "kept.txt", mimeType: "text/plain", data: Data(repeating: 1, count: 300))326 let message = try await store.appendUserMessage(conversationID: conversation.id, text: "m", attachments: [kept])327 // Files left behind by a crash between writing the bytes and committing the row.328 let strayHash = try store.blobs.store(Data(repeating: 2, count: 700))329 // A file the store did not write is never counted nor touched.330 let foreign = store.attachmentsDirectory.appendingPathComponent("not-ours.txt")331 try Data(repeating: 3, count: 50).write(to: foreign)332 #expect(try await store.attachmentStorageBytes() == 1000)333 #expect(try await store.cleanupOrphanAttachments() == 700)334 #expect(try await store.cleanupOrphanAttachments() == 0)335 #expect(try await store.attachmentStorageBytes() == 300)336 await #expect(throws: AstraStoreError.self) { try await store.attachmentData(sha256: strayHash) }337 #expect(try await store.attachmentData(try #require(try await store.attachments(messageID: message.id).first)).count == 300)338 // Removing every attachment keeps the messages.339 let result = try await store.deleteAllAttachments()340 #expect(result.rowsDeleted == 1)341 #expect(result.bytesFreed >= 300)342 #expect(try await store.attachmentStorageBytes() == 0)343 #expect(try await store.activePath(conversationID: conversation.id).count == 1)344 #expect(FileManager.default.fileExists(atPath: foreign.path))345 }346}347348/// Directories under `root` that directly contain at least one regular file.349func directoriesHoldingFiles(under root: URL) -> Set<String> {350 var out = Set<String>()351 guard let enumerator = FileManager.default.enumerator(at: root, includingPropertiesForKeys: [.isRegularFileKey]) else { return out }352 for case let url as URL in enumerator where (try? url.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile) == true {353 out.insert(url.deletingLastPathComponent().standardizedFileURL.path)354 }355 return out356}357