SPB Git

Repository Public MIT

SearchAndAttachmentTests.swift 22.9 KB 357 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraPersistenceTests/SearchAndAttachmentTests.swift10// Module           : AstraPersistenceTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import Testing90import AstraCore91@testable import AstraPersistence9293@Suite struct SearchTests {94    static let hostile = [95        "\"", "\"\"", "\"unbalanced", "it's", "*", "***", "a*", "NEAR", "NEAR(a b, 3)", "a NEAR b", "AND", "OR", "NOT", "a AND", "OR b", "NOT NOT",96        "title:foo", "text : bar", "{text title}: x", "^start", "a + b", "(", ")", "((", "a) OR (b", "-", "--", "- -x", "col:\"x\"*", "\\", "%", "_", "%_%",97        "'; DROP TABLE message; --", "🙂", "🙂🙂 tides", "👨‍👩‍👧‍👦", "潮", "潮汐", "日本語のテキスト", "한국어", "ไทย", "Ⓐ", "①", "\u{0}", "a\u{0}b", "\n\t", "   ",98        "fullwidth", "é", "e\u{301}", "ß", "İ", String(repeating: "x", count: 5000), String(repeating: "\" OR \"", count: 200), "a\"b\"c", "\"*\"", "*\"*",99    ]100101    @Test func findsTitlesAndMessagesWithSnippets() async throws {102        let store = try makeStore()103        let tides = try await store.createConversation(title: "Tide tables", model: gpt, at: t(0))104        let cooking = try await store.createConversation(title: "Cooking", model: gpt, at: t(1))105        let archived = try await store.createConversation(title: "Old harbour notes", model: gpt, at: t(2))106        try await store.exchange(tides.id, "When is high water in Halifax?", "High water is around noon; the harbour gauge confirms it.", at: 10)107        let soup = try await store.exchange(cooking.id, "A soup recipe please", "Simmer leeks and potatoes for twenty minutes, then blend. " + String(repeating: "Filler sentence. ", count: 40) + "Serve near the harbour if you like.", at: 20)108        try await store.setConversationArchived(id: archived.id, true)109110        let title = try await store.searchConversations("tide")111        #expect(title.map(\.conversation.id) == [tides.id])112        #expect(title.first?.matchedTitle == true)113114        let hits = try await store.searchConversations("harbour")115        #expect(Set(hits.map(\.conversation.id)) == [tides.id, cooking.id, archived.id])116        #expect(hits.first?.conversation.id == archived.id)   // title matches come first117        let cookingHit = try #require(hits.first { $0.conversation.id == cooking.id })118        #expect(cookingHit.messageID == soup.assistant.id)119        #expect(cookingHit.snippet.localizedCaseInsensitiveContains("harbour"))120        #expect(cookingHit.snippet.count < soup.assistant.text.count)121        #expect(!cookingHit.matchedTitle)122        #expect(try await store.searchConversations("harbour", includeArchived: false).count == 2)123        #expect(try await store.searchConversations("harbour", limit: 1).count == 1)124125        // Prefixes, several terms (all required), case and diacritics.126        #expect(try await store.searchConversations("harb").count == 3)127        #expect(try await store.searchConversations("LEEKS potatoes").map(\.conversation.id) == [cooking.id])128        #expect(try await store.searchConversations("leeks halifax").isEmpty)129        try await store.exchange(cooking.id, "Et la crème brûlée ?", "Caraméliser le sucre.", at: 30)130        #expect(try await store.searchConversations("creme brulee").map(\.conversation.id) == [cooking.id])131        // Substring fallback when no word starts with the text.132        #expect(try await store.searchConversations("arbou").count == 3)133        #expect(try await store.searchConversations("").isEmpty)134        #expect(try await store.searchConversations("   ").isEmpty)135        #expect(try await store.searchConversations("nothingmatchesthis").isEmpty)136    }137138    @Test func indexFollowsRenamesEditsAndDeletes() async throws {139        let store = try makeStore()140        let conversation = try await store.createConversation(title: "Albatross", model: gpt)141        let pair = try await store.exchange(conversation.id, "about pelicans", "pelicans dive")142        #expect(try await store.searchConversations("albatross").count == 1)143        try await store.renameConversation(id: conversation.id, title: "Cormorant")144        #expect(try await store.searchConversations("albatross").isEmpty)145        #expect(try await store.searchConversations("cormorant").count == 1)146        try await store.setMessageText(id: pair.assistant.id, text: "gannets dive")147        #expect(try await store.searchConversations("gannets").first?.messageID == pair.assistant.id)148        #expect(try await store.searchConversations("pelicans").first?.messageID == pair.user.id)149        try await store.deleteMessageSubtree(messageID: pair.user.id)150        #expect(try await store.searchConversations("pelicans").isEmpty)151        #expect(try await store.searchConversations("gannets").isEmpty)152    }153154    @Test func hostileQueriesNeverBreakSearch() async throws {155        let store = try makeStore()156        let conversation = try await store.createConversation(title: "NEAR AND OR NOT \"quoted\" * title:x", model: gpt)157        try await store.exchange(conversation.id, "100% sure_thing \\ back\\slash 🙂 潮汐 tides", "it's a \"quote\" (paren) col:on a*b ^caret")158        try await store.addDocument(name: "doc", kind: "text", text: "NEAR the * harbour: \"quoted\" 100% 🙂 潮汐表 AND OR NOT")159        for query in Self.hostile {160            // Must not throw, whatever comes back.161            _ = try await store.searchConversations(query)162            _ = try await store.searchChunks(query)163            if let pattern = FTSQuery.pattern(query) {164                // Everything outside the quoted terms is a space, a star or OR: no operator, column filter or parenthesis can appear.165                let outside = pattern.split(separator: "\"", omittingEmptySubsequences: false).enumerated().filter { $0.offset % 2 == 0 }.map(\.element).joined()166                #expect(outside.allSatisfy { $0 == " " || $0 == "*" }, "Unsafe pattern for \(query.prefix(40)): \(pattern.prefix(80))")167            }168        }169        let plain = try await store.createConversation(title: "Plain", model: gpt)170        try await store.exchange(plain.id, "nothing special here", "quote without the other word")171        // Operators typed by a person are plain words: every term must be present, including "AND", "OR", "NOT" and "NEAR".172        #expect(try await store.searchConversations("NEAR").map(\.conversation.id) == [conversation.id])173        #expect(try await store.searchConversations("quote paren").map(\.conversation.id) == [conversation.id])174        #expect(try await store.searchConversations("quote AND paren").isEmpty)175        #expect(try await store.searchConversations("paren OR special").isEmpty)176        #expect(try await store.searchConversations("quote NOT paren").isEmpty)177        #expect(try await store.searchConversations("\"quote\"").count == 2)178        #expect(try await store.searchConversations("text:quote").isEmpty)179        #expect(try await store.searchConversations("it's").map(\.conversation.id) == [conversation.id])180        #expect(try await store.searchConversations("quot*").count == 2)181        // Literal punctuation, emoji and CJK are found by substring; `%` and `_` are literal there, not wildcards.182        #expect(try await store.searchConversations("100%").map(\.conversation.id) == [conversation.id])183        #expect(try await store.searchConversations("%").map(\.conversation.id) == [conversation.id])184        #expect(try await store.searchConversations("_").map(\.conversation.id) == [conversation.id])185        #expect(try await store.searchConversations("\\").map(\.conversation.id) == [conversation.id])186        #expect(try await store.searchConversations("%%").isEmpty)187        #expect(try await store.searchConversations("back\\slash").map(\.conversation.id) == [conversation.id])188        #expect(try await store.searchConversations("🙂").first?.snippet.contains("🙂") == true)189        #expect(try await store.searchConversations("潮").map(\.conversation.id) == [conversation.id])190        #expect(try await store.searchConversations("潮汐").map(\.conversation.id) == [conversation.id])191        #expect(try await store.searchConversations("汐 t").map(\.conversation.id) == [conversation.id])192        #expect(try await store.searchConversations("津波").isEmpty)193        #expect(try await store.searchChunks("潮汐").count == 1)194        #expect(try await store.searchChunks("🙂").count == 1)195        #expect(try await store.searchChunks("NEAR(harbour quoted, 2)").count == 1)196        #expect(try await store.read { db in try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM message") } == 4)197    }198199    @Test func patternBuilder() {200        #expect(FTSQuery.pattern("hello world") == "\"hello\"* \"world\"*")201        #expect(FTSQuery.pattern("Hello hello HELLO") == "\"Hello\"*")202        #expect(FTSQuery.pattern("a \"b\" c*") == "\"a\"* \"b\"* \"c\"*")203        #expect(FTSQuery.pattern("NEAR(a b)", mode: .anyTerm) == "\"NEAR\"")204        #expect(FTSQuery.pattern("what is the tide", mode: .anyTerm) == "\"what\" OR \"is\" OR \"the\" OR \"tide\"")205        #expect(FTSQuery.pattern("* \" ( ) : ^ - 🙂") == nil)206        #expect(FTSQuery.pattern("") == nil)207        #expect(FTSQuery.terms(String(repeating: "word ", count: 100)).count == 24)208        #expect(FTSQuery.terms(String(repeating: "x", count: 500)).first?.count == 64)209        #expect(FTSQuery.likePattern(" 50%_\\ ") == "%50\\%\\_\\\\%")210        #expect(FTSQuery.containsUnsegmentedScript("潮汐"))211        #expect(!FTSQuery.containsUnsegmentedScript("tide"))212    }213}214215@Suite struct AttachmentTests {216    @Test func storesMetadataAndBytes() async throws {217        let store = try makeStore()218        let conversation = try await store.createConversation(model: gpt)219        let bytes = Data("%PDF-1.7 pretend".utf8)220        let draft = AttachmentDraft(kind: .pdf, filename: "Report 2026.pdf", mimeType: "Application/PDF", data: bytes, extractedText: "pretend text")221        let message = try await store.appendUserMessage(conversationID: conversation.id, text: "see file", attachments: [draft], at: t(1))222        let attachment = try #require(try await store.attachments(messageID: message.id).first)223        #expect(attachment.kind == .pdf)224        #expect(attachment.filename == "Report 2026.pdf")225        #expect(attachment.mimeType == "application/pdf")226        #expect(attachment.byteSize == bytes.count)227        #expect(attachment.sha256 == Hashing.sha256(bytes))228        #expect(Hashing.sha256(Data("abc".utf8)) == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")229        #expect(attachment.extractedText == "pretend text")230        #expect(attachment.createdAt == t(1))231        #expect(try await store.attachmentData(attachment) == bytes)232        #expect(try store.attachmentFileURL(sha256: attachment.sha256).lastPathComponent == attachment.sha256)233        #expect(try await store.attachments(conversationID: conversation.id)[message.id]?.count == 1)234        let usage = try await store.storageUsage()235        #expect(usage.attachmentBytes == Int64(bytes.count))236        #expect(usage.attachmentFileCount == 1)237        #expect(usage.databaseBytes > 0)238        await #expect(throws: AstraStoreError.self) { try await store.addAttachment(messageID: "missing", draft) }239    }240241    @Test func identicalContentIsStoredOnce() async throws {242        let store = try makeStore()243        let conversation = try await store.createConversation(model: gpt)244        let data = Data(repeating: 42, count: 4096)245        let a = try await store.appendUserMessage(conversationID: conversation.id, text: "a",246                                                  attachments: [AttachmentDraft(kind: .image, filename: "one.png", mimeType: "image/png", data: data)])247        let b = try await store.appendUserMessage(conversationID: conversation.id, text: "b",248                                                  attachments: [AttachmentDraft(kind: .image, filename: "two.png", mimeType: "image/png", data: data),249                                                                AttachmentDraft(kind: .image, filename: "three.png", mimeType: "image/png", data: data)])250        let first = try #require(try await store.attachments(messageID: a.id).first)251        let others = try await store.attachments(messageID: b.id)252        #expect(Set(others.map(\.sha256)) == [first.sha256])253        #expect(Set(others.map(\.filename)) == ["two.png", "three.png"])254        #expect(try await store.attachmentStorageBytes() == 4096)255        #expect(try FileManager.default.contentsOfDirectory(atPath: store.attachmentsDirectory.path).count == 1)256        // The file stays while any row uses it, and goes with the last one.257        #expect(try await store.deleteAttachment(id: first.id).bytesFreed == 0)258        #expect(try await store.deleteAttachment(id: others[0].id).bytesFreed == 0)259        #expect(try await store.attachmentData(sha256: first.sha256) == data)260        let last = try await store.deleteAttachment(id: others[1].id)261        #expect(last == DeletionResult(rowsDeleted: 1, bytesFreed: 4096))262        await #expect(throws: AstraStoreError.missingAttachmentData(first.sha256)) { try await store.attachmentData(sha256: first.sha256) }263        #expect(try await store.deleteAttachment(id: "missing").rowsDeleted == 0)264    }265266    @Test func sizeCapIsEnforcedAndNothingIsWritten() async throws {267        let store = try makeStore(.init(maxAttachmentBytes: 1000))268        let conversation = try await store.createConversation(model: gpt)269        let exact = AttachmentDraft(kind: .file, filename: "ok.bin", mimeType: "application/octet-stream", data: Data(repeating: 1, count: 1000))270        let tooBig = AttachmentDraft(kind: .file, filename: "big.bin", mimeType: "application/octet-stream", data: Data(repeating: 2, count: 1001))271        try await store.appendUserMessage(conversationID: conversation.id, text: "fits", attachments: [exact])272        await #expect(throws: AstraStoreError.attachmentTooLarge(limit: 1000, actual: 1001)) {273            try await store.appendUserMessage(conversationID: conversation.id, text: "does not fit", attachments: [exact, tooBig])274        }275        // The whole write rolled back: no message, no row, no extra file.276        #expect(try await store.activePath(conversationID: conversation.id).map(\.text) == ["fits"])277        #expect(try await store.attachmentStorageBytes() == 1000)278        #expect(AstraStoreError.attachmentTooLarge(limit: 1000, actual: 1001).errorDescription?.contains("too large") == true)279    }280281    @Test func namesCanNeverEscapeTheAttachmentsDirectory() async throws {282        let outer = try temporaryDirectory()283        defer { try? FileManager.default.removeItem(at: outer) }284        let attachments = outer.appendingPathComponent("inner/Attachments", isDirectory: true)285        let store = try AstraStore(inMemory: .init(), attachmentsDirectory: attachments)286        let conversation = try await store.createConversation(model: gpt)287        let names = ["../../escape.txt", "/etc/passwd", "..", "../", "..\\..\\windows\\system32\\evil.dll", "a/b/../../../c.txt", "\u{0}null.txt",288                     "~/.ssh/authorized_keys", ".hidden", "name\nwith\nnewlines.txt", "", "   ", String(repeating: "long", count: 200) + ".txt",289                     "file:///etc/hosts", "CON", "normal.txt", "%2e%2e%2fencoded.txt", "....//....//x"]290        var messageIDs: [String] = []291        for (index, name) in names.enumerated() {292            let draft = AttachmentDraft(kind: .file, filename: name, mimeType: "../../text/plain", data: Data("content \(index)".utf8))293            messageIDs.append(try await store.appendUserMessage(conversationID: conversation.id, text: "m\(index)", attachments: [draft]).id)294        }295        // Every byte landed directly inside the attachments directory, under a 64-hex-digit name.296        let stored = try FileManager.default.contentsOfDirectory(atPath: attachments.path)297        #expect(stored.count == names.count)298        #expect(stored.allSatisfy { Hashing.isSHA256($0) })299        #expect(directoriesHoldingFiles(under: outer) == [attachments.standardizedFileURL.path])300        // Display names are sanitized: no separators, no control characters, no leading dot, never empty, bounded.301        for id in messageIDs {302            let record = try #require(try await store.attachments(messageID: id).first)303            #expect(!record.filename.isEmpty)304            #expect(!record.filename.contains("/"))305            #expect(!record.filename.contains("\\"))306            #expect(!record.filename.hasPrefix("."))307            #expect(record.filename.unicodeScalars.allSatisfy { !CharacterSet.controlCharacters.contains($0) })308            #expect(record.filename.count <= 200)309            #expect(record.mimeType == "application/octet-stream")310        }311        #expect(BlobDirectory.displayName("../../escape.txt") == "escape.txt")312        #expect(BlobDirectory.displayName("..") == "attachment")313        #expect(BlobDirectory.displayName("C:\\Users\\me\\photo.jpg") == "photo.jpg")314        // Reading only accepts hashes.315        for reference in ["../../etc/passwd", "/etc/passwd", "..", "", "abc", String(repeating: "g", count: 64), String(repeating: "A", count: 64),316                          String(repeating: "a", count: 63) + "/", String(repeating: "a", count: 65)] {317            await #expect(throws: AstraStoreError.invalidAttachmentReference) { try await store.attachmentData(sha256: reference) }318            #expect(throws: AstraStoreError.invalidAttachmentReference) { try store.attachmentFileURL(sha256: reference) }319        }320    }321322    @Test func orphanFilesAreCleanedUp() async throws {323        let store = try makeStore()324        let conversation = try await store.createConversation(model: gpt)325        let kept = AttachmentDraft(kind: .text, filename: "kept.txt", mimeType: "text/plain", data: Data(repeating: 1, count: 300))326        let message = try await store.appendUserMessage(conversationID: conversation.id, text: "m", attachments: [kept])327        // Files left behind by a crash between writing the bytes and committing the row.328        let strayHash = try store.blobs.store(Data(repeating: 2, count: 700))329        // A file the store did not write is never counted nor touched.330        let foreign = store.attachmentsDirectory.appendingPathComponent("not-ours.txt")331        try Data(repeating: 3, count: 50).write(to: foreign)332        #expect(try await store.attachmentStorageBytes() == 1000)333        #expect(try await store.cleanupOrphanAttachments() == 700)334        #expect(try await store.cleanupOrphanAttachments() == 0)335        #expect(try await store.attachmentStorageBytes() == 300)336        await #expect(throws: AstraStoreError.self) { try await store.attachmentData(sha256: strayHash) }337        #expect(try await store.attachmentData(try #require(try await store.attachments(messageID: message.id).first)).count == 300)338        // Removing every attachment keeps the messages.339        let result = try await store.deleteAllAttachments()340        #expect(result.rowsDeleted == 1)341        #expect(result.bytesFreed >= 300)342        #expect(try await store.attachmentStorageBytes() == 0)343        #expect(try await store.activePath(conversationID: conversation.id).count == 1)344        #expect(FileManager.default.fileExists(atPath: foreign.path))345    }346}347348/// Directories under `root` that directly contain at least one regular file.349func directoriesHoldingFiles(under root: URL) -> Set<String> {350    var out = Set<String>()351    guard let enumerator = FileManager.default.enumerator(at: root, includingPropertiesForKeys: [.isRegularFileKey]) else { return out }352    for case let url as URL in enumerator where (try? url.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile) == true {353        out.insert(url.deletingLastPathComponent().standardizedFileURL.path)354    }355    return out356}357
Files main
Menu

Theme