SPB Git

Repository Public MIT

DataTests.swift 23.8 KB 437 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraPersistenceTests/DataTests.swift10// Module           : AstraPersistenceTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import Testing90import AstraCore91@testable import AstraPersistence9293@Suite struct UsageTests {94    static let utc = TimeZone(identifier: "UTC")!95    static func day(_ day: Int, hour: Int = 12) -> Date {96        var calendar = Calendar(identifier: .gregorian); calendar.timeZone = utc97        return calendar.date(from: DateComponents(year: 2026, month: 11, day: day, hour: hour))!98    }99100    static func seed(_ store: AstraStore) async throws {101        let rows: [(Int, Int, ModelRef, TokenUsage?, CostEstimate?, UsageOrigin)] = [102            (1, 9, gpt, TokenUsage(inputTokens: 1000, outputTokens: 200, cachedInputTokens: 600, reasoningTokens: 50), CostEstimate(amountUSD: 0.01, basis: .estimatedFromUsage), .conversation),103            (1, 23, gpt, TokenUsage(inputTokens: 500, outputTokens: 100), CostEstimate(amountUSD: 0.005, basis: .estimatedFromUsage), .arena),104            (2, 1, claude, TokenUsage(inputTokens: 2000, outputTokens: 800), CostEstimate(amountUSD: 0.04, basis: .reportedByProvider), .conversation),105            (2, 10, local, TokenUsage(inputTokens: 300, outputTokens: 900), .onDevice, .agent),106            (3, 10, claude, TokenUsage(inputTokens: 100, outputTokens: 10), .unavailable, .conversation),107            (3, 11, gpt, nil, nil, .promptStudio),108        ]109        for (day, hour, model, usage, cost, origin) in rows {110            try await store.recordUsage(UsageRecord(date: Self.day(day, hour: hour), model: model, usage: usage, cost: cost, origin: origin, originID: "o"))111        }112    }113114    @Test func aggregatesByDayProviderModelAndOrigin() async throws {115        let store = try makeStore()116        try await Self.seed(store)117        let days = try await store.usageAggregates(by: .day, timeZone: Self.utc)118        #expect(days.map(\.key) == ["2026-11-01", "2026-11-02", "2026-11-03"])119        #expect(days.map(\.requests) == [2, 2, 2])120        #expect(days.map(\.inputTokens) == [1500, 2300, 100])121        #expect(days.map(\.outputTokens) == [300, 1700, 10])122        #expect(days[0].cachedInputTokens == 600)123        #expect(days[0].reasoningTokens == 50)124        #expect(abs(days[0].costUSD - 0.015) < 1e-12)125        #expect(abs(days[1].costUSD - 0.04) < 1e-12)126        #expect(days.map(\.unpricedRequests) == [0, 0, 2])127        #expect(days[0].totalTokens == 1800)128129        // Days follow the requested time zone: 23:00 UTC on the 1st is already the 2nd in Paris, and still the 1st in Montréal.130        let paris = try await store.usageAggregates(by: .day, timeZone: TimeZone(identifier: "Europe/Paris")!)131        #expect(paris.map(\.requests) == [1, 3, 2])132        let montreal = try await store.usageAggregates(by: .day, timeZone: TimeZone(identifier: "America/Toronto")!)133        #expect(montreal.map(\.key) == ["2026-11-01", "2026-11-02", "2026-11-03"])134        #expect(montreal.map(\.requests) == [3, 1, 2])135136        let providers = try await store.usageAggregates(by: .provider)137        #expect(providers.map(\.key) == ["anthropic", "openAI", "mlx"])138        #expect(providers.map(\.label) == ["Anthropic", "OpenAI", "On-device (MLX)"])139        #expect(providers.map(\.totalTokens) == [2910, 1800, 1200])140        #expect(providers[1].requests == 3)141        #expect(providers[1].unpricedRequests == 1)142        #expect(providers[2].costUSD == 0)143        #expect(providers[2].unpricedRequests == 0)144145        let models = try await store.usageAggregates(by: .model)146        #expect(models.map(\.key) == ["anthropic|claude-test", "openAI|gpt-test", "mlx|mlx-community/test-4bit"])147        #expect(models.first?.label == "claude-test")148        let origins = try await store.usageAggregates(by: .origin)149        #expect(Dictionary(uniqueKeysWithValues: origins.map { ($0.key, $0.requests) }) == ["conversation": 3, "arena": 1, "agent": 1, "promptStudio": 1])150151        // Ranges are half-open: [from, to).152        let second = try await store.usageAggregates(by: .day, from: Self.day(2, hour: 0), to: Self.day(3, hour: 0), timeZone: Self.utc)153        #expect(second.map(\.key) == ["2026-11-02"])154        #expect(second.first?.requests == 2)155        #expect(try await store.usageRecords(from: Self.day(3, hour: 10), to: Self.day(3, hour: 11)).count == 1)156        #expect(try await store.usageRecords(limit: 2).map(\.date) == [Self.day(3, hour: 11), Self.day(3, hour: 10)])157158        let total = try await store.usageTotal()159        #expect(total.requests == 6)160        #expect(total.inputTokens == 3900)161        #expect(total.outputTokens == 2010)162        #expect(total.cachedInputTokens == 600)163        #expect(abs(total.costUSD - 0.055) < 1e-12)164        #expect(total.unpricedRequests == 2)165        let empty = try await store.usageTotal(from: Self.day(20))166        #expect(empty.requests == 0)167        #expect(empty.costUSD == 0)168        #expect(try await store.usageAggregates(by: .day, from: Self.day(20)).isEmpty)169    }170171    @Test func ledgerExports() async throws {172        let store = try makeStore()173        try await Self.seed(store)174        let rows = parseCSV(try await store.exportUsage(.csv))175        #expect(rows.count == 7)176        #expect(rows[0].first == "date")177        #expect(rows.last?[1] == "openAI")178        #expect(rows.last?[7] == "0.01")179        #expect(rows[1][7] == "")   // unknown cost stays empty, never zero180        let decoded = try ExportCoding.decoder.decode([UsageRecord].self, from: Data(try await store.exportUsage(.json).utf8))181        #expect(decoded.count == 6)182        #expect(try await store.exportUsage(.markdown).split(separator: "\n").count == 8)183        let result = try await store.deleteAllUsage()184        #expect(result.rowsDeleted == 6)185        #expect(try await store.usageRecords().isEmpty)186    }187}188189@Suite struct DiagnosticsAndMemoryTests {190    @Test func diagnosticsAreARingBuffer() async throws {191        let store = try makeStore(.init(diagnosticsCapacity: 25))192        for index in 0..<60 {193            try await store.recordDiagnostic(RequestDiagnostic(provider: .openRouter, modelID: "m\(index)", startedAt: t(Double(index)), httpStatus: 200,194                                                               requestID: "req-\(index)", timeToHeaders: 0.1, timeToFirstToken: 0.2, totalTime: 1, retryCount: index % 2,195                                                               usage: TokenUsage(inputTokens: index), stopReason: .endOfTurn))196        }197        let kept = try await store.diagnostics()198        #expect(kept.count == 25)199        #expect(kept.first?.modelID == "m59")200        #expect(kept.last?.modelID == "m35")201        #expect(kept.first?.usage?.inputTokens == 59)202        #expect(kept.first?.startedAt == t(59))203        #expect(try await store.diagnostics(limit: 3).map(\.modelID) == ["m59", "m58", "m57"])204        // Recording the same diagnostic again replaces it and makes it the newest.205        var updated = kept[10]206        updated.errorTitle = "Failed with Bearer abcdefgh12345678"207        try await store.recordDiagnostic(updated)208        let after = try await store.diagnostics()209        #expect(after.count == 25)210        #expect(after.first?.id == updated.id)211        #expect(after.first?.errorTitle?.contains("abcdefgh12345678") == false)212        #expect(try await store.deleteAllDiagnostics().rowsDeleted == 25)213        #expect(try await store.diagnostics().isEmpty)214    }215216    @Test func defaultCapacityIsFiveHundred() async throws {217        #expect(AstraStore.Options().diagnosticsCapacity == 500)218        #expect(AstraStore.Options(diagnosticsCapacity: 0).diagnosticsCapacity == 1)219        let store = try makeStore()220        for index in 0..<520 { try await store.recordDiagnostic(RequestDiagnostic(provider: .openAI, modelID: "m\(index)")) }221        let kept = try await store.diagnostics(limit: 1000)222        #expect(kept.count == 500)223        #expect(kept.last?.modelID == "m20")224    }225226    @Test func memoryEntries() async throws {227        let store = try makeStore()228        let conversation = try await store.createConversation(model: gpt)229        let first = try await store.addMemory(text: "Prefers metric units", sourceConversationID: conversation.id, at: t(1))230        let second = try await store.addMemory(text: "Lives in Gatineau", at: t(2))231        let disabled = try await store.addMemory(text: "Old fact", enabled: false, at: t(3))232        #expect(try await store.memories().map(\.id) == [disabled.id, second.id, first.id])233        #expect(try await store.memories(enabledOnly: true).map(\.text) == ["Lives in Gatineau", "Prefers metric units"])234        try await store.updateMemory(id: first.id, text: "Prefers metric units and 24 h time", at: t(9))235        try await store.updateMemory(id: second.id, enabled: false)236        let updated = try #require(try await store.memories().first { $0.id == first.id })237        #expect(updated.text == "Prefers metric units and 24 h time")238        #expect(updated.createdAt == t(1))239        #expect(updated.updatedAt == t(9))240        #expect(updated.sourceConversationID == conversation.id)241        #expect(try await store.memories(enabledOnly: true).map(\.id) == [first.id])242        try await store.deleteMemory(id: first.id)243        #expect(try await store.memories().count == 2)244        await #expect(throws: AstraStoreError.self) { try await store.updateMemory(id: "missing", text: "x") }245        #expect(try await store.deleteAllMemory().rowsDeleted == 2)246        #expect(try await store.memories().isEmpty)247    }248}249250@Suite struct DocumentTests {251    @Test func bm25RanksTheMostRelevantChunkFirst() async throws {252        let store = try makeStore()253        let tides = try await store.addDocument(name: "Tides.md", kind: "markdown", byteSize: 900, sha256: "hash-tides", chunks: [254            ChunkDraft(text: "Tides are caused by the gravitational pull of the Moon and the Sun on the oceans."),255            ChunkDraft(text: "A spring tide happens at new and full moon; a neap tide at the quarters. Tide, tide, tide tables list each tide."),256            ChunkDraft(text: "Harbour pilots read tide tables before every departure."),257        ], at: t(1))258        let bread = try await store.addDocument(name: "Bread.txt", kind: "text", byteSize: 300, sha256: "hash-bread", chunks: [259            ChunkDraft(text: "Sourdough needs flour, water, salt and time. The Moon has nothing to do with it."),260        ], at: t(2))261        #expect(try await store.documents().map(\.name) == ["Bread.txt", "Tides.md"])262        #expect(try await store.document(sha256: "hash-tides")?.id == tides.id)263        #expect(try await store.chunkCount() == 4)264        #expect(try await store.chunks(documentID: tides.id).map(\.ordinal) == [0, 1, 2])265266        let hits = try await store.searchChunks("tide tables", limit: 10)267        #expect(hits.map(\.chunk.ordinal).prefix(2) == [1, 2])268        #expect(hits.allSatisfy { $0.documentName == "Tides.md" })269        #expect(hits[0].score > hits[1].score)270        #expect(hits.allSatisfy { $0.score > 0 })271        // A natural-language question matches on any of its terms (OR), ranked.272        let question = try await store.searchChunks("What does the Moon have to do with sourdough bread?", limit: 3)273        #expect(question.first?.documentName == "Bread.txt")274        #expect(question.count >= 2)275        #expect(try await store.searchChunks("moon", limit: 1).count == 1)276        #expect(try await store.searchChunks("moon", limit: 10, documentIDs: [bread.id]).map(\.documentName) == ["Bread.txt"])277        #expect(try await store.searchChunks("moon", limit: 0).isEmpty)278        #expect(try await store.searchChunks("xylophone").isEmpty)279        #expect(try await store.searchChunks("").isEmpty)280        #expect(try await store.chunks(ids: [hits[0].chunk.id, "missing"]).map(\.chunk.text) == [hits[0].chunk.text])281282        // Deleting a document removes its chunks and index entries.283        let result = try await store.deleteDocument(id: tides.id)284        #expect(result.rowsDeleted == 4)285        #expect(try await store.searchChunks("tide").isEmpty)286        #expect(try await store.searchChunks("moon").count == 1)287        #expect(try await store.read { db in try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM chunk_fts") } == 1)288        #expect(try await store.deleteDocument(id: tides.id).rowsDeleted == 0)289        #expect(try await store.deleteAllDocuments().rowsDeleted == 2)290        #expect(try await store.chunkCount() == 0)291    }292293    @Test func embeddingsRoundTripAsBlobs() async throws {294        let store = try makeStore()295        let vector: [Float] = [0, 1, -1, 0.5, .pi, -.greatestFiniteMagnitude, .leastNonzeroMagnitude, 1e-20]296        let document = try await store.addDocument(name: "d", kind: "text", byteSize: 1, sha256: "h", chunks: [297            ChunkDraft(text: "with vector", embedding: vector), ChunkDraft(text: "without vector"),298        ])299        var chunks = try await store.chunks(documentID: document.id, includeEmbeddings: true)300        #expect(chunks[0].embedding == vector)301        #expect(chunks[1].embedding == nil)302        #expect(try await store.chunks(documentID: document.id)[0].embedding == nil)303        try await store.setChunkEmbedding(chunkID: chunks[1].id, [9, 8, 7])304        #expect(try await store.chunkEmbeddings() == [chunks[0].id: vector, chunks[1].id: [9, 8, 7]])305        #expect(try await store.chunkEmbeddings(documentIDs: ["other"]).isEmpty)306        try await store.setChunkEmbedding(chunkID: chunks[0].id, nil)307        chunks = try await store.chunks(documentID: document.id, includeEmbeddings: true)308        #expect(chunks[0].embedding == nil)309        #expect([Float](blob: vector.blob) == vector)310        #expect([Float](blob: Data()) == [])311        // Embeddings are left out of the full export; the text is there.312        let export = try await store.fullExport()313        #expect(export.chunks.map(\.text) == ["with vector", "without vector"])314    }315316    @Test func addingADocumentFromTextChunksIt() async throws {317        let store = try makeStore()318        let text = (0..<30).map { "Paragraph \($0) talks about lighthouse number \($0) on the coast." }.joined(separator: "\n\n")319        let document = try await store.addDocument(name: "coast.txt", kind: "text", text: text, chunker: Chunker(targetSize: 300, overlap: 40))320        #expect(document.byteSize == text.utf8.count)321        #expect(document.sha256 == Hashing.sha256(Data(text.utf8)))322        let chunks = try await store.chunks(documentID: document.id)323        #expect(chunks.count > 4)324        #expect(chunks.allSatisfy { $0.text.count <= 300 })325        #expect(try await store.searchChunks("lighthouse number 17").first?.chunk.text.contains("lighthouse number 17") == true)326    }327}328329@Suite struct ChunkerTests {330    @Test func shortTextIsOneChunk() {331        #expect(Chunker().chunks("Hello world.") == ["Hello world."])332        #expect(Chunker().chunks("") == [])333        #expect(Chunker().chunks("  \n\n \t\n") == [])334        #expect(Chunker().chunks("\n\n  padded  \n\n") == ["padded"])335    }336337    @Test func parametersAreClamped() {338        #expect(Chunker(targetSize: 1, overlap: 500) == Chunker(targetSize: 16, overlap: 8))339        #expect(Chunker(targetSize: 100, overlap: -5).overlap == 0)340        #expect(Chunker().targetSize == 1_200)341    }342343    @Test func paragraphsArePackedWholeUpToTheTargetSize() {344        let paragraphs = (0..<10).map { "Paragraph \($0): " + String(repeating: "word ", count: 15).trimmingCharacters(in: .whitespaces) + "." }345        let text = paragraphs.joined(separator: "\n\n\n")   // several blank lines count as one break346        let chunks = Chunker(targetSize: 220, overlap: 0).chunks(text)347        #expect(chunks.count > 1)348        #expect(chunks.allSatisfy { $0.count <= 220 })349        // No paragraph is ever cut, none is lost, order is kept.350        #expect(chunks.flatMap { $0.components(separatedBy: "\n\n") } == paragraphs)351        #expect(chunks.allSatisfy { $0 == $0.trimmingCharacters(in: .whitespacesAndNewlines) })352        // Windows line endings and whitespace-only separator lines behave the same.353        #expect(Chunker(targetSize: 220, overlap: 0).chunks(paragraphs.joined(separator: "\r\n  \r\n")) == chunks)354        // Single line breaks stay inside a paragraph.355        #expect(Chunker().chunks("line one\nline two\n\nnext") == ["line one\nline two\n\nnext"])356        #expect(Chunker.paragraphs("a\nb\n\n\nc") == ["a\nb", "c"])357    }358359    @Test func overlapCarriesTheEndOfTheChunkIntoTheNext() {360        let paragraphs = (0..<12).map { "Section \($0) " + (0..<10).map { "token\($0)" }.joined(separator: " ") + " end\($0)." }361        let chunker = Chunker(targetSize: 260, overlap: 40)362        let chunks = chunker.chunks(paragraphs.joined(separator: "\n\n"))363        #expect(chunks.count > 2)364        #expect(chunks.allSatisfy { $0.count <= 260 })365        for (previous, next) in zip(chunks, chunks.dropFirst()) {366            // Each chunk starts with a tail of the previous one, cut at a word boundary and at most `overlap` long.367            let head = next.components(separatedBy: "\n\n")[0]368            if head.hasPrefix("Section") { continue }   // no room for the carried tail in this chunk369            #expect(previous.hasSuffix(head))370            #expect(head.count <= 40)371            #expect(!head.hasPrefix(" "))372        }373        #expect(zip(chunks, chunks.dropFirst()).contains { !$1.hasPrefix("Section") })374        // Every paragraph still appears whole somewhere.375        for paragraph in paragraphs { #expect(chunks.contains { $0.contains(paragraph) }) }376    }377378    @Test func aLongParagraphIsCutAtSentenceEnds() {379        let sentences = (0..<40).map { "Sentence number \($0) explains one more thing about tides." }380        let chunks = Chunker(targetSize: 200, overlap: 0).chunks(sentences.joined(separator: " "))381        #expect(chunks.count > 5)382        #expect(chunks.allSatisfy { $0.count <= 200 })383        #expect(chunks.allSatisfy { $0.hasSuffix(".") })384        #expect(chunks.allSatisfy { $0.hasPrefix("Sentence number") })385        #expect(chunks.joined(separator: " ") == sentences.joined(separator: " "))386    }387388    @Test func aLongParagraphWithOverlapRepeatsWholeWords() {389        let words = (0..<400).map { "w\($0)" }390        let chunker = Chunker(targetSize: 120, overlap: 30)391        let chunks = chunker.chunks(words.joined(separator: " "))392        #expect(chunks.count > 10)393        #expect(chunks.allSatisfy { $0.count <= 120 })394        var covered = Set<String>()395        for chunk in chunks {396            let parts = chunk.split(separator: " ").map(String.init)397            #expect(parts.allSatisfy(Set(words).contains), "A word was cut in \(chunk)")398            covered.formUnion(parts)399        }400        #expect(covered == Set(words))401        for (previous, next) in zip(chunks, chunks.dropFirst()) {402            let shared = Set(previous.split(separator: " ")).intersection(next.split(separator: " "))403            #expect(!shared.isEmpty)404            #expect(shared.count <= 8)405            // Progress: every chunk brings something new.406            #expect(next.split(separator: " ").last != previous.split(separator: " ").last)407        }408    }409410    @Test func textWithoutSpacesIsStillCutAndTerminates() {411        let blob = String(repeating: "x", count: 1000)412        let chunks = Chunker(targetSize: 100, overlap: 20).chunks(blob)413        #expect(chunks.allSatisfy { $0.count <= 100 && !$0.isEmpty })414        #expect(chunks.count >= 10 && chunks.count <= 14)415        #expect(Chunker(targetSize: 100, overlap: 0).chunks(blob).joined() == blob)416        // CJK text: cut at full stops, grapheme clusters never split.417        let cjk = String(repeating: "潮汐是由月球引力引起的。", count: 30)418        let cjkChunks = Chunker(targetSize: 50, overlap: 0).chunks(cjk)419        #expect(cjkChunks.joined() == cjk)420        #expect(cjkChunks.allSatisfy { $0.count <= 50 && $0.hasSuffix("。") })421        let emoji = String(repeating: "👨‍👩‍👧‍👦", count: 100)422        let emojiChunks = Chunker(targetSize: 16, overlap: 4).chunks(emoji)423        #expect(emojiChunks.allSatisfy { chunk in chunk.allSatisfy { $0 == "👨‍👩‍👧‍👦" } && chunk.count <= 16 })424    }425426    @Test func chunkingALargeDocumentIsFast() {427        let text = (0..<4000).map { "Paragraph \($0). " + String(repeating: "Some filler sentence about the sea. ", count: 8) }.joined(separator: "\n\n")428        let clock = ContinuousClock()429        var chunks: [String] = []430        let elapsed = clock.measure { chunks = Chunker().chunks(text) }431        print("Chunker: \(text.count) characters → \(chunks.count) chunks in \(elapsed)")432        #expect(chunks.count > 500)433        #expect(chunks.allSatisfy { $0.count <= 1_200 })434        #expect(elapsed < .seconds(2))435    }436}437
Files main
Menu

Theme