SPB Git

Repository Public

EndToEndTests.swift 6.9 KB 122 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatAppTests/EndToEndTests.swift3// Description: End-to-end test of the application layer on a throw-away store: real Hugging Face download, real MLX chat through ChatController, branching, knowledge retrieval.4import Foundation5import Testing6@testable import LocalChatUI7import LocalChatConversations8import LocalChatCore9import LocalChatKnowledge10import LocalChatModelHub1112@MainActor func waitUntil(timeout: TimeInterval, _ condition: @MainActor () -> Bool) async -> Bool {13    let deadline = Date().addingTimeInterval(timeout)14    while Date() < deadline { if condition() { return true }; try? await Task.sleep(for: .milliseconds(150)) }15    return condition()16}1718@Suite(.serialized) @MainActor struct EndToEnd {19    static let repo = "mlx-community/Qwen3-0.6B-4bit"2021    /// One shared app per process: the model is downloaded once (kept between runs in a cache folder so reruns do not download again).22    static let root: URL = {23        let base = URL(fileURLWithPath: NSHomeDirectory()).appendingPathComponent("Library/Caches/LocalChatTests/app-e2e")24        // conversations start fresh on every run; the Models folder is kept25        for name in ["Data", "Attachments", "AgentRuns"] { try? FileManager.default.removeItem(at: base.appendingPathComponent(name)) }26        return base27    }()2829    @Test func downloadChatBranchAndRetrieve() async throws {30        let app = try AppModel(root: Self.root)31        await app.bootstrap()32        #expect(app.isReady)3334        // 1. real download through the hub + download manager (skipped when a previous run already installed it)35        if !app.installedModels.contains(where: { $0.repoID == Self.repo }) {36            let hubModel = try await app.hub.model(Self.repo)37            let candidate = try #require(ModelMetadataResolver().candidates(for: hubModel).first)38            #expect(candidate.descriptor.backend == .mlx)39            #expect(candidate.downloadBytes > 100_000_000)40            let request = try #require(DownloadRequest(candidate: candidate))41            try await app.downloads.enqueue(request)42            let state = await app.downloads.waitUntilSettled(id: request.id)43            print("E2E download:", state?.phase as Any, state?.bytesDone as Any)44            #expect(state?.phase == .completed)45            await app.refreshModels()46        }47        let model = try #require(app.installedModels.first { $0.repoID == Self.repo })48        print("E2E model:", model.name, model.quantization as Any, model.sizeBytes as Any, model.architecture as Any, app.compatibility(model).fit)49        #expect(app.compatibility(model).canAttemptLoad)5051        // 2. a real conversation through the controller the views use52        var conversation = app.newConversation()53        conversation.modelID = model.id; conversation.settings.reasoningEnabled = false; conversation.settings.temperature = 0; conversation.settings.maxTokens = 6454        app.update(conversation)55        let chat = try #require(app.controller(for: conversation.id))56        chat.update { $0.modelID = model.id; $0.settings = conversation.settings }57        chat.draft = "What is the capital of France? Answer in one short sentence."58        chat.send()59        #expect(chat.isGenerating)60        #expect(await waitUntil(timeout: 120) { !chat.isGenerating })61        #expect(chat.errorText == nil)62        #expect(chat.messages.count == 2)63        let answer = try #require(chat.messages.last)64        print("E2E answer:", answer.content, answer.metrics as Any)65        #expect(answer.role == .assistant && answer.status == .complete)66        #expect(answer.content.lowercased().contains("paris"))67        #expect(answer.modelID == model.id && answer.backend == .mlx)68        #expect((answer.metrics?.tokensPerSecond ?? 0) > 1)69        #expect(app.snapshot.isLoaded(model.id) || app.preferences.keepModelLoadedMinutes == 0)70        #expect(await waitUntil(timeout: 30) { chat.conversation.title != "New chat" })71        print("E2E title:", chat.conversation.title)72        #expect(chat.context != nil)7374        // 3. regenerate → a sibling branch, the first answer is kept75        chat.regenerate(answer.id)76        #expect(await waitUntil(timeout: 120) { !chat.isGenerating })77        let second = try #require(chat.messages.last)78        #expect(second.id != answer.id)79        #expect(chat.tree.branchPosition(of: second.id).count == 2)80        chat.switchBranch(from: second.id, offset: -1)81        #expect(chat.messages.last?.id == answer.id)8283        // 4. second turn (KV cache reuse inside the engine) and full-text search84        chat.switchBranch(from: answer.id, offset: 1)85        chat.draft = "And the capital of Italy?"86        chat.send()87        #expect(await waitUntil(timeout: 120) { !chat.isGenerating })88        print("E2E turn 2:", chat.messages.map { "\($0.role.rawValue): \($0.content)" })89        #expect(chat.messages.count == 4)90        #expect(chat.messages.last?.content.lowercased().contains("rome") == true)91        #expect(try app.store.search("Italy").isEmpty == false)92        #expect(try app.metrics.records(modelID: model.id).count >= 3)9394        // 5. knowledge: index a document with the system embedder, then answer from it95        if let provider = AppleNLEmbeddingProvider(language: .english) {96            let collection = KnowledgeCollection(name: "E2E", embedderID: provider.identifier, dimension: provider.dimension)97            try app.knowledgeStore.saveCollection(collection)98            let text = "Project Heron internal memo. The launch code name of the Heron satellite is BLUE-KESTREL-42. The launch window opens on March 3."99            let document = try await app.indexer.add(data: Data(text.utf8), name: "heron.txt", to: collection.id, provider: provider)100            #expect(document.status == .ready && document.chunkCount >= 1)101            app.reloadLists()102            var grounded = app.newConversation()103            grounded.modelID = model.id; grounded.settings = conversation.settings; grounded.collectionIDs = [collection.id]104            app.update(grounded)105            let ragChat = try #require(app.controller(for: grounded.id))106            ragChat.draft = "What is the launch code name of the Heron satellite?"107            ragChat.send()108            #expect(await waitUntil(timeout: 120) { !ragChat.isGenerating })109            let ragAnswer = try #require(ragChat.messages.last)110            print("E2E RAG answer:", ragAnswer.content, "sources:", ragAnswer.sources.count)111            #expect(!ragAnswer.sources.isEmpty)112            #expect(ragAnswer.content.uppercased().contains("BLUE-KESTREL-42"))113        } else { print("SKIPPED knowledge step: no system sentence embedding") }114115        // 6. export round trip116        let markdown = chat.exportMarkdown()117        #expect(markdown.contains("Paris") || markdown.contains("paris"))118        await app.engine.unloadAll()119        #expect(await waitUntil(timeout: 10) { app.snapshot.loaded.isEmpty })120    }121}122
Files main
Menu

Theme