SPB Git

Repository Public

AssistantAskTests.swift 21.8 KB 367 lines swift

1// SP Notes Core2// File: Tests/SPNotesFeaturesTests/AssistantAskTests.swift3// Description: View models of « Demander à mes notes » and of the model picker against the fake backend: streamed grounded4//              answer with block citations, the server's evidence verdicts (none / insufficient), stream and pre-stream5//              errors, the expensive-request confirmation, stop, offline (draft kept), conversations; the catalogue grouped6//              by creator with routes, availability and prices; and the bundled creator / route logos.7//8// Author: Simon-Pierre Boucher9// Contact: contact@spboucher.ai10import Foundation11import SPNotesAPI12import SPNotesSync13import SPNotesTestSupport14import Testing15@testable import SPNotesFeatures16@testable import SPNotesFeaturesCatalog1718private let sseHeaders = ["content-type": "text/event-stream; charset=utf-8"]1920@Suite("Assistant: ask")21@MainActor22struct AssistantAskTests {23    @Test("nothing is sent before send(); a grounded answer streams, ends with the server's text, model, cost and a citation that opens its block")24    func grounded() async throws {25        let world = try await FeatureWorld()26        let router = FeatureRouterRecorder()27        let model = AskModel(repositories: world.repositories, router: router, context: AskContext(pageId: AssistantSamples.askPageId, pageTitle: "Offre de la banque"), scope: .page)28        await model.load()29        model.draft = "  Quelle est la pénalité de remboursement anticipé ?  "30        #expect(world.fake.count("POST", "^/api/ai/chat$") == 0)31        #expect(model.canSend)32        await model.send()33        #expect(world.fake.count("POST", "^/api/ai/chat$") == 1)34        let turn = try #require(model.turns.first)35        #expect(turn.state == .done)36        #expect(turn.question == "Quelle est la pénalité de remboursement anticipé ?")37        #expect(turn.answer == "La pénalité de remboursement anticipé équivaut à trois mois d'intérêts [1].")38        #expect(turn.model == "claude-haiku-5-5" && turn.provider == "anthropic")39        #expect(turn.evidence == .ok && turn.isModelOutput)40        #expect(turn.scope == .page)41        #expect(turn.costUsd != nil && AskModel.cost(turn.costUsd) == "0,0001 $")42        let citation = try #require(turn.citations.first)43        #expect(turn.citations.count == 1 && citation.n == 1 && citation.label == "Offre de la banque › Conditions")44        model.open(citation)45        #expect(router.calls == [.page(id: AssistantSamples.askPageId, blockId: AssistantSamples.askBlockId)])46        #expect(model.draft.isEmpty && !model.isStreaming && model.actionFailure == nil)47        #expect(model.conversationId == AssistantSamples.conversationId)48    }4950    @Test("the request carries the scope, the ids of the context, the chosen model and the conversation to continue")51    func requestBody() async throws {52        let world = try await FeatureWorld()53        let bodies = Bodies()54        world.fake.stub("POST", path: "^/api/ai/chat$") { request in55            bodies.add(request.body)56            return .init(body: AssistantSamples.groundedStream, headers: sseHeaders)57        }58        let context = AskContext(pageId: "p1", projectId: "proj", selection: "Un passage.", selectionBlockId: "b9")59        let model = AskModel(repositories: world.repositories, context: context, scope: .selection)60        #expect(model.availableScopes == [.all, .page, .project, .selection])61        #expect(model.scopeDetail(.selection) == "11 caractères")62        model.draft = "Que dit ce passage ?"63        await model.send()64        var body = try #require(bodies.all.last)65        #expect(body["scope"] as? String == "selection" && body["selection"] as? String == "Un passage." && body["selectionBlockId"] as? String == "b9")66        #expect(body["pageId"] as? String == "p1" && body["projectId"] == nil && body["model"] == nil && body["conversationId"] == nil)67        // second question: same conversation, project scope, an explicit model68        let catalog = ModelPickerModel(repositories: world.repositories)69        await catalog.load()70        model.selectModel(catalog.catalog?.models.first { $0.id == "claude-haiku-5-5" })71        model.selectModel(catalog.catalog?.models.first { !$0.isSelectable })           // refused: unavailable72        #expect(model.modelTitle == "Claude Haiku 5.5")73        model.setScope(.project)74        model.draft = "Et dans le projet ?"75        await model.send()76        body = try #require(bodies.all.last)77        #expect(body["scope"] as? String == "project" && body["projectId"] as? String == "proj" && body["selection"] == nil)78        #expect(body["model"] as? String == "claude-haiku-5-5" && body["conversationId"] as? String == AssistantSamples.conversationId)79        #expect(model.turns.count == 2)80    }8182    @Test("scopes are offered only when the host gave what they need")83    func scopes() async throws {84        let world = try await FeatureWorld(samples: false)85        let bare = AskModel(repositories: world.repositories, scope: .page)86        #expect(bare.availableScopes == [.all] && bare.scope == .all)87        bare.setScope(.selection)88        #expect(bare.scope == .all)89        let page = AskModel(repositories: world.repositories, context: AskContext(pageId: "p", selection: "   "))90        #expect(page.availableScopes == [.all, .page, .project])91        #expect(!page.canSend)92    }9394    @Test("evidence none: no model called, nothing spent; evidence insufficient: the withheld answer is said to be withheld")95    func evidence() async throws {96        let world = try await FeatureWorld()97        let model = AskModel(repositories: world.repositories, context: AskContext(pageId: "p", projectId: "proj"), scope: .project)98        model.draft = "Quel est le taux variable ?"99        await model.send()100        let none = try #require(model.turns.last)101        #expect(none.evidence == AskEvidence.none && none.model == nil && !none.isModelOutput && none.citations.isEmpty && none.state == .done)102        #expect(AskModel.cost(none.costUsd) == "0 $ (aucun appel de modèle)")103        model.setScope(.all)104        model.draft = "Réponds sans citer"105        await model.send()106        let withheld = try #require(model.turns.last)107        #expect(withheld.evidence == .insufficient && withheld.model == "claude-haiku-5-5" && withheld.citations.isEmpty && withheld.isModelOutput)108        #expect(AskEvidence("ok") == .ok && AskEvidence("partial") == .other("partial") && AskEvidence(nil) == nil)109    }110111    @Test("offline: a calm failure, the draft is kept, no turn is invented; back online the same draft goes")112    func offline() async throws {113        let world = try await FeatureWorld()114        let model = AskModel(repositories: world.repositories)115        model.draft = "Une question"116        world.setOffline(true)117        await model.send()118        #expect(model.actionFailure?.kind == .offline && model.draft == "Une question" && model.turns.isEmpty && !model.isStreaming)119        await model.load()120        #expect(model.phase.failure?.isOffline == true && model.canSend)121        world.setOffline(false)122        await model.send()123        #expect(model.actionFailure == nil && model.turns.count == 1 && model.draft.isEmpty)124    }125126    @Test("errors: before the stream (503 not_configured), inside it (page_required, not retryable), a stream cut before done (retry)")127    func errors() async throws {128        let world = try await FeatureWorld()129        let model = AskModel(repositories: world.repositories)130        world.fake.stub("POST", path: "^/api/ai/chat$") { _ in .error(503, code: "not_configured", message: "No provider key") }131        model.draft = "Question"132        await model.send()133        #expect(model.actionFailure?.kind == .server && model.actionFailure?.code == "not_configured" && model.turns.isEmpty && model.draft == "Question")134135        world.fake.stub("POST", path: "^/api/ai/chat$") { _ in .init(body: AssistantSamples.sse([AssistantSamples.event("chat-error-page-required")]), headers: sseHeaders) }136        await model.send()137        var turn = try #require(model.turns.last)138        #expect(turn.state == .failed(FeatureFailure(kind: .server, message: "Cette portée a besoin d'une page ouverte.", code: "page_required"), retryable: false))139140        // the server closes after one delta, without `done`: shown as interrupted, never as a complete answer141        world.fake.stub("POST", path: "^/api/ai/chat$") { _ in .init(body: AssistantSamples.sse([AssistantSamples.event("chat-page-meta"), AssistantSamples.text("La pén")]), headers: sseHeaders) }142        model.draft = "Autre question"143        await model.send()144        turn = try #require(model.turns.last)145        #expect(turn.state == .stopped && turn.answer == "La pén")146147        // nothing at all: a retryable failure; retry asks the same question again148        world.fake.stub("POST", path: "^/api/ai/chat$") { _ in .init(body: AssistantSamples.sse([]), headers: sseHeaders) }149        model.draft = "Encore"150        await model.send()151        turn = try #require(model.turns.last)152        guard case .failed(_, let retryable) = turn.state else { Issue.record("expected a failure"); return }153        #expect(retryable)154        world.fake.stub("POST", path: "^/api/ai/chat$") { _ in .init(body: AssistantSamples.groundedStream, headers: sseHeaders) }155        let before = model.turns.count156        await model.retry(turnId: turn.id)157        #expect(model.turns.count == before && model.turns.last?.state == .done && model.turns.last?.question == "Encore")158    }159160    @Test("an expensive request is announced (nothing sent to a model), then sent only on confirmation")161    func confirmation() async throws {162        let world = try await FeatureWorld()163        let model = AskModel(repositories: world.repositories)164        model.draft = "Résume tout le dossier"165        await model.send()166        let pending = try #require(model.confirmation)167        #expect(pending.estimatedCostUsd == 1.84 && pending.model == "claude-fable-5-1" && pending.question == "Résume tout le dossier")168        #expect(model.turns.isEmpty && model.draft == "Résume tout le dossier")169        model.cancelConfirmation()170        #expect(model.confirmation == nil)171        await model.send()172        await model.confirmExpensive()173        #expect(model.confirmation == nil && model.turns.count == 1 && model.turns[0].state == .done && model.draft.isEmpty)174        #expect(world.fake.count("POST", "^/api/ai/chat$") == 3)175176        // the same announcement arriving as a stream event177        world.fake.stub("POST", path: "^/api/ai/chat$") { _ in178            .init(body: AssistantSamples.sse([#"{"type":"needs_confirmation","reason":"long context","plan":{"estCostUsd":0.5}}"#]), headers: sseHeaders)179        }180        model.draft = "Autre"181        await model.send()182        #expect(model.confirmation?.reason == "long context" && model.turns.count == 1 && model.draft == "Autre")183    }184185    @Test("stop before the answer starts: nothing is kept and the draft stays")186    func stop() async throws {187        let world = try await FeatureWorld()188        let model = AskModel(repositories: world.repositories)189        let gate = Gate()190        world.fake.inject(.hold(gate), method: "POST", path: "^/api/ai/chat$")191        model.draft = "Question lente"192        let sending = Task { await model.send() }193        await gate.reached()194        #expect(model.isStreaming)195        model.stop()196        gate.open()197        await sending.value198        #expect(!model.isStreaming && model.turns.isEmpty && model.draft == "Question lente" && model.actionFailure == nil)199    }200201    @Test("conversations: listed newest first, opened as turns and continued, renamed, deleted with undo; unsupported and empty servers")202    func conversations() async throws {203        let world = try await FeatureWorld()204        let router = FeatureRouterRecorder()205        let model = AskModel(repositories: world.repositories, router: router)206        await model.load()207        #expect(model.phase == .loaded && model.conversations.count == 3)208        #expect(model.conversations.map(AskModel.title) == ["Pénalité de remboursement anticipé", "Hypothèses du modèle GARCH", "Conversation sans titre"])209        await model.open(conversation: model.conversations[0])210        #expect(model.conversationId == AssistantSamples.conversationId && model.turns.count == 1)211        let turn = model.turns[0]212        #expect(turn.question.hasPrefix("Quelle est") && turn.state == .done && !turn.isLive && turn.evidence == .ok && turn.model == "claude-haiku-5-5")213        #expect(turn.citations.count == 1 && turn.citations[0].n == nil)                // stored sources carry no marker number214        model.open(turn.citations[0])215        #expect(router.calls == [.page(id: AssistantSamples.askPageId, blockId: AssistantSamples.askBlockId)])216        await model.rename(conversation: model.conversations[1], title: "  GARCH  ")217        #expect(model.conversations[1].title == "GARCH")218        let first = model.conversations[0]219        await model.delete(conversation: first)220        #expect(model.conversations.count == 2 && model.lastDeleted?.id == first.id && model.conversationId == nil && model.turns.isEmpty)221        await model.undoDelete()222        #expect(model.lastDeleted == nil && model.conversations.count == 3)223        #expect(world.fake.count("DELETE", "^/api/ai/conversations") == 2)224        model.newConversation()225        #expect(model.turns.isEmpty && model.conversationId == nil)226227        world.setOffline(true)228        await model.delete(conversation: model.conversations[0])229        #expect(model.actionFailure?.kind == .offline && model.conversations.count == 3)230231        let empty = try await FeatureWorld(samples: false)232        empty.fake.stub("GET", path: "^/api/ai/conversations$", json: "[]")233        let none = AskModel(repositories: empty.repositories)234        await none.load()235        #expect(none.phase == .loaded && none.conversations.isEmpty && !none.historyUnsupported)236        empty.fake.stub("GET", path: "^/api/ai/conversations$") { _ in .error(501, code: "not_implemented") }237        await none.load()238        #expect(none.phase == .loaded && none.historyUnsupported)239        empty.fake.stub("GET", path: "^/api/ai/conversations$") { _ in .error(500, code: "server_error", message: "boom") }240        await none.load()241        #expect(none.phase.failure?.kind == .server)242    }243}244245/// Bodies of the stubbed requests, in order.246final class Bodies: @unchecked Sendable {247    private let lock = NSLock()248    private var items: [[String: Any]] = []249    func add(_ data: Data) {250        let object = (try? JSONSerialization.jsonObject(with: data) as? [String: Any]) ?? [:]251        lock.withLock { items.append(object) }252    }253    var all: [[String: Any]] { lock.withLock { items } }254}255256@Suite("Assistant: model picker and logos")257@MainActor258struct AssistantModelPickerTests {259    @Test("the catalogue is grouped by creator, the route is separate, an unavailable model is listed but cannot be chosen")260    func happy() async throws {261        let world = try await FeatureWorld()262        let chosen = Chosen()263        let model = ModelPickerModel(repositories: world.repositories, selectedId: "claude-haiku-5-5", onSelect: { chosen.values.append($0?.id ?? "auto") })264        await model.load()265        #expect(model.phase == .loaded && model.freshness?.fromCache == false)266        #expect(model.groups.map(\.id) == ["anthropic", "openai"])267        #expect(model.groups.map(\.name) == ["Anthropic", "OpenAI"])268        let anthropic = model.groups[0]269        #expect(anthropic.models.count == 5 && anthropic.models.last?.id == "openrouter:anthropic/claude-fable-5")   // unavailable last270        let viaRouter = try #require(anthropic.models.last)271        #expect(!viaRouter.isSelectable && ModelPickerModel.isIndirectRoute(viaRouter) && ModelPickerModel.routeLabel(viaRouter) == "via OpenRouter")272        #expect(ModelPickerModel.unavailableReason(viaRouter)?.contains("aucune clé") == true)273        #expect(ModelPickerModel.manualOnlyNote(viaRouter) != nil)274        #expect(!model.select(viaRouter) && model.selectedId == "claude-haiku-5-5" && model.actionFailure != nil && chosen.values.isEmpty)275        let haiku = try #require(model.selected)276        #expect(!ModelPickerModel.isIndirectRoute(haiku) && ModelPickerModel.routeLabel(haiku) == "via Anthropic API")277        #expect(ModelPickerModel.contextLabel(haiku) == "Contexte 1 M")278        #expect(ModelPickerModel.priceLabel(haiku) == "0,1 $ entrée · 0,5 $ sortie / M jetons")279        #expect(ModelPickerModel.capabilities(haiku).map(\.label) == ["Vision", "Outils", "Raisonnement", "Sortie structurée"])280        #expect(ModelPickerModel.capabilitiesMeasured(haiku) == true && ModelPickerModel.capabilitiesMeasured(viaRouter) == false)281        #expect(ModelPickerModel.accessibilityLabel(viaRouter, creator: "Anthropic").contains("Indisponible"))282        let sol = try #require(model.groups[1].models.first { $0.id == "gpt-6.1-sol" })283        #expect(model.select(sol) && model.selectedId == "gpt-6.1-sol" && model.actionFailure == nil)284        #expect(model.select(nil) && model.selectedId == nil && model.selected == nil)285        #expect(chosen.values == ["gpt-6.1-sol", "auto"])286287        model.searchText = "openrouter"288        #expect(model.groups.count == 1 && model.groups[0].models.map(\.id) == ["openrouter:anthropic/claude-fable-5"])289        model.showsUnavailable = false290        #expect(model.isEmpty)291        model.searchText = ""292        #expect(model.groups[0].models.count == 4)293    }294295    @Test("offline: the cached catalogue with its date; never fetched: not cached; server error; empty catalogue; a 2.x catalogue groups by provider")296    func failures() async throws {297        let world = try await FeatureWorld()298        let model = ModelPickerModel(repositories: world.repositories)299        await model.load()300        world.setOffline(true)301        await model.load(forceRefresh: true)302        #expect(model.phase == .loaded && model.freshness?.fromCache == true && model.groups.count == 2)303304        let cold = try await FeatureWorld(samples: false)305        let empty = ModelPickerModel(repositories: cold.repositories)306        cold.setOffline(true)307        await empty.load()308        #expect(empty.phase.failure?.kind == .notCached)309        cold.setOffline(false)310        cold.fake.stub("GET", path: "^/api/ai/models$") { _ in .error(500, code: "server_error", message: "boom") }311        await empty.load()312        #expect(empty.phase.failure?.kind == .server)313        cold.fake.stub("GET", path: "^/api/ai/models$", json: #"{"models":[],"providers":[]}"#)314        await empty.load()315        #expect(empty.phase == .loaded && empty.isEmpty)316        cold.fake.stub("GET", path: "^/api/ai/models$", json: #"{"models":[{"id":"m1","provider":"openai","available":true},{"id":"m2","available":true}],"providers":[]}"#)317        await empty.load(forceRefresh: true)318        #expect(empty.groups.map(\.id) == ["openai", "unknown"] && empty.groups.map(\.name) == ["Openai", "Créateur inconnu"])319        #expect(ModelPickerModel.priceLabel(empty.groups[0].models[0]) == nil && ModelPickerModel.capabilities(empty.groups[0].models[0]).isEmpty)320    }321322    @Test("numbers: tokens and prices")323    func numbers() {324        #expect(ModelPickerModel.tokens(1_000_000) == "1 M" && ModelPickerModel.tokens(1_500_000) == "1,5 M" && ModelPickerModel.tokens(200_000) == "200 k")325        #expect(ModelPickerModel.tokens(8192) == "8192" && ModelPickerModel.tokens(0) == nil && ModelPickerModel.tokens(nil) == nil)326        #expect(ModelPickerModel.price(10) == "10" && ModelPickerModel.price(0.25) == "0,25" && ModelPickerModel.price(2.5) == "2,5" && ModelPickerModel.price(0.05) == "0,05")327    }328329    @Test("logos: every creator and route of the registry resolves to a bundled image; unknown creators get the neutral fallback")330    func logos() throws {331        let registry = ModelLogoRegistry.shared332        #expect(registry.entries.count >= 28 && registry.source?.hasPrefix("simple-icons@") == true)333        for entry in registry.entries {334            let logo = registry.resolve(id: entry.id, kind: entry.kind)335            #expect(logo.isBundled, "no bundled image for \(entry.kind) \(entry.id) (\(logo.imageName))")336            #expect(logo.isGeneric == entry.generic, "\(entry.id)")337            #expect(logo.imageName == ModelLogoRegistry.imageName(file: entry.file))338        }339        let anthropic = registry.resolve(id: "anthropic")340        #expect(anthropic.imageName == "ModelLogo-anthropic" && !anthropic.isGeneric && anthropic.monogram == "AN")341        let openai = registry.resolve(id: "OpenAI")                                     // no mark shipped: generic + monogram342        #expect(openai.isGeneric && openai.imageName == registry.fallbackImage && openai.monogram == "OA")343        let unknown = registry.resolve(id: "acme-labs", logoPath: "/model-logos/acme.svg")344        #expect(unknown.isGeneric && unknown.imageName == "ModelLogo-generic" && unknown.monogram == "AC" && unknown.isBundled)345        #expect(registry.resolve(id: nil).isGeneric && registry.resolve(id: nil).monogram == "AI")346        // a creator the bundled registry does not know, whose file is shipped: the server's path is honoured347        let alias = registry.resolve(id: "google-deepmind", logoPath: "/model-logos/google.svg", monogram: "GD")348        #expect(!alias.isGeneric && alias.imageName == "ModelLogo-google" && alias.monogram == "GD")349        let route = registry.resolve(id: "openrouter", kind: "route")350        #expect(!route.isGeneric && route.imageName == "ModelLogo-openrouter")351        #expect(registry.resolve(id: "maclustr", kind: "route").isGeneric)352        // every creator of the sample catalogue (the live answer) is known to the bundled registry353        let catalog = try JSONDecoder().decode(CreatorList.self, from: Data(FeatureSamples.text("ai", "models-catalog").utf8))354        for creator in catalog.creators { #expect(registry.entry(id: creator.id, kind: "creator") != nil, "\(creator.id)") }355    }356}357358private struct CreatorList: Decodable {359    struct Creator: Decodable { var id: String }360    var creators: [Creator]361}362363@MainActor364final class Chosen {365    var values: [String] = []366}367
Files main
Menu

Theme