Repository Public
1// SP Notes Core2// File: Tests/SPNotesFeaturesTests/AssistantAskTests.swift3// Description: View models of « Demander à mes notes » and of the model picker against the fake backend: streamed grounded4// answer with block citations, the server's evidence verdicts (none / insufficient), stream and pre-stream5// errors, the expensive-request confirmation, stop, offline (draft kept), conversations; the catalogue grouped6// by creator with routes, availability and prices; and the bundled creator / route logos.7//8// Author: Simon-Pierre Boucher9// Contact: contact@spboucher.ai10import Foundation11import SPNotesAPI12import SPNotesSync13import SPNotesTestSupport14import Testing15@testable import SPNotesFeatures16@testable import SPNotesFeaturesCatalog1718private let sseHeaders = ["content-type": "text/event-stream; charset=utf-8"]1920@Suite("Assistant: ask")21@MainActor22struct AssistantAskTests {23 @Test("nothing is sent before send(); a grounded answer streams, ends with the server's text, model, cost and a citation that opens its block")24 func grounded() async throws {25 let world = try await FeatureWorld()26 let router = FeatureRouterRecorder()27 let model = AskModel(repositories: world.repositories, router: router, context: AskContext(pageId: AssistantSamples.askPageId, pageTitle: "Offre de la banque"), scope: .page)28 await model.load()29 model.draft = " Quelle est la pénalité de remboursement anticipé ? "30 #expect(world.fake.count("POST", "^/api/ai/chat$") == 0)31 #expect(model.canSend)32 await model.send()33 #expect(world.fake.count("POST", "^/api/ai/chat$") == 1)34 let turn = try #require(model.turns.first)35 #expect(turn.state == .done)36 #expect(turn.question == "Quelle est la pénalité de remboursement anticipé ?")37 #expect(turn.answer == "La pénalité de remboursement anticipé équivaut à trois mois d'intérêts [1].")38 #expect(turn.model == "claude-haiku-5-5" && turn.provider == "anthropic")39 #expect(turn.evidence == .ok && turn.isModelOutput)40 #expect(turn.scope == .page)41 #expect(turn.costUsd != nil && AskModel.cost(turn.costUsd) == "0,0001 $")42 let citation = try #require(turn.citations.first)43 #expect(turn.citations.count == 1 && citation.n == 1 && citation.label == "Offre de la banque › Conditions")44 model.open(citation)45 #expect(router.calls == [.page(id: AssistantSamples.askPageId, blockId: AssistantSamples.askBlockId)])46 #expect(model.draft.isEmpty && !model.isStreaming && model.actionFailure == nil)47 #expect(model.conversationId == AssistantSamples.conversationId)48 }4950 @Test("the request carries the scope, the ids of the context, the chosen model and the conversation to continue")51 func requestBody() async throws {52 let world = try await FeatureWorld()53 let bodies = Bodies()54 world.fake.stub("POST", path: "^/api/ai/chat$") { request in55 bodies.add(request.body)56 return .init(body: AssistantSamples.groundedStream, headers: sseHeaders)57 }58 let context = AskContext(pageId: "p1", projectId: "proj", selection: "Un passage.", selectionBlockId: "b9")59 let model = AskModel(repositories: world.repositories, context: context, scope: .selection)60 #expect(model.availableScopes == [.all, .page, .project, .selection])61 #expect(model.scopeDetail(.selection) == "11 caractères")62 model.draft = "Que dit ce passage ?"63 await model.send()64 var body = try #require(bodies.all.last)65 #expect(body["scope"] as? String == "selection" && body["selection"] as? String == "Un passage." && body["selectionBlockId"] as? String == "b9")66 #expect(body["pageId"] as? String == "p1" && body["projectId"] == nil && body["model"] == nil && body["conversationId"] == nil)67 // second question: same conversation, project scope, an explicit model68 let catalog = ModelPickerModel(repositories: world.repositories)69 await catalog.load()70 model.selectModel(catalog.catalog?.models.first { $0.id == "claude-haiku-5-5" })71 model.selectModel(catalog.catalog?.models.first { !$0.isSelectable }) // refused: unavailable72 #expect(model.modelTitle == "Claude Haiku 5.5")73 model.setScope(.project)74 model.draft = "Et dans le projet ?"75 await model.send()76 body = try #require(bodies.all.last)77 #expect(body["scope"] as? String == "project" && body["projectId"] as? String == "proj" && body["selection"] == nil)78 #expect(body["model"] as? String == "claude-haiku-5-5" && body["conversationId"] as? String == AssistantSamples.conversationId)79 #expect(model.turns.count == 2)80 }8182 @Test("scopes are offered only when the host gave what they need")83 func scopes() async throws {84 let world = try await FeatureWorld(samples: false)85 let bare = AskModel(repositories: world.repositories, scope: .page)86 #expect(bare.availableScopes == [.all] && bare.scope == .all)87 bare.setScope(.selection)88 #expect(bare.scope == .all)89 let page = AskModel(repositories: world.repositories, context: AskContext(pageId: "p", selection: " "))90 #expect(page.availableScopes == [.all, .page, .project])91 #expect(!page.canSend)92 }9394 @Test("evidence none: no model called, nothing spent; evidence insufficient: the withheld answer is said to be withheld")95 func evidence() async throws {96 let world = try await FeatureWorld()97 let model = AskModel(repositories: world.repositories, context: AskContext(pageId: "p", projectId: "proj"), scope: .project)98 model.draft = "Quel est le taux variable ?"99 await model.send()100 let none = try #require(model.turns.last)101 #expect(none.evidence == AskEvidence.none && none.model == nil && !none.isModelOutput && none.citations.isEmpty && none.state == .done)102 #expect(AskModel.cost(none.costUsd) == "0 $ (aucun appel de modèle)")103 model.setScope(.all)104 model.draft = "Réponds sans citer"105 await model.send()106 let withheld = try #require(model.turns.last)107 #expect(withheld.evidence == .insufficient && withheld.model == "claude-haiku-5-5" && withheld.citations.isEmpty && withheld.isModelOutput)108 #expect(AskEvidence("ok") == .ok && AskEvidence("partial") == .other("partial") && AskEvidence(nil) == nil)109 }110111 @Test("offline: a calm failure, the draft is kept, no turn is invented; back online the same draft goes")112 func offline() async throws {113 let world = try await FeatureWorld()114 let model = AskModel(repositories: world.repositories)115 model.draft = "Une question"116 world.setOffline(true)117 await model.send()118 #expect(model.actionFailure?.kind == .offline && model.draft == "Une question" && model.turns.isEmpty && !model.isStreaming)119 await model.load()120 #expect(model.phase.failure?.isOffline == true && model.canSend)121 world.setOffline(false)122 await model.send()123 #expect(model.actionFailure == nil && model.turns.count == 1 && model.draft.isEmpty)124 }125126 @Test("errors: before the stream (503 not_configured), inside it (page_required, not retryable), a stream cut before done (retry)")127 func errors() async throws {128 let world = try await FeatureWorld()129 let model = AskModel(repositories: world.repositories)130 world.fake.stub("POST", path: "^/api/ai/chat$") { _ in .error(503, code: "not_configured", message: "No provider key") }131 model.draft = "Question"132 await model.send()133 #expect(model.actionFailure?.kind == .server && model.actionFailure?.code == "not_configured" && model.turns.isEmpty && model.draft == "Question")134135 world.fake.stub("POST", path: "^/api/ai/chat$") { _ in .init(body: AssistantSamples.sse([AssistantSamples.event("chat-error-page-required")]), headers: sseHeaders) }136 await model.send()137 var turn = try #require(model.turns.last)138 #expect(turn.state == .failed(FeatureFailure(kind: .server, message: "Cette portée a besoin d'une page ouverte.", code: "page_required"), retryable: false))139140 // the server closes after one delta, without `done`: shown as interrupted, never as a complete answer141 world.fake.stub("POST", path: "^/api/ai/chat$") { _ in .init(body: AssistantSamples.sse([AssistantSamples.event("chat-page-meta"), AssistantSamples.text("La pén")]), headers: sseHeaders) }142 model.draft = "Autre question"143 await model.send()144 turn = try #require(model.turns.last)145 #expect(turn.state == .stopped && turn.answer == "La pén")146147 // nothing at all: a retryable failure; retry asks the same question again148 world.fake.stub("POST", path: "^/api/ai/chat$") { _ in .init(body: AssistantSamples.sse([]), headers: sseHeaders) }149 model.draft = "Encore"150 await model.send()151 turn = try #require(model.turns.last)152 guard case .failed(_, let retryable) = turn.state else { Issue.record("expected a failure"); return }153 #expect(retryable)154 world.fake.stub("POST", path: "^/api/ai/chat$") { _ in .init(body: AssistantSamples.groundedStream, headers: sseHeaders) }155 let before = model.turns.count156 await model.retry(turnId: turn.id)157 #expect(model.turns.count == before && model.turns.last?.state == .done && model.turns.last?.question == "Encore")158 }159160 @Test("an expensive request is announced (nothing sent to a model), then sent only on confirmation")161 func confirmation() async throws {162 let world = try await FeatureWorld()163 let model = AskModel(repositories: world.repositories)164 model.draft = "Résume tout le dossier"165 await model.send()166 let pending = try #require(model.confirmation)167 #expect(pending.estimatedCostUsd == 1.84 && pending.model == "claude-fable-5-1" && pending.question == "Résume tout le dossier")168 #expect(model.turns.isEmpty && model.draft == "Résume tout le dossier")169 model.cancelConfirmation()170 #expect(model.confirmation == nil)171 await model.send()172 await model.confirmExpensive()173 #expect(model.confirmation == nil && model.turns.count == 1 && model.turns[0].state == .done && model.draft.isEmpty)174 #expect(world.fake.count("POST", "^/api/ai/chat$") == 3)175176 // the same announcement arriving as a stream event177 world.fake.stub("POST", path: "^/api/ai/chat$") { _ in178 .init(body: AssistantSamples.sse([#"{"type":"needs_confirmation","reason":"long context","plan":{"estCostUsd":0.5}}"#]), headers: sseHeaders)179 }180 model.draft = "Autre"181 await model.send()182 #expect(model.confirmation?.reason == "long context" && model.turns.count == 1 && model.draft == "Autre")183 }184185 @Test("stop before the answer starts: nothing is kept and the draft stays")186 func stop() async throws {187 let world = try await FeatureWorld()188 let model = AskModel(repositories: world.repositories)189 let gate = Gate()190 world.fake.inject(.hold(gate), method: "POST", path: "^/api/ai/chat$")191 model.draft = "Question lente"192 let sending = Task { await model.send() }193 await gate.reached()194 #expect(model.isStreaming)195 model.stop()196 gate.open()197 await sending.value198 #expect(!model.isStreaming && model.turns.isEmpty && model.draft == "Question lente" && model.actionFailure == nil)199 }200201 @Test("conversations: listed newest first, opened as turns and continued, renamed, deleted with undo; unsupported and empty servers")202 func conversations() async throws {203 let world = try await FeatureWorld()204 let router = FeatureRouterRecorder()205 let model = AskModel(repositories: world.repositories, router: router)206 await model.load()207 #expect(model.phase == .loaded && model.conversations.count == 3)208 #expect(model.conversations.map(AskModel.title) == ["Pénalité de remboursement anticipé", "Hypothèses du modèle GARCH", "Conversation sans titre"])209 await model.open(conversation: model.conversations[0])210 #expect(model.conversationId == AssistantSamples.conversationId && model.turns.count == 1)211 let turn = model.turns[0]212 #expect(turn.question.hasPrefix("Quelle est") && turn.state == .done && !turn.isLive && turn.evidence == .ok && turn.model == "claude-haiku-5-5")213 #expect(turn.citations.count == 1 && turn.citations[0].n == nil) // stored sources carry no marker number214 model.open(turn.citations[0])215 #expect(router.calls == [.page(id: AssistantSamples.askPageId, blockId: AssistantSamples.askBlockId)])216 await model.rename(conversation: model.conversations[1], title: " GARCH ")217 #expect(model.conversations[1].title == "GARCH")218 let first = model.conversations[0]219 await model.delete(conversation: first)220 #expect(model.conversations.count == 2 && model.lastDeleted?.id == first.id && model.conversationId == nil && model.turns.isEmpty)221 await model.undoDelete()222 #expect(model.lastDeleted == nil && model.conversations.count == 3)223 #expect(world.fake.count("DELETE", "^/api/ai/conversations") == 2)224 model.newConversation()225 #expect(model.turns.isEmpty && model.conversationId == nil)226227 world.setOffline(true)228 await model.delete(conversation: model.conversations[0])229 #expect(model.actionFailure?.kind == .offline && model.conversations.count == 3)230231 let empty = try await FeatureWorld(samples: false)232 empty.fake.stub("GET", path: "^/api/ai/conversations$", json: "[]")233 let none = AskModel(repositories: empty.repositories)234 await none.load()235 #expect(none.phase == .loaded && none.conversations.isEmpty && !none.historyUnsupported)236 empty.fake.stub("GET", path: "^/api/ai/conversations$") { _ in .error(501, code: "not_implemented") }237 await none.load()238 #expect(none.phase == .loaded && none.historyUnsupported)239 empty.fake.stub("GET", path: "^/api/ai/conversations$") { _ in .error(500, code: "server_error", message: "boom") }240 await none.load()241 #expect(none.phase.failure?.kind == .server)242 }243}244245/// Bodies of the stubbed requests, in order.246final class Bodies: @unchecked Sendable {247 private let lock = NSLock()248 private var items: [[String: Any]] = []249 func add(_ data: Data) {250 let object = (try? JSONSerialization.jsonObject(with: data) as? [String: Any]) ?? [:]251 lock.withLock { items.append(object) }252 }253 var all: [[String: Any]] { lock.withLock { items } }254}255256@Suite("Assistant: model picker and logos")257@MainActor258struct AssistantModelPickerTests {259 @Test("the catalogue is grouped by creator, the route is separate, an unavailable model is listed but cannot be chosen")260 func happy() async throws {261 let world = try await FeatureWorld()262 let chosen = Chosen()263 let model = ModelPickerModel(repositories: world.repositories, selectedId: "claude-haiku-5-5", onSelect: { chosen.values.append($0?.id ?? "auto") })264 await model.load()265 #expect(model.phase == .loaded && model.freshness?.fromCache == false)266 #expect(model.groups.map(\.id) == ["anthropic", "openai"])267 #expect(model.groups.map(\.name) == ["Anthropic", "OpenAI"])268 let anthropic = model.groups[0]269 #expect(anthropic.models.count == 5 && anthropic.models.last?.id == "openrouter:anthropic/claude-fable-5") // unavailable last270 let viaRouter = try #require(anthropic.models.last)271 #expect(!viaRouter.isSelectable && ModelPickerModel.isIndirectRoute(viaRouter) && ModelPickerModel.routeLabel(viaRouter) == "via OpenRouter")272 #expect(ModelPickerModel.unavailableReason(viaRouter)?.contains("aucune clé") == true)273 #expect(ModelPickerModel.manualOnlyNote(viaRouter) != nil)274 #expect(!model.select(viaRouter) && model.selectedId == "claude-haiku-5-5" && model.actionFailure != nil && chosen.values.isEmpty)275 let haiku = try #require(model.selected)276 #expect(!ModelPickerModel.isIndirectRoute(haiku) && ModelPickerModel.routeLabel(haiku) == "via Anthropic API")277 #expect(ModelPickerModel.contextLabel(haiku) == "Contexte 1 M")278 #expect(ModelPickerModel.priceLabel(haiku) == "0,1 $ entrée · 0,5 $ sortie / M jetons")279 #expect(ModelPickerModel.capabilities(haiku).map(\.label) == ["Vision", "Outils", "Raisonnement", "Sortie structurée"])280 #expect(ModelPickerModel.capabilitiesMeasured(haiku) == true && ModelPickerModel.capabilitiesMeasured(viaRouter) == false)281 #expect(ModelPickerModel.accessibilityLabel(viaRouter, creator: "Anthropic").contains("Indisponible"))282 let sol = try #require(model.groups[1].models.first { $0.id == "gpt-6.1-sol" })283 #expect(model.select(sol) && model.selectedId == "gpt-6.1-sol" && model.actionFailure == nil)284 #expect(model.select(nil) && model.selectedId == nil && model.selected == nil)285 #expect(chosen.values == ["gpt-6.1-sol", "auto"])286287 model.searchText = "openrouter"288 #expect(model.groups.count == 1 && model.groups[0].models.map(\.id) == ["openrouter:anthropic/claude-fable-5"])289 model.showsUnavailable = false290 #expect(model.isEmpty)291 model.searchText = ""292 #expect(model.groups[0].models.count == 4)293 }294295 @Test("offline: the cached catalogue with its date; never fetched: not cached; server error; empty catalogue; a 2.x catalogue groups by provider")296 func failures() async throws {297 let world = try await FeatureWorld()298 let model = ModelPickerModel(repositories: world.repositories)299 await model.load()300 world.setOffline(true)301 await model.load(forceRefresh: true)302 #expect(model.phase == .loaded && model.freshness?.fromCache == true && model.groups.count == 2)303304 let cold = try await FeatureWorld(samples: false)305 let empty = ModelPickerModel(repositories: cold.repositories)306 cold.setOffline(true)307 await empty.load()308 #expect(empty.phase.failure?.kind == .notCached)309 cold.setOffline(false)310 cold.fake.stub("GET", path: "^/api/ai/models$") { _ in .error(500, code: "server_error", message: "boom") }311 await empty.load()312 #expect(empty.phase.failure?.kind == .server)313 cold.fake.stub("GET", path: "^/api/ai/models$", json: #"{"models":[],"providers":[]}"#)314 await empty.load()315 #expect(empty.phase == .loaded && empty.isEmpty)316 cold.fake.stub("GET", path: "^/api/ai/models$", json: #"{"models":[{"id":"m1","provider":"openai","available":true},{"id":"m2","available":true}],"providers":[]}"#)317 await empty.load(forceRefresh: true)318 #expect(empty.groups.map(\.id) == ["openai", "unknown"] && empty.groups.map(\.name) == ["Openai", "Créateur inconnu"])319 #expect(ModelPickerModel.priceLabel(empty.groups[0].models[0]) == nil && ModelPickerModel.capabilities(empty.groups[0].models[0]).isEmpty)320 }321322 @Test("numbers: tokens and prices")323 func numbers() {324 #expect(ModelPickerModel.tokens(1_000_000) == "1 M" && ModelPickerModel.tokens(1_500_000) == "1,5 M" && ModelPickerModel.tokens(200_000) == "200 k")325 #expect(ModelPickerModel.tokens(8192) == "8192" && ModelPickerModel.tokens(0) == nil && ModelPickerModel.tokens(nil) == nil)326 #expect(ModelPickerModel.price(10) == "10" && ModelPickerModel.price(0.25) == "0,25" && ModelPickerModel.price(2.5) == "2,5" && ModelPickerModel.price(0.05) == "0,05")327 }328329 @Test("logos: every creator and route of the registry resolves to a bundled image; unknown creators get the neutral fallback")330 func logos() throws {331 let registry = ModelLogoRegistry.shared332 #expect(registry.entries.count >= 28 && registry.source?.hasPrefix("simple-icons@") == true)333 for entry in registry.entries {334 let logo = registry.resolve(id: entry.id, kind: entry.kind)335 #expect(logo.isBundled, "no bundled image for \(entry.kind) \(entry.id) (\(logo.imageName))")336 #expect(logo.isGeneric == entry.generic, "\(entry.id)")337 #expect(logo.imageName == ModelLogoRegistry.imageName(file: entry.file))338 }339 let anthropic = registry.resolve(id: "anthropic")340 #expect(anthropic.imageName == "ModelLogo-anthropic" && !anthropic.isGeneric && anthropic.monogram == "AN")341 let openai = registry.resolve(id: "OpenAI") // no mark shipped: generic + monogram342 #expect(openai.isGeneric && openai.imageName == registry.fallbackImage && openai.monogram == "OA")343 let unknown = registry.resolve(id: "acme-labs", logoPath: "/model-logos/acme.svg")344 #expect(unknown.isGeneric && unknown.imageName == "ModelLogo-generic" && unknown.monogram == "AC" && unknown.isBundled)345 #expect(registry.resolve(id: nil).isGeneric && registry.resolve(id: nil).monogram == "AI")346 // a creator the bundled registry does not know, whose file is shipped: the server's path is honoured347 let alias = registry.resolve(id: "google-deepmind", logoPath: "/model-logos/google.svg", monogram: "GD")348 #expect(!alias.isGeneric && alias.imageName == "ModelLogo-google" && alias.monogram == "GD")349 let route = registry.resolve(id: "openrouter", kind: "route")350 #expect(!route.isGeneric && route.imageName == "ModelLogo-openrouter")351 #expect(registry.resolve(id: "maclustr", kind: "route").isGeneric)352 // every creator of the sample catalogue (the live answer) is known to the bundled registry353 let catalog = try JSONDecoder().decode(CreatorList.self, from: Data(FeatureSamples.text("ai", "models-catalog").utf8))354 for creator in catalog.creators { #expect(registry.entry(id: creator.id, kind: "creator") != nil, "\(creator.id)") }355 }356}357358private struct CreatorList: Decodable {359 struct Creator: Decodable { var id: String }360 var creators: [Creator]361}362363@MainActor364final class Chosen {365 var values: [String] = []366}367