Repository Public
1//2// SP Chat iOS3// File: SPChat-iOS/Tests/LiveNextGenTests.swift4// Description: End-to-end checks of the backend 1.0 additions against the DEVELOPMENT server (Scripts/test-live.sh):5// unified catalogue, providers, model intelligence, an OpenRouter turn with web search, the sync feed.6//7// Author: Simon-Pierre Boucher8// Contact: contact@spboucher.ai9//1011import Foundation12import Testing13@testable import SPChat1415@Suite(.serialized, .enabled(if: Live.enabled)) @MainActor struct LiveNextGenTests {16 static let openRouterModel = ProcessInfo.processInfo.environment["SPCHAT_LIVE_OR_MODEL"] ?? "or:google/gemini-4-neon"1718 @Test func theWholeCatalogueLoadsWithMakersRoutesAndLogos() async throws {19 let env = Live.environment()20 let clock = ContinuousClock(); var response: ModelsResponse?21 let elapsed = try await clock.measure { response = try await env.api.catalog() }22 let r = try #require(response)23 let or = r.models.filter(\.isOpenRouter)24 #expect(or.count > 300 && r.openRouterConfigured && r.catalogSyncedAt != nil, "catalogue OpenRouter : \(or.count) modèles en \(elapsed)")25 #expect(or.allSatisfy { $0.id.hasPrefix("or:") && $0.provider == "openrouter" && !$0.creator.isEmpty && !$0.creatorName.isEmpty })26 #expect(or.contains { $0.isCatalogOnly } && r.models.contains { $0.isVerified && $0.route == "direct" })27 // every logo slug the server names exists in the bundled registry (no remote image, no hand-drawn logo)28 let slugs = Set(r.models.compactMap(\.logo))29 #expect(slugs.allSatisfy { LogoRegistry.asset($0) != nil }, "logos absents du registre : \(slugs.filter { LogoRegistry.asset($0) == nil })")30 // the 0.3 contract is unchanged: without `routes`, only the direct, benchmarked models31 let direct = try await env.api.models()32 #expect(!direct.models.contains { $0.isOpenRouter } && direct.models.count < r.models.count)33 await env.catalog.refresh()34 #expect(env.catalog.selectable.count > 300 && env.catalog.model(Self.openRouterModel)?.isSelectableForChat == true)35 }3637 @Test func providersModelDetailAndRouteSuggestionDecode() async throws {38 let env = Live.environment()39 let p = try await env.api.providers()40 #expect(p.providers.contains { $0.id == "openrouter" && $0.manageable } && p.providers.contains { $0.id == "tavily" && $0.kind == "tool" } && p.categoryRules["free"] != nil)41 #expect(p.providers.allSatisfy { ($0.keyHint ?? "").count <= 8 }) // a hint, never a key42 let check = try await env.api.testProvider("openrouter")43 #expect(check.ok, "test de la clé OpenRouter : \(check.message)")44 let d = try await env.api.modelDetail(Self.openRouterModel)45 #expect(d.model.id == Self.openRouterModel && d.endpointsSource != nil && !d.endpoints.isEmpty)46 let direct = try await env.api.modelDetail(Live.model)47 #expect(direct.model.isVerified && direct.endpoints.isEmpty && direct.twins.contains { $0.route == "openrouter" })48 let s = try await env.api.routeSuggestion(text: "Écris une fonction Swift qui trie un tableau et explique sa complexité.", hasImages: false, wantsTools: false)49 let suggestion = try #require(s)50 await env.catalog.refresh()51 #expect(env.catalog.model(suggestion.selected)?.isVerified == true && suggestion.basis != nil) // only bench-verified models are proposed52 }5354 @Test func aKeyRefusedByTheProviderChangesNothing() async throws {55 let env = Live.environment()56 let model = ProvidersModel(api: env.api, catalog: env.catalog, settings: env.settings)57 await model.load()58 let before = try #require(model.providers.first { $0.id == "tavily" })59 // the dev test session is freshly authenticated: the server validates the key with Tavily, which refuses it (422)60 let accepted = await model.setKey("tavily", key: "tvly-dev-0000000000000000-invalid")61 #expect(!accepted && model.results["tavily"]?.ok == false)62 await model.load()63 let after = try #require(model.providers.first { $0.id == "tavily" })64 #expect(after.keySource == before.keySource && after.keyHint == before.keyHint && after.configured == before.configured)65 }6667 @Test func anOpenRouterTurnWithWebSearchStreamsSourcesThenFollowUps() async throws {68 let env = Live.environment()69 await env.catalog.refresh(); await env.settings.refresh()70 #expect(env.settings.tools.contains { $0.name == "web_search" } && env.settings.followUpsEnabled)71 env.select(model: Self.openRouterModel); env.preferences.webSearch = true; env.preferences.effort = .low72 let session = env.newSession()73 // the server proposes follow-ups for answers of 200 characters or more74 session.composerText = "Quelle est la capitale de l'Australie ? Réponds en trois ou quatre phrases complètes (au moins 400 caractères), avec la source."75 session.sendComposer(options: env.sendOptions)76 let finished = await TestEnv.wait(timeout: 120) { session.phase == .idle && session.messages.count == 2 }77 #expect(finished)78 let id = try #require(session.conversationId)79 defer { Task { try? await env.api.deleteConversation(id) } }80 let answer = try #require(session.messages.last)81 #expect(answer.status == .complete && answer.modelId == Self.openRouterModel && answer.provider == "openrouter" && answer.text.lowercased().contains("canberra"))82 #expect(!answer.citations.isEmpty && answer.citations.allSatisfy { URL(string: $0.url)?.scheme?.hasPrefix("http") == true }, "sources : \(answer.citations.count)")83 #expect(answer.parts.contains { if case .toolCall(let t) = $0 { return t.name == "web_search" && t.status == .ok } else { return false } })84 #expect(answer.routing?.servedBy?.contains("OpenRouter") == true && answer.exactCost && (answer.costUsd ?? 0) > 0)85 // follow-ups arrive after the status, and are stored with the message86 let suggested = await TestEnv.wait(timeout: 30) { !(session.messages.last?.followups.isEmpty ?? true) }87 #expect(suggested)88 let stored = try await env.api.conversation(id)89 #expect(stored.activeMessages.last?.followups.isEmpty == false && stored.activeMessages.last?.citations.count == answer.citations.count)90 // switching model inside the conversation: the next answer comes from the other model, by the direct route91 env.select(model: Live.model); env.preferences.webSearch = false92 session.composerText = "Réponds exactement par le mot : pong"93 session.sendComposer(options: env.sendOptions)94 let second = await TestEnv.wait(timeout: 60) { session.phase == .idle && session.messages.count == 4 }95 #expect(second && session.messages.last?.modelId == Live.model && session.messages.last?.citations.isEmpty == true)96 let timeline = try await env.api.timeline(conversation: id)97 #expect(timeline.events.contains { $0.kind == .modelSwitch && $0.to == Live.model })98 let recent = await TestEnv.wait(timeout: 10) { env.catalog.recents.first?.id == Live.model }99 #expect(recent)100 }101102 @Test func parametersTheModelDoesNotAcceptAreReportedAsIgnored() async throws {103 let env = Live.environment()104 await env.catalog.refresh()105 // a catalogue model without tools, asked for a web search: the server answers and says what it dropped106 let candidate = env.catalog.openRouter.filter { !$0.tools && !$0.reasoning && $0.priceInput != nil && ($0.priceInput ?? 9) < 1 && !$0.imageGeneration && $0.catalog?.outputModalities == ["text"] }.sorted { ($0.priceInput ?? 0) < ($1.priceInput ?? 0) }.first107 let model = try #require(candidate, "aucun modèle sans outils dans le catalogue")108 var o = SendOptions(modelId: model.id, routingMode: .manual, toolsEnabled: true); o.webSearch = true; o.maxOutputTokens = 1000109 let session = env.newSession()110 session.composerText = "Réponds par le seul mot : oui"111 session.sendComposer(options: o)112 let finished = await TestEnv.wait(timeout: 90) { session.phase == .idle && session.messages.count == 2 }113 #expect(finished, "modèle \(model.id)")114 if let id = session.conversationId { Task { try? await env.api.deleteConversation(id) } }115 let answer = try #require(session.messages.last)116 #expect(!answer.ignoredParams.isEmpty, "modèle \(model.id) : applied = \(String(describing: answer.applied))")117 }118119 @Test func aTurnWithTheLocalRouteCostsNothing() async throws {120 let env = Live.environment()121 await env.catalog.refresh()122 let model = try #require(env.catalog.local.first, "aucun modèle de route locale sur ce serveur")123 #expect(model.id.hasPrefix("local:") && model.provider == "local" && model.isFree && env.catalog.model(model.id)?.isSelectableForChat == true)124 let alone = try await env.api.http.decode(ModelsResponse.self, .get("/api/models", query: [.init(name: "routes", value: "local")]))125 #expect(alone.models.contains { $0.id == model.id })126 let providers = try await env.api.providers().providers127 let row = try #require(providers.first { $0.id == "local" })128 #expect(row.route == "local" && row.manageable && row.kind == "models")129 let check = try await env.api.testProvider("local")130 #expect(check.ok, "test du service local : \(check.message)")131 let direct = try await env.api.testProvider("anthropic")132 #expect(direct.ok, "test d'un fournisseur direct : \(direct.message)")133 env.select(model: model.id)134 let session = env.newSession()135 session.composerText = "Réponds exactement par le mot : pong"136 session.sendComposer(options: env.sendOptions)137 let finished = await TestEnv.wait(timeout: 120) { session.phase == .idle && session.messages.count == 2 }138 #expect(finished)139 if let id = session.conversationId { Task { try? await env.api.deleteConversation(id) } }140 let answer = try #require(session.messages.last)141 #expect(answer.status == .complete && answer.modelId == model.id && answer.provider == "local" && answer.text.lowercased().contains("pong"), "réponse : \(answer.text.prefix(80)) \(answer.error ?? "")")142 #expect(!answer.exactCost && (answer.costUsd ?? 0) == 0)143 #expect(AIModel.costLabel(answer.costUsd, exact: answer.exactCost, route: model.route) == "local · 0 $")144 }145146 @Test func theSyncFeedReportsAChangeMadeByAnotherClient() async throws {147 let env = Live.environment()148 let plans = NSLockBox<[SyncPlan]>([])149 env.sync.onPlan = { plan in plans.withLock { $0.append(plan) } }150 let first = await env.sync.pull()151 let start = try #require(env.sync.cursor)152 #expect(first == SyncPlan())153 env.sync.start()154 let live = await TestEnv.wait(timeout: 15) { env.sync.status == .live }155 #expect(live)156 // "another device": a second environment renames nothing destructive — it creates then deletes a prompt and writes a draft157 let other = Live.environment()158 let marker = "sync-\(UUID().uuidString.prefix(6))"159 try await other.api.createPrompt(title: marker, body: "corps \(marker)", tags: [], favorite: false)160 let seen = await TestEnv.wait(timeout: 15) { plans.withLock { $0 }.contains { $0.prompts } }161 #expect(seen && (env.sync.cursor ?? 0) > start)162 env.sync.stop()163 // while the feed is closed, changes are found by the pull on the next foreground164 if let p = try await other.api.prompts(query: marker).first { try await other.api.deletePrompt(p.id) }165 let pulled = try #require(await env.sync.pull())166 #expect(pulled.prompts)167 let again = await env.sync.pull()168 #expect(again == SyncPlan())169 }170171 @Test func dockFavouritesAndAssistantSettingsRoundTripOnTheServer() async throws {172 let env = Live.environment()173 await env.catalog.refresh(); await env.settings.refresh()174 let dockBefore = env.catalog.dock, followBefore = env.settings.followUpsEnabled175 await env.catalog.setDock([Live.model, Self.openRouterModel])176 let seen = try await Live.environment().api.catalog()177 #expect(seen.dock == [Live.model, Self.openRouterModel])178 await env.catalog.setDock(dockBefore)179 let neon = try #require(env.catalog.model(Self.openRouterModel))180 await env.catalog.toggleFavorite(neon)181 let fav = try await env.api.catalog().models.first { $0.id == neon.id }?.favorite182 #expect(fav == !neon.favorite)183 await env.catalog.toggleFavorite(try #require(env.catalog.model(neon.id)))184 let e = await env.settings.setFollowUps(!followBefore)185 let other = Live.environment(); await other.settings.refresh()186 #expect(e == nil && other.settings.followUpsEnabled == !followBefore)187 _ = await env.settings.setFollowUps(followBefore)188 let c = try await env.api.constellation(limit: 20)189 #expect(!c.basis.isEmpty)190 }191}192