SPB Git

Repository Public

LiveNextGenTests.swift 12.9 KB 192 lines swift

1//2//  SP Chat iOS3//  File: SPChat-iOS/Tests/LiveNextGenTests.swift4//  Description: End-to-end checks of the backend 1.0 additions against the DEVELOPMENT server (Scripts/test-live.sh):5//               unified catalogue, providers, model intelligence, an OpenRouter turn with web search, the sync feed.6//7//  Author: Simon-Pierre Boucher8//  Contact: contact@spboucher.ai9//1011import Foundation12import Testing13@testable import SPChat1415@Suite(.serialized, .enabled(if: Live.enabled)) @MainActor struct LiveNextGenTests {16    static let openRouterModel = ProcessInfo.processInfo.environment["SPCHAT_LIVE_OR_MODEL"] ?? "or:google/gemini-4-neon"1718    @Test func theWholeCatalogueLoadsWithMakersRoutesAndLogos() async throws {19        let env = Live.environment()20        let clock = ContinuousClock(); var response: ModelsResponse?21        let elapsed = try await clock.measure { response = try await env.api.catalog() }22        let r = try #require(response)23        let or = r.models.filter(\.isOpenRouter)24        #expect(or.count > 300 && r.openRouterConfigured && r.catalogSyncedAt != nil, "catalogue OpenRouter : \(or.count) modèles en \(elapsed)")25        #expect(or.allSatisfy { $0.id.hasPrefix("or:") && $0.provider == "openrouter" && !$0.creator.isEmpty && !$0.creatorName.isEmpty })26        #expect(or.contains { $0.isCatalogOnly } && r.models.contains { $0.isVerified && $0.route == "direct" })27        // every logo slug the server names exists in the bundled registry (no remote image, no hand-drawn logo)28        let slugs = Set(r.models.compactMap(\.logo))29        #expect(slugs.allSatisfy { LogoRegistry.asset($0) != nil }, "logos absents du registre : \(slugs.filter { LogoRegistry.asset($0) == nil })")30        // the 0.3 contract is unchanged: without `routes`, only the direct, benchmarked models31        let direct = try await env.api.models()32        #expect(!direct.models.contains { $0.isOpenRouter } && direct.models.count < r.models.count)33        await env.catalog.refresh()34        #expect(env.catalog.selectable.count > 300 && env.catalog.model(Self.openRouterModel)?.isSelectableForChat == true)35    }3637    @Test func providersModelDetailAndRouteSuggestionDecode() async throws {38        let env = Live.environment()39        let p = try await env.api.providers()40        #expect(p.providers.contains { $0.id == "openrouter" && $0.manageable } && p.providers.contains { $0.id == "tavily" && $0.kind == "tool" } && p.categoryRules["free"] != nil)41        #expect(p.providers.allSatisfy { ($0.keyHint ?? "").count <= 8 })                 // a hint, never a key42        let check = try await env.api.testProvider("openrouter")43        #expect(check.ok, "test de la clé OpenRouter : \(check.message)")44        let d = try await env.api.modelDetail(Self.openRouterModel)45        #expect(d.model.id == Self.openRouterModel && d.endpointsSource != nil && !d.endpoints.isEmpty)46        let direct = try await env.api.modelDetail(Live.model)47        #expect(direct.model.isVerified && direct.endpoints.isEmpty && direct.twins.contains { $0.route == "openrouter" })48        let s = try await env.api.routeSuggestion(text: "Écris une fonction Swift qui trie un tableau et explique sa complexité.", hasImages: false, wantsTools: false)49        let suggestion = try #require(s)50        await env.catalog.refresh()51        #expect(env.catalog.model(suggestion.selected)?.isVerified == true && suggestion.basis != nil)      // only bench-verified models are proposed52    }5354    @Test func aKeyRefusedByTheProviderChangesNothing() async throws {55        let env = Live.environment()56        let model = ProvidersModel(api: env.api, catalog: env.catalog, settings: env.settings)57        await model.load()58        let before = try #require(model.providers.first { $0.id == "tavily" })59        // the dev test session is freshly authenticated: the server validates the key with Tavily, which refuses it (422)60        let accepted = await model.setKey("tavily", key: "tvly-dev-0000000000000000-invalid")61        #expect(!accepted && model.results["tavily"]?.ok == false)62        await model.load()63        let after = try #require(model.providers.first { $0.id == "tavily" })64        #expect(after.keySource == before.keySource && after.keyHint == before.keyHint && after.configured == before.configured)65    }6667    @Test func anOpenRouterTurnWithWebSearchStreamsSourcesThenFollowUps() async throws {68        let env = Live.environment()69        await env.catalog.refresh(); await env.settings.refresh()70        #expect(env.settings.tools.contains { $0.name == "web_search" } && env.settings.followUpsEnabled)71        env.select(model: Self.openRouterModel); env.preferences.webSearch = true; env.preferences.effort = .low72        let session = env.newSession()73        // the server proposes follow-ups for answers of 200 characters or more74        session.composerText = "Quelle est la capitale de l'Australie ? Réponds en trois ou quatre phrases complètes (au moins 400 caractères), avec la source."75        session.sendComposer(options: env.sendOptions)76        let finished = await TestEnv.wait(timeout: 120) { session.phase == .idle && session.messages.count == 2 }77        #expect(finished)78        let id = try #require(session.conversationId)79        defer { Task { try? await env.api.deleteConversation(id) } }80        let answer = try #require(session.messages.last)81        #expect(answer.status == .complete && answer.modelId == Self.openRouterModel && answer.provider == "openrouter" && answer.text.lowercased().contains("canberra"))82        #expect(!answer.citations.isEmpty && answer.citations.allSatisfy { URL(string: $0.url)?.scheme?.hasPrefix("http") == true }, "sources : \(answer.citations.count)")83        #expect(answer.parts.contains { if case .toolCall(let t) = $0 { return t.name == "web_search" && t.status == .ok } else { return false } })84        #expect(answer.routing?.servedBy?.contains("OpenRouter") == true && answer.exactCost && (answer.costUsd ?? 0) > 0)85        // follow-ups arrive after the status, and are stored with the message86        let suggested = await TestEnv.wait(timeout: 30) { !(session.messages.last?.followups.isEmpty ?? true) }87        #expect(suggested)88        let stored = try await env.api.conversation(id)89        #expect(stored.activeMessages.last?.followups.isEmpty == false && stored.activeMessages.last?.citations.count == answer.citations.count)90        // switching model inside the conversation: the next answer comes from the other model, by the direct route91        env.select(model: Live.model); env.preferences.webSearch = false92        session.composerText = "Réponds exactement par le mot : pong"93        session.sendComposer(options: env.sendOptions)94        let second = await TestEnv.wait(timeout: 60) { session.phase == .idle && session.messages.count == 4 }95        #expect(second && session.messages.last?.modelId == Live.model && session.messages.last?.citations.isEmpty == true)96        let timeline = try await env.api.timeline(conversation: id)97        #expect(timeline.events.contains { $0.kind == .modelSwitch && $0.to == Live.model })98        let recent = await TestEnv.wait(timeout: 10) { env.catalog.recents.first?.id == Live.model }99        #expect(recent)100    }101102    @Test func parametersTheModelDoesNotAcceptAreReportedAsIgnored() async throws {103        let env = Live.environment()104        await env.catalog.refresh()105        // a catalogue model without tools, asked for a web search: the server answers and says what it dropped106        let candidate = env.catalog.openRouter.filter { !$0.tools && !$0.reasoning && $0.priceInput != nil && ($0.priceInput ?? 9) < 1 && !$0.imageGeneration && $0.catalog?.outputModalities == ["text"] }.sorted { ($0.priceInput ?? 0) < ($1.priceInput ?? 0) }.first107        let model = try #require(candidate, "aucun modèle sans outils dans le catalogue")108        var o = SendOptions(modelId: model.id, routingMode: .manual, toolsEnabled: true); o.webSearch = true; o.maxOutputTokens = 1000109        let session = env.newSession()110        session.composerText = "Réponds par le seul mot : oui"111        session.sendComposer(options: o)112        let finished = await TestEnv.wait(timeout: 90) { session.phase == .idle && session.messages.count == 2 }113        #expect(finished, "modèle \(model.id)")114        if let id = session.conversationId { Task { try? await env.api.deleteConversation(id) } }115        let answer = try #require(session.messages.last)116        #expect(!answer.ignoredParams.isEmpty, "modèle \(model.id) : applied = \(String(describing: answer.applied))")117    }118119    @Test func aTurnWithTheLocalRouteCostsNothing() async throws {120        let env = Live.environment()121        await env.catalog.refresh()122        let model = try #require(env.catalog.local.first, "aucun modèle de route locale sur ce serveur")123        #expect(model.id.hasPrefix("local:") && model.provider == "local" && model.isFree && env.catalog.model(model.id)?.isSelectableForChat == true)124        let alone = try await env.api.http.decode(ModelsResponse.self, .get("/api/models", query: [.init(name: "routes", value: "local")]))125        #expect(alone.models.contains { $0.id == model.id })126        let providers = try await env.api.providers().providers127        let row = try #require(providers.first { $0.id == "local" })128        #expect(row.route == "local" && row.manageable && row.kind == "models")129        let check = try await env.api.testProvider("local")130        #expect(check.ok, "test du service local : \(check.message)")131        let direct = try await env.api.testProvider("anthropic")132        #expect(direct.ok, "test d'un fournisseur direct : \(direct.message)")133        env.select(model: model.id)134        let session = env.newSession()135        session.composerText = "Réponds exactement par le mot : pong"136        session.sendComposer(options: env.sendOptions)137        let finished = await TestEnv.wait(timeout: 120) { session.phase == .idle && session.messages.count == 2 }138        #expect(finished)139        if let id = session.conversationId { Task { try? await env.api.deleteConversation(id) } }140        let answer = try #require(session.messages.last)141        #expect(answer.status == .complete && answer.modelId == model.id && answer.provider == "local" && answer.text.lowercased().contains("pong"), "réponse : \(answer.text.prefix(80)) \(answer.error ?? "")")142        #expect(!answer.exactCost && (answer.costUsd ?? 0) == 0)143        #expect(AIModel.costLabel(answer.costUsd, exact: answer.exactCost, route: model.route) == "local · 0 $")144    }145146    @Test func theSyncFeedReportsAChangeMadeByAnotherClient() async throws {147        let env = Live.environment()148        let plans = NSLockBox<[SyncPlan]>([])149        env.sync.onPlan = { plan in plans.withLock { $0.append(plan) } }150        let first = await env.sync.pull()151        let start = try #require(env.sync.cursor)152        #expect(first == SyncPlan())153        env.sync.start()154        let live = await TestEnv.wait(timeout: 15) { env.sync.status == .live }155        #expect(live)156        // "another device": a second environment renames nothing destructive — it creates then deletes a prompt and writes a draft157        let other = Live.environment()158        let marker = "sync-\(UUID().uuidString.prefix(6))"159        try await other.api.createPrompt(title: marker, body: "corps \(marker)", tags: [], favorite: false)160        let seen = await TestEnv.wait(timeout: 15) { plans.withLock { $0 }.contains { $0.prompts } }161        #expect(seen && (env.sync.cursor ?? 0) > start)162        env.sync.stop()163        // while the feed is closed, changes are found by the pull on the next foreground164        if let p = try await other.api.prompts(query: marker).first { try await other.api.deletePrompt(p.id) }165        let pulled = try #require(await env.sync.pull())166        #expect(pulled.prompts)167        let again = await env.sync.pull()168        #expect(again == SyncPlan())169    }170171    @Test func dockFavouritesAndAssistantSettingsRoundTripOnTheServer() async throws {172        let env = Live.environment()173        await env.catalog.refresh(); await env.settings.refresh()174        let dockBefore = env.catalog.dock, followBefore = env.settings.followUpsEnabled175        await env.catalog.setDock([Live.model, Self.openRouterModel])176        let seen = try await Live.environment().api.catalog()177        #expect(seen.dock == [Live.model, Self.openRouterModel])178        await env.catalog.setDock(dockBefore)179        let neon = try #require(env.catalog.model(Self.openRouterModel))180        await env.catalog.toggleFavorite(neon)181        let fav = try await env.api.catalog().models.first { $0.id == neon.id }?.favorite182        #expect(fav == !neon.favorite)183        await env.catalog.toggleFavorite(try #require(env.catalog.model(neon.id)))184        let e = await env.settings.setFollowUps(!followBefore)185        let other = Live.environment(); await other.settings.refresh()186        #expect(e == nil && other.settings.followUpsEnabled == !followBefore)187        _ = await env.settings.setFollowUps(followBefore)188        let c = try await env.api.constellation(limit: 20)189        #expect(!c.basis.isEmpty)190    }191}192
Files main
Menu

Theme