SPB Git

Repository Public

NextGenIntegrationTests.swift 19.1 KB 234 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: SPChat/Tests/CoreTests/NextGenIntegrationTests.swift3// Description: End-to-end tests of the backend 1.0 additions against the real DEVELOPMENT backend: whole catalogue, providers, model detail, route suggestion, a real turn on an OpenRouter model with web search (citations, follow-ups, billed cost), lens, timeline, tree, context, constellation, usage, sync feed, arena and its export.4import Foundation5import Testing6@testable import SPChatCore78/// Runs only when `SPCHAT_IT_URL` (a loopback development server, never production) and `SPCHAT_IT_COOKIE` (a session9/// minted for the development database) are set — see Scripts/integration-test.sh. Real model calls are made with the10/// cheapest suitable models (a few cents in total). Nothing is deleted except what the test created; no key is touched.11private let itURL = ProcessInfo.processInfo.environment["SPCHAT_IT_URL"].flatMap(URL.init(string:))12private let itCookie = ProcessInfo.processInfo.environment["SPCHAT_IT_COOKIE"]13private let cheapOpenRouter = "or:anthropic/claude-haiku-5.5"14private let cheapDirect = "claude-haiku-5-5"15private let localModel = "local:maclustr-qwen3"1617@Suite(.serialized, .enabled(if: itURL != nil && itCookie != nil)) struct NextGenIntegrationTests {18    private func client() throws -> APIClient {19        let server = try #require(ServerConfiguration(url: itURL!))20        #expect(server.host == "localhost" || server.host == "127.0.0.1", "integration tests must never target production")21        let api = APIClient(server: server, transport: URLSessionTransport(), secrets: InMemorySecretStore(), userAgent: "SPChat/2.0 (Macintosh) native tests")22        api.cookies.ingest(setCookie: itCookie! + "; Path=/; Max-Age=3600")23        return api24    }2526    @Test func catalogueProvidersDetailAndSuggestion() async throws {27        let api = try client()28        let direct = try await api.models(includeHidden: true)29        let all = try await api.models(includeHidden: true, routes: "all")30        #expect(direct.models.allSatisfy { $0.route == "direct" }, "0.3 contract: direct models only")31        let or = all.models.filter(\.isOpenRouter)32        #expect(or.count > 300, "the OpenRouter catalogue is listed (\(or.count))"); #expect(all.models.count == direct.models.count + or.count + all.models.filter(\.isLocal).count)33        #expect(or.allSatisfy { $0.id.hasPrefix("or:") && $0.isCatalogOnly && $0.catalog?.slug.isEmpty == false })34        #expect(all.openrouter?.configured == true); #expect((all.openrouter?.catalog?.total ?? 0) == or.count)35        #expect(Set(all.models.map(\.status)).isSubset(of: ["verified", "catalog", "untested", "unavailable", "unsupported"]))36        // filtering and instant search over the real catalogue37        let index = ModelSearchIndex(all.chatModels)38        var q = ModelQuery(); q.text = "claude haiku"39        #expect(index.search(q).prefix(4).allSatisfy { $0.displayName.lowercased().contains("haiku") })40        q = ModelQuery(); q.freeOnly = true; q.routes = ["openrouter"]41        #expect(index.search(q).allSatisfy { $0.isFree && $0.isOpenRouter }); #expect(!index.search(q).isEmpty)42        #expect(all.models.makers.first?.count ?? 0 > 20)4344        let providers = try await api.providers()45        #expect(Set(["openai", "anthropic", "xai", "openrouter", "tavily"]).isSubset(of: Set(providers.providers.map(\.id))))46        let openrouter = try #require(providers.provider("openrouter"))47        #expect(openrouter.configured); #expect(openrouter.manageable); #expect((openrouter.keyHint ?? "").count <= 8, "only the last characters of a key ever reach a client")48        #expect(providers.provider("tavily")?.kind == "tool"); #expect(!(providers.categoryRules["long-context"] ?? "").isEmpty); #expect((providers.logos?.makers.count ?? 0) > 30)49        let test = try await api.testProvider("tavily"); #expect(test.ok, "\(test.message)")5051        let detail = try await api.modelDetail("or:google/gemini-4-neon")52        #expect(detail.model.id == "or:google/gemini-4-neon"); #expect(!(detail.endpoints?.list.isEmpty ?? true)); #expect(!(detail.endpoints?.source.isEmpty ?? true))53        #expect(detail.thirdPartyBenchmarks.allSatisfy { !$0.source.isEmpty && !$0.values.isEmpty })54        let haiku = try await api.modelDetail(cheapDirect)55        #expect(haiku.endpoints == nil); #expect(haiku.twins.contains { $0.id == cheapOpenRouter && $0.route == "openrouter" }); #expect(haiku.params?.effort.isEmpty == false)56        if let m = haiku.measured { #expect(!m.source.isEmpty); #expect(m.requests > 0) }5758        let suggestion = try await api.suggestRoute(text: "Écris un quicksort en Rust et explique sa complexité.")59        let selected = try #require(suggestion.decision?.selected)60        #expect(all.model(selected)?.isVerified == true, "the router only proposes verified models"); #expect(!(suggestion.basis ?? "").isEmpty); #expect(!suggestion.candidates.isEmpty)6162        // dock: synced list, restored afterwards63        let original = all.dock64        try await api.setDock([cheapDirect, cheapOpenRouter])65        #expect(try await api.models(routes: "all").dock == [cheapDirect, cheapOpenRouter])66        try await api.setDock(original)67        #expect(try await api.models(routes: "all").dock == original)68    }6970    @Test func openRouterTurnWithWebSearchThenLensTimelineContextTreeAndSync() async throws {71        let api = try client()72        let sync = SyncClient(api: api)73        let head = try await sync.pull(after: 0).head74        let engine = GenerationEngine(api: api)75        var acc = MessageAccumulator(conversationId: ""); var conversationId = ""; var end: GenerationEnd?; var sawFollowups = false76        let options = TurnOptions(modelId: cheapOpenRouter, routingMode: .manual, effort: .low, maxOutputTokens: 600, toolsEnabled: true, toolNames: ["web_search"], webSearch: true)77        for await u in engine.start(.send, payload: ["text": "Quelle est la capitale de l'Australie et depuis quand ? Fais une seule recherche web, puis réponds en trois phrases."].merging(options.payload) { $1 }) {78            switch u {79            case .event(let e):80                if case .conversation(let id, _, let uid, _, _, _) = e { conversationId = id; acc.conversationId = id; acc.parentId = uid }81                if case .followups = e { sawFollowups = true }82                acc.apply(e)83            case .ended(let e): end = e84            default: break85            }86        }87        #expect(end == .completed)88        let live = try #require(acc.message)89        #expect(live.modelId == cheapOpenRouter); #expect(live.status == .complete, "\(live.error ?? "")"); #expect(live.visibleText.localizedCaseInsensitiveContains("Canberra"))90        #expect(!live.citations.isEmpty, "web_search returned sources as citation parts"); #expect(live.routing?.servedBy?.hasPrefix("OpenRouter") == true)91        #expect(live.isCostExact, "OpenRouter reports the billed amount"); #expect((live.costUsd ?? 0) > 0)92        #expect(live.parts.contains { if case .toolCall(let c) = $0 { return c.name == "web_search" && c.status == .ok }; return false })93        #expect(sawFollowups); #expect((live.followups?.count ?? 0) >= 1 && (live.followups?.count ?? 9) <= 3)9495        // the stored rows hold the same truth96        let detail = try await api.conversation(conversationId)97        let stored = try #require(detail.activeMessages.last)98        #expect(stored.id == live.id); #expect(stored.followups == live.followups); #expect(stored.citations.count == live.citations.count); #expect(stored.isCostExact)99        #expect(detail.conversation.modelId == cheapOpenRouter)100101        // switching model inside the conversation = the next turn with another modelId (direct route this time)102        var second = MessageAccumulator(conversationId: conversationId, parentId: stored.id)103        for await u in engine.start(.send, payload: ["conversationId": .string(conversationId), "text": "Et celle de la Nouvelle-Zélande ? Un seul mot."].merging(TurnOptions(modelId: cheapDirect, routingMode: .manual, effort: .low, toolsEnabled: false).payload) { $1 }) {104            if case .event(let e) = u { second.apply(e) }105        }106        #expect(second.message?.modelId == cheapDirect); #expect(second.message?.visibleText.localizedCaseInsensitiveContains("Wellington") == true)107108        // timeline, context, tree109        let timeline = try await api.conversationTimeline(conversationId)110        let kinds = timeline.events.map(\.kind)111        #expect(kinds.contains(.prompt)); #expect(kinds.contains(.response)); #expect(kinds.contains(.tool)); #expect(kinds.contains(.modelSwitch), "\(timeline.events.map(\.type))")112        #expect(!(timeline.note ?? "").isEmpty); #expect(timeline.events.contains { $0.messageId == stored.id })113        let context = try await api.conversationContext(conversationId, model: cheapDirect)114        #expect(context.model?.id == cheapDirect); #expect(context.messages.count == 4); #expect((context.estimate?.nextRequestTokens ?? 0) > 0); #expect(context.estimate?.source?.isEmpty == false, "the estimate says what it is based on")115        let fork = try await api.forkConversation(conversationId, at: stored.id, label: "Test natif")116        let tree = try await api.conversationTree(conversationId)117        #expect(tree.current == conversationId); #expect(tree.family.contains { $0.id == fork.id && $0.branchLabel == "Test natif" })118        try await api.updateConversation(fork.id, ConversationPatch(branchLabel: .some("Renommée")))119        #expect(try await api.conversationTree(fork.id).family.first { $0.id == fork.id }?.branchLabel == "Renommée")120121        // Response Lens: a real, stored summary of the first answer122        let kindsOfLens = try await api.lenses(messageId: stored.id)123        #expect(Set(kindsOfLens.kinds.map(\.id)) == ["summary", "explain", "keypoints", "table", "verify"]); #expect(kindsOfLens.kinds.first { $0.id == "verify" }?.usesWeb == true)124        var lens = LensAccumulator(lens: "summary")125        guard case .events(_, let bytes) = try await api.openLens(messageId: stored.id, lens: "summary", modelId: cheapDirect) else { Issue.record("lens did not stream"); return }126        var parser = SSEParser()127        for try await chunk in bytes { for sse in parser.feed(chunk) { _ = lens.apply(LensEvent(sse)) }; if lens.isFinished { break } }128        #expect(!lens.failed, "\(lens.error ?? "")"); #expect(!lens.displayText.isEmpty)129        let lensId = try #require(lens.result?.id)130        #expect(try await api.lenses(messageId: stored.id).lenses.contains { $0.id == lensId && $0.lens == "summary" })131        try await api.deleteLens(lensId)132        #expect(try await api.lenses(messageId: stored.id).lenses.contains { $0.id == lensId } == false)133134        // advanced search, constellation, usage by route135        #expect(try await api.search("Canberra", filters: SearchFilters(semantic: false, provider: "openrouter", role: "assistant")).contains { $0.conversationId == conversationId })136        #expect(try await api.search("Canberra", filters: SearchFilters(semantic: false, provider: "xai")).contains { $0.conversationId == conversationId } == false)137        let constellation = try await api.constellation(limit: 80)138        #expect(constellation.nodes.contains { $0.id == conversationId }); #expect(constellation.links.contains { $0.kind == "branch" && ($0.source == fork.id || $0.target == fork.id) }); #expect(!constellation.basis.isEmpty)139        let usage = try await api.usage()140        #expect(usage.byRoute.contains { $0.route == "openrouter" && $0.requests > 0 }); #expect((usage.costBasis?.exact ?? 0) > 0); #expect(!usage.byConversation.isEmpty)141142        // sync feed: everything above is in the change feed, by pull and by the live stream (resumed from `head`)143        let feed = try await sync.pull(after: head)144        let inv = SyncInvalidation.reduce(feed, after: head)145        #expect(feed.head > head); #expect(inv.conversationList); #expect(inv.conversations.contains(conversationId) || !inv.messages.isEmpty)146        var streamed = SyncInvalidation(); var connected = false147        let reader = Task { () -> (Bool, SyncInvalidation) in148            for await u in sync.events(after: head) {149                if case .connected = u { connected = true }150                if case .invalidate(let i) = u { streamed.merge(i); if streamed.head >= feed.head { break } }151            }152            return (connected, streamed)153        }154        let timeout = Task { try await Task.sleep(for: .seconds(20)); reader.cancel() }155        let (wasConnected, got) = await reader.value; timeout.cancel()156        #expect(wasConnected); #expect(got.head >= feed.head); #expect(got.conversationList)157158        // clean up what the test created159        try await api.deleteConversation(fork.id); try await api.deleteConversation(conversationId)160        #expect(SyncInvalidation.reduce(try await sync.pull(after: feed.head), after: feed.head).deletedConversations.contains(conversationId))161    }162163    /// The v1 journey covers this with a freshly enrolled passkey; here the same stop is checked on backend 1.0 with the164    /// minted session (the journey signs every other development session out, so it is not part of the default run).165    @Test func stopKeepsThePartialAnswerAsCancelled() async throws {166        let api = try client(); let engine = GenerationEngine(api: api)167        var acc = MessageAccumulator(conversationId: ""); var runId: String?; var conversationId = ""; var stopped = false168        let options = TurnOptions(modelId: cheapDirect, routingMode: .manual, maxOutputTokens: 4000, toolsEnabled: false)169        for await u in engine.start(.send, payload: ["text": "Écris un texte de 1500 mots sur l'histoire des phares."].merging(options.payload) { $1 }) {170            if case .event(.run(let id)) = u { runId = id }171            if case .event(.conversation(let id, _, _, _, _, _)) = u { conversationId = id; acc.conversationId = id }172            if case .event(let e) = u { acc.apply(e); if case .text = e, (acc.message?.visibleText.count ?? 0) > 200, !stopped { stopped = true; try await api.cancelGeneration(runId: runId, messageId: acc.message?.id) } }173        }174        #expect(stopped, "status \(String(describing: acc.message?.status)) error \(acc.message?.error ?? "—")")175        let cancelled = try #require(try await api.conversation(conversationId).messages.first { $0.id == acc.message?.id })176        #expect(cancelled.status == .cancelled); #expect(cancelled.visibleText.isEmpty == false); #expect(cancelled.visibleText.count < 9000)177        try await api.deleteConversation(conversationId)178    }179180    /// Third route: MacLustr AI, the owner's own MLX inference service (`local:` ids). A real turn costs nothing.181    @Test func localRouteCatalogueProviderAndRealTurn() async throws {182        let api = try client()183        let onlyLocal = try await api.models(includeHidden: true, routes: "local")184        #expect(!onlyLocal.models.isEmpty); #expect(onlyLocal.models.allSatisfy { $0.isLocal && $0.id.hasPrefix("local:") && $0.provider == "local" })185        let model = try #require(onlyLocal.model(localModel))186        #expect(model.categories.contains("local")); #expect(model.inputPrice == 0 && model.outputPrice == 0); #expect(model.catalog?.source == "local")187        #expect(model.catalog?.quantization?.isEmpty == false); #expect(model.catalog?.license?.isEmpty == false); #expect(model.catalog?.servedBy?.isEmpty == false); #expect(model.isSelectableForChat)188        let all = try await api.models(includeHidden: true, routes: "all")189        #expect(Set(all.models.map(\.route)) == ["direct", "openrouter", "local"]); #expect(all.model(localModel) != nil)190        var q = ModelQuery(); q.routes = ["local"]191        #expect(ModelSearchIndex(all.chatModels).search(q).map(\.id) == onlyLocal.chatModels.filter { !$0.hidden }.map(\.id).sorted())192193        let providers = try await api.providers()194        let local = try #require(providers.provider("local"))195        #expect(local.route == "local"); #expect(local.manageable); #expect(local.kind == "models"); #expect((local.logo ?? "").isEmpty); #expect(providers.categoryRules["local"]?.isEmpty == false)196        let test = try await api.testProvider("local"); #expect(test.ok, "\(test.message)")197        // « Tester » now works for the direct providers too198        let anthropic = try await api.testProvider("anthropic"); #expect(anthropic.ok, "\(anthropic.message)")199        #expect(try await api.modelDetail(localModel).model.id == localModel)200201        var acc = MessageAccumulator(conversationId: ""); var conversationId = ""; var end: GenerationEnd?202        let options = TurnOptions(modelId: localModel, routingMode: .manual, maxOutputTokens: 300, toolsEnabled: false)203        for await u in GenerationEngine(api: api).start(.send, payload: ["text": "Réponds en une courte phrase : quelle est la capitale du Canada ?"].merging(options.payload) { $1 }) {204            if case .event(let e) = u { if case .conversation(let id, _, _, _, _, _) = e { conversationId = id; acc.conversationId = id }; acc.apply(e) }205            if case .ended(let e) = u { end = e }206        }207        #expect(end == .completed)208        let answer = try #require(acc.message)209        #expect(answer.modelId == localModel); #expect(answer.status == .complete, "\(answer.error ?? "")"); #expect(answer.visibleText.localizedCaseInsensitiveContains("Ottawa"))210        #expect((answer.costUsd ?? 0) == 0); #expect(!answer.isCostExact); #expect(answer.costLabel(route: model.route) == "local · 0 $")211        let stored = try #require(try await api.conversation(conversationId).activeMessages.last)212        #expect(stored.modelId == localModel); #expect(stored.provider == "local"); #expect((stored.usage?.output ?? 0) > 0)213        try await api.deleteConversation(conversationId)214    }215216    @Test func arenaAcrossRoutesAndExport() async throws {217        let api = try client()218        guard case .events(_, let bytes) = try await api.openArena(prompt: "Réponds par un seul mot : bonjour", modelIds: [cheapDirect, cheapOpenRouter], effort: "low") else { Issue.record("arena did not stream"); return }219        var acc = ArenaAccumulator(modelIds: [cheapDirect, cheapOpenRouter]); var parser = SSEParser()220        for try await chunk in bytes { for sse in parser.feed(chunk) { acc.apply(ArenaEvent(sse)) }; if acc.finished || acc.failure != nil { break } }221        #expect(acc.failure == nil, "\(acc.failure ?? "")"); #expect(acc.finished)222        let id = try #require(acc.comparisonId)223        #expect(acc.columns.count == 2)224        for c in acc.columns { #expect(c.error == nil, "\(c.modelId): \(c.error ?? "")"); #expect(!c.text.isEmpty); #expect(c.done); #expect((c.latencyMs ?? 0) > 0) }225        #expect(acc.columns.first { $0.modelId == cheapOpenRouter }?.route == "openrouter"); #expect(acc.columns.first { $0.modelId == cheapOpenRouter }?.exactCost == true)226        try await api.preferArenaAnswer(comparisonId: id, modelId: cheapOpenRouter)227        #expect(try await api.arenaHistory().contains { $0.id == id })228        let md = try await api.exportArena(comparisonId: id, format: .markdown)229        #expect(String(decoding: md.data, as: UTF8.self).contains("bonjour") || String(decoding: md.data, as: UTF8.self).contains("Bonjour")); #expect(md.filename?.hasSuffix(".md") == true)230        let json = try await api.exportArena(comparisonId: id, format: .json)231        #expect(JSONValue.parse(json.data) != nil)232    }233}234
Files main
Menu

Theme