SPB Git

Repository Public

LiveBackendTests.swift 12.4 KB 196 lines swift

1//2//  SP Chat iOS3//  File: SPChat-iOS/Tests/LiveBackendTests.swift4//  Description: End-to-end tests against the real backend (development server + *_dev database): real HTTP, real SSE, real models.5//6//  Author: Simon-Pierre Boucher7//  Contact: contact@spboucher.ai8//910import Foundation11import Testing12import UIKit13@testable import SPChat1415/// Enabled only when the runner provides a development server and a session minted for its database:16///   Scripts/test-live.sh   (tunnel to the dev server, TEST_RUNNER_SPCHAT_BASE_URL, TEST_RUNNER_SPCHAT_DEBUG_SESSION)17/// The suite makes a handful of real model calls on the cheapest verified model (a fraction of a cent).18enum Live {19    static let baseURL = ProcessInfo.processInfo.environment["SPCHAT_BASE_URL"].flatMap(URL.init(string:))20    static let token = ProcessInfo.processInfo.environment["SPCHAT_DEBUG_SESSION"]21    static var enabled: Bool { baseURL != nil && (token?.count ?? 0) == 43 }22    static let model = ProcessInfo.processInfo.environment["SPCHAT_LIVE_MODEL"] ?? "claude-haiku-5-5"2324    @MainActor static func environment(token: String? = Live.token) -> AppEnvironment {25        let suite = "spchat.tests.live.\(UUID().uuidString)"26        let env = AppEnvironment(configuration: AppConfiguration(baseURL: baseURL!, userAgent: "SPChat/1.0 (iPhone; iOS tests; native)"), tokens: MemorySessionTokenStore(token), storeDirectory: nil, defaults: UserDefaults(suiteName: suite)!)27        env.network.simulate(online: true)28        return env29    }30    static var options: SendOptions { SendOptions(modelId: model, routingMode: .manual, effort: .low, toolsEnabled: false) }31}3233@Suite(.serialized, .enabled(if: Live.enabled)) @MainActor struct LiveBackendTests {34    @Test func accountEndpointsDecode() async throws {35        let env = Live.environment()36        let models = try await env.api.models()37        #expect(models.models.count > 5 && models.models.contains { $0.isSelectableForChat } && models.models.contains { $0.id == Live.model && $0.available })38        #expect(try await env.api.presets().contains { $0.builtin })39        _ = try await env.api.projects()40        _ = try await env.api.conversations()41        let usage = try await env.api.usage(days: 30)42        #expect(usage.spending.limits.dailyLimitUsd >= 0)43        #expect(try await env.api.passkeys().passkeys.count >= 0)44        #expect(try await env.api.sessions().contains { $0.current })45        #expect(try await env.api.settings()["settings"]?["spending"] != nil)46        _ = try await env.api.memories(); _ = try await env.api.prompts(); _ = try await env.api.files()47        #expect(try await env.api.authState().signedIn)48    }4950    @Test func aRevokedOrUnknownTokenIsReportedAsLocked() async {51        let env = Live.environment(token: String(repeating: "A", count: 43))52        let lost = NSLockBox(false)53        await env.http.setSessionLostHandler { lost.withLock { $0 = true } }54        await #expect(throws: APIError.locked) { _ = try await env.api.conversations() }55        let has = await env.http.hasSession56        #expect(lost.withLock { $0 } && !has)57    }5859    @Test func theServerRefusesMutationsWithoutItsOrigin() async throws {60        // documents why HTTPClient states the Origin: without it every mutation is rejected (403 bad_origin)61        var req = URLRequest(url: Live.baseURL!.appendingPathComponent("/api/drafts")); req.httpMethod = "PUT"62        req.setValue("csa_session=\(Live.token!)", forHTTPHeaderField: "Cookie"); req.setValue("application/json", forHTTPHeaderField: "Content-Type")63        req.httpBody = Data(#"{"scope":"new","text":"x","attachmentIds":[]}"#.utf8)64        let config = URLSessionConfiguration.ephemeral; config.httpShouldSetCookies = false65        let (_, resp) = try await URLSession(configuration: config).data(for: req)66        #expect((resp as? HTTPURLResponse)?.statusCode == 403)67    }6869    @Test func aFullConversationLifecycle() async throws {70        let env = Live.environment()71        let session = env.newSession()72        session.composerText = "Réponds exactement par le mot : pong"73        session.sendComposer(options: Live.options)74        #expect(await TestEnv.wait(timeout: 60) { session.phase == .idle && session.messages.count == 2 })75        let id = try #require(session.conversationId)76        defer { Task { try? await env.api.deleteConversation(id) } }77        let answer = try #require(session.messages.last)78        #expect(answer.role == .assistant && answer.status == .complete && answer.text.lowercased().contains("pong"))79        #expect(answer.modelId == Live.model && (answer.usage?.output ?? 0) > 0 && !session.messages[0].id.hasPrefix("tmp_"))8081        // synchronised: another client (fresh environment = the web app, the Mac) sees the same conversation82        let other = Live.environment()83        let seen = try await other.api.conversation(id)84        #expect(seen.activeMessages.map(\.id) == session.messages.map(\.id) && seen.liveRun == nil)85        #expect(try await other.api.conversations().contains { $0.id == id })8687        // regenerate: a second version of the answer, the first is kept as a sibling88        session.regenerate(answer, options: Live.options)89        #expect(await TestEnv.wait(timeout: 60) { session.phase == .idle && session.messages.count == 2 && session.messages.last?.id != answer.id })90        #expect(await TestEnv.wait(timeout: 10) { session.versions(of: session.messages[1]).count == 2 })91        await session.switchBranch(to: answer.id)92        #expect(session.messages.last?.id == answer.id)9394        // edit the question: a new branch, answered again95        session.edit(session.messages[0], text: "Réponds exactement par le mot : ping", options: Live.options)96        #expect(await TestEnv.wait(timeout: 60) { session.phase == .idle && session.messages.last?.role == .assistant && session.messages.last?.text.lowercased().contains("ping") == true })97        #expect(await TestEnv.wait(timeout: 10) { session.versions(of: session.messages[0]).count == 2 })9899        // metadata, export, search100        await session.update(ConversationPatch(title: "Essai iOS \(id.suffix(5))", pinned: true, favorite: true))101        #expect(session.conversation?.pinned == true && session.conversation?.favorite == true && session.title.hasPrefix("Essai iOS"))102        let md = String(decoding: try await env.api.export(conversation: id, format: "md"), as: UTF8.self)103        #expect(md.contains("ping"))104        let fork = try #require(await session.fork(at: session.messages[0]))105        try await env.api.deleteConversation(fork)106107        // archive then delete: gone for every client108        await env.conversations.refresh()109        let conv = try #require(env.conversations.conversation(id))110        await env.conversations.setArchived(conv, true)111        #expect(env.conversations.archived.contains { $0.id == id } && !env.conversations.active.contains { $0.id == id })112        await env.conversations.delete(try #require(env.conversations.conversation(id)))113        #expect(env.conversations.conversation(id) == nil)114        do { _ = try await other.api.conversation(id); Issue.record("deleted conversation still served") } catch { #expect((error as? APIError)?.status == 404) }115    }116117    @Test func stopReallyEndsTheGenerationOnTheServer() async throws {118        let env = Live.environment()119        let session = env.newSession()120        session.composerText = "Écris un texte de 1500 mots sur l'histoire de la typographie."121        session.sendComposer(options: Live.options)122        #expect(await TestEnv.wait(timeout: 60) { session.messages.last?.role == .assistant && (session.messages.last?.text.count ?? 0) > 40 })123        let id = try #require(session.conversationId)124        defer { Task { try? await env.api.deleteConversation(id) } }125        session.stop()126        #expect(session.phase == .idle && session.messages.last?.status == .cancelled)127        try await Task.sleep(for: .seconds(3))128        let detail = try await env.api.conversation(id)129        // the answer is cancelled at once; with backend 1.0 the run itself may stay registered a few more seconds130        // while the server finishes its post-processing (title, memory), so its end is awaited separately131        #expect(detail.activeMessages.last?.status == .cancelled && !(detail.activeMessages.last?.text.isEmpty ?? true),132                "état \(String(describing: detail.activeMessages.last?.status)), \(detail.activeMessages.last?.text.count ?? 0) caractères")133        var live = detail.liveRun134        for _ in 0..<20 where live != nil { try await Task.sleep(for: .seconds(1)); live = try await env.api.conversation(id).liveRun }135        #expect(live == nil, "le run \(live ?? "") est encore vivant 20 s après l'arrêt")136        let settled = try await env.api.conversation(id)137        #expect(settled.activeMessages.last?.status == .cancelled)138    }139140    @Test func attachmentsUploadWithProgressAndAreUsedByTheModel() async throws {141        let env = Live.environment()142        let session = env.newSession()143        let note = "Le code secret du coffre est MANGUE-42."144        session.addAttachment(data: Data(note.utf8), filename: "note.txt", mime: "text/plain", thumbnail: nil)145        let image = UIGraphicsImageRenderer(size: CGSize(width: 64, height: 64)).image { ctx in UIColor.systemRed.setFill(); ctx.fill(CGRect(x: 0, y: 0, width: 64, height: 64)) }.pngData()!146        session.addAttachment(data: image, filename: "carre.png", mime: "image/png", thumbnail: nil)147        #expect(!session.canSend)                                                  // uploads in flight block the send148        #expect(await TestEnv.wait(timeout: 30) { session.composerAttachments.allSatisfy { $0.uploaded != nil } })149        let uploaded = session.composerAttachments.compactMap(\.uploaded)150        #expect(uploaded.count == 2 && uploaded.contains { $0.mime == "text/plain" } && uploaded.contains { $0.mime == "image/png" })151        // bytes come back identical through the authenticated download + cache152        let png = try #require(uploaded.first { $0.isImage })153        #expect(try await env.files.data(for: png.id) == image)154        #expect(await env.files.thumbnail(for: png.id, maxPixel: 64) != nil)155156        session.composerText = "Quel est le code secret indiqué dans le fichier joint ? Réponds seulement par le code."157        session.sendComposer(options: Live.options)158        #expect(await TestEnv.wait(timeout: 90) { session.phase == .idle && session.messages.count == 2 })159        let id = try #require(session.conversationId)160        defer { Task { try? await env.api.deleteConversation(id) } }161        #expect(session.messages[1].text.contains("MANGUE-42"))162        #expect(session.messages[0].parts.contains { if case .file = $0 { return true } else { return false } } && session.messages[0].parts.contains { if case .image = $0 { return true } else { return false } })163164        // a file the server cannot accept is refused with its reason, not silently dropped165        let other = env.newSession()166        other.addAttachment(data: Data([0x00, 0x01, 0x02, 0x03, 0xFF, 0xFE]), filename: "inconnu.bin", mime: "application/octet-stream", thumbnail: nil)167        #expect(await TestEnv.wait(timeout: 20) { if case .failed = other.composerAttachments.first?.state { return true } else { return false } })168    }169170    @Test func draftsFollowTheAccount() async throws {171        let env = Live.environment()172        let text = "brouillon iOS \(UUID().uuidString.prefix(8))"173        try await env.api.saveDraft(scope: "new", text: text)174        let fresh = Live.environment().newSession()175        await fresh.load()176        #expect(fresh.composerText == text)177        try await env.api.saveDraft(scope: "new", text: "")178        #expect(try await env.api.draft(scope: "new") == nil)179    }180181    @Test func searchFindsMessagesAndPromptsAndMemoriesRoundTrip() async throws {182        let env = Live.environment()183        let marker = "zx\(UUID().uuidString.prefix(6).lowercased())"184        try await env.api.createPrompt(title: "Prompt \(marker)", body: "Corps \(marker)")185        let created = try #require(try await env.api.prompts(query: marker).first)186        try await env.api.markPromptUsed(created.id)187        try await env.api.deletePrompt(created.id)188        #expect(try await env.api.prompts(query: marker).isEmpty)189        try await env.api.createMemory(content: "Le propriétaire teste l'app iOS \(marker).")190        let mem = try #require(try await env.api.memories().first { $0.content.contains(marker) })191        try await env.api.setMemory(mem.id, status: "disabled")192        try await env.api.deleteMemory(mem.id)193        _ = try await env.api.search("bonjour")194    }195}196
Files main
Menu

Theme