SPB Git

Repository Public

ArenaTests.swift 19.7 KB 283 lines swift

1//2//  SP Chat iOS3//  File: SPChat-iOS/Tests/ArenaTests.swift4//  Description: Arena — decoding of the `/api/compare` SSE vocabulary, the ArenaState reducer, ArenaSession against5//               a scripted server, and one end-to-end comparison against the development backend.6//7//  Author: Simon-Pierre Boucher8//  Contact: contact@spboucher.ai9//1011import Foundation12import Testing13@testable import SPChat1415private enum ArenaFixtures {16    static let start = #"{"id":"cmp_test1","models":[{"id":"or:google/gemini-4-neon","provider":"openrouter","route":"openrouter","creator":"google","logo":"google","displayName":"Gemini 4 Neon","estimate":{"inputTokens":825,"inputCostUsd":0.002475,"basis":"Estimation avant envoi"}},{"id":"claude-haiku-5-5","provider":"anthropic","route":"direct","creator":"anthropic","logo":"anthropic","displayName":"Claude Haiku 5.5","estimate":null}]}"#17    static let haikuDone = #"{"modelId":"claude-haiku-5-5","provider":"anthropic","route":"direct","creator":"anthropic","logo":"anthropic","displayName":"Claude Haiku 5.5","servedBy":null,"text":"Pong","usage":{"input":1041,"output":5,"cached":0,"reasoning":0},"costUsd":0.0001066,"exactCost":false,"latencyMs":614,"ttftMs":611,"error":null}"#18    static let neonDone = #"{"modelId":"or:google/gemini-4-neon","provider":"openrouter","route":"openrouter","creator":"google","logo":"google","displayName":"Gemini 4 Neon","servedBy":"OpenRouter · Google","text":"Pong !","reasoning":"court","usage":{"input":684,"output":38,"cached":0,"reasoning":33,"costUsd":0.002622},"costUsd":0.002622,"exactCost":true,"latencyMs":2832,"ttftMs":2746,"error":null}"#19    static func ev(_ name: String, _ json: String) -> ArenaEvent { ArenaEvent(sse: SSEEvent(event: name, data: json, id: nil)) }2021    /// A complete numbered run, as the backend emits it.22    static func run(end: Bool = true) -> [String] {23        var e: [String] = ["retry: 2000\n\n"]; var i = 024        func add(_ n: String, _ j: String) { e.append(SSE.event(n, j, id: i)); i += 1 }25        add("run", #"{"runId":"run_cmp1"}"#); add("start", start)26        add("text", #"{"model":"claude-haiku-5-5","delta":"Po"}"#); add("text", #"{"model":"claude-haiku-5-5","delta":"ng"}"#)27        add("reasoning", #"{"model":"or:google/gemini-4-neon","delta":"court"}"#)28        add("text", #"{"model":"or:google/gemini-4-neon","delta":"Pong !"}"#)29        if end { add("model_done", haikuDone); add("model_done", neonDone); add("done", #"{"id":"cmp_test1"}"#); add("end", "{}") }30        return e31    }32    static var models: [AIModel] {33        [AIModel(id: "claude-haiku-5-5", provider: "anthropic", displayName: "Claude Haiku 5.5"), AIModel(id: "or:google/gemini-4-neon", provider: "openrouter", displayName: "Gemini 4 Neon", creator: "google", status: "catalog")]34    }35}3637@Suite struct ArenaEventTests {38    @Test func decodesEveryEventOfTheStream() {39        #expect(ArenaFixtures.ev("run", #"{"runId":"run_x"}"#) == .run(id: "run_x"))40        guard case .start(let id, let cols) = ArenaFixtures.ev("start", ArenaFixtures.start) else { Issue.record("start"); return }41        #expect(id == "cmp_test1" && cols.map(\.id) == ["or:google/gemini-4-neon", "claude-haiku-5-5"])42        #expect(cols.first?.route == "openrouter" && cols.first?.logo == "google" && cols.first?.estimatedInputCostUsd == 0.002475 && cols.first?.estimateBasis == "Estimation avant envoi")43        #expect(cols.last?.estimatedInputCostUsd == nil && cols.last?.isDone == false)44        #expect(ArenaFixtures.ev("text", #"{"model":"m","delta":"a"}"#) == .text(model: "m", delta: "a"))45        #expect(ArenaFixtures.ev("reasoning", #"{"model":"m","delta":"r"}"#) == .reasoning(model: "m", delta: "r"))46        #expect(ArenaFixtures.ev("model_error", #"{"model":"m","message":"clé refusée"}"#) == .modelError(model: "m", message: "clé refusée"))47        guard case .modelDone(let c) = ArenaFixtures.ev("model_done", ArenaFixtures.neonDone) else { Issue.record("model_done"); return }48        #expect(c.id == "or:google/gemini-4-neon" && c.isDone && c.text == "Pong !" && c.reasoning == "court" && c.servedBy == "OpenRouter · Google")49        #expect(c.inputTokens == 684 && c.outputTokens == 38 && c.costUsd == 0.002622 && c.exactCost && c.latencyMs == 2832 && c.ttftMs == 2746 && c.error == nil)50        #expect(ArenaFixtures.ev("done", #"{"id":"cmp_1"}"#) == .done(id: "cmp_1"))51        #expect(ArenaFixtures.ev("end", "{}") == .end)52        #expect(ArenaFixtures.ev("error", #"{"message":"budget"}"#) == .error("budget"))53    }5455    @Test func unknownOrMalformedEventsNeverBreakTheStream() {56        #expect(ArenaFixtures.ev("something_new", #"{"x":1}"#) == .other)57        #expect(ArenaFixtures.ev("model_done", "not json") == .other)58        #expect(ArenaFixtures.ev("text", "{}") == .text(model: "", delta: ""))59        guard case .start(_, let cols) = ArenaFixtures.ev("start", #"{"id":"c","models":[{"provider":"x"},{"id":"ok"}]}"#) else { Issue.record("start"); return }60        #expect(cols.map(\.id) == ["ok"] && cols.first?.displayName == "ok")61    }6263    @Test func judgementDecodesItsMappingAndDisclaimer() {64        let j = try? JSONDecoder().decode(JSONValue.self, from: Data(#"{"judgeModel":"claude-haiku-5-5","text":"RANKING: A > B","mapping":{"A":"m1","B":"m2"},"costUsd":0.001,"disclaimer":"Une opinion."}"#.utf8))65        let judgement = j.flatMap(ArenaJudgement.init)66        #expect(judgement?.mapping == ["A": "m1", "B": "m2"] && judgement?.disclaimer == "Une opinion." && judgement?.judgeModel == "claude-haiku-5-5")67        #expect(ArenaJudgement(.object(["judgeModel": .string("x")])) == nil)68    }69}7071@Suite struct ArenaStateTests {72    private func played(_ events: [(String, String)]) -> ArenaState {73        var s = ArenaState(prompt: "ping", models: ArenaFixtures.models)74        for (n, j) in events { s.apply(ArenaFixtures.ev(n, j)) }75        return s76    }7778    @Test func startTakesTheServerOrderAndKeepsTextAlreadyReceived() {79        var s = ArenaState(prompt: "ping", models: ArenaFixtures.models)80        #expect(s.columns.map(\.id) == ["claude-haiku-5-5", "or:google/gemini-4-neon"] && s.isRunning && s.id == nil)81        s.apply(.text(model: "claude-haiku-5-5", delta: "tôt"))82        s.apply(ArenaFixtures.ev("run", #"{"runId":"run_1"}"#))83        s.apply(ArenaFixtures.ev("start", ArenaFixtures.start))84        #expect(s.id == "cmp_test1" && s.runId == "run_1" && s.columns.map(\.id) == ["or:google/gemini-4-neon", "claude-haiku-5-5"])85        #expect(s.columns.last?.text == "tôt" && s.columns.first?.estimatedInputCostUsd == 0.002475)86    }8788    @Test func deltasGoToTheirOwnColumn() {89        let s = played([("start", ArenaFixtures.start), ("text", #"{"model":"claude-haiku-5-5","delta":"Po"}"#), ("text", #"{"model":"or:google/gemini-4-neon","delta":"X"}"#),90                        ("text", #"{"model":"claude-haiku-5-5","delta":"ng"}"#), ("reasoning", #"{"model":"or:google/gemini-4-neon","delta":"r"}"#), ("text", #"{"model":"inconnu","delta":"?"}"#)])91        #expect(s.columns.first { $0.id == "claude-haiku-5-5" }?.text == "Pong")92        let neon = s.columns.first { $0.id == "or:google/gemini-4-neon" }93        #expect(neon?.text == "X" && neon?.reasoning == "r" && s.finishedCount == 0 && s.totalCostUsd == nil)94    }9596    @Test func modelDoneKeepsTheEstimateAndTellsBilledFromEstimatedCosts() {97        var s = played([("start", ArenaFixtures.start), ("text", #"{"model":"or:google/gemini-4-neon","delta":"Pong"}"#), ("model_done", ArenaFixtures.neonDone)])98        let neon = s.columns.first { $0.id == "or:google/gemini-4-neon" }99        #expect(neon?.isDone == true && neon?.exactCost == true && neon?.costUsd == 0.002622 && neon?.estimatedInputCostUsd == 0.002475 && neon?.estimateBasis != nil && neon?.text == "Pong !")100        #expect(s.finishedCount == 1 && s.allCostsExact && s.isRunning)101        s.apply(ArenaFixtures.ev("model_done", ArenaFixtures.haikuDone))102        #expect(s.finishedCount == 2 && !s.allCostsExact)                              // one billed, one estimated: the total is an estimate103        #expect(abs((s.totalCostUsd ?? 0) - 0.0027286) < 1e-9 && s.fastestFirstToken == "claude-haiku-5-5")104        s.apply(ArenaFixtures.ev("done", #"{"id":"cmp_test1"}"#))105        #expect(!s.isRunning && s.runId == nil && s.id == "cmp_test1" && s.isStored)106    }107108    @Test func aModelErrorStaysInItsColumnAndTheOthersGoOn() {109        var s = played([("start", ArenaFixtures.start), ("model_error", #"{"model":"or:google/gemini-4-neon","message":"Clé refusée"}"#), ("text", #"{"model":"claude-haiku-5-5","delta":"ok"}"#)])110        #expect(s.columns.first?.error == "Clé refusée" && s.columns.last?.error == nil && s.columns.last?.text == "ok" && s.error == nil && s.isRunning)111        let failed = #"{"modelId":"or:google/gemini-4-neon","displayName":"Gemini 4 Neon","text":"","usage":{"input":0,"output":0},"costUsd":0,"exactCost":false,"latencyMs":90,"ttftMs":null,"error":"Clé refusée"}"#112        s.apply(ArenaFixtures.ev("model_done", failed))113        #expect(s.columns.first?.isDone == true && s.columns.first?.error == "Clé refusée" && s.columns.first?.ttftMs == nil && s.fastestFirstToken == nil)114    }115116    @Test func aGlobalErrorOrAnInterruptedStreamEndsTheRunWithoutInventingResults() {117        var s = played([("start", ArenaFixtures.start), ("error", #"{"message":"Limite atteinte"}"#)])118        #expect(!s.isRunning && s.error == "Limite atteinte")119        var t = played([("run", #"{"runId":"run_9"}"#), ("start", ArenaFixtures.start), ("text", #"{"model":"claude-haiku-5-5","delta":"dé"}"#), ("model_done", ArenaFixtures.neonDone)])120        t.interrupt(reason: "Arrêtée")121        #expect(!t.isRunning && t.runId == nil && !t.isStored && t.columns.allSatisfy(\.isDone))122        let haiku = t.columns.first { $0.id == "claude-haiku-5-5" }123        #expect(haiku?.error == "Arrêtée" && haiku?.text == "dé" && haiku?.costUsd == nil)124        #expect(t.columns.first { $0.id == "or:google/gemini-4-neon" }?.error == nil)125        s.apply(.end)126        #expect(!s.isRunning)127    }128}129130@Suite(.serialized) @MainActor struct ArenaSessionTests {131    private func session(host: String) -> ArenaSession {132        let env = TestEnv.make(host: host)133        let known = Dictionary(uniqueKeysWithValues: ArenaFixtures.models.map { ($0.id, $0) })134        let s = ArenaSession(api: env.api, model: { known[$0] })135        s.setModels(ArenaFixtures.models.map(\.id))136        return s137    }138    nonisolated private static func json(_ body: Data?) -> [String: JSONValue] { (body.flatMap { try? JSONDecoder().decode([String: JSONValue].self, from: $0) }) ?? [:] }139140    @Test func theLineUpAcceptsTwoToSixKnownModels() {141        let s = session(host: "arena-lineup.test")142        s.setModels(["claude-haiku-5-5", "inconnu", "claude-haiku-5-5"])143        #expect(s.modelIds == ["claude-haiku-5-5"])144        s.prompt = "ping"145        #expect(!s.canStart)                                                           // one model is not a comparison146        s.add("or:google/gemini-4-neon")147        #expect(s.canStart && s.models.count == 2)148        s.prompt = "   "149        #expect(!s.canStart)150        s.remove("claude-haiku-5-5")151        #expect(s.modelIds == ["or:google/gemini-4-neon"])152    }153154    @Test func aComparisonStreamsThenIsStoredAndCanBePreferredJudgedAdoptedAndExported() async throws {155        let host = "arena-run.test"156        StubProtocol.install(host: host) { req, body in157            let path = req.url?.path ?? ""158            switch (req.httpMethod ?? "GET", path) {159            case ("POST", "/api/compare"): return .sse(ArenaFixtures.run(), delay: 0.02)160            case ("PATCH", "/api/compare"): return .json(#"{"ok":true}"#)161            case ("POST", "/api/compare/judge"): return .json(#"{"judge":{"judgeModel":"claude-haiku-5-5","text":"RANKING: A > B","mapping":{"A":"or:google/gemini-4-neon","B":"claude-haiku-5-5"},"costUsd":0.0004,"disclaimer":"Évaluation générée par un modèle."}}"#)162            case ("GET", "/api/compare/export"): return StubProtocol.Response(headers: ["Content-Type": "text/markdown; charset=utf-8"], chunks: [(0, Data("# Arène multi-modèles\n\nPong".utf8))])163            case ("POST", "/api/conversations/conv_test1/adopt"): return .json(#"{"ok":true}"#)164            default: return .json("{}")165            }166        }167        let s = session(host: host)168        s.prompt = "  Réponds : ping  "; s.effort = .low; s.conversationId = "conv_test1"169        s.start()170        #expect(s.isRunning && s.prompt.isEmpty && s.state?.prompt == "Réponds : ping" && !s.isStored)171        #expect(await TestEnv.wait { !s.isRunning })172        let state = try #require(s.state)173        #expect(state.id == "cmp_test1" && !state.isRunning && state.columns.map(\.id) == ["or:google/gemini-4-neon", "claude-haiku-5-5"])174        #expect(state.columns.first?.text == "Pong !" && state.columns.first?.reasoning == "court" && state.columns.first?.exactCost == true && state.columns.last?.text == "Pong" && state.columns.last?.exactCost == false)175        #expect(s.isStored && s.error == nil)176177        // the request: trimmed prompt, the line-up in order, effort, the conversation it came from178        let post = try #require(StubProtocol.requests(host: host).first { $0.url?.path == "/api/compare" && $0.httpMethod == "POST" })179        #expect(post.value(forHTTPHeaderField: "Accept") == "text/event-stream")180181        await s.prefer("claude-haiku-5-5")182        #expect(s.state?.preferred == "claude-haiku-5-5")183        await s.judge(with: "claude-haiku-5-5")184        #expect(s.state?.judgement?.mapping["A"] == "or:google/gemini-4-neon" && s.state?.judgement?.disclaimer?.isEmpty == false && !s.isJudging)185        #expect(await s.adopt("or:google/gemini-4-neon") && s.adoptedModel == "or:google/gemini-4-neon")186        let url = try #require(await s.export(format: "md"))187        let exported = try String(contentsOf: url, encoding: .utf8)188        #expect(url.pathExtension == "md" && exported.contains("Arène multi-modèles"))189        try? FileManager.default.removeItem(at: url)190        let paths = StubProtocol.requests(host: host).map { "\($0.httpMethod ?? "") \($0.url?.path ?? "")" }191        #expect(paths.contains("PATCH /api/compare") && paths.contains("POST /api/compare/judge") && paths.contains("POST /api/conversations/conv_test1/adopt") && paths.contains("GET /api/compare/export"))192193        // same prompt again, then a clean slate194        s.rerun()195        #expect(s.isRunning && s.state?.prompt == "Réponds : ping" && s.adoptedModel == nil)196        #expect(await TestEnv.wait { !s.isRunning })197        s.reset()198        #expect(s.state == nil && s.modelIds.count == 2)199    }200201    @Test func theRequestBodyCarriesPromptModelsEffortAndConversation() async throws {202        let host = "arena-body.test"203        let seen = NSLockBox<[String: JSONValue]>([:])204        StubProtocol.install(host: host) { req, body in205            if req.url?.path == "/api/compare" { let j = Self.json(body); seen.withLock { $0 = j } }206            return .sse(ArenaFixtures.run(), delay: 0)207        }208        let s = session(host: host)209        s.prompt = "ping"; s.effort = .high; s.conversationId = "conv_test1"; s.projectId = "proj_1"210        s.start()211        #expect(await TestEnv.wait { !s.isRunning })212        let body = seen.withLock { $0 }213        #expect(body["prompt"]?.stringValue == "ping" && body["modelIds"]?.arrayValue?.compactMap(\.stringValue) == ["claude-haiku-5-5", "or:google/gemini-4-neon"])214        #expect(body["settings"]?["effort"]?.stringValue == "high" && body["conversationId"]?.stringValue == "conv_test1" && body["projectId"]?.stringValue == "proj_1")215    }216217    @Test func stopCancelsTheServerRunAndClosesTheOpenColumnsAsTheyStand() async throws {218        let host = "arena-stop.test"219        StubProtocol.install(host: host) { req, _ in220            if req.url?.path == "/api/compare" { return .sse(ArenaFixtures.run(end: false) + Array(repeating: ": ping\n\n", count: 60), delay: 0.05) }221            return .json(#"{"ok":true,"cancelled":true}"#)222        }223        let s = session(host: host)224        s.prompt = "ping"225        s.start()226        #expect(await TestEnv.wait { s.state?.columns.contains { $0.text == "Pong" } == true })227        s.stop()228        #expect(!s.isRunning && s.state?.isRunning == false && !s.isStored)            // never reported `done`: nothing to prefer or export229        #expect(s.state?.columns.allSatisfy { $0.isDone && $0.error != nil && $0.costUsd == nil } == true)230        #expect(await TestEnv.wait { StubProtocol.requests(host: host).contains { $0.url?.path == "/api/chat/cancel" } })231        await s.prefer("claude-haiku-5-5")232        #expect(s.state?.preferred == nil && !StubProtocol.requests(host: host).contains { $0.httpMethod == "PATCH" })233    }234235    @Test func aRefusedOrDroppedComparisonIsReportedNotInvented() async throws {236        let refused = "arena-refused.test"237        StubProtocol.install(host: refused) { _, _ in .json(#"{"error":"Limite quotidienne atteinte","code":"budget"}"#, status: 402) }238        let a = session(host: refused)239        a.prompt = "ping"; a.start()240        #expect(await TestEnv.wait { !a.isRunning })241        #expect(a.error != nil && a.state?.id == nil && !a.isStored && a.state?.columns.allSatisfy { $0.text.isEmpty } == true)242243        let dropped = "arena-dropped.test"244        StubProtocol.install(host: dropped) { _, _ in .sse(ArenaFixtures.run(end: false), delay: 0.01, drop: true) }245        let b = session(host: dropped)246        b.prompt = "ping"; b.start()247        #expect(await TestEnv.wait { !b.isRunning })248        #expect(b.state?.isRunning == false && !b.isStored && b.state?.columns.allSatisfy { $0.isDone && $0.error != nil } == true)249        #expect(b.state?.columns.first { $0.id == "claude-haiku-5-5" }?.text == "Pong")   // what was received is kept250    }251}252253@Suite(.serialized, .enabled(if: Live.enabled)) @MainActor struct ArenaLiveTests {254    @Test func aRealComparisonOfTwoRoutesIsMeasuredPreferredAndExported() async throws {255        let env = Live.environment()256        await env.catalog.refreshModels()257        let s = ArenaSession(api: env.api, model: { [catalog = env.catalog] in catalog.model($0) })258        s.setModels(["or:google/gemini-4-neon", "claude-haiku-5-5"])259        #expect(s.models.count == 2 && s.models.first?.isOpenRouter == true)260        s.prompt = "Réponds en un mot : ping"; s.effort = .low261        s.start()262        #expect(await TestEnv.wait(timeout: 120) { !s.isRunning })263        let state = try #require(s.state)264        #expect(s.isStored && state.id?.hasPrefix("cmp_") == true && s.error == nil)265        for c in state.columns {266            #expect(c.isDone && c.error == nil && !c.text.isEmpty && (c.outputTokens ?? 0) > 0 && (c.latencyMs ?? 0) > 0 && c.ttftMs != nil && c.costUsd != nil)267        }268        let neon = try #require(state.columns.first { $0.id == "or:google/gemini-4-neon" })269        let haiku = try #require(state.columns.first { $0.id == "claude-haiku-5-5" })270        #expect(neon.route == "openrouter" && neon.logo == "google" && neon.exactCost && neon.servedBy?.contains("OpenRouter") == true && neon.estimatedInputCostUsd != nil)271        #expect(haiku.route == "direct" && !haiku.exactCost)                             // Anthropic direct: tokens × list price272        await s.prefer("claude-haiku-5-5")273        #expect(s.state?.preferred == "claude-haiku-5-5" && s.error == nil)274        let url = try #require(await s.export(format: "md"))275        let md = try String(contentsOf: url, encoding: .utf8)276        #expect(md.contains("Arène multi-modèles") && md.contains("claude-haiku-5-5 ★") && md.contains("(facturé)") && md.contains("(estimé)"))277        let jsonURL = try #require(await s.export(format: "json"))278        let exported = try JSONDecoder().decode(JSONValue.self, from: Data(contentsOf: jsonURL))279        #expect(exported.str("preferred_model") == "claude-haiku-5-5" && exported.list("results").count == 2)280        try? FileManager.default.removeItem(at: url); try? FileManager.default.removeItem(at: jsonURL)281    }282}283
Files main
Menu

Theme